diff --git a/.gitattributes b/.gitattributes index a6344aac8c09253b3b630fb776ae94478aa0275b..927f47ef68c2bda38a41a71577644754f82e2ebd 100644 --- a/.gitattributes +++ b/.gitattributes @@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text *.zip filter=lfs diff=lfs merge=lfs -text *.zst filter=lfs diff=lfs merge=lfs -text *tfevents* filter=lfs diff=lfs merge=lfs -text +*.wandb filter=lfs diff=lfs merge=lfs -text diff --git a/qformer2_128/checkpoints/latest-checkpoint.pt b/qformer2_128/checkpoints/latest-checkpoint.pt new file mode 100644 index 0000000000000000000000000000000000000000..b61d89d1ce27d2b21cb8b3b35c673a43fda76e39 --- /dev/null +++ b/qformer2_128/checkpoints/latest-checkpoint.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6d4c74b79c6a0a254825b6fd9a8a9376c0972356dc57e58f270bba29110fdbe3 +size 28822409398 diff --git a/qformer2_128/config.json b/qformer2_128/config.json new file mode 100644 index 0000000000000000000000000000000000000000..f56b1e78fe645422979e9adb7f20827b3a7970cf --- /dev/null +++ b/qformer2_128/config.json @@ -0,0 +1,60 @@ +{ + "dataset": { + "align_stage_components": [ + "download/llava-laion-cc-sbu-558k/chat.json", + "download/llava-laion-cc-sbu-558k" + ], + "dataset_id": "llava-v15", + "dataset_root_dir": "data", + "finetune_stage_components": [ + "download/llava-v1.5-instruct/llava_v1_5_mix665k.json", + "download/llava-v1.5-instruct" + ], + "type": "llava-v15" + }, + "hf_token": ".hf_token", + "max_length": 4096, + "model": { + "align_epochs": 1, + "align_global_batch_size": 256, + "align_learning_rate": 0.001, + "align_lr_scheduler_type": "linear-warmup+cosine-decay", + "align_max_grad_norm": 1.0, + "align_max_steps": null, + "align_per_device_batch_size": 16, + "align_train_strategy": "fsdp-shard-grad-op", + "align_warmup_ratio": 0.03, + "align_weight_decay": 0.0, + "arch_specifier": "qformer2_128", + "enable_gradient_checkpointing": true, + "enable_mixed_precision_training": true, + "finetune_epochs": 1, + "finetune_global_batch_size": 128, + "finetune_learning_rate": 2e-05, + "finetune_lr_scheduler_type": "linear-warmup+cosine-decay", + "finetune_max_grad_norm": 1.0, + "finetune_max_steps": null, + "finetune_per_device_batch_size": 16, + "finetune_train_strategy": "fsdp-full-shard", + "finetune_warmup_ratio": 0.03, + "finetune_weight_decay": 0.1, + "image_resize_strategy": "letterbox", + "llm_backbone_id": "vicuna-v15-7b", + "llm_max_length": 2048, + "model_id": "one-stage+7b_qformer2_128", + "reduce_in_full_precision": false, + "type": "one-stage+7b", + "vision_backbone_id": "clip-vit-l-336px" + }, + "pretrained_checkpoint": null, + "run_id": "qformer2_128", + "run_root_dir": "runs", + "seed": 7, + "stage": "finetune", + "trackers": [ + "jsonl", + "wandb" + ], + "wandb_entity": "lilei_stones", + "wandb_project": "hf-vlms" +} \ No newline at end of file diff --git a/qformer2_128/config.yaml b/qformer2_128/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a4a3474a264c6f1368145252e4a2562aed2e59ea --- /dev/null +++ b/qformer2_128/config.yaml @@ -0,0 +1,53 @@ +dataset: + align_stage_components: + - download/llava-laion-cc-sbu-558k/chat.json + - download/llava-laion-cc-sbu-558k + dataset_id: llava-v15 + dataset_root_dir: data + finetune_stage_components: + - download/llava-v1.5-instruct/llava_v1_5_mix665k.json + - download/llava-v1.5-instruct + type: llava-v15 +hf_token: .hf_token +max_length: 4096 +model: + align_epochs: 1 + align_global_batch_size: 256 + align_learning_rate: 0.001 + align_lr_scheduler_type: linear-warmup+cosine-decay + align_max_grad_norm: 1.0 + align_max_steps: null + align_per_device_batch_size: 16 + align_train_strategy: fsdp-shard-grad-op + align_warmup_ratio: 0.03 + align_weight_decay: 0.0 + arch_specifier: qformer2_128 + enable_gradient_checkpointing: true + enable_mixed_precision_training: true + finetune_epochs: 1 + finetune_global_batch_size: 128 + finetune_learning_rate: 2.0e-05 + finetune_lr_scheduler_type: linear-warmup+cosine-decay + finetune_max_grad_norm: 1.0 + finetune_max_steps: null + finetune_per_device_batch_size: 16 + finetune_train_strategy: fsdp-full-shard + finetune_warmup_ratio: 0.03 + finetune_weight_decay: 0.1 + image_resize_strategy: letterbox + llm_backbone_id: vicuna-v15-7b + llm_max_length: 2048 + model_id: one-stage+7b_qformer2_128 + reduce_in_full_precision: false + type: one-stage+7b + vision_backbone_id: clip-vit-l-336px +pretrained_checkpoint: null +run_id: qformer2_128 +run_root_dir: runs +seed: 7 +stage: finetune +trackers: +- jsonl +- wandb +wandb_entity: lilei_stones +wandb_project: hf-vlms diff --git a/qformer2_128/qformer2_128.jsonl b/qformer2_128/qformer2_128.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..c6cb09f0e7e49b05ebd9feb370846e48ca94b5d8 --- /dev/null +++ b/qformer2_128/qformer2_128.jsonl @@ -0,0 +1,5198 @@ +{"Finetune/Learning Rate": 1.2903225806451614e-07, "Finetune/Loss": 6.627801418304443, "Finetune/Loss (Raw)": 6.627801418304443, "Finetune/Step": 1, "Finetune/Step Time": 25.845428705215454} +{"Finetune/Learning Rate": 2.580645161290323e-07, "Finetune/Loss": 7.242424964904785, "Finetune/Loss (Raw)": 7.857048034667969, "Finetune/Step": 2, "Finetune/Step Time": 16.512165307998657} +{"Finetune/Learning Rate": 3.870967741935484e-07, "Finetune/Loss": 6.438145637512207, "Finetune/Loss (Raw)": 4.829586982727051, "Finetune/Step": 3, "Finetune/Step Time": 12.223726511001587} +{"Finetune/Learning Rate": 5.161290322580646e-07, "Finetune/Loss": 5.705477237701416, "Finetune/Loss (Raw)": 3.5074715614318848, "Finetune/Step": 4, "Finetune/Step Time": 10.115513622760773} +{"Finetune/Learning Rate": 6.451612903225807e-07, "Finetune/Loss": 5.086579322814941, "Finetune/Loss (Raw)": 2.6109883785247803, "Finetune/Step": 5, "Finetune/Step Time": 9.042057847976684} +{"Finetune/Learning Rate": 7.741935483870968e-07, "Finetune/Loss": 4.641938209533691, "Finetune/Loss (Raw)": 2.418732166290283, "Finetune/Step": 6, "Finetune/Step Time": 8.277850111325582} +{"Finetune/Learning Rate": 9.032258064516129e-07, "Finetune/Loss": 4.223259925842285, "Finetune/Loss (Raw)": 1.7111893892288208, "Finetune/Step": 7, "Finetune/Step Time": 7.701050826481411} +{"Finetune/Learning Rate": 1.0322580645161291e-06, "Finetune/Loss": 3.904858112335205, "Finetune/Loss (Raw)": 1.6760460138320923, "Finetune/Step": 8, "Finetune/Step Time": 7.450239598751068} +{"Finetune/Learning Rate": 1.1612903225806454e-06, "Finetune/Loss": 3.610988140106201, "Finetune/Loss (Raw)": 1.2600281238555908, "Finetune/Step": 9, "Finetune/Step Time": 7.101741075515747} +{"Finetune/Learning Rate": 1.2903225806451614e-06, "Finetune/Loss": 3.4228127002716064, "Finetune/Loss (Raw)": 1.7292319536209106, "Finetune/Step": 10, "Finetune/Step Time": 7.087594270706177} +{"Finetune/Learning Rate": 1.4193548387096776e-06, "Finetune/Loss": 3.2471349239349365, "Finetune/Loss (Raw)": 1.4903604984283447, "Finetune/Step": 11, "Finetune/Step Time": 6.809898766604337} +{"Finetune/Learning Rate": 1.5483870967741937e-06, "Finetune/Loss": 3.0870978832244873, "Finetune/Loss (Raw)": 1.3266866207122803, "Finetune/Step": 12, "Finetune/Step Time": 6.592208047707875} +{"Finetune/Learning Rate": 1.67741935483871e-06, "Finetune/Loss": 2.941387176513672, "Finetune/Loss (Raw)": 1.1928627490997314, "Finetune/Step": 13, "Finetune/Step Time": 6.404186413838313} +{"Finetune/Learning Rate": 1.8064516129032258e-06, "Finetune/Loss": 2.8336985111236572, "Finetune/Loss (Raw)": 1.433739185333252, "Finetune/Step": 14, "Finetune/Step Time": 6.221049155507769} +{"Finetune/Learning Rate": 1.935483870967742e-06, "Finetune/Loss": 2.741560935974121, "Finetune/Loss (Raw)": 1.4516397714614868, "Finetune/Step": 15, "Finetune/Step Time": 6.159964243570964} +{"Finetune/Learning Rate": 2.0645161290322582e-06, "Finetune/Loss": 2.635183811187744, "Finetune/Loss (Raw)": 1.0395286083221436, "Finetune/Step": 16, "Finetune/Step Time": 6.1164669543504715} +{"Finetune/Learning Rate": 2.1935483870967745e-06, "Finetune/Loss": 2.550624132156372, "Finetune/Loss (Raw)": 1.1976659297943115, "Finetune/Step": 17, "Finetune/Step Time": 6.011370210086598} +{"Finetune/Learning Rate": 2.3225806451612907e-06, "Finetune/Loss": 2.4668610095977783, "Finetune/Loss (Raw)": 1.0428839921951294, "Finetune/Step": 18, "Finetune/Step Time": 5.94944167137146} +{"Finetune/Learning Rate": 2.4516129032258066e-06, "Finetune/Loss": 2.3994176387786865, "Finetune/Loss (Raw)": 1.1854417324066162, "Finetune/Step": 19, "Finetune/Step Time": 5.840171914351614} +{"Finetune/Learning Rate": 2.580645161290323e-06, "Finetune/Loss": 2.338010549545288, "Finetune/Loss (Raw)": 1.1712745428085327, "Finetune/Step": 20, "Finetune/Step Time": 5.72707747220993} +{"Finetune/Learning Rate": 2.709677419354839e-06, "Finetune/Loss": 2.2827279567718506, "Finetune/Loss (Raw)": 1.1770777702331543, "Finetune/Step": 21, "Finetune/Step Time": 5.7165932314736505} +{"Finetune/Learning Rate": 2.8387096774193553e-06, "Finetune/Loss": 2.2282750606536865, "Finetune/Loss (Raw)": 1.084766149520874, "Finetune/Step": 22, "Finetune/Step Time": 5.654028231447393} +{"Finetune/Learning Rate": 2.967741935483871e-06, "Finetune/Loss": 2.1414754390716553, "Finetune/Loss (Raw)": 0.23188716173171997, "Finetune/Step": 23, "Finetune/Step Time": 5.7787411005600635} +{"Finetune/Learning Rate": 3.0967741935483874e-06, "Finetune/Loss": 2.102695941925049, "Finetune/Loss (Raw)": 1.2107614278793335, "Finetune/Step": 24, "Finetune/Step Time": 5.6918855011463165} +{"Finetune/Learning Rate": 3.225806451612903e-06, "Finetune/Loss": 2.061882734298706, "Finetune/Loss (Raw)": 1.0823686122894287, "Finetune/Step": 25, "Finetune/Step Time": 5.637165288925171} +{"Finetune/Learning Rate": 3.35483870967742e-06, "Finetune/Loss": 1.992916464805603, "Finetune/Loss (Raw)": 0.26875612139701843, "Finetune/Step": 26, "Finetune/Step Time": 5.778037878183218} +{"Finetune/Learning Rate": 3.4838709677419357e-06, "Finetune/Loss": 1.965890645980835, "Finetune/Loss (Raw)": 1.2632218599319458, "Finetune/Step": 27, "Finetune/Step Time": 5.6919625511875855} +{"Finetune/Learning Rate": 3.6129032258064515e-06, "Finetune/Loss": 1.9363267421722412, "Finetune/Loss (Raw)": 1.1381011009216309, "Finetune/Step": 28, "Finetune/Step Time": 5.622569118227277} +{"Finetune/Learning Rate": 3.741935483870968e-06, "Finetune/Loss": 1.9136208295822144, "Finetune/Loss (Raw)": 1.2778525352478027, "Finetune/Step": 29, "Finetune/Step Time": 5.609435015711291} +{"Finetune/Learning Rate": 3.870967741935484e-06, "Finetune/Loss": 1.8851193189620972, "Finetune/Loss (Raw)": 1.0585753917694092, "Finetune/Step": 30, "Finetune/Step Time": 5.5990467468897505} +{"Finetune/Learning Rate": 4.000000000000001e-06, "Finetune/Loss": 1.8636664152145386, "Finetune/Loss (Raw)": 1.220081090927124, "Finetune/Step": 31, "Finetune/Step Time": 5.5629159250567035} +{"Finetune/Learning Rate": 4.1290322580645165e-06, "Finetune/Loss": 1.8404020071029663, "Finetune/Loss (Raw)": 1.119208812713623, "Finetune/Step": 32, "Finetune/Step Time": 5.586056374013424} +{"Finetune/Learning Rate": 4.258064516129032e-06, "Finetune/Loss": 1.813931941986084, "Finetune/Loss (Raw)": 0.966888427734375, "Finetune/Step": 33, "Finetune/Step Time": 5.559108466813059} +{"Finetune/Learning Rate": 4.387096774193549e-06, "Finetune/Loss": 1.7904483079910278, "Finetune/Loss (Raw)": 1.0154842138290405, "Finetune/Step": 34, "Finetune/Step Time": 5.517073757508221} +{"Finetune/Learning Rate": 4.516129032258065e-06, "Finetune/Loss": 1.7768268585205078, "Finetune/Loss (Raw)": 1.31369948387146, "Finetune/Step": 35, "Finetune/Step Time": 5.4884429454803465} +{"Finetune/Learning Rate": 4.6451612903225815e-06, "Finetune/Loss": 1.759299874305725, "Finetune/Loss (Raw)": 1.145856499671936, "Finetune/Step": 36, "Finetune/Step Time": 5.495686398612128} +{"Finetune/Learning Rate": 4.774193548387097e-06, "Finetune/Loss": 1.7460987567901611, "Finetune/Loss (Raw)": 1.2708544731140137, "Finetune/Step": 37, "Finetune/Step Time": 5.544374169530095} +{"Finetune/Learning Rate": 4.903225806451613e-06, "Finetune/Loss": 1.7299604415893555, "Finetune/Loss (Raw)": 1.1328449249267578, "Finetune/Step": 38, "Finetune/Step Time": 5.49751573487332} +{"Finetune/Learning Rate": 5.032258064516129e-06, "Finetune/Loss": 1.7152162790298462, "Finetune/Loss (Raw)": 1.1549419164657593, "Finetune/Step": 39, "Finetune/Step Time": 5.450664318524874} +{"Finetune/Learning Rate": 5.161290322580646e-06, "Finetune/Loss": 1.7010669708251953, "Finetune/Loss (Raw)": 1.1492505073547363, "Finetune/Step": 40, "Finetune/Step Time": 5.457089430093765} +{"Finetune/Learning Rate": 5.290322580645162e-06, "Finetune/Loss": 1.6905275583267212, "Finetune/Loss (Raw)": 1.268944263458252, "Finetune/Step": 41, "Finetune/Step Time": 5.401769608986087} +{"Finetune/Learning Rate": 5.419354838709678e-06, "Finetune/Loss": 1.6795185804367065, "Finetune/Loss (Raw)": 1.2281551361083984, "Finetune/Step": 42, "Finetune/Step Time": 5.362019658088684} +{"Finetune/Learning Rate": 5.548387096774194e-06, "Finetune/Loss": 1.6673297882080078, "Finetune/Loss (Raw)": 1.155400276184082, "Finetune/Step": 43, "Finetune/Step Time": 5.358730781910031} +{"Finetune/Learning Rate": 5.677419354838711e-06, "Finetune/Loss": 1.6562732458114624, "Finetune/Loss (Raw)": 1.1808379888534546, "Finetune/Step": 44, "Finetune/Step Time": 5.346404541622508} +{"Finetune/Learning Rate": 5.806451612903226e-06, "Finetune/Loss": 1.643001675605774, "Finetune/Loss (Raw)": 1.0590459108352661, "Finetune/Step": 45, "Finetune/Step Time": 5.404946046405368} +{"Finetune/Learning Rate": 5.935483870967742e-06, "Finetune/Loss": 1.613691806793213, "Finetune/Loss (Raw)": 0.2947537302970886, "Finetune/Step": 46, "Finetune/Step Time": 5.5108664709588755} +{"Finetune/Learning Rate": 6.064516129032259e-06, "Finetune/Loss": 1.6037567853927612, "Finetune/Loss (Raw)": 1.1467455625534058, "Finetune/Step": 47, "Finetune/Step Time": 5.497648340590457} +{"Finetune/Learning Rate": 6.193548387096775e-06, "Finetune/Loss": 1.593829870223999, "Finetune/Loss (Raw)": 1.1272660493850708, "Finetune/Step": 48, "Finetune/Step Time": 5.502229859431584} +{"Finetune/Learning Rate": 6.3225806451612906e-06, "Finetune/Loss": 1.5849926471710205, "Finetune/Loss (Raw)": 1.1608046293258667, "Finetune/Step": 49, "Finetune/Step Time": 5.475935828929045} +{"Finetune/Learning Rate": 6.451612903225806e-06, "Finetune/Loss": 1.574001431465149, "Finetune/Loss (Raw)": 1.0354323387145996, "Finetune/Step": 50, "Finetune/Step Time": 5.453482313156128} +{"Finetune/Learning Rate": 6.580645161290323e-06, "Finetune/Loss": 1.5655978918075562, "Finetune/Loss (Raw)": 1.1454222202301025, "Finetune/Step": 51, "Finetune/Step Time": 5.424796001583922} +{"Finetune/Learning Rate": 6.70967741935484e-06, "Finetune/Loss": 1.5587385892868042, "Finetune/Loss (Raw)": 1.2089120149612427, "Finetune/Step": 52, "Finetune/Step Time": 5.4147028143589315} +{"Finetune/Learning Rate": 6.838709677419355e-06, "Finetune/Loss": 1.5467036962509155, "Finetune/Loss (Raw)": 0.9208900928497314, "Finetune/Step": 53, "Finetune/Step Time": 5.426983091066468} +{"Finetune/Learning Rate": 6.967741935483871e-06, "Finetune/Loss": 1.535413146018982, "Finetune/Loss (Raw)": 0.9370174407958984, "Finetune/Step": 54, "Finetune/Step Time": 5.420302792831704} +{"Finetune/Learning Rate": 7.096774193548388e-06, "Finetune/Loss": 1.5287030935287476, "Finetune/Loss (Raw)": 1.1663625240325928, "Finetune/Step": 55, "Finetune/Step Time": 5.419956532391635} +{"Finetune/Learning Rate": 7.225806451612903e-06, "Finetune/Loss": 1.5212801694869995, "Finetune/Loss (Raw)": 1.1130155324935913, "Finetune/Step": 56, "Finetune/Step Time": 5.429584324359894} +{"Finetune/Learning Rate": 7.35483870967742e-06, "Finetune/Loss": 1.512488842010498, "Finetune/Loss (Raw)": 1.0201728343963623, "Finetune/Step": 57, "Finetune/Step Time": 5.405688796127052} +{"Finetune/Learning Rate": 7.483870967741936e-06, "Finetune/Loss": 1.504714012145996, "Finetune/Loss (Raw)": 1.0615499019622803, "Finetune/Step": 58, "Finetune/Step Time": 5.431568084091976} +{"Finetune/Learning Rate": 7.612903225806451e-06, "Finetune/Loss": 1.4979177713394165, "Finetune/Loss (Raw)": 1.103742241859436, "Finetune/Step": 59, "Finetune/Step Time": 5.438267061265848} +{"Finetune/Learning Rate": 7.741935483870968e-06, "Finetune/Loss": 1.4911679029464722, "Finetune/Loss (Raw)": 1.092914342880249, "Finetune/Step": 60, "Finetune/Step Time": 5.412197216351827} +{"Finetune/Learning Rate": 7.870967741935484e-06, "Finetune/Loss": 1.4856239557266235, "Finetune/Loss (Raw)": 1.1530022621154785, "Finetune/Step": 61, "Finetune/Step Time": 5.394853349591865} +{"Finetune/Learning Rate": 8.000000000000001e-06, "Finetune/Loss": 1.4809809923171997, "Finetune/Loss (Raw)": 1.1977512836456299, "Finetune/Step": 62, "Finetune/Step Time": 5.3698390606910955} +{"Finetune/Learning Rate": 8.129032258064517e-06, "Finetune/Loss": 1.4766671657562256, "Finetune/Loss (Raw)": 1.2092055082321167, "Finetune/Step": 63, "Finetune/Step Time": 5.347862705351814} +{"Finetune/Learning Rate": 8.258064516129033e-06, "Finetune/Loss": 1.4702539443969727, "Finetune/Loss (Raw)": 1.0662204027175903, "Finetune/Step": 64, "Finetune/Step Time": 5.383430786430836} +{"Finetune/Learning Rate": 8.387096774193549e-06, "Finetune/Loss": 1.4638001918792725, "Finetune/Loss (Raw)": 1.0507605075836182, "Finetune/Step": 65, "Finetune/Step Time": 5.3770224021031305} +{"Finetune/Learning Rate": 8.516129032258065e-06, "Finetune/Loss": 1.456912636756897, "Finetune/Loss (Raw)": 1.009225606918335, "Finetune/Step": 66, "Finetune/Step Time": 5.362484892209371} +{"Finetune/Learning Rate": 8.64516129032258e-06, "Finetune/Loss": 1.4486056566238403, "Finetune/Loss (Raw)": 0.9003441333770752, "Finetune/Step": 67, "Finetune/Step Time": 5.3371285965193564} +{"Finetune/Learning Rate": 8.774193548387098e-06, "Finetune/Loss": 1.4439328908920288, "Finetune/Loss (Raw)": 1.1308627128601074, "Finetune/Step": 68, "Finetune/Step Time": 5.333888243226444} +{"Finetune/Learning Rate": 8.903225806451614e-06, "Finetune/Loss": 1.4397802352905273, "Finetune/Loss (Raw)": 1.1573833227157593, "Finetune/Step": 69, "Finetune/Step Time": 5.309885847395745} +{"Finetune/Learning Rate": 9.03225806451613e-06, "Finetune/Loss": 1.436307668685913, "Finetune/Loss (Raw)": 1.196703553199768, "Finetune/Step": 70, "Finetune/Step Time": 5.289958480426243} +{"Finetune/Learning Rate": 9.161290322580645e-06, "Finetune/Loss": 1.4291867017745972, "Finetune/Loss (Raw)": 0.9307218790054321, "Finetune/Step": 71, "Finetune/Step Time": 5.2723012373481} +{"Finetune/Learning Rate": 9.290322580645163e-06, "Finetune/Loss": 1.4235550165176392, "Finetune/Loss (Raw)": 1.0237034559249878, "Finetune/Step": 72, "Finetune/Step Time": 5.2481436961227} +{"Finetune/Learning Rate": 9.419354838709677e-06, "Finetune/Loss": 1.4167137145996094, "Finetune/Loss (Raw)": 0.9241412281990051, "Finetune/Step": 73, "Finetune/Step Time": 5.229742628254303} +{"Finetune/Learning Rate": 9.548387096774195e-06, "Finetune/Loss": 1.4102206230163574, "Finetune/Loss (Raw)": 0.9362214803695679, "Finetune/Step": 74, "Finetune/Step Time": 5.211855276210888} +{"Finetune/Learning Rate": 9.67741935483871e-06, "Finetune/Loss": 1.4024447202682495, "Finetune/Loss (Raw)": 0.8270233273506165, "Finetune/Step": 75, "Finetune/Step Time": 5.206651592254639} +{"Finetune/Learning Rate": 9.806451612903226e-06, "Finetune/Loss": 1.3968650102615356, "Finetune/Loss (Raw)": 0.9784058928489685, "Finetune/Step": 76, "Finetune/Step Time": 5.198697190535696} +{"Finetune/Learning Rate": 9.935483870967742e-06, "Finetune/Loss": 1.3932033777236938, "Finetune/Loss (Raw)": 1.114914894104004, "Finetune/Step": 77, "Finetune/Step Time": 5.178399342995186} +{"Finetune/Learning Rate": 1.0064516129032258e-05, "Finetune/Loss": 1.3909927606582642, "Finetune/Loss (Raw)": 1.2207703590393066, "Finetune/Step": 78, "Finetune/Step Time": 5.157914516253349} +{"Finetune/Learning Rate": 1.0193548387096774e-05, "Finetune/Loss": 1.3884012699127197, "Finetune/Loss (Raw)": 1.1862646341323853, "Finetune/Step": 79, "Finetune/Step Time": 5.142443690118911} +{"Finetune/Learning Rate": 1.0322580645161291e-05, "Finetune/Loss": 1.3832553625106812, "Finetune/Loss (Raw)": 0.9767332077026367, "Finetune/Step": 80, "Finetune/Step Time": 5.148260778188705} +{"Finetune/Learning Rate": 1.0451612903225807e-05, "Finetune/Loss": 1.3803091049194336, "Finetune/Loss (Raw)": 1.1446141004562378, "Finetune/Step": 81, "Finetune/Step Time": 5.1456316577063665} +{"Finetune/Learning Rate": 1.0580645161290325e-05, "Finetune/Loss": 1.3769443035125732, "Finetune/Loss (Raw)": 1.1043931245803833, "Finetune/Step": 82, "Finetune/Step Time": 5.128858836685739} +{"Finetune/Learning Rate": 1.070967741935484e-05, "Finetune/Loss": 1.3719439506530762, "Finetune/Loss (Raw)": 0.9619176387786865, "Finetune/Step": 83, "Finetune/Step Time": 5.137374648128647} +{"Finetune/Learning Rate": 1.0838709677419356e-05, "Finetune/Loss": 1.3697726726531982, "Finetune/Loss (Raw)": 1.1895452737808228, "Finetune/Step": 84, "Finetune/Step Time": 5.139200045948937} +{"Finetune/Learning Rate": 1.096774193548387e-05, "Finetune/Loss": 1.3659751415252686, "Finetune/Loss (Raw)": 1.0469920635223389, "Finetune/Step": 85, "Finetune/Step Time": 5.125322097890518} +{"Finetune/Learning Rate": 1.1096774193548388e-05, "Finetune/Loss": 1.3541884422302246, "Finetune/Loss (Raw)": 0.35231873393058777, "Finetune/Step": 86, "Finetune/Step Time": 5.173687812893889} +{"Finetune/Learning Rate": 1.1225806451612904e-05, "Finetune/Loss": 1.3504647016525269, "Finetune/Loss (Raw)": 1.0302209854125977, "Finetune/Step": 87, "Finetune/Step Time": 5.15919138097215} +{"Finetune/Learning Rate": 1.1354838709677421e-05, "Finetune/Loss": 1.3490430116653442, "Finetune/Loss (Raw)": 1.2253497838974, "Finetune/Step": 88, "Finetune/Step Time": 5.166935343633998} +{"Finetune/Learning Rate": 1.1483870967741937e-05, "Finetune/Loss": 1.3459404706954956, "Finetune/Loss (Raw)": 1.072920799255371, "Finetune/Step": 89, "Finetune/Step Time": 5.153400300593859} +{"Finetune/Learning Rate": 1.1612903225806453e-05, "Finetune/Loss": 1.343704104423523, "Finetune/Loss (Raw)": 1.144665002822876, "Finetune/Step": 90, "Finetune/Step Time": 5.161374409993489} +{"Finetune/Learning Rate": 1.1741935483870967e-05, "Finetune/Loss": 1.3416945934295654, "Finetune/Loss (Raw)": 1.1608388423919678, "Finetune/Step": 91, "Finetune/Step Time": 5.16405490728525} +{"Finetune/Learning Rate": 1.1870967741935484e-05, "Finetune/Loss": 1.3402098417282104, "Finetune/Loss (Raw)": 1.2050986289978027, "Finetune/Step": 92, "Finetune/Step Time": 5.152811509111653} +{"Finetune/Learning Rate": 1.2e-05, "Finetune/Loss": 1.3357137441635132, "Finetune/Loss (Raw)": 0.9220702052116394, "Finetune/Step": 93, "Finetune/Step Time": 5.143715138076454} +{"Finetune/Learning Rate": 1.2129032258064518e-05, "Finetune/Loss": 1.3338638544082642, "Finetune/Loss (Raw)": 1.161834716796875, "Finetune/Step": 94, "Finetune/Step Time": 5.127554629711395} +{"Finetune/Learning Rate": 1.2258064516129034e-05, "Finetune/Loss": 1.3313827514648438, "Finetune/Loss (Raw)": 1.0981526374816895, "Finetune/Step": 95, "Finetune/Step Time": 5.11558995999788} +{"Finetune/Learning Rate": 1.238709677419355e-05, "Finetune/Loss": 1.328823447227478, "Finetune/Loss (Raw)": 1.0856821537017822, "Finetune/Step": 96, "Finetune/Step Time": 5.118417223294576} +{"Finetune/Learning Rate": 1.2516129032258067e-05, "Finetune/Loss": 1.3257231712341309, "Finetune/Loss (Raw)": 1.0281091928482056, "Finetune/Step": 97, "Finetune/Step Time": 5.112987879625301} +{"Finetune/Learning Rate": 1.2645161290322581e-05, "Finetune/Loss": 1.323736310005188, "Finetune/Loss (Raw)": 1.131007194519043, "Finetune/Step": 98, "Finetune/Step Time": 5.100828903062003} +{"Finetune/Learning Rate": 1.2774193548387097e-05, "Finetune/Loss": 1.312605857849121, "Finetune/Loss (Raw)": 0.22181521356105804, "Finetune/Step": 99, "Finetune/Step Time": 5.133544221068874} +{"Finetune/Learning Rate": 1.2903225806451613e-05, "Finetune/Loss": 1.311640739440918, "Finetune/Loss (Raw)": 1.2160974740982056, "Finetune/Step": 100, "Finetune/Step Time": 5.125469648838044} +{"Finetune/Learning Rate": 1.303225806451613e-05, "Finetune/Loss": 1.3090763092041016, "Finetune/Loss (Raw)": 1.0526306629180908, "Finetune/Step": 101, "Finetune/Step Time": 5.127137070835227} +{"Finetune/Learning Rate": 1.3161290322580646e-05, "Finetune/Loss": 1.3080980777740479, "Finetune/Loss (Raw)": 1.2092816829681396, "Finetune/Step": 102, "Finetune/Step Time": 5.11008552943959} +{"Finetune/Learning Rate": 1.3290322580645164e-05, "Finetune/Loss": 1.2982550859451294, "Finetune/Loss (Raw)": 0.2942924499511719, "Finetune/Step": 103, "Finetune/Step Time": 5.142518726367395} +{"Finetune/Learning Rate": 1.341935483870968e-05, "Finetune/Loss": 1.2969112396240234, "Finetune/Loss (Raw)": 1.1584950685501099, "Finetune/Step": 104, "Finetune/Step Time": 5.12893698307184} +{"Finetune/Learning Rate": 1.3548387096774194e-05, "Finetune/Loss": 1.2947020530700684, "Finetune/Loss (Raw)": 1.0649343729019165, "Finetune/Step": 105, "Finetune/Step Time": 5.111913020270212} +{"Finetune/Learning Rate": 1.367741935483871e-05, "Finetune/Loss": 1.2925381660461426, "Finetune/Loss (Raw)": 1.0653393268585205, "Finetune/Step": 106, "Finetune/Step Time": 5.098917567505027} +{"Finetune/Learning Rate": 1.3806451612903227e-05, "Finetune/Loss": 1.291047215461731, "Finetune/Loss (Raw)": 1.1330112218856812, "Finetune/Step": 107, "Finetune/Step Time": 5.086693402762725} +{"Finetune/Learning Rate": 1.3935483870967743e-05, "Finetune/Loss": 1.2885140180587769, "Finetune/Loss (Raw)": 1.0174578428268433, "Finetune/Step": 108, "Finetune/Step Time": 5.07814512429414} +{"Finetune/Learning Rate": 1.406451612903226e-05, "Finetune/Loss": 1.2872076034545898, "Finetune/Loss (Raw)": 1.146122932434082, "Finetune/Step": 109, "Finetune/Step Time": 5.073611430071909} +{"Finetune/Learning Rate": 1.4193548387096776e-05, "Finetune/Loss": 1.2850843667984009, "Finetune/Loss (Raw)": 1.0536409616470337, "Finetune/Step": 110, "Finetune/Step Time": 5.061184800754894} +{"Finetune/Learning Rate": 1.4322580645161292e-05, "Finetune/Loss": 1.2835853099822998, "Finetune/Loss (Raw)": 1.1186776161193848, "Finetune/Step": 111, "Finetune/Step Time": 5.0666029195527775} +{"Finetune/Learning Rate": 1.4451612903225806e-05, "Finetune/Loss": 1.281812310218811, "Finetune/Loss (Raw)": 1.0850204229354858, "Finetune/Step": 112, "Finetune/Step Time": 5.067273597632136} +{"Finetune/Learning Rate": 1.4580645161290324e-05, "Finetune/Loss": 1.2804559469223022, "Finetune/Loss (Raw)": 1.1285582780838013, "Finetune/Step": 113, "Finetune/Step Time": 5.07032821663713} +{"Finetune/Learning Rate": 1.470967741935484e-05, "Finetune/Loss": 1.271403193473816, "Finetune/Loss (Raw)": 0.24842682480812073, "Finetune/Step": 114, "Finetune/Step Time": 5.111067907852039} +{"Finetune/Learning Rate": 1.4838709677419357e-05, "Finetune/Loss": 1.2698370218276978, "Finetune/Loss (Raw)": 1.0913023948669434, "Finetune/Step": 115, "Finetune/Step Time": 5.106641331962917} +{"Finetune/Learning Rate": 1.4967741935483873e-05, "Finetune/Loss": 1.2668861150741577, "Finetune/Loss (Raw)": 0.927520751953125, "Finetune/Step": 116, "Finetune/Step Time": 5.117536088515973} +{"Finetune/Learning Rate": 1.5096774193548389e-05, "Finetune/Loss": 1.2655534744262695, "Finetune/Loss (Raw)": 1.1109691858291626, "Finetune/Step": 117, "Finetune/Step Time": 5.111977827854646} +{"Finetune/Learning Rate": 1.5225806451612903e-05, "Finetune/Loss": 1.2644739151000977, "Finetune/Loss (Raw)": 1.1381580829620361, "Finetune/Step": 118, "Finetune/Step Time": 5.101148688187034} +{"Finetune/Learning Rate": 1.535483870967742e-05, "Finetune/Loss": 1.2626584768295288, "Finetune/Loss (Raw)": 1.0484404563903809, "Finetune/Step": 119, "Finetune/Step Time": 5.092124400018644} +{"Finetune/Learning Rate": 1.5483870967741936e-05, "Finetune/Loss": 1.2612535953521729, "Finetune/Loss (Raw)": 1.0940650701522827, "Finetune/Step": 120, "Finetune/Step Time": 5.08721632361412} +{"Finetune/Learning Rate": 1.5612903225806454e-05, "Finetune/Loss": 1.2593293190002441, "Finetune/Loss (Raw)": 1.0284479856491089, "Finetune/Step": 121, "Finetune/Step Time": 5.094304955695287} +{"Finetune/Learning Rate": 1.5741935483870968e-05, "Finetune/Loss": 1.2569141387939453, "Finetune/Loss (Raw)": 0.9646660089492798, "Finetune/Step": 122, "Finetune/Step Time": 5.084837243205211} +{"Finetune/Learning Rate": 1.5870967741935485e-05, "Finetune/Loss": 1.2549645900726318, "Finetune/Loss (Raw)": 1.0171369314193726, "Finetune/Step": 123, "Finetune/Step Time": 5.074740762633037} +{"Finetune/Learning Rate": 1.6000000000000003e-05, "Finetune/Loss": 1.2539485692977905, "Finetune/Loss (Raw)": 1.128950834274292, "Finetune/Step": 124, "Finetune/Step Time": 5.06711438778908} +{"Finetune/Learning Rate": 1.6129032258064517e-05, "Finetune/Loss": 1.2520262002944946, "Finetune/Loss (Raw)": 1.013637661933899, "Finetune/Step": 125, "Finetune/Step Time": 5.059962739944458} +{"Finetune/Learning Rate": 1.6258064516129034e-05, "Finetune/Loss": 1.2508370876312256, "Finetune/Loss (Raw)": 1.1022218465805054, "Finetune/Step": 126, "Finetune/Step Time": 5.071698816995772} +{"Finetune/Learning Rate": 1.638709677419355e-05, "Finetune/Loss": 1.2487620115280151, "Finetune/Loss (Raw)": 0.9873037934303284, "Finetune/Step": 127, "Finetune/Step Time": 5.061093771551538} +{"Finetune/Learning Rate": 1.6516129032258066e-05, "Finetune/Loss": 1.247830867767334, "Finetune/Loss (Raw)": 1.1295770406723022, "Finetune/Step": 128, "Finetune/Step Time": 5.054079122841358} +{"Finetune/Learning Rate": 1.6645161290322583e-05, "Finetune/Loss": 1.2050447463989258, "Finetune/Loss (Raw)": 1.1511603593826294, "Finetune/Step": 129, "Finetune/Step Time": 4.882643545046449} +{"Finetune/Learning Rate": 1.6774193548387098e-05, "Finetune/Loss": 1.1526343822479248, "Finetune/Loss (Raw)": 1.1485238075256348, "Finetune/Step": 130, "Finetune/Step Time": 4.865792080760002} +{"Finetune/Learning Rate": 1.6903225806451615e-05, "Finetune/Loss": 1.1235235929489136, "Finetune/Loss (Raw)": 1.1034058332443237, "Finetune/Step": 131, "Finetune/Step Time": 4.86551901884377} +{"Finetune/Learning Rate": 1.703225806451613e-05, "Finetune/Loss": 1.1047817468643188, "Finetune/Loss (Raw)": 1.108521580696106, "Finetune/Step": 132, "Finetune/Step Time": 4.865829203277826} +{"Finetune/Learning Rate": 1.7161290322580647e-05, "Finetune/Loss": 1.0914106369018555, "Finetune/Loss (Raw)": 0.8994982838630676, "Finetune/Step": 133, "Finetune/Step Time": 4.8760611694306135} +{"Finetune/Learning Rate": 1.729032258064516e-05, "Finetune/Loss": 1.082240104675293, "Finetune/Loss (Raw)": 1.2448985576629639, "Finetune/Step": 134, "Finetune/Step Time": 4.871481366455555} +{"Finetune/Learning Rate": 1.741935483870968e-05, "Finetune/Loss": 1.0766940116882324, "Finetune/Loss (Raw)": 1.0012848377227783, "Finetune/Step": 135, "Finetune/Step Time": 4.87361534871161} +{"Finetune/Learning Rate": 1.7548387096774196e-05, "Finetune/Loss": 1.0720865726470947, "Finetune/Loss (Raw)": 1.0862969160079956, "Finetune/Step": 136, "Finetune/Step Time": 4.858353430405259} +{"Finetune/Learning Rate": 1.7677419354838713e-05, "Finetune/Loss": 1.0698726177215576, "Finetune/Loss (Raw)": 0.9766507148742676, "Finetune/Step": 137, "Finetune/Step Time": 4.8613841366022825} +{"Finetune/Learning Rate": 1.7806451612903228e-05, "Finetune/Loss": 1.0651159286499023, "Finetune/Loss (Raw)": 1.1203557252883911, "Finetune/Step": 138, "Finetune/Step Time": 4.83238460123539} +{"Finetune/Learning Rate": 1.7935483870967742e-05, "Finetune/Loss": 1.0606296062469482, "Finetune/Loss (Raw)": 0.9161070585250854, "Finetune/Step": 139, "Finetune/Step Time": 4.834015879780054} +{"Finetune/Learning Rate": 1.806451612903226e-05, "Finetune/Loss": 1.0603196620941162, "Finetune/Loss (Raw)": 1.2870174646377563, "Finetune/Step": 140, "Finetune/Step Time": 4.827933115884662} +{"Finetune/Learning Rate": 1.8193548387096777e-05, "Finetune/Loss": 1.0591342449188232, "Finetune/Loss (Raw)": 1.0411297082901, "Finetune/Step": 141, "Finetune/Step Time": 4.822006033733487} +{"Finetune/Learning Rate": 1.832258064516129e-05, "Finetune/Loss": 1.0559831857681274, "Finetune/Loss (Raw)": 1.03040611743927, "Finetune/Step": 142, "Finetune/Step Time": 4.8649564031511545} +{"Finetune/Learning Rate": 1.845161290322581e-05, "Finetune/Loss": 1.0522053241729736, "Finetune/Loss (Raw)": 0.9680788516998291, "Finetune/Step": 143, "Finetune/Step Time": 4.879515325650573} +{"Finetune/Learning Rate": 1.8580645161290326e-05, "Finetune/Loss": 1.0470017194747925, "Finetune/Loss (Raw)": 0.3734655976295471, "Finetune/Step": 144, "Finetune/Step Time": 4.903130294755101} +{"Finetune/Learning Rate": 1.870967741935484e-05, "Finetune/Loss": 1.0403450727462769, "Finetune/Loss (Raw)": 0.34561026096343994, "Finetune/Step": 145, "Finetune/Step Time": 4.939006723463535} +{"Finetune/Learning Rate": 1.8838709677419354e-05, "Finetune/Loss": 1.0397989749908447, "Finetune/Loss (Raw)": 0.9729813933372498, "Finetune/Step": 146, "Finetune/Step Time": 4.9292329754680395} +{"Finetune/Learning Rate": 1.896774193548387e-05, "Finetune/Loss": 1.037576675415039, "Finetune/Loss (Raw)": 0.9009859561920166, "Finetune/Step": 147, "Finetune/Step Time": 4.935478726401925} +{"Finetune/Learning Rate": 1.909677419354839e-05, "Finetune/Loss": 1.0368237495422363, "Finetune/Loss (Raw)": 1.07490873336792, "Finetune/Step": 148, "Finetune/Step Time": 4.938347350805998} +{"Finetune/Learning Rate": 1.9225806451612907e-05, "Finetune/Loss": 1.036386251449585, "Finetune/Loss (Raw)": 1.1210795640945435, "Finetune/Step": 149, "Finetune/Step Time": 4.927860962226987} +{"Finetune/Learning Rate": 1.935483870967742e-05, "Finetune/Loss": 1.0367376804351807, "Finetune/Loss (Raw)": 1.1297411918640137, "Finetune/Step": 150, "Finetune/Step Time": 4.931786248460412} +{"Finetune/Learning Rate": 1.948387096774194e-05, "Finetune/Loss": 1.043483018875122, "Finetune/Loss (Raw)": 1.0952924489974976, "Finetune/Step": 151, "Finetune/Step Time": 4.899690192192793} +{"Finetune/Learning Rate": 1.9612903225806452e-05, "Finetune/Loss": 1.042272925376892, "Finetune/Loss (Raw)": 1.055879831314087, "Finetune/Step": 152, "Finetune/Step Time": 4.905847020447254} +{"Finetune/Learning Rate": 1.974193548387097e-05, "Finetune/Loss": 1.042823076248169, "Finetune/Loss (Raw)": 1.1527940034866333, "Finetune/Step": 153, "Finetune/Step Time": 4.915484270080924} +{"Finetune/Learning Rate": 1.9870967741935484e-05, "Finetune/Loss": 1.049612045288086, "Finetune/Loss (Raw)": 1.1377232074737549, "Finetune/Step": 154, "Finetune/Step Time": 4.874381950125098} +{"Finetune/Learning Rate": 2e-05, "Finetune/Loss": 1.0492198467254639, "Finetune/Loss (Raw)": 1.2130179405212402, "Finetune/Step": 155, "Finetune/Step Time": 4.881550371646881} +{"Finetune/Learning Rate": 1.9999998059597617e-05, "Finetune/Loss": 1.0426740646362305, "Finetune/Loss (Raw)": 0.3002665936946869, "Finetune/Step": 156, "Finetune/Step Time": 4.919926868751645} +{"Finetune/Learning Rate": 1.9999992238391222e-05, "Finetune/Loss": 1.0411535501480103, "Finetune/Loss (Raw)": 1.0832167863845825, "Finetune/Step": 157, "Finetune/Step Time": 4.909776523709297} +{"Finetune/Learning Rate": 1.999998253638307e-05, "Finetune/Loss": 1.0415507555007935, "Finetune/Loss (Raw)": 1.1094059944152832, "Finetune/Step": 158, "Finetune/Step Time": 4.900313215330243} +{"Finetune/Learning Rate": 1.999996895357693e-05, "Finetune/Loss": 1.0404374599456787, "Finetune/Loss (Raw)": 1.0775864124298096, "Finetune/Step": 159, "Finetune/Step Time": 4.893392909318209} +{"Finetune/Learning Rate": 1.9999951489978074e-05, "Finetune/Loss": 1.0396497249603271, "Finetune/Loss (Raw)": 1.0183899402618408, "Finetune/Step": 160, "Finetune/Step Time": 4.882644638419151} +{"Finetune/Learning Rate": 1.9999930145593275e-05, "Finetune/Loss": 1.0401835441589355, "Finetune/Loss (Raw)": 1.0352085828781128, "Finetune/Step": 161, "Finetune/Step Time": 4.8789524007588625} +{"Finetune/Learning Rate": 1.9999904920430818e-05, "Finetune/Loss": 1.0421369075775146, "Finetune/Loss (Raw)": 1.2655086517333984, "Finetune/Step": 162, "Finetune/Step Time": 4.877847578376532} +{"Finetune/Learning Rate": 1.9999875814500492e-05, "Finetune/Loss": 1.0403311252593994, "Finetune/Loss (Raw)": 1.0825608968734741, "Finetune/Step": 163, "Finetune/Step Time": 4.878256751224399} +{"Finetune/Learning Rate": 1.99998428278136e-05, "Finetune/Loss": 1.0389189720153809, "Finetune/Loss (Raw)": 0.9651105403900146, "Finetune/Step": 164, "Finetune/Step Time": 4.87629528529942} +{"Finetune/Learning Rate": 1.9999805960382928e-05, "Finetune/Loss": 1.0372519493103027, "Finetune/Loss (Raw)": 1.0574740171432495, "Finetune/Step": 165, "Finetune/Step Time": 4.8532164096832275} +{"Finetune/Learning Rate": 1.99997652122228e-05, "Finetune/Loss": 1.0362164974212646, "Finetune/Loss (Raw)": 1.0002861022949219, "Finetune/Step": 166, "Finetune/Step Time": 4.8551058657467365} +{"Finetune/Learning Rate": 1.9999720583349015e-05, "Finetune/Loss": 1.0361663103103638, "Finetune/Loss (Raw)": 1.148529052734375, "Finetune/Step": 167, "Finetune/Step Time": 4.857086945325136} +{"Finetune/Learning Rate": 1.9999672073778905e-05, "Finetune/Loss": 1.0349159240722656, "Finetune/Loss (Raw)": 0.9892014861106873, "Finetune/Step": 168, "Finetune/Step Time": 4.845231218263507} +{"Finetune/Learning Rate": 1.999961968353129e-05, "Finetune/Loss": 1.0325875282287598, "Finetune/Loss (Raw)": 0.9709007143974304, "Finetune/Step": 169, "Finetune/Step Time": 4.850058684125543} +{"Finetune/Learning Rate": 1.9999563412626503e-05, "Finetune/Loss": 1.0258092880249023, "Finetune/Loss (Raw)": 0.3605409860610962, "Finetune/Step": 170, "Finetune/Step Time": 4.8848461620509624} +{"Finetune/Learning Rate": 1.9999503261086378e-05, "Finetune/Loss": 1.0246542692184448, "Finetune/Loss (Raw)": 1.0075633525848389, "Finetune/Step": 171, "Finetune/Step Time": 4.882433565333486} +{"Finetune/Learning Rate": 1.999943922893426e-05, "Finetune/Loss": 1.0231890678405762, "Finetune/Loss (Raw)": 0.9933118224143982, "Finetune/Step": 172, "Finetune/Step Time": 4.885873444378376} +{"Finetune/Learning Rate": 1.9999371316195005e-05, "Finetune/Loss": 1.0243210792541504, "Finetune/Loss (Raw)": 1.2039366960525513, "Finetune/Step": 173, "Finetune/Step Time": 4.85303782299161} +{"Finetune/Learning Rate": 1.9999299522894957e-05, "Finetune/Loss": 1.0290712118148804, "Finetune/Loss (Raw)": 0.902762234210968, "Finetune/Step": 174, "Finetune/Step Time": 4.807938136160374} +{"Finetune/Learning Rate": 1.9999223849061987e-05, "Finetune/Loss": 1.0294768810272217, "Finetune/Loss (Raw)": 1.1986688375473022, "Finetune/Step": 175, "Finetune/Step Time": 4.7997902277857065} +{"Finetune/Learning Rate": 1.999914429472546e-05, "Finetune/Loss": 1.023460865020752, "Finetune/Loss (Raw)": 0.35722431540489197, "Finetune/Step": 176, "Finetune/Step Time": 4.826578795909882} +{"Finetune/Learning Rate": 1.9999060859916247e-05, "Finetune/Loss": 1.0220345258712769, "Finetune/Loss (Raw)": 0.9782225489616394, "Finetune/Step": 177, "Finetune/Step Time": 4.829213947057724} +{"Finetune/Learning Rate": 1.9998973544666734e-05, "Finetune/Loss": 1.0234105587005615, "Finetune/Loss (Raw)": 1.2115668058395386, "Finetune/Step": 178, "Finetune/Step Time": 4.831404339522123} +{"Finetune/Learning Rate": 1.9998882349010798e-05, "Finetune/Loss": 1.0215449333190918, "Finetune/Loss (Raw)": 0.906628429889679, "Finetune/Step": 179, "Finetune/Step Time": 4.835313184186816} +{"Finetune/Learning Rate": 1.999878727298384e-05, "Finetune/Loss": 1.0204557180404663, "Finetune/Loss (Raw)": 1.0694953203201294, "Finetune/Step": 180, "Finetune/Step Time": 4.837907092645764} +{"Finetune/Learning Rate": 1.999868831662274e-05, "Finetune/Loss": 1.021263837814331, "Finetune/Loss (Raw)": 1.0243293046951294, "Finetune/Step": 181, "Finetune/Step Time": 4.82361794821918} +{"Finetune/Learning Rate": 1.9998585479965918e-05, "Finetune/Loss": 1.0222303867340088, "Finetune/Loss (Raw)": 1.06074059009552, "Finetune/Step": 182, "Finetune/Step Time": 4.828259147703648} +{"Finetune/Learning Rate": 1.999847876305328e-05, "Finetune/Loss": 1.021065354347229, "Finetune/Loss (Raw)": 1.0172228813171387, "Finetune/Step": 183, "Finetune/Step Time": 4.811271045356989} +{"Finetune/Learning Rate": 1.9998368165926237e-05, "Finetune/Loss": 1.021470308303833, "Finetune/Loss (Raw)": 1.1648606061935425, "Finetune/Step": 184, "Finetune/Step Time": 4.7977079432457685} +{"Finetune/Learning Rate": 1.9998253688627706e-05, "Finetune/Loss": 1.0207548141479492, "Finetune/Loss (Raw)": 0.9285845756530762, "Finetune/Step": 185, "Finetune/Step Time": 4.795624483376741} +{"Finetune/Learning Rate": 1.9998135331202124e-05, "Finetune/Loss": 1.0204360485076904, "Finetune/Loss (Raw)": 1.0207569599151611, "Finetune/Step": 186, "Finetune/Step Time": 4.785952236503363} +{"Finetune/Learning Rate": 1.9998013093695413e-05, "Finetune/Loss": 1.019735336303711, "Finetune/Loss (Raw)": 1.0140419006347656, "Finetune/Step": 187, "Finetune/Step Time": 4.771390713751316} +{"Finetune/Learning Rate": 1.9997886976155018e-05, "Finetune/Loss": 1.02009916305542, "Finetune/Loss (Raw)": 1.1394914388656616, "Finetune/Step": 188, "Finetune/Step Time": 4.770196473225951} +{"Finetune/Learning Rate": 1.9997756978629876e-05, "Finetune/Loss": 1.018998384475708, "Finetune/Loss (Raw)": 1.012096881866455, "Finetune/Step": 189, "Finetune/Step Time": 4.7816699501127005} +{"Finetune/Learning Rate": 1.999762310117044e-05, "Finetune/Loss": 1.018020510673523, "Finetune/Loss (Raw)": 1.0725817680358887, "Finetune/Step": 190, "Finetune/Step Time": 4.786520007997751} +{"Finetune/Learning Rate": 1.999748534382867e-05, "Finetune/Loss": 1.0170810222625732, "Finetune/Loss (Raw)": 1.0889593362808228, "Finetune/Step": 191, "Finetune/Step Time": 4.792039280757308} +{"Finetune/Learning Rate": 1.9997343706658023e-05, "Finetune/Loss": 1.0171451568603516, "Finetune/Loss (Raw)": 1.0744283199310303, "Finetune/Step": 192, "Finetune/Step Time": 4.771758023649454} +{"Finetune/Learning Rate": 1.9997198189713462e-05, "Finetune/Loss": 1.0163289308547974, "Finetune/Loss (Raw)": 0.9462729096412659, "Finetune/Step": 193, "Finetune/Step Time": 4.76411391235888} +{"Finetune/Learning Rate": 1.9997048793051463e-05, "Finetune/Loss": 1.016742467880249, "Finetune/Loss (Raw)": 1.0621600151062012, "Finetune/Step": 194, "Finetune/Step Time": 4.762590365484357} +{"Finetune/Learning Rate": 1.9996895516730005e-05, "Finetune/Loss": 1.0176447629928589, "Finetune/Loss (Raw)": 1.015832781791687, "Finetune/Step": 195, "Finetune/Step Time": 4.769241558387876} +{"Finetune/Learning Rate": 1.9996738360808566e-05, "Finetune/Loss": 1.0164334774017334, "Finetune/Loss (Raw)": 0.9758197069168091, "Finetune/Step": 196, "Finetune/Step Time": 4.7656072080135345} +{"Finetune/Learning Rate": 1.999657732534814e-05, "Finetune/Loss": 1.0156728029251099, "Finetune/Loss (Raw)": 1.0600210428237915, "Finetune/Step": 197, "Finetune/Step Time": 4.771891823038459} +{"Finetune/Learning Rate": 1.9996412410411224e-05, "Finetune/Loss": 1.0142498016357422, "Finetune/Loss (Raw)": 1.0145606994628906, "Finetune/Step": 198, "Finetune/Step Time": 4.771180922165513} +{"Finetune/Learning Rate": 1.9996243616061812e-05, "Finetune/Loss": 1.014807939529419, "Finetune/Loss (Raw)": 1.0021690130233765, "Finetune/Step": 199, "Finetune/Step Time": 4.775125170126557} +{"Finetune/Learning Rate": 1.9996070942365416e-05, "Finetune/Loss": 1.0149016380310059, "Finetune/Loss (Raw)": 1.0356810092926025, "Finetune/Step": 200, "Finetune/Step Time": 4.7796330489218235} +{"Finetune/Learning Rate": 1.9995894389389038e-05, "Finetune/Loss": 1.0165996551513672, "Finetune/Loss (Raw)": 1.1415072679519653, "Finetune/Step": 201, "Finetune/Step Time": 4.783850768581033} +{"Finetune/Learning Rate": 1.9995713957201208e-05, "Finetune/Loss": 1.0181224346160889, "Finetune/Loss (Raw)": 1.1311284303665161, "Finetune/Step": 202, "Finetune/Step Time": 4.787017310038209} +{"Finetune/Learning Rate": 1.9995529645871938e-05, "Finetune/Loss": 1.0200581550598145, "Finetune/Loss (Raw)": 1.0748003721237183, "Finetune/Step": 203, "Finetune/Step Time": 4.78119258210063} +{"Finetune/Learning Rate": 1.999534145547276e-05, "Finetune/Loss": 1.0207533836364746, "Finetune/Loss (Raw)": 1.067391276359558, "Finetune/Step": 204, "Finetune/Step Time": 4.775924468412995} +{"Finetune/Learning Rate": 1.99951493860767e-05, "Finetune/Loss": 1.014543056488037, "Finetune/Loss (Raw)": 0.3199898302555084, "Finetune/Step": 205, "Finetune/Step Time": 4.817376956343651} +{"Finetune/Learning Rate": 1.9994953437758305e-05, "Finetune/Loss": 1.0129551887512207, "Finetune/Loss (Raw)": 1.0175269842147827, "Finetune/Step": 206, "Finetune/Step Time": 4.825996110215783} +{"Finetune/Learning Rate": 1.999475361059362e-05, "Finetune/Loss": 1.0130159854888916, "Finetune/Loss (Raw)": 1.194056510925293, "Finetune/Step": 207, "Finetune/Step Time": 4.848897257819772} +{"Finetune/Learning Rate": 1.9994549904660187e-05, "Finetune/Loss": 1.0137044191360474, "Finetune/Loss (Raw)": 1.0648349523544312, "Finetune/Step": 208, "Finetune/Step Time": 4.833794327452779} +{"Finetune/Learning Rate": 1.9994342320037065e-05, "Finetune/Loss": 1.0126914978027344, "Finetune/Loss (Raw)": 1.0149608850479126, "Finetune/Step": 209, "Finetune/Step Time": 4.827653456479311} +{"Finetune/Learning Rate": 1.9994130856804808e-05, "Finetune/Loss": 1.0065786838531494, "Finetune/Loss (Raw)": 0.32195135951042175, "Finetune/Step": 210, "Finetune/Step Time": 4.8689441457390785} +{"Finetune/Learning Rate": 1.9993915515045485e-05, "Finetune/Loss": 1.006834864616394, "Finetune/Loss (Raw)": 0.9947118759155273, "Finetune/Step": 211, "Finetune/Step Time": 4.8614448849111795} +{"Finetune/Learning Rate": 1.999369629484267e-05, "Finetune/Loss": 1.0059868097305298, "Finetune/Loss (Raw)": 1.0809961557388306, "Finetune/Step": 212, "Finetune/Step Time": 4.859126741066575} +{"Finetune/Learning Rate": 1.999347319628143e-05, "Finetune/Loss": 1.0071451663970947, "Finetune/Loss (Raw)": 1.1952693462371826, "Finetune/Step": 213, "Finetune/Step Time": 4.861535206437111} +{"Finetune/Learning Rate": 1.9993246219448347e-05, "Finetune/Loss": 1.0121405124664307, "Finetune/Loss (Raw)": 0.9917219877243042, "Finetune/Step": 214, "Finetune/Step Time": 4.837958266958594} +{"Finetune/Learning Rate": 1.999301536443151e-05, "Finetune/Loss": 1.011547327041626, "Finetune/Loss (Raw)": 0.9542887806892395, "Finetune/Step": 215, "Finetune/Step Time": 4.837659938260913} +{"Finetune/Learning Rate": 1.9992780631320508e-05, "Finetune/Loss": 1.0093655586242676, "Finetune/Loss (Raw)": 0.9460722804069519, "Finetune/Step": 216, "Finetune/Step Time": 4.836353808641434} +{"Finetune/Learning Rate": 1.9992542020206432e-05, "Finetune/Loss": 1.0097301006317139, "Finetune/Loss (Raw)": 1.119598627090454, "Finetune/Step": 217, "Finetune/Step Time": 4.834226205945015} +{"Finetune/Learning Rate": 1.9992299531181892e-05, "Finetune/Loss": 1.0092147588729858, "Finetune/Loss (Raw)": 1.078692078590393, "Finetune/Step": 218, "Finetune/Step Time": 4.823323057964444} +{"Finetune/Learning Rate": 1.999205316434098e-05, "Finetune/Loss": 1.0096116065979004, "Finetune/Loss (Raw)": 1.211639165878296, "Finetune/Step": 219, "Finetune/Step Time": 4.811580061912537} +{"Finetune/Learning Rate": 1.9991802919779318e-05, "Finetune/Loss": 1.0078270435333252, "Finetune/Loss (Raw)": 0.9766735434532166, "Finetune/Step": 220, "Finetune/Step Time": 4.808448009192944} +{"Finetune/Learning Rate": 1.9991548797594017e-05, "Finetune/Loss": 1.00937020778656, "Finetune/Loss (Raw)": 1.1195920705795288, "Finetune/Step": 221, "Finetune/Step Time": 4.8118627946823835} +{"Finetune/Learning Rate": 1.9991290797883693e-05, "Finetune/Loss": 1.007800817489624, "Finetune/Loss (Raw)": 0.9609492421150208, "Finetune/Step": 222, "Finetune/Step Time": 4.817671289667487} +{"Finetune/Learning Rate": 1.9991028920748475e-05, "Finetune/Loss": 1.0069916248321533, "Finetune/Loss (Raw)": 0.9945836067199707, "Finetune/Step": 223, "Finetune/Step Time": 4.81893596611917} +{"Finetune/Learning Rate": 1.9990763166289993e-05, "Finetune/Loss": 1.0013036727905273, "Finetune/Loss (Raw)": 0.3576147258281708, "Finetune/Step": 224, "Finetune/Step Time": 4.8430486880242825} +{"Finetune/Learning Rate": 1.9990493534611376e-05, "Finetune/Loss": 1.0015368461608887, "Finetune/Loss (Raw)": 1.0579627752304077, "Finetune/Step": 225, "Finetune/Step Time": 4.843162000179291} +{"Finetune/Learning Rate": 1.9990220025817268e-05, "Finetune/Loss": 1.002030611038208, "Finetune/Loss (Raw)": 1.1942018270492554, "Finetune/Step": 226, "Finetune/Step Time": 4.8409212827682495} +{"Finetune/Learning Rate": 1.998994264001381e-05, "Finetune/Loss": 1.0096449851989746, "Finetune/Loss (Raw)": 1.1964656114578247, "Finetune/Step": 227, "Finetune/Step Time": 4.804918721318245} +{"Finetune/Learning Rate": 1.9989661377308654e-05, "Finetune/Loss": 1.009204387664795, "Finetune/Loss (Raw)": 1.1597074270248413, "Finetune/Step": 228, "Finetune/Step Time": 4.800188226625323} +{"Finetune/Learning Rate": 1.9989376237810944e-05, "Finetune/Loss": 1.0090923309326172, "Finetune/Loss (Raw)": 1.0382673740386963, "Finetune/Step": 229, "Finetune/Step Time": 4.79347612336278} +{"Finetune/Learning Rate": 1.9989087221631344e-05, "Finetune/Loss": 1.0087308883666992, "Finetune/Loss (Raw)": 1.1630107164382935, "Finetune/Step": 230, "Finetune/Step Time": 4.7961524818092585} +{"Finetune/Learning Rate": 1.9988794328882017e-05, "Finetune/Loss": 1.0143967866897583, "Finetune/Loss (Raw)": 1.019540786743164, "Finetune/Step": 231, "Finetune/Step Time": 4.761441884562373} +{"Finetune/Learning Rate": 1.9988497559676622e-05, "Finetune/Loss": 1.0136921405792236, "Finetune/Loss (Raw)": 1.0683162212371826, "Finetune/Step": 232, "Finetune/Step Time": 4.7648388016968966} +{"Finetune/Learning Rate": 1.9988196914130332e-05, "Finetune/Loss": 1.007799744606018, "Finetune/Loss (Raw)": 0.31069010496139526, "Finetune/Step": 233, "Finetune/Step Time": 4.8042896296828985} +{"Finetune/Learning Rate": 1.9987892392359827e-05, "Finetune/Loss": 1.0085943937301636, "Finetune/Loss (Raw)": 1.1670602560043335, "Finetune/Step": 234, "Finetune/Step Time": 4.82376679033041} +{"Finetune/Learning Rate": 1.998758399448328e-05, "Finetune/Loss": 1.0091677904129028, "Finetune/Loss (Raw)": 1.2064013481140137, "Finetune/Step": 235, "Finetune/Step Time": 4.865012254565954} +{"Finetune/Learning Rate": 1.9987271720620376e-05, "Finetune/Loss": 1.0092337131500244, "Finetune/Loss (Raw)": 1.0258996486663818, "Finetune/Step": 236, "Finetune/Step Time": 4.865047287195921} +{"Finetune/Learning Rate": 1.9986955570892302e-05, "Finetune/Loss": 1.0088646411895752, "Finetune/Loss (Raw)": 1.0988832712173462, "Finetune/Step": 237, "Finetune/Step Time": 4.8930548299103975} +{"Finetune/Learning Rate": 1.998663554542175e-05, "Finetune/Loss": 1.008601188659668, "Finetune/Loss (Raw)": 1.019920825958252, "Finetune/Step": 238, "Finetune/Step Time": 4.896785002201796} +{"Finetune/Learning Rate": 1.9986311644332913e-05, "Finetune/Loss": 1.006899356842041, "Finetune/Loss (Raw)": 0.9008240699768066, "Finetune/Step": 239, "Finetune/Step Time": 4.880852818489075} +{"Finetune/Learning Rate": 1.9985983867751497e-05, "Finetune/Loss": 1.006001591682434, "Finetune/Loss (Raw)": 0.9701222777366638, "Finetune/Step": 240, "Finetune/Step Time": 4.873681189492345} +{"Finetune/Learning Rate": 1.99856522158047e-05, "Finetune/Loss": 1.005833625793457, "Finetune/Loss (Raw)": 1.1070668697357178, "Finetune/Step": 241, "Finetune/Step Time": 4.861052790656686} +{"Finetune/Learning Rate": 1.998531668862123e-05, "Finetune/Loss": 1.012269139289856, "Finetune/Loss (Raw)": 1.072162389755249, "Finetune/Step": 242, "Finetune/Step Time": 4.818269852548838} +{"Finetune/Learning Rate": 1.9984977286331302e-05, "Finetune/Loss": 1.012197494506836, "Finetune/Loss (Raw)": 1.0821377038955688, "Finetune/Step": 243, "Finetune/Step Time": 4.845863031223416} +{"Finetune/Learning Rate": 1.998463400906663e-05, "Finetune/Loss": 1.012194275856018, "Finetune/Loss (Raw)": 0.9271067976951599, "Finetune/Step": 244, "Finetune/Step Time": 4.825233394280076} +{"Finetune/Learning Rate": 1.998428685696043e-05, "Finetune/Loss": 1.0129998922348022, "Finetune/Loss (Raw)": 1.2140793800354004, "Finetune/Step": 245, "Finetune/Step Time": 4.8176498506218195} +{"Finetune/Learning Rate": 1.998393583014743e-05, "Finetune/Loss": 1.0132920742034912, "Finetune/Loss (Raw)": 1.1755642890930176, "Finetune/Step": 246, "Finetune/Step Time": 4.826828608289361} +{"Finetune/Learning Rate": 1.9983580928763852e-05, "Finetune/Loss": 1.01344895362854, "Finetune/Loss (Raw)": 1.068515658378601, "Finetune/Step": 247, "Finetune/Step Time": 4.826703481376171} +{"Finetune/Learning Rate": 1.998322215294743e-05, "Finetune/Loss": 1.0127772092819214, "Finetune/Loss (Raw)": 1.0080920457839966, "Finetune/Step": 248, "Finetune/Step Time": 4.8351213950663805} +{"Finetune/Learning Rate": 1.9982859502837395e-05, "Finetune/Loss": 1.0127558708190918, "Finetune/Loss (Raw)": 1.0257126092910767, "Finetune/Step": 249, "Finetune/Step Time": 4.817378453910351} +{"Finetune/Learning Rate": 1.998249297857449e-05, "Finetune/Loss": 1.0137122869491577, "Finetune/Loss (Raw)": 1.0870895385742188, "Finetune/Step": 250, "Finetune/Step Time": 4.815754935145378} +{"Finetune/Learning Rate": 1.9982122580300947e-05, "Finetune/Loss": 1.0147535800933838, "Finetune/Loss (Raw)": 1.1504123210906982, "Finetune/Step": 251, "Finetune/Step Time": 4.83079494908452} +{"Finetune/Learning Rate": 1.9981748308160517e-05, "Finetune/Loss": 1.0140891075134277, "Finetune/Loss (Raw)": 1.043907880783081, "Finetune/Step": 252, "Finetune/Step Time": 4.8298006542027} +{"Finetune/Learning Rate": 1.9981370162298444e-05, "Finetune/Loss": 1.0146992206573486, "Finetune/Loss (Raw)": 1.091740608215332, "Finetune/Step": 253, "Finetune/Step Time": 4.824828194454312} +{"Finetune/Learning Rate": 1.9980988142861486e-05, "Finetune/Loss": 1.0137603282928467, "Finetune/Loss (Raw)": 0.9820236563682556, "Finetune/Step": 254, "Finetune/Step Time": 4.804557869210839} +{"Finetune/Learning Rate": 1.998060224999789e-05, "Finetune/Loss": 1.008522868156433, "Finetune/Loss (Raw)": 0.31691351532936096, "Finetune/Step": 255, "Finetune/Step Time": 4.839970832690597} +{"Finetune/Learning Rate": 1.9980212483857412e-05, "Finetune/Loss": 1.007168173789978, "Finetune/Loss (Raw)": 0.9561819434165955, "Finetune/Step": 256, "Finetune/Step Time": 4.840363243594766} +{"Finetune/Learning Rate": 1.9979818844591318e-05, "Finetune/Loss": 1.0073513984680176, "Finetune/Loss (Raw)": 1.1746110916137695, "Finetune/Step": 257, "Finetune/Step Time": 4.840043308213353} +{"Finetune/Learning Rate": 1.9979421332352372e-05, "Finetune/Loss": 1.0072218179702759, "Finetune/Loss (Raw)": 1.1319338083267212, "Finetune/Step": 258, "Finetune/Step Time": 4.839964067563415} +{"Finetune/Learning Rate": 1.9979019947294836e-05, "Finetune/Loss": 1.0070757865905762, "Finetune/Loss (Raw)": 1.0847105979919434, "Finetune/Step": 259, "Finetune/Step Time": 4.842470264062285} +{"Finetune/Learning Rate": 1.9978614689574486e-05, "Finetune/Loss": 1.0070579051971436, "Finetune/Loss (Raw)": 1.1062374114990234, "Finetune/Step": 260, "Finetune/Step Time": 4.86279596015811} +{"Finetune/Learning Rate": 1.9978205559348586e-05, "Finetune/Loss": 1.0069271326065063, "Finetune/Loss (Raw)": 0.8827531337738037, "Finetune/Step": 261, "Finetune/Step Time": 4.845550984144211} +{"Finetune/Learning Rate": 1.997779255677592e-05, "Finetune/Loss": 1.003871202468872, "Finetune/Loss (Raw)": 0.8537563681602478, "Finetune/Step": 262, "Finetune/Step Time": 4.856960315257311} +{"Finetune/Learning Rate": 1.997737568201676e-05, "Finetune/Loss": 1.0041255950927734, "Finetune/Loss (Raw)": 1.0338306427001953, "Finetune/Step": 263, "Finetune/Step Time": 4.855067107826471} +{"Finetune/Learning Rate": 1.9976954935232892e-05, "Finetune/Loss": 1.0034276247024536, "Finetune/Loss (Raw)": 0.9969619512557983, "Finetune/Step": 264, "Finetune/Step Time": 4.869632674381137} +{"Finetune/Learning Rate": 1.99765303165876e-05, "Finetune/Loss": 1.003580093383789, "Finetune/Loss (Raw)": 0.9961600303649902, "Finetune/Step": 265, "Finetune/Step Time": 4.866041447967291} +{"Finetune/Learning Rate": 1.997610182624566e-05, "Finetune/Loss": 1.0044519901275635, "Finetune/Loss (Raw)": 1.2319679260253906, "Finetune/Step": 266, "Finetune/Step Time": 4.877500653266907} +{"Finetune/Learning Rate": 1.9975669464373377e-05, "Finetune/Loss": 1.0043926239013672, "Finetune/Loss (Raw)": 0.9085119962692261, "Finetune/Step": 267, "Finetune/Step Time": 4.875272227451205} +{"Finetune/Learning Rate": 1.9975233231138525e-05, "Finetune/Loss": 1.0030590295791626, "Finetune/Loss (Raw)": 1.1163121461868286, "Finetune/Step": 268, "Finetune/Step Time": 4.879291292279959} +{"Finetune/Learning Rate": 1.997479312671041e-05, "Finetune/Loss": 1.0030992031097412, "Finetune/Loss (Raw)": 1.0462665557861328, "Finetune/Step": 269, "Finetune/Step Time": 4.885025447234511} +{"Finetune/Learning Rate": 1.9974349151259824e-05, "Finetune/Loss": 1.003335952758789, "Finetune/Loss (Raw)": 1.060725212097168, "Finetune/Step": 270, "Finetune/Step Time": 4.847437780350447} +{"Finetune/Learning Rate": 1.9973901304959066e-05, "Finetune/Loss": 1.0038447380065918, "Finetune/Loss (Raw)": 1.0331984758377075, "Finetune/Step": 271, "Finetune/Step Time": 4.838370272889733} +{"Finetune/Learning Rate": 1.9973449587981935e-05, "Finetune/Loss": 1.0096484422683716, "Finetune/Loss (Raw)": 1.1163361072540283, "Finetune/Step": 272, "Finetune/Step Time": 4.8063692562282085} +{"Finetune/Learning Rate": 1.9972994000503734e-05, "Finetune/Loss": 1.0151879787445068, "Finetune/Loss (Raw)": 1.0546693801879883, "Finetune/Step": 273, "Finetune/Step Time": 4.770397303625941} +{"Finetune/Learning Rate": 1.997253454270127e-05, "Finetune/Loss": 1.0161890983581543, "Finetune/Loss (Raw)": 1.1011276245117188, "Finetune/Step": 274, "Finetune/Step Time": 4.776528852060437} +{"Finetune/Learning Rate": 1.9972071214752843e-05, "Finetune/Loss": 1.017159104347229, "Finetune/Loss (Raw)": 1.025151252746582, "Finetune/Step": 275, "Finetune/Step Time": 4.7736970745027065} +{"Finetune/Learning Rate": 1.9971604016838264e-05, "Finetune/Loss": 1.015285849571228, "Finetune/Loss (Raw)": 0.8351308703422546, "Finetune/Step": 276, "Finetune/Step Time": 4.806874819099903} +{"Finetune/Learning Rate": 1.997113294913885e-05, "Finetune/Loss": 1.0145689249038696, "Finetune/Loss (Raw)": 1.0293028354644775, "Finetune/Step": 277, "Finetune/Step Time": 4.805499324575067} +{"Finetune/Learning Rate": 1.9970658011837404e-05, "Finetune/Loss": 1.0138652324676514, "Finetune/Loss (Raw)": 1.0396803617477417, "Finetune/Step": 278, "Finetune/Step Time": 4.798171680420637} +{"Finetune/Learning Rate": 1.997017920511825e-05, "Finetune/Loss": 1.0148073434829712, "Finetune/Loss (Raw)": 1.215876579284668, "Finetune/Step": 279, "Finetune/Step Time": 4.804608331993222} +{"Finetune/Learning Rate": 1.9969696529167197e-05, "Finetune/Loss": 1.0143113136291504, "Finetune/Loss (Raw)": 0.9923790693283081, "Finetune/Step": 280, "Finetune/Step Time": 4.802821438759565} +{"Finetune/Learning Rate": 1.996920998417156e-05, "Finetune/Loss": 1.0137999057769775, "Finetune/Loss (Raw)": 1.0873379707336426, "Finetune/Step": 281, "Finetune/Step Time": 4.788371108472347} +{"Finetune/Learning Rate": 1.996871957032016e-05, "Finetune/Loss": 1.0129101276397705, "Finetune/Loss (Raw)": 1.023831844329834, "Finetune/Step": 282, "Finetune/Step Time": 4.787540635094047} +{"Finetune/Learning Rate": 1.996822528780332e-05, "Finetune/Loss": 1.0123610496520996, "Finetune/Loss (Raw)": 1.1427464485168457, "Finetune/Step": 283, "Finetune/Step Time": 4.786078995093703} +{"Finetune/Learning Rate": 1.996772713681286e-05, "Finetune/Loss": 1.019076943397522, "Finetune/Loss (Raw)": 1.1598931550979614, "Finetune/Step": 284, "Finetune/Step Time": 4.747539324685931} +{"Finetune/Learning Rate": 1.9967225117542097e-05, "Finetune/Loss": 1.018505334854126, "Finetune/Loss (Raw)": 1.010054588317871, "Finetune/Step": 285, "Finetune/Step Time": 4.765451719984412} +{"Finetune/Learning Rate": 1.9966719230185864e-05, "Finetune/Loss": 1.0182892084121704, "Finetune/Loss (Raw)": 1.0817457437515259, "Finetune/Step": 286, "Finetune/Step Time": 4.780847918242216} +{"Finetune/Learning Rate": 1.9966209474940477e-05, "Finetune/Loss": 1.0183398723602295, "Finetune/Loss (Raw)": 1.0840709209442139, "Finetune/Step": 287, "Finetune/Step Time": 4.787611790001392} +{"Finetune/Learning Rate": 1.996569585200377e-05, "Finetune/Loss": 1.0174764394760132, "Finetune/Loss (Raw)": 0.9078693389892578, "Finetune/Step": 288, "Finetune/Step Time": 4.795610599219799} +{"Finetune/Learning Rate": 1.9965178361575068e-05, "Finetune/Loss": 1.018219232559204, "Finetune/Loss (Raw)": 1.1302947998046875, "Finetune/Step": 289, "Finetune/Step Time": 4.800149554386735} +{"Finetune/Learning Rate": 1.9964657003855197e-05, "Finetune/Loss": 1.0158542394638062, "Finetune/Loss (Raw)": 0.9627829194068909, "Finetune/Step": 290, "Finetune/Step Time": 4.811346501111984} +{"Finetune/Learning Rate": 1.9964131779046484e-05, "Finetune/Loss": 1.0162074565887451, "Finetune/Loss (Raw)": 1.1277713775634766, "Finetune/Step": 291, "Finetune/Step Time": 4.805079810321331} +{"Finetune/Learning Rate": 1.996360268735276e-05, "Finetune/Loss": 1.017332673072815, "Finetune/Loss (Raw)": 1.1091307401657104, "Finetune/Step": 292, "Finetune/Step Time": 4.793247716501355} +{"Finetune/Learning Rate": 1.9963069728979357e-05, "Finetune/Loss": 1.018059492111206, "Finetune/Loss (Raw)": 1.1505107879638672, "Finetune/Step": 293, "Finetune/Step Time": 4.788916260004044} +{"Finetune/Learning Rate": 1.9962532904133107e-05, "Finetune/Loss": 1.016221046447754, "Finetune/Loss (Raw)": 0.7649643421173096, "Finetune/Step": 294, "Finetune/Step Time": 4.795366844162345} +{"Finetune/Learning Rate": 1.996199221302234e-05, "Finetune/Loss": 1.0158305168151855, "Finetune/Loss (Raw)": 1.0985543727874756, "Finetune/Step": 295, "Finetune/Step Time": 4.801362069323659} +{"Finetune/Learning Rate": 1.9961447655856883e-05, "Finetune/Loss": 1.0160942077636719, "Finetune/Loss (Raw)": 1.0229402780532837, "Finetune/Step": 296, "Finetune/Step Time": 4.806039368733764} +{"Finetune/Learning Rate": 1.9960899232848074e-05, "Finetune/Loss": 1.017385721206665, "Finetune/Loss (Raw)": 1.1362172365188599, "Finetune/Step": 297, "Finetune/Step Time": 4.809172719717026} +{"Finetune/Learning Rate": 1.9960346944208743e-05, "Finetune/Loss": 1.0217134952545166, "Finetune/Loss (Raw)": 0.9144951701164246, "Finetune/Step": 298, "Finetune/Step Time": 4.7994728069752455} +{"Finetune/Learning Rate": 1.995979079015322e-05, "Finetune/Loss": 1.0162535905838013, "Finetune/Loss (Raw)": 0.30869486927986145, "Finetune/Step": 299, "Finetune/Step Time": 4.8307423163205385} +{"Finetune/Learning Rate": 1.995923077089734e-05, "Finetune/Loss": 1.0113564729690552, "Finetune/Loss (Raw)": 0.366478830575943, "Finetune/Step": 300, "Finetune/Step Time": 4.854255750775337} +{"Finetune/Learning Rate": 1.9958666886658436e-05, "Finetune/Loss": 1.005061388015747, "Finetune/Loss (Raw)": 0.398165225982666, "Finetune/Step": 301, "Finetune/Step Time": 4.890719462186098} +{"Finetune/Learning Rate": 1.995809913765534e-05, "Finetune/Loss": 1.0049843788146973, "Finetune/Loss (Raw)": 0.8929084539413452, "Finetune/Step": 302, "Finetune/Step Time": 4.904545959085226} +{"Finetune/Learning Rate": 1.9957527524108386e-05, "Finetune/Loss": 0.9981368780136108, "Finetune/Loss (Raw)": 0.3221999406814575, "Finetune/Step": 303, "Finetune/Step Time": 4.939644854515791} +{"Finetune/Learning Rate": 1.99569520462394e-05, "Finetune/Loss": 1.0039267539978027, "Finetune/Loss (Raw)": 1.0983179807662964, "Finetune/Step": 304, "Finetune/Step Time": 4.89979924634099} +{"Finetune/Learning Rate": 1.995637270427172e-05, "Finetune/Loss": 1.0044865608215332, "Finetune/Loss (Raw)": 1.049882173538208, "Finetune/Step": 305, "Finetune/Step Time": 4.897319016978145} +{"Finetune/Learning Rate": 1.9955789498430178e-05, "Finetune/Loss": 1.0023895502090454, "Finetune/Loss (Raw)": 0.9431445002555847, "Finetune/Step": 306, "Finetune/Step Time": 4.897751780226827} +{"Finetune/Learning Rate": 1.99552024289411e-05, "Finetune/Loss": 1.0044264793395996, "Finetune/Loss (Raw)": 1.167359709739685, "Finetune/Step": 307, "Finetune/Step Time": 4.905918195843697} +{"Finetune/Learning Rate": 1.995461149603232e-05, "Finetune/Loss": 1.0042970180511475, "Finetune/Loss (Raw)": 1.0529085397720337, "Finetune/Step": 308, "Finetune/Step Time": 4.900589158758521} +{"Finetune/Learning Rate": 1.9954016699933163e-05, "Finetune/Loss": 1.005305528640747, "Finetune/Loss (Raw)": 1.1534397602081299, "Finetune/Step": 309, "Finetune/Step Time": 4.914692437276244} +{"Finetune/Learning Rate": 1.995341804087446e-05, "Finetune/Loss": 1.004960536956787, "Finetune/Loss (Raw)": 1.0165812969207764, "Finetune/Step": 310, "Finetune/Step Time": 4.9038992915302515} +{"Finetune/Learning Rate": 1.9952815519088544e-05, "Finetune/Loss": 1.005100131034851, "Finetune/Loss (Raw)": 1.0350896120071411, "Finetune/Step": 311, "Finetune/Step Time": 4.916434062644839} +{"Finetune/Learning Rate": 1.9952209134809234e-05, "Finetune/Loss": 1.0037217140197754, "Finetune/Loss (Raw)": 0.9884116649627686, "Finetune/Step": 312, "Finetune/Step Time": 4.915145214647055} +{"Finetune/Learning Rate": 1.995159888827186e-05, "Finetune/Loss": 1.005194902420044, "Finetune/Loss (Raw)": 1.1171631813049316, "Finetune/Step": 313, "Finetune/Step Time": 4.922990126535296} +{"Finetune/Learning Rate": 1.9950984779713244e-05, "Finetune/Loss": 1.0065035820007324, "Finetune/Loss (Raw)": 1.1882535219192505, "Finetune/Step": 314, "Finetune/Step Time": 4.906564539298415} +{"Finetune/Learning Rate": 1.9950366809371713e-05, "Finetune/Loss": 1.0063345432281494, "Finetune/Loss (Raw)": 0.992415726184845, "Finetune/Step": 315, "Finetune/Step Time": 4.912775846198201} +{"Finetune/Learning Rate": 1.9949744977487087e-05, "Finetune/Loss": 1.0003875494003296, "Finetune/Loss (Raw)": 0.3782693147659302, "Finetune/Step": 316, "Finetune/Step Time": 4.948899015784264} +{"Finetune/Learning Rate": 1.994911928430069e-05, "Finetune/Loss": 1.0000795125961304, "Finetune/Loss (Raw)": 0.9726719260215759, "Finetune/Step": 317, "Finetune/Step Time": 4.938765771687031} +{"Finetune/Learning Rate": 1.994848973005534e-05, "Finetune/Loss": 1.0000190734863281, "Finetune/Loss (Raw)": 1.0648345947265625, "Finetune/Step": 318, "Finetune/Step Time": 4.932230459526181} +{"Finetune/Learning Rate": 1.994785631499535e-05, "Finetune/Loss": 0.9992620944976807, "Finetune/Loss (Raw)": 0.9920819401741028, "Finetune/Step": 319, "Finetune/Step Time": 4.939848056063056} +{"Finetune/Learning Rate": 1.994721903936654e-05, "Finetune/Loss": 0.9982771873474121, "Finetune/Loss (Raw)": 0.9483577609062195, "Finetune/Step": 320, "Finetune/Step Time": 4.934711284935474} +{"Finetune/Learning Rate": 1.9946577903416224e-05, "Finetune/Loss": 0.993471622467041, "Finetune/Loss (Raw)": 0.3311627209186554, "Finetune/Step": 321, "Finetune/Step Time": 4.978515770286322} +{"Finetune/Learning Rate": 1.994593290739322e-05, "Finetune/Loss": 0.9929993748664856, "Finetune/Loss (Raw)": 1.0017036199569702, "Finetune/Step": 322, "Finetune/Step Time": 4.979402860626578} +{"Finetune/Learning Rate": 1.9945284051547823e-05, "Finetune/Loss": 0.9934435486793518, "Finetune/Loss (Raw)": 1.072683334350586, "Finetune/Step": 323, "Finetune/Step Time": 4.97369210422039} +{"Finetune/Learning Rate": 1.9944631336131856e-05, "Finetune/Loss": 0.9944618344306946, "Finetune/Loss (Raw)": 1.1061643362045288, "Finetune/Step": 324, "Finetune/Step Time": 4.972665261477232} +{"Finetune/Learning Rate": 1.994397476139862e-05, "Finetune/Loss": 0.9943755269050598, "Finetune/Loss (Raw)": 1.048974871635437, "Finetune/Step": 325, "Finetune/Step Time": 4.969766654074192} +{"Finetune/Learning Rate": 1.994331432760292e-05, "Finetune/Loss": 0.9959875345230103, "Finetune/Loss (Raw)": 1.220895767211914, "Finetune/Step": 326, "Finetune/Step Time": 4.970140393823385} +{"Finetune/Learning Rate": 1.9942650035001053e-05, "Finetune/Loss": 0.9961209297180176, "Finetune/Loss (Raw)": 1.0192492008209229, "Finetune/Step": 327, "Finetune/Step Time": 4.964552488178015} +{"Finetune/Learning Rate": 1.994198188385082e-05, "Finetune/Loss": 0.9964043498039246, "Finetune/Loss (Raw)": 1.0719573497772217, "Finetune/Step": 328, "Finetune/Step Time": 4.971379486843944} +{"Finetune/Learning Rate": 1.9941309874411525e-05, "Finetune/Loss": 0.9949183464050293, "Finetune/Loss (Raw)": 0.951299786567688, "Finetune/Step": 329, "Finetune/Step Time": 4.98810338228941} +{"Finetune/Learning Rate": 1.9940634006943952e-05, "Finetune/Loss": 0.99297696352005, "Finetune/Loss (Raw)": 0.8826269507408142, "Finetune/Step": 330, "Finetune/Step Time": 4.992394773289561} +{"Finetune/Learning Rate": 1.9939954281710395e-05, "Finetune/Loss": 0.992847204208374, "Finetune/Loss (Raw)": 1.0581889152526855, "Finetune/Step": 331, "Finetune/Step Time": 4.996373379603028} +{"Finetune/Learning Rate": 1.9939270698974643e-05, "Finetune/Loss": 0.9909743070602417, "Finetune/Loss (Raw)": 0.8276589512825012, "Finetune/Step": 332, "Finetune/Step Time": 5.012874776497483} +{"Finetune/Learning Rate": 1.9938583259001983e-05, "Finetune/Loss": 0.9965864419937134, "Finetune/Loss (Raw)": 1.0383461713790894, "Finetune/Step": 333, "Finetune/Step Time": 4.978375738486648} +{"Finetune/Learning Rate": 1.993789196205919e-05, "Finetune/Loss": 0.9964638948440552, "Finetune/Loss (Raw)": 1.0018336772918701, "Finetune/Step": 334, "Finetune/Step Time": 4.973802980035543} +{"Finetune/Learning Rate": 1.9937196808414552e-05, "Finetune/Loss": 0.9950702786445618, "Finetune/Loss (Raw)": 1.0156842470169067, "Finetune/Step": 335, "Finetune/Step Time": 4.961404073983431} +{"Finetune/Learning Rate": 1.993649779833784e-05, "Finetune/Loss": 0.9953713417053223, "Finetune/Loss (Raw)": 1.1033636331558228, "Finetune/Step": 336, "Finetune/Step Time": 4.964655261486769} +{"Finetune/Learning Rate": 1.993579493210032e-05, "Finetune/Loss": 0.9970636367797852, "Finetune/Loss (Raw)": 1.231582760810852, "Finetune/Step": 337, "Finetune/Step Time": 4.96037701703608} +{"Finetune/Learning Rate": 1.9935088209974774e-05, "Finetune/Loss": 1.0023787021636963, "Finetune/Loss (Raw)": 1.002279281616211, "Finetune/Step": 338, "Finetune/Step Time": 4.928338550031185} +{"Finetune/Learning Rate": 1.9934377632235455e-05, "Finetune/Loss": 1.002167820930481, "Finetune/Loss (Raw)": 0.9677096009254456, "Finetune/Step": 339, "Finetune/Step Time": 4.9313140120357275} +{"Finetune/Learning Rate": 1.9933663199158135e-05, "Finetune/Loss": 1.0008740425109863, "Finetune/Loss (Raw)": 0.9154032468795776, "Finetune/Step": 340, "Finetune/Step Time": 4.9296942465007305} +{"Finetune/Learning Rate": 1.993294491102006e-05, "Finetune/Loss": 0.9995672106742859, "Finetune/Loss (Raw)": 1.0279908180236816, "Finetune/Step": 341, "Finetune/Step Time": 4.931028572842479} +{"Finetune/Learning Rate": 1.9932222768099992e-05, "Finetune/Loss": 1.0003864765167236, "Finetune/Loss (Raw)": 1.0965766906738281, "Finetune/Step": 342, "Finetune/Step Time": 4.9123743530362844} +{"Finetune/Learning Rate": 1.9931496770678178e-05, "Finetune/Loss": 1.0009113550186157, "Finetune/Loss (Raw)": 1.0214788913726807, "Finetune/Step": 343, "Finetune/Step Time": 4.910938452929258} +{"Finetune/Learning Rate": 1.993076691903636e-05, "Finetune/Loss": 1.0020339488983154, "Finetune/Loss (Raw)": 1.0897727012634277, "Finetune/Step": 344, "Finetune/Step Time": 4.896002434194088} +{"Finetune/Learning Rate": 1.993003321345779e-05, "Finetune/Loss": 1.0020945072174072, "Finetune/Loss (Raw)": 1.1273595094680786, "Finetune/Step": 345, "Finetune/Step Time": 4.898556465283036} +{"Finetune/Learning Rate": 1.9929295654227193e-05, "Finetune/Loss": 1.0024759769439697, "Finetune/Loss (Raw)": 1.127502679824829, "Finetune/Step": 346, "Finetune/Step Time": 4.905467990785837} +{"Finetune/Learning Rate": 1.9928554241630804e-05, "Finetune/Loss": 1.0020462274551392, "Finetune/Loss (Raw)": 1.1566256284713745, "Finetune/Step": 347, "Finetune/Step Time": 4.908712949603796} +{"Finetune/Learning Rate": 1.9927808975956354e-05, "Finetune/Loss": 1.0028800964355469, "Finetune/Loss (Raw)": 1.0834133625030518, "Finetune/Step": 348, "Finetune/Step Time": 4.929172519594431} +{"Finetune/Learning Rate": 1.9927059857493068e-05, "Finetune/Loss": 1.0031991004943848, "Finetune/Loss (Raw)": 1.1604337692260742, "Finetune/Step": 349, "Finetune/Step Time": 4.937881786376238} +{"Finetune/Learning Rate": 1.9926306886531658e-05, "Finetune/Loss": 1.0034284591674805, "Finetune/Loss (Raw)": 0.9903117418289185, "Finetune/Step": 350, "Finetune/Step Time": 4.952947203069925} +{"Finetune/Learning Rate": 1.9925550063364337e-05, "Finetune/Loss": 1.0034997463226318, "Finetune/Loss (Raw)": 1.0037020444869995, "Finetune/Step": 351, "Finetune/Step Time": 4.966374268755317} +{"Finetune/Learning Rate": 1.992478938828482e-05, "Finetune/Loss": 1.0086078643798828, "Finetune/Loss (Raw)": 1.011447548866272, "Finetune/Step": 352, "Finetune/Step Time": 4.940945355221629} +{"Finetune/Learning Rate": 1.9924024861588307e-05, "Finetune/Loss": 1.0080153942108154, "Finetune/Loss (Raw)": 0.9821395874023438, "Finetune/Step": 353, "Finetune/Step Time": 4.940056150779128} +{"Finetune/Learning Rate": 1.9923256483571495e-05, "Finetune/Loss": 1.0058083534240723, "Finetune/Loss (Raw)": 0.9116818904876709, "Finetune/Step": 354, "Finetune/Step Time": 4.946760771796107} +{"Finetune/Learning Rate": 1.9922484254532578e-05, "Finetune/Loss": 1.0048515796661377, "Finetune/Loss (Raw)": 1.0740045309066772, "Finetune/Step": 355, "Finetune/Step Time": 4.956500884145498} +{"Finetune/Learning Rate": 1.9921708174771237e-05, "Finetune/Loss": 1.0029807090759277, "Finetune/Loss (Raw)": 0.9202373623847961, "Finetune/Step": 356, "Finetune/Step Time": 4.96386213041842} +{"Finetune/Learning Rate": 1.9920928244588662e-05, "Finetune/Loss": 1.0021297931671143, "Finetune/Loss (Raw)": 0.9293422102928162, "Finetune/Step": 357, "Finetune/Step Time": 4.9781320951879025} +{"Finetune/Learning Rate": 1.9920144464287527e-05, "Finetune/Loss": 0.9959354400634766, "Finetune/Loss (Raw)": 0.3701484799385071, "Finetune/Step": 358, "Finetune/Step Time": 5.012749461457133} +{"Finetune/Learning Rate": 1.9919356834171996e-05, "Finetune/Loss": 0.9946130514144897, "Finetune/Loss (Raw)": 0.8502735495567322, "Finetune/Step": 359, "Finetune/Step Time": 5.014020560309291} +{"Finetune/Learning Rate": 1.9918565354547738e-05, "Finetune/Loss": 0.9941955804824829, "Finetune/Loss (Raw)": 1.014885425567627, "Finetune/Step": 360, "Finetune/Step Time": 5.013734167441726} +{"Finetune/Learning Rate": 1.991777002572191e-05, "Finetune/Loss": 0.9987751841545105, "Finetune/Loss (Raw)": 0.8968779444694519, "Finetune/Step": 361, "Finetune/Step Time": 4.97815678268671} +{"Finetune/Learning Rate": 1.9916970848003166e-05, "Finetune/Loss": 0.9973307847976685, "Finetune/Loss (Raw)": 0.9821655750274658, "Finetune/Step": 362, "Finetune/Step Time": 4.970565363764763} +{"Finetune/Learning Rate": 1.9916167821701645e-05, "Finetune/Loss": 0.9963480234146118, "Finetune/Loss (Raw)": 1.0806089639663696, "Finetune/Step": 363, "Finetune/Step Time": 4.929233258590102} +{"Finetune/Learning Rate": 1.991536094712899e-05, "Finetune/Loss": 0.9969522356987, "Finetune/Loss (Raw)": 1.1032476425170898, "Finetune/Step": 364, "Finetune/Step Time": 4.927948731929064} +{"Finetune/Learning Rate": 1.991455022459833e-05, "Finetune/Loss": 0.9971528649330139, "Finetune/Loss (Raw)": 1.1245582103729248, "Finetune/Step": 365, "Finetune/Step Time": 4.902648076415062} +{"Finetune/Learning Rate": 1.9913735654424297e-05, "Finetune/Loss": 0.9962636828422546, "Finetune/Loss (Raw)": 0.9061123728752136, "Finetune/Step": 366, "Finetune/Step Time": 4.907847622409463} +{"Finetune/Learning Rate": 1.9912917236923003e-05, "Finetune/Loss": 0.9970836639404297, "Finetune/Loss (Raw)": 1.0057682991027832, "Finetune/Step": 367, "Finetune/Step Time": 4.913320414721966} +{"Finetune/Learning Rate": 1.9912094972412067e-05, "Finetune/Loss": 0.997519850730896, "Finetune/Loss (Raw)": 1.0259621143341064, "Finetune/Step": 368, "Finetune/Step Time": 4.913496408611536} +{"Finetune/Learning Rate": 1.9911268861210584e-05, "Finetune/Loss": 0.9966315031051636, "Finetune/Loss (Raw)": 0.9933605790138245, "Finetune/Step": 369, "Finetune/Step Time": 4.913433656096458} +{"Finetune/Learning Rate": 1.991043890363916e-05, "Finetune/Loss": 0.9973733425140381, "Finetune/Loss (Raw)": 1.1671098470687866, "Finetune/Step": 370, "Finetune/Step Time": 4.9184944126755} +{"Finetune/Learning Rate": 1.9909605100019883e-05, "Finetune/Loss": 0.9979912042617798, "Finetune/Loss (Raw)": 1.1612259149551392, "Finetune/Step": 371, "Finetune/Step Time": 4.886325148865581} +{"Finetune/Learning Rate": 1.9908767450676336e-05, "Finetune/Loss": 0.9989683628082275, "Finetune/Loss (Raw)": 1.0521897077560425, "Finetune/Step": 372, "Finetune/Step Time": 4.895167265087366} +{"Finetune/Learning Rate": 1.9907925955933593e-05, "Finetune/Loss": 0.992963433265686, "Finetune/Loss (Raw)": 0.44543907046318054, "Finetune/Step": 373, "Finetune/Step Time": 4.934257870540023} +{"Finetune/Learning Rate": 1.9907080616118223e-05, "Finetune/Loss": 0.9866392016410828, "Finetune/Loss (Raw)": 0.3660750985145569, "Finetune/Step": 374, "Finetune/Step Time": 4.9604111928492785} +{"Finetune/Learning Rate": 1.9906231431558283e-05, "Finetune/Loss": 0.9869813323020935, "Finetune/Loss (Raw)": 1.1123051643371582, "Finetune/Step": 375, "Finetune/Step Time": 4.957749141380191} +{"Finetune/Learning Rate": 1.990537840258333e-05, "Finetune/Loss": 0.9861664175987244, "Finetune/Loss (Raw)": 0.9037861227989197, "Finetune/Step": 376, "Finetune/Step Time": 4.957245424389839} +{"Finetune/Learning Rate": 1.9904521529524402e-05, "Finetune/Loss": 0.9857638478279114, "Finetune/Loss (Raw)": 0.9741759300231934, "Finetune/Step": 377, "Finetune/Step Time": 4.9633614923805} +{"Finetune/Learning Rate": 1.990366081271404e-05, "Finetune/Loss": 0.9868239760398865, "Finetune/Loss (Raw)": 1.2227874994277954, "Finetune/Step": 378, "Finetune/Step Time": 4.962988752871752} +{"Finetune/Learning Rate": 1.990279625248627e-05, "Finetune/Loss": 0.9868866205215454, "Finetune/Loss (Raw)": 1.1584317684173584, "Finetune/Step": 379, "Finetune/Step Time": 4.949482576921582} +{"Finetune/Learning Rate": 1.990192784917661e-05, "Finetune/Loss": 0.9868746995925903, "Finetune/Loss (Raw)": 1.0423825979232788, "Finetune/Step": 380, "Finetune/Step Time": 4.9481414668262005} +{"Finetune/Learning Rate": 1.9901055603122067e-05, "Finetune/Loss": 0.9866301417350769, "Finetune/Loss (Raw)": 1.0604333877563477, "Finetune/Step": 381, "Finetune/Step Time": 4.9565145168453455} +{"Finetune/Learning Rate": 1.9900179514661148e-05, "Finetune/Loss": 0.9862006306648254, "Finetune/Loss (Raw)": 0.9270538091659546, "Finetune/Step": 382, "Finetune/Step Time": 5.008529622107744} +{"Finetune/Learning Rate": 1.9899299584133848e-05, "Finetune/Loss": 0.9910470247268677, "Finetune/Loss (Raw)": 0.9372491240501404, "Finetune/Step": 383, "Finetune/Step Time": 5.0067779161036015} +{"Finetune/Learning Rate": 1.9898415811881643e-05, "Finetune/Loss": 0.9874619841575623, "Finetune/Loss (Raw)": 0.4972997009754181, "Finetune/Step": 384, "Finetune/Step Time": 5.038381932303309} +{"Finetune/Learning Rate": 1.989752819824751e-05, "Finetune/Loss": 0.986211359500885, "Finetune/Loss (Raw)": 1.014527678489685, "Finetune/Step": 385, "Finetune/Step Time": 5.035729384049773} +{"Finetune/Learning Rate": 1.989663674357592e-05, "Finetune/Loss": 0.9853184819221497, "Finetune/Loss (Raw)": 1.0176447629928589, "Finetune/Step": 386, "Finetune/Step Time": 5.040314760059118} +{"Finetune/Learning Rate": 1.9895741448212822e-05, "Finetune/Loss": 0.9837007522583008, "Finetune/Loss (Raw)": 0.8776412010192871, "Finetune/Step": 387, "Finetune/Step Time": 5.041802665218711} +{"Finetune/Learning Rate": 1.9894842312505673e-05, "Finetune/Loss": 0.9825187921524048, "Finetune/Loss (Raw)": 0.9549481868743896, "Finetune/Step": 388, "Finetune/Step Time": 5.019835587590933} +{"Finetune/Learning Rate": 1.9893939336803396e-05, "Finetune/Loss": 0.9829837083816528, "Finetune/Loss (Raw)": 0.9422567486763, "Finetune/Step": 389, "Finetune/Step Time": 5.0195435006171465} +{"Finetune/Learning Rate": 1.989303252145643e-05, "Finetune/Loss": 0.9828976988792419, "Finetune/Loss (Raw)": 0.8427552580833435, "Finetune/Step": 390, "Finetune/Step Time": 5.00604479201138} +{"Finetune/Learning Rate": 1.9892121866816682e-05, "Finetune/Loss": 0.9827014207839966, "Finetune/Loss (Raw)": 1.0087082386016846, "Finetune/Step": 391, "Finetune/Step Time": 5.001538136973977} +{"Finetune/Learning Rate": 1.989120737323757e-05, "Finetune/Loss": 0.9825384616851807, "Finetune/Loss (Raw)": 0.976100742816925, "Finetune/Step": 392, "Finetune/Step Time": 4.998238107189536} +{"Finetune/Learning Rate": 1.9890289041073984e-05, "Finetune/Loss": 0.9814094305038452, "Finetune/Loss (Raw)": 0.8516339063644409, "Finetune/Step": 393, "Finetune/Step Time": 5.010728668421507} +{"Finetune/Learning Rate": 1.988936687068231e-05, "Finetune/Loss": 0.978101909160614, "Finetune/Loss (Raw)": 0.8086121082305908, "Finetune/Step": 394, "Finetune/Step Time": 5.026028927415609} +{"Finetune/Learning Rate": 1.9888440862420433e-05, "Finetune/Loss": 0.9804427623748779, "Finetune/Loss (Raw)": 1.208140254020691, "Finetune/Step": 395, "Finetune/Step Time": 5.024671137332916} +{"Finetune/Learning Rate": 1.988751101664771e-05, "Finetune/Loss": 0.9803910255432129, "Finetune/Loss (Raw)": 1.1096923351287842, "Finetune/Step": 396, "Finetune/Step Time": 5.025196462869644} +{"Finetune/Learning Rate": 1.9886577333724995e-05, "Finetune/Loss": 0.9788943529129028, "Finetune/Loss (Raw)": 0.8546915054321289, "Finetune/Step": 397, "Finetune/Step Time": 5.041030116379261} +{"Finetune/Learning Rate": 1.9885639814014642e-05, "Finetune/Loss": 0.9798429012298584, "Finetune/Loss (Raw)": 1.1821428537368774, "Finetune/Step": 398, "Finetune/Step Time": 5.034215798601508} +{"Finetune/Learning Rate": 1.9884698457880476e-05, "Finetune/Loss": 0.9790006875991821, "Finetune/Loss (Raw)": 0.9253978133201599, "Finetune/Step": 399, "Finetune/Step Time": 5.021788019686937} +{"Finetune/Learning Rate": 1.988375326568782e-05, "Finetune/Loss": 0.9776982665061951, "Finetune/Loss (Raw)": 0.9496223330497742, "Finetune/Step": 400, "Finetune/Step Time": 5.019138786941767} +{"Finetune/Learning Rate": 1.9882804237803487e-05, "Finetune/Loss": 0.9781817197799683, "Finetune/Loss (Raw)": 1.1165454387664795, "Finetune/Step": 401, "Finetune/Step Time": 5.021697329357266} +{"Finetune/Learning Rate": 1.9881851374595774e-05, "Finetune/Loss": 0.9774669408798218, "Finetune/Loss (Raw)": 1.0096375942230225, "Finetune/Step": 402, "Finetune/Step Time": 5.033793691545725} +{"Finetune/Learning Rate": 1.988089467643447e-05, "Finetune/Loss": 0.9770830869674683, "Finetune/Loss (Raw)": 0.9760212898254395, "Finetune/Step": 403, "Finetune/Step Time": 5.0332906395196915} +{"Finetune/Learning Rate": 1.9879934143690848e-05, "Finetune/Loss": 0.9790161848068237, "Finetune/Loss (Raw)": 1.082568883895874, "Finetune/Step": 404, "Finetune/Step Time": 5.000522349029779} +{"Finetune/Learning Rate": 1.9878969776737675e-05, "Finetune/Loss": 0.9767524003982544, "Finetune/Loss (Raw)": 0.7395386695861816, "Finetune/Step": 405, "Finetune/Step Time": 5.012972870841622} +{"Finetune/Learning Rate": 1.9878001575949205e-05, "Finetune/Loss": 0.9724022746086121, "Finetune/Loss (Raw)": 0.48286327719688416, "Finetune/Step": 406, "Finetune/Step Time": 5.049014803022146} +{"Finetune/Learning Rate": 1.9877029541701174e-05, "Finetune/Loss": 0.9661937952041626, "Finetune/Loss (Raw)": 0.4211890995502472, "Finetune/Step": 407, "Finetune/Step Time": 5.072443932294846} +{"Finetune/Learning Rate": 1.9876053674370806e-05, "Finetune/Loss": 0.9661401510238647, "Finetune/Loss (Raw)": 0.9855164885520935, "Finetune/Step": 408, "Finetune/Step Time": 5.072034019976854} +{"Finetune/Learning Rate": 1.9875073974336827e-05, "Finetune/Loss": 0.9669915437698364, "Finetune/Loss (Raw)": 1.196311354637146, "Finetune/Step": 409, "Finetune/Step Time": 5.072930892929435} +{"Finetune/Learning Rate": 1.9874090441979433e-05, "Finetune/Loss": 0.9686034321784973, "Finetune/Loss (Raw)": 1.2301563024520874, "Finetune/Step": 410, "Finetune/Step Time": 5.07122166082263} +{"Finetune/Learning Rate": 1.987310307768031e-05, "Finetune/Loss": 0.9674838781356812, "Finetune/Loss (Raw)": 0.9994461536407471, "Finetune/Step": 411, "Finetune/Step Time": 5.0742884781211615} +{"Finetune/Learning Rate": 1.987211188182264e-05, "Finetune/Loss": 0.9613533020019531, "Finetune/Loss (Raw)": 0.37517520785331726, "Finetune/Step": 412, "Finetune/Step Time": 5.114578764885664} +{"Finetune/Learning Rate": 1.9871116854791085e-05, "Finetune/Loss": 0.9622379541397095, "Finetune/Loss (Raw)": 1.1232943534851074, "Finetune/Step": 413, "Finetune/Step Time": 5.092102773487568} +{"Finetune/Learning Rate": 1.9870117996971797e-05, "Finetune/Loss": 0.9611680507659912, "Finetune/Loss (Raw)": 0.9447952508926392, "Finetune/Step": 414, "Finetune/Step Time": 5.077201519161463} +{"Finetune/Learning Rate": 1.986911530875241e-05, "Finetune/Loss": 0.9609876871109009, "Finetune/Loss (Raw)": 1.0609854459762573, "Finetune/Step": 415, "Finetune/Step Time": 5.07413873821497} +{"Finetune/Learning Rate": 1.9868108790522052e-05, "Finetune/Loss": 0.9627268314361572, "Finetune/Loss (Raw)": 1.1304824352264404, "Finetune/Step": 416, "Finetune/Step Time": 5.059283059090376} +{"Finetune/Learning Rate": 1.986709844267133e-05, "Finetune/Loss": 0.9621272087097168, "Finetune/Loss (Raw)": 1.053544282913208, "Finetune/Step": 417, "Finetune/Step Time": 5.053902870044112} +{"Finetune/Learning Rate": 1.9866084265592343e-05, "Finetune/Loss": 0.9632193446159363, "Finetune/Loss (Raw)": 1.1025679111480713, "Finetune/Step": 418, "Finetune/Step Time": 5.041196582838893} +{"Finetune/Learning Rate": 1.986506625967867e-05, "Finetune/Loss": 0.9620200395584106, "Finetune/Loss (Raw)": 0.9742718935012817, "Finetune/Step": 419, "Finetune/Step Time": 5.059124546125531} +{"Finetune/Learning Rate": 1.9864044425325382e-05, "Finetune/Loss": 0.9562022089958191, "Finetune/Loss (Raw)": 0.3644373118877411, "Finetune/Step": 420, "Finetune/Step Time": 5.093101212754846} +{"Finetune/Learning Rate": 1.986301876292903e-05, "Finetune/Loss": 0.9553147554397583, "Finetune/Loss (Raw)": 1.0369246006011963, "Finetune/Step": 421, "Finetune/Step Time": 5.093335373327136} +{"Finetune/Learning Rate": 1.986198927288766e-05, "Finetune/Loss": 0.9572219848632812, "Finetune/Loss (Raw)": 1.009084701538086, "Finetune/Step": 422, "Finetune/Step Time": 5.098832448944449} +{"Finetune/Learning Rate": 1.9860955955600784e-05, "Finetune/Loss": 0.9574918150901794, "Finetune/Loss (Raw)": 1.1330972909927368, "Finetune/Step": 423, "Finetune/Step Time": 5.0923157427459955} +{"Finetune/Learning Rate": 1.9859918811469424e-05, "Finetune/Loss": 0.95765620470047, "Finetune/Loss (Raw)": 1.0439785718917847, "Finetune/Step": 424, "Finetune/Step Time": 5.091579966247082} +{"Finetune/Learning Rate": 1.9858877840896076e-05, "Finetune/Loss": 0.9547674655914307, "Finetune/Loss (Raw)": 0.7664567828178406, "Finetune/Step": 425, "Finetune/Step Time": 5.127441871911287} +{"Finetune/Learning Rate": 1.9857833044284714e-05, "Finetune/Loss": 0.955615758895874, "Finetune/Loss (Raw)": 1.0230756998062134, "Finetune/Step": 426, "Finetune/Step Time": 5.106626419350505} +{"Finetune/Learning Rate": 1.9856784422040802e-05, "Finetune/Loss": 0.9608767032623291, "Finetune/Loss (Raw)": 0.9820966720581055, "Finetune/Step": 427, "Finetune/Step Time": 5.075134381651878} +{"Finetune/Learning Rate": 1.98557319745713e-05, "Finetune/Loss": 0.966781497001648, "Finetune/Loss (Raw)": 1.122288703918457, "Finetune/Step": 428, "Finetune/Step Time": 5.04044097289443} +{"Finetune/Learning Rate": 1.985467570228463e-05, "Finetune/Loss": 0.9720914959907532, "Finetune/Loss (Raw)": 1.0778526067733765, "Finetune/Step": 429, "Finetune/Step Time": 5.008955026045442} +{"Finetune/Learning Rate": 1.985361560559072e-05, "Finetune/Loss": 0.9721809029579163, "Finetune/Loss (Raw)": 0.904347836971283, "Finetune/Step": 430, "Finetune/Step Time": 4.9974685329943895} +{"Finetune/Learning Rate": 1.9852551684900964e-05, "Finetune/Loss": 0.9781897664070129, "Finetune/Loss (Raw)": 1.0913383960723877, "Finetune/Step": 431, "Finetune/Step Time": 4.971526807174087} +{"Finetune/Learning Rate": 1.9851483940628256e-05, "Finetune/Loss": 0.9770022630691528, "Finetune/Loss (Raw)": 0.9463168382644653, "Finetune/Step": 432, "Finetune/Step Time": 4.977873915806413} +{"Finetune/Learning Rate": 1.9850412373186967e-05, "Finetune/Loss": 0.9759987592697144, "Finetune/Loss (Raw)": 0.9214318990707397, "Finetune/Step": 433, "Finetune/Step Time": 4.9758387096226215} +{"Finetune/Learning Rate": 1.9849336982992948e-05, "Finetune/Loss": 0.9763355851173401, "Finetune/Loss (Raw)": 0.9862513542175293, "Finetune/Step": 434, "Finetune/Step Time": 4.982574865221977} +{"Finetune/Learning Rate": 1.984825777046354e-05, "Finetune/Loss": 0.9737883806228638, "Finetune/Loss (Raw)": 0.8413249850273132, "Finetune/Step": 435, "Finetune/Step Time": 4.967350801452994} +{"Finetune/Learning Rate": 1.9847174736017557e-05, "Finetune/Loss": 0.9727699756622314, "Finetune/Loss (Raw)": 0.9225490689277649, "Finetune/Step": 436, "Finetune/Step Time": 4.964107656851411} +{"Finetune/Learning Rate": 1.9846087880075314e-05, "Finetune/Loss": 0.9717782735824585, "Finetune/Loss (Raw)": 1.0265045166015625, "Finetune/Step": 437, "Finetune/Step Time": 4.955401023849845} +{"Finetune/Learning Rate": 1.984499720305859e-05, "Finetune/Loss": 0.9714096784591675, "Finetune/Loss (Raw)": 0.9693970680236816, "Finetune/Step": 438, "Finetune/Step Time": 4.952859848737717} +{"Finetune/Learning Rate": 1.984390270539066e-05, "Finetune/Loss": 0.970974326133728, "Finetune/Loss (Raw)": 0.9793646335601807, "Finetune/Step": 439, "Finetune/Step Time": 4.951903291046619} +{"Finetune/Learning Rate": 1.9842804387496276e-05, "Finetune/Loss": 0.9717159271240234, "Finetune/Loss (Raw)": 1.0833414793014526, "Finetune/Step": 440, "Finetune/Step Time": 4.958330055698752} +{"Finetune/Learning Rate": 1.9841702249801677e-05, "Finetune/Loss": 0.9712361097335815, "Finetune/Loss (Raw)": 1.0557477474212646, "Finetune/Step": 441, "Finetune/Step Time": 4.955619994550943} +{"Finetune/Learning Rate": 1.9840596292734573e-05, "Finetune/Loss": 0.9654132127761841, "Finetune/Loss (Raw)": 0.4429188370704651, "Finetune/Step": 442, "Finetune/Step Time": 4.993151281028986} +{"Finetune/Learning Rate": 1.9839486516724173e-05, "Finetune/Loss": 0.9660760760307312, "Finetune/Loss (Raw)": 1.0772626399993896, "Finetune/Step": 443, "Finetune/Step Time": 4.997026277706027} +{"Finetune/Learning Rate": 1.983837292220115e-05, "Finetune/Loss": 0.9720442295074463, "Finetune/Loss (Raw)": 1.1421914100646973, "Finetune/Step": 444, "Finetune/Step Time": 4.961860066279769} +{"Finetune/Learning Rate": 1.983725550959768e-05, "Finetune/Loss": 0.9724264144897461, "Finetune/Loss (Raw)": 1.021592140197754, "Finetune/Step": 445, "Finetune/Step Time": 4.958159303292632} +{"Finetune/Learning Rate": 1.98361342793474e-05, "Finetune/Loss": 0.9728578329086304, "Finetune/Loss (Raw)": 1.1200577020645142, "Finetune/Step": 446, "Finetune/Step Time": 4.960892608389258} +{"Finetune/Learning Rate": 1.9835009231885444e-05, "Finetune/Loss": 0.9732413291931152, "Finetune/Loss (Raw)": 1.04116952419281, "Finetune/Step": 447, "Finetune/Step Time": 4.950502244755626} +{"Finetune/Learning Rate": 1.9833880367648412e-05, "Finetune/Loss": 0.9733452796936035, "Finetune/Loss (Raw)": 0.961668074131012, "Finetune/Step": 448, "Finetune/Step Time": 4.962268935516477} +{"Finetune/Learning Rate": 1.9832747687074403e-05, "Finetune/Loss": 0.9796591997146606, "Finetune/Loss (Raw)": 1.1393364667892456, "Finetune/Step": 449, "Finetune/Step Time": 4.927498586475849} +{"Finetune/Learning Rate": 1.9831611190602984e-05, "Finetune/Loss": 0.9792416095733643, "Finetune/Loss (Raw)": 0.9482488036155701, "Finetune/Step": 450, "Finetune/Step Time": 4.926356166601181} +{"Finetune/Learning Rate": 1.983047087867521e-05, "Finetune/Loss": 0.9785181283950806, "Finetune/Loss (Raw)": 0.980084240436554, "Finetune/Step": 451, "Finetune/Step Time": 4.931151479482651} +{"Finetune/Learning Rate": 1.9829326751733607e-05, "Finetune/Loss": 0.9776976704597473, "Finetune/Loss (Raw)": 1.0011414289474487, "Finetune/Step": 452, "Finetune/Step Time": 4.924171537160873} +{"Finetune/Learning Rate": 1.9828178810222197e-05, "Finetune/Loss": 0.97910475730896, "Finetune/Loss (Raw)": 1.2290884256362915, "Finetune/Step": 453, "Finetune/Step Time": 4.91937736235559} +{"Finetune/Learning Rate": 1.9827027054586468e-05, "Finetune/Loss": 0.9773242473602295, "Finetune/Loss (Raw)": 0.9929923415184021, "Finetune/Step": 454, "Finetune/Step Time": 4.934052191674709} +{"Finetune/Learning Rate": 1.9825871485273395e-05, "Finetune/Loss": 0.9779248237609863, "Finetune/Loss (Raw)": 1.0961233377456665, "Finetune/Step": 455, "Finetune/Step Time": 4.932357124984264} +{"Finetune/Learning Rate": 1.9824712102731435e-05, "Finetune/Loss": 0.9722492098808289, "Finetune/Loss (Raw)": 0.3454785943031311, "Finetune/Step": 456, "Finetune/Step Time": 4.9556403663009405} +{"Finetune/Learning Rate": 1.982354890741052e-05, "Finetune/Loss": 0.9720232486724854, "Finetune/Loss (Raw)": 0.9223717451095581, "Finetune/Step": 457, "Finetune/Step Time": 4.935882207006216} +{"Finetune/Learning Rate": 1.982238189976206e-05, "Finetune/Loss": 0.9746432304382324, "Finetune/Loss (Raw)": 1.2179844379425049, "Finetune/Step": 458, "Finetune/Step Time": 4.93301404453814} +{"Finetune/Learning Rate": 1.9821211080238953e-05, "Finetune/Loss": 0.9742040634155273, "Finetune/Loss (Raw)": 1.0019769668579102, "Finetune/Step": 459, "Finetune/Step Time": 4.935992777347565} +{"Finetune/Learning Rate": 1.9820036449295564e-05, "Finetune/Loss": 0.974384605884552, "Finetune/Loss (Raw)": 0.8507646322250366, "Finetune/Step": 460, "Finetune/Step Time": 4.935045639052987} +{"Finetune/Learning Rate": 1.9818858007387753e-05, "Finetune/Loss": 0.9739649891853333, "Finetune/Loss (Raw)": 0.9846392869949341, "Finetune/Step": 461, "Finetune/Step Time": 4.93414918705821} +{"Finetune/Learning Rate": 1.9817675754972846e-05, "Finetune/Loss": 0.973917543888092, "Finetune/Loss (Raw)": 0.9957587122917175, "Finetune/Step": 462, "Finetune/Step Time": 4.9334701634943485} +{"Finetune/Learning Rate": 1.9816489692509654e-05, "Finetune/Loss": 0.9753121733665466, "Finetune/Loss (Raw)": 1.1941977739334106, "Finetune/Step": 463, "Finetune/Step Time": 4.927341423928738} +{"Finetune/Learning Rate": 1.9815299820458458e-05, "Finetune/Loss": 0.9689687490463257, "Finetune/Loss (Raw)": 0.2914087474346161, "Finetune/Step": 464, "Finetune/Step Time": 4.9613605029881} +{"Finetune/Learning Rate": 1.9814106139281033e-05, "Finetune/Loss": 0.967537522315979, "Finetune/Loss (Raw)": 1.0483787059783936, "Finetune/Step": 465, "Finetune/Step Time": 4.968741582706571} +{"Finetune/Learning Rate": 1.9812908649440618e-05, "Finetune/Loss": 0.9670924544334412, "Finetune/Loss (Raw)": 0.9453155994415283, "Finetune/Step": 466, "Finetune/Step Time": 4.973159417510033} +{"Finetune/Learning Rate": 1.981170735140194e-05, "Finetune/Loss": 0.9680298566818237, "Finetune/Loss (Raw)": 1.087691307067871, "Finetune/Step": 467, "Finetune/Step Time": 4.973083978518844} +{"Finetune/Learning Rate": 1.9810502245631194e-05, "Finetune/Loss": 0.9674084186553955, "Finetune/Loss (Raw)": 0.8358675241470337, "Finetune/Step": 468, "Finetune/Step Time": 4.984917087480426} +{"Finetune/Learning Rate": 1.980929333259606e-05, "Finetune/Loss": 0.962167501449585, "Finetune/Loss (Raw)": 0.35714593529701233, "Finetune/Step": 469, "Finetune/Step Time": 5.016448769718409} +{"Finetune/Learning Rate": 1.9808080612765694e-05, "Finetune/Loss": 0.9612905979156494, "Finetune/Loss (Raw)": 0.9843416213989258, "Finetune/Step": 470, "Finetune/Step Time": 5.015362439677119} +{"Finetune/Learning Rate": 1.980686408661073e-05, "Finetune/Loss": 0.9612341523170471, "Finetune/Loss (Raw)": 1.0142501592636108, "Finetune/Step": 471, "Finetune/Step Time": 5.030835147947073} +{"Finetune/Learning Rate": 1.9805643754603275e-05, "Finetune/Loss": 0.9616441130638123, "Finetune/Loss (Raw)": 1.1422494649887085, "Finetune/Step": 472, "Finetune/Step Time": 5.031933473423123} +{"Finetune/Learning Rate": 1.9804419617216922e-05, "Finetune/Loss": 0.9600435495376587, "Finetune/Loss (Raw)": 0.9224904775619507, "Finetune/Step": 473, "Finetune/Step Time": 5.0303534008562565} +{"Finetune/Learning Rate": 1.9803191674926727e-05, "Finetune/Loss": 0.9594780206680298, "Finetune/Loss (Raw)": 1.0551092624664307, "Finetune/Step": 474, "Finetune/Step Time": 5.026381451636553} +{"Finetune/Learning Rate": 1.9801959928209236e-05, "Finetune/Loss": 0.9588032960891724, "Finetune/Loss (Raw)": 1.0702656507492065, "Finetune/Step": 475, "Finetune/Step Time": 5.025726169347763} +{"Finetune/Learning Rate": 1.9800724377542463e-05, "Finetune/Loss": 0.9584192037582397, "Finetune/Loss (Raw)": 1.0342390537261963, "Finetune/Step": 476, "Finetune/Step Time": 5.009620491415262} +{"Finetune/Learning Rate": 1.9799485023405904e-05, "Finetune/Loss": 0.9580084085464478, "Finetune/Loss (Raw)": 1.107853651046753, "Finetune/Step": 477, "Finetune/Step Time": 5.0152880642563105} +{"Finetune/Learning Rate": 1.9798241866280523e-05, "Finetune/Loss": 0.9578933119773865, "Finetune/Loss (Raw)": 0.9755814075469971, "Finetune/Step": 478, "Finetune/Step Time": 4.9915643110871315} +{"Finetune/Learning Rate": 1.979699490664877e-05, "Finetune/Loss": 0.9580928087234497, "Finetune/Loss (Raw)": 1.0292363166809082, "Finetune/Step": 479, "Finetune/Step Time": 4.976000381633639} +{"Finetune/Learning Rate": 1.9795744144994562e-05, "Finetune/Loss": 0.9587584733963013, "Finetune/Loss (Raw)": 1.096663475036621, "Finetune/Step": 480, "Finetune/Step Time": 4.967681009322405} +{"Finetune/Learning Rate": 1.97944895818033e-05, "Finetune/Loss": 0.9591057300567627, "Finetune/Loss (Raw)": 1.0265767574310303, "Finetune/Step": 481, "Finetune/Step Time": 4.972563488408923} +{"Finetune/Learning Rate": 1.979323121756185e-05, "Finetune/Loss": 0.9602094888687134, "Finetune/Loss (Raw)": 1.0529719591140747, "Finetune/Step": 482, "Finetune/Step Time": 4.981648100540042} +{"Finetune/Learning Rate": 1.9791969052758563e-05, "Finetune/Loss": 0.9599276781082153, "Finetune/Loss (Raw)": 1.0379254817962646, "Finetune/Step": 483, "Finetune/Step Time": 4.992316847667098} +{"Finetune/Learning Rate": 1.9790703087883258e-05, "Finetune/Loss": 0.960389256477356, "Finetune/Loss (Raw)": 0.9793252348899841, "Finetune/Step": 484, "Finetune/Step Time": 4.9965114910155535} +{"Finetune/Learning Rate": 1.9789433323427233e-05, "Finetune/Loss": 0.9598171710968018, "Finetune/Loss (Raw)": 0.856113851070404, "Finetune/Step": 485, "Finetune/Step Time": 5.000169260427356} +{"Finetune/Learning Rate": 1.9788159759883258e-05, "Finetune/Loss": 0.9647018909454346, "Finetune/Loss (Raw)": 0.9953896403312683, "Finetune/Step": 486, "Finetune/Step Time": 4.97837133333087} +{"Finetune/Learning Rate": 1.9786882397745573e-05, "Finetune/Loss": 0.9662042856216431, "Finetune/Loss (Raw)": 1.0425812005996704, "Finetune/Step": 487, "Finetune/Step Time": 4.977123556658626} +{"Finetune/Learning Rate": 1.9785601237509907e-05, "Finetune/Loss": 0.9664598703384399, "Finetune/Loss (Raw)": 1.047603726387024, "Finetune/Step": 488, "Finetune/Step Time": 4.974520813673735} +{"Finetune/Learning Rate": 1.978431627967345e-05, "Finetune/Loss": 0.9668548107147217, "Finetune/Loss (Raw)": 0.9474320411682129, "Finetune/Step": 489, "Finetune/Step Time": 4.979024820029736} +{"Finetune/Learning Rate": 1.9783027524734865e-05, "Finetune/Loss": 0.9665538668632507, "Finetune/Loss (Raw)": 0.9436405301094055, "Finetune/Step": 490, "Finetune/Step Time": 4.985205275937915} +{"Finetune/Learning Rate": 1.9781734973194294e-05, "Finetune/Loss": 0.9634243249893188, "Finetune/Loss (Raw)": 0.680025577545166, "Finetune/Step": 491, "Finetune/Step Time": 5.013560751453042} +{"Finetune/Learning Rate": 1.9780438625553352e-05, "Finetune/Loss": 0.9623574018478394, "Finetune/Loss (Raw)": 0.9666816592216492, "Finetune/Step": 492, "Finetune/Step Time": 5.0189828388392925} +{"Finetune/Learning Rate": 1.9779138482315128e-05, "Finetune/Loss": 0.9620517492294312, "Finetune/Loss (Raw)": 1.0854344367980957, "Finetune/Step": 493, "Finetune/Step Time": 5.015609448775649} +{"Finetune/Learning Rate": 1.977783454398418e-05, "Finetune/Loss": 0.9626172780990601, "Finetune/Loss (Raw)": 0.9785018563270569, "Finetune/Step": 494, "Finetune/Step Time": 5.026840033009648} +{"Finetune/Learning Rate": 1.9776526811066543e-05, "Finetune/Loss": 0.9627399444580078, "Finetune/Loss (Raw)": 1.0214697122573853, "Finetune/Step": 495, "Finetune/Step Time": 5.0270856898278} +{"Finetune/Learning Rate": 1.977521528406972e-05, "Finetune/Loss": 0.9628925323486328, "Finetune/Loss (Raw)": 1.0455012321472168, "Finetune/Step": 496, "Finetune/Step Time": 5.040659165009856} +{"Finetune/Learning Rate": 1.9773899963502693e-05, "Finetune/Loss": 0.9627319574356079, "Finetune/Loss (Raw)": 0.9728024005889893, "Finetune/Step": 497, "Finetune/Step Time": 5.042499344795942} +{"Finetune/Learning Rate": 1.9772580849875905e-05, "Finetune/Loss": 0.9618280529975891, "Finetune/Loss (Raw)": 1.0514075756072998, "Finetune/Step": 498, "Finetune/Step Time": 5.0349681582301855} +{"Finetune/Learning Rate": 1.9771257943701284e-05, "Finetune/Loss": 0.9597216844558716, "Finetune/Loss (Raw)": 0.8916063904762268, "Finetune/Step": 499, "Finetune/Step Time": 5.036973800510168} +{"Finetune/Learning Rate": 1.9769931245492225e-05, "Finetune/Loss": 0.9592874050140381, "Finetune/Loss (Raw)": 0.9966074824333191, "Finetune/Step": 500, "Finetune/Step Time": 5.0416206531226635} +{"Finetune/Learning Rate": 1.976860075576359e-05, "Finetune/Loss": 0.9620548486709595, "Finetune/Loss (Raw)": 0.7996684312820435, "Finetune/Step": 501, "Finetune/Step Time": 5.0277168694883585} +{"Finetune/Learning Rate": 1.9767266475031716e-05, "Finetune/Loss": 0.9661914110183716, "Finetune/Loss (Raw)": 0.8955609798431396, "Finetune/Step": 502, "Finetune/Step Time": 5.010655857622623} +{"Finetune/Learning Rate": 1.9765928403814416e-05, "Finetune/Loss": 0.9656057953834534, "Finetune/Loss (Raw)": 1.0373423099517822, "Finetune/Step": 503, "Finetune/Step Time": 5.032327134162188} +{"Finetune/Learning Rate": 1.9764586542630965e-05, "Finetune/Loss": 0.9669144153594971, "Finetune/Loss (Raw)": 1.0712889432907104, "Finetune/Step": 504, "Finetune/Step Time": 5.022746384143829} +{"Finetune/Learning Rate": 1.9763240892002113e-05, "Finetune/Loss": 0.967226505279541, "Finetune/Loss (Raw)": 1.0141191482543945, "Finetune/Step": 505, "Finetune/Step Time": 5.018102092668414} +{"Finetune/Learning Rate": 1.9761891452450082e-05, "Finetune/Loss": 0.9641741514205933, "Finetune/Loss (Raw)": 0.8320844173431396, "Finetune/Step": 506, "Finetune/Step Time": 5.030329015105963} +{"Finetune/Learning Rate": 1.9760538224498563e-05, "Finetune/Loss": 0.963105320930481, "Finetune/Loss (Raw)": 1.0216264724731445, "Finetune/Step": 507, "Finetune/Step Time": 5.025864083319902} +{"Finetune/Learning Rate": 1.975918120867272e-05, "Finetune/Loss": 0.9627729654312134, "Finetune/Loss (Raw)": 0.9998437762260437, "Finetune/Step": 508, "Finetune/Step Time": 5.025429952889681} +{"Finetune/Learning Rate": 1.975782040549918e-05, "Finetune/Loss": 0.9621037244796753, "Finetune/Loss (Raw)": 0.9747638702392578, "Finetune/Step": 509, "Finetune/Step Time": 5.028472002595663} +{"Finetune/Learning Rate": 1.9756455815506045e-05, "Finetune/Loss": 0.9621771574020386, "Finetune/Loss (Raw)": 0.9364564418792725, "Finetune/Step": 510, "Finetune/Step Time": 4.983376795426011} +{"Finetune/Learning Rate": 1.9755087439222888e-05, "Finetune/Loss": 0.9634660482406616, "Finetune/Loss (Raw)": 1.1022238731384277, "Finetune/Step": 511, "Finetune/Step Time": 4.953898081555963} +{"Finetune/Learning Rate": 1.975371527718075e-05, "Finetune/Loss": 0.9676332473754883, "Finetune/Loss (Raw)": 1.0307013988494873, "Finetune/Step": 512, "Finetune/Step Time": 4.9261745028197765} +{"Finetune/Learning Rate": 1.9752339329912137e-05, "Finetune/Loss": 0.9680429100990295, "Finetune/Loss (Raw)": 1.066965937614441, "Finetune/Step": 513, "Finetune/Step Time": 4.9301144387573} +{"Finetune/Learning Rate": 1.9750959597951026e-05, "Finetune/Loss": 0.9686988592147827, "Finetune/Loss (Raw)": 1.1016113758087158, "Finetune/Step": 514, "Finetune/Step Time": 4.938376046717167} +{"Finetune/Learning Rate": 1.9749576081832865e-05, "Finetune/Loss": 0.9649884700775146, "Finetune/Loss (Raw)": 0.4027065634727478, "Finetune/Step": 515, "Finetune/Step Time": 4.971843097358942} +{"Finetune/Learning Rate": 1.9748188782094576e-05, "Finetune/Loss": 0.966003954410553, "Finetune/Loss (Raw)": 1.08493173122406, "Finetune/Step": 516, "Finetune/Step Time": 4.974693903699517} +{"Finetune/Learning Rate": 1.9746797699274535e-05, "Finetune/Loss": 0.9668369293212891, "Finetune/Loss (Raw)": 1.0488770008087158, "Finetune/Step": 517, "Finetune/Step Time": 4.9735476244241} +{"Finetune/Learning Rate": 1.9745402833912598e-05, "Finetune/Loss": 0.9685455560684204, "Finetune/Loss (Raw)": 1.0614538192749023, "Finetune/Step": 518, "Finetune/Step Time": 4.975788984447718} +{"Finetune/Learning Rate": 1.9744004186550086e-05, "Finetune/Loss": 0.9690231680870056, "Finetune/Loss (Raw)": 1.069853663444519, "Finetune/Step": 519, "Finetune/Step Time": 4.9909854009747505} +{"Finetune/Learning Rate": 1.974260175772978e-05, "Finetune/Loss": 0.9693166017532349, "Finetune/Loss (Raw)": 1.0136581659317017, "Finetune/Step": 520, "Finetune/Step Time": 4.990406984463334} +{"Finetune/Learning Rate": 1.9741195547995944e-05, "Finetune/Loss": 0.9705760478973389, "Finetune/Loss (Raw)": 1.0128366947174072, "Finetune/Step": 521, "Finetune/Step Time": 4.98127381131053} +{"Finetune/Learning Rate": 1.9739785557894296e-05, "Finetune/Loss": 0.9720971584320068, "Finetune/Loss (Raw)": 1.0033198595046997, "Finetune/Step": 522, "Finetune/Step Time": 4.967999991029501} +{"Finetune/Learning Rate": 1.9738371787972025e-05, "Finetune/Loss": 0.9697693586349487, "Finetune/Loss (Raw)": 0.9101749062538147, "Finetune/Step": 523, "Finetune/Step Time": 4.97527670301497} +{"Finetune/Learning Rate": 1.9736954238777793e-05, "Finetune/Loss": 0.9690766334533691, "Finetune/Loss (Raw)": 1.0210304260253906, "Finetune/Step": 524, "Finetune/Step Time": 4.971637764945626} +{"Finetune/Learning Rate": 1.9735532910861713e-05, "Finetune/Loss": 0.9704020023345947, "Finetune/Loss (Raw)": 1.0243293046951294, "Finetune/Step": 525, "Finetune/Step Time": 4.954964444041252} +{"Finetune/Learning Rate": 1.9734107804775385e-05, "Finetune/Loss": 0.968471884727478, "Finetune/Loss (Raw)": 0.9350985288619995, "Finetune/Step": 526, "Finetune/Step Time": 4.967832131311297} +{"Finetune/Learning Rate": 1.9732678921071858e-05, "Finetune/Loss": 0.9688350558280945, "Finetune/Loss (Raw)": 0.9718828797340393, "Finetune/Step": 527, "Finetune/Step Time": 4.9616796635091305} +{"Finetune/Learning Rate": 1.9731246260305657e-05, "Finetune/Loss": 0.969608724117279, "Finetune/Loss (Raw)": 1.0486445426940918, "Finetune/Step": 528, "Finetune/Step Time": 4.956769287586212} +{"Finetune/Learning Rate": 1.9729809823032767e-05, "Finetune/Loss": 0.9694943428039551, "Finetune/Loss (Raw)": 1.1019113063812256, "Finetune/Step": 529, "Finetune/Step Time": 4.95654440484941} +{"Finetune/Learning Rate": 1.9728369609810648e-05, "Finetune/Loss": 0.9702099561691284, "Finetune/Loss (Raw)": 1.1012399196624756, "Finetune/Step": 530, "Finetune/Step Time": 4.9404820539057255} +{"Finetune/Learning Rate": 1.972692562119821e-05, "Finetune/Loss": 0.9716472625732422, "Finetune/Loss (Raw)": 1.159984827041626, "Finetune/Step": 531, "Finetune/Step Time": 4.937108250334859} +{"Finetune/Learning Rate": 1.9725477857755842e-05, "Finetune/Loss": 0.9709041714668274, "Finetune/Loss (Raw)": 0.9874573945999146, "Finetune/Step": 532, "Finetune/Step Time": 4.935113999992609} +{"Finetune/Learning Rate": 1.9724026320045393e-05, "Finetune/Loss": 0.972527265548706, "Finetune/Loss (Raw)": 0.9472962021827698, "Finetune/Step": 533, "Finetune/Step Time": 4.931335911154747} +{"Finetune/Learning Rate": 1.9722571008630172e-05, "Finetune/Loss": 0.976122260093689, "Finetune/Loss (Raw)": 0.9430251717567444, "Finetune/Step": 534, "Finetune/Step Time": 4.9033686183393} +{"Finetune/Learning Rate": 1.9721111924074958e-05, "Finetune/Loss": 0.9801041483879089, "Finetune/Loss (Raw)": 0.9308740496635437, "Finetune/Step": 535, "Finetune/Step Time": 4.880308944731951} +{"Finetune/Learning Rate": 1.9719649066946e-05, "Finetune/Loss": 0.9787951707839966, "Finetune/Loss (Raw)": 0.8179625868797302, "Finetune/Step": 536, "Finetune/Step Time": 4.903216568753123} +{"Finetune/Learning Rate": 1.9718182437810996e-05, "Finetune/Loss": 0.9777882099151611, "Finetune/Loss (Raw)": 1.0674165487289429, "Finetune/Step": 537, "Finetune/Step Time": 4.904392570257187} +{"Finetune/Learning Rate": 1.971671203723912e-05, "Finetune/Loss": 0.9761524796485901, "Finetune/Loss (Raw)": 1.0207849740982056, "Finetune/Step": 538, "Finetune/Step Time": 4.9133040979504585} +{"Finetune/Learning Rate": 1.9715237865801007e-05, "Finetune/Loss": 0.9769261479377747, "Finetune/Loss (Raw)": 1.0984760522842407, "Finetune/Step": 539, "Finetune/Step Time": 4.907281793653965} +{"Finetune/Learning Rate": 1.971375992406875e-05, "Finetune/Loss": 0.98160719871521, "Finetune/Loss (Raw)": 0.974342942237854, "Finetune/Step": 540, "Finetune/Step Time": 4.8670718017965555} +{"Finetune/Learning Rate": 1.9712278212615913e-05, "Finetune/Loss": 0.9806826114654541, "Finetune/Loss (Raw)": 1.0049476623535156, "Finetune/Step": 541, "Finetune/Step Time": 4.87429803609848} +{"Finetune/Learning Rate": 1.9710792732017517e-05, "Finetune/Loss": 0.9813151359558105, "Finetune/Loss (Raw)": 1.02576744556427, "Finetune/Step": 542, "Finetune/Step Time": 4.873321659862995} +{"Finetune/Learning Rate": 1.9709303482850047e-05, "Finetune/Loss": 0.9818046689033508, "Finetune/Loss (Raw)": 1.1236484050750732, "Finetune/Step": 543, "Finetune/Step Time": 4.873340101912618} +{"Finetune/Learning Rate": 1.9707810465691456e-05, "Finetune/Loss": 0.9814019203186035, "Finetune/Loss (Raw)": 1.078921914100647, "Finetune/Step": 544, "Finetune/Step Time": 4.873781817033887} +{"Finetune/Learning Rate": 1.970631368112115e-05, "Finetune/Loss": 0.9809114933013916, "Finetune/Loss (Raw)": 0.9907656311988831, "Finetune/Step": 545, "Finetune/Step Time": 4.87124609015882} +{"Finetune/Learning Rate": 1.970481312972001e-05, "Finetune/Loss": 0.979648768901825, "Finetune/Loss (Raw)": 0.9409499168395996, "Finetune/Step": 546, "Finetune/Step Time": 4.872634079307318} +{"Finetune/Learning Rate": 1.9703308812070358e-05, "Finetune/Loss": 0.9804521203041077, "Finetune/Loss (Raw)": 1.0770936012268066, "Finetune/Step": 547, "Finetune/Step Time": 4.863281272351742} +{"Finetune/Learning Rate": 1.9701800728756e-05, "Finetune/Loss": 0.9850931167602539, "Finetune/Loss (Raw)": 0.9584895968437195, "Finetune/Step": 548, "Finetune/Step Time": 4.84361145272851} +{"Finetune/Learning Rate": 1.970028888036219e-05, "Finetune/Loss": 0.9835537672042847, "Finetune/Loss (Raw)": 0.8398874402046204, "Finetune/Step": 549, "Finetune/Step Time": 4.852363033220172} +{"Finetune/Learning Rate": 1.9698773267475652e-05, "Finetune/Loss": 0.9834871292114258, "Finetune/Loss (Raw)": 1.000546932220459, "Finetune/Step": 550, "Finetune/Step Time": 4.834773268550634} +{"Finetune/Learning Rate": 1.9697253890684556e-05, "Finetune/Loss": 0.9823017716407776, "Finetune/Loss (Raw)": 0.981383740901947, "Finetune/Step": 551, "Finetune/Step Time": 4.836265090852976} +{"Finetune/Learning Rate": 1.969573075057855e-05, "Finetune/Loss": 0.9827479124069214, "Finetune/Loss (Raw)": 1.1010823249816895, "Finetune/Step": 552, "Finetune/Step Time": 4.831756547093391} +{"Finetune/Learning Rate": 1.9694203847748734e-05, "Finetune/Loss": 0.9827436804771423, "Finetune/Loss (Raw)": 0.7659168243408203, "Finetune/Step": 553, "Finetune/Step Time": 4.795077700167894} +{"Finetune/Learning Rate": 1.969267318278767e-05, "Finetune/Loss": 0.9837054014205933, "Finetune/Loss (Raw)": 1.1461749076843262, "Finetune/Step": 554, "Finetune/Step Time": 4.7927664667367935} +{"Finetune/Learning Rate": 1.969113875628937e-05, "Finetune/Loss": 0.9834914207458496, "Finetune/Loss (Raw)": 0.9547083377838135, "Finetune/Step": 555, "Finetune/Step Time": 4.807061679661274} +{"Finetune/Learning Rate": 1.9689600568849328e-05, "Finetune/Loss": 0.9826469421386719, "Finetune/Loss (Raw)": 1.0141966342926025, "Finetune/Step": 556, "Finetune/Step Time": 4.819072458893061} +{"Finetune/Learning Rate": 1.968805862106448e-05, "Finetune/Loss": 0.9817091226577759, "Finetune/Loss (Raw)": 0.9578092694282532, "Finetune/Step": 557, "Finetune/Step Time": 4.8155408930033445} +{"Finetune/Learning Rate": 1.968651291353322e-05, "Finetune/Loss": 0.9765993356704712, "Finetune/Loss (Raw)": 0.25029096007347107, "Finetune/Step": 558, "Finetune/Step Time": 4.843734325841069} +{"Finetune/Learning Rate": 1.968496344685541e-05, "Finetune/Loss": 0.975723147392273, "Finetune/Loss (Raw)": 0.9791943430900574, "Finetune/Step": 559, "Finetune/Step Time": 4.836903406307101} +{"Finetune/Learning Rate": 1.9683410221632374e-05, "Finetune/Loss": 0.9749410152435303, "Finetune/Loss (Raw)": 0.8462016582489014, "Finetune/Step": 560, "Finetune/Step Time": 4.832130271941423} +{"Finetune/Learning Rate": 1.9681853238466883e-05, "Finetune/Loss": 0.974689245223999, "Finetune/Loss (Raw)": 0.8892033100128174, "Finetune/Step": 561, "Finetune/Step Time": 4.838721016421914} +{"Finetune/Learning Rate": 1.9680292497963172e-05, "Finetune/Loss": 0.9743445515632629, "Finetune/Loss (Raw)": 0.9421278834342957, "Finetune/Step": 562, "Finetune/Step Time": 4.830569764599204} +{"Finetune/Learning Rate": 1.9678728000726936e-05, "Finetune/Loss": 0.9759793281555176, "Finetune/Loss (Raw)": 1.0505777597427368, "Finetune/Step": 563, "Finetune/Step Time": 4.831413052976131} +{"Finetune/Learning Rate": 1.967715974736532e-05, "Finetune/Loss": 0.9768567085266113, "Finetune/Loss (Raw)": 1.0348546504974365, "Finetune/Step": 564, "Finetune/Step Time": 4.841761242598295} +{"Finetune/Learning Rate": 1.9675587738486935e-05, "Finetune/Loss": 0.9763031005859375, "Finetune/Loss (Raw)": 0.9556466341018677, "Finetune/Step": 565, "Finetune/Step Time": 4.846033051609993} +{"Finetune/Learning Rate": 1.9674011974701855e-05, "Finetune/Loss": 0.9771508574485779, "Finetune/Loss (Raw)": 1.0779004096984863, "Finetune/Step": 566, "Finetune/Step Time": 4.847681360319257} +{"Finetune/Learning Rate": 1.9672432456621597e-05, "Finetune/Loss": 0.9773908853530884, "Finetune/Loss (Raw)": 1.0100953578948975, "Finetune/Step": 567, "Finetune/Step Time": 4.850590266287327} +{"Finetune/Learning Rate": 1.9670849184859136e-05, "Finetune/Loss": 0.9711778163909912, "Finetune/Loss (Raw)": 0.2880694270133972, "Finetune/Step": 568, "Finetune/Step Time": 4.876203764230013} +{"Finetune/Learning Rate": 1.9669262160028916e-05, "Finetune/Loss": 0.9702098965644836, "Finetune/Loss (Raw)": 0.9318477511405945, "Finetune/Step": 569, "Finetune/Step Time": 4.882127311080694} +{"Finetune/Learning Rate": 1.966767138274683e-05, "Finetune/Loss": 0.9747052192687988, "Finetune/Loss (Raw)": 1.0183231830596924, "Finetune/Step": 570, "Finetune/Step Time": 4.856747606769204} +{"Finetune/Learning Rate": 1.966607685363022e-05, "Finetune/Loss": 0.9737504720687866, "Finetune/Loss (Raw)": 0.9550540447235107, "Finetune/Step": 571, "Finetune/Step Time": 4.863174717873335} +{"Finetune/Learning Rate": 1.96644785732979e-05, "Finetune/Loss": 0.972955584526062, "Finetune/Loss (Raw)": 1.040444254875183, "Finetune/Step": 572, "Finetune/Step Time": 4.862591436132789} +{"Finetune/Learning Rate": 1.966287654237013e-05, "Finetune/Loss": 0.9733678102493286, "Finetune/Loss (Raw)": 1.0743613243103027, "Finetune/Step": 573, "Finetune/Step Time": 4.866417873650789} +{"Finetune/Learning Rate": 1.9661270761468625e-05, "Finetune/Loss": 0.973209023475647, "Finetune/Loss (Raw)": 1.0997328758239746, "Finetune/Step": 574, "Finetune/Step Time": 4.86974285915494} +{"Finetune/Learning Rate": 1.965966123121656e-05, "Finetune/Loss": 0.9739489555358887, "Finetune/Loss (Raw)": 1.135883092880249, "Finetune/Step": 575, "Finetune/Step Time": 4.888344336301088} +{"Finetune/Learning Rate": 1.965804795223856e-05, "Finetune/Loss": 0.9741760492324829, "Finetune/Loss (Raw)": 0.9907337427139282, "Finetune/Step": 576, "Finetune/Step Time": 4.883819008246064} +{"Finetune/Learning Rate": 1.9656430925160704e-05, "Finetune/Loss": 0.9730157256126404, "Finetune/Loss (Raw)": 0.990816056728363, "Finetune/Step": 577, "Finetune/Step Time": 4.874028189107776} +{"Finetune/Learning Rate": 1.9654810150610536e-05, "Finetune/Loss": 0.9730734825134277, "Finetune/Loss (Raw)": 0.955639123916626, "Finetune/Step": 578, "Finetune/Step Time": 4.870012613013387} +{"Finetune/Learning Rate": 1.9653185629217038e-05, "Finetune/Loss": 0.9720812439918518, "Finetune/Loss (Raw)": 0.8530804514884949, "Finetune/Step": 579, "Finetune/Step Time": 4.8831219878047705} +{"Finetune/Learning Rate": 1.9651557361610663e-05, "Finetune/Loss": 0.9724127054214478, "Finetune/Loss (Raw)": 1.0435712337493896, "Finetune/Step": 580, "Finetune/Step Time": 4.890549940988421} +{"Finetune/Learning Rate": 1.9649925348423308e-05, "Finetune/Loss": 0.9717187881469727, "Finetune/Loss (Raw)": 1.140260100364685, "Finetune/Step": 581, "Finetune/Step Time": 4.894759366288781} +{"Finetune/Learning Rate": 1.964828959028832e-05, "Finetune/Loss": 0.9721572399139404, "Finetune/Loss (Raw)": 1.0491154193878174, "Finetune/Step": 582, "Finetune/Step Time": 4.89507326669991} +{"Finetune/Learning Rate": 1.964665008784051e-05, "Finetune/Loss": 0.9706229567527771, "Finetune/Loss (Raw)": 0.899734616279602, "Finetune/Step": 583, "Finetune/Step Time": 4.901534346863627} +{"Finetune/Learning Rate": 1.9645006841716137e-05, "Finetune/Loss": 0.9763602614402771, "Finetune/Loss (Raw)": 1.079857349395752, "Finetune/Step": 584, "Finetune/Step Time": 4.8819331508129835} +{"Finetune/Learning Rate": 1.9643359852552912e-05, "Finetune/Loss": 0.9776006937026978, "Finetune/Loss (Raw)": 1.0811452865600586, "Finetune/Step": 585, "Finetune/Step Time": 4.883635213598609} +{"Finetune/Learning Rate": 1.9641709120989996e-05, "Finetune/Loss": 0.970792293548584, "Finetune/Loss (Raw)": 0.34650465846061707, "Finetune/Step": 586, "Finetune/Step Time": 4.914350485429168} +{"Finetune/Learning Rate": 1.9640054647668015e-05, "Finetune/Loss": 0.9722476601600647, "Finetune/Loss (Raw)": 1.1882669925689697, "Finetune/Step": 587, "Finetune/Step Time": 4.905583905056119} +{"Finetune/Learning Rate": 1.9638396433229025e-05, "Finetune/Loss": 0.9734557271003723, "Finetune/Loss (Raw)": 1.0053974390029907, "Finetune/Step": 588, "Finetune/Step Time": 4.913959922268987} +{"Finetune/Learning Rate": 1.963673447831656e-05, "Finetune/Loss": 0.9742696285247803, "Finetune/Loss (Raw)": 1.0888195037841797, "Finetune/Step": 589, "Finetune/Step Time": 4.909162990748882} +{"Finetune/Learning Rate": 1.9635068783575577e-05, "Finetune/Loss": 0.9750320315361023, "Finetune/Loss (Raw)": 1.0933468341827393, "Finetune/Step": 590, "Finetune/Step Time": 4.910422455519438} +{"Finetune/Learning Rate": 1.9633399349652512e-05, "Finetune/Loss": 0.973153829574585, "Finetune/Loss (Raw)": 0.9537940621376038, "Finetune/Step": 591, "Finetune/Step Time": 4.912700628861785} +{"Finetune/Learning Rate": 1.9631726177195235e-05, "Finetune/Loss": 0.9784945845603943, "Finetune/Loss (Raw)": 0.9750147461891174, "Finetune/Step": 592, "Finetune/Step Time": 4.8761339597404} +{"Finetune/Learning Rate": 1.9630049266853076e-05, "Finetune/Loss": 0.9781139492988586, "Finetune/Loss (Raw)": 0.999665379524231, "Finetune/Step": 593, "Finetune/Step Time": 4.890816502273083} +{"Finetune/Learning Rate": 1.96283686192768e-05, "Finetune/Loss": 0.978654682636261, "Finetune/Loss (Raw)": 1.014525294303894, "Finetune/Step": 594, "Finetune/Step Time": 4.878406239673495} +{"Finetune/Learning Rate": 1.9626684235118646e-05, "Finetune/Loss": 0.9780838489532471, "Finetune/Loss (Raw)": 1.014622688293457, "Finetune/Step": 595, "Finetune/Step Time": 4.868383029475808} +{"Finetune/Learning Rate": 1.9624996115032287e-05, "Finetune/Loss": 0.9788792133331299, "Finetune/Loss (Raw)": 0.9376736879348755, "Finetune/Step": 596, "Finetune/Step Time": 4.850657034665346} +{"Finetune/Learning Rate": 1.9623304259672844e-05, "Finetune/Loss": 0.9829501509666443, "Finetune/Loss (Raw)": 0.8782320022583008, "Finetune/Step": 597, "Finetune/Step Time": 4.817781575024128} +{"Finetune/Learning Rate": 1.96216086696969e-05, "Finetune/Loss": 0.9831018447875977, "Finetune/Loss (Raw)": 1.0037496089935303, "Finetune/Step": 598, "Finetune/Step Time": 4.8410249054431915} +{"Finetune/Learning Rate": 1.9619909345762474e-05, "Finetune/Loss": 0.9831691980361938, "Finetune/Loss (Raw)": 1.022872805595398, "Finetune/Step": 599, "Finetune/Step Time": 4.822715360671282} +{"Finetune/Learning Rate": 1.961820628852905e-05, "Finetune/Loss": 0.9829273223876953, "Finetune/Loss (Raw)": 1.1112948656082153, "Finetune/Step": 600, "Finetune/Step Time": 4.8272849172353745} +{"Finetune/Learning Rate": 1.9616499498657537e-05, "Finetune/Loss": 0.9833906292915344, "Finetune/Loss (Raw)": 0.9817994236946106, "Finetune/Step": 601, "Finetune/Step Time": 4.850977556779981} +{"Finetune/Learning Rate": 1.961478897681032e-05, "Finetune/Loss": 0.9835190176963806, "Finetune/Loss (Raw)": 1.071532964706421, "Finetune/Step": 602, "Finetune/Step Time": 4.843961218371987} +{"Finetune/Learning Rate": 1.9613074723651214e-05, "Finetune/Loss": 0.9825097918510437, "Finetune/Loss (Raw)": 0.9410859942436218, "Finetune/Step": 603, "Finetune/Step Time": 4.848900469020009} +{"Finetune/Learning Rate": 1.9611356739845484e-05, "Finetune/Loss": 0.9827814102172852, "Finetune/Loss (Raw)": 1.0690107345581055, "Finetune/Step": 604, "Finetune/Step Time": 4.848129330202937} +{"Finetune/Learning Rate": 1.9609635026059855e-05, "Finetune/Loss": 0.9815301895141602, "Finetune/Loss (Raw)": 0.9476980566978455, "Finetune/Step": 605, "Finetune/Step Time": 4.843209117650986} +{"Finetune/Learning Rate": 1.9607909582962478e-05, "Finetune/Loss": 0.9811431765556335, "Finetune/Loss (Raw)": 0.9260387420654297, "Finetune/Step": 606, "Finetune/Step Time": 4.860486486926675} +{"Finetune/Learning Rate": 1.9606180411222974e-05, "Finetune/Loss": 0.9795660972595215, "Finetune/Loss (Raw)": 0.8273733854293823, "Finetune/Step": 607, "Finetune/Step Time": 4.890799678862095} +{"Finetune/Learning Rate": 1.9604447511512396e-05, "Finetune/Loss": 0.9795502424240112, "Finetune/Loss (Raw)": 1.0946261882781982, "Finetune/Step": 608, "Finetune/Step Time": 4.888292791321874} +{"Finetune/Learning Rate": 1.960271088450325e-05, "Finetune/Loss": 0.9787050485610962, "Finetune/Loss (Raw)": 0.9184046387672424, "Finetune/Step": 609, "Finetune/Step Time": 4.883172746747732} +{"Finetune/Learning Rate": 1.960097053086949e-05, "Finetune/Loss": 0.9780670404434204, "Finetune/Loss (Raw)": 0.9712997674942017, "Finetune/Step": 610, "Finetune/Step Time": 4.873774491250515} +{"Finetune/Learning Rate": 1.9599226451286504e-05, "Finetune/Loss": 0.9780580997467041, "Finetune/Loss (Raw)": 1.0367841720581055, "Finetune/Step": 611, "Finetune/Step Time": 4.8518800381571054} +{"Finetune/Learning Rate": 1.9597478646431147e-05, "Finetune/Loss": 0.9798622727394104, "Finetune/Loss (Raw)": 1.210253119468689, "Finetune/Step": 612, "Finetune/Step Time": 4.848917003720999} +{"Finetune/Learning Rate": 1.95957271169817e-05, "Finetune/Loss": 0.9802382588386536, "Finetune/Loss (Raw)": 0.9042487740516663, "Finetune/Step": 613, "Finetune/Step Time": 4.841341549530625} +{"Finetune/Learning Rate": 1.95939718636179e-05, "Finetune/Loss": 0.9801742434501648, "Finetune/Loss (Raw)": 0.9871916770935059, "Finetune/Step": 614, "Finetune/Step Time": 4.826591659337282} +{"Finetune/Learning Rate": 1.9592212887020923e-05, "Finetune/Loss": 0.9801125526428223, "Finetune/Loss (Raw)": 1.0346804857254028, "Finetune/Step": 615, "Finetune/Step Time": 4.833558699116111} +{"Finetune/Learning Rate": 1.9590450187873403e-05, "Finetune/Loss": 0.979759693145752, "Finetune/Loss (Raw)": 1.0024478435516357, "Finetune/Step": 616, "Finetune/Step Time": 4.835658302530646} +{"Finetune/Learning Rate": 1.95886837668594e-05, "Finetune/Loss": 0.9801745414733887, "Finetune/Loss (Raw)": 1.0005306005477905, "Finetune/Step": 617, "Finetune/Step Time": 4.827708741649985} +{"Finetune/Learning Rate": 1.958691362466443e-05, "Finetune/Loss": 0.9809466600418091, "Finetune/Loss (Raw)": 1.0424660444259644, "Finetune/Step": 618, "Finetune/Step Time": 4.821820670738816} +{"Finetune/Learning Rate": 1.958513976197545e-05, "Finetune/Loss": 0.9828619956970215, "Finetune/Loss (Raw)": 0.9251905083656311, "Finetune/Step": 619, "Finetune/Step Time": 4.816488783806562} +{"Finetune/Learning Rate": 1.9583362179480866e-05, "Finetune/Loss": 0.9833941459655762, "Finetune/Loss (Raw)": 1.0347946882247925, "Finetune/Step": 620, "Finetune/Step Time": 4.807195292785764} +{"Finetune/Learning Rate": 1.9581580877870514e-05, "Finetune/Loss": 0.9828857183456421, "Finetune/Loss (Raw)": 1.0203640460968018, "Finetune/Step": 621, "Finetune/Step Time": 4.826521569862962} +{"Finetune/Learning Rate": 1.9579795857835692e-05, "Finetune/Loss": 0.9828168749809265, "Finetune/Loss (Raw)": 0.9696852564811707, "Finetune/Step": 622, "Finetune/Step Time": 4.813928196206689} +{"Finetune/Learning Rate": 1.9578007120069123e-05, "Finetune/Loss": 0.983255922794342, "Finetune/Loss (Raw)": 1.0776691436767578, "Finetune/Step": 623, "Finetune/Step Time": 4.815009314566851} +{"Finetune/Learning Rate": 1.957621466526499e-05, "Finetune/Loss": 0.9828827381134033, "Finetune/Loss (Raw)": 0.997725784778595, "Finetune/Step": 624, "Finetune/Step Time": 4.800836695358157} +{"Finetune/Learning Rate": 1.9574418494118905e-05, "Finetune/Loss": 0.9835505485534668, "Finetune/Loss (Raw)": 1.0582914352416992, "Finetune/Step": 625, "Finetune/Step Time": 4.802265243604779} +{"Finetune/Learning Rate": 1.9572618607327928e-05, "Finetune/Loss": 0.983676552772522, "Finetune/Loss (Raw)": 1.0675283670425415, "Finetune/Step": 626, "Finetune/Step Time": 4.800140155479312} +{"Finetune/Learning Rate": 1.9570815005590556e-05, "Finetune/Loss": 0.9843484163284302, "Finetune/Loss (Raw)": 0.9776095747947693, "Finetune/Step": 627, "Finetune/Step Time": 4.80246121250093} +{"Finetune/Learning Rate": 1.9569007689606735e-05, "Finetune/Loss": 0.9843287467956543, "Finetune/Loss (Raw)": 0.994085967540741, "Finetune/Step": 628, "Finetune/Step Time": 4.792070748284459} +{"Finetune/Learning Rate": 1.9567196660077848e-05, "Finetune/Loss": 0.986113429069519, "Finetune/Loss (Raw)": 1.028120994567871, "Finetune/Step": 629, "Finetune/Step Time": 4.777028851211071} +{"Finetune/Learning Rate": 1.9565381917706726e-05, "Finetune/Loss": 0.9880272150039673, "Finetune/Loss (Raw)": 1.1405097246170044, "Finetune/Step": 630, "Finetune/Step Time": 4.758522298187017} +{"Finetune/Learning Rate": 1.9563563463197625e-05, "Finetune/Loss": 0.9870573282241821, "Finetune/Loss (Raw)": 0.9132083654403687, "Finetune/Step": 631, "Finetune/Step Time": 4.738671829923987} +{"Finetune/Learning Rate": 1.9561741297256257e-05, "Finetune/Loss": 0.9878634214401245, "Finetune/Loss (Raw)": 1.174465298652649, "Finetune/Step": 632, "Finetune/Step Time": 4.743090184405446} +{"Finetune/Learning Rate": 1.955991542058977e-05, "Finetune/Loss": 0.9898138046264648, "Finetune/Loss (Raw)": 1.2637693881988525, "Finetune/Step": 633, "Finetune/Step Time": 4.744060847908258} +{"Finetune/Learning Rate": 1.9558085833906753e-05, "Finetune/Loss": 0.9907028675079346, "Finetune/Loss (Raw)": 0.9458772540092468, "Finetune/Step": 634, "Finetune/Step Time": 4.734535237774253} +{"Finetune/Learning Rate": 1.9556252537917228e-05, "Finetune/Loss": 0.9905366897583008, "Finetune/Loss (Raw)": 1.0003540515899658, "Finetune/Step": 635, "Finetune/Step Time": 4.738823778927326} +{"Finetune/Learning Rate": 1.955441553333266e-05, "Finetune/Loss": 0.9911589026451111, "Finetune/Loss (Raw)": 1.0794918537139893, "Finetune/Step": 636, "Finetune/Step Time": 4.763475362211466} +{"Finetune/Learning Rate": 1.955257482086596e-05, "Finetune/Loss": 0.9921483993530273, "Finetune/Loss (Raw)": 1.1014221906661987, "Finetune/Step": 637, "Finetune/Step Time": 4.75324285030365} +{"Finetune/Learning Rate": 1.955073040123147e-05, "Finetune/Loss": 0.9934349656105042, "Finetune/Loss (Raw)": 1.1011426448822021, "Finetune/Step": 638, "Finetune/Step Time": 4.747501492500305} +{"Finetune/Learning Rate": 1.9548882275144974e-05, "Finetune/Loss": 0.9927948117256165, "Finetune/Loss (Raw)": 1.0202831029891968, "Finetune/Step": 639, "Finetune/Step Time": 4.744095524773002} +{"Finetune/Learning Rate": 1.9547030443323695e-05, "Finetune/Loss": 0.9926317930221558, "Finetune/Loss (Raw)": 1.0098291635513306, "Finetune/Step": 640, "Finetune/Step Time": 4.749473791569471} +{"Finetune/Learning Rate": 1.9545174906486287e-05, "Finetune/Loss": 0.9929267168045044, "Finetune/Loss (Raw)": 1.104722023010254, "Finetune/Step": 641, "Finetune/Step Time": 4.7572366036474705} +{"Finetune/Learning Rate": 1.9543315665352855e-05, "Finetune/Loss": 0.9870001077651978, "Finetune/Loss (Raw)": 0.34300556778907776, "Finetune/Step": 642, "Finetune/Step Time": 4.776142382994294} +{"Finetune/Learning Rate": 1.954145272064493e-05, "Finetune/Loss": 0.9916746020317078, "Finetune/Loss (Raw)": 1.001033902168274, "Finetune/Step": 643, "Finetune/Step Time": 4.7465960793197155} +{"Finetune/Learning Rate": 1.9539586073085482e-05, "Finetune/Loss": 0.9913982152938843, "Finetune/Loss (Raw)": 1.0495506525039673, "Finetune/Step": 644, "Finetune/Step Time": 4.751343261450529} +{"Finetune/Learning Rate": 1.9537715723398926e-05, "Finetune/Loss": 0.9917005300521851, "Finetune/Loss (Raw)": 1.0875780582427979, "Finetune/Step": 645, "Finetune/Step Time": 4.749935623258352} +{"Finetune/Learning Rate": 1.9535841672311106e-05, "Finetune/Loss": 0.9914148449897766, "Finetune/Loss (Raw)": 1.024885892868042, "Finetune/Step": 646, "Finetune/Step Time": 4.7601730059832335} +{"Finetune/Learning Rate": 1.9533963920549307e-05, "Finetune/Loss": 0.9905191659927368, "Finetune/Loss (Raw)": 0.9552145600318909, "Finetune/Step": 647, "Finetune/Step Time": 4.760353513062} +{"Finetune/Learning Rate": 1.9532082468842243e-05, "Finetune/Loss": 0.9906834363937378, "Finetune/Loss (Raw)": 1.0346760749816895, "Finetune/Step": 648, "Finetune/Step Time": 4.764004837721586} +{"Finetune/Learning Rate": 1.953019731792007e-05, "Finetune/Loss": 0.9908493757247925, "Finetune/Loss (Raw)": 1.034080982208252, "Finetune/Step": 649, "Finetune/Step Time": 4.759902149438858} +{"Finetune/Learning Rate": 1.9528308468514383e-05, "Finetune/Loss": 0.9907734394073486, "Finetune/Loss (Raw)": 0.9935925006866455, "Finetune/Step": 650, "Finetune/Step Time": 4.76819971203804} +{"Finetune/Learning Rate": 1.9526415921358202e-05, "Finetune/Loss": 0.9917008876800537, "Finetune/Loss (Raw)": 1.0288845300674438, "Finetune/Step": 651, "Finetune/Step Time": 4.776592830196023} +{"Finetune/Learning Rate": 1.952451967718599e-05, "Finetune/Loss": 0.9916117191314697, "Finetune/Loss (Raw)": 1.009619951248169, "Finetune/Step": 652, "Finetune/Step Time": 4.7806428130716085} +{"Finetune/Learning Rate": 1.952261973673364e-05, "Finetune/Loss": 0.9908066391944885, "Finetune/Loss (Raw)": 0.9212839007377625, "Finetune/Step": 653, "Finetune/Step Time": 4.782960152253509} +{"Finetune/Learning Rate": 1.9520716100738487e-05, "Finetune/Loss": 0.99297696352005, "Finetune/Loss (Raw)": 1.2129050493240356, "Finetune/Step": 654, "Finetune/Step Time": 4.774207517504692} +{"Finetune/Learning Rate": 1.9518808769939287e-05, "Finetune/Loss": 0.9931206107139587, "Finetune/Loss (Raw)": 0.9902677536010742, "Finetune/Step": 655, "Finetune/Step Time": 4.783930368721485} +{"Finetune/Learning Rate": 1.9516897745076243e-05, "Finetune/Loss": 0.9925488233566284, "Finetune/Loss (Raw)": 0.9754564166069031, "Finetune/Step": 656, "Finetune/Step Time": 4.78714389167726} +{"Finetune/Learning Rate": 1.9514983026890988e-05, "Finetune/Loss": 0.991431713104248, "Finetune/Loss (Raw)": 0.958925187587738, "Finetune/Step": 657, "Finetune/Step Time": 4.807513201609254} +{"Finetune/Learning Rate": 1.9513064616126588e-05, "Finetune/Loss": 0.9904991388320923, "Finetune/Loss (Raw)": 0.9818629622459412, "Finetune/Step": 658, "Finetune/Step Time": 4.809295142069459} +{"Finetune/Learning Rate": 1.951114251352753e-05, "Finetune/Loss": 0.9877281188964844, "Finetune/Loss (Raw)": 0.8052882552146912, "Finetune/Step": 659, "Finetune/Step Time": 4.831267369911075} +{"Finetune/Learning Rate": 1.950921671983976e-05, "Finetune/Loss": 0.9888617992401123, "Finetune/Loss (Raw)": 1.132576584815979, "Finetune/Step": 660, "Finetune/Step Time": 4.835563542321324} +{"Finetune/Learning Rate": 1.9507287235810625e-05, "Finetune/Loss": 0.9893853664398193, "Finetune/Loss (Raw)": 1.0143170356750488, "Finetune/Step": 661, "Finetune/Step Time": 4.828080279752612} +{"Finetune/Learning Rate": 1.9505354062188934e-05, "Finetune/Loss": 0.9903154373168945, "Finetune/Loss (Raw)": 1.0620750188827515, "Finetune/Step": 662, "Finetune/Step Time": 4.816854625940323} +{"Finetune/Learning Rate": 1.9503417199724904e-05, "Finetune/Loss": 0.9917401671409607, "Finetune/Loss (Raw)": 1.1132326126098633, "Finetune/Step": 663, "Finetune/Step Time": 4.805348990485072} +{"Finetune/Learning Rate": 1.95014766491702e-05, "Finetune/Loss": 0.9933688640594482, "Finetune/Loss (Raw)": 1.0264334678649902, "Finetune/Step": 664, "Finetune/Step Time": 4.7928489074110985} +{"Finetune/Learning Rate": 1.9499532411277907e-05, "Finetune/Loss": 0.9941114187240601, "Finetune/Loss (Raw)": 1.162465214729309, "Finetune/Step": 665, "Finetune/Step Time": 4.801336228847504} +{"Finetune/Learning Rate": 1.9497584486802548e-05, "Finetune/Loss": 0.9953002333641052, "Finetune/Loss (Raw)": 1.1729555130004883, "Finetune/Step": 666, "Finetune/Step Time": 4.793494759127498} +{"Finetune/Learning Rate": 1.9495632876500076e-05, "Finetune/Loss": 0.995500385761261, "Finetune/Loss (Raw)": 1.12409245967865, "Finetune/Step": 667, "Finetune/Step Time": 4.807066569104791} +{"Finetune/Learning Rate": 1.949367758112787e-05, "Finetune/Loss": 0.9946611523628235, "Finetune/Loss (Raw)": 0.8669209480285645, "Finetune/Step": 668, "Finetune/Step Time": 4.8121892139315605} +{"Finetune/Learning Rate": 1.949171860144474e-05, "Finetune/Loss": 0.9949300289154053, "Finetune/Loss (Raw)": 1.0393664836883545, "Finetune/Step": 669, "Finetune/Step Time": 4.806507706642151} +{"Finetune/Learning Rate": 1.9489755938210936e-05, "Finetune/Loss": 0.9940923452377319, "Finetune/Loss (Raw)": 0.9185458421707153, "Finetune/Step": 670, "Finetune/Step Time": 4.808588266372681} +{"Finetune/Learning Rate": 1.9487789592188123e-05, "Finetune/Loss": 0.9930973052978516, "Finetune/Loss (Raw)": 0.9962795376777649, "Finetune/Step": 671, "Finetune/Step Time": 4.807109706103802} +{"Finetune/Learning Rate": 1.9485819564139404e-05, "Finetune/Loss": 0.9929606318473816, "Finetune/Loss (Raw)": 1.061432957649231, "Finetune/Step": 672, "Finetune/Step Time": 4.804015440866351} +{"Finetune/Learning Rate": 1.9483845854829304e-05, "Finetune/Loss": 0.9939700365066528, "Finetune/Loss (Raw)": 1.1199718713760376, "Finetune/Step": 673, "Finetune/Step Time": 4.804900381714106} +{"Finetune/Learning Rate": 1.9481868465023788e-05, "Finetune/Loss": 0.9953683614730835, "Finetune/Loss (Raw)": 1.1199350357055664, "Finetune/Step": 674, "Finetune/Step Time": 4.805336864665151} +{"Finetune/Learning Rate": 1.9479887395490234e-05, "Finetune/Loss": 0.9953116178512573, "Finetune/Loss (Raw)": 1.0698281526565552, "Finetune/Step": 675, "Finetune/Step Time": 4.807190578430891} +{"Finetune/Learning Rate": 1.9477902646997464e-05, "Finetune/Loss": 0.9959233999252319, "Finetune/Loss (Raw)": 1.0367892980575562, "Finetune/Step": 676, "Finetune/Step Time": 4.801385421305895} +{"Finetune/Learning Rate": 1.947591422031572e-05, "Finetune/Loss": 0.9965434074401855, "Finetune/Loss (Raw)": 0.9192500710487366, "Finetune/Step": 677, "Finetune/Step Time": 4.8340303637087345} +{"Finetune/Learning Rate": 1.9473922116216664e-05, "Finetune/Loss": 0.9976155757904053, "Finetune/Loss (Raw)": 1.1377962827682495, "Finetune/Step": 678, "Finetune/Step Time": 4.845609921962023} +{"Finetune/Learning Rate": 1.9471926335473396e-05, "Finetune/Loss": 0.9989067316055298, "Finetune/Loss (Raw)": 1.1466425657272339, "Finetune/Step": 679, "Finetune/Step Time": 4.850733157247305} +{"Finetune/Learning Rate": 1.9469926878860447e-05, "Finetune/Loss": 0.9978525638580322, "Finetune/Loss (Raw)": 0.9661507606506348, "Finetune/Step": 680, "Finetune/Step Time": 4.849373575299978} +{"Finetune/Learning Rate": 1.9467923747153756e-05, "Finetune/Loss": 1.0002373456954956, "Finetune/Loss (Raw)": 1.0711700916290283, "Finetune/Step": 681, "Finetune/Step Time": 4.847827197983861} +{"Finetune/Learning Rate": 1.9465916941130705e-05, "Finetune/Loss": 0.999753475189209, "Finetune/Loss (Raw)": 1.0842427015304565, "Finetune/Step": 682, "Finetune/Step Time": 4.844626162201166} +{"Finetune/Learning Rate": 1.9463906461570098e-05, "Finetune/Loss": 1.000983715057373, "Finetune/Loss (Raw)": 1.112178921699524, "Finetune/Step": 683, "Finetune/Step Time": 4.825340824201703} +{"Finetune/Learning Rate": 1.9461892309252158e-05, "Finetune/Loss": 1.000589370727539, "Finetune/Loss (Raw)": 0.9637149572372437, "Finetune/Step": 684, "Finetune/Step Time": 4.813831435516477} +{"Finetune/Learning Rate": 1.945987448495854e-05, "Finetune/Loss": 0.9990823268890381, "Finetune/Loss (Raw)": 0.7649127840995789, "Finetune/Step": 685, "Finetune/Step Time": 4.829246371984482} +{"Finetune/Learning Rate": 1.945785298947233e-05, "Finetune/Loss": 1.0044474601745605, "Finetune/Loss (Raw)": 0.9370135068893433, "Finetune/Step": 686, "Finetune/Step Time": 4.808124719187617} +{"Finetune/Learning Rate": 1.9455827823578017e-05, "Finetune/Loss": 1.004439115524292, "Finetune/Loss (Raw)": 0.9781346917152405, "Finetune/Step": 687, "Finetune/Step Time": 4.805399185046554} +{"Finetune/Learning Rate": 1.9453798988061535e-05, "Finetune/Loss": 1.0057919025421143, "Finetune/Loss (Raw)": 1.0193604230880737, "Finetune/Step": 688, "Finetune/Step Time": 4.8040243200957775} +{"Finetune/Learning Rate": 1.9451766483710236e-05, "Finetune/Loss": 1.0046241283416748, "Finetune/Loss (Raw)": 0.7397341728210449, "Finetune/Step": 689, "Finetune/Step Time": 4.8115578554570675} +{"Finetune/Learning Rate": 1.94497303113129e-05, "Finetune/Loss": 1.003831386566162, "Finetune/Loss (Raw)": 0.8406558036804199, "Finetune/Step": 690, "Finetune/Step Time": 4.857384825125337} +{"Finetune/Learning Rate": 1.9447690471659715e-05, "Finetune/Loss": 1.0039100646972656, "Finetune/Loss (Raw)": 1.060648798942566, "Finetune/Step": 691, "Finetune/Step Time": 4.859030241146684} +{"Finetune/Learning Rate": 1.944564696554231e-05, "Finetune/Loss": 1.0031681060791016, "Finetune/Loss (Raw)": 0.9398748874664307, "Finetune/Step": 692, "Finetune/Step Time": 4.876677943393588} +{"Finetune/Learning Rate": 1.944359979375373e-05, "Finetune/Loss": 1.0034245252609253, "Finetune/Loss (Raw)": 0.9884611964225769, "Finetune/Step": 693, "Finetune/Step Time": 4.875915190204978} +{"Finetune/Learning Rate": 1.9441548957088438e-05, "Finetune/Loss": 0.9981406331062317, "Finetune/Loss (Raw)": 0.4015762209892273, "Finetune/Step": 694, "Finetune/Step Time": 4.909690359607339} +{"Finetune/Learning Rate": 1.9439494456342327e-05, "Finetune/Loss": 0.9970513582229614, "Finetune/Loss (Raw)": 0.8706592917442322, "Finetune/Step": 695, "Finetune/Step Time": 4.916065827012062} +{"Finetune/Learning Rate": 1.943743629231271e-05, "Finetune/Loss": 1.0035187005996704, "Finetune/Loss (Raw)": 1.115899682044983, "Finetune/Step": 696, "Finetune/Step Time": 4.886114316061139} +{"Finetune/Learning Rate": 1.9435374465798318e-05, "Finetune/Loss": 1.0036033391952515, "Finetune/Loss (Raw)": 0.9426776766777039, "Finetune/Step": 697, "Finetune/Step Time": 4.892902931198478} +{"Finetune/Learning Rate": 1.9433308977599306e-05, "Finetune/Loss": 1.004956841468811, "Finetune/Loss (Raw)": 1.1915676593780518, "Finetune/Step": 698, "Finetune/Step Time": 4.880491612479091} +{"Finetune/Learning Rate": 1.943123982851725e-05, "Finetune/Loss": 1.0069305896759033, "Finetune/Loss (Raw)": 1.207698106765747, "Finetune/Step": 699, "Finetune/Step Time": 4.877180954441428} +{"Finetune/Learning Rate": 1.9429167019355143e-05, "Finetune/Loss": 1.0065909624099731, "Finetune/Loss (Raw)": 0.9969688653945923, "Finetune/Step": 700, "Finetune/Step Time": 4.897004805505276} +{"Finetune/Learning Rate": 1.9427090550917407e-05, "Finetune/Loss": 1.0006473064422607, "Finetune/Loss (Raw)": 0.3135650157928467, "Finetune/Step": 701, "Finetune/Step Time": 4.942491102963686} +{"Finetune/Learning Rate": 1.9425010424009876e-05, "Finetune/Loss": 0.9997079968452454, "Finetune/Loss (Raw)": 0.9795029759407043, "Finetune/Step": 702, "Finetune/Step Time": 4.944207785651088} +{"Finetune/Learning Rate": 1.9422926639439806e-05, "Finetune/Loss": 0.9977201223373413, "Finetune/Loss (Raw)": 0.8814290165901184, "Finetune/Step": 703, "Finetune/Step Time": 4.934285273775458} +{"Finetune/Learning Rate": 1.9420839198015878e-05, "Finetune/Loss": 0.9977365136146545, "Finetune/Loss (Raw)": 0.992839515209198, "Finetune/Step": 704, "Finetune/Step Time": 4.9514609184116125} +{"Finetune/Learning Rate": 1.941874810054818e-05, "Finetune/Loss": 0.9979436993598938, "Finetune/Loss (Raw)": 1.017339825630188, "Finetune/Step": 705, "Finetune/Step Time": 4.956644024699926} +{"Finetune/Learning Rate": 1.941665334784823e-05, "Finetune/Loss": 0.9966377019882202, "Finetune/Loss (Raw)": 0.788476288318634, "Finetune/Step": 706, "Finetune/Step Time": 4.964861521497369} +{"Finetune/Learning Rate": 1.9414554940728963e-05, "Finetune/Loss": 0.9970561265945435, "Finetune/Loss (Raw)": 0.9066319465637207, "Finetune/Step": 707, "Finetune/Step Time": 4.949478939175606} +{"Finetune/Learning Rate": 1.9412452880004728e-05, "Finetune/Loss": 0.9979786276817322, "Finetune/Loss (Raw)": 1.1616551876068115, "Finetune/Step": 708, "Finetune/Step Time": 4.948868501931429} +{"Finetune/Learning Rate": 1.941034716649129e-05, "Finetune/Loss": 0.9966533184051514, "Finetune/Loss (Raw)": 0.9706171154975891, "Finetune/Step": 709, "Finetune/Step Time": 4.950059467926621} +{"Finetune/Learning Rate": 1.9408237801005837e-05, "Finetune/Loss": 0.9953716993331909, "Finetune/Loss (Raw)": 0.885064959526062, "Finetune/Step": 710, "Finetune/Step Time": 4.934359883889556} +{"Finetune/Learning Rate": 1.9406124784366976e-05, "Finetune/Loss": 0.9968101978302002, "Finetune/Loss (Raw)": 1.0838701725006104, "Finetune/Step": 711, "Finetune/Step Time": 4.94746982678771} +{"Finetune/Learning Rate": 1.9404008117394726e-05, "Finetune/Loss": 0.9940096139907837, "Finetune/Loss (Raw)": 0.721375584602356, "Finetune/Step": 712, "Finetune/Step Time": 4.952530732378364} +{"Finetune/Learning Rate": 1.9401887800910525e-05, "Finetune/Loss": 0.9932540655136108, "Finetune/Loss (Raw)": 0.9844390153884888, "Finetune/Step": 713, "Finetune/Step Time": 4.95819347165525} +{"Finetune/Learning Rate": 1.939976383573722e-05, "Finetune/Loss": 0.9978786706924438, "Finetune/Loss (Raw)": 0.9384517073631287, "Finetune/Step": 714, "Finetune/Step Time": 4.929048260673881} +{"Finetune/Learning Rate": 1.9397636222699088e-05, "Finetune/Loss": 0.9973495006561279, "Finetune/Loss (Raw)": 1.1205374002456665, "Finetune/Step": 715, "Finetune/Step Time": 4.928286602720618} +{"Finetune/Learning Rate": 1.939550496262181e-05, "Finetune/Loss": 0.9963055849075317, "Finetune/Loss (Raw)": 0.8717790842056274, "Finetune/Step": 716, "Finetune/Step Time": 4.902830503880978} +{"Finetune/Learning Rate": 1.939337005633249e-05, "Finetune/Loss": 0.9966902732849121, "Finetune/Loss (Raw)": 1.1380516290664673, "Finetune/Step": 717, "Finetune/Step Time": 4.9153059888631105} +{"Finetune/Learning Rate": 1.939123150465964e-05, "Finetune/Loss": 0.9964131116867065, "Finetune/Loss (Raw)": 1.05787193775177, "Finetune/Step": 718, "Finetune/Step Time": 4.919120604172349} +{"Finetune/Learning Rate": 1.938908930843319e-05, "Finetune/Loss": 0.9970706701278687, "Finetune/Loss (Raw)": 1.0379642248153687, "Finetune/Step": 719, "Finetune/Step Time": 4.9195669535547495} +{"Finetune/Learning Rate": 1.9386943468484487e-05, "Finetune/Loss": 0.9989212155342102, "Finetune/Loss (Raw)": 1.2118849754333496, "Finetune/Step": 720, "Finetune/Step Time": 4.922071015462279} +{"Finetune/Learning Rate": 1.938479398564629e-05, "Finetune/Loss": 0.9983530044555664, "Finetune/Loss (Raw)": 0.9269345998764038, "Finetune/Step": 721, "Finetune/Step Time": 4.926072366535664} +{"Finetune/Learning Rate": 1.938264086075277e-05, "Finetune/Loss": 0.9981850385665894, "Finetune/Loss (Raw)": 0.993016242980957, "Finetune/Step": 722, "Finetune/Step Time": 4.952917546033859} +{"Finetune/Learning Rate": 1.938048409463951e-05, "Finetune/Loss": 0.998999834060669, "Finetune/Loss (Raw)": 1.1189203262329102, "Finetune/Step": 723, "Finetune/Step Time": 4.950013777241111} +{"Finetune/Learning Rate": 1.9378323688143515e-05, "Finetune/Loss": 1.000147819519043, "Finetune/Loss (Raw)": 1.0846295356750488, "Finetune/Step": 724, "Finetune/Step Time": 4.965687660500407} +{"Finetune/Learning Rate": 1.937615964210319e-05, "Finetune/Loss": 1.002211570739746, "Finetune/Loss (Raw)": 1.1423667669296265, "Finetune/Step": 725, "Finetune/Step Time": 4.964572889730334} +{"Finetune/Learning Rate": 1.9373991957358362e-05, "Finetune/Loss": 1.0004045963287354, "Finetune/Loss (Raw)": 0.7724626064300537, "Finetune/Step": 726, "Finetune/Step Time": 4.95205057784915} +{"Finetune/Learning Rate": 1.9371820634750266e-05, "Finetune/Loss": 1.0004795789718628, "Finetune/Loss (Raw)": 1.0324817895889282, "Finetune/Step": 727, "Finetune/Step Time": 4.958741525188088} +{"Finetune/Learning Rate": 1.9369645675121554e-05, "Finetune/Loss": 1.0011805295944214, "Finetune/Loss (Raw)": 1.2010101079940796, "Finetune/Step": 728, "Finetune/Step Time": 4.9584026876837015} +{"Finetune/Learning Rate": 1.936746707931628e-05, "Finetune/Loss": 0.9985544681549072, "Finetune/Loss (Raw)": 0.6456649899482727, "Finetune/Step": 729, "Finetune/Step Time": 4.9475404266268015} +{"Finetune/Learning Rate": 1.9365284848179917e-05, "Finetune/Loss": 0.9980708360671997, "Finetune/Loss (Raw)": 1.0096341371536255, "Finetune/Step": 730, "Finetune/Step Time": 4.951253375038505} +{"Finetune/Learning Rate": 1.9363098982559346e-05, "Finetune/Loss": 0.999251127243042, "Finetune/Loss (Raw)": 1.0921685695648193, "Finetune/Step": 731, "Finetune/Step Time": 4.947171742096543} +{"Finetune/Learning Rate": 1.9360909483302862e-05, "Finetune/Loss": 0.9979923963546753, "Finetune/Loss (Raw)": 0.9078896045684814, "Finetune/Step": 732, "Finetune/Step Time": 4.979528896510601} +{"Finetune/Learning Rate": 1.9358716351260164e-05, "Finetune/Loss": 0.9980872869491577, "Finetune/Loss (Raw)": 0.9598336815834045, "Finetune/Step": 733, "Finetune/Step Time": 4.9775511752814054} +{"Finetune/Learning Rate": 1.935651958728236e-05, "Finetune/Loss": 0.9976664185523987, "Finetune/Loss (Raw)": 0.8721736669540405, "Finetune/Step": 734, "Finetune/Step Time": 4.965085972100496} +{"Finetune/Learning Rate": 1.9354319192221975e-05, "Finetune/Loss": 0.9982717037200928, "Finetune/Loss (Raw)": 0.9048501253128052, "Finetune/Step": 735, "Finetune/Step Time": 4.942317761480808} +{"Finetune/Learning Rate": 1.9352115166932943e-05, "Finetune/Loss": 0.9969884157180786, "Finetune/Loss (Raw)": 0.9303719401359558, "Finetune/Step": 736, "Finetune/Step Time": 4.978427205234766} +{"Finetune/Learning Rate": 1.9349907512270597e-05, "Finetune/Loss": 0.998599648475647, "Finetune/Loss (Raw)": 1.1246377229690552, "Finetune/Step": 737, "Finetune/Step Time": 4.974178662523627} +{"Finetune/Learning Rate": 1.9347696229091685e-05, "Finetune/Loss": 0.9989702105522156, "Finetune/Loss (Raw)": 1.0187275409698486, "Finetune/Step": 738, "Finetune/Step Time": 4.97013053484261} +{"Finetune/Learning Rate": 1.934548131825437e-05, "Finetune/Loss": 0.9944700002670288, "Finetune/Loss (Raw)": 0.4607602655887604, "Finetune/Step": 739, "Finetune/Step Time": 5.010291509330273} +{"Finetune/Learning Rate": 1.9343262780618206e-05, "Finetune/Loss": 0.9939361810684204, "Finetune/Loss (Raw)": 1.1419281959533691, "Finetune/Step": 740, "Finetune/Step Time": 5.012450711801648} +{"Finetune/Learning Rate": 1.934104061704417e-05, "Finetune/Loss": 0.9932137727737427, "Finetune/Loss (Raw)": 0.8117660284042358, "Finetune/Step": 741, "Finetune/Step Time": 5.030391275882721} +{"Finetune/Learning Rate": 1.9338814828394638e-05, "Finetune/Loss": 0.9931978583335876, "Finetune/Loss (Raw)": 0.9851623177528381, "Finetune/Step": 742, "Finetune/Step Time": 5.035610236227512} +{"Finetune/Learning Rate": 1.9336585415533398e-05, "Finetune/Loss": 0.9934992790222168, "Finetune/Loss (Raw)": 1.0732556581497192, "Finetune/Step": 743, "Finetune/Step Time": 5.028877891600132} +{"Finetune/Learning Rate": 1.9334352379325638e-05, "Finetune/Loss": 0.9934492111206055, "Finetune/Loss (Raw)": 0.9960490465164185, "Finetune/Step": 744, "Finetune/Step Time": 5.03068963624537} +{"Finetune/Learning Rate": 1.9332115720637957e-05, "Finetune/Loss": 0.9926365613937378, "Finetune/Loss (Raw)": 0.8965076208114624, "Finetune/Step": 745, "Finetune/Step Time": 5.048667466267943} +{"Finetune/Learning Rate": 1.9329875440338363e-05, "Finetune/Loss": 0.9931033253669739, "Finetune/Loss (Raw)": 1.1022168397903442, "Finetune/Step": 746, "Finetune/Step Time": 5.038975011557341} +{"Finetune/Learning Rate": 1.9327631539296258e-05, "Finetune/Loss": 0.9946231842041016, "Finetune/Loss (Raw)": 1.119739055633545, "Finetune/Step": 747, "Finetune/Step Time": 5.018712859600782} +{"Finetune/Learning Rate": 1.932538401838246e-05, "Finetune/Loss": 0.9951435923576355, "Finetune/Loss (Raw)": 1.1013975143432617, "Finetune/Step": 748, "Finetune/Step Time": 5.020640540868044} +{"Finetune/Learning Rate": 1.9323132878469185e-05, "Finetune/Loss": 0.995061993598938, "Finetune/Loss (Raw)": 1.0099135637283325, "Finetune/Step": 749, "Finetune/Step Time": 5.010008126497269} +{"Finetune/Learning Rate": 1.9320878120430063e-05, "Finetune/Loss": 0.9954855442047119, "Finetune/Loss (Raw)": 1.0239015817642212, "Finetune/Step": 750, "Finetune/Step Time": 5.015773717314005} +{"Finetune/Learning Rate": 1.9318619745140115e-05, "Finetune/Loss": 0.9940518140792847, "Finetune/Loss (Raw)": 0.8941459059715271, "Finetune/Step": 751, "Finetune/Step Time": 5.018619397655129} +{"Finetune/Learning Rate": 1.9316357753475776e-05, "Finetune/Loss": 0.9947143197059631, "Finetune/Loss (Raw)": 1.0825361013412476, "Finetune/Step": 752, "Finetune/Step Time": 5.0192894116044044} +{"Finetune/Learning Rate": 1.9314092146314872e-05, "Finetune/Loss": 0.9895581007003784, "Finetune/Loss (Raw)": 0.3982999920845032, "Finetune/Step": 753, "Finetune/Step Time": 5.051947880536318} +{"Finetune/Learning Rate": 1.9311822924536656e-05, "Finetune/Loss": 0.9880682229995728, "Finetune/Loss (Raw)": 0.8768162727355957, "Finetune/Step": 754, "Finetune/Step Time": 5.051837140694261} +{"Finetune/Learning Rate": 1.930955008902176e-05, "Finetune/Loss": 0.9877280592918396, "Finetune/Loss (Raw)": 0.934075117111206, "Finetune/Step": 755, "Finetune/Step Time": 5.073706278577447} +{"Finetune/Learning Rate": 1.9307273640652228e-05, "Finetune/Loss": 0.9874012470245361, "Finetune/Loss (Raw)": 0.9522483944892883, "Finetune/Step": 756, "Finetune/Step Time": 5.07087842002511} +{"Finetune/Learning Rate": 1.9304993580311505e-05, "Finetune/Loss": 0.9871427416801453, "Finetune/Loss (Raw)": 0.9950311779975891, "Finetune/Step": 757, "Finetune/Step Time": 5.076744740828872} +{"Finetune/Learning Rate": 1.9302709908884437e-05, "Finetune/Loss": 0.9872831106185913, "Finetune/Loss (Raw)": 1.1584763526916504, "Finetune/Step": 758, "Finetune/Step Time": 5.085367722436786} +{"Finetune/Learning Rate": 1.9300422627257275e-05, "Finetune/Loss": 0.9885210990905762, "Finetune/Loss (Raw)": 1.0716737508773804, "Finetune/Step": 759, "Finetune/Step Time": 5.085014156997204} +{"Finetune/Learning Rate": 1.9298131736317666e-05, "Finetune/Loss": 0.9873449802398682, "Finetune/Loss (Raw)": 1.0239289999008179, "Finetune/Step": 760, "Finetune/Step Time": 5.0720257721841335} +{"Finetune/Learning Rate": 1.929583723695466e-05, "Finetune/Loss": 0.9847659468650818, "Finetune/Loss (Raw)": 0.9336463809013367, "Finetune/Step": 761, "Finetune/Step Time": 5.081034449860454} +{"Finetune/Learning Rate": 1.9293539130058708e-05, "Finetune/Loss": 0.9803714156150818, "Finetune/Loss (Raw)": 0.38337668776512146, "Finetune/Step": 762, "Finetune/Step Time": 5.119678653776646} +{"Finetune/Learning Rate": 1.9291237416521662e-05, "Finetune/Loss": 0.9795587062835693, "Finetune/Loss (Raw)": 0.8963289856910706, "Finetune/Step": 763, "Finetune/Step Time": 5.12312456779182} +{"Finetune/Learning Rate": 1.9288932097236772e-05, "Finetune/Loss": 0.9789758920669556, "Finetune/Loss (Raw)": 1.0048905611038208, "Finetune/Step": 764, "Finetune/Step Time": 5.100444218143821} +{"Finetune/Learning Rate": 1.9286623173098683e-05, "Finetune/Loss": 0.9780091643333435, "Finetune/Loss (Raw)": 0.977677583694458, "Finetune/Step": 765, "Finetune/Step Time": 5.109964109957218} +{"Finetune/Learning Rate": 1.928431064500345e-05, "Finetune/Loss": 0.9779928922653198, "Finetune/Loss (Raw)": 1.0990612506866455, "Finetune/Step": 766, "Finetune/Step Time": 5.128329269587994} +{"Finetune/Learning Rate": 1.9281994513848515e-05, "Finetune/Loss": 0.977253258228302, "Finetune/Loss (Raw)": 0.925608217716217, "Finetune/Step": 767, "Finetune/Step Time": 5.142998121678829} +{"Finetune/Learning Rate": 1.9279674780532726e-05, "Finetune/Loss": 0.9766963124275208, "Finetune/Loss (Raw)": 0.9385460615158081, "Finetune/Step": 768, "Finetune/Step Time": 5.141831114888191} +{"Finetune/Learning Rate": 1.9277351445956323e-05, "Finetune/Loss": 0.9745356440544128, "Finetune/Loss (Raw)": 0.8281554579734802, "Finetune/Step": 769, "Finetune/Step Time": 5.149484625086188} +{"Finetune/Learning Rate": 1.927502451102095e-05, "Finetune/Loss": 0.9799062013626099, "Finetune/Loss (Raw)": 1.0304315090179443, "Finetune/Step": 770, "Finetune/Step Time": 5.127749420702457} +{"Finetune/Learning Rate": 1.9272693976629642e-05, "Finetune/Loss": 0.9805049896240234, "Finetune/Loss (Raw)": 1.0776846408843994, "Finetune/Step": 771, "Finetune/Step Time": 5.127408506348729} +{"Finetune/Learning Rate": 1.9270359843686837e-05, "Finetune/Loss": 0.9801443815231323, "Finetune/Loss (Raw)": 1.0033912658691406, "Finetune/Step": 772, "Finetune/Step Time": 5.121114680543542} +{"Finetune/Learning Rate": 1.9268022113098366e-05, "Finetune/Loss": 0.9799292683601379, "Finetune/Loss (Raw)": 1.060042381286621, "Finetune/Step": 773, "Finetune/Step Time": 5.131305234506726} +{"Finetune/Learning Rate": 1.9265680785771455e-05, "Finetune/Loss": 0.9787600040435791, "Finetune/Loss (Raw)": 0.8752239346504211, "Finetune/Step": 774, "Finetune/Step Time": 5.142752589657903} +{"Finetune/Learning Rate": 1.926333586261473e-05, "Finetune/Loss": 0.9793838262557983, "Finetune/Loss (Raw)": 1.035064458847046, "Finetune/Step": 775, "Finetune/Step Time": 5.150161473080516} +{"Finetune/Learning Rate": 1.9260987344538205e-05, "Finetune/Loss": 0.9775415658950806, "Finetune/Loss (Raw)": 0.7988722324371338, "Finetune/Step": 776, "Finetune/Step Time": 5.146291213110089} +{"Finetune/Learning Rate": 1.9258635232453296e-05, "Finetune/Loss": 0.9784183502197266, "Finetune/Loss (Raw)": 1.146301507949829, "Finetune/Step": 777, "Finetune/Step Time": 5.163201721385121} +{"Finetune/Learning Rate": 1.9256279527272815e-05, "Finetune/Loss": 0.9792419075965881, "Finetune/Loss (Raw)": 1.0990017652511597, "Finetune/Step": 778, "Finetune/Step Time": 5.147775506600738} +{"Finetune/Learning Rate": 1.9253920229910966e-05, "Finetune/Loss": 0.9785172939300537, "Finetune/Loss (Raw)": 0.9361431002616882, "Finetune/Step": 779, "Finetune/Step Time": 5.144545126706362} +{"Finetune/Learning Rate": 1.925155734128334e-05, "Finetune/Loss": 0.9732644557952881, "Finetune/Loss (Raw)": 0.33725279569625854, "Finetune/Step": 780, "Finetune/Step Time": 5.1793375834822655} +{"Finetune/Learning Rate": 1.9249190862306935e-05, "Finetune/Loss": 0.9733937382698059, "Finetune/Loss (Raw)": 0.937826931476593, "Finetune/Step": 781, "Finetune/Step Time": 5.179741462692618} +{"Finetune/Learning Rate": 1.9246820793900127e-05, "Finetune/Loss": 0.9696903228759766, "Finetune/Loss (Raw)": 0.738869845867157, "Finetune/Step": 782, "Finetune/Step Time": 5.21052105166018} +{"Finetune/Learning Rate": 1.92444471369827e-05, "Finetune/Loss": 0.969202995300293, "Finetune/Loss (Raw)": 0.9278907179832458, "Finetune/Step": 783, "Finetune/Step Time": 5.256995828822255} +{"Finetune/Learning Rate": 1.924206989247582e-05, "Finetune/Loss": 0.967455267906189, "Finetune/Loss (Raw)": 0.7517579793930054, "Finetune/Step": 784, "Finetune/Step Time": 5.274478288367391} +{"Finetune/Learning Rate": 1.9239689061302053e-05, "Finetune/Loss": 0.9684292078018188, "Finetune/Loss (Raw)": 1.0835802555084229, "Finetune/Step": 785, "Finetune/Step Time": 5.270031470805407} +{"Finetune/Learning Rate": 1.923730464438535e-05, "Finetune/Loss": 0.969701886177063, "Finetune/Loss (Raw)": 1.1447604894638062, "Finetune/Step": 786, "Finetune/Step Time": 5.267816295847297} +{"Finetune/Learning Rate": 1.923491664265106e-05, "Finetune/Loss": 0.9721693992614746, "Finetune/Loss (Raw)": 1.1211297512054443, "Finetune/Step": 787, "Finetune/Step Time": 5.27205865830183} +{"Finetune/Learning Rate": 1.9232525057025915e-05, "Finetune/Loss": 0.9721307754516602, "Finetune/Loss (Raw)": 1.1276428699493408, "Finetune/Step": 788, "Finetune/Step Time": 5.268798474222422} +{"Finetune/Learning Rate": 1.9230129888438046e-05, "Finetune/Loss": 0.9720774292945862, "Finetune/Loss (Raw)": 1.007483720779419, "Finetune/Step": 789, "Finetune/Step Time": 5.267186472192407} +{"Finetune/Learning Rate": 1.922773113781697e-05, "Finetune/Loss": 0.9697521924972534, "Finetune/Loss (Raw)": 0.7644444704055786, "Finetune/Step": 790, "Finetune/Step Time": 5.297864800319076} +{"Finetune/Learning Rate": 1.9225328806093596e-05, "Finetune/Loss": 0.9683843851089478, "Finetune/Loss (Raw)": 0.9381561279296875, "Finetune/Step": 791, "Finetune/Step Time": 5.299507299438119} +{"Finetune/Learning Rate": 1.922292289420022e-05, "Finetune/Loss": 0.9669860601425171, "Finetune/Loss (Raw)": 0.8474454283714294, "Finetune/Step": 792, "Finetune/Step Time": 5.290224451571703} +{"Finetune/Learning Rate": 1.9220513403070534e-05, "Finetune/Loss": 0.9663783311843872, "Finetune/Loss (Raw)": 1.0846749544143677, "Finetune/Step": 793, "Finetune/Step Time": 5.292678385972977} +{"Finetune/Learning Rate": 1.9218100333639608e-05, "Finetune/Loss": 0.9645745754241943, "Finetune/Loss (Raw)": 0.9420725107192993, "Finetune/Step": 794, "Finetune/Step Time": 5.295423639938235} +{"Finetune/Learning Rate": 1.9215683686843912e-05, "Finetune/Loss": 0.9635137915611267, "Finetune/Loss (Raw)": 0.9883154034614563, "Finetune/Step": 795, "Finetune/Step Time": 5.29828997515142} +{"Finetune/Learning Rate": 1.92132634636213e-05, "Finetune/Loss": 0.9641846418380737, "Finetune/Loss (Raw)": 0.9527900218963623, "Finetune/Step": 796, "Finetune/Step Time": 5.294223561882973} +{"Finetune/Learning Rate": 1.9210839664911013e-05, "Finetune/Loss": 0.9637213945388794, "Finetune/Loss (Raw)": 0.980065643787384, "Finetune/Step": 797, "Finetune/Step Time": 5.3001519199460745} +{"Finetune/Learning Rate": 1.9208412291653674e-05, "Finetune/Loss": 0.9645062685012817, "Finetune/Loss (Raw)": 1.0190093517303467, "Finetune/Step": 798, "Finetune/Step Time": 5.30228403583169} +{"Finetune/Learning Rate": 1.9205981344791307e-05, "Finetune/Loss": 0.9638303518295288, "Finetune/Loss (Raw)": 0.9097663164138794, "Finetune/Step": 799, "Finetune/Step Time": 5.309363998472691} +{"Finetune/Learning Rate": 1.9203546825267313e-05, "Finetune/Loss": 0.9638274908065796, "Finetune/Loss (Raw)": 1.0610675811767578, "Finetune/Step": 800, "Finetune/Step Time": 5.314825305715203} +{"Finetune/Learning Rate": 1.9201108734026477e-05, "Finetune/Loss": 0.9634997844696045, "Finetune/Loss (Raw)": 1.0780309438705444, "Finetune/Step": 801, "Finetune/Step Time": 5.340311290696263} +{"Finetune/Learning Rate": 1.9198667072014978e-05, "Finetune/Loss": 0.9625194072723389, "Finetune/Loss (Raw)": 0.9944387078285217, "Finetune/Step": 802, "Finetune/Step Time": 5.343821285292506} +{"Finetune/Learning Rate": 1.9196221840180378e-05, "Finetune/Loss": 0.9604116678237915, "Finetune/Loss (Raw)": 0.8000436425209045, "Finetune/Step": 803, "Finetune/Step Time": 5.350265277549624} +{"Finetune/Learning Rate": 1.9193773039471622e-05, "Finetune/Loss": 0.9600605964660645, "Finetune/Loss (Raw)": 0.991849958896637, "Finetune/Step": 804, "Finetune/Step Time": 5.341529242694378} +{"Finetune/Learning Rate": 1.9191320670839047e-05, "Finetune/Loss": 0.9609929323196411, "Finetune/Loss (Raw)": 1.0385912656784058, "Finetune/Step": 805, "Finetune/Step Time": 5.310778684914112} +{"Finetune/Learning Rate": 1.918886473523436e-05, "Finetune/Loss": 0.9587118625640869, "Finetune/Loss (Raw)": 0.8458172082901001, "Finetune/Step": 806, "Finetune/Step Time": 5.325374471023679} +{"Finetune/Learning Rate": 1.918640523361067e-05, "Finetune/Loss": 0.9575071334838867, "Finetune/Loss (Raw)": 0.9924420118331909, "Finetune/Step": 807, "Finetune/Step Time": 5.317396506667137} +{"Finetune/Learning Rate": 1.9183942166922455e-05, "Finetune/Loss": 0.9581321477890015, "Finetune/Loss (Raw)": 1.0461459159851074, "Finetune/Step": 808, "Finetune/Step Time": 5.3371075969189405} +{"Finetune/Learning Rate": 1.9181475536125588e-05, "Finetune/Loss": 0.95722496509552, "Finetune/Loss (Raw)": 0.9550567865371704, "Finetune/Step": 809, "Finetune/Step Time": 5.34096802957356} +{"Finetune/Learning Rate": 1.917900534217732e-05, "Finetune/Loss": 0.9573192596435547, "Finetune/Loss (Raw)": 1.0963058471679688, "Finetune/Step": 810, "Finetune/Step Time": 5.344350501894951} +{"Finetune/Learning Rate": 1.9176531586036282e-05, "Finetune/Loss": 0.9552673101425171, "Finetune/Loss (Raw)": 0.849527895450592, "Finetune/Step": 811, "Finetune/Step Time": 5.341580927371979} +{"Finetune/Learning Rate": 1.917405426866249e-05, "Finetune/Loss": 0.9573540091514587, "Finetune/Loss (Raw)": 1.2308189868927002, "Finetune/Step": 812, "Finetune/Step Time": 5.339259205386043} +{"Finetune/Learning Rate": 1.9171573391017348e-05, "Finetune/Loss": 0.9591079354286194, "Finetune/Loss (Raw)": 0.9894137382507324, "Finetune/Step": 813, "Finetune/Step Time": 5.326047016307712} +{"Finetune/Learning Rate": 1.9169088954063634e-05, "Finetune/Loss": 0.9593037962913513, "Finetune/Loss (Raw)": 0.962084174156189, "Finetune/Step": 814, "Finetune/Step Time": 5.315304089337587} +{"Finetune/Learning Rate": 1.9166600958765508e-05, "Finetune/Loss": 0.9593299031257629, "Finetune/Loss (Raw)": 0.9814768433570862, "Finetune/Step": 815, "Finetune/Step Time": 5.319195043295622} +{"Finetune/Learning Rate": 1.916410940608851e-05, "Finetune/Loss": 0.9597513675689697, "Finetune/Loss (Raw)": 1.0733113288879395, "Finetune/Step": 816, "Finetune/Step Time": 5.319276003167033} +{"Finetune/Learning Rate": 1.9161614296999565e-05, "Finetune/Loss": 0.9623985290527344, "Finetune/Loss (Raw)": 1.0785672664642334, "Finetune/Step": 817, "Finetune/Step Time": 5.3081794288009405} +{"Finetune/Learning Rate": 1.9159115632466978e-05, "Finetune/Loss": 0.9584169387817383, "Finetune/Loss (Raw)": 0.33101028203964233, "Finetune/Step": 818, "Finetune/Step Time": 5.292951378971338} +{"Finetune/Learning Rate": 1.9156613413460432e-05, "Finetune/Loss": 0.9565472602844238, "Finetune/Loss (Raw)": 0.8213281035423279, "Finetune/Step": 819, "Finetune/Step Time": 5.293173497542739} +{"Finetune/Learning Rate": 1.9154107640950986e-05, "Finetune/Loss": 0.9549781680107117, "Finetune/Loss (Raw)": 0.7390363216400146, "Finetune/Step": 820, "Finetune/Step Time": 5.273684876039624} +{"Finetune/Learning Rate": 1.9151598315911088e-05, "Finetune/Loss": 0.9550483822822571, "Finetune/Loss (Raw)": 0.9974498748779297, "Finetune/Step": 821, "Finetune/Step Time": 5.274082202464342} +{"Finetune/Learning Rate": 1.9149085439314548e-05, "Finetune/Loss": 0.9591823816299438, "Finetune/Loss (Raw)": 0.9307243824005127, "Finetune/Step": 822, "Finetune/Step Time": 5.242355398833752} +{"Finetune/Learning Rate": 1.9146569012136572e-05, "Finetune/Loss": 0.9609411358833313, "Finetune/Loss (Raw)": 1.0957801342010498, "Finetune/Step": 823, "Finetune/Step Time": 5.228224562481046} +{"Finetune/Learning Rate": 1.914404903535373e-05, "Finetune/Loss": 0.960726261138916, "Finetune/Loss (Raw)": 1.0883980989456177, "Finetune/Step": 824, "Finetune/Step Time": 5.227111669257283} +{"Finetune/Learning Rate": 1.9141525509943987e-05, "Finetune/Loss": 0.960538387298584, "Finetune/Loss (Raw)": 0.9186257719993591, "Finetune/Step": 825, "Finetune/Step Time": 5.217047840356827} +{"Finetune/Learning Rate": 1.9138998436886664e-05, "Finetune/Loss": 0.9579210877418518, "Finetune/Loss (Raw)": 0.8565553426742554, "Finetune/Step": 826, "Finetune/Step Time": 5.2232910096645355} +{"Finetune/Learning Rate": 1.913646781716247e-05, "Finetune/Loss": 0.956330418586731, "Finetune/Loss (Raw)": 1.0040884017944336, "Finetune/Step": 827, "Finetune/Step Time": 5.209741184487939} +{"Finetune/Learning Rate": 1.9133933651753493e-05, "Finetune/Loss": 0.9515266418457031, "Finetune/Loss (Raw)": 0.38209667801856995, "Finetune/Step": 828, "Finetune/Step Time": 5.2295798026025295} +{"Finetune/Learning Rate": 1.913139594164319e-05, "Finetune/Loss": 0.9574564695358276, "Finetune/Loss (Raw)": 1.0725749731063843, "Finetune/Step": 829, "Finetune/Step Time": 5.192860769107938} +{"Finetune/Learning Rate": 1.9128854687816394e-05, "Finetune/Loss": 0.9575859904289246, "Finetune/Loss (Raw)": 0.9960843920707703, "Finetune/Step": 830, "Finetune/Step Time": 5.1898331716656685} +{"Finetune/Learning Rate": 1.9126309891259324e-05, "Finetune/Loss": 0.9588609933853149, "Finetune/Loss (Raw)": 1.0446323156356812, "Finetune/Step": 831, "Finetune/Step Time": 5.206051344051957} +{"Finetune/Learning Rate": 1.9123761552959562e-05, "Finetune/Loss": 0.9596130847930908, "Finetune/Loss (Raw)": 1.0890989303588867, "Finetune/Step": 832, "Finetune/Step Time": 5.18252033367753} +{"Finetune/Learning Rate": 1.9121209673906064e-05, "Finetune/Loss": 0.954993724822998, "Finetune/Loss (Raw)": 0.4260670244693756, "Finetune/Step": 833, "Finetune/Step Time": 5.21341460570693} +{"Finetune/Learning Rate": 1.911865425508917e-05, "Finetune/Loss": 0.956584632396698, "Finetune/Loss (Raw)": 0.9921127557754517, "Finetune/Step": 834, "Finetune/Step Time": 5.215992372483015} +{"Finetune/Learning Rate": 1.9116095297500588e-05, "Finetune/Loss": 0.9572432041168213, "Finetune/Loss (Raw)": 0.9909285306930542, "Finetune/Step": 835, "Finetune/Step Time": 5.22149364463985} +{"Finetune/Learning Rate": 1.9113532802133394e-05, "Finetune/Loss": 0.9560850262641907, "Finetune/Loss (Raw)": 1.0134122371673584, "Finetune/Step": 836, "Finetune/Step Time": 5.216170271858573} +{"Finetune/Learning Rate": 1.911096676998205e-05, "Finetune/Loss": 0.9558022022247314, "Finetune/Loss (Raw)": 0.934410810470581, "Finetune/Step": 837, "Finetune/Step Time": 5.230864468961954} +{"Finetune/Learning Rate": 1.9108397202042376e-05, "Finetune/Loss": 0.9571876525878906, "Finetune/Loss (Raw)": 1.0623996257781982, "Finetune/Step": 838, "Finetune/Step Time": 5.23518000356853} +{"Finetune/Learning Rate": 1.910582409931158e-05, "Finetune/Loss": 0.9558357000350952, "Finetune/Loss (Raw)": 0.9108242988586426, "Finetune/Step": 839, "Finetune/Step Time": 5.227130556479096} +{"Finetune/Learning Rate": 1.910324746278822e-05, "Finetune/Loss": 0.9593362212181091, "Finetune/Loss (Raw)": 1.1694401502609253, "Finetune/Step": 840, "Finetune/Step Time": 5.220928767696023} +{"Finetune/Learning Rate": 1.9100667293472252e-05, "Finetune/Loss": 0.9586538076400757, "Finetune/Loss (Raw)": 0.8970925211906433, "Finetune/Step": 841, "Finetune/Step Time": 5.219846453517675} +{"Finetune/Learning Rate": 1.9098083592364976e-05, "Finetune/Loss": 0.9596871137619019, "Finetune/Loss (Raw)": 1.0707123279571533, "Finetune/Step": 842, "Finetune/Step Time": 5.2184169832617044} +{"Finetune/Learning Rate": 1.9095496360469087e-05, "Finetune/Loss": 0.9591033458709717, "Finetune/Loss (Raw)": 1.0458126068115234, "Finetune/Step": 843, "Finetune/Step Time": 5.237881109118462} +{"Finetune/Learning Rate": 1.9092905598788633e-05, "Finetune/Loss": 0.9600910544395447, "Finetune/Loss (Raw)": 0.9982101917266846, "Finetune/Step": 844, "Finetune/Step Time": 5.239482000470161} +{"Finetune/Learning Rate": 1.9090311308329045e-05, "Finetune/Loss": 0.9596470594406128, "Finetune/Loss (Raw)": 1.0812160968780518, "Finetune/Step": 845, "Finetune/Step Time": 5.225086925551295} +{"Finetune/Learning Rate": 1.9087713490097108e-05, "Finetune/Loss": 0.9595385789871216, "Finetune/Loss (Raw)": 1.0439866781234741, "Finetune/Step": 846, "Finetune/Step Time": 5.216955939307809} +{"Finetune/Learning Rate": 1.9085112145100987e-05, "Finetune/Loss": 0.9550637006759644, "Finetune/Loss (Raw)": 0.46518996357917786, "Finetune/Step": 847, "Finetune/Step Time": 5.2497263960540295} +{"Finetune/Learning Rate": 1.9082507274350216e-05, "Finetune/Loss": 0.9522544145584106, "Finetune/Loss (Raw)": 0.8522880673408508, "Finetune/Step": 848, "Finetune/Step Time": 5.25772107206285} +{"Finetune/Learning Rate": 1.9079898878855695e-05, "Finetune/Loss": 0.952856183052063, "Finetune/Loss (Raw)": 1.0039640665054321, "Finetune/Step": 849, "Finetune/Step Time": 5.235235944390297} +{"Finetune/Learning Rate": 1.9077286959629684e-05, "Finetune/Loss": 0.9520513415336609, "Finetune/Loss (Raw)": 0.8899961709976196, "Finetune/Step": 850, "Finetune/Step Time": 5.219223832711577} +{"Finetune/Learning Rate": 1.907467151768583e-05, "Finetune/Loss": 0.9512882232666016, "Finetune/Loss (Raw)": 1.021248459815979, "Finetune/Step": 851, "Finetune/Step Time": 5.225545799359679} +{"Finetune/Learning Rate": 1.9072052554039123e-05, "Finetune/Loss": 0.9497181177139282, "Finetune/Loss (Raw)": 0.8836547136306763, "Finetune/Step": 852, "Finetune/Step Time": 5.210140015929937} +{"Finetune/Learning Rate": 1.9069430069705936e-05, "Finetune/Loss": 0.9482698440551758, "Finetune/Loss (Raw)": 0.9569802284240723, "Finetune/Step": 853, "Finetune/Step Time": 5.206484258174896} +{"Finetune/Learning Rate": 1.9066804065704006e-05, "Finetune/Loss": 0.9495933055877686, "Finetune/Loss (Raw)": 0.9418607354164124, "Finetune/Step": 854, "Finetune/Step Time": 5.209132494404912} +{"Finetune/Learning Rate": 1.9064174543052435e-05, "Finetune/Loss": 0.9477933645248413, "Finetune/Loss (Raw)": 0.8020903468132019, "Finetune/Step": 855, "Finetune/Step Time": 5.245313806459308} +{"Finetune/Learning Rate": 1.9061541502771682e-05, "Finetune/Loss": 0.9461472034454346, "Finetune/Loss (Raw)": 0.9903080463409424, "Finetune/Step": 856, "Finetune/Step Time": 5.242433933541179} +{"Finetune/Learning Rate": 1.905890494588359e-05, "Finetune/Loss": 0.948509931564331, "Finetune/Loss (Raw)": 0.9480907917022705, "Finetune/Step": 857, "Finetune/Step Time": 5.232376979663968} +{"Finetune/Learning Rate": 1.9056264873411343e-05, "Finetune/Loss": 0.9499175548553467, "Finetune/Loss (Raw)": 1.1898093223571777, "Finetune/Step": 858, "Finetune/Step Time": 5.229810917750001} +{"Finetune/Learning Rate": 1.905362128637951e-05, "Finetune/Loss": 0.9499955177307129, "Finetune/Loss (Raw)": 1.1021525859832764, "Finetune/Step": 859, "Finetune/Step Time": 5.24473375454545} +{"Finetune/Learning Rate": 1.905097418581401e-05, "Finetune/Loss": 0.9503581523895264, "Finetune/Loss (Raw)": 0.9543057680130005, "Finetune/Step": 860, "Finetune/Step Time": 5.212647644802928} +{"Finetune/Learning Rate": 1.904832357274214e-05, "Finetune/Loss": 0.9485514760017395, "Finetune/Loss (Raw)": 0.7285761833190918, "Finetune/Step": 861, "Finetune/Step Time": 5.202578108757734} +{"Finetune/Learning Rate": 1.9045669448192537e-05, "Finetune/Loss": 0.9497032165527344, "Finetune/Loss (Raw)": 1.0195887088775635, "Finetune/Step": 862, "Finetune/Step Time": 5.205148970708251} +{"Finetune/Learning Rate": 1.904301181319523e-05, "Finetune/Loss": 0.9491969347000122, "Finetune/Loss (Raw)": 0.8400558829307556, "Finetune/Step": 863, "Finetune/Step Time": 5.21080213598907} +{"Finetune/Learning Rate": 1.9040350668781584e-05, "Finetune/Loss": 0.9500964879989624, "Finetune/Loss (Raw)": 1.0455143451690674, "Finetune/Step": 864, "Finetune/Step Time": 5.177763482555747} +{"Finetune/Learning Rate": 1.9037686015984343e-05, "Finetune/Loss": 0.9488067626953125, "Finetune/Loss (Raw)": 0.9595489501953125, "Finetune/Step": 865, "Finetune/Step Time": 5.199847096577287} +{"Finetune/Learning Rate": 1.9035017855837604e-05, "Finetune/Loss": 0.9496256113052368, "Finetune/Loss (Raw)": 1.1235398054122925, "Finetune/Step": 866, "Finetune/Step Time": 5.2014702670276165} +{"Finetune/Learning Rate": 1.9032346189376828e-05, "Finetune/Loss": 0.9543378353118896, "Finetune/Loss (Raw)": 1.0639371871948242, "Finetune/Step": 867, "Finetune/Step Time": 5.164667377248406} +{"Finetune/Learning Rate": 1.902967101763884e-05, "Finetune/Loss": 0.952797532081604, "Finetune/Loss (Raw)": 0.9447556138038635, "Finetune/Step": 868, "Finetune/Step Time": 5.1579577177762985} +{"Finetune/Learning Rate": 1.902699234166182e-05, "Finetune/Loss": 0.9549903273582458, "Finetune/Loss (Raw)": 1.0924514532089233, "Finetune/Step": 869, "Finetune/Step Time": 5.148923186585307} +{"Finetune/Learning Rate": 1.9024310162485308e-05, "Finetune/Loss": 0.9543457627296448, "Finetune/Loss (Raw)": 0.9026516675949097, "Finetune/Step": 870, "Finetune/Step Time": 5.152604427188635} +{"Finetune/Learning Rate": 1.9021624481150208e-05, "Finetune/Loss": 0.9526077508926392, "Finetune/Loss (Raw)": 0.850795328617096, "Finetune/Step": 871, "Finetune/Step Time": 5.156005976721644} +{"Finetune/Learning Rate": 1.9018935298698777e-05, "Finetune/Loss": 0.9526318311691284, "Finetune/Loss (Raw)": 0.9991310834884644, "Finetune/Step": 872, "Finetune/Step Time": 5.155261328443885} +{"Finetune/Learning Rate": 1.9016242616174637e-05, "Finetune/Loss": 0.9538247585296631, "Finetune/Loss (Raw)": 1.049200177192688, "Finetune/Step": 873, "Finetune/Step Time": 5.139051802456379} +{"Finetune/Learning Rate": 1.9013546434622766e-05, "Finetune/Loss": 0.9521031975746155, "Finetune/Loss (Raw)": 0.8818553686141968, "Finetune/Step": 874, "Finetune/Step Time": 5.144468193873763} +{"Finetune/Learning Rate": 1.90108467550895e-05, "Finetune/Loss": 0.9507647752761841, "Finetune/Loss (Raw)": 0.9484171271324158, "Finetune/Step": 875, "Finetune/Step Time": 5.1556013487279415} +{"Finetune/Learning Rate": 1.9008143578622527e-05, "Finetune/Loss": 0.949488639831543, "Finetune/Loss (Raw)": 0.938053548336029, "Finetune/Step": 876, "Finetune/Step Time": 5.1567196398973465} +{"Finetune/Learning Rate": 1.9005436906270902e-05, "Finetune/Loss": 0.9496620893478394, "Finetune/Loss (Raw)": 1.032117486000061, "Finetune/Step": 877, "Finetune/Step Time": 5.155267680063844} +{"Finetune/Learning Rate": 1.9002726739085027e-05, "Finetune/Loss": 0.9494426250457764, "Finetune/Loss (Raw)": 0.9958024621009827, "Finetune/Step": 878, "Finetune/Step Time": 5.147775128483772} +{"Finetune/Learning Rate": 1.900001307811667e-05, "Finetune/Loss": 0.9494185447692871, "Finetune/Loss (Raw)": 0.8910661339759827, "Finetune/Step": 879, "Finetune/Step Time": 5.169141186401248} +{"Finetune/Learning Rate": 1.899729592441895e-05, "Finetune/Loss": 0.9488871097564697, "Finetune/Loss (Raw)": 1.0145211219787598, "Finetune/Step": 880, "Finetune/Step Time": 5.173553796485066} +{"Finetune/Learning Rate": 1.899457527904634e-05, "Finetune/Loss": 0.952715277671814, "Finetune/Loss (Raw)": 0.8883066177368164, "Finetune/Step": 881, "Finetune/Step Time": 5.152323812246323} +{"Finetune/Learning Rate": 1.8991851143054662e-05, "Finetune/Loss": 0.9542078971862793, "Finetune/Loss (Raw)": 1.067866325378418, "Finetune/Step": 882, "Finetune/Step Time": 5.152179686352611} +{"Finetune/Learning Rate": 1.8989123517501112e-05, "Finetune/Loss": 0.9551854133605957, "Finetune/Loss (Raw)": 1.0591953992843628, "Finetune/Step": 883, "Finetune/Step Time": 5.123002598062158} +{"Finetune/Learning Rate": 1.8986392403444222e-05, "Finetune/Loss": 0.9561042189598083, "Finetune/Loss (Raw)": 1.0698590278625488, "Finetune/Step": 884, "Finetune/Step Time": 5.123694580048323} +{"Finetune/Learning Rate": 1.8983657801943884e-05, "Finetune/Loss": 0.9501329660415649, "Finetune/Loss (Raw)": 0.23071534931659698, "Finetune/Step": 885, "Finetune/Step Time": 5.146024268120527} +{"Finetune/Learning Rate": 1.8980919714061343e-05, "Finetune/Loss": 0.9490258693695068, "Finetune/Loss (Raw)": 1.0167605876922607, "Finetune/Step": 886, "Finetune/Step Time": 5.149193761870265} +{"Finetune/Learning Rate": 1.89781781408592e-05, "Finetune/Loss": 0.9438313245773315, "Finetune/Loss (Raw)": 0.4067729711532593, "Finetune/Step": 887, "Finetune/Step Time": 5.18738554045558} +{"Finetune/Learning Rate": 1.8975433083401403e-05, "Finetune/Loss": 0.9443157911300659, "Finetune/Loss (Raw)": 1.0859397649765015, "Finetune/Step": 888, "Finetune/Step Time": 5.193278992548585} +{"Finetune/Learning Rate": 1.897268454275326e-05, "Finetune/Loss": 0.9452013373374939, "Finetune/Loss (Raw)": 1.0470002889633179, "Finetune/Step": 889, "Finetune/Step Time": 5.193053171038628} +{"Finetune/Learning Rate": 1.896993251998142e-05, "Finetune/Loss": 0.9498244524002075, "Finetune/Loss (Raw)": 0.9751304984092712, "Finetune/Step": 890, "Finetune/Step Time": 5.155728995800018} +{"Finetune/Learning Rate": 1.8967177016153895e-05, "Finetune/Loss": 0.9505740404129028, "Finetune/Loss (Raw)": 0.9922778606414795, "Finetune/Step": 891, "Finetune/Step Time": 5.159192962571979} +{"Finetune/Learning Rate": 1.8964418032340037e-05, "Finetune/Loss": 0.9496957659721375, "Finetune/Loss (Raw)": 0.892473578453064, "Finetune/Step": 892, "Finetune/Step Time": 5.163453662768006} +{"Finetune/Learning Rate": 1.8961655569610557e-05, "Finetune/Loss": 0.9508664608001709, "Finetune/Loss (Raw)": 1.1275265216827393, "Finetune/Step": 893, "Finetune/Step Time": 5.154268929734826} +{"Finetune/Learning Rate": 1.8958889629037514e-05, "Finetune/Loss": 0.9498803019523621, "Finetune/Loss (Raw)": 0.9728276133537292, "Finetune/Step": 894, "Finetune/Step Time": 5.144171291962266} +{"Finetune/Learning Rate": 1.895612021169431e-05, "Finetune/Loss": 0.9502938985824585, "Finetune/Loss (Raw)": 0.9785502552986145, "Finetune/Step": 895, "Finetune/Step Time": 5.150757346302271} +{"Finetune/Learning Rate": 1.8953347318655704e-05, "Finetune/Loss": 0.9513174295425415, "Finetune/Loss (Raw)": 1.0695618391036987, "Finetune/Step": 896, "Finetune/Step Time": 5.148141985759139} +{"Finetune/Learning Rate": 1.8950570950997808e-05, "Finetune/Loss": 0.951511025428772, "Finetune/Loss (Raw)": 0.8529355525970459, "Finetune/Step": 897, "Finetune/Step Time": 5.1517349276691675} +{"Finetune/Learning Rate": 1.894779110979807e-05, "Finetune/Loss": 0.9462876319885254, "Finetune/Loss (Raw)": 0.3618374168872833, "Finetune/Step": 898, "Finetune/Step Time": 5.173249954357743} +{"Finetune/Learning Rate": 1.894500779613529e-05, "Finetune/Loss": 0.9443185329437256, "Finetune/Loss (Raw)": 0.8256357312202454, "Finetune/Step": 899, "Finetune/Step Time": 5.251079060137272} +{"Finetune/Learning Rate": 1.8942221011089623e-05, "Finetune/Loss": 0.9451932907104492, "Finetune/Loss (Raw)": 1.115369200706482, "Finetune/Step": 900, "Finetune/Step Time": 5.250496478751302} +{"Finetune/Learning Rate": 1.8939430755742564e-05, "Finetune/Loss": 0.9455603957176208, "Finetune/Loss (Raw)": 1.1070259809494019, "Finetune/Step": 901, "Finetune/Step Time": 5.244980836287141} +{"Finetune/Learning Rate": 1.8936637031176957e-05, "Finetune/Loss": 0.9470219612121582, "Finetune/Loss (Raw)": 1.0623103380203247, "Finetune/Step": 902, "Finetune/Step Time": 5.232993267476559} +{"Finetune/Learning Rate": 1.893383983847699e-05, "Finetune/Loss": 0.9473856687545776, "Finetune/Loss (Raw)": 1.0816044807434082, "Finetune/Step": 903, "Finetune/Step Time": 5.237460220232606} +{"Finetune/Learning Rate": 1.89310391787282e-05, "Finetune/Loss": 0.948663592338562, "Finetune/Loss (Raw)": 0.9624612331390381, "Finetune/Step": 904, "Finetune/Step Time": 5.244561003521085} +{"Finetune/Learning Rate": 1.892823505301747e-05, "Finetune/Loss": 0.9479888677597046, "Finetune/Loss (Raw)": 1.059928297996521, "Finetune/Step": 905, "Finetune/Step Time": 5.22414056956768} +{"Finetune/Learning Rate": 1.8925427462433027e-05, "Finetune/Loss": 0.9467629194259644, "Finetune/Loss (Raw)": 0.9420847296714783, "Finetune/Step": 906, "Finetune/Step Time": 5.221617538481951} +{"Finetune/Learning Rate": 1.8922616408064436e-05, "Finetune/Loss": 0.9476549625396729, "Finetune/Loss (Raw)": 1.050324559211731, "Finetune/Step": 907, "Finetune/Step Time": 5.208944348618388} +{"Finetune/Learning Rate": 1.8919801891002615e-05, "Finetune/Loss": 0.9526435732841492, "Finetune/Loss (Raw)": 0.9757972955703735, "Finetune/Step": 908, "Finetune/Step Time": 5.17394676245749} +{"Finetune/Learning Rate": 1.891698391233983e-05, "Finetune/Loss": 0.9531195163726807, "Finetune/Loss (Raw)": 0.9987488389015198, "Finetune/Step": 909, "Finetune/Step Time": 5.178312176838517} +{"Finetune/Learning Rate": 1.8914162473169676e-05, "Finetune/Loss": 0.9549182057380676, "Finetune/Loss (Raw)": 0.969098687171936, "Finetune/Step": 910, "Finetune/Step Time": 5.149597931653261} +{"Finetune/Learning Rate": 1.8911337574587098e-05, "Finetune/Loss": 0.9549834728240967, "Finetune/Loss (Raw)": 0.9362438321113586, "Finetune/Step": 911, "Finetune/Step Time": 5.097738452255726} +{"Finetune/Learning Rate": 1.8908509217688387e-05, "Finetune/Loss": 0.9579064249992371, "Finetune/Loss (Raw)": 1.1258947849273682, "Finetune/Step": 912, "Finetune/Step Time": 5.08858297765255} +{"Finetune/Learning Rate": 1.8905677403571177e-05, "Finetune/Loss": 0.9550995230674744, "Finetune/Loss (Raw)": 0.7242985367774963, "Finetune/Step": 913, "Finetune/Step Time": 5.088299628347158} +{"Finetune/Learning Rate": 1.890284213333443e-05, "Finetune/Loss": 0.9549233913421631, "Finetune/Loss (Raw)": 1.1222150325775146, "Finetune/Step": 914, "Finetune/Step Time": 5.0922268237918615} +{"Finetune/Learning Rate": 1.8900003408078468e-05, "Finetune/Loss": 0.9540549516677856, "Finetune/Loss (Raw)": 1.0099633932113647, "Finetune/Step": 915, "Finetune/Step Time": 5.064738005399704} +{"Finetune/Learning Rate": 1.889716122890494e-05, "Finetune/Loss": 0.9533365964889526, "Finetune/Loss (Raw)": 1.0356981754302979, "Finetune/Step": 916, "Finetune/Step Time": 5.071383120492101} +{"Finetune/Learning Rate": 1.889431559691684e-05, "Finetune/Loss": 0.954777181148529, "Finetune/Loss (Raw)": 1.1918779611587524, "Finetune/Step": 917, "Finetune/Step Time": 5.1136809922754765} +{"Finetune/Learning Rate": 1.8891466513218507e-05, "Finetune/Loss": 0.9564089775085449, "Finetune/Loss (Raw)": 0.9733188152313232, "Finetune/Step": 918, "Finetune/Step Time": 5.090039648115635} +{"Finetune/Learning Rate": 1.888861397891561e-05, "Finetune/Loss": 0.9550755023956299, "Finetune/Loss (Raw)": 0.7674677968025208, "Finetune/Step": 919, "Finetune/Step Time": 5.104643449187279} +{"Finetune/Learning Rate": 1.888575799511516e-05, "Finetune/Loss": 0.9550731182098389, "Finetune/Loss (Raw)": 0.847136914730072, "Finetune/Step": 920, "Finetune/Step Time": 5.110479960218072} +{"Finetune/Learning Rate": 1.888289856292552e-05, "Finetune/Loss": 0.9542965292930603, "Finetune/Loss (Raw)": 0.9852777123451233, "Finetune/Step": 921, "Finetune/Step Time": 5.10011070035398} +{"Finetune/Learning Rate": 1.8880035683456365e-05, "Finetune/Loss": 0.9541518688201904, "Finetune/Loss (Raw)": 0.9235610365867615, "Finetune/Step": 922, "Finetune/Step Time": 5.0989777110517025} +{"Finetune/Learning Rate": 1.887716935781873e-05, "Finetune/Loss": 0.9545471668243408, "Finetune/Loss (Raw)": 1.038906455039978, "Finetune/Step": 923, "Finetune/Step Time": 5.091068036854267} +{"Finetune/Learning Rate": 1.887429958712498e-05, "Finetune/Loss": 0.9544413089752197, "Finetune/Loss (Raw)": 0.9392374157905579, "Finetune/Step": 924, "Finetune/Step Time": 5.0999486874789} +{"Finetune/Learning Rate": 1.8871426372488818e-05, "Finetune/Loss": 0.9531607627868652, "Finetune/Loss (Raw)": 0.8161652684211731, "Finetune/Step": 925, "Finetune/Step Time": 5.11992191337049} +{"Finetune/Learning Rate": 1.886854971502528e-05, "Finetune/Loss": 0.9526030421257019, "Finetune/Loss (Raw)": 0.9476215243339539, "Finetune/Step": 926, "Finetune/Step Time": 5.124009344726801} +{"Finetune/Learning Rate": 1.886566961585074e-05, "Finetune/Loss": 0.9536802768707275, "Finetune/Loss (Raw)": 1.0476423501968384, "Finetune/Step": 927, "Finetune/Step Time": 5.116688966751099} +{"Finetune/Learning Rate": 1.886278607608291e-05, "Finetune/Loss": 0.9533106088638306, "Finetune/Loss (Raw)": 1.013746976852417, "Finetune/Step": 928, "Finetune/Step Time": 5.109092071652412} +{"Finetune/Learning Rate": 1.8859899096840836e-05, "Finetune/Loss": 0.952967643737793, "Finetune/Loss (Raw)": 1.0341355800628662, "Finetune/Step": 929, "Finetune/Step Time": 5.084416650235653} +{"Finetune/Learning Rate": 1.8857008679244897e-05, "Finetune/Loss": 0.9522933959960938, "Finetune/Loss (Raw)": 0.9081391096115112, "Finetune/Step": 930, "Finetune/Step Time": 5.08932214602828} +{"Finetune/Learning Rate": 1.885411482441681e-05, "Finetune/Loss": 0.9540646076202393, "Finetune/Loss (Raw)": 1.026763677597046, "Finetune/Step": 931, "Finetune/Step Time": 5.070536518469453} +{"Finetune/Learning Rate": 1.885121753347962e-05, "Finetune/Loss": 0.9546195268630981, "Finetune/Loss (Raw)": 1.06287682056427, "Finetune/Step": 932, "Finetune/Step Time": 5.076524414122105} +{"Finetune/Learning Rate": 1.884831680755771e-05, "Finetune/Loss": 0.9546434879302979, "Finetune/Loss (Raw)": 1.0416548252105713, "Finetune/Step": 933, "Finetune/Step Time": 5.0865778010338545} +{"Finetune/Learning Rate": 1.8845412647776795e-05, "Finetune/Loss": 0.9559822082519531, "Finetune/Loss (Raw)": 1.0171722173690796, "Finetune/Step": 934, "Finetune/Step Time": 5.072755081579089} +{"Finetune/Learning Rate": 1.8842505055263928e-05, "Finetune/Loss": 0.9550108313560486, "Finetune/Loss (Raw)": 0.8681078553199768, "Finetune/Step": 935, "Finetune/Step Time": 5.073606913909316} +{"Finetune/Learning Rate": 1.883959403114748e-05, "Finetune/Loss": 0.9542415738105774, "Finetune/Loss (Raw)": 0.9476789236068726, "Finetune/Step": 936, "Finetune/Step Time": 5.065713353455067} +{"Finetune/Learning Rate": 1.8836679576557165e-05, "Finetune/Loss": 0.9556012153625488, "Finetune/Loss (Raw)": 1.1290969848632812, "Finetune/Step": 937, "Finetune/Step Time": 5.060523223131895} +{"Finetune/Learning Rate": 1.8833761692624034e-05, "Finetune/Loss": 0.9559084177017212, "Finetune/Loss (Raw)": 1.1356191635131836, "Finetune/Step": 938, "Finetune/Step Time": 5.0642211474478245} +{"Finetune/Learning Rate": 1.883084038048045e-05, "Finetune/Loss": 0.9565651416778564, "Finetune/Loss (Raw)": 0.9335960745811462, "Finetune/Step": 939, "Finetune/Step Time": 5.065213609486818} +{"Finetune/Learning Rate": 1.8827915641260127e-05, "Finetune/Loss": 0.9549338817596436, "Finetune/Loss (Raw)": 1.0220134258270264, "Finetune/Step": 940, "Finetune/Step Time": 5.067778050899506} +{"Finetune/Learning Rate": 1.882498747609809e-05, "Finetune/Loss": 0.9548285007476807, "Finetune/Loss (Raw)": 0.9759258031845093, "Finetune/Step": 941, "Finetune/Step Time": 5.067223751917481} +{"Finetune/Learning Rate": 1.8822055886130708e-05, "Finetune/Loss": 0.9546858668327332, "Finetune/Loss (Raw)": 0.9438282251358032, "Finetune/Step": 942, "Finetune/Step Time": 5.073159297928214} +{"Finetune/Learning Rate": 1.8819120872495675e-05, "Finetune/Loss": 0.9544799327850342, "Finetune/Loss (Raw)": 0.9551117420196533, "Finetune/Step": 943, "Finetune/Step Time": 5.072253804653883} +{"Finetune/Learning Rate": 1.8816182436332006e-05, "Finetune/Loss": 0.9538959264755249, "Finetune/Loss (Raw)": 0.9985695481300354, "Finetune/Step": 944, "Finetune/Step Time": 5.06741857714951} +{"Finetune/Learning Rate": 1.881324057878006e-05, "Finetune/Loss": 0.95224928855896, "Finetune/Loss (Raw)": 0.8677968978881836, "Finetune/Step": 945, "Finetune/Step Time": 5.077547010034323} +{"Finetune/Learning Rate": 1.8810295300981504e-05, "Finetune/Loss": 0.9567404985427856, "Finetune/Loss (Raw)": 0.9058798551559448, "Finetune/Step": 946, "Finetune/Step Time": 5.059461126103997} +{"Finetune/Learning Rate": 1.8807346604079352e-05, "Finetune/Loss": 0.9578953981399536, "Finetune/Loss (Raw)": 0.9691529273986816, "Finetune/Step": 947, "Finetune/Step Time": 5.078917799517512} +{"Finetune/Learning Rate": 1.880439448921793e-05, "Finetune/Loss": 0.9551291465759277, "Finetune/Loss (Raw)": 0.38495728373527527, "Finetune/Step": 948, "Finetune/Step Time": 5.103769067674875} +{"Finetune/Learning Rate": 1.88014389575429e-05, "Finetune/Loss": 0.9562404155731201, "Finetune/Loss (Raw)": 1.1396942138671875, "Finetune/Step": 949, "Finetune/Step Time": 5.088908897712827} +{"Finetune/Learning Rate": 1.879848001020124e-05, "Finetune/Loss": 0.9560848474502563, "Finetune/Loss (Raw)": 0.910808265209198, "Finetune/Step": 950, "Finetune/Step Time": 5.086941482499242} +{"Finetune/Learning Rate": 1.8795517648341267e-05, "Finetune/Loss": 0.956779420375824, "Finetune/Loss (Raw)": 1.184687614440918, "Finetune/Step": 951, "Finetune/Step Time": 5.080355478450656} +{"Finetune/Learning Rate": 1.8792551873112614e-05, "Finetune/Loss": 0.9549240469932556, "Finetune/Loss (Raw)": 0.8509113788604736, "Finetune/Step": 952, "Finetune/Step Time": 5.085402620956302} +{"Finetune/Learning Rate": 1.8789582685666237e-05, "Finetune/Loss": 0.9554407596588135, "Finetune/Loss (Raw)": 0.984761118888855, "Finetune/Step": 953, "Finetune/Step Time": 5.077938042581081} +{"Finetune/Learning Rate": 1.878661008715442e-05, "Finetune/Loss": 0.9559048414230347, "Finetune/Loss (Raw)": 0.9159546494483948, "Finetune/Step": 954, "Finetune/Step Time": 5.0748970452696085} +{"Finetune/Learning Rate": 1.8783634078730774e-05, "Finetune/Loss": 0.9558641314506531, "Finetune/Loss (Raw)": 0.998887300491333, "Finetune/Step": 955, "Finetune/Step Time": 5.079675858840346} +{"Finetune/Learning Rate": 1.878065466155023e-05, "Finetune/Loss": 0.9609402418136597, "Finetune/Loss (Raw)": 1.0318286418914795, "Finetune/Step": 956, "Finetune/Step Time": 5.041336419060826} +{"Finetune/Learning Rate": 1.877767183676904e-05, "Finetune/Loss": 0.9596211910247803, "Finetune/Loss (Raw)": 0.9037460684776306, "Finetune/Step": 957, "Finetune/Step Time": 5.03761306963861} +{"Finetune/Learning Rate": 1.8774685605544776e-05, "Finetune/Loss": 0.9594486951828003, "Finetune/Loss (Raw)": 0.9739993810653687, "Finetune/Step": 958, "Finetune/Step Time": 5.046669162809849} +{"Finetune/Learning Rate": 1.8771695969036342e-05, "Finetune/Loss": 0.9585484266281128, "Finetune/Loss (Raw)": 0.9293972253799438, "Finetune/Step": 959, "Finetune/Step Time": 5.017419079318643} +{"Finetune/Learning Rate": 1.8768702928403954e-05, "Finetune/Loss": 0.9575207233428955, "Finetune/Loss (Raw)": 0.9575572609901428, "Finetune/Step": 960, "Finetune/Step Time": 5.013814423233271} +{"Finetune/Learning Rate": 1.8765706484809155e-05, "Finetune/Loss": 0.9610674381256104, "Finetune/Loss (Raw)": 0.8800485134124756, "Finetune/Step": 961, "Finetune/Step Time": 4.984826786443591} +{"Finetune/Learning Rate": 1.8762706639414806e-05, "Finetune/Loss": 0.9619578123092651, "Finetune/Loss (Raw)": 1.1060782670974731, "Finetune/Step": 962, "Finetune/Step Time": 4.981673795729876} +{"Finetune/Learning Rate": 1.8759703393385087e-05, "Finetune/Loss": 0.9613313674926758, "Finetune/Loss (Raw)": 0.9107430577278137, "Finetune/Step": 963, "Finetune/Step Time": 4.974409528076649} +{"Finetune/Learning Rate": 1.87566967478855e-05, "Finetune/Loss": 0.9606913328170776, "Finetune/Loss (Raw)": 0.9314821362495422, "Finetune/Step": 964, "Finetune/Step Time": 4.97389892488718} +{"Finetune/Learning Rate": 1.8753686704082868e-05, "Finetune/Loss": 0.9597026705741882, "Finetune/Loss (Raw)": 0.8078712821006775, "Finetune/Step": 965, "Finetune/Step Time": 4.968533426523209} +{"Finetune/Learning Rate": 1.8750673263145323e-05, "Finetune/Loss": 0.9588093757629395, "Finetune/Loss (Raw)": 0.948052704334259, "Finetune/Step": 966, "Finetune/Step Time": 4.969246404245496} +{"Finetune/Learning Rate": 1.874765642624233e-05, "Finetune/Loss": 0.9594775438308716, "Finetune/Loss (Raw)": 0.996346652507782, "Finetune/Step": 967, "Finetune/Step Time": 4.9798814337700605} +{"Finetune/Learning Rate": 1.874463619454466e-05, "Finetune/Loss": 0.9586391448974609, "Finetune/Loss (Raw)": 1.062132716178894, "Finetune/Step": 968, "Finetune/Step Time": 4.972952283918858} +{"Finetune/Learning Rate": 1.874161256922441e-05, "Finetune/Loss": 0.9589011669158936, "Finetune/Loss (Raw)": 0.9306312799453735, "Finetune/Step": 969, "Finetune/Step Time": 4.97998508438468} +{"Finetune/Learning Rate": 1.8738585551454982e-05, "Finetune/Loss": 0.9560436010360718, "Finetune/Loss (Raw)": 0.7049461603164673, "Finetune/Step": 970, "Finetune/Step Time": 4.998420994728804} +{"Finetune/Learning Rate": 1.873555514241111e-05, "Finetune/Loss": 0.9557470083236694, "Finetune/Loss (Raw)": 1.0078452825546265, "Finetune/Step": 971, "Finetune/Step Time": 5.006123593077064} +{"Finetune/Learning Rate": 1.8732521343268838e-05, "Finetune/Loss": 0.9510177373886108, "Finetune/Loss (Raw)": 0.39286231994628906, "Finetune/Step": 972, "Finetune/Step Time": 5.039846220985055} +{"Finetune/Learning Rate": 1.8729484155205517e-05, "Finetune/Loss": 0.9494980573654175, "Finetune/Loss (Raw)": 0.8866934180259705, "Finetune/Step": 973, "Finetune/Step Time": 5.045469589531422} +{"Finetune/Learning Rate": 1.8726443579399828e-05, "Finetune/Loss": 0.9488990306854248, "Finetune/Loss (Raw)": 0.9673140048980713, "Finetune/Step": 974, "Finetune/Step Time": 5.066830264404416} +{"Finetune/Learning Rate": 1.8723399617031754e-05, "Finetune/Loss": 0.9539560675621033, "Finetune/Loss (Raw)": 1.1124845743179321, "Finetune/Step": 975, "Finetune/Step Time": 5.045959271490574} +{"Finetune/Learning Rate": 1.8720352269282592e-05, "Finetune/Loss": 0.9550143480300903, "Finetune/Loss (Raw)": 0.9877547025680542, "Finetune/Step": 976, "Finetune/Step Time": 5.042966241016984} +{"Finetune/Learning Rate": 1.8717301537334975e-05, "Finetune/Loss": 0.9555701613426208, "Finetune/Loss (Raw)": 1.0751070976257324, "Finetune/Step": 977, "Finetune/Step Time": 5.0485749412328005} +{"Finetune/Learning Rate": 1.8714247422372816e-05, "Finetune/Loss": 0.9560466408729553, "Finetune/Loss (Raw)": 0.9509869813919067, "Finetune/Step": 978, "Finetune/Step Time": 5.033544788137078} +{"Finetune/Learning Rate": 1.8711189925581366e-05, "Finetune/Loss": 0.9549787640571594, "Finetune/Loss (Raw)": 0.8845517635345459, "Finetune/Step": 979, "Finetune/Step Time": 5.034394785761833} +{"Finetune/Learning Rate": 1.8708129048147176e-05, "Finetune/Loss": 0.9572190046310425, "Finetune/Loss (Raw)": 1.1704131364822388, "Finetune/Step": 980, "Finetune/Step Time": 5.032744355499744} +{"Finetune/Learning Rate": 1.8705064791258118e-05, "Finetune/Loss": 0.9574292898178101, "Finetune/Loss (Raw)": 0.9839008450508118, "Finetune/Step": 981, "Finetune/Step Time": 5.033561835065484} +{"Finetune/Learning Rate": 1.870199715610336e-05, "Finetune/Loss": 0.9581183195114136, "Finetune/Loss (Raw)": 1.0300543308258057, "Finetune/Step": 982, "Finetune/Step Time": 5.0210969764739275} +{"Finetune/Learning Rate": 1.8698926143873404e-05, "Finetune/Loss": 0.9599286317825317, "Finetune/Loss (Raw)": 1.0338072776794434, "Finetune/Step": 983, "Finetune/Step Time": 4.984605802223086} +{"Finetune/Learning Rate": 1.8695851755760042e-05, "Finetune/Loss": 0.9598530530929565, "Finetune/Loss (Raw)": 0.9806342720985413, "Finetune/Step": 984, "Finetune/Step Time": 4.995547318831086} +{"Finetune/Learning Rate": 1.8692773992956383e-05, "Finetune/Loss": 0.9598422050476074, "Finetune/Loss (Raw)": 0.9467062950134277, "Finetune/Step": 985, "Finetune/Step Time": 4.994241751730442} +{"Finetune/Learning Rate": 1.8689692856656854e-05, "Finetune/Loss": 0.957470178604126, "Finetune/Loss (Raw)": 0.8861868381500244, "Finetune/Step": 986, "Finetune/Step Time": 4.997804241254926} +{"Finetune/Learning Rate": 1.8686608348057178e-05, "Finetune/Loss": 0.9567421674728394, "Finetune/Loss (Raw)": 1.0089622735977173, "Finetune/Step": 987, "Finetune/Step Time": 4.99492652527988} +{"Finetune/Learning Rate": 1.868352046835439e-05, "Finetune/Loss": 0.9560370445251465, "Finetune/Loss (Raw)": 0.8640492558479309, "Finetune/Step": 988, "Finetune/Step Time": 4.997655024752021} +{"Finetune/Learning Rate": 1.8680429218746842e-05, "Finetune/Loss": 0.958454966545105, "Finetune/Loss (Raw)": 1.0380712747573853, "Finetune/Step": 989, "Finetune/Step Time": 5.000835003331304} +{"Finetune/Learning Rate": 1.867733460043419e-05, "Finetune/Loss": 0.9563561677932739, "Finetune/Loss (Raw)": 0.7509527802467346, "Finetune/Step": 990, "Finetune/Step Time": 5.0065625458955765} +{"Finetune/Learning Rate": 1.8674236614617386e-05, "Finetune/Loss": 0.9580068588256836, "Finetune/Loss (Raw)": 1.051332950592041, "Finetune/Step": 991, "Finetune/Step Time": 4.997303137555718} +{"Finetune/Learning Rate": 1.86711352624987e-05, "Finetune/Loss": 0.9575294256210327, "Finetune/Loss (Raw)": 0.9844077825546265, "Finetune/Step": 992, "Finetune/Step Time": 4.995059283450246} +{"Finetune/Learning Rate": 1.866803054528171e-05, "Finetune/Loss": 0.9577836990356445, "Finetune/Loss (Raw)": 0.9920929074287415, "Finetune/Step": 993, "Finetune/Step Time": 5.005917267873883} +{"Finetune/Learning Rate": 1.8664922464171295e-05, "Finetune/Loss": 0.9550697803497314, "Finetune/Loss (Raw)": 0.7761582732200623, "Finetune/Step": 994, "Finetune/Step Time": 5.014618648216128} +{"Finetune/Learning Rate": 1.866181102037364e-05, "Finetune/Loss": 0.9541106820106506, "Finetune/Loss (Raw)": 0.9411728978157043, "Finetune/Step": 995, "Finetune/Step Time": 5.013113360852003} +{"Finetune/Learning Rate": 1.8658696215096235e-05, "Finetune/Loss": 0.9542489051818848, "Finetune/Loss (Raw)": 0.9624557495117188, "Finetune/Step": 996, "Finetune/Step Time": 5.017141986638308} +{"Finetune/Learning Rate": 1.8655578049547878e-05, "Finetune/Loss": 0.9530283212661743, "Finetune/Loss (Raw)": 0.9362103939056396, "Finetune/Step": 997, "Finetune/Step Time": 4.996572177857161} +{"Finetune/Learning Rate": 1.8652456524938665e-05, "Finetune/Loss": 0.9534947276115417, "Finetune/Loss (Raw)": 0.962356686592102, "Finetune/Step": 998, "Finetune/Step Time": 5.009665219113231} +{"Finetune/Learning Rate": 1.864933164248e-05, "Finetune/Loss": 0.9556015729904175, "Finetune/Loss (Raw)": 1.1204702854156494, "Finetune/Step": 999, "Finetune/Step Time": 5.003120129927993} +{"Finetune/Learning Rate": 1.8646203403384583e-05, "Finetune/Loss": 0.9567354917526245, "Finetune/Loss (Raw)": 1.144270658493042, "Finetune/Step": 1000, "Finetune/Step Time": 4.9969427194446325} +{"Finetune/Learning Rate": 1.8643071808866434e-05, "Finetune/Loss": 0.9510704874992371, "Finetune/Loss (Raw)": 0.3240794837474823, "Finetune/Step": 1001, "Finetune/Step Time": 5.035156542435288} +{"Finetune/Learning Rate": 1.8639936860140856e-05, "Finetune/Loss": 0.9518787264823914, "Finetune/Loss (Raw)": 0.9853082895278931, "Finetune/Step": 1002, "Finetune/Step Time": 5.035678027197719} +{"Finetune/Learning Rate": 1.8636798558424464e-05, "Finetune/Loss": 0.952733039855957, "Finetune/Loss (Raw)": 1.0577659606933594, "Finetune/Step": 1003, "Finetune/Step Time": 5.039015980437398} +{"Finetune/Learning Rate": 1.8633656904935168e-05, "Finetune/Loss": 0.9537377953529358, "Finetune/Loss (Raw)": 1.0666695833206177, "Finetune/Step": 1004, "Finetune/Step Time": 5.038731846958399} +{"Finetune/Learning Rate": 1.863051190089219e-05, "Finetune/Loss": 0.95365309715271, "Finetune/Loss (Raw)": 1.0212756395339966, "Finetune/Step": 1005, "Finetune/Step Time": 5.030908714979887} +{"Finetune/Learning Rate": 1.8627363547516032e-05, "Finetune/Loss": 0.9527336359024048, "Finetune/Loss (Raw)": 0.8781131505966187, "Finetune/Step": 1006, "Finetune/Step Time": 5.047227883711457} +{"Finetune/Learning Rate": 1.8624211846028522e-05, "Finetune/Loss": 0.9530828595161438, "Finetune/Loss (Raw)": 0.9357638359069824, "Finetune/Step": 1007, "Finetune/Step Time": 5.02769866026938} +{"Finetune/Learning Rate": 1.862105679765277e-05, "Finetune/Loss": 0.9528591632843018, "Finetune/Loss (Raw)": 0.9858848452568054, "Finetune/Step": 1008, "Finetune/Step Time": 5.027047218754888} +{"Finetune/Learning Rate": 1.8617898403613183e-05, "Finetune/Loss": 0.9492627382278442, "Finetune/Loss (Raw)": 0.42796966433525085, "Finetune/Step": 1009, "Finetune/Step Time": 5.048322049900889} +{"Finetune/Learning Rate": 1.8614736665135476e-05, "Finetune/Loss": 0.9479902982711792, "Finetune/Loss (Raw)": 0.9049879908561707, "Finetune/Step": 1010, "Finetune/Step Time": 5.050613317638636} +{"Finetune/Learning Rate": 1.861157158344666e-05, "Finetune/Loss": 0.9469336271286011, "Finetune/Loss (Raw)": 0.9239472150802612, "Finetune/Step": 1011, "Finetune/Step Time": 5.049905201420188} +{"Finetune/Learning Rate": 1.8608403159775037e-05, "Finetune/Loss": 0.9464849233627319, "Finetune/Loss (Raw)": 1.0124210119247437, "Finetune/Step": 1012, "Finetune/Step Time": 5.054847223684192} +{"Finetune/Learning Rate": 1.8605231395350213e-05, "Finetune/Loss": 0.9518632888793945, "Finetune/Loss (Raw)": 0.9191474914550781, "Finetune/Step": 1013, "Finetune/Step Time": 5.0194671880453825} +{"Finetune/Learning Rate": 1.860205629140309e-05, "Finetune/Loss": 0.9521065354347229, "Finetune/Loss (Raw)": 1.0478936433792114, "Finetune/Step": 1014, "Finetune/Step Time": 5.014444401487708} +{"Finetune/Learning Rate": 1.8598877849165857e-05, "Finetune/Loss": 0.951392412185669, "Finetune/Loss (Raw)": 0.31537237763404846, "Finetune/Step": 1015, "Finetune/Step Time": 5.014222268015146} +{"Finetune/Learning Rate": 1.8595696069872013e-05, "Finetune/Loss": 0.9508029818534851, "Finetune/Loss (Raw)": 1.0104875564575195, "Finetune/Step": 1016, "Finetune/Step Time": 5.012809790670872} +{"Finetune/Learning Rate": 1.859251095475634e-05, "Finetune/Loss": 0.9510814547538757, "Finetune/Loss (Raw)": 1.0826451778411865, "Finetune/Step": 1017, "Finetune/Step Time": 5.004129473119974} +{"Finetune/Learning Rate": 1.858932250505492e-05, "Finetune/Loss": 0.9513028860092163, "Finetune/Loss (Raw)": 1.0034762620925903, "Finetune/Step": 1018, "Finetune/Step Time": 5.048420237377286} +{"Finetune/Learning Rate": 1.8586130722005128e-05, "Finetune/Loss": 0.950760543346405, "Finetune/Loss (Raw)": 0.9228555560112, "Finetune/Step": 1019, "Finetune/Step Time": 5.046261396259069} +{"Finetune/Learning Rate": 1.8582935606845636e-05, "Finetune/Loss": 0.9515541195869446, "Finetune/Loss (Raw)": 0.994053065776825, "Finetune/Step": 1020, "Finetune/Step Time": 5.0418008007109165} +{"Finetune/Learning Rate": 1.8579737160816397e-05, "Finetune/Loss": 0.9495282173156738, "Finetune/Loss (Raw)": 0.8682062029838562, "Finetune/Step": 1021, "Finetune/Step Time": 5.053513314574957} +{"Finetune/Learning Rate": 1.8576535385158675e-05, "Finetune/Loss": 0.9499001502990723, "Finetune/Loss (Raw)": 1.0204336643218994, "Finetune/Step": 1022, "Finetune/Step Time": 5.05413712374866} +{"Finetune/Learning Rate": 1.8573330281115008e-05, "Finetune/Loss": 0.9488651752471924, "Finetune/Loss (Raw)": 0.846079409122467, "Finetune/Step": 1023, "Finetune/Step Time": 5.052403239533305} +{"Finetune/Learning Rate": 1.8570121849929245e-05, "Finetune/Loss": 0.9478039741516113, "Finetune/Loss (Raw)": 0.9337298274040222, "Finetune/Step": 1024, "Finetune/Step Time": 5.054107239469886} +{"Finetune/Learning Rate": 1.8566910092846508e-05, "Finetune/Loss": 0.9480725526809692, "Finetune/Loss (Raw)": 0.887310802936554, "Finetune/Step": 1025, "Finetune/Step Time": 5.038632288575172} +{"Finetune/Learning Rate": 1.8563695011113217e-05, "Finetune/Loss": 0.9532349705696106, "Finetune/Loss (Raw)": 1.022627592086792, "Finetune/Step": 1026, "Finetune/Step Time": 5.00709075294435} +{"Finetune/Learning Rate": 1.8560476605977085e-05, "Finetune/Loss": 0.9497071504592896, "Finetune/Loss (Raw)": 0.374070405960083, "Finetune/Step": 1027, "Finetune/Step Time": 4.960154067724943} +{"Finetune/Learning Rate": 1.8557254878687114e-05, "Finetune/Loss": 0.9492115378379822, "Finetune/Loss (Raw)": 1.051932692527771, "Finetune/Step": 1028, "Finetune/Step Time": 4.963991126045585} +{"Finetune/Learning Rate": 1.8554029830493587e-05, "Finetune/Loss": 0.9474397301673889, "Finetune/Loss (Raw)": 0.8802321553230286, "Finetune/Step": 1029, "Finetune/Step Time": 4.97162895463407} +{"Finetune/Learning Rate": 1.855080146264809e-05, "Finetune/Loss": 0.9472082853317261, "Finetune/Loss (Raw)": 1.0326879024505615, "Finetune/Step": 1030, "Finetune/Step Time": 4.970527796074748} +{"Finetune/Learning Rate": 1.854756977640348e-05, "Finetune/Loss": 0.9464945197105408, "Finetune/Loss (Raw)": 0.9902418255805969, "Finetune/Step": 1031, "Finetune/Step Time": 4.950707979500294} +{"Finetune/Learning Rate": 1.8544334773013924e-05, "Finetune/Loss": 0.9417213201522827, "Finetune/Loss (Raw)": 0.3514880836009979, "Finetune/Step": 1032, "Finetune/Step Time": 4.980709683150053} +{"Finetune/Learning Rate": 1.8541096453734852e-05, "Finetune/Loss": 0.9413118958473206, "Finetune/Loss (Raw)": 1.0075294971466064, "Finetune/Step": 1033, "Finetune/Step Time": 5.012217925861478} +{"Finetune/Learning Rate": 1.8537854819822998e-05, "Finetune/Loss": 0.9405440092086792, "Finetune/Loss (Raw)": 0.8437972068786621, "Finetune/Step": 1034, "Finetune/Step Time": 5.025470653548837} +{"Finetune/Learning Rate": 1.853460987253638e-05, "Finetune/Loss": 0.9402028322219849, "Finetune/Loss (Raw)": 1.0066502094268799, "Finetune/Step": 1035, "Finetune/Step Time": 5.0375798884779215} +{"Finetune/Learning Rate": 1.8531361613134286e-05, "Finetune/Loss": 0.9401286244392395, "Finetune/Loss (Raw)": 0.9662941694259644, "Finetune/Step": 1036, "Finetune/Step Time": 5.047421092167497} +{"Finetune/Learning Rate": 1.8528110042877316e-05, "Finetune/Loss": 0.9391903281211853, "Finetune/Loss (Raw)": 0.8786503076553345, "Finetune/Step": 1037, "Finetune/Step Time": 5.055565234273672} +{"Finetune/Learning Rate": 1.8524855163027337e-05, "Finetune/Loss": 0.9407048225402832, "Finetune/Loss (Raw)": 1.1629538536071777, "Finetune/Step": 1038, "Finetune/Step Time": 5.048616956919432} +{"Finetune/Learning Rate": 1.85215969748475e-05, "Finetune/Loss": 0.9413008689880371, "Finetune/Loss (Raw)": 1.0125333070755005, "Finetune/Step": 1039, "Finetune/Step Time": 5.04793075658381} +{"Finetune/Learning Rate": 1.851833547960225e-05, "Finetune/Loss": 0.9398089051246643, "Finetune/Loss (Raw)": 0.9349262118339539, "Finetune/Step": 1040, "Finetune/Step Time": 5.044068373739719} +{"Finetune/Learning Rate": 1.8515070678557304e-05, "Finetune/Loss": 0.942615270614624, "Finetune/Loss (Raw)": 1.0835150480270386, "Finetune/Step": 1041, "Finetune/Step Time": 5.020591842010617} +{"Finetune/Learning Rate": 1.8511802572979672e-05, "Finetune/Loss": 0.942301332950592, "Finetune/Loss (Raw)": 1.0820331573486328, "Finetune/Step": 1042, "Finetune/Step Time": 5.021584929898381} +{"Finetune/Learning Rate": 1.8508531164137644e-05, "Finetune/Loss": 0.9374058246612549, "Finetune/Loss (Raw)": 0.38333991169929504, "Finetune/Step": 1043, "Finetune/Step Time": 5.06122681684792} +{"Finetune/Learning Rate": 1.8505256453300778e-05, "Finetune/Loss": 0.9360157251358032, "Finetune/Loss (Raw)": 0.8577632904052734, "Finetune/Step": 1044, "Finetune/Step Time": 5.061470614746213} +{"Finetune/Learning Rate": 1.850197844173994e-05, "Finetune/Loss": 0.9349262714385986, "Finetune/Loss (Raw)": 1.052428126335144, "Finetune/Step": 1045, "Finetune/Step Time": 5.037674767896533} +{"Finetune/Learning Rate": 1.8498697130727255e-05, "Finetune/Loss": 0.9347586631774902, "Finetune/Loss (Raw)": 0.9518639445304871, "Finetune/Step": 1046, "Finetune/Step Time": 5.040860895067453} +{"Finetune/Learning Rate": 1.8495412521536135e-05, "Finetune/Loss": 0.9366092681884766, "Finetune/Loss (Raw)": 1.0043433904647827, "Finetune/Step": 1047, "Finetune/Step Time": 5.032398968935013} +{"Finetune/Learning Rate": 1.8492124615441276e-05, "Finetune/Loss": 0.9391337633132935, "Finetune/Loss (Raw)": 1.170276403427124, "Finetune/Step": 1048, "Finetune/Step Time": 5.023957632482052} +{"Finetune/Learning Rate": 1.8488833413718646e-05, "Finetune/Loss": 0.9382839202880859, "Finetune/Loss (Raw)": 0.8764950037002563, "Finetune/Step": 1049, "Finetune/Step Time": 5.021474581211805} +{"Finetune/Learning Rate": 1.84855389176455e-05, "Finetune/Loss": 0.9374372363090515, "Finetune/Loss (Raw)": 0.8151907324790955, "Finetune/Step": 1050, "Finetune/Step Time": 5.033605992794037} +{"Finetune/Learning Rate": 1.8482241128500366e-05, "Finetune/Loss": 0.9367140531539917, "Finetune/Loss (Raw)": 0.9463287591934204, "Finetune/Step": 1051, "Finetune/Step Time": 5.037437427788973} +{"Finetune/Learning Rate": 1.847894004756305e-05, "Finetune/Loss": 0.938382089138031, "Finetune/Loss (Raw)": 1.1527458429336548, "Finetune/Step": 1052, "Finetune/Step Time": 5.035663353279233} +{"Finetune/Learning Rate": 1.8475635676114643e-05, "Finetune/Loss": 0.9396601915359497, "Finetune/Loss (Raw)": 0.9797656536102295, "Finetune/Step": 1053, "Finetune/Step Time": 5.022950084879994} +{"Finetune/Learning Rate": 1.8472328015437503e-05, "Finetune/Loss": 0.9409652352333069, "Finetune/Loss (Raw)": 1.1146718263626099, "Finetune/Step": 1054, "Finetune/Step Time": 5.018277792260051} +{"Finetune/Learning Rate": 1.8469017066815265e-05, "Finetune/Loss": 0.939839243888855, "Finetune/Loss (Raw)": 0.9035144448280334, "Finetune/Step": 1055, "Finetune/Step Time": 5.019204227253795} +{"Finetune/Learning Rate": 1.8465702831532848e-05, "Finetune/Loss": 0.9404575824737549, "Finetune/Loss (Raw)": 1.0928939580917358, "Finetune/Step": 1056, "Finetune/Step Time": 5.020441796630621} +{"Finetune/Learning Rate": 1.8462385310876444e-05, "Finetune/Loss": 0.9395742416381836, "Finetune/Loss (Raw)": 0.9210687875747681, "Finetune/Step": 1057, "Finetune/Step Time": 5.01494050770998} +{"Finetune/Learning Rate": 1.845906450613351e-05, "Finetune/Loss": 0.9397395849227905, "Finetune/Loss (Raw)": 0.9293040633201599, "Finetune/Step": 1058, "Finetune/Step Time": 5.007295411080122} +{"Finetune/Learning Rate": 1.8455740418592793e-05, "Finetune/Loss": 0.9390920400619507, "Finetune/Loss (Raw)": 0.9438750743865967, "Finetune/Step": 1059, "Finetune/Step Time": 5.0259078443050385} +{"Finetune/Learning Rate": 1.8452413049544304e-05, "Finetune/Loss": 0.93844074010849, "Finetune/Loss (Raw)": 0.9795054793357849, "Finetune/Step": 1060, "Finetune/Step Time": 5.023948762565851} +{"Finetune/Learning Rate": 1.844908240027933e-05, "Finetune/Loss": 0.9369488954544067, "Finetune/Loss (Raw)": 0.8507013916969299, "Finetune/Step": 1061, "Finetune/Step Time": 5.004603277891874} +{"Finetune/Learning Rate": 1.8445748472090432e-05, "Finetune/Loss": 0.9368047714233398, "Finetune/Loss (Raw)": 0.9987238645553589, "Finetune/Step": 1062, "Finetune/Step Time": 4.999887244775891} +{"Finetune/Learning Rate": 1.844241126627144e-05, "Finetune/Loss": 0.9377071857452393, "Finetune/Loss (Raw)": 0.9836238026618958, "Finetune/Step": 1063, "Finetune/Step Time": 5.007769845426083} +{"Finetune/Learning Rate": 1.843907078411746e-05, "Finetune/Loss": 0.9395776987075806, "Finetune/Loss (Raw)": 1.18709135055542, "Finetune/Step": 1064, "Finetune/Step Time": 4.994583236053586} +{"Finetune/Learning Rate": 1.8435727026924864e-05, "Finetune/Loss": 0.9396262168884277, "Finetune/Loss (Raw)": 1.1353161334991455, "Finetune/Step": 1065, "Finetune/Step Time": 4.995927887037396} +{"Finetune/Learning Rate": 1.8432379995991307e-05, "Finetune/Loss": 0.9377868175506592, "Finetune/Loss (Raw)": 0.900174081325531, "Finetune/Step": 1066, "Finetune/Step Time": 4.989899026229978} +{"Finetune/Learning Rate": 1.8429029692615702e-05, "Finetune/Loss": 0.9391576051712036, "Finetune/Loss (Raw)": 1.1090505123138428, "Finetune/Step": 1067, "Finetune/Step Time": 4.9969888012856245} +{"Finetune/Learning Rate": 1.8425676118098236e-05, "Finetune/Loss": 0.9382666349411011, "Finetune/Loss (Raw)": 0.9079727530479431, "Finetune/Step": 1068, "Finetune/Step Time": 5.001589143648744} +{"Finetune/Learning Rate": 1.8422319273740364e-05, "Finetune/Loss": 0.9382318258285522, "Finetune/Loss (Raw)": 0.9714694619178772, "Finetune/Step": 1069, "Finetune/Step Time": 5.027947995811701} +{"Finetune/Learning Rate": 1.8418959160844813e-05, "Finetune/Loss": 0.9389286637306213, "Finetune/Loss (Raw)": 1.0330333709716797, "Finetune/Step": 1070, "Finetune/Step Time": 5.017427092418075} +{"Finetune/Learning Rate": 1.8415595780715583e-05, "Finetune/Loss": 0.9396074414253235, "Finetune/Loss (Raw)": 1.0419939756393433, "Finetune/Step": 1071, "Finetune/Step Time": 5.01515381410718} +{"Finetune/Learning Rate": 1.8412229134657926e-05, "Finetune/Loss": 0.9399763941764832, "Finetune/Loss (Raw)": 1.0457936525344849, "Finetune/Step": 1072, "Finetune/Step Time": 5.024643629789352} +{"Finetune/Learning Rate": 1.8408859223978378e-05, "Finetune/Loss": 0.9406788349151611, "Finetune/Loss (Raw)": 0.9577072858810425, "Finetune/Step": 1073, "Finetune/Step Time": 5.007378092035651} +{"Finetune/Learning Rate": 1.840548604998474e-05, "Finetune/Loss": 0.9418340921401978, "Finetune/Loss (Raw)": 1.053755760192871, "Finetune/Step": 1074, "Finetune/Step Time": 4.99259152635932} +{"Finetune/Learning Rate": 1.840210961398606e-05, "Finetune/Loss": 0.9424645900726318, "Finetune/Loss (Raw)": 1.049850344657898, "Finetune/Step": 1075, "Finetune/Step Time": 4.987093476578593} +{"Finetune/Learning Rate": 1.8398729917292683e-05, "Finetune/Loss": 0.9478791952133179, "Finetune/Loss (Raw)": 1.0780284404754639, "Finetune/Step": 1076, "Finetune/Step Time": 4.952626401558518} +{"Finetune/Learning Rate": 1.8395346961216193e-05, "Finetune/Loss": 0.9475394487380981, "Finetune/Loss (Raw)": 1.096206545829773, "Finetune/Step": 1077, "Finetune/Step Time": 4.956576654687524} +{"Finetune/Learning Rate": 1.8391960747069453e-05, "Finetune/Loss": 0.94874107837677, "Finetune/Loss (Raw)": 1.0646252632141113, "Finetune/Step": 1078, "Finetune/Step Time": 4.958977876231074} +{"Finetune/Learning Rate": 1.8388571276166588e-05, "Finetune/Loss": 0.946252167224884, "Finetune/Loss (Raw)": 0.8661049604415894, "Finetune/Step": 1079, "Finetune/Step Time": 4.963191607967019} +{"Finetune/Learning Rate": 1.8385178549822982e-05, "Finetune/Loss": 0.9472289085388184, "Finetune/Loss (Raw)": 0.9759236574172974, "Finetune/Step": 1080, "Finetune/Step Time": 4.96289349347353} +{"Finetune/Learning Rate": 1.8381782569355288e-05, "Finetune/Loss": 0.9479764699935913, "Finetune/Loss (Raw)": 1.0804561376571655, "Finetune/Step": 1081, "Finetune/Step Time": 4.966416716575623} +{"Finetune/Learning Rate": 1.837838333608142e-05, "Finetune/Loss": 0.9474425315856934, "Finetune/Loss (Raw)": 0.847612202167511, "Finetune/Step": 1082, "Finetune/Step Time": 4.9871345441788435} +{"Finetune/Learning Rate": 1.8374980851320555e-05, "Finetune/Loss": 0.9476240873336792, "Finetune/Loss (Raw)": 1.0221171379089355, "Finetune/Step": 1083, "Finetune/Step Time": 4.980379989370704} +{"Finetune/Learning Rate": 1.8371575116393125e-05, "Finetune/Loss": 0.9478915333747864, "Finetune/Loss (Raw)": 1.0660685300827026, "Finetune/Step": 1084, "Finetune/Step Time": 4.995490398257971} +{"Finetune/Learning Rate": 1.8368166132620838e-05, "Finetune/Loss": 0.9488035440444946, "Finetune/Loss (Raw)": 1.020485758781433, "Finetune/Step": 1085, "Finetune/Step Time": 4.985576532781124} +{"Finetune/Learning Rate": 1.8364753901326647e-05, "Finetune/Loss": 0.9492961168289185, "Finetune/Loss (Raw)": 1.0370513200759888, "Finetune/Step": 1086, "Finetune/Step Time": 4.972021261230111} +{"Finetune/Learning Rate": 1.8361338423834776e-05, "Finetune/Loss": 0.9500105381011963, "Finetune/Loss (Raw)": 1.0208452939987183, "Finetune/Step": 1087, "Finetune/Step Time": 4.974322082474828} +{"Finetune/Learning Rate": 1.8357919701470702e-05, "Finetune/Loss": 0.9491671919822693, "Finetune/Loss (Raw)": 0.8496060371398926, "Finetune/Step": 1088, "Finetune/Step Time": 4.98247011192143} +{"Finetune/Learning Rate": 1.835449773556117e-05, "Finetune/Loss": 0.9501668214797974, "Finetune/Loss (Raw)": 1.0079994201660156, "Finetune/Step": 1089, "Finetune/Step Time": 4.972729869186878} +{"Finetune/Learning Rate": 1.8351072527434173e-05, "Finetune/Loss": 0.9501591920852661, "Finetune/Loss (Raw)": 1.1051065921783447, "Finetune/Step": 1090, "Finetune/Step Time": 4.9829095639288425} +{"Finetune/Learning Rate": 1.8347644078418968e-05, "Finetune/Loss": 0.9503644108772278, "Finetune/Loss (Raw)": 0.9370026588439941, "Finetune/Step": 1091, "Finetune/Step Time": 4.987344712018967} +{"Finetune/Learning Rate": 1.834421238984607e-05, "Finetune/Loss": 0.9511725902557373, "Finetune/Loss (Raw)": 1.034934639930725, "Finetune/Step": 1092, "Finetune/Step Time": 4.986049162223935} +{"Finetune/Learning Rate": 1.8340777463047253e-05, "Finetune/Loss": 0.951772928237915, "Finetune/Loss (Raw)": 0.8847123384475708, "Finetune/Step": 1093, "Finetune/Step Time": 4.983270883560181} +{"Finetune/Learning Rate": 1.833733929935554e-05, "Finetune/Loss": 0.9520843625068665, "Finetune/Loss (Raw)": 0.9879157543182373, "Finetune/Step": 1094, "Finetune/Step Time": 4.983217857778072} +{"Finetune/Learning Rate": 1.833389790010522e-05, "Finetune/Loss": 0.9524380564689636, "Finetune/Loss (Raw)": 1.0416209697723389, "Finetune/Step": 1095, "Finetune/Step Time": 4.961039990186691} +{"Finetune/Learning Rate": 1.8330453266631828e-05, "Finetune/Loss": 0.9524766206741333, "Finetune/Loss (Raw)": 1.0670732259750366, "Finetune/Step": 1096, "Finetune/Step Time": 4.9567544888705015} +{"Finetune/Learning Rate": 1.8327005400272165e-05, "Finetune/Loss": 0.9529200196266174, "Finetune/Loss (Raw)": 0.9873785376548767, "Finetune/Step": 1097, "Finetune/Step Time": 4.957893712446094} +{"Finetune/Learning Rate": 1.8323554302364273e-05, "Finetune/Loss": 0.9551109075546265, "Finetune/Loss (Raw)": 0.9853775501251221, "Finetune/Step": 1098, "Finetune/Step Time": 4.959762040525675} +{"Finetune/Learning Rate": 1.8320099974247462e-05, "Finetune/Loss": 0.955081582069397, "Finetune/Loss (Raw)": 1.0040960311889648, "Finetune/Step": 1099, "Finetune/Step Time": 4.938192378729582} +{"Finetune/Learning Rate": 1.831664241726229e-05, "Finetune/Loss": 0.959548830986023, "Finetune/Loss (Raw)": 0.9646664261817932, "Finetune/Step": 1100, "Finetune/Step Time": 4.907720470800996} +{"Finetune/Learning Rate": 1.831318163275056e-05, "Finetune/Loss": 0.9611067771911621, "Finetune/Loss (Raw)": 1.0861058235168457, "Finetune/Step": 1101, "Finetune/Step Time": 4.907487638294697} +{"Finetune/Learning Rate": 1.8309717622055343e-05, "Finetune/Loss": 0.9598397612571716, "Finetune/Loss (Raw)": 0.8051438331604004, "Finetune/Step": 1102, "Finetune/Step Time": 4.888676896691322} +{"Finetune/Learning Rate": 1.830625038652095e-05, "Finetune/Loss": 0.9599822759628296, "Finetune/Loss (Raw)": 1.1307272911071777, "Finetune/Step": 1103, "Finetune/Step Time": 4.881007166579366} +{"Finetune/Learning Rate": 1.8302779927492945e-05, "Finetune/Loss": 0.9606201648712158, "Finetune/Loss (Raw)": 1.069405198097229, "Finetune/Step": 1104, "Finetune/Step Time": 4.884516824036837} +{"Finetune/Learning Rate": 1.829930624631815e-05, "Finetune/Loss": 0.9609900712966919, "Finetune/Loss (Raw)": 1.122450351715088, "Finetune/Step": 1105, "Finetune/Step Time": 4.889481667429209} +{"Finetune/Learning Rate": 1.829582934434463e-05, "Finetune/Loss": 0.9613643884658813, "Finetune/Loss (Raw)": 0.9988966584205627, "Finetune/Step": 1106, "Finetune/Step Time": 4.907173214480281} +{"Finetune/Learning Rate": 1.829234922292171e-05, "Finetune/Loss": 0.9624828696250916, "Finetune/Loss (Raw)": 1.0277208089828491, "Finetune/Step": 1107, "Finetune/Step Time": 4.90191438421607} +{"Finetune/Learning Rate": 1.8288865883399946e-05, "Finetune/Loss": 0.9618881940841675, "Finetune/Loss (Raw)": 1.0942927598953247, "Finetune/Step": 1108, "Finetune/Step Time": 4.902278579771519} +{"Finetune/Learning Rate": 1.8285379327131157e-05, "Finetune/Loss": 0.9606283903121948, "Finetune/Loss (Raw)": 0.8226451277732849, "Finetune/Step": 1109, "Finetune/Step Time": 4.919298628345132} +{"Finetune/Learning Rate": 1.828188955546841e-05, "Finetune/Loss": 0.9603098034858704, "Finetune/Loss (Raw)": 0.9892779588699341, "Finetune/Step": 1110, "Finetune/Step Time": 4.918168572708964} +{"Finetune/Learning Rate": 1.827839656976602e-05, "Finetune/Loss": 0.9611183404922485, "Finetune/Loss (Raw)": 1.1372958421707153, "Finetune/Step": 1111, "Finetune/Step Time": 4.913664674386382} +{"Finetune/Learning Rate": 1.8274900371379543e-05, "Finetune/Loss": 0.9607077836990356, "Finetune/Loss (Raw)": 0.9280872941017151, "Finetune/Step": 1112, "Finetune/Step Time": 4.90165095962584} +{"Finetune/Learning Rate": 1.8271400961665784e-05, "Finetune/Loss": 0.9607195854187012, "Finetune/Loss (Raw)": 0.9482137560844421, "Finetune/Step": 1113, "Finetune/Step Time": 4.898030487820506} +{"Finetune/Learning Rate": 1.8267898341982796e-05, "Finetune/Loss": 0.9618855714797974, "Finetune/Loss (Raw)": 1.0354334115982056, "Finetune/Step": 1114, "Finetune/Step Time": 4.9215536043047905} +{"Finetune/Learning Rate": 1.8264392513689882e-05, "Finetune/Loss": 0.961984395980835, "Finetune/Loss (Raw)": 1.0216171741485596, "Finetune/Step": 1115, "Finetune/Step Time": 4.918461561203003} +{"Finetune/Learning Rate": 1.8260883478147578e-05, "Finetune/Loss": 0.9638070464134216, "Finetune/Loss (Raw)": 1.0973477363586426, "Finetune/Step": 1116, "Finetune/Step Time": 4.912882516160607} +{"Finetune/Learning Rate": 1.8257371236717682e-05, "Finetune/Loss": 0.9626635313034058, "Finetune/Loss (Raw)": 0.8916972875595093, "Finetune/Step": 1117, "Finetune/Step Time": 4.912209203466773} +{"Finetune/Learning Rate": 1.8253855790763215e-05, "Finetune/Loss": 0.9653275012969971, "Finetune/Loss (Raw)": 1.0919502973556519, "Finetune/Step": 1118, "Finetune/Step Time": 4.898987211287022} +{"Finetune/Learning Rate": 1.8250337141648464e-05, "Finetune/Loss": 0.9628555774688721, "Finetune/Loss (Raw)": 0.7349217534065247, "Finetune/Step": 1119, "Finetune/Step Time": 4.908956604078412} +{"Finetune/Learning Rate": 1.8246815290738937e-05, "Finetune/Loss": 0.9628692865371704, "Finetune/Loss (Raw)": 0.986161470413208, "Finetune/Step": 1120, "Finetune/Step Time": 4.911834044381976} +{"Finetune/Learning Rate": 1.8243290239401402e-05, "Finetune/Loss": 0.964177131652832, "Finetune/Loss (Raw)": 1.1594936847686768, "Finetune/Step": 1121, "Finetune/Step Time": 4.8804184682667255} +{"Finetune/Learning Rate": 1.8239761989003866e-05, "Finetune/Loss": 0.9662918448448181, "Finetune/Loss (Raw)": 1.0468451976776123, "Finetune/Step": 1122, "Finetune/Step Time": 4.886715073138475} +{"Finetune/Learning Rate": 1.8236230540915567e-05, "Finetune/Loss": 0.9662879705429077, "Finetune/Loss (Raw)": 0.9406751990318298, "Finetune/Step": 1123, "Finetune/Step Time": 4.904965162277222} +{"Finetune/Learning Rate": 1.8232695896506995e-05, "Finetune/Loss": 0.9683732986450195, "Finetune/Loss (Raw)": 1.2293853759765625, "Finetune/Step": 1124, "Finetune/Step Time": 4.901122869923711} +{"Finetune/Learning Rate": 1.8229158057149875e-05, "Finetune/Loss": 0.9684540033340454, "Finetune/Loss (Raw)": 0.9465339183807373, "Finetune/Step": 1125, "Finetune/Step Time": 4.897626098245382} +{"Finetune/Learning Rate": 1.8225617024217174e-05, "Finetune/Loss": 0.9686188697814941, "Finetune/Loss (Raw)": 0.9834564924240112, "Finetune/Step": 1126, "Finetune/Step Time": 4.879309648647904} +{"Finetune/Learning Rate": 1.8222072799083096e-05, "Finetune/Loss": 0.9678822755813599, "Finetune/Loss (Raw)": 1.026184320449829, "Finetune/Step": 1127, "Finetune/Step Time": 4.878944259136915} +{"Finetune/Learning Rate": 1.821852538312309e-05, "Finetune/Loss": 0.9665787816047668, "Finetune/Loss (Raw)": 0.9774296879768372, "Finetune/Step": 1128, "Finetune/Step Time": 4.891184510663152} +{"Finetune/Learning Rate": 1.8214974777713834e-05, "Finetune/Loss": 0.9727128744125366, "Finetune/Loss (Raw)": 1.1092406511306763, "Finetune/Step": 1129, "Finetune/Step Time": 4.850257556885481} +{"Finetune/Learning Rate": 1.821142098423325e-05, "Finetune/Loss": 0.9712682962417603, "Finetune/Loss (Raw)": 0.8004099726676941, "Finetune/Step": 1130, "Finetune/Step Time": 4.858666975051165} +{"Finetune/Learning Rate": 1.8207864004060497e-05, "Finetune/Loss": 0.9714629054069519, "Finetune/Loss (Raw)": 1.082669973373413, "Finetune/Step": 1131, "Finetune/Step Time": 4.840902455151081} +{"Finetune/Learning Rate": 1.820430383857597e-05, "Finetune/Loss": 0.9711581468582153, "Finetune/Loss (Raw)": 1.0276552438735962, "Finetune/Step": 1132, "Finetune/Step Time": 4.842716913670301} +{"Finetune/Learning Rate": 1.8200740489161296e-05, "Finetune/Loss": 0.970856785774231, "Finetune/Loss (Raw)": 0.9826998114585876, "Finetune/Step": 1133, "Finetune/Step Time": 4.837619431316853} +{"Finetune/Learning Rate": 1.819717395719935e-05, "Finetune/Loss": 0.9714701175689697, "Finetune/Loss (Raw)": 0.9566280841827393, "Finetune/Step": 1134, "Finetune/Step Time": 4.819221701472998} +{"Finetune/Learning Rate": 1.819360424407422e-05, "Finetune/Loss": 0.9727792739868164, "Finetune/Loss (Raw)": 1.1033380031585693, "Finetune/Step": 1135, "Finetune/Step Time": 4.8143531288951635} +{"Finetune/Learning Rate": 1.8190031351171256e-05, "Finetune/Loss": 0.9727365970611572, "Finetune/Loss (Raw)": 0.9804222583770752, "Finetune/Step": 1136, "Finetune/Step Time": 4.811636114493012} +{"Finetune/Learning Rate": 1.818645527987702e-05, "Finetune/Loss": 0.9775940179824829, "Finetune/Loss (Raw)": 1.0497159957885742, "Finetune/Step": 1137, "Finetune/Step Time": 4.777192296460271} +{"Finetune/Learning Rate": 1.8182876031579312e-05, "Finetune/Loss": 0.9785604476928711, "Finetune/Loss (Raw)": 1.0286904573440552, "Finetune/Step": 1138, "Finetune/Step Time": 4.786484466865659} +{"Finetune/Learning Rate": 1.8179293607667177e-05, "Finetune/Loss": 0.978875994682312, "Finetune/Loss (Raw)": 0.9643419981002808, "Finetune/Step": 1139, "Finetune/Step Time": 4.799390254542232} +{"Finetune/Learning Rate": 1.817570800953088e-05, "Finetune/Loss": 0.9789507389068604, "Finetune/Loss (Raw)": 1.0219898223876953, "Finetune/Step": 1140, "Finetune/Step Time": 4.792557818815112} +{"Finetune/Learning Rate": 1.8172119238561924e-05, "Finetune/Loss": 0.9792409539222717, "Finetune/Loss (Raw)": 0.9562870860099792, "Finetune/Step": 1141, "Finetune/Step Time": 4.797991409897804} +{"Finetune/Learning Rate": 1.8168527296153037e-05, "Finetune/Loss": 0.9734477996826172, "Finetune/Loss (Raw)": 0.30637621879577637, "Finetune/Step": 1142, "Finetune/Step Time": 4.827301740646362} +{"Finetune/Learning Rate": 1.8164932183698184e-05, "Finetune/Loss": 0.9782960414886475, "Finetune/Loss (Raw)": 0.935937225818634, "Finetune/Step": 1143, "Finetune/Step Time": 4.803663173690438} +{"Finetune/Learning Rate": 1.8161333902592554e-05, "Finetune/Loss": 0.978309690952301, "Finetune/Loss (Raw)": 1.0122451782226562, "Finetune/Step": 1144, "Finetune/Step Time": 4.807724483311176} +{"Finetune/Learning Rate": 1.8157732454232577e-05, "Finetune/Loss": 0.9773863554000854, "Finetune/Loss (Raw)": 0.9644500017166138, "Finetune/Step": 1145, "Finetune/Step Time": 4.808378912508488} +{"Finetune/Learning Rate": 1.81541278400159e-05, "Finetune/Loss": 0.9778233170509338, "Finetune/Loss (Raw)": 1.0594122409820557, "Finetune/Step": 1146, "Finetune/Step Time": 4.767844213172793} +{"Finetune/Learning Rate": 1.81505200613414e-05, "Finetune/Loss": 0.9769407510757446, "Finetune/Loss (Raw)": 0.8098882436752319, "Finetune/Step": 1147, "Finetune/Step Time": 4.771024964749813} +{"Finetune/Learning Rate": 1.8146909119609196e-05, "Finetune/Loss": 0.9719008803367615, "Finetune/Loss (Raw)": 0.3489459156990051, "Finetune/Step": 1148, "Finetune/Step Time": 4.810799445956945} +{"Finetune/Learning Rate": 1.8143295016220612e-05, "Finetune/Loss": 0.9737534523010254, "Finetune/Loss (Raw)": 1.1053334474563599, "Finetune/Step": 1149, "Finetune/Step Time": 4.811214866116643} +{"Finetune/Learning Rate": 1.813967775257822e-05, "Finetune/Loss": 0.9734266996383667, "Finetune/Loss (Raw)": 0.9786126613616943, "Finetune/Step": 1150, "Finetune/Step Time": 4.801621595397592} +{"Finetune/Learning Rate": 1.8136057330085805e-05, "Finetune/Loss": 0.9737142324447632, "Finetune/Loss (Raw)": 0.8828847408294678, "Finetune/Step": 1151, "Finetune/Step Time": 4.785973938181996} +{"Finetune/Learning Rate": 1.8132433750148383e-05, "Finetune/Loss": 0.9742045998573303, "Finetune/Loss (Raw)": 0.996493935585022, "Finetune/Step": 1152, "Finetune/Step Time": 4.798360066488385} +{"Finetune/Learning Rate": 1.8128807014172197e-05, "Finetune/Loss": 0.9758243560791016, "Finetune/Loss (Raw)": 1.0946418046951294, "Finetune/Step": 1153, "Finetune/Step Time": 4.812506163492799} +{"Finetune/Learning Rate": 1.812517712356471e-05, "Finetune/Loss": 0.9704421758651733, "Finetune/Loss (Raw)": 0.33370664715766907, "Finetune/Step": 1154, "Finetune/Step Time": 4.839774599298835} +{"Finetune/Learning Rate": 1.812154407973461e-05, "Finetune/Loss": 0.9702169895172119, "Finetune/Loss (Raw)": 0.34525030851364136, "Finetune/Step": 1155, "Finetune/Step Time": 4.8382260873913765} +{"Finetune/Learning Rate": 1.8117907884091815e-05, "Finetune/Loss": 0.9710156321525574, "Finetune/Loss (Raw)": 1.1541579961776733, "Finetune/Step": 1156, "Finetune/Step Time": 4.840024890378118} +{"Finetune/Learning Rate": 1.8114268538047457e-05, "Finetune/Loss": 0.9722913503646851, "Finetune/Loss (Raw)": 1.0435214042663574, "Finetune/Step": 1157, "Finetune/Step Time": 4.831646207720041} +{"Finetune/Learning Rate": 1.81106260430139e-05, "Finetune/Loss": 0.9714183807373047, "Finetune/Loss (Raw)": 0.9209532141685486, "Finetune/Step": 1158, "Finetune/Step Time": 4.833394968882203} +{"Finetune/Learning Rate": 1.810698040040472e-05, "Finetune/Loss": 0.9703837037086487, "Finetune/Loss (Raw)": 0.8577958941459656, "Finetune/Step": 1159, "Finetune/Step Time": 4.827823216095567} +{"Finetune/Learning Rate": 1.810333161163472e-05, "Finetune/Loss": 0.9751498103141785, "Finetune/Loss (Raw)": 0.9615546464920044, "Finetune/Step": 1160, "Finetune/Step Time": 4.805338414385915} +{"Finetune/Learning Rate": 1.809967967811993e-05, "Finetune/Loss": 0.9749575853347778, "Finetune/Loss (Raw)": 0.9829208850860596, "Finetune/Step": 1161, "Finetune/Step Time": 4.792900087311864} +{"Finetune/Learning Rate": 1.8096024601277586e-05, "Finetune/Loss": 0.9762596487998962, "Finetune/Loss (Raw)": 1.0104645490646362, "Finetune/Step": 1162, "Finetune/Step Time": 4.784166352823377} +{"Finetune/Learning Rate": 1.8092366382526157e-05, "Finetune/Loss": 0.9746687412261963, "Finetune/Loss (Raw)": 0.8030138611793518, "Finetune/Step": 1163, "Finetune/Step Time": 4.797894891351461} +{"Finetune/Learning Rate": 1.8088705023285324e-05, "Finetune/Loss": 0.9765080213546753, "Finetune/Loss (Raw)": 1.2017267942428589, "Finetune/Step": 1164, "Finetune/Step Time": 4.7905353885143995} +{"Finetune/Learning Rate": 1.8085040524975994e-05, "Finetune/Loss": 0.9725228548049927, "Finetune/Loss (Raw)": 0.3685491383075714, "Finetune/Step": 1165, "Finetune/Step Time": 4.809764480218291} +{"Finetune/Learning Rate": 1.8081372889020282e-05, "Finetune/Loss": 0.9703059196472168, "Finetune/Loss (Raw)": 0.8791877031326294, "Finetune/Step": 1166, "Finetune/Step Time": 4.82463750988245} +{"Finetune/Learning Rate": 1.8077702116841524e-05, "Finetune/Loss": 0.9689257144927979, "Finetune/Loss (Raw)": 0.835862398147583, "Finetune/Step": 1167, "Finetune/Step Time": 4.839687881991267} +{"Finetune/Learning Rate": 1.807402820986428e-05, "Finetune/Loss": 0.9691141843795776, "Finetune/Loss (Raw)": 0.9590463042259216, "Finetune/Step": 1168, "Finetune/Step Time": 4.838344918563962} +{"Finetune/Learning Rate": 1.8070351169514324e-05, "Finetune/Loss": 0.9684580564498901, "Finetune/Loss (Raw)": 0.9995349049568176, "Finetune/Step": 1169, "Finetune/Step Time": 4.845738081261516} +{"Finetune/Learning Rate": 1.8066670997218633e-05, "Finetune/Loss": 0.9636898040771484, "Finetune/Loss (Raw)": 0.4716905653476715, "Finetune/Step": 1170, "Finetune/Step Time": 4.876825151965022} +{"Finetune/Learning Rate": 1.806298769440542e-05, "Finetune/Loss": 0.9694885611534119, "Finetune/Loss (Raw)": 1.125585913658142, "Finetune/Step": 1171, "Finetune/Step Time": 4.84185086004436} +{"Finetune/Learning Rate": 1.8059301262504097e-05, "Finetune/Loss": 0.9709739685058594, "Finetune/Loss (Raw)": 1.0478986501693726, "Finetune/Step": 1172, "Finetune/Step Time": 4.836365718394518} +{"Finetune/Learning Rate": 1.80556117029453e-05, "Finetune/Loss": 0.9705049395561218, "Finetune/Loss (Raw)": 0.9923909306526184, "Finetune/Step": 1173, "Finetune/Step Time": 4.819783788174391} +{"Finetune/Learning Rate": 1.8051919017160875e-05, "Finetune/Loss": 0.9659290313720703, "Finetune/Loss (Raw)": 0.36614876985549927, "Finetune/Step": 1174, "Finetune/Step Time": 4.846446422860026} +{"Finetune/Learning Rate": 1.804822320658388e-05, "Finetune/Loss": 0.9655437469482422, "Finetune/Loss (Raw)": 0.9550232887268066, "Finetune/Step": 1175, "Finetune/Step Time": 4.855033623054624} +{"Finetune/Learning Rate": 1.8044524272648588e-05, "Finetune/Loss": 0.963721752166748, "Finetune/Loss (Raw)": 0.9370580911636353, "Finetune/Step": 1176, "Finetune/Step Time": 4.85579358972609} +{"Finetune/Learning Rate": 1.8040822216790475e-05, "Finetune/Loss": 0.9591599702835083, "Finetune/Loss (Raw)": 0.29259321093559265, "Finetune/Step": 1177, "Finetune/Step Time": 4.893501153215766} +{"Finetune/Learning Rate": 1.803711704044625e-05, "Finetune/Loss": 0.9553370475769043, "Finetune/Loss (Raw)": 0.32585540413856506, "Finetune/Step": 1178, "Finetune/Step Time": 4.913887778297067} +{"Finetune/Learning Rate": 1.803340874505381e-05, "Finetune/Loss": 0.957029402256012, "Finetune/Loss (Raw)": 1.162949562072754, "Finetune/Step": 1179, "Finetune/Step Time": 4.9062989223748446} +{"Finetune/Learning Rate": 1.8029697332052277e-05, "Finetune/Loss": 0.9552918672561646, "Finetune/Loss (Raw)": 0.9303387999534607, "Finetune/Step": 1180, "Finetune/Step Time": 4.922394372522831} +{"Finetune/Learning Rate": 1.8025982802881976e-05, "Finetune/Loss": 0.9561190605163574, "Finetune/Loss (Raw)": 1.0856528282165527, "Finetune/Step": 1181, "Finetune/Step Time": 4.909813519567251} +{"Finetune/Learning Rate": 1.802226515898444e-05, "Finetune/Loss": 0.9529756307601929, "Finetune/Loss (Raw)": 0.7123126983642578, "Finetune/Step": 1182, "Finetune/Step Time": 4.911808995530009} +{"Finetune/Learning Rate": 1.8018544401802422e-05, "Finetune/Loss": 0.9540107250213623, "Finetune/Loss (Raw)": 1.036002278327942, "Finetune/Step": 1183, "Finetune/Step Time": 4.913187511265278} +{"Finetune/Learning Rate": 1.801482053277987e-05, "Finetune/Loss": 0.9518758058547974, "Finetune/Loss (Raw)": 0.8196206092834473, "Finetune/Step": 1184, "Finetune/Step Time": 4.943019315600395} +{"Finetune/Learning Rate": 1.8011093553361944e-05, "Finetune/Loss": 0.9526379108428955, "Finetune/Loss (Raw)": 1.0186153650283813, "Finetune/Step": 1185, "Finetune/Step Time": 4.94639603421092} +{"Finetune/Learning Rate": 1.8007363464995015e-05, "Finetune/Loss": 0.9541451930999756, "Finetune/Loss (Raw)": 1.1222442388534546, "Finetune/Step": 1186, "Finetune/Step Time": 4.9467782862484455} +{"Finetune/Learning Rate": 1.800363026912665e-05, "Finetune/Loss": 0.9495375156402588, "Finetune/Loss (Raw)": 0.35408738255500793, "Finetune/Step": 1187, "Finetune/Step Time": 4.967859109863639} +{"Finetune/Learning Rate": 1.7999893967205643e-05, "Finetune/Loss": 0.9501597881317139, "Finetune/Loss (Raw)": 1.0591554641723633, "Finetune/Step": 1188, "Finetune/Step Time": 4.96714099124074} +{"Finetune/Learning Rate": 1.7996154560681963e-05, "Finetune/Loss": 0.9518231153488159, "Finetune/Loss (Raw)": 1.0636096000671387, "Finetune/Step": 1189, "Finetune/Step Time": 4.9716885685920715} +{"Finetune/Learning Rate": 1.7992412051006817e-05, "Finetune/Loss": 0.951689600944519, "Finetune/Loss (Raw)": 0.9816321134567261, "Finetune/Step": 1190, "Finetune/Step Time": 4.969985790550709} +{"Finetune/Learning Rate": 1.7988666439632593e-05, "Finetune/Loss": 0.9522315859794617, "Finetune/Loss (Raw)": 1.0529979467391968, "Finetune/Step": 1191, "Finetune/Step Time": 4.963500719517469} +{"Finetune/Learning Rate": 1.7984917728012886e-05, "Finetune/Loss": 0.9510114192962646, "Finetune/Loss (Raw)": 1.030913233757019, "Finetune/Step": 1192, "Finetune/Step Time": 4.971233807504177} +{"Finetune/Learning Rate": 1.7981165917602502e-05, "Finetune/Loss": 0.9500985741615295, "Finetune/Loss (Raw)": 1.0184701681137085, "Finetune/Step": 1193, "Finetune/Step Time": 4.97021371498704} +{"Finetune/Learning Rate": 1.7977411009857446e-05, "Finetune/Loss": 0.9514119029045105, "Finetune/Loss (Raw)": 1.0682796239852905, "Finetune/Step": 1194, "Finetune/Step Time": 4.984145861119032} +{"Finetune/Learning Rate": 1.797365300623492e-05, "Finetune/Loss": 0.9505506157875061, "Finetune/Loss (Raw)": 0.9988060593605042, "Finetune/Step": 1195, "Finetune/Step Time": 4.973917504772544} +{"Finetune/Learning Rate": 1.7969891908193338e-05, "Finetune/Loss": 0.9511721134185791, "Finetune/Loss (Raw)": 0.9875177145004272, "Finetune/Step": 1196, "Finetune/Step Time": 4.970476208254695} +{"Finetune/Learning Rate": 1.7966127717192305e-05, "Finetune/Loss": 0.9522621631622314, "Finetune/Loss (Raw)": 1.1109998226165771, "Finetune/Step": 1197, "Finetune/Step Time": 4.937180100008845} +{"Finetune/Learning Rate": 1.796236043469263e-05, "Finetune/Loss": 0.9527329206466675, "Finetune/Loss (Raw)": 1.0932985544204712, "Finetune/Step": 1198, "Finetune/Step Time": 4.942583624273539} +{"Finetune/Learning Rate": 1.7958590062156318e-05, "Finetune/Loss": 0.9527355432510376, "Finetune/Loss (Raw)": 1.0423251390457153, "Finetune/Step": 1199, "Finetune/Step Time": 4.938754381611943} +{"Finetune/Learning Rate": 1.7954816601046587e-05, "Finetune/Loss": 0.9528024792671204, "Finetune/Loss (Raw)": 1.0543612241744995, "Finetune/Step": 1200, "Finetune/Step Time": 4.935098014771938} +{"Finetune/Learning Rate": 1.7951040052827836e-05, "Finetune/Loss": 0.9534463882446289, "Finetune/Loss (Raw)": 1.040122151374817, "Finetune/Step": 1201, "Finetune/Step Time": 4.948764950037003} +{"Finetune/Learning Rate": 1.794726041896567e-05, "Finetune/Loss": 0.9543015956878662, "Finetune/Loss (Raw)": 1.1632306575775146, "Finetune/Step": 1202, "Finetune/Step Time": 4.95236724242568} +{"Finetune/Learning Rate": 1.7943477700926895e-05, "Finetune/Loss": 0.954888105392456, "Finetune/Loss (Raw)": 1.12491774559021, "Finetune/Step": 1203, "Finetune/Step Time": 4.942792883142829} +{"Finetune/Learning Rate": 1.7939691900179507e-05, "Finetune/Loss": 0.9549319744110107, "Finetune/Loss (Raw)": 1.0836457014083862, "Finetune/Step": 1204, "Finetune/Step Time": 4.951091352850199} +{"Finetune/Learning Rate": 1.79359030181927e-05, "Finetune/Loss": 0.9539440870285034, "Finetune/Loss (Raw)": 0.9697629809379578, "Finetune/Step": 1205, "Finetune/Step Time": 4.968777593225241} +{"Finetune/Learning Rate": 1.7932111056436868e-05, "Finetune/Loss": 0.954603374004364, "Finetune/Loss (Raw)": 1.1490098237991333, "Finetune/Step": 1206, "Finetune/Step Time": 4.961007889360189} +{"Finetune/Learning Rate": 1.79283160163836e-05, "Finetune/Loss": 0.9553401470184326, "Finetune/Loss (Raw)": 0.9604055285453796, "Finetune/Step": 1207, "Finetune/Step Time": 4.9678516369313} +{"Finetune/Learning Rate": 1.792451789950567e-05, "Finetune/Loss": 0.9556667804718018, "Finetune/Loss (Raw)": 1.0177439451217651, "Finetune/Step": 1208, "Finetune/Step Time": 4.968233358114958} +{"Finetune/Learning Rate": 1.7920716707277054e-05, "Finetune/Loss": 0.9554259777069092, "Finetune/Loss (Raw)": 1.0496302843093872, "Finetune/Step": 1209, "Finetune/Step Time": 4.966940922662616} +{"Finetune/Learning Rate": 1.7916912441172924e-05, "Finetune/Loss": 0.9566062688827515, "Finetune/Loss (Raw)": 0.9986864924430847, "Finetune/Step": 1210, "Finetune/Step Time": 4.9448609706014395} +{"Finetune/Learning Rate": 1.7913105102669643e-05, "Finetune/Loss": 0.9560536742210388, "Finetune/Loss (Raw)": 0.9513893127441406, "Finetune/Step": 1211, "Finetune/Step Time": 4.947725087404251} +{"Finetune/Learning Rate": 1.790929469324476e-05, "Finetune/Loss": 0.9564905166625977, "Finetune/Loss (Raw)": 1.1219768524169922, "Finetune/Step": 1212, "Finetune/Step Time": 4.935466410592198} +{"Finetune/Learning Rate": 1.7905481214377027e-05, "Finetune/Loss": 0.9560004472732544, "Finetune/Loss (Raw)": 0.9577575922012329, "Finetune/Step": 1213, "Finetune/Step Time": 4.934798516333103} +{"Finetune/Learning Rate": 1.790166466754637e-05, "Finetune/Loss": 0.9545934200286865, "Finetune/Loss (Raw)": 0.856961190700531, "Finetune/Step": 1214, "Finetune/Step Time": 4.949018631130457} +{"Finetune/Learning Rate": 1.7897845054233927e-05, "Finetune/Loss": 0.9543454647064209, "Finetune/Loss (Raw)": 0.9891017079353333, "Finetune/Step": 1215, "Finetune/Step Time": 4.949827237054706} +{"Finetune/Learning Rate": 1.7894022375922006e-05, "Finetune/Loss": 0.9504585862159729, "Finetune/Loss (Raw)": 0.35208335518836975, "Finetune/Step": 1216, "Finetune/Step Time": 4.975255882367492} +{"Finetune/Learning Rate": 1.7890196634094122e-05, "Finetune/Loss": 0.9512540102005005, "Finetune/Loss (Raw)": 1.1098114252090454, "Finetune/Step": 1217, "Finetune/Step Time": 4.978766152635217} +{"Finetune/Learning Rate": 1.7886367830234968e-05, "Finetune/Loss": 0.9508610367774963, "Finetune/Loss (Raw)": 1.0548120737075806, "Finetune/Step": 1218, "Finetune/Step Time": 4.963260201737285} +{"Finetune/Learning Rate": 1.788253596583043e-05, "Finetune/Loss": 0.9459470510482788, "Finetune/Loss (Raw)": 0.30801382660865784, "Finetune/Step": 1219, "Finetune/Step Time": 4.994637755677104} +{"Finetune/Learning Rate": 1.787870104236757e-05, "Finetune/Loss": 0.9455916881561279, "Finetune/Loss (Raw)": 0.9894474148750305, "Finetune/Step": 1220, "Finetune/Step Time": 4.993894033133984} +{"Finetune/Learning Rate": 1.7874863061334658e-05, "Finetune/Loss": 0.9467929601669312, "Finetune/Loss (Raw)": 1.0384750366210938, "Finetune/Step": 1221, "Finetune/Step Time": 4.999197382479906} +{"Finetune/Learning Rate": 1.7871022024221133e-05, "Finetune/Loss": 0.9468718767166138, "Finetune/Loss (Raw)": 0.9980108737945557, "Finetune/Step": 1222, "Finetune/Step Time": 5.004254290834069} +{"Finetune/Learning Rate": 1.7867177932517634e-05, "Finetune/Loss": 0.946738064289093, "Finetune/Loss (Raw)": 1.024492621421814, "Finetune/Step": 1223, "Finetune/Step Time": 5.004773747175932} +{"Finetune/Learning Rate": 1.786333078771597e-05, "Finetune/Loss": 0.9462358951568604, "Finetune/Loss (Raw)": 1.0028034448623657, "Finetune/Step": 1224, "Finetune/Step Time": 5.011406444013119} +{"Finetune/Learning Rate": 1.785948059130915e-05, "Finetune/Loss": 0.9453861713409424, "Finetune/Loss (Raw)": 0.8786132335662842, "Finetune/Step": 1225, "Finetune/Step Time": 5.00184422545135} +{"Finetune/Learning Rate": 1.785562734479135e-05, "Finetune/Loss": 0.9446812272071838, "Finetune/Loss (Raw)": 0.8951446413993835, "Finetune/Step": 1226, "Finetune/Step Time": 4.986915480345488} +{"Finetune/Learning Rate": 1.785177104965795e-05, "Finetune/Loss": 0.9436191320419312, "Finetune/Loss (Raw)": 0.8681504130363464, "Finetune/Step": 1227, "Finetune/Step Time": 4.995550941675901} +{"Finetune/Learning Rate": 1.7847911707405494e-05, "Finetune/Loss": 0.9442889094352722, "Finetune/Loss (Raw)": 1.0503942966461182, "Finetune/Step": 1228, "Finetune/Step Time": 5.002126406878233} +{"Finetune/Learning Rate": 1.7844049319531725e-05, "Finetune/Loss": 0.9442178010940552, "Finetune/Loss (Raw)": 1.077003836631775, "Finetune/Step": 1229, "Finetune/Step Time": 4.998507011681795} +{"Finetune/Learning Rate": 1.7840183887535558e-05, "Finetune/Loss": 0.9411002397537231, "Finetune/Loss (Raw)": 0.40609294176101685, "Finetune/Step": 1230, "Finetune/Step Time": 5.03649971447885} +{"Finetune/Learning Rate": 1.7836315412917088e-05, "Finetune/Loss": 0.9409602880477905, "Finetune/Loss (Raw)": 1.1128119230270386, "Finetune/Step": 1231, "Finetune/Step Time": 5.029759196564555} +{"Finetune/Learning Rate": 1.78324438971776e-05, "Finetune/Loss": 0.9405263066291809, "Finetune/Loss (Raw)": 1.0138565301895142, "Finetune/Step": 1232, "Finetune/Step Time": 5.023970261216164} +{"Finetune/Learning Rate": 1.782856934181955e-05, "Finetune/Loss": 0.9398510456085205, "Finetune/Loss (Raw)": 1.0360181331634521, "Finetune/Step": 1233, "Finetune/Step Time": 5.0089034996926785} +{"Finetune/Learning Rate": 1.782469174834658e-05, "Finetune/Loss": 0.9383624792098999, "Finetune/Loss (Raw)": 0.8083620071411133, "Finetune/Step": 1234, "Finetune/Step Time": 5.012755343690515} +{"Finetune/Learning Rate": 1.7820811118263503e-05, "Finetune/Loss": 0.937756359577179, "Finetune/Loss (Raw)": 0.9501402974128723, "Finetune/Step": 1235, "Finetune/Step Time": 5.011293603107333} +{"Finetune/Learning Rate": 1.7816927453076324e-05, "Finetune/Loss": 0.9378427863121033, "Finetune/Loss (Raw)": 1.1053508520126343, "Finetune/Step": 1236, "Finetune/Step Time": 5.007191672921181} +{"Finetune/Learning Rate": 1.781304075429221e-05, "Finetune/Loss": 0.939385712146759, "Finetune/Loss (Raw)": 1.0201404094696045, "Finetune/Step": 1237, "Finetune/Step Time": 5.004369856789708} +{"Finetune/Learning Rate": 1.7809151023419517e-05, "Finetune/Loss": 0.9408183693885803, "Finetune/Loss (Raw)": 1.1726592779159546, "Finetune/Step": 1238, "Finetune/Step Time": 5.007079506292939} +{"Finetune/Learning Rate": 1.7805258261967774e-05, "Finetune/Loss": 0.939556360244751, "Finetune/Loss (Raw)": 0.9757534861564636, "Finetune/Step": 1239, "Finetune/Step Time": 5.020851472392678} +{"Finetune/Learning Rate": 1.780136247144768e-05, "Finetune/Loss": 0.9399297833442688, "Finetune/Loss (Raw)": 0.9758924245834351, "Finetune/Step": 1240, "Finetune/Step Time": 5.019183678552508} +{"Finetune/Learning Rate": 1.7797463653371125e-05, "Finetune/Loss": 0.9395338892936707, "Finetune/Loss (Raw)": 0.8975390195846558, "Finetune/Step": 1241, "Finetune/Step Time": 5.028905091807246} +{"Finetune/Learning Rate": 1.7793561809251153e-05, "Finetune/Loss": 0.9390683174133301, "Finetune/Loss (Raw)": 0.9758330583572388, "Finetune/Step": 1242, "Finetune/Step Time": 5.001708829775453} +{"Finetune/Learning Rate": 1.7789656940602003e-05, "Finetune/Loss": 0.9393261671066284, "Finetune/Loss (Raw)": 1.0546296834945679, "Finetune/Step": 1243, "Finetune/Step Time": 4.995639054104686} +{"Finetune/Learning Rate": 1.7785749048939076e-05, "Finetune/Loss": 0.9383350610733032, "Finetune/Loss (Raw)": 0.9704859852790833, "Finetune/Step": 1244, "Finetune/Step Time": 5.004682132974267} +{"Finetune/Learning Rate": 1.7781838135778945e-05, "Finetune/Loss": 0.938959538936615, "Finetune/Loss (Raw)": 0.9716255068778992, "Finetune/Step": 1245, "Finetune/Step Time": 4.994856661185622} +{"Finetune/Learning Rate": 1.777792420263936e-05, "Finetune/Loss": 0.9376764297485352, "Finetune/Loss (Raw)": 0.927710771560669, "Finetune/Step": 1246, "Finetune/Step Time": 5.017169814556837} +{"Finetune/Learning Rate": 1.7774007251039246e-05, "Finetune/Loss": 0.940280556678772, "Finetune/Loss (Raw)": 1.0682517290115356, "Finetune/Step": 1247, "Finetune/Step Time": 5.002446578815579} +{"Finetune/Learning Rate": 1.777008728249869e-05, "Finetune/Loss": 0.9389426708221436, "Finetune/Loss (Raw)": 0.8149150609970093, "Finetune/Step": 1248, "Finetune/Step Time": 5.006992448121309} +{"Finetune/Learning Rate": 1.776616429853896e-05, "Finetune/Loss": 0.937031626701355, "Finetune/Loss (Raw)": 0.9148775935173035, "Finetune/Step": 1249, "Finetune/Step Time": 5.007948290556669} +{"Finetune/Learning Rate": 1.7762238300682484e-05, "Finetune/Loss": 0.936935544013977, "Finetune/Loss (Raw)": 1.0345447063446045, "Finetune/Step": 1250, "Finetune/Step Time": 4.9881318882107735} +{"Finetune/Learning Rate": 1.775830929045287e-05, "Finetune/Loss": 0.9375923871994019, "Finetune/Loss (Raw)": 1.0247573852539062, "Finetune/Step": 1251, "Finetune/Step Time": 4.973924167454243} +{"Finetune/Learning Rate": 1.7754377269374893e-05, "Finetune/Loss": 0.9349762201309204, "Finetune/Loss (Raw)": 0.8945072889328003, "Finetune/Step": 1252, "Finetune/Step Time": 4.969863370060921} +{"Finetune/Learning Rate": 1.7750442238974483e-05, "Finetune/Loss": 0.9348462820053101, "Finetune/Loss (Raw)": 0.9299046993255615, "Finetune/Step": 1253, "Finetune/Step Time": 4.988557852804661} +{"Finetune/Learning Rate": 1.7746504200778756e-05, "Finetune/Loss": 0.9348249435424805, "Finetune/Loss (Raw)": 0.9807317852973938, "Finetune/Step": 1254, "Finetune/Step Time": 4.991902127861977} +{"Finetune/Learning Rate": 1.7742563156315987e-05, "Finetune/Loss": 0.93539959192276, "Finetune/Loss (Raw)": 1.0997376441955566, "Finetune/Step": 1255, "Finetune/Step Time": 4.994095062837005} +{"Finetune/Learning Rate": 1.773861910711562e-05, "Finetune/Loss": 0.9357243776321411, "Finetune/Loss (Raw)": 1.0189943313598633, "Finetune/Step": 1256, "Finetune/Step Time": 4.989480692893267} +{"Finetune/Learning Rate": 1.773467205470826e-05, "Finetune/Loss": 0.9325047135353088, "Finetune/Loss (Raw)": 0.6971272826194763, "Finetune/Step": 1257, "Finetune/Step Time": 5.01585884205997} +{"Finetune/Learning Rate": 1.7730722000625683e-05, "Finetune/Loss": 0.9336954355239868, "Finetune/Loss (Raw)": 0.9528205394744873, "Finetune/Step": 1258, "Finetune/Step Time": 5.004218837246299} +{"Finetune/Learning Rate": 1.7726768946400825e-05, "Finetune/Loss": 0.932253360748291, "Finetune/Loss (Raw)": 0.8980908989906311, "Finetune/Step": 1259, "Finetune/Step Time": 5.015284055843949} +{"Finetune/Learning Rate": 1.772281289356779e-05, "Finetune/Loss": 0.9333975911140442, "Finetune/Loss (Raw)": 1.1741145849227905, "Finetune/Step": 1260, "Finetune/Step Time": 5.017177507281303} +{"Finetune/Learning Rate": 1.771885384366185e-05, "Finetune/Loss": 0.9323440194129944, "Finetune/Loss (Raw)": 0.8478386402130127, "Finetune/Step": 1261, "Finetune/Step Time": 5.017420148476958} +{"Finetune/Learning Rate": 1.7714891798219432e-05, "Finetune/Loss": 0.9326095581054688, "Finetune/Loss (Raw)": 0.9906250238418579, "Finetune/Step": 1262, "Finetune/Step Time": 5.012868518009782} +{"Finetune/Learning Rate": 1.7710926758778123e-05, "Finetune/Loss": 0.9322183132171631, "Finetune/Loss (Raw)": 1.0532594919204712, "Finetune/Step": 1263, "Finetune/Step Time": 5.009583165869117} +{"Finetune/Learning Rate": 1.7706958726876685e-05, "Finetune/Loss": 0.9323379993438721, "Finetune/Loss (Raw)": 0.9957387447357178, "Finetune/Step": 1264, "Finetune/Step Time": 5.007890820503235} +{"Finetune/Learning Rate": 1.770298770405503e-05, "Finetune/Loss": 0.9271109700202942, "Finetune/Loss (Raw)": 0.3806505799293518, "Finetune/Step": 1265, "Finetune/Step Time": 5.042561525478959} +{"Finetune/Learning Rate": 1.7699013691854238e-05, "Finetune/Loss": 0.9265755414962769, "Finetune/Loss (Raw)": 0.9601597785949707, "Finetune/Step": 1266, "Finetune/Step Time": 5.048125309869647} +{"Finetune/Learning Rate": 1.769503669181654e-05, "Finetune/Loss": 0.9271990060806274, "Finetune/Loss (Raw)": 1.044150948524475, "Finetune/Step": 1267, "Finetune/Step Time": 5.034090055152774} +{"Finetune/Learning Rate": 1.7691056705485333e-05, "Finetune/Loss": 0.9275614023208618, "Finetune/Loss (Raw)": 1.0683733224868774, "Finetune/Step": 1268, "Finetune/Step Time": 5.037591176107526} +{"Finetune/Learning Rate": 1.7687073734405177e-05, "Finetune/Loss": 0.9275487065315247, "Finetune/Loss (Raw)": 0.9546542763710022, "Finetune/Step": 1269, "Finetune/Step Time": 5.029388654977083} +{"Finetune/Learning Rate": 1.768308778012178e-05, "Finetune/Loss": 0.9339535236358643, "Finetune/Loss (Raw)": 1.12619948387146, "Finetune/Step": 1270, "Finetune/Step Time": 5.003529317677021} +{"Finetune/Learning Rate": 1.767909884418202e-05, "Finetune/Loss": 0.9352390170097351, "Finetune/Loss (Raw)": 1.1004838943481445, "Finetune/Step": 1271, "Finetune/Step Time": 4.988543653860688} +{"Finetune/Learning Rate": 1.7675106928133915e-05, "Finetune/Loss": 0.933722734451294, "Finetune/Loss (Raw)": 0.8181567192077637, "Finetune/Step": 1272, "Finetune/Step Time": 4.997435785830021} +{"Finetune/Learning Rate": 1.7671112033526655e-05, "Finetune/Loss": 0.9334961175918579, "Finetune/Loss (Raw)": 0.9354390501976013, "Finetune/Step": 1273, "Finetune/Step Time": 5.007420716807246} +{"Finetune/Learning Rate": 1.7667114161910586e-05, "Finetune/Loss": 0.9320242404937744, "Finetune/Loss (Raw)": 0.8710175156593323, "Finetune/Step": 1274, "Finetune/Step Time": 4.998548684641719} +{"Finetune/Learning Rate": 1.7663113314837192e-05, "Finetune/Loss": 0.9348129630088806, "Finetune/Loss (Raw)": 1.166846752166748, "Finetune/Step": 1275, "Finetune/Step Time": 4.989759622141719} +{"Finetune/Learning Rate": 1.7659109493859134e-05, "Finetune/Loss": 0.9394937753677368, "Finetune/Loss (Raw)": 0.9480884075164795, "Finetune/Step": 1276, "Finetune/Step Time": 4.951319936662912} +{"Finetune/Learning Rate": 1.7655102700530212e-05, "Finetune/Loss": 0.9383742809295654, "Finetune/Loss (Raw)": 0.9620282053947449, "Finetune/Step": 1277, "Finetune/Step Time": 4.9546926487237215} +{"Finetune/Learning Rate": 1.7651092936405385e-05, "Finetune/Loss": 0.936521053314209, "Finetune/Loss (Raw)": 0.741411566734314, "Finetune/Step": 1278, "Finetune/Step Time": 4.961114885285497} +{"Finetune/Learning Rate": 1.7647080203040765e-05, "Finetune/Loss": 0.9373811483383179, "Finetune/Loss (Raw)": 0.9929695725440979, "Finetune/Step": 1279, "Finetune/Step Time": 4.963058220222592} +{"Finetune/Learning Rate": 1.7643064501993614e-05, "Finetune/Loss": 0.938633918762207, "Finetune/Loss (Raw)": 1.1568528413772583, "Finetune/Step": 1280, "Finetune/Step Time": 4.9415485095232725} +{"Finetune/Learning Rate": 1.763904583482235e-05, "Finetune/Loss": 0.9375187158584595, "Finetune/Loss (Raw)": 0.9518929719924927, "Finetune/Step": 1281, "Finetune/Step Time": 4.925957588478923} +{"Finetune/Learning Rate": 1.7635024203086532e-05, "Finetune/Loss": 0.9426558613777161, "Finetune/Loss (Raw)": 0.9912586212158203, "Finetune/Step": 1282, "Finetune/Step Time": 4.891373855993152} +{"Finetune/Learning Rate": 1.7630999608346888e-05, "Finetune/Loss": 0.9468250870704651, "Finetune/Loss (Raw)": 0.8789179921150208, "Finetune/Step": 1283, "Finetune/Step Time": 4.868059867992997} +{"Finetune/Learning Rate": 1.7626972052165275e-05, "Finetune/Loss": 0.9453816413879395, "Finetune/Loss (Raw)": 0.9693970084190369, "Finetune/Step": 1284, "Finetune/Step Time": 4.86547202989459} +{"Finetune/Learning Rate": 1.7622941536104715e-05, "Finetune/Loss": 0.9454708099365234, "Finetune/Loss (Raw)": 1.0549278259277344, "Finetune/Step": 1285, "Finetune/Step Time": 4.865512229502201} +{"Finetune/Learning Rate": 1.7618908061729373e-05, "Finetune/Loss": 0.9457787871360779, "Finetune/Loss (Raw)": 0.9603818655014038, "Finetune/Step": 1286, "Finetune/Step Time": 4.869615716859698} +{"Finetune/Learning Rate": 1.7614871630604554e-05, "Finetune/Loss": 0.9467462301254272, "Finetune/Loss (Raw)": 0.9816257357597351, "Finetune/Step": 1287, "Finetune/Step Time": 4.869100425392389} +{"Finetune/Learning Rate": 1.761083224429672e-05, "Finetune/Loss": 0.946334958076477, "Finetune/Loss (Raw)": 0.9089115858078003, "Finetune/Step": 1288, "Finetune/Step Time": 4.849067954346538} +{"Finetune/Learning Rate": 1.760678990437349e-05, "Finetune/Loss": 0.9457669258117676, "Finetune/Loss (Raw)": 0.9102115631103516, "Finetune/Step": 1289, "Finetune/Step Time": 4.833659375086427} +{"Finetune/Learning Rate": 1.76027446124036e-05, "Finetune/Loss": 0.9464985132217407, "Finetune/Loss (Raw)": 1.1041051149368286, "Finetune/Step": 1290, "Finetune/Step Time": 4.828115342184901} +{"Finetune/Learning Rate": 1.759869636995696e-05, "Finetune/Loss": 0.9479641318321228, "Finetune/Loss (Raw)": 0.9906156659126282, "Finetune/Step": 1291, "Finetune/Step Time": 4.825600383803248} +{"Finetune/Learning Rate": 1.7594645178604614e-05, "Finetune/Loss": 0.9461272954940796, "Finetune/Loss (Raw)": 0.966616690158844, "Finetune/Step": 1292, "Finetune/Step Time": 4.825682492926717} +{"Finetune/Learning Rate": 1.7590591039918744e-05, "Finetune/Loss": 0.9506711959838867, "Finetune/Loss (Raw)": 0.9501582980155945, "Finetune/Step": 1293, "Finetune/Step Time": 4.795100700110197} +{"Finetune/Learning Rate": 1.7586533955472686e-05, "Finetune/Loss": 0.9515547156333923, "Finetune/Loss (Raw)": 0.9922874569892883, "Finetune/Step": 1294, "Finetune/Step Time": 4.791888777166605} +{"Finetune/Learning Rate": 1.7582473926840916e-05, "Finetune/Loss": 0.9522728323936462, "Finetune/Loss (Raw)": 0.927778422832489, "Finetune/Step": 1295, "Finetune/Step Time": 4.790288234129548} +{"Finetune/Learning Rate": 1.7578410955599046e-05, "Finetune/Loss": 0.9522822499275208, "Finetune/Loss (Raw)": 0.9602497220039368, "Finetune/Step": 1296, "Finetune/Step Time": 4.7939551174640656} +{"Finetune/Learning Rate": 1.7574345043323843e-05, "Finetune/Loss": 0.9522122740745544, "Finetune/Loss (Raw)": 0.990576982498169, "Finetune/Step": 1297, "Finetune/Step Time": 4.803661732003093} +{"Finetune/Learning Rate": 1.7570276191593202e-05, "Finetune/Loss": 0.9551492929458618, "Finetune/Loss (Raw)": 0.8476305603981018, "Finetune/Step": 1298, "Finetune/Step Time": 4.770134912803769} +{"Finetune/Learning Rate": 1.7566204401986172e-05, "Finetune/Loss": 0.952147901058197, "Finetune/Loss (Raw)": 0.7414052486419678, "Finetune/Step": 1299, "Finetune/Step Time": 4.774020634591579} +{"Finetune/Learning Rate": 1.7562129676082933e-05, "Finetune/Loss": 0.9509385824203491, "Finetune/Loss (Raw)": 0.8931099772453308, "Finetune/Step": 1300, "Finetune/Step Time": 4.7776816207915545} +{"Finetune/Learning Rate": 1.75580520154648e-05, "Finetune/Loss": 0.9511687755584717, "Finetune/Loss (Raw)": 1.0218589305877686, "Finetune/Step": 1301, "Finetune/Step Time": 4.795698678120971} +{"Finetune/Learning Rate": 1.755397142171424e-05, "Finetune/Loss": 0.9556582570075989, "Finetune/Loss (Raw)": 0.940798819065094, "Finetune/Step": 1302, "Finetune/Step Time": 4.786821823567152} +{"Finetune/Learning Rate": 1.7549887896414853e-05, "Finetune/Loss": 0.9558037519454956, "Finetune/Loss (Raw)": 0.9736443758010864, "Finetune/Step": 1303, "Finetune/Step Time": 4.77011195383966} +{"Finetune/Learning Rate": 1.7545801441151368e-05, "Finetune/Loss": 0.9563049077987671, "Finetune/Loss (Raw)": 1.0012058019638062, "Finetune/Step": 1304, "Finetune/Step Time": 4.770934639498591} +{"Finetune/Learning Rate": 1.7541712057509666e-05, "Finetune/Loss": 0.961542546749115, "Finetune/Loss (Raw)": 0.9630113840103149, "Finetune/Step": 1305, "Finetune/Step Time": 4.7386705949902534} +{"Finetune/Learning Rate": 1.7537619747076752e-05, "Finetune/Loss": 0.9667754173278809, "Finetune/Loss (Raw)": 0.9956676363945007, "Finetune/Step": 1306, "Finetune/Step Time": 4.726214600726962} +{"Finetune/Learning Rate": 1.7533524511440777e-05, "Finetune/Loss": 0.9661128520965576, "Finetune/Loss (Raw)": 1.0781292915344238, "Finetune/Step": 1307, "Finetune/Step Time": 4.720325196161866} +{"Finetune/Learning Rate": 1.7529426352191014e-05, "Finetune/Loss": 0.967227041721344, "Finetune/Loss (Raw)": 1.072962999343872, "Finetune/Step": 1308, "Finetune/Step Time": 4.702806789427996} +{"Finetune/Learning Rate": 1.752532527091789e-05, "Finetune/Loss": 0.9662469625473022, "Finetune/Loss (Raw)": 0.9602102637290955, "Finetune/Step": 1309, "Finetune/Step Time": 4.721675705164671} +{"Finetune/Learning Rate": 1.7521221269212944e-05, "Finetune/Loss": 0.9677847623825073, "Finetune/Loss (Raw)": 0.9091470241546631, "Finetune/Step": 1310, "Finetune/Step Time": 4.719542833045125} +{"Finetune/Learning Rate": 1.751711434866886e-05, "Finetune/Loss": 0.9675338864326477, "Finetune/Loss (Raw)": 1.0038862228393555, "Finetune/Step": 1311, "Finetune/Step Time": 4.721737403422594} +{"Finetune/Learning Rate": 1.751300451087946e-05, "Finetune/Loss": 0.9670977592468262, "Finetune/Loss (Raw)": 0.7637962102890015, "Finetune/Step": 1312, "Finetune/Step Time": 4.716350028291345} +{"Finetune/Learning Rate": 1.7508891757439687e-05, "Finetune/Loss": 0.9670582413673401, "Finetune/Loss (Raw)": 1.013551950454712, "Finetune/Step": 1313, "Finetune/Step Time": 4.722673803567886} +{"Finetune/Learning Rate": 1.7504776089945626e-05, "Finetune/Loss": 0.9660825133323669, "Finetune/Loss (Raw)": 0.9973505139350891, "Finetune/Step": 1314, "Finetune/Step Time": 4.720214830711484} +{"Finetune/Learning Rate": 1.7500657509994478e-05, "Finetune/Loss": 0.9710367918014526, "Finetune/Loss (Raw)": 0.9882357120513916, "Finetune/Step": 1315, "Finetune/Step Time": 4.686404127627611} +{"Finetune/Learning Rate": 1.7496536019184588e-05, "Finetune/Loss": 0.9718339443206787, "Finetune/Loss (Raw)": 1.1611963510513306, "Finetune/Step": 1316, "Finetune/Step Time": 4.691709546372294} +{"Finetune/Learning Rate": 1.749241161911543e-05, "Finetune/Loss": 0.971794068813324, "Finetune/Loss (Raw)": 1.0585051774978638, "Finetune/Step": 1317, "Finetune/Step Time": 4.684845680370927} +{"Finetune/Learning Rate": 1.7488284311387597e-05, "Finetune/Loss": 0.9715944528579712, "Finetune/Loss (Raw)": 0.9560803771018982, "Finetune/Step": 1318, "Finetune/Step Time": 4.684930728748441} +{"Finetune/Learning Rate": 1.7484154097602822e-05, "Finetune/Loss": 0.9711349010467529, "Finetune/Loss (Raw)": 0.9941726326942444, "Finetune/Step": 1319, "Finetune/Step Time": 4.682437812909484} +{"Finetune/Learning Rate": 1.7480020979363956e-05, "Finetune/Loss": 0.9706993699073792, "Finetune/Loss (Raw)": 0.975164532661438, "Finetune/Step": 1320, "Finetune/Step Time": 4.684119826182723} +{"Finetune/Learning Rate": 1.747588495827498e-05, "Finetune/Loss": 0.9705746173858643, "Finetune/Loss (Raw)": 1.0025049448013306, "Finetune/Step": 1321, "Finetune/Step Time": 4.687210341915488} +{"Finetune/Learning Rate": 1.7471746035941007e-05, "Finetune/Loss": 0.9698495268821716, "Finetune/Loss (Raw)": 0.9754663705825806, "Finetune/Step": 1322, "Finetune/Step Time": 4.676669521257281} +{"Finetune/Learning Rate": 1.7467604213968274e-05, "Finetune/Loss": 0.9702430963516235, "Finetune/Loss (Raw)": 1.0491769313812256, "Finetune/Step": 1323, "Finetune/Step Time": 4.6776793617755175} +{"Finetune/Learning Rate": 1.746345949396414e-05, "Finetune/Loss": 0.9704999923706055, "Finetune/Loss (Raw)": 1.0204098224639893, "Finetune/Step": 1324, "Finetune/Step Time": 4.679728478193283} +{"Finetune/Learning Rate": 1.7459311877537085e-05, "Finetune/Loss": 0.9700840711593628, "Finetune/Loss (Raw)": 1.0577573776245117, "Finetune/Step": 1325, "Finetune/Step Time": 4.697357498109341} +{"Finetune/Learning Rate": 1.745516136629672e-05, "Finetune/Loss": 0.9696284532546997, "Finetune/Loss (Raw)": 1.0349864959716797, "Finetune/Step": 1326, "Finetune/Step Time": 4.690056558698416} +{"Finetune/Learning Rate": 1.7451007961853778e-05, "Finetune/Loss": 0.9686189889907837, "Finetune/Loss (Raw)": 0.9131067991256714, "Finetune/Step": 1327, "Finetune/Step Time": 4.7278580497950315} +{"Finetune/Learning Rate": 1.7446851665820118e-05, "Finetune/Loss": 0.9692751169204712, "Finetune/Loss (Raw)": 1.1383531093597412, "Finetune/Step": 1328, "Finetune/Step Time": 4.723924338817596} +{"Finetune/Learning Rate": 1.744269247980871e-05, "Finetune/Loss": 0.9703613519668579, "Finetune/Loss (Raw)": 1.1791536808013916, "Finetune/Step": 1329, "Finetune/Step Time": 4.715575190261006} +{"Finetune/Learning Rate": 1.7438530405433663e-05, "Finetune/Loss": 0.9673805236816406, "Finetune/Loss (Raw)": 0.7816810011863708, "Finetune/Step": 1330, "Finetune/Step Time": 4.709265664219856} +{"Finetune/Learning Rate": 1.7434365444310185e-05, "Finetune/Loss": 0.9668368697166443, "Finetune/Loss (Raw)": 1.0553367137908936, "Finetune/Step": 1331, "Finetune/Step Time": 4.703271329402924} +{"Finetune/Learning Rate": 1.7430197598054622e-05, "Finetune/Loss": 0.9651415944099426, "Finetune/Loss (Raw)": 0.8666527271270752, "Finetune/Step": 1332, "Finetune/Step Time": 4.7062827087938786} +{"Finetune/Learning Rate": 1.7426026868284438e-05, "Finetune/Loss": 0.9651002883911133, "Finetune/Loss (Raw)": 0.9644753932952881, "Finetune/Step": 1333, "Finetune/Step Time": 4.692809714004397} +{"Finetune/Learning Rate": 1.7421853256618204e-05, "Finetune/Loss": 0.9640130996704102, "Finetune/Loss (Raw)": 1.0098484754562378, "Finetune/Step": 1334, "Finetune/Step Time": 4.696759132668376} +{"Finetune/Learning Rate": 1.741767676467562e-05, "Finetune/Loss": 0.9645543098449707, "Finetune/Loss (Raw)": 1.0296790599822998, "Finetune/Step": 1335, "Finetune/Step Time": 4.689092883840203} +{"Finetune/Learning Rate": 1.7413497394077502e-05, "Finetune/Loss": 0.9646254181861877, "Finetune/Loss (Raw)": 1.0268465280532837, "Finetune/Step": 1336, "Finetune/Step Time": 4.68451532907784} +{"Finetune/Learning Rate": 1.7409315146445783e-05, "Finetune/Loss": 0.9652024507522583, "Finetune/Loss (Raw)": 1.1234861612319946, "Finetune/Step": 1337, "Finetune/Step Time": 4.687063524499536} +{"Finetune/Learning Rate": 1.7405130023403513e-05, "Finetune/Loss": 0.9598474502563477, "Finetune/Loss (Raw)": 0.31324347853660583, "Finetune/Step": 1338, "Finetune/Step Time": 4.724053731188178} +{"Finetune/Learning Rate": 1.740094202657485e-05, "Finetune/Loss": 0.9594539999961853, "Finetune/Loss (Raw)": 0.9010331034660339, "Finetune/Step": 1339, "Finetune/Step Time": 4.729790600016713} +{"Finetune/Learning Rate": 1.7396751157585078e-05, "Finetune/Loss": 0.9584787487983704, "Finetune/Loss (Raw)": 0.997146725654602, "Finetune/Step": 1340, "Finetune/Step Time": 4.736081840470433} +{"Finetune/Learning Rate": 1.739255741806059e-05, "Finetune/Loss": 0.9583946466445923, "Finetune/Loss (Raw)": 0.9469882845878601, "Finetune/Step": 1341, "Finetune/Step Time": 4.737417943775654} +{"Finetune/Learning Rate": 1.73883608096289e-05, "Finetune/Loss": 0.9602753520011902, "Finetune/Loss (Raw)": 1.0976948738098145, "Finetune/Step": 1342, "Finetune/Step Time": 4.734555771574378} +{"Finetune/Learning Rate": 1.738416133391862e-05, "Finetune/Loss": 0.9596364498138428, "Finetune/Loss (Raw)": 0.9073213338851929, "Finetune/Step": 1343, "Finetune/Step Time": 4.756922505795956} +{"Finetune/Learning Rate": 1.7379958992559494e-05, "Finetune/Loss": 0.9642990827560425, "Finetune/Loss (Raw)": 0.9489068388938904, "Finetune/Step": 1344, "Finetune/Step Time": 4.727944258600473} +{"Finetune/Learning Rate": 1.7375753787182364e-05, "Finetune/Loss": 0.9638824462890625, "Finetune/Loss (Raw)": 1.0564751625061035, "Finetune/Step": 1345, "Finetune/Step Time": 4.728005385026336} +{"Finetune/Learning Rate": 1.7371545719419188e-05, "Finetune/Loss": 0.9630526304244995, "Finetune/Loss (Raw)": 0.948599636554718, "Finetune/Step": 1346, "Finetune/Step Time": 4.730240328237414} +{"Finetune/Learning Rate": 1.7367334790903034e-05, "Finetune/Loss": 0.969660758972168, "Finetune/Loss (Raw)": 1.1538606882095337, "Finetune/Step": 1347, "Finetune/Step Time": 4.712079796940088} +{"Finetune/Learning Rate": 1.736312100326808e-05, "Finetune/Loss": 0.9653140902519226, "Finetune/Loss (Raw)": 0.4330717921257019, "Finetune/Step": 1348, "Finetune/Step Time": 4.7502958830446005} +{"Finetune/Learning Rate": 1.7358904358149625e-05, "Finetune/Loss": 0.9656197428703308, "Finetune/Loss (Raw)": 1.077597737312317, "Finetune/Step": 1349, "Finetune/Step Time": 4.730231493711472} +{"Finetune/Learning Rate": 1.7354684857184056e-05, "Finetune/Loss": 0.965981125831604, "Finetune/Loss (Raw)": 1.0442652702331543, "Finetune/Step": 1350, "Finetune/Step Time": 4.728775817900896} +{"Finetune/Learning Rate": 1.735046250200888e-05, "Finetune/Loss": 0.9660900831222534, "Finetune/Loss (Raw)": 1.0384379625320435, "Finetune/Step": 1351, "Finetune/Step Time": 4.732564402744174} +{"Finetune/Learning Rate": 1.7346237294262717e-05, "Finetune/Loss": 0.9615811109542847, "Finetune/Loss (Raw)": 0.425655335187912, "Finetune/Step": 1352, "Finetune/Step Time": 4.767494056373835} +{"Finetune/Learning Rate": 1.734200923558528e-05, "Finetune/Loss": 0.9620922207832336, "Finetune/Loss (Raw)": 0.9440353512763977, "Finetune/Step": 1353, "Finetune/Step Time": 4.762189058586955} +{"Finetune/Learning Rate": 1.7337778327617397e-05, "Finetune/Loss": 0.9628452658653259, "Finetune/Loss (Raw)": 0.9915323257446289, "Finetune/Step": 1354, "Finetune/Step Time": 4.75925824418664} +{"Finetune/Learning Rate": 1.733354457200101e-05, "Finetune/Loss": 0.9647405743598938, "Finetune/Loss (Raw)": 1.1107559204101562, "Finetune/Step": 1355, "Finetune/Step Time": 4.752268744632602} +{"Finetune/Learning Rate": 1.7329307970379145e-05, "Finetune/Loss": 0.9648547172546387, "Finetune/Loss (Raw)": 1.0649992227554321, "Finetune/Step": 1356, "Finetune/Step Time": 4.745968801900744} +{"Finetune/Learning Rate": 1.732506852439595e-05, "Finetune/Loss": 0.9646888375282288, "Finetune/Loss (Raw)": 1.0557780265808105, "Finetune/Step": 1357, "Finetune/Step Time": 4.758649563416839} +{"Finetune/Learning Rate": 1.7320826235696672e-05, "Finetune/Loss": 0.9691542983055115, "Finetune/Loss (Raw)": 0.9776631593704224, "Finetune/Step": 1358, "Finetune/Step Time": 4.720169130712748} +{"Finetune/Learning Rate": 1.7316581105927658e-05, "Finetune/Loss": 0.9671356678009033, "Finetune/Loss (Raw)": 0.8544253706932068, "Finetune/Step": 1359, "Finetune/Step Time": 4.734330967068672} +{"Finetune/Learning Rate": 1.731233313673636e-05, "Finetune/Loss": 0.9670122861862183, "Finetune/Loss (Raw)": 0.9980732798576355, "Finetune/Step": 1360, "Finetune/Step Time": 4.731761649250984} +{"Finetune/Learning Rate": 1.7308082329771335e-05, "Finetune/Loss": 0.9673368334770203, "Finetune/Loss (Raw)": 1.0775539875030518, "Finetune/Step": 1361, "Finetune/Step Time": 4.732765285298228} +{"Finetune/Learning Rate": 1.7303828686682233e-05, "Finetune/Loss": 0.9684885740280151, "Finetune/Loss (Raw)": 0.9557872414588928, "Finetune/Step": 1362, "Finetune/Step Time": 4.714871093630791} +{"Finetune/Learning Rate": 1.7299572209119813e-05, "Finetune/Loss": 0.9687293767929077, "Finetune/Loss (Raw)": 0.9809624552726746, "Finetune/Step": 1363, "Finetune/Step Time": 4.715635554865003} +{"Finetune/Learning Rate": 1.7295312898735933e-05, "Finetune/Loss": 0.96753990650177, "Finetune/Loss (Raw)": 0.9530952572822571, "Finetune/Step": 1364, "Finetune/Step Time": 4.749223001301289} +{"Finetune/Learning Rate": 1.7291050757183546e-05, "Finetune/Loss": 0.9665517807006836, "Finetune/Loss (Raw)": 0.8936694264411926, "Finetune/Step": 1365, "Finetune/Step Time": 4.74022881872952} +{"Finetune/Learning Rate": 1.7286785786116707e-05, "Finetune/Loss": 0.9648312330245972, "Finetune/Loss (Raw)": 0.952420711517334, "Finetune/Step": 1366, "Finetune/Step Time": 4.735045524314046} +{"Finetune/Learning Rate": 1.7282517987190567e-05, "Finetune/Loss": 0.9643493890762329, "Finetune/Loss (Raw)": 0.9140792489051819, "Finetune/Step": 1367, "Finetune/Step Time": 4.763462703675032} +{"Finetune/Learning Rate": 1.7278247362061372e-05, "Finetune/Loss": 0.9647655487060547, "Finetune/Loss (Raw)": 1.0291563272476196, "Finetune/Step": 1368, "Finetune/Step Time": 4.763457141816616} +{"Finetune/Learning Rate": 1.7273973912386473e-05, "Finetune/Loss": 0.9659404754638672, "Finetune/Loss (Raw)": 1.0479391813278198, "Finetune/Step": 1369, "Finetune/Step Time": 4.767168672755361} +{"Finetune/Learning Rate": 1.726969763982431e-05, "Finetune/Loss": 0.9649244546890259, "Finetune/Loss (Raw)": 0.8457821607589722, "Finetune/Step": 1370, "Finetune/Step Time": 4.770044948905706} +{"Finetune/Learning Rate": 1.7265418546034424e-05, "Finetune/Loss": 0.9642919898033142, "Finetune/Loss (Raw)": 0.9736735224723816, "Finetune/Step": 1371, "Finetune/Step Time": 4.775709856301546} +{"Finetune/Learning Rate": 1.7261136632677445e-05, "Finetune/Loss": 0.9597111940383911, "Finetune/Loss (Raw)": 0.38414445519447327, "Finetune/Step": 1372, "Finetune/Step Time": 4.802812926471233} +{"Finetune/Learning Rate": 1.7256851901415102e-05, "Finetune/Loss": 0.960837721824646, "Finetune/Loss (Raw)": 1.1158146858215332, "Finetune/Step": 1373, "Finetune/Step Time": 4.803102781996131} +{"Finetune/Learning Rate": 1.725256435391021e-05, "Finetune/Loss": 0.9608441591262817, "Finetune/Loss (Raw)": 0.9285354614257812, "Finetune/Step": 1374, "Finetune/Step Time": 4.794546959921718} +{"Finetune/Learning Rate": 1.7248273991826692e-05, "Finetune/Loss": 0.9606207013130188, "Finetune/Loss (Raw)": 1.0396512746810913, "Finetune/Step": 1375, "Finetune/Step Time": 4.792156543582678} +{"Finetune/Learning Rate": 1.7243980816829543e-05, "Finetune/Loss": 0.9567573070526123, "Finetune/Loss (Raw)": 0.3203967213630676, "Finetune/Step": 1376, "Finetune/Step Time": 4.819510651752353} +{"Finetune/Learning Rate": 1.723968483058487e-05, "Finetune/Loss": 0.9579054713249207, "Finetune/Loss (Raw)": 1.0618419647216797, "Finetune/Step": 1377, "Finetune/Step Time": 4.827508157119155} +{"Finetune/Learning Rate": 1.7235386034759853e-05, "Finetune/Loss": 0.9587177038192749, "Finetune/Loss (Raw)": 1.1385124921798706, "Finetune/Step": 1378, "Finetune/Step Time": 4.84590614028275} +{"Finetune/Learning Rate": 1.723108443102278e-05, "Finetune/Loss": 0.9563278555870056, "Finetune/Loss (Raw)": 0.7188618779182434, "Finetune/Step": 1379, "Finetune/Step Time": 4.862451868131757} +{"Finetune/Learning Rate": 1.722678002104301e-05, "Finetune/Loss": 0.9565800428390503, "Finetune/Loss (Raw)": 0.9267758131027222, "Finetune/Step": 1380, "Finetune/Step Time": 4.862554607912898} +{"Finetune/Learning Rate": 1.722247280649101e-05, "Finetune/Loss": 0.9574069976806641, "Finetune/Loss (Raw)": 1.035754919052124, "Finetune/Step": 1381, "Finetune/Step Time": 4.847695410251617} +{"Finetune/Learning Rate": 1.7218162789038312e-05, "Finetune/Loss": 0.9573651552200317, "Finetune/Loss (Raw)": 0.9753829836845398, "Finetune/Step": 1382, "Finetune/Step Time": 4.847215499728918} +{"Finetune/Learning Rate": 1.7213849970357565e-05, "Finetune/Loss": 0.9558622241020203, "Finetune/Loss (Raw)": 0.9073616862297058, "Finetune/Step": 1383, "Finetune/Step Time": 4.851137351244688} +{"Finetune/Learning Rate": 1.7209534352122484e-05, "Finetune/Loss": 0.9554053544998169, "Finetune/Loss (Raw)": 0.9605173468589783, "Finetune/Step": 1384, "Finetune/Step Time": 4.85403560847044} +{"Finetune/Learning Rate": 1.720521593600787e-05, "Finetune/Loss": 0.9579406976699829, "Finetune/Loss (Raw)": 1.0216481685638428, "Finetune/Step": 1385, "Finetune/Step Time": 4.829914499074221} +{"Finetune/Learning Rate": 1.7200894723689626e-05, "Finetune/Loss": 0.9577081203460693, "Finetune/Loss (Raw)": 0.9230526685714722, "Finetune/Step": 1386, "Finetune/Step Time": 4.830435656011105} +{"Finetune/Learning Rate": 1.7196570716844724e-05, "Finetune/Loss": 0.9574356079101562, "Finetune/Loss (Raw)": 0.863207995891571, "Finetune/Step": 1387, "Finetune/Step Time": 4.824425205588341} +{"Finetune/Learning Rate": 1.7192243917151228e-05, "Finetune/Loss": 0.9562121629714966, "Finetune/Loss (Raw)": 1.0175119638442993, "Finetune/Step": 1388, "Finetune/Step Time": 4.820100674405694} +{"Finetune/Learning Rate": 1.7187914326288286e-05, "Finetune/Loss": 0.9574265480041504, "Finetune/Loss (Raw)": 1.0032819509506226, "Finetune/Step": 1389, "Finetune/Step Time": 4.8206054251641035} +{"Finetune/Learning Rate": 1.718358194593612e-05, "Finetune/Loss": 0.958716094493866, "Finetune/Loss (Raw)": 1.1556859016418457, "Finetune/Step": 1390, "Finetune/Step Time": 4.828506791964173} +{"Finetune/Learning Rate": 1.7179246777776053e-05, "Finetune/Loss": 0.957302451133728, "Finetune/Loss (Raw)": 0.8723152875900269, "Finetune/Step": 1391, "Finetune/Step Time": 4.843999911099672} +{"Finetune/Learning Rate": 1.7174908823490475e-05, "Finetune/Loss": 0.9565401077270508, "Finetune/Loss (Raw)": 0.8981513381004333, "Finetune/Step": 1392, "Finetune/Step Time": 4.877287974581122} +{"Finetune/Learning Rate": 1.717056808476286e-05, "Finetune/Loss": 0.9610834121704102, "Finetune/Loss (Raw)": 0.9621950387954712, "Finetune/Step": 1393, "Finetune/Step Time": 4.880311539396644} +{"Finetune/Learning Rate": 1.716622456327776e-05, "Finetune/Loss": 0.9616333842277527, "Finetune/Loss (Raw)": 1.0305622816085815, "Finetune/Step": 1394, "Finetune/Step Time": 4.866739142686129} +{"Finetune/Learning Rate": 1.7161878260720826e-05, "Finetune/Loss": 0.960956871509552, "Finetune/Loss (Raw)": 0.9575556516647339, "Finetune/Step": 1395, "Finetune/Step Time": 4.869528790935874} +{"Finetune/Learning Rate": 1.7157529178778757e-05, "Finetune/Loss": 0.9582892656326294, "Finetune/Loss (Raw)": 0.7269156575202942, "Finetune/Step": 1396, "Finetune/Step Time": 4.877502782270312} +{"Finetune/Learning Rate": 1.7153177319139352e-05, "Finetune/Loss": 0.9588609337806702, "Finetune/Loss (Raw)": 1.027830958366394, "Finetune/Step": 1397, "Finetune/Step Time": 4.901117445901036} +{"Finetune/Learning Rate": 1.7148822683491483e-05, "Finetune/Loss": 0.9578472971916199, "Finetune/Loss (Raw)": 0.996455192565918, "Finetune/Step": 1398, "Finetune/Step Time": 4.890667013823986} +{"Finetune/Learning Rate": 1.7144465273525105e-05, "Finetune/Loss": 0.9556190967559814, "Finetune/Loss (Raw)": 0.8152696490287781, "Finetune/Step": 1399, "Finetune/Step Time": 4.896991206333041} +{"Finetune/Learning Rate": 1.7140105090931232e-05, "Finetune/Loss": 0.957378089427948, "Finetune/Loss (Raw)": 1.0433118343353271, "Finetune/Step": 1400, "Finetune/Step Time": 4.891468679532409} +{"Finetune/Learning Rate": 1.7135742137401976e-05, "Finetune/Loss": 0.957165002822876, "Finetune/Loss (Raw)": 0.9081647992134094, "Finetune/Step": 1401, "Finetune/Step Time": 4.8898536171764135} +{"Finetune/Learning Rate": 1.7131376414630506e-05, "Finetune/Loss": 0.9534528255462646, "Finetune/Loss (Raw)": 0.3958585262298584, "Finetune/Step": 1402, "Finetune/Step Time": 4.927113911136985} +{"Finetune/Learning Rate": 1.712700792431108e-05, "Finetune/Loss": 0.9516105651855469, "Finetune/Loss (Raw)": 0.9310362935066223, "Finetune/Step": 1403, "Finetune/Step Time": 4.927870497107506} +{"Finetune/Learning Rate": 1.7122636668139022e-05, "Finetune/Loss": 0.9517823457717896, "Finetune/Loss (Raw)": 0.9700779318809509, "Finetune/Step": 1404, "Finetune/Step Time": 4.924634084105492} +{"Finetune/Learning Rate": 1.711826264781073e-05, "Finetune/Loss": 0.9507913589477539, "Finetune/Loss (Raw)": 0.8351792097091675, "Finetune/Step": 1405, "Finetune/Step Time": 4.922633573412895} +{"Finetune/Learning Rate": 1.711388586502368e-05, "Finetune/Loss": 0.9525296688079834, "Finetune/Loss (Raw)": 0.9639127254486084, "Finetune/Step": 1406, "Finetune/Step Time": 4.915842197835445} +{"Finetune/Learning Rate": 1.710950632147641e-05, "Finetune/Loss": 0.9477014541625977, "Finetune/Loss (Raw)": 0.3749617338180542, "Finetune/Step": 1407, "Finetune/Step Time": 4.94534863345325} +{"Finetune/Learning Rate": 1.710512401886854e-05, "Finetune/Loss": 0.9453842043876648, "Finetune/Loss (Raw)": 0.8602437376976013, "Finetune/Step": 1408, "Finetune/Step Time": 4.948245057836175} +{"Finetune/Learning Rate": 1.7100738958900752e-05, "Finetune/Loss": 0.94570392370224, "Finetune/Loss (Raw)": 0.9928148984909058, "Finetune/Step": 1409, "Finetune/Step Time": 4.945437241345644} +{"Finetune/Learning Rate": 1.7096351143274804e-05, "Finetune/Loss": 0.9450305700302124, "Finetune/Loss (Raw)": 0.9050739407539368, "Finetune/Step": 1410, "Finetune/Step Time": 4.974570652469993} +{"Finetune/Learning Rate": 1.7091960573693523e-05, "Finetune/Loss": 0.9462735056877136, "Finetune/Loss (Raw)": 1.0380091667175293, "Finetune/Step": 1411, "Finetune/Step Time": 4.963049348443747} +{"Finetune/Learning Rate": 1.70875672518608e-05, "Finetune/Loss": 0.9445042610168457, "Finetune/Loss (Raw)": 0.7429362535476685, "Finetune/Step": 1412, "Finetune/Step Time": 4.963000405579805} +{"Finetune/Learning Rate": 1.7083171179481603e-05, "Finetune/Loss": 0.9442188739776611, "Finetune/Loss (Raw)": 1.0184032917022705, "Finetune/Step": 1413, "Finetune/Step Time": 4.9668321795761585} +{"Finetune/Learning Rate": 1.7078772358261957e-05, "Finetune/Loss": 0.9452508091926575, "Finetune/Loss (Raw)": 1.092462420463562, "Finetune/Step": 1414, "Finetune/Step Time": 4.956135788932443} +{"Finetune/Learning Rate": 1.707437078990896e-05, "Finetune/Loss": 0.9451360702514648, "Finetune/Loss (Raw)": 0.9669342041015625, "Finetune/Step": 1415, "Finetune/Step Time": 4.961047822609544} +{"Finetune/Learning Rate": 1.7069966476130773e-05, "Finetune/Loss": 0.9464758634567261, "Finetune/Loss (Raw)": 1.0804181098937988, "Finetune/Step": 1416, "Finetune/Step Time": 4.964890379458666} +{"Finetune/Learning Rate": 1.706555941863663e-05, "Finetune/Loss": 0.94647216796875, "Finetune/Loss (Raw)": 0.9097291231155396, "Finetune/Step": 1417, "Finetune/Step Time": 4.9640163984149694} +{"Finetune/Learning Rate": 1.7061149619136818e-05, "Finetune/Loss": 0.9444947838783264, "Finetune/Loss (Raw)": 0.8510053753852844, "Finetune/Step": 1418, "Finetune/Step Time": 4.981748281046748} +{"Finetune/Learning Rate": 1.7056737079342694e-05, "Finetune/Loss": 0.9435724020004272, "Finetune/Loss (Raw)": 0.8725476264953613, "Finetune/Step": 1419, "Finetune/Step Time": 4.958822259679437} +{"Finetune/Learning Rate": 1.7052321800966684e-05, "Finetune/Loss": 0.9437491297721863, "Finetune/Loss (Raw)": 0.9892428517341614, "Finetune/Step": 1420, "Finetune/Step Time": 4.953892124816775} +{"Finetune/Learning Rate": 1.7047903785722268e-05, "Finetune/Loss": 0.9434193968772888, "Finetune/Loss (Raw)": 0.9079474806785583, "Finetune/Step": 1421, "Finetune/Step Time": 4.9488551914691925} +{"Finetune/Learning Rate": 1.704348303532399e-05, "Finetune/Loss": 0.9428902864456177, "Finetune/Loss (Raw)": 0.9245588183403015, "Finetune/Step": 1422, "Finetune/Step Time": 4.945649517700076} +{"Finetune/Learning Rate": 1.7039059551487457e-05, "Finetune/Loss": 0.9436509609222412, "Finetune/Loss (Raw)": 1.0251399278640747, "Finetune/Step": 1423, "Finetune/Step Time": 4.935823412612081} +{"Finetune/Learning Rate": 1.703463333592934e-05, "Finetune/Loss": 0.9427858591079712, "Finetune/Loss (Raw)": 0.8495253920555115, "Finetune/Step": 1424, "Finetune/Step Time": 4.93049562163651} +{"Finetune/Learning Rate": 1.7030204390367368e-05, "Finetune/Loss": 0.9427202939987183, "Finetune/Loss (Raw)": 0.9821776151657104, "Finetune/Step": 1425, "Finetune/Step Time": 4.924309337511659} +{"Finetune/Learning Rate": 1.7025772716520324e-05, "Finetune/Loss": 0.9429919123649597, "Finetune/Loss (Raw)": 0.8824047446250916, "Finetune/Step": 1426, "Finetune/Step Time": 4.931562611833215} +{"Finetune/Learning Rate": 1.702133831610805e-05, "Finetune/Loss": 0.9445820450782776, "Finetune/Loss (Raw)": 0.9449372887611389, "Finetune/Step": 1427, "Finetune/Step Time": 4.9256565272808075} +{"Finetune/Learning Rate": 1.701690119085146e-05, "Finetune/Loss": 0.9448763132095337, "Finetune/Loss (Raw)": 0.9307829737663269, "Finetune/Step": 1428, "Finetune/Step Time": 4.92650874517858} +{"Finetune/Learning Rate": 1.701246134247251e-05, "Finetune/Loss": 0.9435179233551025, "Finetune/Loss (Raw)": 0.8479777574539185, "Finetune/Step": 1429, "Finetune/Step Time": 4.9222422037273645} +{"Finetune/Learning Rate": 1.700801877269422e-05, "Finetune/Loss": 0.9435946941375732, "Finetune/Loss (Raw)": 0.9506294131278992, "Finetune/Step": 1430, "Finetune/Step Time": 4.9082008469849825} +{"Finetune/Learning Rate": 1.7003573483240666e-05, "Finetune/Loss": 0.9429423213005066, "Finetune/Loss (Raw)": 0.8901378512382507, "Finetune/Step": 1431, "Finetune/Step Time": 4.919449958950281} +{"Finetune/Learning Rate": 1.6999125475836973e-05, "Finetune/Loss": 0.9425045251846313, "Finetune/Loss (Raw)": 0.9451722502708435, "Finetune/Step": 1432, "Finetune/Step Time": 4.922351032495499} +{"Finetune/Learning Rate": 1.6994674752209334e-05, "Finetune/Loss": 0.9427268505096436, "Finetune/Loss (Raw)": 0.9914736747741699, "Finetune/Step": 1433, "Finetune/Step Time": 4.923941260203719} +{"Finetune/Learning Rate": 1.6990221314084976e-05, "Finetune/Loss": 0.9378992319107056, "Finetune/Loss (Raw)": 0.3777313232421875, "Finetune/Step": 1434, "Finetune/Step Time": 4.937238873913884} +{"Finetune/Learning Rate": 1.6985765163192204e-05, "Finetune/Loss": 0.9373895525932312, "Finetune/Loss (Raw)": 1.012880802154541, "Finetune/Step": 1435, "Finetune/Step Time": 4.944040486589074} +{"Finetune/Learning Rate": 1.698130630126036e-05, "Finetune/Loss": 0.935227632522583, "Finetune/Loss (Raw)": 0.7962382435798645, "Finetune/Step": 1436, "Finetune/Step Time": 4.960059981793165} +{"Finetune/Learning Rate": 1.697684473001983e-05, "Finetune/Loss": 0.9358218312263489, "Finetune/Loss (Raw)": 1.036274790763855, "Finetune/Step": 1437, "Finetune/Step Time": 4.942277496680617} +{"Finetune/Learning Rate": 1.6972380451202083e-05, "Finetune/Loss": 0.9362685680389404, "Finetune/Loss (Raw)": 0.9663287401199341, "Finetune/Step": 1438, "Finetune/Step Time": 4.942995592951775} +{"Finetune/Learning Rate": 1.69679134665396e-05, "Finetune/Loss": 0.9364888668060303, "Finetune/Loss (Raw)": 1.032076120376587, "Finetune/Step": 1439, "Finetune/Step Time": 4.939652070403099} +{"Finetune/Learning Rate": 1.696344377776594e-05, "Finetune/Loss": 0.9360090494155884, "Finetune/Loss (Raw)": 0.7023874521255493, "Finetune/Step": 1440, "Finetune/Step Time": 4.949364792555571} +{"Finetune/Learning Rate": 1.69589713866157e-05, "Finetune/Loss": 0.9354026317596436, "Finetune/Loss (Raw)": 0.9359285235404968, "Finetune/Step": 1441, "Finetune/Step Time": 4.945622788742185} +{"Finetune/Learning Rate": 1.6954496294824526e-05, "Finetune/Loss": 0.9352332353591919, "Finetune/Loss (Raw)": 0.9756669402122498, "Finetune/Step": 1442, "Finetune/Step Time": 4.951042450964451} +{"Finetune/Learning Rate": 1.6950018504129112e-05, "Finetune/Loss": 0.9345871210098267, "Finetune/Loss (Raw)": 0.9055300354957581, "Finetune/Step": 1443, "Finetune/Step Time": 4.959611088037491} +{"Finetune/Learning Rate": 1.6945538016267206e-05, "Finetune/Loss": 0.9325947165489197, "Finetune/Loss (Raw)": 0.906173586845398, "Finetune/Step": 1444, "Finetune/Step Time": 4.956762012094259} +{"Finetune/Learning Rate": 1.6941054832977598e-05, "Finetune/Loss": 0.9314411878585815, "Finetune/Loss (Raw)": 0.9108577370643616, "Finetune/Step": 1445, "Finetune/Step Time": 4.959302302449942} +{"Finetune/Learning Rate": 1.6936568956000123e-05, "Finetune/Loss": 0.9306148290634155, "Finetune/Loss (Raw)": 0.8502972722053528, "Finetune/Step": 1446, "Finetune/Step Time": 4.974751323461533} +{"Finetune/Learning Rate": 1.693208038707566e-05, "Finetune/Loss": 0.9303733110427856, "Finetune/Loss (Raw)": 0.963257372379303, "Finetune/Step": 1447, "Finetune/Step Time": 4.979298077523708} +{"Finetune/Learning Rate": 1.6927589127946136e-05, "Finetune/Loss": 0.9309194087982178, "Finetune/Loss (Raw)": 1.0450700521469116, "Finetune/Step": 1448, "Finetune/Step Time": 4.97026327252388} +{"Finetune/Learning Rate": 1.692309518035452e-05, "Finetune/Loss": 0.9296951293945312, "Finetune/Loss (Raw)": 0.8457934260368347, "Finetune/Step": 1449, "Finetune/Step Time": 4.972611427307129} +{"Finetune/Learning Rate": 1.691859854604483e-05, "Finetune/Loss": 0.9287526607513428, "Finetune/Loss (Raw)": 0.8548384308815002, "Finetune/Step": 1450, "Finetune/Step Time": 5.0084397830069065} +{"Finetune/Learning Rate": 1.6914099226762116e-05, "Finetune/Loss": 0.9276574850082397, "Finetune/Loss (Raw)": 0.9089837670326233, "Finetune/Step": 1451, "Finetune/Step Time": 5.011560266837478} +{"Finetune/Learning Rate": 1.6909597224252478e-05, "Finetune/Loss": 0.927855372428894, "Finetune/Loss (Raw)": 1.0457420349121094, "Finetune/Step": 1452, "Finetune/Step Time": 5.0036345068365335} +{"Finetune/Learning Rate": 1.6905092540263055e-05, "Finetune/Loss": 0.9266808032989502, "Finetune/Loss (Raw)": 0.907425582408905, "Finetune/Step": 1453, "Finetune/Step Time": 4.998420976102352} +{"Finetune/Learning Rate": 1.690058517654203e-05, "Finetune/Loss": 0.9278988838195801, "Finetune/Loss (Raw)": 1.1908926963806152, "Finetune/Step": 1454, "Finetune/Step Time": 5.0010235495865345} +{"Finetune/Learning Rate": 1.689607513483862e-05, "Finetune/Loss": 0.9286212921142578, "Finetune/Loss (Raw)": 1.0055725574493408, "Finetune/Step": 1455, "Finetune/Step Time": 4.968164496123791} +{"Finetune/Learning Rate": 1.6891562416903083e-05, "Finetune/Loss": 0.9269865155220032, "Finetune/Loss (Raw)": 0.929107666015625, "Finetune/Step": 1456, "Finetune/Step Time": 4.969627372920513} +{"Finetune/Learning Rate": 1.688704702448672e-05, "Finetune/Loss": 0.9207242131233215, "Finetune/Loss (Raw)": 0.3775826394557953, "Finetune/Step": 1457, "Finetune/Step Time": 5.0035192016512156} +{"Finetune/Learning Rate": 1.6882528959341865e-05, "Finetune/Loss": 0.9219512939453125, "Finetune/Loss (Raw)": 0.9387402534484863, "Finetune/Step": 1458, "Finetune/Step Time": 5.00053708255291} +{"Finetune/Learning Rate": 1.6878008223221895e-05, "Finetune/Loss": 0.9221486449241638, "Finetune/Loss (Raw)": 1.080597162246704, "Finetune/Step": 1459, "Finetune/Step Time": 5.008907154202461} +{"Finetune/Learning Rate": 1.687348481788121e-05, "Finetune/Loss": 0.9239124059677124, "Finetune/Loss (Raw)": 1.092416524887085, "Finetune/Step": 1460, "Finetune/Step Time": 4.998364539816976} +{"Finetune/Learning Rate": 1.6868958745075262e-05, "Finetune/Loss": 0.9250792264938354, "Finetune/Loss (Raw)": 1.1138249635696411, "Finetune/Step": 1461, "Finetune/Step Time": 4.991548333317041} +{"Finetune/Learning Rate": 1.6864430006560527e-05, "Finetune/Loss": 0.9252886772155762, "Finetune/Loss (Raw)": 1.036655306816101, "Finetune/Step": 1462, "Finetune/Step Time": 4.993583424016833} +{"Finetune/Learning Rate": 1.685989860409453e-05, "Finetune/Loss": 0.9253453016281128, "Finetune/Loss (Raw)": 1.0369316339492798, "Finetune/Step": 1463, "Finetune/Step Time": 4.994309764355421} +{"Finetune/Learning Rate": 1.6855364539435805e-05, "Finetune/Loss": 0.9241915941238403, "Finetune/Loss (Raw)": 0.8791732788085938, "Finetune/Step": 1464, "Finetune/Step Time": 4.999254709109664} +{"Finetune/Learning Rate": 1.685082781434395e-05, "Finetune/Loss": 0.9229358434677124, "Finetune/Loss (Raw)": 0.9627513885498047, "Finetune/Step": 1465, "Finetune/Step Time": 4.99214605614543} +{"Finetune/Learning Rate": 1.6846288430579565e-05, "Finetune/Loss": 0.9279446005821228, "Finetune/Loss (Raw)": 0.9543634653091431, "Finetune/Step": 1466, "Finetune/Step Time": 4.961922738701105} +{"Finetune/Learning Rate": 1.6841746389904306e-05, "Finetune/Loss": 0.9282358884811401, "Finetune/Loss (Raw)": 0.9383145570755005, "Finetune/Step": 1467, "Finetune/Step Time": 4.9797663651406765} +{"Finetune/Learning Rate": 1.683720169408085e-05, "Finetune/Loss": 0.9284193515777588, "Finetune/Loss (Raw)": 1.020631194114685, "Finetune/Step": 1468, "Finetune/Step Time": 4.972221108153462} +{"Finetune/Learning Rate": 1.6832654344872893e-05, "Finetune/Loss": 0.9240584373474121, "Finetune/Loss (Raw)": 0.3887953758239746, "Finetune/Step": 1469, "Finetune/Step Time": 5.01123321056366} +{"Finetune/Learning Rate": 1.6828104344045186e-05, "Finetune/Loss": 0.9230706095695496, "Finetune/Loss (Raw)": 0.971244752407074, "Finetune/Step": 1470, "Finetune/Step Time": 4.998875143006444} +{"Finetune/Learning Rate": 1.682355169336349e-05, "Finetune/Loss": 0.9236003160476685, "Finetune/Loss (Raw)": 0.9751254916191101, "Finetune/Step": 1471, "Finetune/Step Time": 4.972793731838465} +{"Finetune/Learning Rate": 1.6818996394594603e-05, "Finetune/Loss": 0.922840416431427, "Finetune/Loss (Raw)": 0.8516440987586975, "Finetune/Step": 1472, "Finetune/Step Time": 4.965486725792289} +{"Finetune/Learning Rate": 1.681443844950634e-05, "Finetune/Loss": 0.9200557470321655, "Finetune/Loss (Raw)": 0.7000387907028198, "Finetune/Step": 1473, "Finetune/Step Time": 4.97175969183445} +{"Finetune/Learning Rate": 1.6809877859867557e-05, "Finetune/Loss": 0.9213120341300964, "Finetune/Loss (Raw)": 1.109403371810913, "Finetune/Step": 1474, "Finetune/Step Time": 4.966512642800808} +{"Finetune/Learning Rate": 1.680531462744813e-05, "Finetune/Loss": 0.9181889295578003, "Finetune/Loss (Raw)": 0.7541066408157349, "Finetune/Step": 1475, "Finetune/Step Time": 4.959262318909168} +{"Finetune/Learning Rate": 1.680074875401896e-05, "Finetune/Loss": 0.9225069284439087, "Finetune/Loss (Raw)": 0.9857771992683411, "Finetune/Step": 1476, "Finetune/Step Time": 4.926637610420585} +{"Finetune/Learning Rate": 1.679618024135197e-05, "Finetune/Loss": 0.9165606498718262, "Finetune/Loss (Raw)": 0.3164714574813843, "Finetune/Step": 1477, "Finetune/Step Time": 4.967961436137557} +{"Finetune/Learning Rate": 1.679160909122011e-05, "Finetune/Loss": 0.9167056679725647, "Finetune/Loss (Raw)": 1.062821388244629, "Finetune/Step": 1478, "Finetune/Step Time": 4.959576269611716} +{"Finetune/Learning Rate": 1.6787035305397363e-05, "Finetune/Loss": 0.9155261516571045, "Finetune/Loss (Raw)": 0.8874602317810059, "Finetune/Step": 1479, "Finetune/Step Time": 4.974076092243195} +{"Finetune/Learning Rate": 1.6782458885658716e-05, "Finetune/Loss": 0.9197050929069519, "Finetune/Loss (Raw)": 0.9605581164360046, "Finetune/Step": 1480, "Finetune/Step Time": 4.933152997866273} +{"Finetune/Learning Rate": 1.6777879833780192e-05, "Finetune/Loss": 0.9199947118759155, "Finetune/Loss (Raw)": 0.981113076210022, "Finetune/Step": 1481, "Finetune/Step Time": 4.938942462205887} +{"Finetune/Learning Rate": 1.677329815153883e-05, "Finetune/Loss": 0.9196224212646484, "Finetune/Loss (Raw)": 0.9438828825950623, "Finetune/Step": 1482, "Finetune/Step Time": 4.9360225312411785} +{"Finetune/Learning Rate": 1.6768713840712692e-05, "Finetune/Loss": 0.9179004430770874, "Finetune/Loss (Raw)": 0.8903354406356812, "Finetune/Step": 1483, "Finetune/Step Time": 4.955230005085468} +{"Finetune/Learning Rate": 1.6764126903080865e-05, "Finetune/Loss": 0.9174933433532715, "Finetune/Loss (Raw)": 1.012895107269287, "Finetune/Step": 1484, "Finetune/Step Time": 4.951118782162666} +{"Finetune/Learning Rate": 1.675953734042344e-05, "Finetune/Loss": 0.9163108468055725, "Finetune/Loss (Raw)": 0.9044121503829956, "Finetune/Step": 1485, "Finetune/Step Time": 4.938817946240306} +{"Finetune/Learning Rate": 1.6754945154521548e-05, "Finetune/Loss": 0.9110990762710571, "Finetune/Loss (Raw)": 0.310556560754776, "Finetune/Step": 1486, "Finetune/Step Time": 4.973672786727548} +{"Finetune/Learning Rate": 1.6750350347157315e-05, "Finetune/Loss": 0.9129471182823181, "Finetune/Loss (Raw)": 1.090976357460022, "Finetune/Step": 1487, "Finetune/Step Time": 4.968962932005525} +{"Finetune/Learning Rate": 1.6745752920113907e-05, "Finetune/Loss": 0.9143784642219543, "Finetune/Loss (Raw)": 1.1812881231307983, "Finetune/Step": 1488, "Finetune/Step Time": 4.972129242494702} +{"Finetune/Learning Rate": 1.6741152875175485e-05, "Finetune/Loss": 0.9122449159622192, "Finetune/Loss (Raw)": 0.8044597506523132, "Finetune/Step": 1489, "Finetune/Step Time": 4.986610548570752} +{"Finetune/Learning Rate": 1.6736550214127247e-05, "Finetune/Loss": 0.9120050668716431, "Finetune/Loss (Raw)": 0.9250843524932861, "Finetune/Step": 1490, "Finetune/Step Time": 5.008451018482447} +{"Finetune/Learning Rate": 1.673194493875539e-05, "Finetune/Loss": 0.9106712341308594, "Finetune/Loss (Raw)": 0.8102337718009949, "Finetune/Step": 1491, "Finetune/Step Time": 5.022271890193224} +{"Finetune/Learning Rate": 1.672733705084713e-05, "Finetune/Loss": 0.9115512371063232, "Finetune/Loss (Raw)": 1.0657399892807007, "Finetune/Step": 1492, "Finetune/Step Time": 4.989463448524475} +{"Finetune/Learning Rate": 1.67227265521907e-05, "Finetune/Loss": 0.9071838855743408, "Finetune/Loss (Raw)": 0.33464494347572327, "Finetune/Step": 1493, "Finetune/Step Time": 5.020613867789507} +{"Finetune/Learning Rate": 1.671811344457535e-05, "Finetune/Loss": 0.9070068597793579, "Finetune/Loss (Raw)": 0.9297550916671753, "Finetune/Step": 1494, "Finetune/Step Time": 5.0244180504232645} +{"Finetune/Learning Rate": 1.671349772979133e-05, "Finetune/Loss": 0.907849133014679, "Finetune/Loss (Raw)": 1.021891713142395, "Finetune/Step": 1495, "Finetune/Step Time": 4.9951025899499655} +{"Finetune/Learning Rate": 1.6708879409629908e-05, "Finetune/Loss": 0.9075025320053101, "Finetune/Loss (Raw)": 0.9847902655601501, "Finetune/Step": 1496, "Finetune/Step Time": 5.000912845134735} +{"Finetune/Learning Rate": 1.670425848588337e-05, "Finetune/Loss": 0.9058836102485657, "Finetune/Loss (Raw)": 0.8407205939292908, "Finetune/Step": 1497, "Finetune/Step Time": 4.994846248999238} +{"Finetune/Learning Rate": 1.6699634960345005e-05, "Finetune/Loss": 0.907582700252533, "Finetune/Loss (Raw)": 1.0632681846618652, "Finetune/Step": 1498, "Finetune/Step Time": 4.9976061806082726} +{"Finetune/Learning Rate": 1.6695008834809107e-05, "Finetune/Loss": 0.9033328294754028, "Finetune/Loss (Raw)": 0.4296826422214508, "Finetune/Step": 1499, "Finetune/Step Time": 5.022093757987022} +{"Finetune/Learning Rate": 1.669038011107099e-05, "Finetune/Loss": 0.9031058549880981, "Finetune/Loss (Raw)": 0.35510164499282837, "Finetune/Step": 1500, "Finetune/Step Time": 5.021130453795195} +{"Finetune/Learning Rate": 1.6685748790926975e-05, "Finetune/Loss": 0.9023990631103516, "Finetune/Loss (Raw)": 1.0253411531448364, "Finetune/Step": 1501, "Finetune/Step Time": 5.0589329320937395} +{"Finetune/Learning Rate": 1.6681114876174377e-05, "Finetune/Loss": 0.903668224811554, "Finetune/Loss (Raw)": 1.0909894704818726, "Finetune/Step": 1502, "Finetune/Step Time": 5.051080649718642} +{"Finetune/Learning Rate": 1.6676478368611536e-05, "Finetune/Loss": 0.9021843671798706, "Finetune/Loss (Raw)": 0.849716067314148, "Finetune/Step": 1503, "Finetune/Step Time": 5.054182797670364} +{"Finetune/Learning Rate": 1.667183927003779e-05, "Finetune/Loss": 0.9073470234870911, "Finetune/Loss (Raw)": 0.981221616268158, "Finetune/Step": 1504, "Finetune/Step Time": 5.036055674776435} +{"Finetune/Learning Rate": 1.6667197582253474e-05, "Finetune/Loss": 0.9066317677497864, "Finetune/Loss (Raw)": 0.9702891111373901, "Finetune/Step": 1505, "Finetune/Step Time": 5.027906712144613} +{"Finetune/Learning Rate": 1.6662553307059947e-05, "Finetune/Loss": 0.9045480489730835, "Finetune/Loss (Raw)": 0.8717992305755615, "Finetune/Step": 1506, "Finetune/Step Time": 5.022522576153278} +{"Finetune/Learning Rate": 1.6657906446259554e-05, "Finetune/Loss": 0.9071439504623413, "Finetune/Loss (Raw)": 1.051133394241333, "Finetune/Step": 1507, "Finetune/Step Time": 5.001040177419782} +{"Finetune/Learning Rate": 1.6653257001655652e-05, "Finetune/Loss": 0.9077510833740234, "Finetune/Loss (Raw)": 1.0044822692871094, "Finetune/Step": 1508, "Finetune/Step Time": 5.004746958613396} +{"Finetune/Learning Rate": 1.664860497505261e-05, "Finetune/Loss": 0.9074971675872803, "Finetune/Loss (Raw)": 1.0032597780227661, "Finetune/Step": 1509, "Finetune/Step Time": 4.998601607978344} +{"Finetune/Learning Rate": 1.664395036825577e-05, "Finetune/Loss": 0.9082045555114746, "Finetune/Loss (Raw)": 1.0659316778182983, "Finetune/Step": 1510, "Finetune/Step Time": 4.997767519205809} +{"Finetune/Learning Rate": 1.6639293183071514e-05, "Finetune/Loss": 0.909716010093689, "Finetune/Loss (Raw)": 1.1008206605911255, "Finetune/Step": 1511, "Finetune/Step Time": 4.997130587697029} +{"Finetune/Learning Rate": 1.6634633421307187e-05, "Finetune/Loss": 0.9091463088989258, "Finetune/Loss (Raw)": 0.8875932097434998, "Finetune/Step": 1512, "Finetune/Step Time": 5.0007843393832445} +{"Finetune/Learning Rate": 1.6629971084771165e-05, "Finetune/Loss": 0.9100512266159058, "Finetune/Loss (Raw)": 1.1374858617782593, "Finetune/Step": 1513, "Finetune/Step Time": 5.002948930487037} +{"Finetune/Learning Rate": 1.66253061752728e-05, "Finetune/Loss": 0.9101734161376953, "Finetune/Loss (Raw)": 0.938688337802887, "Finetune/Step": 1514, "Finetune/Step Time": 5.012687813490629} +{"Finetune/Learning Rate": 1.662063869462246e-05, "Finetune/Loss": 0.9108390808105469, "Finetune/Loss (Raw)": 0.9484153389930725, "Finetune/Step": 1515, "Finetune/Step Time": 5.011223468929529} +{"Finetune/Learning Rate": 1.6615968644631498e-05, "Finetune/Loss": 0.910285234451294, "Finetune/Loss (Raw)": 0.9466175436973572, "Finetune/Step": 1516, "Finetune/Step Time": 5.0134341809898615} +{"Finetune/Learning Rate": 1.661129602711227e-05, "Finetune/Loss": 0.9091609120368958, "Finetune/Loss (Raw)": 0.8593714237213135, "Finetune/Step": 1517, "Finetune/Step Time": 5.0153388772159815} +{"Finetune/Learning Rate": 1.6606620843878128e-05, "Finetune/Loss": 0.9085033535957336, "Finetune/Loss (Raw)": 1.0715172290802002, "Finetune/Step": 1518, "Finetune/Step Time": 5.009215200319886} +{"Finetune/Learning Rate": 1.6601943096743426e-05, "Finetune/Loss": 0.9081871509552002, "Finetune/Loss (Raw)": 0.8318400979042053, "Finetune/Step": 1519, "Finetune/Step Time": 4.994802109897137} +{"Finetune/Learning Rate": 1.6597262787523493e-05, "Finetune/Loss": 0.904254674911499, "Finetune/Loss (Raw)": 0.394793301820755, "Finetune/Step": 1520, "Finetune/Step Time": 4.99391033872962} +{"Finetune/Learning Rate": 1.6592579918034678e-05, "Finetune/Loss": 0.9047181606292725, "Finetune/Loss (Raw)": 1.0215227603912354, "Finetune/Step": 1521, "Finetune/Step Time": 4.95604744926095} +{"Finetune/Learning Rate": 1.65878944900943e-05, "Finetune/Loss": 0.9036523103713989, "Finetune/Loss (Raw)": 0.8941354155540466, "Finetune/Step": 1522, "Finetune/Step Time": 4.967734778299928} +{"Finetune/Learning Rate": 1.6583206505520695e-05, "Finetune/Loss": 0.9023345112800598, "Finetune/Loss (Raw)": 0.7888703942298889, "Finetune/Step": 1523, "Finetune/Step Time": 4.981521921232343} +{"Finetune/Learning Rate": 1.6578515966133167e-05, "Finetune/Loss": 0.9035670757293701, "Finetune/Loss (Raw)": 0.8846915364265442, "Finetune/Step": 1524, "Finetune/Step Time": 4.999468984082341} +{"Finetune/Learning Rate": 1.6573822873752026e-05, "Finetune/Loss": 0.9023174047470093, "Finetune/Loss (Raw)": 0.8678775429725647, "Finetune/Step": 1525, "Finetune/Step Time": 4.987933522090316} +{"Finetune/Learning Rate": 1.6569127230198575e-05, "Finetune/Loss": 0.9029216170310974, "Finetune/Loss (Raw)": 1.0737882852554321, "Finetune/Step": 1526, "Finetune/Step Time": 4.98726743273437} +{"Finetune/Learning Rate": 1.6564429037295097e-05, "Finetune/Loss": 0.9038420915603638, "Finetune/Loss (Raw)": 0.9330887198448181, "Finetune/Step": 1527, "Finetune/Step Time": 4.981623712927103} +{"Finetune/Learning Rate": 1.6559728296864868e-05, "Finetune/Loss": 0.8989630937576294, "Finetune/Loss (Raw)": 0.41879427433013916, "Finetune/Step": 1528, "Finetune/Step Time": 5.011280160397291} +{"Finetune/Learning Rate": 1.6555025010732156e-05, "Finetune/Loss": 0.8999648094177246, "Finetune/Loss (Raw)": 1.0363833904266357, "Finetune/Step": 1529, "Finetune/Step Time": 5.001700859516859} +{"Finetune/Learning Rate": 1.6550319180722213e-05, "Finetune/Loss": 0.9027172923088074, "Finetune/Loss (Raw)": 0.7481839060783386, "Finetune/Step": 1530, "Finetune/Step Time": 4.985012877732515} +{"Finetune/Learning Rate": 1.6545610808661278e-05, "Finetune/Loss": 0.9032154679298401, "Finetune/Loss (Raw)": 0.9948022961616516, "Finetune/Step": 1531, "Finetune/Step Time": 4.9884329084306955} +{"Finetune/Learning Rate": 1.654089989637658e-05, "Finetune/Loss": 0.9030377864837646, "Finetune/Loss (Raw)": 0.947333574295044, "Finetune/Step": 1532, "Finetune/Step Time": 4.997779976576567} +{"Finetune/Learning Rate": 1.6536186445696337e-05, "Finetune/Loss": 0.9033185243606567, "Finetune/Loss (Raw)": 0.8711176514625549, "Finetune/Step": 1533, "Finetune/Step Time": 4.99141606874764} +{"Finetune/Learning Rate": 1.653147045844974e-05, "Finetune/Loss": 0.9031550884246826, "Finetune/Loss (Raw)": 0.9429920315742493, "Finetune/Step": 1534, "Finetune/Step Time": 4.9993609469383955} +{"Finetune/Learning Rate": 1.6526751936466974e-05, "Finetune/Loss": 0.9071388244628906, "Finetune/Loss (Raw)": 0.8848814368247986, "Finetune/Step": 1535, "Finetune/Step Time": 4.966490415856242} +{"Finetune/Learning Rate": 1.6522030881579207e-05, "Finetune/Loss": 0.9073061943054199, "Finetune/Loss (Raw)": 0.8816601037979126, "Finetune/Step": 1536, "Finetune/Step Time": 4.972374394536018} +{"Finetune/Learning Rate": 1.6517307295618582e-05, "Finetune/Loss": 0.9064755439758301, "Finetune/Loss (Raw)": 0.8864948153495789, "Finetune/Step": 1537, "Finetune/Step Time": 4.973006907850504} +{"Finetune/Learning Rate": 1.6512581180418245e-05, "Finetune/Loss": 0.9028350114822388, "Finetune/Loss (Raw)": 0.4390862286090851, "Finetune/Step": 1538, "Finetune/Step Time": 4.976942783221602} +{"Finetune/Learning Rate": 1.650785253781229e-05, "Finetune/Loss": 0.9023429155349731, "Finetune/Loss (Raw)": 0.9750145673751831, "Finetune/Step": 1539, "Finetune/Step Time": 4.977864481508732} +{"Finetune/Learning Rate": 1.6503121369635822e-05, "Finetune/Loss": 0.9040670990943909, "Finetune/Loss (Raw)": 0.9636378288269043, "Finetune/Step": 1540, "Finetune/Step Time": 4.977538963779807} +{"Finetune/Learning Rate": 1.6498387677724917e-05, "Finetune/Loss": 0.904472827911377, "Finetune/Loss (Raw)": 1.0703386068344116, "Finetune/Step": 1541, "Finetune/Step Time": 4.97796231508255} +{"Finetune/Learning Rate": 1.6493651463916622e-05, "Finetune/Loss": 0.9028514623641968, "Finetune/Loss (Raw)": 0.8849266767501831, "Finetune/Step": 1542, "Finetune/Step Time": 4.985856125131249} +{"Finetune/Learning Rate": 1.6488912730048973e-05, "Finetune/Loss": 0.9018498063087463, "Finetune/Loss (Raw)": 0.8387205600738525, "Finetune/Step": 1543, "Finetune/Step Time": 4.981703272089362} +{"Finetune/Learning Rate": 1.6484171477960978e-05, "Finetune/Loss": 0.9010910987854004, "Finetune/Loss (Raw)": 0.9832989573478699, "Finetune/Step": 1544, "Finetune/Step Time": 4.974687019363046} +{"Finetune/Learning Rate": 1.6479427709492622e-05, "Finetune/Loss": 0.9009613990783691, "Finetune/Loss (Raw)": 0.8931265473365784, "Finetune/Step": 1545, "Finetune/Step Time": 4.97746541723609} +{"Finetune/Learning Rate": 1.6474681426484878e-05, "Finetune/Loss": 0.902515709400177, "Finetune/Loss (Raw)": 1.0499627590179443, "Finetune/Step": 1546, "Finetune/Step Time": 4.965874180197716} +{"Finetune/Learning Rate": 1.6469932630779677e-05, "Finetune/Loss": 0.9031949043273926, "Finetune/Loss (Raw)": 0.9594886302947998, "Finetune/Step": 1547, "Finetune/Step Time": 4.971559789031744} +{"Finetune/Learning Rate": 1.6465181324219937e-05, "Finetune/Loss": 0.903404712677002, "Finetune/Loss (Raw)": 1.016095519065857, "Finetune/Step": 1548, "Finetune/Step Time": 4.981340805068612} +{"Finetune/Learning Rate": 1.6460427508649546e-05, "Finetune/Loss": 0.899444580078125, "Finetune/Loss (Raw)": 0.40104940533638, "Finetune/Step": 1549, "Finetune/Step Time": 5.027195882052183} +{"Finetune/Learning Rate": 1.6455671185913367e-05, "Finetune/Loss": 0.8999959826469421, "Finetune/Loss (Raw)": 0.9951368570327759, "Finetune/Step": 1550, "Finetune/Step Time": 5.025018189102411} +{"Finetune/Learning Rate": 1.645091235785724e-05, "Finetune/Loss": 0.8986421227455139, "Finetune/Loss (Raw)": 0.8518489599227905, "Finetune/Step": 1551, "Finetune/Step Time": 5.019343068823218} +{"Finetune/Learning Rate": 1.644615102632797e-05, "Finetune/Loss": 0.8997414112091064, "Finetune/Loss (Raw)": 0.9902276396751404, "Finetune/Step": 1552, "Finetune/Step Time": 5.020009750500321} +{"Finetune/Learning Rate": 1.6441387193173337e-05, "Finetune/Loss": 0.8989641070365906, "Finetune/Loss (Raw)": 0.8826920986175537, "Finetune/Step": 1553, "Finetune/Step Time": 5.03159343637526} +{"Finetune/Learning Rate": 1.6436620860242093e-05, "Finetune/Loss": 0.8987233638763428, "Finetune/Loss (Raw)": 0.8515813946723938, "Finetune/Step": 1554, "Finetune/Step Time": 5.021982569247484} +{"Finetune/Learning Rate": 1.6431852029383955e-05, "Finetune/Loss": 0.8994553089141846, "Finetune/Loss (Raw)": 1.0386273860931396, "Finetune/Step": 1555, "Finetune/Step Time": 5.02986160479486} +{"Finetune/Learning Rate": 1.6427080702449616e-05, "Finetune/Loss": 0.898844838142395, "Finetune/Loss (Raw)": 0.8526403307914734, "Finetune/Step": 1556, "Finetune/Step Time": 5.036108808591962} +{"Finetune/Learning Rate": 1.6422306881290734e-05, "Finetune/Loss": 0.8987711668014526, "Finetune/Loss (Raw)": 0.8385508060455322, "Finetune/Step": 1557, "Finetune/Step Time": 5.029452281072736} +{"Finetune/Learning Rate": 1.641753056775994e-05, "Finetune/Loss": 0.898518443107605, "Finetune/Loss (Raw)": 0.9182847142219543, "Finetune/Step": 1558, "Finetune/Step Time": 5.01908715814352} +{"Finetune/Learning Rate": 1.641275176371082e-05, "Finetune/Loss": 0.9004905223846436, "Finetune/Loss (Raw)": 1.142561435699463, "Finetune/Step": 1559, "Finetune/Step Time": 5.009714528918266} +{"Finetune/Learning Rate": 1.6407970470997943e-05, "Finetune/Loss": 0.9009787440299988, "Finetune/Loss (Raw)": 1.0076653957366943, "Finetune/Step": 1560, "Finetune/Step Time": 5.0149334538728} +{"Finetune/Learning Rate": 1.6403186691476828e-05, "Finetune/Loss": 0.9010494947433472, "Finetune/Loss (Raw)": 1.000525951385498, "Finetune/Step": 1561, "Finetune/Step Time": 5.011850990355015} +{"Finetune/Learning Rate": 1.639840042700397e-05, "Finetune/Loss": 0.9057510495185852, "Finetune/Loss (Raw)": 0.9795355796813965, "Finetune/Step": 1562, "Finetune/Step Time": 4.986043093726039} +{"Finetune/Learning Rate": 1.6393611679436828e-05, "Finetune/Loss": 0.9049991369247437, "Finetune/Loss (Raw)": 0.9166322946548462, "Finetune/Step": 1563, "Finetune/Step Time": 5.004649117588997} +{"Finetune/Learning Rate": 1.6388820450633813e-05, "Finetune/Loss": 0.9058021306991577, "Finetune/Loss (Raw)": 0.899021327495575, "Finetune/Step": 1564, "Finetune/Step Time": 4.985214099287987} +{"Finetune/Learning Rate": 1.6384026742454315e-05, "Finetune/Loss": 0.9043487310409546, "Finetune/Loss (Raw)": 0.8502463102340698, "Finetune/Step": 1565, "Finetune/Step Time": 4.996229214593768} +{"Finetune/Learning Rate": 1.6379230556758676e-05, "Finetune/Loss": 0.9040921926498413, "Finetune/Loss (Raw)": 0.9334908127784729, "Finetune/Step": 1566, "Finetune/Step Time": 4.998150860890746} +{"Finetune/Learning Rate": 1.63744318954082e-05, "Finetune/Loss": 0.9036238193511963, "Finetune/Loss (Raw)": 0.9721189141273499, "Finetune/Step": 1567, "Finetune/Step Time": 5.009698027744889} +{"Finetune/Learning Rate": 1.636963076026516e-05, "Finetune/Loss": 0.905585765838623, "Finetune/Loss (Raw)": 0.9535213708877563, "Finetune/Step": 1568, "Finetune/Step Time": 4.982313238084316} +{"Finetune/Learning Rate": 1.6364827153192773e-05, "Finetune/Loss": 0.9046449661254883, "Finetune/Loss (Raw)": 0.8155059814453125, "Finetune/Step": 1569, "Finetune/Step Time": 4.995074072852731} +{"Finetune/Learning Rate": 1.6360021076055234e-05, "Finetune/Loss": 0.905031681060791, "Finetune/Loss (Raw)": 1.025168776512146, "Finetune/Step": 1570, "Finetune/Step Time": 4.997169446200132} +{"Finetune/Learning Rate": 1.6355212530717683e-05, "Finetune/Loss": 0.9054068326950073, "Finetune/Loss (Raw)": 0.9535417556762695, "Finetune/Step": 1571, "Finetune/Step Time": 4.9881930984556675} +{"Finetune/Learning Rate": 1.6350401519046226e-05, "Finetune/Loss": 0.9060962200164795, "Finetune/Loss (Raw)": 0.994418740272522, "Finetune/Step": 1572, "Finetune/Step Time": 4.981903847306967} +{"Finetune/Learning Rate": 1.634558804290792e-05, "Finetune/Loss": 0.9065921306610107, "Finetune/Loss (Raw)": 0.9743337631225586, "Finetune/Step": 1573, "Finetune/Step Time": 4.997854612767696} +{"Finetune/Learning Rate": 1.634077210417078e-05, "Finetune/Loss": 0.9062191843986511, "Finetune/Loss (Raw)": 0.8025583028793335, "Finetune/Step": 1574, "Finetune/Step Time": 4.980898478999734} +{"Finetune/Learning Rate": 1.633595370470378e-05, "Finetune/Loss": 0.9077592492103577, "Finetune/Loss (Raw)": 1.1603891849517822, "Finetune/Step": 1575, "Finetune/Step Time": 4.99091924726963} +{"Finetune/Learning Rate": 1.6331132846376843e-05, "Finetune/Loss": 0.9069980382919312, "Finetune/Loss (Raw)": 0.9476338624954224, "Finetune/Step": 1576, "Finetune/Step Time": 4.994035683572292} +{"Finetune/Learning Rate": 1.632630953106086e-05, "Finetune/Loss": 0.9076946377754211, "Finetune/Loss (Raw)": 0.9349554777145386, "Finetune/Step": 1577, "Finetune/Step Time": 4.985443426296115} +{"Finetune/Learning Rate": 1.6321483760627658e-05, "Finetune/Loss": 0.9088042974472046, "Finetune/Loss (Raw)": 0.9968721270561218, "Finetune/Step": 1578, "Finetune/Step Time": 4.943150935694575} +{"Finetune/Learning Rate": 1.6316655536950017e-05, "Finetune/Loss": 0.9091100692749023, "Finetune/Loss (Raw)": 0.9481248259544373, "Finetune/Step": 1579, "Finetune/Step Time": 4.949088424444199} +{"Finetune/Learning Rate": 1.6311824861901693e-05, "Finetune/Loss": 0.906938910484314, "Finetune/Loss (Raw)": 0.7678419351577759, "Finetune/Step": 1580, "Finetune/Step Time": 4.984187660738826} +{"Finetune/Learning Rate": 1.6306991737357366e-05, "Finetune/Loss": 0.9063825607299805, "Finetune/Loss (Raw)": 0.8362051248550415, "Finetune/Step": 1581, "Finetune/Step Time": 4.979034522548318} +{"Finetune/Learning Rate": 1.630215616519268e-05, "Finetune/Loss": 0.9045214653015137, "Finetune/Loss (Raw)": 0.952677309513092, "Finetune/Step": 1582, "Finetune/Step Time": 4.98009530082345} +{"Finetune/Learning Rate": 1.6297318147284223e-05, "Finetune/Loss": 0.9044169783592224, "Finetune/Loss (Raw)": 0.9921973943710327, "Finetune/Step": 1583, "Finetune/Step Time": 4.983274191617966} +{"Finetune/Learning Rate": 1.6292477685509538e-05, "Finetune/Loss": 0.9045443534851074, "Finetune/Loss (Raw)": 0.9454083442687988, "Finetune/Step": 1584, "Finetune/Step Time": 4.985336311161518} +{"Finetune/Learning Rate": 1.6287634781747113e-05, "Finetune/Loss": 0.9097917675971985, "Finetune/Loss (Raw)": 1.0492538213729858, "Finetune/Step": 1585, "Finetune/Step Time": 4.950575882568955} +{"Finetune/Learning Rate": 1.628278943787639e-05, "Finetune/Loss": 0.9110552072525024, "Finetune/Loss (Raw)": 1.1004608869552612, "Finetune/Step": 1586, "Finetune/Step Time": 5.008963420987129} +{"Finetune/Learning Rate": 1.6277941655777744e-05, "Finetune/Loss": 0.9107645153999329, "Finetune/Loss (Raw)": 1.0433884859085083, "Finetune/Step": 1587, "Finetune/Step Time": 5.001316422596574} +{"Finetune/Learning Rate": 1.6273091437332506e-05, "Finetune/Loss": 0.9095665216445923, "Finetune/Loss (Raw)": 0.939074695110321, "Finetune/Step": 1588, "Finetune/Step Time": 5.011669894680381} +{"Finetune/Learning Rate": 1.6268238784422954e-05, "Finetune/Loss": 0.9080736637115479, "Finetune/Loss (Raw)": 0.9227409958839417, "Finetune/Step": 1589, "Finetune/Step Time": 5.027886174619198} +{"Finetune/Learning Rate": 1.6263383698932307e-05, "Finetune/Loss": 0.9078317880630493, "Finetune/Loss (Raw)": 1.005686640739441, "Finetune/Step": 1590, "Finetune/Step Time": 5.033866666257381} +{"Finetune/Learning Rate": 1.6258526182744727e-05, "Finetune/Loss": 0.9059723615646362, "Finetune/Loss (Raw)": 0.7989305853843689, "Finetune/Step": 1591, "Finetune/Step Time": 5.049874056130648} +{"Finetune/Learning Rate": 1.6253666237745326e-05, "Finetune/Loss": 0.9058467149734497, "Finetune/Loss (Raw)": 0.8630858063697815, "Finetune/Step": 1592, "Finetune/Step Time": 5.051592951640487} +{"Finetune/Learning Rate": 1.6248803865820152e-05, "Finetune/Loss": 0.9054257273674011, "Finetune/Loss (Raw)": 0.9088695645332336, "Finetune/Step": 1593, "Finetune/Step Time": 5.054725036025047} +{"Finetune/Learning Rate": 1.624393906885619e-05, "Finetune/Loss": 0.9051942825317383, "Finetune/Loss (Raw)": 0.9247400760650635, "Finetune/Step": 1594, "Finetune/Step Time": 5.0599831491708755} +{"Finetune/Learning Rate": 1.6239071848741385e-05, "Finetune/Loss": 0.9044501185417175, "Finetune/Loss (Raw)": 0.8430619239807129, "Finetune/Step": 1595, "Finetune/Step Time": 5.037725105881691} +{"Finetune/Learning Rate": 1.6234202207364596e-05, "Finetune/Loss": 0.9048779010772705, "Finetune/Loss (Raw)": 1.0753837823867798, "Finetune/Step": 1596, "Finetune/Step Time": 5.042817834764719} +{"Finetune/Learning Rate": 1.6229330146615646e-05, "Finetune/Loss": 0.908393383026123, "Finetune/Loss (Raw)": 0.8387793302536011, "Finetune/Step": 1597, "Finetune/Step Time": 5.009560756385326} +{"Finetune/Learning Rate": 1.6224455668385283e-05, "Finetune/Loss": 0.9083718657493591, "Finetune/Loss (Raw)": 0.9684900045394897, "Finetune/Step": 1598, "Finetune/Step Time": 5.023017825558782} +{"Finetune/Learning Rate": 1.62195787745652e-05, "Finetune/Loss": 0.9037143588066101, "Finetune/Loss (Raw)": 0.3789633810520172, "Finetune/Step": 1599, "Finetune/Step Time": 5.059999108314514} +{"Finetune/Learning Rate": 1.6214699467048017e-05, "Finetune/Loss": 0.9046187400817871, "Finetune/Loss (Raw)": 0.9674019813537598, "Finetune/Step": 1600, "Finetune/Step Time": 5.063583802431822} +{"Finetune/Learning Rate": 1.6209817747727304e-05, "Finetune/Loss": 0.9021726846694946, "Finetune/Loss (Raw)": 0.38695022463798523, "Finetune/Step": 1601, "Finetune/Step Time": 5.095766780897975} +{"Finetune/Learning Rate": 1.620493361849756e-05, "Finetune/Loss": 0.9018552303314209, "Finetune/Loss (Raw)": 1.0687620639801025, "Finetune/Step": 1602, "Finetune/Step Time": 5.105640962719917} +{"Finetune/Learning Rate": 1.6200047081254223e-05, "Finetune/Loss": 0.9038246870040894, "Finetune/Loss (Raw)": 1.0061945915222168, "Finetune/Step": 1603, "Finetune/Step Time": 5.101180609315634} +{"Finetune/Learning Rate": 1.6195158137893656e-05, "Finetune/Loss": 0.9030860662460327, "Finetune/Loss (Raw)": 0.891241192817688, "Finetune/Step": 1604, "Finetune/Step Time": 5.1006575636565685} +{"Finetune/Learning Rate": 1.6190266790313165e-05, "Finetune/Loss": 0.9068906307220459, "Finetune/Loss (Raw)": 0.8034478425979614, "Finetune/Step": 1605, "Finetune/Step Time": 5.074186619371176} +{"Finetune/Learning Rate": 1.618537304041099e-05, "Finetune/Loss": 0.9045042991638184, "Finetune/Loss (Raw)": 0.7573763132095337, "Finetune/Step": 1606, "Finetune/Step Time": 5.105806829407811} +{"Finetune/Learning Rate": 1.6180476890086297e-05, "Finetune/Loss": 0.9045060873031616, "Finetune/Loss (Raw)": 0.8876849412918091, "Finetune/Step": 1607, "Finetune/Step Time": 5.092135084792972} +{"Finetune/Learning Rate": 1.6175578341239188e-05, "Finetune/Loss": 0.900019109249115, "Finetune/Loss (Raw)": 0.38622817397117615, "Finetune/Step": 1608, "Finetune/Step Time": 5.127725940197706} +{"Finetune/Learning Rate": 1.617067739577069e-05, "Finetune/Loss": 0.8997258543968201, "Finetune/Loss (Raw)": 0.9435825347900391, "Finetune/Step": 1609, "Finetune/Step Time": 5.121719852089882} +{"Finetune/Learning Rate": 1.616577405558277e-05, "Finetune/Loss": 0.8996248841285706, "Finetune/Loss (Raw)": 0.9309544563293457, "Finetune/Step": 1610, "Finetune/Step Time": 5.117928523570299} +{"Finetune/Learning Rate": 1.616086832257831e-05, "Finetune/Loss": 0.899357795715332, "Finetune/Loss (Raw)": 0.8561485409736633, "Finetune/Step": 1611, "Finetune/Step Time": 5.109696893021464} +{"Finetune/Learning Rate": 1.615596019866114e-05, "Finetune/Loss": 0.89857017993927, "Finetune/Loss (Raw)": 0.9120795130729675, "Finetune/Step": 1612, "Finetune/Step Time": 5.112044472247362} +{"Finetune/Learning Rate": 1.6151049685736e-05, "Finetune/Loss": 0.8980141878128052, "Finetune/Loss (Raw)": 0.8332514762878418, "Finetune/Step": 1613, "Finetune/Step Time": 5.124830204993486} +{"Finetune/Learning Rate": 1.6146136785708564e-05, "Finetune/Loss": 0.9032400846481323, "Finetune/Loss (Raw)": 0.979472279548645, "Finetune/Step": 1614, "Finetune/Step Time": 5.089127292856574} +{"Finetune/Learning Rate": 1.6141221500485437e-05, "Finetune/Loss": 0.9029040932655334, "Finetune/Loss (Raw)": 1.0479624271392822, "Finetune/Step": 1615, "Finetune/Step Time": 5.0772871524095535} +{"Finetune/Learning Rate": 1.6136303831974146e-05, "Finetune/Loss": 0.901034951210022, "Finetune/Loss (Raw)": 0.9420386552810669, "Finetune/Step": 1616, "Finetune/Step Time": 5.071102265268564} +{"Finetune/Learning Rate": 1.6131383782083135e-05, "Finetune/Loss": 0.9016481041908264, "Finetune/Loss (Raw)": 0.882950484752655, "Finetune/Step": 1617, "Finetune/Step Time": 5.078660311177373} +{"Finetune/Learning Rate": 1.6126461352721784e-05, "Finetune/Loss": 0.9014283418655396, "Finetune/Loss (Raw)": 0.8969483971595764, "Finetune/Step": 1618, "Finetune/Step Time": 5.057565715163946} +{"Finetune/Learning Rate": 1.612153654580039e-05, "Finetune/Loss": 0.9014124870300293, "Finetune/Loss (Raw)": 0.8082007169723511, "Finetune/Step": 1619, "Finetune/Step Time": 5.046436294913292} +{"Finetune/Learning Rate": 1.6116609363230176e-05, "Finetune/Loss": 0.9019502401351929, "Finetune/Loss (Raw)": 1.1345725059509277, "Finetune/Step": 1620, "Finetune/Step Time": 5.049233818426728} +{"Finetune/Learning Rate": 1.6111679806923287e-05, "Finetune/Loss": 0.9058746695518494, "Finetune/Loss (Raw)": 0.8369741439819336, "Finetune/Step": 1621, "Finetune/Step Time": 5.019311472773552} +{"Finetune/Learning Rate": 1.6106747878792784e-05, "Finetune/Loss": 0.906808614730835, "Finetune/Loss (Raw)": 1.0492981672286987, "Finetune/Step": 1622, "Finetune/Step Time": 5.017462827265263} +{"Finetune/Learning Rate": 1.610181358075265e-05, "Finetune/Loss": 0.9075971841812134, "Finetune/Loss (Raw)": 1.1228290796279907, "Finetune/Step": 1623, "Finetune/Step Time": 5.01073707267642} +{"Finetune/Learning Rate": 1.60968769147178e-05, "Finetune/Loss": 0.9069929718971252, "Finetune/Loss (Raw)": 0.9074522256851196, "Finetune/Step": 1624, "Finetune/Step Time": 5.007310843095183} +{"Finetune/Learning Rate": 1.6091937882604047e-05, "Finetune/Loss": 0.9080438613891602, "Finetune/Loss (Raw)": 0.9752376675605774, "Finetune/Step": 1625, "Finetune/Step Time": 5.012919010594487} +{"Finetune/Learning Rate": 1.608699648632814e-05, "Finetune/Loss": 0.9073953628540039, "Finetune/Loss (Raw)": 0.9802603125572205, "Finetune/Step": 1626, "Finetune/Step Time": 5.0035997331142426} +{"Finetune/Learning Rate": 1.6082052727807732e-05, "Finetune/Loss": 0.9120853543281555, "Finetune/Loss (Raw)": 1.0299960374832153, "Finetune/Step": 1627, "Finetune/Step Time": 4.964679652824998} +{"Finetune/Learning Rate": 1.60771066089614e-05, "Finetune/Loss": 0.91688072681427, "Finetune/Loss (Raw)": 0.9689085483551025, "Finetune/Step": 1628, "Finetune/Step Time": 4.92899189889431} +{"Finetune/Learning Rate": 1.6072158131708643e-05, "Finetune/Loss": 0.916063129901886, "Finetune/Loss (Raw)": 0.9206977486610413, "Finetune/Step": 1629, "Finetune/Step Time": 4.894760061055422} +{"Finetune/Learning Rate": 1.6067207297969864e-05, "Finetune/Loss": 0.9154958724975586, "Finetune/Loss (Raw)": 1.018371343612671, "Finetune/Step": 1630, "Finetune/Step Time": 4.8926068395376205} +{"Finetune/Learning Rate": 1.6062254109666383e-05, "Finetune/Loss": 0.9161272048950195, "Finetune/Loss (Raw)": 0.9305357336997986, "Finetune/Step": 1631, "Finetune/Step Time": 4.895005472004414} +{"Finetune/Learning Rate": 1.6057298568720436e-05, "Finetune/Loss": 0.915595293045044, "Finetune/Loss (Raw)": 0.9131303429603577, "Finetune/Step": 1632, "Finetune/Step Time": 4.880403568968177} +{"Finetune/Learning Rate": 1.6052340677055175e-05, "Finetune/Loss": 0.9147872924804688, "Finetune/Loss (Raw)": 0.8668673038482666, "Finetune/Step": 1633, "Finetune/Step Time": 4.8863238506019115} +{"Finetune/Learning Rate": 1.6047380436594663e-05, "Finetune/Loss": 0.9156652688980103, "Finetune/Loss (Raw)": 0.9841786623001099, "Finetune/Step": 1634, "Finetune/Step Time": 4.8735692128539085} +{"Finetune/Learning Rate": 1.6042417849263863e-05, "Finetune/Loss": 0.9148701429367065, "Finetune/Loss (Raw)": 0.9493566155433655, "Finetune/Step": 1635, "Finetune/Step Time": 4.895566402003169} +{"Finetune/Learning Rate": 1.6037452916988665e-05, "Finetune/Loss": 0.9146069288253784, "Finetune/Loss (Raw)": 0.9707966446876526, "Finetune/Step": 1636, "Finetune/Step Time": 4.891964098438621} +{"Finetune/Learning Rate": 1.6032485641695862e-05, "Finetune/Loss": 0.914115846157074, "Finetune/Loss (Raw)": 0.9403994679450989, "Finetune/Step": 1637, "Finetune/Step Time": 4.899093586951494} +{"Finetune/Learning Rate": 1.602751602531316e-05, "Finetune/Loss": 0.9127714037895203, "Finetune/Loss (Raw)": 0.8938395380973816, "Finetune/Step": 1638, "Finetune/Step Time": 4.900708353146911} +{"Finetune/Learning Rate": 1.602254406976916e-05, "Finetune/Loss": 0.9105830788612366, "Finetune/Loss (Raw)": 0.8207176923751831, "Finetune/Step": 1639, "Finetune/Step Time": 4.898089597001672} +{"Finetune/Learning Rate": 1.6017569776993388e-05, "Finetune/Loss": 0.9115302562713623, "Finetune/Loss (Raw)": 1.0088274478912354, "Finetune/Step": 1640, "Finetune/Step Time": 4.892043447121978} +{"Finetune/Learning Rate": 1.601259314891627e-05, "Finetune/Loss": 0.9105925559997559, "Finetune/Loss (Raw)": 1.0174607038497925, "Finetune/Step": 1641, "Finetune/Step Time": 4.889569528400898} +{"Finetune/Learning Rate": 1.6007614187469137e-05, "Finetune/Loss": 0.9107553958892822, "Finetune/Loss (Raw)": 0.9595256447792053, "Finetune/Step": 1642, "Finetune/Step Time": 4.883931960910559} +{"Finetune/Learning Rate": 1.6002632894584227e-05, "Finetune/Loss": 0.911681592464447, "Finetune/Loss (Raw)": 1.066978096961975, "Finetune/Step": 1643, "Finetune/Step Time": 4.8857622761279345} +{"Finetune/Learning Rate": 1.599764927219468e-05, "Finetune/Loss": 0.9116978645324707, "Finetune/Loss (Raw)": 0.9487040638923645, "Finetune/Step": 1644, "Finetune/Step Time": 4.91153964586556} +{"Finetune/Learning Rate": 1.5992663322234548e-05, "Finetune/Loss": 0.9121139049530029, "Finetune/Loss (Raw)": 0.9126251935958862, "Finetune/Step": 1645, "Finetune/Step Time": 4.912066349759698} +{"Finetune/Learning Rate": 1.5987675046638777e-05, "Finetune/Loss": 0.9116441011428833, "Finetune/Loss (Raw)": 1.0113706588745117, "Finetune/Step": 1646, "Finetune/Step Time": 4.915657687932253} +{"Finetune/Learning Rate": 1.5982684447343214e-05, "Finetune/Loss": 0.9120074510574341, "Finetune/Loss (Raw)": 0.8783522248268127, "Finetune/Step": 1647, "Finetune/Step Time": 4.928915796801448} +{"Finetune/Learning Rate": 1.5977691526284623e-05, "Finetune/Loss": 0.9169853925704956, "Finetune/Loss (Raw)": 1.0319664478302002, "Finetune/Step": 1648, "Finetune/Step Time": 4.897629640996456} +{"Finetune/Learning Rate": 1.5972696285400656e-05, "Finetune/Loss": 0.9162477254867554, "Finetune/Loss (Raw)": 0.9271085858345032, "Finetune/Step": 1649, "Finetune/Step Time": 4.899783471599221} +{"Finetune/Learning Rate": 1.596769872662987e-05, "Finetune/Loss": 0.91668701171875, "Finetune/Loss (Raw)": 0.9503681063652039, "Finetune/Step": 1650, "Finetune/Step Time": 4.888101631775498} +{"Finetune/Learning Rate": 1.5962698851911715e-05, "Finetune/Loss": 0.9172574877738953, "Finetune/Loss (Raw)": 0.8618845343589783, "Finetune/Step": 1651, "Finetune/Step Time": 4.889854529872537} +{"Finetune/Learning Rate": 1.5957696663186547e-05, "Finetune/Loss": 0.9171873331069946, "Finetune/Loss (Raw)": 0.8757140040397644, "Finetune/Step": 1652, "Finetune/Step Time": 4.87218551337719} +{"Finetune/Learning Rate": 1.595269216239562e-05, "Finetune/Loss": 0.9178054332733154, "Finetune/Loss (Raw)": 0.9469953775405884, "Finetune/Step": 1653, "Finetune/Step Time": 4.8636208195239305} +{"Finetune/Learning Rate": 1.594768535148108e-05, "Finetune/Loss": 0.9172055721282959, "Finetune/Loss (Raw)": 0.9970096945762634, "Finetune/Step": 1654, "Finetune/Step Time": 4.877228546887636} +{"Finetune/Learning Rate": 1.5942676232385973e-05, "Finetune/Loss": 0.9129177331924438, "Finetune/Loss (Raw)": 0.3842432498931885, "Finetune/Step": 1655, "Finetune/Step Time": 4.911547003313899} +{"Finetune/Learning Rate": 1.5937664807054242e-05, "Finetune/Loss": 0.91716468334198, "Finetune/Loss (Raw)": 0.9623978137969971, "Finetune/Step": 1656, "Finetune/Step Time": 4.878979969769716} +{"Finetune/Learning Rate": 1.593265107743072e-05, "Finetune/Loss": 0.9158452153205872, "Finetune/Loss (Raw)": 0.8674933314323425, "Finetune/Step": 1657, "Finetune/Step Time": 4.877641076222062} +{"Finetune/Learning Rate": 1.5927635045461145e-05, "Finetune/Loss": 0.9174594283103943, "Finetune/Loss (Raw)": 0.9548012614250183, "Finetune/Step": 1658, "Finetune/Step Time": 4.862035807222128} +{"Finetune/Learning Rate": 1.5922616713092135e-05, "Finetune/Loss": 0.9180704951286316, "Finetune/Loss (Raw)": 1.0730257034301758, "Finetune/Step": 1659, "Finetune/Step Time": 4.8626319374889135} +{"Finetune/Learning Rate": 1.5917596082271206e-05, "Finetune/Loss": 0.9181115031242371, "Finetune/Loss (Raw)": 0.9525766968727112, "Finetune/Step": 1660, "Finetune/Step Time": 4.8592136185616255} +{"Finetune/Learning Rate": 1.5912573154946768e-05, "Finetune/Loss": 0.9184131622314453, "Finetune/Loss (Raw)": 0.9097322225570679, "Finetune/Step": 1661, "Finetune/Step Time": 4.8544103018939495} +{"Finetune/Learning Rate": 1.5907547933068125e-05, "Finetune/Loss": 0.919127345085144, "Finetune/Loss (Raw)": 1.0344047546386719, "Finetune/Step": 1662, "Finetune/Step Time": 4.853487526997924} +{"Finetune/Learning Rate": 1.590252041858546e-05, "Finetune/Loss": 0.9176380634307861, "Finetune/Loss (Raw)": 0.6942570209503174, "Finetune/Step": 1663, "Finetune/Step Time": 4.863111279904842} +{"Finetune/Learning Rate": 1.5897490613449864e-05, "Finetune/Loss": 0.9169193506240845, "Finetune/Loss (Raw)": 0.7896696925163269, "Finetune/Step": 1664, "Finetune/Step Time": 4.856062885373831} +{"Finetune/Learning Rate": 1.5892458519613296e-05, "Finetune/Loss": 0.9182433485984802, "Finetune/Loss (Raw)": 1.055957555770874, "Finetune/Step": 1665, "Finetune/Step Time": 4.8558909967541695} +{"Finetune/Learning Rate": 1.588742413902862e-05, "Finetune/Loss": 0.9217793941497803, "Finetune/Loss (Raw)": 0.8917095065116882, "Finetune/Step": 1666, "Finetune/Step Time": 4.816022364422679} +{"Finetune/Learning Rate": 1.5882387473649572e-05, "Finetune/Loss": 0.9218325614929199, "Finetune/Loss (Raw)": 0.9818120002746582, "Finetune/Step": 1667, "Finetune/Step Time": 4.814011627808213} +{"Finetune/Learning Rate": 1.5877348525430793e-05, "Finetune/Loss": 0.9224487543106079, "Finetune/Loss (Raw)": 1.042507529258728, "Finetune/Step": 1668, "Finetune/Step Time": 4.818845175206661} +{"Finetune/Learning Rate": 1.5872307296327795e-05, "Finetune/Loss": 0.9214788675308228, "Finetune/Loss (Raw)": 0.9461979269981384, "Finetune/Step": 1669, "Finetune/Step Time": 4.815630946308374} +{"Finetune/Learning Rate": 1.5867263788296984e-05, "Finetune/Loss": 0.9222917556762695, "Finetune/Loss (Raw)": 0.9889789819717407, "Finetune/Step": 1670, "Finetune/Step Time": 4.802049247547984} +{"Finetune/Learning Rate": 1.5862218003295647e-05, "Finetune/Loss": 0.9226133823394775, "Finetune/Loss (Raw)": 0.8798861503601074, "Finetune/Step": 1671, "Finetune/Step Time": 4.8100996389985085} +{"Finetune/Learning Rate": 1.5857169943281948e-05, "Finetune/Loss": 0.9228132963180542, "Finetune/Loss (Raw)": 1.0088852643966675, "Finetune/Step": 1672, "Finetune/Step Time": 4.827112287282944} +{"Finetune/Learning Rate": 1.585211961021495e-05, "Finetune/Loss": 0.9222163558006287, "Finetune/Loss (Raw)": 0.8167251348495483, "Finetune/Step": 1673, "Finetune/Step Time": 4.832269236445427} +{"Finetune/Learning Rate": 1.5847067006054584e-05, "Finetune/Loss": 0.9200767278671265, "Finetune/Loss (Raw)": 0.7760828733444214, "Finetune/Step": 1674, "Finetune/Step Time": 4.8391328155994415} +{"Finetune/Learning Rate": 1.584201213276167e-05, "Finetune/Loss": 0.9195457100868225, "Finetune/Loss (Raw)": 0.8915247917175293, "Finetune/Step": 1675, "Finetune/Step Time": 4.843488696962595} +{"Finetune/Learning Rate": 1.5836954992297897e-05, "Finetune/Loss": 0.9198504090309143, "Finetune/Loss (Raw)": 1.0550907850265503, "Finetune/Step": 1676, "Finetune/Step Time": 4.842871030792594} +{"Finetune/Learning Rate": 1.583189558662585e-05, "Finetune/Loss": 0.924524188041687, "Finetune/Loss (Raw)": 0.999298632144928, "Finetune/Step": 1677, "Finetune/Step Time": 4.796618167310953} +{"Finetune/Learning Rate": 1.5826833917708983e-05, "Finetune/Loss": 0.9233593940734863, "Finetune/Loss (Raw)": 0.8460432887077332, "Finetune/Step": 1678, "Finetune/Step Time": 4.794552642852068} +{"Finetune/Learning Rate": 1.5821769987511634e-05, "Finetune/Loss": 0.9197617769241333, "Finetune/Loss (Raw)": 0.39135217666625977, "Finetune/Step": 1679, "Finetune/Step Time": 4.831896653398871} +{"Finetune/Learning Rate": 1.5816703797999015e-05, "Finetune/Loss": 0.9193030595779419, "Finetune/Loss (Raw)": 0.9315080642700195, "Finetune/Step": 1680, "Finetune/Step Time": 4.838850766420364} +{"Finetune/Learning Rate": 1.5811635351137212e-05, "Finetune/Loss": 0.9198868870735168, "Finetune/Loss (Raw)": 0.9574279189109802, "Finetune/Step": 1681, "Finetune/Step Time": 4.819877086207271} +{"Finetune/Learning Rate": 1.580656464889319e-05, "Finetune/Loss": 0.9218392372131348, "Finetune/Loss (Raw)": 1.1014759540557861, "Finetune/Step": 1682, "Finetune/Step Time": 4.822389639914036} +{"Finetune/Learning Rate": 1.5801491693234793e-05, "Finetune/Loss": 0.9203411340713501, "Finetune/Loss (Raw)": 0.8468695878982544, "Finetune/Step": 1683, "Finetune/Step Time": 4.83010315336287} +{"Finetune/Learning Rate": 1.5796416486130735e-05, "Finetune/Loss": 0.9207541942596436, "Finetune/Loss (Raw)": 0.9055091738700867, "Finetune/Step": 1684, "Finetune/Step Time": 4.82120774127543} +{"Finetune/Learning Rate": 1.5791339029550604e-05, "Finetune/Loss": 0.9208654165267944, "Finetune/Loss (Raw)": 0.8527979254722595, "Finetune/Step": 1685, "Finetune/Step Time": 4.825156223028898} +{"Finetune/Learning Rate": 1.578625932546486e-05, "Finetune/Loss": 0.9198918342590332, "Finetune/Loss (Raw)": 0.7936637997627258, "Finetune/Step": 1686, "Finetune/Step Time": 4.836603291332722} +{"Finetune/Learning Rate": 1.578117737584484e-05, "Finetune/Loss": 0.918827474117279, "Finetune/Loss (Raw)": 1.0063233375549316, "Finetune/Step": 1687, "Finetune/Step Time": 4.839064791798592} +{"Finetune/Learning Rate": 1.577609318266275e-05, "Finetune/Loss": 0.9189077615737915, "Finetune/Loss (Raw)": 1.0179427862167358, "Finetune/Step": 1688, "Finetune/Step Time": 4.841105120256543} +{"Finetune/Learning Rate": 1.5771006747891663e-05, "Finetune/Loss": 0.9135012030601501, "Finetune/Loss (Raw)": 0.3084871172904968, "Finetune/Step": 1689, "Finetune/Step Time": 4.873470256105065} +{"Finetune/Learning Rate": 1.576591807350553e-05, "Finetune/Loss": 0.9141328930854797, "Finetune/Loss (Raw)": 1.060390591621399, "Finetune/Step": 1690, "Finetune/Step Time": 4.863964535295963} +{"Finetune/Learning Rate": 1.5760827161479158e-05, "Finetune/Loss": 0.9142892360687256, "Finetune/Loss (Raw)": 0.9366440773010254, "Finetune/Step": 1691, "Finetune/Step Time": 4.845522614195943} +{"Finetune/Learning Rate": 1.5755734013788235e-05, "Finetune/Loss": 0.9140343070030212, "Finetune/Loss (Raw)": 0.8663877248764038, "Finetune/Step": 1692, "Finetune/Step Time": 4.867509305477142} +{"Finetune/Learning Rate": 1.5750638632409315e-05, "Finetune/Loss": 0.9138743281364441, "Finetune/Loss (Raw)": 0.8297718167304993, "Finetune/Step": 1693, "Finetune/Step Time": 4.874918878078461} +{"Finetune/Learning Rate": 1.574554101931981e-05, "Finetune/Loss": 0.9122008085250854, "Finetune/Loss (Raw)": 0.7192807197570801, "Finetune/Step": 1694, "Finetune/Step Time": 4.8771559074521065} +{"Finetune/Learning Rate": 1.5740441176498015e-05, "Finetune/Loss": 0.9114418029785156, "Finetune/Loss (Raw)": 0.8749628663063049, "Finetune/Step": 1695, "Finetune/Step Time": 4.878256931900978} +{"Finetune/Learning Rate": 1.5735339105923068e-05, "Finetune/Loss": 0.9114898443222046, "Finetune/Loss (Raw)": 0.9596712589263916, "Finetune/Step": 1696, "Finetune/Step Time": 4.8704748172312975} +{"Finetune/Learning Rate": 1.5730234809574986e-05, "Finetune/Loss": 0.9128555655479431, "Finetune/Loss (Raw)": 0.9903208017349243, "Finetune/Step": 1697, "Finetune/Step Time": 4.867554591968656} +{"Finetune/Learning Rate": 1.5725128289434647e-05, "Finetune/Loss": 0.9121010899543762, "Finetune/Loss (Raw)": 0.9285941123962402, "Finetune/Step": 1698, "Finetune/Step Time": 4.865007499232888} +{"Finetune/Learning Rate": 1.5720019547483795e-05, "Finetune/Loss": 0.912545919418335, "Finetune/Loss (Raw)": 1.0104776620864868, "Finetune/Step": 1699, "Finetune/Step Time": 4.874057367444038} +{"Finetune/Learning Rate": 1.571490858570503e-05, "Finetune/Loss": 0.9125089645385742, "Finetune/Loss (Raw)": 0.9896946549415588, "Finetune/Step": 1700, "Finetune/Step Time": 4.883895022794604} +{"Finetune/Learning Rate": 1.5709795406081818e-05, "Finetune/Loss": 0.9130672216415405, "Finetune/Loss (Raw)": 1.0457847118377686, "Finetune/Step": 1701, "Finetune/Step Time": 4.868946459144354} +{"Finetune/Learning Rate": 1.570468001059848e-05, "Finetune/Loss": 0.9144558906555176, "Finetune/Loss (Raw)": 0.9803130626678467, "Finetune/Step": 1702, "Finetune/Step Time": 4.868667162954807} +{"Finetune/Learning Rate": 1.5699562401240206e-05, "Finetune/Loss": 0.911986231803894, "Finetune/Loss (Raw)": 0.8442650437355042, "Finetune/Step": 1703, "Finetune/Step Time": 4.875989882275462} +{"Finetune/Learning Rate": 1.5694442579993042e-05, "Finetune/Loss": 0.9072985649108887, "Finetune/Loss (Raw)": 0.34761056303977966, "Finetune/Step": 1704, "Finetune/Step Time": 4.906712986528873} +{"Finetune/Learning Rate": 1.5689320548843882e-05, "Finetune/Loss": 0.907258927822113, "Finetune/Loss (Raw)": 0.9298874139785767, "Finetune/Step": 1705, "Finetune/Step Time": 4.909051356837153} +{"Finetune/Learning Rate": 1.5684196309780495e-05, "Finetune/Loss": 0.9074870347976685, "Finetune/Loss (Raw)": 1.0260728597640991, "Finetune/Step": 1706, "Finetune/Step Time": 4.916578100994229} +{"Finetune/Learning Rate": 1.5679069864791493e-05, "Finetune/Loss": 0.9074003100395203, "Finetune/Loss (Raw)": 0.9370197057723999, "Finetune/Step": 1707, "Finetune/Step Time": 4.913297273218632} +{"Finetune/Learning Rate": 1.567394121586635e-05, "Finetune/Loss": 0.9087467193603516, "Finetune/Loss (Raw)": 0.9401792287826538, "Finetune/Step": 1708, "Finetune/Step Time": 4.893323576077819} +{"Finetune/Learning Rate": 1.56688103649954e-05, "Finetune/Loss": 0.9106765985488892, "Finetune/Loss (Raw)": 1.0832313299179077, "Finetune/Step": 1709, "Finetune/Step Time": 4.891336522996426} +{"Finetune/Learning Rate": 1.566367731416982e-05, "Finetune/Loss": 0.9109059572219849, "Finetune/Loss (Raw)": 0.982036828994751, "Finetune/Step": 1710, "Finetune/Step Time": 4.90104116499424} +{"Finetune/Learning Rate": 1.5658542065381644e-05, "Finetune/Loss": 0.910913348197937, "Finetune/Loss (Raw)": 0.9931473135948181, "Finetune/Step": 1711, "Finetune/Step Time": 4.904178643599153} +{"Finetune/Learning Rate": 1.565340462062377e-05, "Finetune/Loss": 0.9122909307479858, "Finetune/Loss (Raw)": 1.1217364072799683, "Finetune/Step": 1712, "Finetune/Step Time": 4.904247464612126} +{"Finetune/Learning Rate": 1.5648264981889936e-05, "Finetune/Loss": 0.9117327928543091, "Finetune/Loss (Raw)": 0.9778112173080444, "Finetune/Step": 1713, "Finetune/Step Time": 4.903868846595287} +{"Finetune/Learning Rate": 1.5643123151174736e-05, "Finetune/Loss": 0.9108246564865112, "Finetune/Loss (Raw)": 0.984218955039978, "Finetune/Step": 1714, "Finetune/Step Time": 4.853768330067396} +{"Finetune/Learning Rate": 1.5637979130473612e-05, "Finetune/Loss": 0.9089490175247192, "Finetune/Loss (Raw)": 0.8033071756362915, "Finetune/Step": 1715, "Finetune/Step Time": 4.86334359459579} +{"Finetune/Learning Rate": 1.563283292178286e-05, "Finetune/Loss": 0.9087958335876465, "Finetune/Loss (Raw)": 0.9194737672805786, "Finetune/Step": 1716, "Finetune/Step Time": 4.851128539070487} +{"Finetune/Learning Rate": 1.5627684527099624e-05, "Finetune/Loss": 0.9095983505249023, "Finetune/Loss (Raw)": 1.0254629850387573, "Finetune/Step": 1717, "Finetune/Step Time": 4.851187519729137} +{"Finetune/Learning Rate": 1.5622533948421895e-05, "Finetune/Loss": 0.9093391299247742, "Finetune/Loss (Raw)": 0.9725015759468079, "Finetune/Step": 1718, "Finetune/Step Time": 4.841326026245952} +{"Finetune/Learning Rate": 1.5617381187748507e-05, "Finetune/Loss": 0.9104220867156982, "Finetune/Loss (Raw)": 0.9375486373901367, "Finetune/Step": 1719, "Finetune/Step Time": 4.8271073289215565} +{"Finetune/Learning Rate": 1.5612226247079155e-05, "Finetune/Loss": 0.9117872714996338, "Finetune/Loss (Raw)": 1.0378341674804688, "Finetune/Step": 1720, "Finetune/Step Time": 4.825524738058448} +{"Finetune/Learning Rate": 1.5607069128414366e-05, "Finetune/Loss": 0.9123601317405701, "Finetune/Loss (Raw)": 0.9821901321411133, "Finetune/Step": 1721, "Finetune/Step Time": 4.8266016729176044} +{"Finetune/Learning Rate": 1.5601909833755514e-05, "Finetune/Loss": 0.9127506017684937, "Finetune/Loss (Raw)": 0.9747256636619568, "Finetune/Step": 1722, "Finetune/Step Time": 4.817380666732788} +{"Finetune/Learning Rate": 1.559674836510482e-05, "Finetune/Loss": 0.9140154123306274, "Finetune/Loss (Raw)": 1.0049482583999634, "Finetune/Step": 1723, "Finetune/Step Time": 4.833667173981667} +{"Finetune/Learning Rate": 1.5591584724465363e-05, "Finetune/Loss": 0.9124554991722107, "Finetune/Loss (Raw)": 0.875722348690033, "Finetune/Step": 1724, "Finetune/Step Time": 4.82616863027215} +{"Finetune/Learning Rate": 1.5586418913841034e-05, "Finetune/Loss": 0.9133842587471008, "Finetune/Loss (Raw)": 0.9576622843742371, "Finetune/Step": 1725, "Finetune/Step Time": 4.8157433196902275} +{"Finetune/Learning Rate": 1.558125093523659e-05, "Finetune/Loss": 0.9126176238059998, "Finetune/Loss (Raw)": 0.870358407497406, "Finetune/Step": 1726, "Finetune/Step Time": 4.80778288282454} +{"Finetune/Learning Rate": 1.5576080790657626e-05, "Finetune/Loss": 0.9172148704528809, "Finetune/Loss (Raw)": 0.9674065709114075, "Finetune/Step": 1727, "Finetune/Step Time": 4.788228623569012} +{"Finetune/Learning Rate": 1.5570908482110566e-05, "Finetune/Loss": 0.9178709983825684, "Finetune/Loss (Raw)": 1.0513972043991089, "Finetune/Step": 1728, "Finetune/Step Time": 4.785284506157041} +{"Finetune/Learning Rate": 1.5565734011602693e-05, "Finetune/Loss": 0.9226511716842651, "Finetune/Loss (Raw)": 0.9988093972206116, "Finetune/Step": 1729, "Finetune/Step Time": 4.748957939445972} +{"Finetune/Learning Rate": 1.556055738114211e-05, "Finetune/Loss": 0.9206702709197998, "Finetune/Loss (Raw)": 0.8151952624320984, "Finetune/Step": 1730, "Finetune/Step Time": 4.7411597073078156} +{"Finetune/Learning Rate": 1.555537859273777e-05, "Finetune/Loss": 0.9213643074035645, "Finetune/Loss (Raw)": 1.0950331687927246, "Finetune/Step": 1731, "Finetune/Step Time": 4.737908728420734} +{"Finetune/Learning Rate": 1.5550197648399453e-05, "Finetune/Loss": 0.9227720499038696, "Finetune/Loss (Raw)": 1.071440577507019, "Finetune/Step": 1732, "Finetune/Step Time": 4.753041926771402} +{"Finetune/Learning Rate": 1.554501455013779e-05, "Finetune/Loss": 0.9240601062774658, "Finetune/Loss (Raw)": 0.9683123826980591, "Finetune/Step": 1733, "Finetune/Step Time": 4.743936797603965} +{"Finetune/Learning Rate": 1.5539829299964236e-05, "Finetune/Loss": 0.9259076714515686, "Finetune/Loss (Raw)": 0.9938687086105347, "Finetune/Step": 1734, "Finetune/Step Time": 4.727825498208404} +{"Finetune/Learning Rate": 1.5534641899891084e-05, "Finetune/Loss": 0.9260964393615723, "Finetune/Loss (Raw)": 0.9118514060974121, "Finetune/Step": 1735, "Finetune/Step Time": 4.727955978363752} +{"Finetune/Learning Rate": 1.5529452351931473e-05, "Finetune/Loss": 0.9305477142333984, "Finetune/Loss (Raw)": 0.9559904932975769, "Finetune/Step": 1736, "Finetune/Step Time": 4.692146852612495} +{"Finetune/Learning Rate": 1.5524260658099352e-05, "Finetune/Loss": 0.9300904273986816, "Finetune/Loss (Raw)": 0.8850430846214294, "Finetune/Step": 1737, "Finetune/Step Time": 4.691715380176902} +{"Finetune/Learning Rate": 1.551906682040952e-05, "Finetune/Loss": 0.9256048798561096, "Finetune/Loss (Raw)": 0.3568085730075836, "Finetune/Step": 1738, "Finetune/Step Time": 4.723699524998665} +{"Finetune/Learning Rate": 1.5513870840877608e-05, "Finetune/Loss": 0.9260597229003906, "Finetune/Loss (Raw)": 0.9143648743629456, "Finetune/Step": 1739, "Finetune/Step Time": 4.7019002344459295} +{"Finetune/Learning Rate": 1.5508672721520076e-05, "Finetune/Loss": 0.925625205039978, "Finetune/Loss (Raw)": 0.8564719557762146, "Finetune/Step": 1740, "Finetune/Step Time": 4.718512328341603} +{"Finetune/Learning Rate": 1.5503472464354207e-05, "Finetune/Loss": 0.9277169108390808, "Finetune/Loss (Raw)": 1.1009880304336548, "Finetune/Step": 1741, "Finetune/Step Time": 4.719833567738533} +{"Finetune/Learning Rate": 1.5498270071398116e-05, "Finetune/Loss": 0.9266873598098755, "Finetune/Loss (Raw)": 0.8476870059967041, "Finetune/Step": 1742, "Finetune/Step Time": 4.728289891034365} +{"Finetune/Learning Rate": 1.549306554467076e-05, "Finetune/Loss": 0.9258704781532288, "Finetune/Loss (Raw)": 0.9434006214141846, "Finetune/Step": 1743, "Finetune/Step Time": 4.744074944406748} +{"Finetune/Learning Rate": 1.5487858886191903e-05, "Finetune/Loss": 0.9255174398422241, "Finetune/Loss (Raw)": 0.8968530893325806, "Finetune/Step": 1744, "Finetune/Step Time": 4.7452789302915335} +{"Finetune/Learning Rate": 1.548265009798216e-05, "Finetune/Loss": 0.9262253046035767, "Finetune/Loss (Raw)": 0.9735559821128845, "Finetune/Step": 1745, "Finetune/Step Time": 4.729677138850093} +{"Finetune/Learning Rate": 1.547743918206295e-05, "Finetune/Loss": 0.9272359609603882, "Finetune/Loss (Raw)": 1.0263104438781738, "Finetune/Step": 1746, "Finetune/Step Time": 4.7296238746494055} +{"Finetune/Learning Rate": 1.5472226140456537e-05, "Finetune/Loss": 0.9243296384811401, "Finetune/Loss (Raw)": 0.43619027733802795, "Finetune/Step": 1747, "Finetune/Step Time": 4.764044318348169} +{"Finetune/Learning Rate": 1.546701097518599e-05, "Finetune/Loss": 0.9220253229141235, "Finetune/Loss (Raw)": 0.8396183848381042, "Finetune/Step": 1748, "Finetune/Step Time": 4.77438772469759} +{"Finetune/Learning Rate": 1.546179368827522e-05, "Finetune/Loss": 0.9230279326438904, "Finetune/Loss (Raw)": 0.9653080701828003, "Finetune/Step": 1749, "Finetune/Step Time": 4.7724799159914255} +{"Finetune/Learning Rate": 1.545657428174895e-05, "Finetune/Loss": 0.9232727289199829, "Finetune/Loss (Raw)": 1.0806347131729126, "Finetune/Step": 1750, "Finetune/Step Time": 4.791684195399284} +{"Finetune/Learning Rate": 1.5451352757632734e-05, "Finetune/Loss": 0.9223962426185608, "Finetune/Loss (Raw)": 1.0106374025344849, "Finetune/Step": 1751, "Finetune/Step Time": 4.796174081042409} +{"Finetune/Learning Rate": 1.544612911795294e-05, "Finetune/Loss": 0.9244099855422974, "Finetune/Loss (Raw)": 1.1652106046676636, "Finetune/Step": 1752, "Finetune/Step Time": 4.79530611820519} +{"Finetune/Learning Rate": 1.5440903364736763e-05, "Finetune/Loss": 0.9239633083343506, "Finetune/Loss (Raw)": 0.9180687069892883, "Finetune/Step": 1753, "Finetune/Step Time": 4.787854820489883} +{"Finetune/Learning Rate": 1.5435675500012212e-05, "Finetune/Loss": 0.9239959716796875, "Finetune/Loss (Raw)": 0.9844344258308411, "Finetune/Step": 1754, "Finetune/Step Time": 4.795280631631613} +{"Finetune/Learning Rate": 1.5430445525808123e-05, "Finetune/Loss": 0.9235015511512756, "Finetune/Loss (Raw)": 0.966713011264801, "Finetune/Step": 1755, "Finetune/Step Time": 4.801883487030864} +{"Finetune/Learning Rate": 1.5425213444154152e-05, "Finetune/Loss": 0.924263060092926, "Finetune/Loss (Raw)": 1.0663838386535645, "Finetune/Step": 1756, "Finetune/Step Time": 4.810826955363154} +{"Finetune/Learning Rate": 1.5419979257080755e-05, "Finetune/Loss": 0.9198975563049316, "Finetune/Loss (Raw)": 0.3619188070297241, "Finetune/Step": 1757, "Finetune/Step Time": 4.846260899677873} +{"Finetune/Learning Rate": 1.541474296661923e-05, "Finetune/Loss": 0.9184457063674927, "Finetune/Loss (Raw)": 0.8325268030166626, "Finetune/Step": 1758, "Finetune/Step Time": 4.850051008164883} +{"Finetune/Learning Rate": 1.5409504574801667e-05, "Finetune/Loss": 0.9190727472305298, "Finetune/Loss (Raw)": 1.0107992887496948, "Finetune/Step": 1759, "Finetune/Step Time": 4.8609429486095905} +{"Finetune/Learning Rate": 1.5404264083660993e-05, "Finetune/Loss": 0.9200537800788879, "Finetune/Loss (Raw)": 1.0386992692947388, "Finetune/Step": 1760, "Finetune/Step Time": 4.858643218874931} +{"Finetune/Learning Rate": 1.539902149523094e-05, "Finetune/Loss": 0.9199515581130981, "Finetune/Loss (Raw)": 0.8537800312042236, "Finetune/Step": 1761, "Finetune/Step Time": 4.852348236367106} +{"Finetune/Learning Rate": 1.5393776811546047e-05, "Finetune/Loss": 0.9187448024749756, "Finetune/Loss (Raw)": 0.829718291759491, "Finetune/Step": 1762, "Finetune/Step Time": 4.866971226409078} +{"Finetune/Learning Rate": 1.538853003464168e-05, "Finetune/Loss": 0.9197647571563721, "Finetune/Loss (Raw)": 1.0799120664596558, "Finetune/Step": 1763, "Finetune/Step Time": 4.866932932287455} +{"Finetune/Learning Rate": 1.538328116655401e-05, "Finetune/Loss": 0.9202482104301453, "Finetune/Loss (Raw)": 1.0326744318008423, "Finetune/Step": 1764, "Finetune/Step Time": 4.887562073767185} +{"Finetune/Learning Rate": 1.5378030209320018e-05, "Finetune/Loss": 0.9198306202888489, "Finetune/Loss (Raw)": 0.8869466185569763, "Finetune/Step": 1765, "Finetune/Step Time": 4.895027449354529} +{"Finetune/Learning Rate": 1.53727771649775e-05, "Finetune/Loss": 0.9205000996589661, "Finetune/Loss (Raw)": 0.9795379042625427, "Finetune/Step": 1766, "Finetune/Step Time": 4.896880377084017} +{"Finetune/Learning Rate": 1.5367522035565055e-05, "Finetune/Loss": 0.920456051826477, "Finetune/Loss (Raw)": 0.8150774836540222, "Finetune/Step": 1767, "Finetune/Step Time": 4.897400306537747} +{"Finetune/Learning Rate": 1.5362264823122107e-05, "Finetune/Loss": 0.9194761514663696, "Finetune/Loss (Raw)": 0.8833996653556824, "Finetune/Step": 1768, "Finetune/Step Time": 4.897788807749748} +{"Finetune/Learning Rate": 1.5357005529688868e-05, "Finetune/Loss": 0.9177163243293762, "Finetune/Loss (Raw)": 0.7922006845474243, "Finetune/Step": 1769, "Finetune/Step Time": 4.912866985425353} +{"Finetune/Learning Rate": 1.5351744157306366e-05, "Finetune/Loss": 0.9180499315261841, "Finetune/Loss (Raw)": 1.0022292137145996, "Finetune/Step": 1770, "Finetune/Step Time": 4.908798100426793} +{"Finetune/Learning Rate": 1.5346480708016445e-05, "Finetune/Loss": 0.9166010022163391, "Finetune/Loss (Raw)": 0.8815151453018188, "Finetune/Step": 1771, "Finetune/Step Time": 4.906623220071197} +{"Finetune/Learning Rate": 1.5341215183861738e-05, "Finetune/Loss": 0.9176369905471802, "Finetune/Loss (Raw)": 1.0813145637512207, "Finetune/Step": 1772, "Finetune/Step Time": 4.880514279007912} +{"Finetune/Learning Rate": 1.53359475868857e-05, "Finetune/Loss": 0.9178235530853271, "Finetune/Loss (Raw)": 0.9364964365959167, "Finetune/Step": 1773, "Finetune/Step Time": 4.890262618660927} +{"Finetune/Learning Rate": 1.5330677919132577e-05, "Finetune/Loss": 0.9172499179840088, "Finetune/Loss (Raw)": 0.937954306602478, "Finetune/Step": 1774, "Finetune/Step Time": 4.90363641269505} +{"Finetune/Learning Rate": 1.5325406182647426e-05, "Finetune/Loss": 0.9183996319770813, "Finetune/Loss (Raw)": 1.0255125761032104, "Finetune/Step": 1775, "Finetune/Step Time": 4.8900461960583925} +{"Finetune/Learning Rate": 1.5320132379476107e-05, "Finetune/Loss": 0.9176603555679321, "Finetune/Loss (Raw)": 0.9373414516448975, "Finetune/Step": 1776, "Finetune/Step Time": 4.889465287327766} +{"Finetune/Learning Rate": 1.531485651166528e-05, "Finetune/Loss": 0.9181533455848694, "Finetune/Loss (Raw)": 0.9902057647705078, "Finetune/Step": 1777, "Finetune/Step Time": 4.88969730399549} +{"Finetune/Learning Rate": 1.5309578581262404e-05, "Finetune/Loss": 0.9180483818054199, "Finetune/Loss (Raw)": 0.9369404911994934, "Finetune/Step": 1778, "Finetune/Step Time": 4.891914501786232} +{"Finetune/Learning Rate": 1.530429859031574e-05, "Finetune/Loss": 0.9197976589202881, "Finetune/Loss (Raw)": 1.0857847929000854, "Finetune/Step": 1779, "Finetune/Step Time": 4.876554707065225} +{"Finetune/Learning Rate": 1.5299016540874346e-05, "Finetune/Loss": 0.9199589490890503, "Finetune/Loss (Raw)": 0.8963603973388672, "Finetune/Step": 1780, "Finetune/Step Time": 4.869068430736661} +{"Finetune/Learning Rate": 1.5293732434988095e-05, "Finetune/Loss": 0.9188030958175659, "Finetune/Loss (Raw)": 0.7990401387214661, "Finetune/Step": 1781, "Finetune/Step Time": 4.870869027450681} +{"Finetune/Learning Rate": 1.5288446274707635e-05, "Finetune/Loss": 0.9181606769561768, "Finetune/Loss (Raw)": 0.9147860407829285, "Finetune/Step": 1782, "Finetune/Step Time": 4.857579868286848} +{"Finetune/Learning Rate": 1.5283158062084423e-05, "Finetune/Loss": 0.9217932820320129, "Finetune/Loss (Raw)": 0.8492189645767212, "Finetune/Step": 1783, "Finetune/Step Time": 4.823298191651702} +{"Finetune/Learning Rate": 1.5277867799170708e-05, "Finetune/Loss": 0.9225361347198486, "Finetune/Loss (Raw)": 1.0574779510498047, "Finetune/Step": 1784, "Finetune/Step Time": 4.827344462275505} +{"Finetune/Learning Rate": 1.527257548801955e-05, "Finetune/Loss": 0.9216740131378174, "Finetune/Loss (Raw)": 0.75714510679245, "Finetune/Step": 1785, "Finetune/Step Time": 4.8417529202997684} +{"Finetune/Learning Rate": 1.526728113068478e-05, "Finetune/Loss": 0.9212697744369507, "Finetune/Loss (Raw)": 0.903060793876648, "Finetune/Step": 1786, "Finetune/Step Time": 4.839511020109057} +{"Finetune/Learning Rate": 1.526198472922104e-05, "Finetune/Loss": 0.9190382361412048, "Finetune/Loss (Raw)": 0.7873849868774414, "Finetune/Step": 1787, "Finetune/Step Time": 4.844954684376717} +{"Finetune/Learning Rate": 1.5256686285683753e-05, "Finetune/Loss": 0.9183045625686646, "Finetune/Loss (Raw)": 0.8586729168891907, "Finetune/Step": 1788, "Finetune/Step Time": 4.840809667482972} +{"Finetune/Learning Rate": 1.5251385802129149e-05, "Finetune/Loss": 0.9135194420814514, "Finetune/Loss (Raw)": 0.2972261309623718, "Finetune/Step": 1789, "Finetune/Step Time": 4.883056120947003} +{"Finetune/Learning Rate": 1.5246083280614241e-05, "Finetune/Loss": 0.9133581519126892, "Finetune/Loss (Raw)": 1.0137661695480347, "Finetune/Step": 1790, "Finetune/Step Time": 4.886015772819519} +{"Finetune/Learning Rate": 1.5240778723196833e-05, "Finetune/Loss": 0.9161916971206665, "Finetune/Loss (Raw)": 1.0569499731063843, "Finetune/Step": 1791, "Finetune/Step Time": 4.878392092883587} +{"Finetune/Learning Rate": 1.523547213193552e-05, "Finetune/Loss": 0.9177305102348328, "Finetune/Loss (Raw)": 0.9866368770599365, "Finetune/Step": 1792, "Finetune/Step Time": 4.878618540242314} +{"Finetune/Learning Rate": 1.5230163508889686e-05, "Finetune/Loss": 0.9157569408416748, "Finetune/Loss (Raw)": 0.8033356070518494, "Finetune/Step": 1793, "Finetune/Step Time": 4.8936870861798525} +{"Finetune/Learning Rate": 1.5224852856119503e-05, "Finetune/Loss": 0.916773796081543, "Finetune/Loss (Raw)": 1.0218784809112549, "Finetune/Step": 1794, "Finetune/Step Time": 4.899898111820221} +{"Finetune/Learning Rate": 1.5219540175685938e-05, "Finetune/Loss": 0.9119985103607178, "Finetune/Loss (Raw)": 0.3705662488937378, "Finetune/Step": 1795, "Finetune/Step Time": 4.935722576454282} +{"Finetune/Learning Rate": 1.5214225469650727e-05, "Finetune/Loss": 0.9109508991241455, "Finetune/Loss (Raw)": 0.9084169268608093, "Finetune/Step": 1796, "Finetune/Step Time": 4.9284714832901955} +{"Finetune/Learning Rate": 1.5208908740076416e-05, "Finetune/Loss": 0.9113306403160095, "Finetune/Loss (Raw)": 0.9948016405105591, "Finetune/Step": 1797, "Finetune/Step Time": 4.9282272066921} +{"Finetune/Learning Rate": 1.5203589989026318e-05, "Finetune/Loss": 0.9108447432518005, "Finetune/Loss (Raw)": 0.926787257194519, "Finetune/Step": 1798, "Finetune/Step Time": 4.9323622062802315} +{"Finetune/Learning Rate": 1.5198269218564535e-05, "Finetune/Loss": 0.9109787940979004, "Finetune/Loss (Raw)": 0.8970434069633484, "Finetune/Step": 1799, "Finetune/Step Time": 4.9231878798455} +{"Finetune/Learning Rate": 1.5192946430755955e-05, "Finetune/Loss": 0.9101924300193787, "Finetune/Loss (Raw)": 0.9082278609275818, "Finetune/Step": 1800, "Finetune/Step Time": 4.917211962863803} +{"Finetune/Learning Rate": 1.518762162766625e-05, "Finetune/Loss": 0.9120751023292542, "Finetune/Loss (Raw)": 1.0577080249786377, "Finetune/Step": 1801, "Finetune/Step Time": 4.921540670096874} +{"Finetune/Learning Rate": 1.5182294811361871e-05, "Finetune/Loss": 0.9134089350700378, "Finetune/Loss (Raw)": 0.9468116760253906, "Finetune/Step": 1802, "Finetune/Step Time": 4.923594165593386} +{"Finetune/Learning Rate": 1.5176965983910053e-05, "Finetune/Loss": 0.9136087894439697, "Finetune/Loss (Raw)": 0.9171076416969299, "Finetune/Step": 1803, "Finetune/Step Time": 4.936001284047961} +{"Finetune/Learning Rate": 1.5171635147378804e-05, "Finetune/Loss": 0.9128667116165161, "Finetune/Loss (Raw)": 0.9601114988327026, "Finetune/Step": 1804, "Finetune/Step Time": 4.933663763105869} +{"Finetune/Learning Rate": 1.5166302303836928e-05, "Finetune/Loss": 0.910094141960144, "Finetune/Loss (Raw)": 0.6444018483161926, "Finetune/Step": 1805, "Finetune/Step Time": 4.942626537755132} +{"Finetune/Learning Rate": 1.5160967455353985e-05, "Finetune/Loss": 0.9101812839508057, "Finetune/Loss (Raw)": 0.8571962118148804, "Finetune/Step": 1806, "Finetune/Step Time": 4.959907131269574} +{"Finetune/Learning Rate": 1.5155630604000338e-05, "Finetune/Loss": 0.9144010543823242, "Finetune/Loss (Raw)": 0.9314850568771362, "Finetune/Step": 1807, "Finetune/Step Time": 4.923675734549761} +{"Finetune/Learning Rate": 1.5150291751847106e-05, "Finetune/Loss": 0.9144781231880188, "Finetune/Loss (Raw)": 0.9413723945617676, "Finetune/Step": 1808, "Finetune/Step Time": 4.912954796105623} +{"Finetune/Learning Rate": 1.5144950900966198e-05, "Finetune/Loss": 0.9143394231796265, "Finetune/Loss (Raw)": 0.9396752119064331, "Finetune/Step": 1809, "Finetune/Step Time": 4.912832470610738} +{"Finetune/Learning Rate": 1.5139608053430288e-05, "Finetune/Loss": 0.9131184816360474, "Finetune/Loss (Raw)": 0.9451950192451477, "Finetune/Step": 1810, "Finetune/Step Time": 4.915173694491386} +{"Finetune/Learning Rate": 1.5134263211312837e-05, "Finetune/Loss": 0.9137817025184631, "Finetune/Loss (Raw)": 0.931765079498291, "Finetune/Step": 1811, "Finetune/Step Time": 4.908106409013271} +{"Finetune/Learning Rate": 1.5128916376688071e-05, "Finetune/Loss": 0.9137011766433716, "Finetune/Loss (Raw)": 0.8951983451843262, "Finetune/Step": 1812, "Finetune/Step Time": 4.915383949875832} +{"Finetune/Learning Rate": 1.5123567551630993e-05, "Finetune/Loss": 0.9141851663589478, "Finetune/Loss (Raw)": 0.9147489666938782, "Finetune/Step": 1813, "Finetune/Step Time": 4.904804700985551} +{"Finetune/Learning Rate": 1.511821673821738e-05, "Finetune/Loss": 0.9156023263931274, "Finetune/Loss (Raw)": 0.9750553965568542, "Finetune/Step": 1814, "Finetune/Step Time": 4.892934983596206} +{"Finetune/Learning Rate": 1.5112863938523774e-05, "Finetune/Loss": 0.9146335124969482, "Finetune/Loss (Raw)": 0.882321834564209, "Finetune/Step": 1815, "Finetune/Step Time": 4.89146064966917} +{"Finetune/Learning Rate": 1.5107509154627491e-05, "Finetune/Loss": 0.9147144556045532, "Finetune/Loss (Raw)": 1.0283011198043823, "Finetune/Step": 1816, "Finetune/Step Time": 4.881034525111318} +{"Finetune/Learning Rate": 1.5102152388606625e-05, "Finetune/Loss": 0.9193997383117676, "Finetune/Loss (Raw)": 0.9082045555114746, "Finetune/Step": 1817, "Finetune/Step Time": 4.848219465464354} +{"Finetune/Learning Rate": 1.5096793642540027e-05, "Finetune/Loss": 0.9181122183799744, "Finetune/Loss (Raw)": 0.8955824971199036, "Finetune/Step": 1818, "Finetune/Step Time": 4.849089251831174} +{"Finetune/Learning Rate": 1.5091432918507322e-05, "Finetune/Loss": 0.9183432459831238, "Finetune/Loss (Raw)": 0.9662214517593384, "Finetune/Step": 1819, "Finetune/Step Time": 4.846589995548129} +{"Finetune/Learning Rate": 1.5086070218588904e-05, "Finetune/Loss": 0.9181475639343262, "Finetune/Loss (Raw)": 0.8413347601890564, "Finetune/Step": 1820, "Finetune/Step Time": 4.824485270306468} +{"Finetune/Learning Rate": 1.5080705544865932e-05, "Finetune/Loss": 0.9192065596580505, "Finetune/Loss (Raw)": 0.9653319716453552, "Finetune/Step": 1821, "Finetune/Step Time": 4.809610698372126} +{"Finetune/Learning Rate": 1.5075338899420326e-05, "Finetune/Loss": 0.9228475689888, "Finetune/Loss (Raw)": 1.1853269338607788, "Finetune/Step": 1822, "Finetune/Step Time": 4.8071417566388845} +{"Finetune/Learning Rate": 1.5069970284334785e-05, "Finetune/Loss": 0.9233800172805786, "Finetune/Loss (Raw)": 0.9431173801422119, "Finetune/Step": 1823, "Finetune/Step Time": 4.803444813936949} +{"Finetune/Learning Rate": 1.506459970169276e-05, "Finetune/Loss": 0.9224268198013306, "Finetune/Loss (Raw)": 0.8376544117927551, "Finetune/Step": 1824, "Finetune/Step Time": 4.808034241199493} +{"Finetune/Learning Rate": 1.5059227153578466e-05, "Finetune/Loss": 0.9217396378517151, "Finetune/Loss (Raw)": 0.9023671746253967, "Finetune/Step": 1825, "Finetune/Step Time": 4.797887098044157} +{"Finetune/Learning Rate": 1.505385264207689e-05, "Finetune/Loss": 0.9224306344985962, "Finetune/Loss (Raw)": 1.017039179801941, "Finetune/Step": 1826, "Finetune/Step Time": 4.798756239935756} +{"Finetune/Learning Rate": 1.5048476169273767e-05, "Finetune/Loss": 0.9218246936798096, "Finetune/Loss (Raw)": 0.9329244494438171, "Finetune/Step": 1827, "Finetune/Step Time": 4.820372788235545} +{"Finetune/Learning Rate": 1.5043097737255607e-05, "Finetune/Loss": 0.9224374294281006, "Finetune/Loss (Raw)": 1.0681148767471313, "Finetune/Step": 1828, "Finetune/Step Time": 4.818157710134983} +{"Finetune/Learning Rate": 1.5037717348109677e-05, "Finetune/Loss": 0.9230378866195679, "Finetune/Loss (Raw)": 1.1226541996002197, "Finetune/Step": 1829, "Finetune/Step Time": 4.827403297647834} +{"Finetune/Learning Rate": 1.5032335003923994e-05, "Finetune/Loss": 0.9235100746154785, "Finetune/Loss (Raw)": 1.040746808052063, "Finetune/Step": 1830, "Finetune/Step Time": 4.844688173383474} +{"Finetune/Learning Rate": 1.5026950706787346e-05, "Finetune/Loss": 0.9240607023239136, "Finetune/Loss (Raw)": 0.9147445559501648, "Finetune/Step": 1831, "Finetune/Step Time": 4.829137729480863} +{"Finetune/Learning Rate": 1.502156445878927e-05, "Finetune/Loss": 0.9294289946556091, "Finetune/Loss (Raw)": 1.0347532033920288, "Finetune/Step": 1832, "Finetune/Step Time": 4.80351966060698} +{"Finetune/Learning Rate": 1.5016176262020069e-05, "Finetune/Loss": 0.9287630319595337, "Finetune/Loss (Raw)": 0.8446494340896606, "Finetune/Step": 1833, "Finetune/Step Time": 4.82294431887567} +{"Finetune/Learning Rate": 1.5010786118570791e-05, "Finetune/Loss": 0.9292361736297607, "Finetune/Loss (Raw)": 1.0866292715072632, "Finetune/Step": 1834, "Finetune/Step Time": 4.817114418372512} +{"Finetune/Learning Rate": 1.5005394030533246e-05, "Finetune/Loss": 0.9293609261512756, "Finetune/Loss (Raw)": 0.9529918432235718, "Finetune/Step": 1835, "Finetune/Step Time": 4.826101750135422} +{"Finetune/Learning Rate": 1.5000000000000002e-05, "Finetune/Loss": 0.9296318292617798, "Finetune/Loss (Raw)": 0.9748474359512329, "Finetune/Step": 1836, "Finetune/Step Time": 4.821217266842723} +{"Finetune/Learning Rate": 1.4994604029064372e-05, "Finetune/Loss": 0.9285142421722412, "Finetune/Loss (Raw)": 0.9401856064796448, "Finetune/Step": 1837, "Finetune/Step Time": 4.822845082730055} +{"Finetune/Learning Rate": 1.4989206119820432e-05, "Finetune/Loss": 0.9284579157829285, "Finetune/Loss (Raw)": 0.974828839302063, "Finetune/Step": 1838, "Finetune/Step Time": 4.815327173098922} +{"Finetune/Learning Rate": 1.4983806274363005e-05, "Finetune/Loss": 0.9283273220062256, "Finetune/Loss (Raw)": 0.9764361381530762, "Finetune/Step": 1839, "Finetune/Step Time": 4.822473494336009} +{"Finetune/Learning Rate": 1.497840449478766e-05, "Finetune/Loss": 0.9273483157157898, "Finetune/Loss (Raw)": 0.9964143633842468, "Finetune/Step": 1840, "Finetune/Step Time": 4.832534126937389} +{"Finetune/Learning Rate": 1.4973000783190728e-05, "Finetune/Loss": 0.9261600971221924, "Finetune/Loss (Raw)": 0.8257242441177368, "Finetune/Step": 1841, "Finetune/Step Time": 4.834110375493765} +{"Finetune/Learning Rate": 1.4967595141669278e-05, "Finetune/Loss": 0.9266869425773621, "Finetune/Loss (Raw)": 1.0516506433486938, "Finetune/Step": 1842, "Finetune/Step Time": 4.832578480243683} +{"Finetune/Learning Rate": 1.4962187572321143e-05, "Finetune/Loss": 0.928046703338623, "Finetune/Loss (Raw)": 0.9773628115653992, "Finetune/Step": 1843, "Finetune/Step Time": 4.837931368499994} +{"Finetune/Learning Rate": 1.4956778077244884e-05, "Finetune/Loss": 0.9281592965126038, "Finetune/Loss (Raw)": 0.9338778853416443, "Finetune/Step": 1844, "Finetune/Step Time": 4.84974005445838} +{"Finetune/Learning Rate": 1.4951366658539832e-05, "Finetune/Loss": 0.9273315668106079, "Finetune/Loss (Raw)": 0.9195239543914795, "Finetune/Step": 1845, "Finetune/Step Time": 4.84339814633131} +{"Finetune/Learning Rate": 1.4945953318306044e-05, "Finetune/Loss": 0.9280039072036743, "Finetune/Loss (Raw)": 1.0585507154464722, "Finetune/Step": 1846, "Finetune/Step Time": 4.844899060204625} +{"Finetune/Learning Rate": 1.4940538058644332e-05, "Finetune/Loss": 0.92851722240448, "Finetune/Loss (Raw)": 1.0032570362091064, "Finetune/Step": 1847, "Finetune/Step Time": 4.8534746784716845} +{"Finetune/Learning Rate": 1.4935120881656263e-05, "Finetune/Loss": 0.9278056025505066, "Finetune/Loss (Raw)": 0.9467487335205078, "Finetune/Step": 1848, "Finetune/Step Time": 4.845583897083998} +{"Finetune/Learning Rate": 1.4929701789444122e-05, "Finetune/Loss": 0.9282714128494263, "Finetune/Loss (Raw)": 1.041811227798462, "Finetune/Step": 1849, "Finetune/Step Time": 4.839528558775783} +{"Finetune/Learning Rate": 1.4924280784110965e-05, "Finetune/Loss": 0.9286861419677734, "Finetune/Loss (Raw)": 1.0278141498565674, "Finetune/Step": 1850, "Finetune/Step Time": 4.845294587314129} +{"Finetune/Learning Rate": 1.4918857867760571e-05, "Finetune/Loss": 0.927462637424469, "Finetune/Loss (Raw)": 0.8483388423919678, "Finetune/Step": 1851, "Finetune/Step Time": 4.848442263901234} +{"Finetune/Learning Rate": 1.4913433042497473e-05, "Finetune/Loss": 0.9283277988433838, "Finetune/Loss (Raw)": 0.9864673614501953, "Finetune/Step": 1852, "Finetune/Step Time": 4.849833441898227} +{"Finetune/Learning Rate": 1.4908006310426942e-05, "Finetune/Loss": 0.9284105896949768, "Finetune/Loss (Raw)": 0.9682543873786926, "Finetune/Step": 1853, "Finetune/Step Time": 4.85077366605401} +{"Finetune/Learning Rate": 1.4902577673654977e-05, "Finetune/Loss": 0.9292747974395752, "Finetune/Loss (Raw)": 0.9809800386428833, "Finetune/Step": 1854, "Finetune/Step Time": 4.844672104343772} +{"Finetune/Learning Rate": 1.4897147134288334e-05, "Finetune/Loss": 0.9297090768814087, "Finetune/Loss (Raw)": 1.0229932069778442, "Finetune/Step": 1855, "Finetune/Step Time": 4.829674769192934} +{"Finetune/Learning Rate": 1.4891714694434495e-05, "Finetune/Loss": 0.9297937154769897, "Finetune/Loss (Raw)": 1.062225103378296, "Finetune/Step": 1856, "Finetune/Step Time": 4.829925240948796} +{"Finetune/Learning Rate": 1.4886280356201688e-05, "Finetune/Loss": 0.9288768768310547, "Finetune/Loss (Raw)": 0.8814609050750732, "Finetune/Step": 1857, "Finetune/Step Time": 4.867042101919651} +{"Finetune/Learning Rate": 1.4880844121698872e-05, "Finetune/Loss": 0.9302963614463806, "Finetune/Loss (Raw)": 0.9968827962875366, "Finetune/Step": 1858, "Finetune/Step Time": 4.874768251553178} +{"Finetune/Learning Rate": 1.487540599303574e-05, "Finetune/Loss": 0.9299641847610474, "Finetune/Loss (Raw)": 1.052522897720337, "Finetune/Step": 1859, "Finetune/Step Time": 4.869433123618364} +{"Finetune/Learning Rate": 1.4869965972322728e-05, "Finetune/Loss": 0.9286054372787476, "Finetune/Loss (Raw)": 0.8975142240524292, "Finetune/Step": 1860, "Finetune/Step Time": 4.857936631888151} +{"Finetune/Learning Rate": 1.4864524061670999e-05, "Finetune/Loss": 0.9276818037033081, "Finetune/Loss (Raw)": 0.8500943779945374, "Finetune/Step": 1861, "Finetune/Step Time": 4.85857267677784} +{"Finetune/Learning Rate": 1.4859080263192456e-05, "Finetune/Loss": 0.9265991449356079, "Finetune/Loss (Raw)": 0.8552838563919067, "Finetune/Step": 1862, "Finetune/Step Time": 4.842754054814577} +{"Finetune/Learning Rate": 1.4853634578999728e-05, "Finetune/Loss": 0.9259172677993774, "Finetune/Loss (Raw)": 0.8245751857757568, "Finetune/Step": 1863, "Finetune/Step Time": 4.8421501070261} +{"Finetune/Learning Rate": 1.4848187011206177e-05, "Finetune/Loss": 0.9249902963638306, "Finetune/Loss (Raw)": 0.837333619594574, "Finetune/Step": 1864, "Finetune/Step Time": 4.846862215548754} +{"Finetune/Learning Rate": 1.4842737561925901e-05, "Finetune/Loss": 0.9261119961738586, "Finetune/Loss (Raw)": 1.0286216735839844, "Finetune/Step": 1865, "Finetune/Step Time": 4.849368983879685} +{"Finetune/Learning Rate": 1.4837286233273722e-05, "Finetune/Loss": 0.9314182996749878, "Finetune/Loss (Raw)": 1.036012887954712, "Finetune/Step": 1866, "Finetune/Step Time": 4.827906791120768} +{"Finetune/Learning Rate": 1.48318330273652e-05, "Finetune/Loss": 0.9327718019485474, "Finetune/Loss (Raw)": 1.0876195430755615, "Finetune/Step": 1867, "Finetune/Step Time": 4.834337271749973} +{"Finetune/Learning Rate": 1.4826377946316611e-05, "Finetune/Loss": 0.9335348010063171, "Finetune/Loss (Raw)": 0.9541333317756653, "Finetune/Step": 1868, "Finetune/Step Time": 4.835845140740275} +{"Finetune/Learning Rate": 1.4820920992244964e-05, "Finetune/Loss": 0.9330908060073853, "Finetune/Loss (Raw)": 1.0441581010818481, "Finetune/Step": 1869, "Finetune/Step Time": 4.8216213919222355} +{"Finetune/Learning Rate": 1.481546216726801e-05, "Finetune/Loss": 0.9334150552749634, "Finetune/Loss (Raw)": 0.8891847729682922, "Finetune/Step": 1870, "Finetune/Step Time": 4.813984317705035} +{"Finetune/Learning Rate": 1.4810001473504194e-05, "Finetune/Loss": 0.9333446025848389, "Finetune/Loss (Raw)": 0.9343876838684082, "Finetune/Step": 1871, "Finetune/Step Time": 4.827672636136413} +{"Finetune/Learning Rate": 1.4804538913072719e-05, "Finetune/Loss": 0.9327505826950073, "Finetune/Loss (Raw)": 0.8208122253417969, "Finetune/Step": 1872, "Finetune/Step Time": 4.832449162378907} +{"Finetune/Learning Rate": 1.479907448809349e-05, "Finetune/Loss": 0.9280272722244263, "Finetune/Loss (Raw)": 0.3689757287502289, "Finetune/Step": 1873, "Finetune/Step Time": 4.862434297800064} +{"Finetune/Learning Rate": 1.4793608200687146e-05, "Finetune/Loss": 0.9281679391860962, "Finetune/Loss (Raw)": 1.0443220138549805, "Finetune/Step": 1874, "Finetune/Step Time": 4.868834692984819} +{"Finetune/Learning Rate": 1.478814005297505e-05, "Finetune/Loss": 0.9317430853843689, "Finetune/Loss (Raw)": 0.8938041925430298, "Finetune/Step": 1875, "Finetune/Step Time": 4.8501497730612755} +{"Finetune/Learning Rate": 1.4782670047079275e-05, "Finetune/Loss": 0.9326198101043701, "Finetune/Loss (Raw)": 0.9518411755561829, "Finetune/Step": 1876, "Finetune/Step Time": 4.8366591446101665} +{"Finetune/Learning Rate": 1.477719818512263e-05, "Finetune/Loss": 0.9325037002563477, "Finetune/Loss (Raw)": 0.9504426121711731, "Finetune/Step": 1877, "Finetune/Step Time": 4.841215880587697} +{"Finetune/Learning Rate": 1.4771724469228638e-05, "Finetune/Loss": 0.9302605390548706, "Finetune/Loss (Raw)": 0.7935137748718262, "Finetune/Step": 1878, "Finetune/Step Time": 4.833852892741561} +{"Finetune/Learning Rate": 1.4766248901521539e-05, "Finetune/Loss": 0.9306461215019226, "Finetune/Loss (Raw)": 1.0599901676177979, "Finetune/Step": 1879, "Finetune/Step Time": 4.824710493907332} +{"Finetune/Learning Rate": 1.4760771484126296e-05, "Finetune/Loss": 0.927818775177002, "Finetune/Loss (Raw)": 0.8033108711242676, "Finetune/Step": 1880, "Finetune/Step Time": 4.845634115859866} +{"Finetune/Learning Rate": 1.4755292219168581e-05, "Finetune/Loss": 0.927099883556366, "Finetune/Loss (Raw)": 0.8260495066642761, "Finetune/Step": 1881, "Finetune/Step Time": 4.85341764241457} +{"Finetune/Learning Rate": 1.4749811108774798e-05, "Finetune/Loss": 0.9271048307418823, "Finetune/Loss (Raw)": 0.9850718975067139, "Finetune/Step": 1882, "Finetune/Step Time": 4.849541988223791} +{"Finetune/Learning Rate": 1.4744328155072058e-05, "Finetune/Loss": 0.9262943267822266, "Finetune/Loss (Raw)": 0.8629629611968994, "Finetune/Step": 1883, "Finetune/Step Time": 4.858134103938937} +{"Finetune/Learning Rate": 1.4738843360188184e-05, "Finetune/Loss": 0.9207658171653748, "Finetune/Loss (Raw)": 0.35873329639434814, "Finetune/Step": 1884, "Finetune/Step Time": 4.886225402355194} +{"Finetune/Learning Rate": 1.4733356726251722e-05, "Finetune/Loss": 0.9255507588386536, "Finetune/Loss (Raw)": 0.9743921756744385, "Finetune/Step": 1885, "Finetune/Step Time": 4.852237097918987} +{"Finetune/Learning Rate": 1.4727868255391924e-05, "Finetune/Loss": 0.9274652004241943, "Finetune/Loss (Raw)": 1.0775785446166992, "Finetune/Step": 1886, "Finetune/Step Time": 4.856348956003785} +{"Finetune/Learning Rate": 1.472237794973876e-05, "Finetune/Loss": 0.9258689880371094, "Finetune/Loss (Raw)": 0.8064823150634766, "Finetune/Step": 1887, "Finetune/Step Time": 4.84602003544569} +{"Finetune/Learning Rate": 1.471688581142291e-05, "Finetune/Loss": 0.9237054586410522, "Finetune/Loss (Raw)": 0.7617631554603577, "Finetune/Step": 1888, "Finetune/Step Time": 4.856450203806162} +{"Finetune/Learning Rate": 1.4711391842575765e-05, "Finetune/Loss": 0.9243152141571045, "Finetune/Loss (Raw)": 0.9318289756774902, "Finetune/Step": 1889, "Finetune/Step Time": 4.86717283539474} +{"Finetune/Learning Rate": 1.4705896045329432e-05, "Finetune/Loss": 0.9249545335769653, "Finetune/Loss (Raw)": 0.9115543961524963, "Finetune/Step": 1890, "Finetune/Step Time": 4.855708556249738} +{"Finetune/Learning Rate": 1.4700398421816714e-05, "Finetune/Loss": 0.9244054555892944, "Finetune/Loss (Raw)": 1.0096321105957031, "Finetune/Step": 1891, "Finetune/Step Time": 4.834575166925788} +{"Finetune/Learning Rate": 1.469489897417114e-05, "Finetune/Loss": 0.9249525666236877, "Finetune/Loss (Raw)": 1.1027017831802368, "Finetune/Step": 1892, "Finetune/Step Time": 4.814724318683147} +{"Finetune/Learning Rate": 1.468939770452693e-05, "Finetune/Loss": 0.9252094626426697, "Finetune/Loss (Raw)": 0.9198296666145325, "Finetune/Step": 1893, "Finetune/Step Time": 4.816311310976744} +{"Finetune/Learning Rate": 1.4683894615019028e-05, "Finetune/Loss": 0.9239646196365356, "Finetune/Loss (Raw)": 0.8201929330825806, "Finetune/Step": 1894, "Finetune/Step Time": 4.813094332814217} +{"Finetune/Learning Rate": 1.4678389707783072e-05, "Finetune/Loss": 0.924638032913208, "Finetune/Loss (Raw)": 0.9012824296951294, "Finetune/Step": 1895, "Finetune/Step Time": 4.812815831974149} +{"Finetune/Learning Rate": 1.4672882984955404e-05, "Finetune/Loss": 0.925230860710144, "Finetune/Loss (Raw)": 0.9592804908752441, "Finetune/Step": 1896, "Finetune/Step Time": 4.815442711114883} +{"Finetune/Learning Rate": 1.4667374448673081e-05, "Finetune/Loss": 0.9275599122047424, "Finetune/Loss (Raw)": 1.0903173685073853, "Finetune/Step": 1897, "Finetune/Step Time": 4.802159555256367} +{"Finetune/Learning Rate": 1.4661864101073857e-05, "Finetune/Loss": 0.9274584650993347, "Finetune/Loss (Raw)": 0.9892430305480957, "Finetune/Step": 1898, "Finetune/Step Time": 4.812029926106334} +{"Finetune/Learning Rate": 1.465635194429619e-05, "Finetune/Loss": 0.9269661903381348, "Finetune/Loss (Raw)": 0.818503737449646, "Finetune/Step": 1899, "Finetune/Step Time": 4.831635896116495} +{"Finetune/Learning Rate": 1.4650837980479246e-05, "Finetune/Loss": 0.9254547357559204, "Finetune/Loss (Raw)": 0.8878435492515564, "Finetune/Step": 1900, "Finetune/Step Time": 4.832796731963754} +{"Finetune/Learning Rate": 1.4645322211762876e-05, "Finetune/Loss": 0.9252310991287231, "Finetune/Loss (Raw)": 0.9078775644302368, "Finetune/Step": 1901, "Finetune/Step Time": 4.828767912462354} +{"Finetune/Learning Rate": 1.4639804640287652e-05, "Finetune/Loss": 0.9257012009620667, "Finetune/Loss (Raw)": 0.9981247186660767, "Finetune/Step": 1902, "Finetune/Step Time": 4.8191490694880486} +{"Finetune/Learning Rate": 1.4634285268194828e-05, "Finetune/Loss": 0.9243646860122681, "Finetune/Loss (Raw)": 0.8544378876686096, "Finetune/Step": 1903, "Finetune/Step Time": 4.83941794000566} +{"Finetune/Learning Rate": 1.462876409762637e-05, "Finetune/Loss": 0.9242236614227295, "Finetune/Loss (Raw)": 0.919289231300354, "Finetune/Step": 1904, "Finetune/Step Time": 4.843929626047611} +{"Finetune/Learning Rate": 1.4623241130724937e-05, "Finetune/Loss": 0.9189980626106262, "Finetune/Loss (Raw)": 0.32133468985557556, "Finetune/Step": 1905, "Finetune/Step Time": 4.881833661347628} +{"Finetune/Learning Rate": 1.4617716369633876e-05, "Finetune/Loss": 0.9187942743301392, "Finetune/Loss (Raw)": 0.9108561873435974, "Finetune/Step": 1906, "Finetune/Step Time": 4.8937637731432915} +{"Finetune/Learning Rate": 1.461218981649725e-05, "Finetune/Loss": 0.9176836013793945, "Finetune/Loss (Raw)": 0.9436128735542297, "Finetune/Step": 1907, "Finetune/Step Time": 4.90910611115396} +{"Finetune/Learning Rate": 1.4606661473459801e-05, "Finetune/Loss": 0.917578935623169, "Finetune/Loss (Raw)": 0.8829668164253235, "Finetune/Step": 1908, "Finetune/Step Time": 4.906180385500193} +{"Finetune/Learning Rate": 1.4601131342666973e-05, "Finetune/Loss": 0.9171066284179688, "Finetune/Loss (Raw)": 0.7385892868041992, "Finetune/Step": 1909, "Finetune/Step Time": 4.906678458675742} +{"Finetune/Learning Rate": 1.45955994262649e-05, "Finetune/Loss": 0.9171802401542664, "Finetune/Loss (Raw)": 0.9242020845413208, "Finetune/Step": 1910, "Finetune/Step Time": 4.908954890444875} +{"Finetune/Learning Rate": 1.459006572640041e-05, "Finetune/Loss": 0.918210506439209, "Finetune/Loss (Raw)": 0.9810909628868103, "Finetune/Step": 1911, "Finetune/Step Time": 4.914488635957241} +{"Finetune/Learning Rate": 1.4584530245221025e-05, "Finetune/Loss": 0.9175236821174622, "Finetune/Loss (Raw)": 0.9695672988891602, "Finetune/Step": 1912, "Finetune/Step Time": 4.908292479813099} +{"Finetune/Learning Rate": 1.4578992984874956e-05, "Finetune/Loss": 0.9191031455993652, "Finetune/Loss (Raw)": 0.9593187570571899, "Finetune/Step": 1913, "Finetune/Step Time": 4.892465300858021} +{"Finetune/Learning Rate": 1.457345394751111e-05, "Finetune/Loss": 0.9204574823379517, "Finetune/Loss (Raw)": 1.0764133930206299, "Finetune/Step": 1914, "Finetune/Step Time": 4.890282895416021} +{"Finetune/Learning Rate": 1.4567913135279075e-05, "Finetune/Loss": 0.9224656820297241, "Finetune/Loss (Raw)": 1.0444355010986328, "Finetune/Step": 1915, "Finetune/Step Time": 4.912371067330241} +{"Finetune/Learning Rate": 1.456237055032913e-05, "Finetune/Loss": 0.9229342937469482, "Finetune/Loss (Raw)": 0.9186487197875977, "Finetune/Step": 1916, "Finetune/Step Time": 4.912189280614257} +{"Finetune/Learning Rate": 1.455682619481225e-05, "Finetune/Loss": 0.9271669983863831, "Finetune/Loss (Raw)": 0.839018702507019, "Finetune/Step": 1917, "Finetune/Step Time": 4.869574818760157} +{"Finetune/Learning Rate": 1.4551280070880089e-05, "Finetune/Loss": 0.9261982440948486, "Finetune/Loss (Raw)": 0.889765739440918, "Finetune/Step": 1918, "Finetune/Step Time": 4.85735135525465} +{"Finetune/Learning Rate": 1.4545732180684989e-05, "Finetune/Loss": 0.9252363443374634, "Finetune/Loss (Raw)": 0.9338216185569763, "Finetune/Step": 1919, "Finetune/Step Time": 4.853217024356127} +{"Finetune/Learning Rate": 1.4540182526379976e-05, "Finetune/Loss": 0.9241278171539307, "Finetune/Loss (Raw)": 0.8447469472885132, "Finetune/Step": 1920, "Finetune/Step Time": 4.863201415166259} +{"Finetune/Learning Rate": 1.4534631110118763e-05, "Finetune/Loss": 0.9257379174232483, "Finetune/Loss (Raw)": 1.0094321966171265, "Finetune/Step": 1921, "Finetune/Step Time": 4.846770657226443} +{"Finetune/Learning Rate": 1.4529077934055754e-05, "Finetune/Loss": 0.9201419949531555, "Finetune/Loss (Raw)": 0.3055996000766754, "Finetune/Step": 1922, "Finetune/Step Time": 4.880358409136534} +{"Finetune/Learning Rate": 1.4523523000346012e-05, "Finetune/Loss": 0.9247624278068542, "Finetune/Loss (Raw)": 0.9619795083999634, "Finetune/Step": 1923, "Finetune/Step Time": 4.845823597162962} +{"Finetune/Learning Rate": 1.4517966311145313e-05, "Finetune/Loss": 0.9257863759994507, "Finetune/Loss (Raw)": 1.039483666419983, "Finetune/Step": 1924, "Finetune/Step Time": 4.844546476379037} +{"Finetune/Learning Rate": 1.4512407868610094e-05, "Finetune/Loss": 0.9269396066665649, "Finetune/Loss (Raw)": 1.1424120664596558, "Finetune/Step": 1925, "Finetune/Step Time": 4.84198541380465} +{"Finetune/Learning Rate": 1.450684767489748e-05, "Finetune/Loss": 0.9223470687866211, "Finetune/Loss (Raw)": 0.3389508128166199, "Finetune/Step": 1926, "Finetune/Step Time": 4.872276274487376} +{"Finetune/Learning Rate": 1.4501285732165271e-05, "Finetune/Loss": 0.9184687733650208, "Finetune/Loss (Raw)": 0.40061816573143005, "Finetune/Step": 1927, "Finetune/Step Time": 4.909215843304992} +{"Finetune/Learning Rate": 1.4495722042571946e-05, "Finetune/Loss": 0.9193724393844604, "Finetune/Loss (Raw)": 1.0238964557647705, "Finetune/Step": 1928, "Finetune/Step Time": 4.896988181397319} +{"Finetune/Learning Rate": 1.4490156608276674e-05, "Finetune/Loss": 0.9174050688743591, "Finetune/Loss (Raw)": 0.8058861494064331, "Finetune/Step": 1929, "Finetune/Step Time": 4.888152526691556} +{"Finetune/Learning Rate": 1.4484589431439283e-05, "Finetune/Loss": 0.9169018268585205, "Finetune/Loss (Raw)": 0.88239586353302, "Finetune/Step": 1930, "Finetune/Step Time": 4.87392408400774} +{"Finetune/Learning Rate": 1.4479020514220285e-05, "Finetune/Loss": 0.917248547077179, "Finetune/Loss (Raw)": 0.9614815711975098, "Finetune/Step": 1931, "Finetune/Step Time": 4.851492220535874} +{"Finetune/Learning Rate": 1.4473449858780874e-05, "Finetune/Loss": 0.9162156581878662, "Finetune/Loss (Raw)": 0.8279100060462952, "Finetune/Step": 1932, "Finetune/Step Time": 4.866011127829552} +{"Finetune/Learning Rate": 1.4467877467282906e-05, "Finetune/Loss": 0.9187313318252563, "Finetune/Loss (Raw)": 0.9664052128791809, "Finetune/Step": 1933, "Finetune/Step Time": 4.858905995264649} +{"Finetune/Learning Rate": 1.4462303341888925e-05, "Finetune/Loss": 0.91986483335495, "Finetune/Loss (Raw)": 1.0022830963134766, "Finetune/Step": 1934, "Finetune/Step Time": 4.839717382565141} +{"Finetune/Learning Rate": 1.4456727484762137e-05, "Finetune/Loss": 0.9206924438476562, "Finetune/Loss (Raw)": 1.037416934967041, "Finetune/Step": 1935, "Finetune/Step Time": 4.845996122807264} +{"Finetune/Learning Rate": 1.4451149898066418e-05, "Finetune/Loss": 0.9198600649833679, "Finetune/Loss (Raw)": 0.8348288536071777, "Finetune/Step": 1936, "Finetune/Step Time": 4.85479730181396} +{"Finetune/Learning Rate": 1.4445570583966329e-05, "Finetune/Loss": 0.9200647473335266, "Finetune/Loss (Raw)": 0.9658771753311157, "Finetune/Step": 1937, "Finetune/Step Time": 4.853536017239094} +{"Finetune/Learning Rate": 1.4439989544627085e-05, "Finetune/Loss": 0.9158622622489929, "Finetune/Loss (Raw)": 0.40727677941322327, "Finetune/Step": 1938, "Finetune/Step Time": 4.883364696055651} +{"Finetune/Learning Rate": 1.4434406782214584e-05, "Finetune/Loss": 0.915839433670044, "Finetune/Loss (Raw)": 0.9288436770439148, "Finetune/Step": 1939, "Finetune/Step Time": 4.873398249968886} +{"Finetune/Learning Rate": 1.4428822298895386e-05, "Finetune/Loss": 0.9134195446968079, "Finetune/Loss (Raw)": 0.5854547619819641, "Finetune/Step": 1940, "Finetune/Step Time": 4.874304508790374} +{"Finetune/Learning Rate": 1.442323609683672e-05, "Finetune/Loss": 0.9133546948432922, "Finetune/Loss (Raw)": 0.9064441919326782, "Finetune/Step": 1941, "Finetune/Step Time": 4.873203184455633} +{"Finetune/Learning Rate": 1.4417648178206481e-05, "Finetune/Loss": 0.9145278930664062, "Finetune/Loss (Raw)": 1.1252223253250122, "Finetune/Step": 1942, "Finetune/Step Time": 4.874992240220308} +{"Finetune/Learning Rate": 1.4412058545173228e-05, "Finetune/Loss": 0.9156766533851624, "Finetune/Loss (Raw)": 1.0293703079223633, "Finetune/Step": 1943, "Finetune/Step Time": 4.879772838205099} +{"Finetune/Learning Rate": 1.4406467199906194e-05, "Finetune/Loss": 0.915096640586853, "Finetune/Loss (Raw)": 0.954062283039093, "Finetune/Step": 1944, "Finetune/Step Time": 4.88455256074667} +{"Finetune/Learning Rate": 1.440087414457527e-05, "Finetune/Loss": 0.9110087752342224, "Finetune/Loss (Raw)": 0.38495033979415894, "Finetune/Step": 1945, "Finetune/Step Time": 4.919658862054348} +{"Finetune/Learning Rate": 1.4395279381351008e-05, "Finetune/Loss": 0.9112800359725952, "Finetune/Loss (Raw)": 0.9303086400032043, "Finetune/Step": 1946, "Finetune/Step Time": 4.932175297290087} +{"Finetune/Learning Rate": 1.438968291240463e-05, "Finetune/Loss": 0.9124717712402344, "Finetune/Loss (Raw)": 1.118760108947754, "Finetune/Step": 1947, "Finetune/Step Time": 4.930119272321463} +{"Finetune/Learning Rate": 1.4384084739908016e-05, "Finetune/Loss": 0.9143770933151245, "Finetune/Loss (Raw)": 1.085210919380188, "Finetune/Step": 1948, "Finetune/Step Time": 4.935355495661497} +{"Finetune/Learning Rate": 1.4378484866033705e-05, "Finetune/Loss": 0.9151613712310791, "Finetune/Loss (Raw)": 1.065723180770874, "Finetune/Step": 1949, "Finetune/Step Time": 4.933817902579904} +{"Finetune/Learning Rate": 1.43728832929549e-05, "Finetune/Loss": 0.9143373370170593, "Finetune/Loss (Raw)": 1.0798485279083252, "Finetune/Step": 1950, "Finetune/Step Time": 4.924431631341577} +{"Finetune/Learning Rate": 1.4367280022845462e-05, "Finetune/Loss": 0.9151973724365234, "Finetune/Loss (Raw)": 1.053205966949463, "Finetune/Step": 1951, "Finetune/Step Time": 4.9290716126561165} +{"Finetune/Learning Rate": 1.4361675057879913e-05, "Finetune/Loss": 0.9169853925704956, "Finetune/Loss (Raw)": 1.066519856452942, "Finetune/Step": 1952, "Finetune/Step Time": 4.923284037038684} +{"Finetune/Learning Rate": 1.4356068400233429e-05, "Finetune/Loss": 0.9154878258705139, "Finetune/Loss (Raw)": 0.7106804251670837, "Finetune/Step": 1953, "Finetune/Step Time": 4.935124360024929} +{"Finetune/Learning Rate": 1.435046005208184e-05, "Finetune/Loss": 0.9140756130218506, "Finetune/Loss (Raw)": 0.8362773656845093, "Finetune/Step": 1954, "Finetune/Step Time": 4.943571651354432} +{"Finetune/Learning Rate": 1.4344850015601644e-05, "Finetune/Loss": 0.9132488965988159, "Finetune/Loss (Raw)": 0.827101469039917, "Finetune/Step": 1955, "Finetune/Step Time": 4.9279438722878695} +{"Finetune/Learning Rate": 1.4339238292969981e-05, "Finetune/Loss": 0.9115418195724487, "Finetune/Loss (Raw)": 0.8496142625808716, "Finetune/Step": 1956, "Finetune/Step Time": 4.921963065862656} +{"Finetune/Learning Rate": 1.4333624886364655e-05, "Finetune/Loss": 0.9111481308937073, "Finetune/Loss (Raw)": 1.0722546577453613, "Finetune/Step": 1957, "Finetune/Step Time": 4.9094958901405334} +{"Finetune/Learning Rate": 1.4328009797964115e-05, "Finetune/Loss": 0.9112713932991028, "Finetune/Loss (Raw)": 1.0565299987792969, "Finetune/Step": 1958, "Finetune/Step Time": 4.894652724266052} +{"Finetune/Learning Rate": 1.432239302994747e-05, "Finetune/Loss": 0.9106374382972717, "Finetune/Loss (Raw)": 0.8336002826690674, "Finetune/Step": 1959, "Finetune/Step Time": 4.89727003313601} +{"Finetune/Learning Rate": 1.431677458449448e-05, "Finetune/Loss": 0.9112246036529541, "Finetune/Loss (Raw)": 1.1099097728729248, "Finetune/Step": 1960, "Finetune/Step Time": 4.8971754796803} +{"Finetune/Learning Rate": 1.431115446378555e-05, "Finetune/Loss": 0.910982072353363, "Finetune/Loss (Raw)": 0.8136007785797119, "Finetune/Step": 1961, "Finetune/Step Time": 4.884832112118602} +{"Finetune/Learning Rate": 1.4305532670001743e-05, "Finetune/Loss": 0.9056994915008545, "Finetune/Loss (Raw)": 0.41046327352523804, "Finetune/Step": 1962, "Finetune/Step Time": 4.928071698173881} +{"Finetune/Learning Rate": 1.4299909205324762e-05, "Finetune/Loss": 0.901366114616394, "Finetune/Loss (Raw)": 0.39831680059432983, "Finetune/Step": 1963, "Finetune/Step Time": 4.949870742857456} +{"Finetune/Learning Rate": 1.4294284071936968e-05, "Finetune/Loss": 0.9016512632369995, "Finetune/Loss (Raw)": 1.0113424062728882, "Finetune/Step": 1964, "Finetune/Step Time": 4.956489223986864} +{"Finetune/Learning Rate": 1.4288657272021367e-05, "Finetune/Loss": 0.9006823301315308, "Finetune/Loss (Raw)": 0.8161572217941284, "Finetune/Step": 1965, "Finetune/Step Time": 4.9555173479020596} +{"Finetune/Learning Rate": 1.4283028807761603e-05, "Finetune/Loss": 0.8999056816101074, "Finetune/Loss (Raw)": 0.8754236102104187, "Finetune/Step": 1966, "Finetune/Step Time": 4.9559968970716} +{"Finetune/Learning Rate": 1.4277398681341982e-05, "Finetune/Loss": 0.8988370895385742, "Finetune/Loss (Raw)": 0.83965665102005, "Finetune/Step": 1967, "Finetune/Step Time": 4.938654121011496} +{"Finetune/Learning Rate": 1.4271766894947441e-05, "Finetune/Loss": 0.8989426493644714, "Finetune/Loss (Raw)": 1.0099307298660278, "Finetune/Step": 1968, "Finetune/Step Time": 4.930750275030732} +{"Finetune/Learning Rate": 1.4266133450763564e-05, "Finetune/Loss": 0.8959969282150269, "Finetune/Loss (Raw)": 0.4486735463142395, "Finetune/Step": 1969, "Finetune/Step Time": 4.964474393054843} +{"Finetune/Learning Rate": 1.426049835097659e-05, "Finetune/Loss": 0.8948422074317932, "Finetune/Loss (Raw)": 0.9038426876068115, "Finetune/Step": 1970, "Finetune/Step Time": 4.959313729777932} +{"Finetune/Learning Rate": 1.425486159777338e-05, "Finetune/Loss": 0.8903826475143433, "Finetune/Loss (Raw)": 0.4065403342247009, "Finetune/Step": 1971, "Finetune/Step Time": 4.983870880678296} +{"Finetune/Learning Rate": 1.4249223193341456e-05, "Finetune/Loss": 0.8902336359024048, "Finetune/Loss (Raw)": 0.9148015975952148, "Finetune/Step": 1972, "Finetune/Step Time": 4.971584100276232} +{"Finetune/Learning Rate": 1.4243583139868969e-05, "Finetune/Loss": 0.8898822069168091, "Finetune/Loss (Raw)": 0.8745352625846863, "Finetune/Step": 1973, "Finetune/Step Time": 4.998523559421301} +{"Finetune/Learning Rate": 1.423794143954471e-05, "Finetune/Loss": 0.8900086879730225, "Finetune/Loss (Raw)": 1.0747435092926025, "Finetune/Step": 1974, "Finetune/Step Time": 4.99750492721796} +{"Finetune/Learning Rate": 1.4232298094558125e-05, "Finetune/Loss": 0.8893179893493652, "Finetune/Loss (Raw)": 0.9148582816123962, "Finetune/Step": 1975, "Finetune/Step Time": 4.987119771540165} +{"Finetune/Learning Rate": 1.4226653107099273e-05, "Finetune/Loss": 0.889296293258667, "Finetune/Loss (Raw)": 0.9439612030982971, "Finetune/Step": 1976, "Finetune/Step Time": 4.9900266751646996} +{"Finetune/Learning Rate": 1.422100647935887e-05, "Finetune/Loss": 0.8888707160949707, "Finetune/Loss (Raw)": 0.9873448014259338, "Finetune/Step": 1977, "Finetune/Step Time": 4.998173911124468} +{"Finetune/Learning Rate": 1.421535821352826e-05, "Finetune/Loss": 0.8890207409858704, "Finetune/Loss (Raw)": 1.0470138788223267, "Finetune/Step": 1978, "Finetune/Step Time": 4.991545706987381} +{"Finetune/Learning Rate": 1.4209708311799424e-05, "Finetune/Loss": 0.8853597640991211, "Finetune/Loss (Raw)": 0.37973150610923767, "Finetune/Step": 1979, "Finetune/Step Time": 5.003307729959488} +{"Finetune/Learning Rate": 1.4204056776364985e-05, "Finetune/Loss": 0.8839700222015381, "Finetune/Loss (Raw)": 0.8085788488388062, "Finetune/Step": 1980, "Finetune/Step Time": 5.016439583152533} +{"Finetune/Learning Rate": 1.4198403609418184e-05, "Finetune/Loss": 0.8842295408248901, "Finetune/Loss (Raw)": 1.0014785528182983, "Finetune/Step": 1981, "Finetune/Step Time": 5.016909210011363} +{"Finetune/Learning Rate": 1.419274881315291e-05, "Finetune/Loss": 0.8841267824172974, "Finetune/Loss (Raw)": 0.9678247570991516, "Finetune/Step": 1982, "Finetune/Step Time": 5.024757811799645} +{"Finetune/Learning Rate": 1.418709238976368e-05, "Finetune/Loss": 0.8831614255905151, "Finetune/Loss (Raw)": 0.8994283080101013, "Finetune/Step": 1983, "Finetune/Step Time": 5.0233652330935} +{"Finetune/Learning Rate": 1.4181434341445634e-05, "Finetune/Loss": 0.8834108114242554, "Finetune/Loss (Raw)": 1.0941516160964966, "Finetune/Step": 1984, "Finetune/Step Time": 5.023001821711659} +{"Finetune/Learning Rate": 1.4175774670394561e-05, "Finetune/Loss": 0.8843964338302612, "Finetune/Loss (Raw)": 1.0076125860214233, "Finetune/Step": 1985, "Finetune/Step Time": 4.983151545748115} +{"Finetune/Learning Rate": 1.4170113378806862e-05, "Finetune/Loss": 0.8858035802841187, "Finetune/Loss (Raw)": 1.1769956350326538, "Finetune/Step": 1986, "Finetune/Step Time": 4.973159775137901} +{"Finetune/Learning Rate": 1.4164450468879578e-05, "Finetune/Loss": 0.8849105834960938, "Finetune/Loss (Raw)": 0.938221275806427, "Finetune/Step": 1987, "Finetune/Step Time": 4.999763239175081} +{"Finetune/Learning Rate": 1.4158785942810367e-05, "Finetune/Loss": 0.8847280144691467, "Finetune/Loss (Raw)": 0.8741490244865417, "Finetune/Step": 1988, "Finetune/Step Time": 4.997534723952413} +{"Finetune/Learning Rate": 1.4153119802797526e-05, "Finetune/Loss": 0.885630190372467, "Finetune/Loss (Raw)": 0.965566098690033, "Finetune/Step": 1989, "Finetune/Step Time": 4.99893899820745} +{"Finetune/Learning Rate": 1.4147452051039974e-05, "Finetune/Loss": 0.8866481781005859, "Finetune/Loss (Raw)": 0.9855872988700867, "Finetune/Step": 1990, "Finetune/Step Time": 4.9995395597070456} +{"Finetune/Learning Rate": 1.4141782689737252e-05, "Finetune/Loss": 0.8865705728530884, "Finetune/Loss (Raw)": 0.8146471381187439, "Finetune/Step": 1991, "Finetune/Step Time": 4.995854994282126} +{"Finetune/Learning Rate": 1.413611172108953e-05, "Finetune/Loss": 0.8876227140426636, "Finetune/Loss (Raw)": 0.9720070362091064, "Finetune/Step": 1992, "Finetune/Step Time": 4.990921813994646} +{"Finetune/Learning Rate": 1.41304391472976e-05, "Finetune/Loss": 0.886314868927002, "Finetune/Loss (Raw)": 0.861208975315094, "Finetune/Step": 1993, "Finetune/Step Time": 4.995409136638045} +{"Finetune/Learning Rate": 1.4124764970562874e-05, "Finetune/Loss": 0.8841116428375244, "Finetune/Loss (Raw)": 0.754004955291748, "Finetune/Step": 1994, "Finetune/Step Time": 4.991331577301025} +{"Finetune/Learning Rate": 1.4119089193087396e-05, "Finetune/Loss": 0.8817922472953796, "Finetune/Loss (Raw)": 0.7907376289367676, "Finetune/Step": 1995, "Finetune/Step Time": 4.9949713964015245} +{"Finetune/Learning Rate": 1.4113411817073818e-05, "Finetune/Loss": 0.8808460235595703, "Finetune/Loss (Raw)": 0.833021342754364, "Finetune/Step": 1996, "Finetune/Step Time": 4.982179570943117} +{"Finetune/Learning Rate": 1.4107732844725424e-05, "Finetune/Loss": 0.8810049295425415, "Finetune/Loss (Raw)": 1.0644971132278442, "Finetune/Step": 1997, "Finetune/Step Time": 4.9842229783535} +{"Finetune/Learning Rate": 1.4102052278246107e-05, "Finetune/Loss": 0.8811236619949341, "Finetune/Loss (Raw)": 0.904385507106781, "Finetune/Step": 1998, "Finetune/Step Time": 4.984845055267215} +{"Finetune/Learning Rate": 1.4096370119840389e-05, "Finetune/Loss": 0.8770601749420166, "Finetune/Loss (Raw)": 0.4142589569091797, "Finetune/Step": 1999, "Finetune/Step Time": 4.988528190180659} +{"Finetune/Learning Rate": 1.4090686371713403e-05, "Finetune/Loss": 0.8774679899215698, "Finetune/Loss (Raw)": 0.8730071187019348, "Finetune/Step": 2000, "Finetune/Step Time": 5.0122409872710705} +{"Finetune/Learning Rate": 1.4085001036070896e-05, "Finetune/Loss": 0.8819805383682251, "Finetune/Loss (Raw)": 0.946584165096283, "Finetune/Step": 2001, "Finetune/Step Time": 4.981082815676928} +{"Finetune/Learning Rate": 1.4079314115119242e-05, "Finetune/Loss": 0.8811060190200806, "Finetune/Loss (Raw)": 0.9323808550834656, "Finetune/Step": 2002, "Finetune/Step Time": 4.973651660606265} +{"Finetune/Learning Rate": 1.4073625611065422e-05, "Finetune/Loss": 0.8809128999710083, "Finetune/Loss (Raw)": 0.8690909743309021, "Finetune/Step": 2003, "Finetune/Step Time": 4.967303993180394} +{"Finetune/Learning Rate": 1.4067935526117033e-05, "Finetune/Loss": 0.8804193735122681, "Finetune/Loss (Raw)": 0.8886686563491821, "Finetune/Step": 2004, "Finetune/Step Time": 4.9716678615659475} +{"Finetune/Learning Rate": 1.4062243862482286e-05, "Finetune/Loss": 0.8800801038742065, "Finetune/Loss (Raw)": 0.9070125222206116, "Finetune/Step": 2005, "Finetune/Step Time": 4.965408198535442} +{"Finetune/Learning Rate": 1.4056550622370004e-05, "Finetune/Loss": 0.8811586499214172, "Finetune/Loss (Raw)": 0.9315724968910217, "Finetune/Step": 2006, "Finetune/Step Time": 4.9546935353428125} +{"Finetune/Learning Rate": 1.4050855807989626e-05, "Finetune/Loss": 0.8816655278205872, "Finetune/Loss (Raw)": 1.1248726844787598, "Finetune/Step": 2007, "Finetune/Step Time": 4.954950544983149} +{"Finetune/Learning Rate": 1.4045159421551191e-05, "Finetune/Loss": 0.8829804062843323, "Finetune/Loss (Raw)": 0.9716090559959412, "Finetune/Step": 2008, "Finetune/Step Time": 4.945550175383687} +{"Finetune/Learning Rate": 1.4039461465265362e-05, "Finetune/Loss": 0.8834556341171265, "Finetune/Loss (Raw)": 0.8868815898895264, "Finetune/Step": 2009, "Finetune/Step Time": 4.934262687340379} +{"Finetune/Learning Rate": 1.4033761941343403e-05, "Finetune/Loss": 0.883742094039917, "Finetune/Loss (Raw)": 1.0217365026474, "Finetune/Step": 2010, "Finetune/Step Time": 4.934051996096969} +{"Finetune/Learning Rate": 1.4028060851997183e-05, "Finetune/Loss": 0.8855577111244202, "Finetune/Loss (Raw)": 1.0953601598739624, "Finetune/Step": 2011, "Finetune/Step Time": 4.9326567985117435} +{"Finetune/Learning Rate": 1.4022358199439193e-05, "Finetune/Loss": 0.8899719715118408, "Finetune/Loss (Raw)": 0.9237591028213501, "Finetune/Step": 2012, "Finetune/Step Time": 4.895812556147575} +{"Finetune/Learning Rate": 1.4016653985882512e-05, "Finetune/Loss": 0.8902269601821899, "Finetune/Loss (Raw)": 1.0070394277572632, "Finetune/Step": 2013, "Finetune/Step Time": 4.912593120709062} +{"Finetune/Learning Rate": 1.4010948213540838e-05, "Finetune/Loss": 0.8879265785217285, "Finetune/Loss (Raw)": 0.7831207513809204, "Finetune/Step": 2014, "Finetune/Step Time": 4.921771103516221} +{"Finetune/Learning Rate": 1.4005240884628473e-05, "Finetune/Loss": 0.8847082257270813, "Finetune/Loss (Raw)": 0.3945389986038208, "Finetune/Step": 2015, "Finetune/Step Time": 4.954360909759998} +{"Finetune/Learning Rate": 1.3999532001360309e-05, "Finetune/Loss": 0.8819152116775513, "Finetune/Loss (Raw)": 0.40425169467926025, "Finetune/Step": 2016, "Finetune/Step Time": 4.979369612410665} +{"Finetune/Learning Rate": 1.3993821565951868e-05, "Finetune/Loss": 0.881252110004425, "Finetune/Loss (Raw)": 0.8469547033309937, "Finetune/Step": 2017, "Finetune/Step Time": 4.979686841368675} +{"Finetune/Learning Rate": 1.3988109580619244e-05, "Finetune/Loss": 0.88161301612854, "Finetune/Loss (Raw)": 0.9577447772026062, "Finetune/Step": 2018, "Finetune/Step Time": 4.981090005487204} +{"Finetune/Learning Rate": 1.3982396047579156e-05, "Finetune/Loss": 0.8805031776428223, "Finetune/Loss (Raw)": 0.8675820231437683, "Finetune/Step": 2019, "Finetune/Step Time": 4.993124829605222} +{"Finetune/Learning Rate": 1.3976680969048912e-05, "Finetune/Loss": 0.8794693946838379, "Finetune/Loss (Raw)": 0.9703785181045532, "Finetune/Step": 2020, "Finetune/Step Time": 5.0022191517055035} +{"Finetune/Learning Rate": 1.3970964347246419e-05, "Finetune/Loss": 0.8797376155853271, "Finetune/Loss (Raw)": 0.9541615843772888, "Finetune/Step": 2021, "Finetune/Step Time": 4.998535133898258} +{"Finetune/Learning Rate": 1.3965246184390193e-05, "Finetune/Loss": 0.8804575204849243, "Finetune/Loss (Raw)": 0.9123350977897644, "Finetune/Step": 2022, "Finetune/Step Time": 4.999481119215488} +{"Finetune/Learning Rate": 1.3959526482699335e-05, "Finetune/Loss": 0.8793730735778809, "Finetune/Loss (Raw)": 0.7624735236167908, "Finetune/Step": 2023, "Finetune/Step Time": 5.024514447897673} +{"Finetune/Learning Rate": 1.3953805244393554e-05, "Finetune/Loss": 0.8791424632072449, "Finetune/Loss (Raw)": 0.9297575950622559, "Finetune/Step": 2024, "Finetune/Step Time": 5.018009530380368} +{"Finetune/Learning Rate": 1.394808247169315e-05, "Finetune/Loss": 0.877400279045105, "Finetune/Loss (Raw)": 0.8673200607299805, "Finetune/Step": 2025, "Finetune/Step Time": 5.030291648581624} +{"Finetune/Learning Rate": 1.3942358166819016e-05, "Finetune/Loss": 0.8771613836288452, "Finetune/Loss (Raw)": 0.9586613774299622, "Finetune/Step": 2026, "Finetune/Step Time": 5.011866310611367} +{"Finetune/Learning Rate": 1.3936632331992648e-05, "Finetune/Loss": 0.8774515390396118, "Finetune/Loss (Raw)": 0.8556513786315918, "Finetune/Step": 2027, "Finetune/Step Time": 4.991393253207207} +{"Finetune/Learning Rate": 1.3930904969436125e-05, "Finetune/Loss": 0.8780034780502319, "Finetune/Loss (Raw)": 0.9584871530532837, "Finetune/Step": 2028, "Finetune/Step Time": 5.0107677560299635} +{"Finetune/Learning Rate": 1.3925176081372129e-05, "Finetune/Loss": 0.879009485244751, "Finetune/Loss (Raw)": 1.0366567373275757, "Finetune/Step": 2029, "Finetune/Step Time": 5.003757864236832} +{"Finetune/Learning Rate": 1.3919445670023933e-05, "Finetune/Loss": 0.8739542961120605, "Finetune/Loss (Raw)": 0.3510454595088959, "Finetune/Step": 2030, "Finetune/Step Time": 5.042561201378703} +{"Finetune/Learning Rate": 1.3913713737615387e-05, "Finetune/Loss": 0.8742567896842957, "Finetune/Loss (Raw)": 0.8931691646575928, "Finetune/Step": 2031, "Finetune/Step Time": 5.020980475470424} +{"Finetune/Learning Rate": 1.3907980286370952e-05, "Finetune/Loss": 0.8751866817474365, "Finetune/Loss (Raw)": 1.0383116006851196, "Finetune/Step": 2032, "Finetune/Step Time": 5.02133521810174} +{"Finetune/Learning Rate": 1.3902245318515661e-05, "Finetune/Loss": 0.880584716796875, "Finetune/Loss (Raw)": 1.0122812986373901, "Finetune/Step": 2033, "Finetune/Step Time": 4.983915578573942} +{"Finetune/Learning Rate": 1.389650883627515e-05, "Finetune/Loss": 0.8816670179367065, "Finetune/Loss (Raw)": 1.0493853092193604, "Finetune/Step": 2034, "Finetune/Step Time": 4.971486520022154} +{"Finetune/Learning Rate": 1.3890770841875632e-05, "Finetune/Loss": 0.8807380795478821, "Finetune/Loss (Raw)": 0.8247138261795044, "Finetune/Step": 2035, "Finetune/Step Time": 4.981348589062691} +{"Finetune/Learning Rate": 1.388503133754391e-05, "Finetune/Loss": 0.8814723491668701, "Finetune/Loss (Raw)": 0.9769607186317444, "Finetune/Step": 2036, "Finetune/Step Time": 4.995082734152675} +{"Finetune/Learning Rate": 1.3879290325507378e-05, "Finetune/Loss": 0.8831583857536316, "Finetune/Loss (Raw)": 0.9543994665145874, "Finetune/Step": 2037, "Finetune/Step Time": 5.009736765176058} +{"Finetune/Learning Rate": 1.3873547807994e-05, "Finetune/Loss": 0.8822054862976074, "Finetune/Loss (Raw)": 0.8022215962409973, "Finetune/Step": 2038, "Finetune/Step Time": 5.012139601632953} +{"Finetune/Learning Rate": 1.386780378723235e-05, "Finetune/Loss": 0.8827377557754517, "Finetune/Loss (Raw)": 1.0492336750030518, "Finetune/Step": 2039, "Finetune/Step Time": 5.006418127566576} +{"Finetune/Learning Rate": 1.3862058265451561e-05, "Finetune/Loss": 0.8830628395080566, "Finetune/Loss (Raw)": 1.0111669301986694, "Finetune/Step": 2040, "Finetune/Step Time": 5.00492887198925} +{"Finetune/Learning Rate": 1.385631124488136e-05, "Finetune/Loss": 0.8831925988197327, "Finetune/Loss (Raw)": 0.9759343266487122, "Finetune/Step": 2041, "Finetune/Step Time": 5.049180708825588} +{"Finetune/Learning Rate": 1.385056272775205e-05, "Finetune/Loss": 0.8811413049697876, "Finetune/Loss (Raw)": 0.8138405084609985, "Finetune/Step": 2042, "Finetune/Step Time": 5.05009445361793} +{"Finetune/Learning Rate": 1.3844812716294523e-05, "Finetune/Loss": 0.880338191986084, "Finetune/Loss (Raw)": 0.9416438341140747, "Finetune/Step": 2043, "Finetune/Step Time": 5.020107598975301} +{"Finetune/Learning Rate": 1.3839061212740244e-05, "Finetune/Loss": 0.8804789781570435, "Finetune/Loss (Raw)": 0.9366728663444519, "Finetune/Step": 2044, "Finetune/Step Time": 5.019627697765827} +{"Finetune/Learning Rate": 1.383330821932126e-05, "Finetune/Loss": 0.881954550743103, "Finetune/Loss (Raw)": 1.0278925895690918, "Finetune/Step": 2045, "Finetune/Step Time": 5.031798524782062} +{"Finetune/Learning Rate": 1.3827553738270196e-05, "Finetune/Loss": 0.8831101059913635, "Finetune/Loss (Raw)": 1.0376701354980469, "Finetune/Step": 2046, "Finetune/Step Time": 5.031955590471625} +{"Finetune/Learning Rate": 1.382179777182025e-05, "Finetune/Loss": 0.8833267688751221, "Finetune/Loss (Raw)": 0.9615534543991089, "Finetune/Step": 2047, "Finetune/Step Time": 5.0355428997427225} +{"Finetune/Learning Rate": 1.3816040322205206e-05, "Finetune/Loss": 0.8830233812332153, "Finetune/Loss (Raw)": 0.8059123158454895, "Finetune/Step": 2048, "Finetune/Step Time": 5.024506215006113} +{"Finetune/Learning Rate": 1.3810281391659414e-05, "Finetune/Loss": 0.8806319236755371, "Finetune/Loss (Raw)": 0.703326940536499, "Finetune/Step": 2049, "Finetune/Step Time": 5.039189798757434} +{"Finetune/Learning Rate": 1.3804520982417802e-05, "Finetune/Loss": 0.884678304195404, "Finetune/Loss (Raw)": 0.8235353231430054, "Finetune/Step": 2050, "Finetune/Step Time": 5.0171422474086285} +{"Finetune/Learning Rate": 1.3798759096715877e-05, "Finetune/Loss": 0.8855456113815308, "Finetune/Loss (Raw)": 1.073001503944397, "Finetune/Step": 2051, "Finetune/Step Time": 5.021125920116901} +{"Finetune/Learning Rate": 1.379299573678971e-05, "Finetune/Loss": 0.8850516080856323, "Finetune/Loss (Raw)": 0.9762485027313232, "Finetune/Step": 2052, "Finetune/Step Time": 5.032942280173302} +{"Finetune/Learning Rate": 1.3787230904875947e-05, "Finetune/Loss": 0.8834372758865356, "Finetune/Loss (Raw)": 0.9357784986495972, "Finetune/Step": 2053, "Finetune/Step Time": 5.035311637446284} +{"Finetune/Learning Rate": 1.3781464603211811e-05, "Finetune/Loss": 0.8889293074607849, "Finetune/Loss (Raw)": 1.041925311088562, "Finetune/Step": 2054, "Finetune/Step Time": 5.001920567825437} +{"Finetune/Learning Rate": 1.3775696834035088e-05, "Finetune/Loss": 0.8936727046966553, "Finetune/Loss (Raw)": 1.0077790021896362, "Finetune/Step": 2055, "Finetune/Step Time": 4.971722912043333} +{"Finetune/Learning Rate": 1.3769927599584141e-05, "Finetune/Loss": 0.8934818506240845, "Finetune/Loss (Raw)": 0.9994596242904663, "Finetune/Step": 2056, "Finetune/Step Time": 4.972243145108223} +{"Finetune/Learning Rate": 1.376415690209789e-05, "Finetune/Loss": 0.8941099643707275, "Finetune/Loss (Raw)": 0.8862951397895813, "Finetune/Step": 2057, "Finetune/Step Time": 4.981242373585701} +{"Finetune/Learning Rate": 1.3758384743815838e-05, "Finetune/Loss": 0.8942796587944031, "Finetune/Loss (Raw)": 0.904109001159668, "Finetune/Step": 2058, "Finetune/Step Time": 4.992232831194997} +{"Finetune/Learning Rate": 1.3752611126978045e-05, "Finetune/Loss": 0.8932777643203735, "Finetune/Loss (Raw)": 0.833237886428833, "Finetune/Step": 2059, "Finetune/Step Time": 4.993677021935582} +{"Finetune/Learning Rate": 1.3746836053825133e-05, "Finetune/Loss": 0.8940969705581665, "Finetune/Loss (Raw)": 0.9327752590179443, "Finetune/Step": 2060, "Finetune/Step Time": 4.973391396924853} +{"Finetune/Learning Rate": 1.3741059526598299e-05, "Finetune/Loss": 0.8945504426956177, "Finetune/Loss (Raw)": 1.0244508981704712, "Finetune/Step": 2061, "Finetune/Step Time": 4.972300328314304} +{"Finetune/Learning Rate": 1.3735281547539304e-05, "Finetune/Loss": 0.8934747576713562, "Finetune/Loss (Raw)": 0.8645930290222168, "Finetune/Step": 2062, "Finetune/Step Time": 4.972166679799557} +{"Finetune/Learning Rate": 1.3729502118890463e-05, "Finetune/Loss": 0.8931877613067627, "Finetune/Loss (Raw)": 1.000685214996338, "Finetune/Step": 2063, "Finetune/Step Time": 4.96419938467443} +{"Finetune/Learning Rate": 1.3723721242894663e-05, "Finetune/Loss": 0.8938663005828857, "Finetune/Loss (Raw)": 0.9216802716255188, "Finetune/Step": 2064, "Finetune/Step Time": 4.977981893345714} +{"Finetune/Learning Rate": 1.3717938921795347e-05, "Finetune/Loss": 0.8939018845558167, "Finetune/Loss (Raw)": 0.9704305529594421, "Finetune/Step": 2065, "Finetune/Step Time": 4.985691452398896} +{"Finetune/Learning Rate": 1.3712155157836522e-05, "Finetune/Loss": 0.8933727145195007, "Finetune/Loss (Raw)": 0.33953651785850525, "Finetune/Step": 2066, "Finetune/Step Time": 4.985861115157604} +{"Finetune/Learning Rate": 1.3706369953262754e-05, "Finetune/Loss": 0.8934266567230225, "Finetune/Loss (Raw)": 0.9357536435127258, "Finetune/Step": 2067, "Finetune/Step Time": 4.987205624580383} +{"Finetune/Learning Rate": 1.3700583310319171e-05, "Finetune/Loss": 0.8960678577423096, "Finetune/Loss (Raw)": 0.9235273003578186, "Finetune/Step": 2068, "Finetune/Step Time": 4.976193629205227} +{"Finetune/Learning Rate": 1.3694795231251453e-05, "Finetune/Loss": 0.8960825800895691, "Finetune/Loss (Raw)": 0.9083274602890015, "Finetune/Step": 2069, "Finetune/Step Time": 4.972588632255793} +{"Finetune/Learning Rate": 1.368900571830584e-05, "Finetune/Loss": 0.8927091360092163, "Finetune/Loss (Raw)": 0.6934264302253723, "Finetune/Step": 2070, "Finetune/Step Time": 4.984586825594306} +{"Finetune/Learning Rate": 1.3683214773729123e-05, "Finetune/Loss": 0.8922804594039917, "Finetune/Loss (Raw)": 0.9744930267333984, "Finetune/Step": 2071, "Finetune/Step Time": 4.985229941084981} +{"Finetune/Learning Rate": 1.3677422399768667e-05, "Finetune/Loss": 0.8934073448181152, "Finetune/Loss (Raw)": 1.098304033279419, "Finetune/Step": 2072, "Finetune/Step Time": 4.982929339632392} +{"Finetune/Learning Rate": 1.3671628598672373e-05, "Finetune/Loss": 0.8975129127502441, "Finetune/Loss (Raw)": 0.9104588031768799, "Finetune/Step": 2073, "Finetune/Step Time": 4.957727130502462} +{"Finetune/Learning Rate": 1.3665833372688702e-05, "Finetune/Loss": 0.8982734680175781, "Finetune/Loss (Raw)": 1.027667760848999, "Finetune/Step": 2074, "Finetune/Step Time": 4.948826145380735} +{"Finetune/Learning Rate": 1.366003672406667e-05, "Finetune/Loss": 0.8963979482650757, "Finetune/Loss (Raw)": 0.8786913752555847, "Finetune/Step": 2075, "Finetune/Step Time": 4.948612365871668} +{"Finetune/Learning Rate": 1.3654238655055836e-05, "Finetune/Loss": 0.893988847732544, "Finetune/Loss (Raw)": 0.7768502235412598, "Finetune/Step": 2076, "Finetune/Step Time": 4.955600634217262} +{"Finetune/Learning Rate": 1.364843916790633e-05, "Finetune/Loss": 0.8924534320831299, "Finetune/Loss (Raw)": 0.8691877722740173, "Finetune/Step": 2077, "Finetune/Step Time": 4.9555340595543385} +{"Finetune/Learning Rate": 1.3642638264868808e-05, "Finetune/Loss": 0.8909310698509216, "Finetune/Loss (Raw)": 0.8849834203720093, "Finetune/Step": 2078, "Finetune/Step Time": 4.955342769622803} +{"Finetune/Learning Rate": 1.3636835948194494e-05, "Finetune/Loss": 0.8899604082107544, "Finetune/Loss (Raw)": 0.9289677739143372, "Finetune/Step": 2079, "Finetune/Step Time": 4.944816987961531} +{"Finetune/Learning Rate": 1.3631032220135152e-05, "Finetune/Loss": 0.8876802325248718, "Finetune/Loss (Raw)": 0.7746517062187195, "Finetune/Step": 2080, "Finetune/Step Time": 4.963764324784279} +{"Finetune/Learning Rate": 1.3625227082943093e-05, "Finetune/Loss": 0.8893287181854248, "Finetune/Loss (Raw)": 0.921690046787262, "Finetune/Step": 2081, "Finetune/Step Time": 4.963664244860411} +{"Finetune/Learning Rate": 1.361942053887118e-05, "Finetune/Loss": 0.8894432783126831, "Finetune/Loss (Raw)": 0.8509352803230286, "Finetune/Step": 2082, "Finetune/Step Time": 4.95053363032639} +{"Finetune/Learning Rate": 1.361361259017282e-05, "Finetune/Loss": 0.8905963897705078, "Finetune/Loss (Raw)": 0.9747003316879272, "Finetune/Step": 2083, "Finetune/Step Time": 4.933203984051943} +{"Finetune/Learning Rate": 1.3607803239101965e-05, "Finetune/Loss": 0.8925843238830566, "Finetune/Loss (Raw)": 1.104067087173462, "Finetune/Step": 2084, "Finetune/Step Time": 4.9291185233742} +{"Finetune/Learning Rate": 1.360199248791311e-05, "Finetune/Loss": 0.8867154121398926, "Finetune/Loss (Raw)": 0.32103225588798523, "Finetune/Step": 2085, "Finetune/Step Time": 4.96392235532403} +{"Finetune/Learning Rate": 1.359618033886129e-05, "Finetune/Loss": 0.8862152099609375, "Finetune/Loss (Raw)": 0.9925156235694885, "Finetune/Step": 2086, "Finetune/Step Time": 4.963179161772132} +{"Finetune/Learning Rate": 1.359036679420209e-05, "Finetune/Loss": 0.8879929780960083, "Finetune/Loss (Raw)": 1.0611509084701538, "Finetune/Step": 2087, "Finetune/Step Time": 4.958544816821814} +{"Finetune/Learning Rate": 1.3584551856191635e-05, "Finetune/Loss": 0.887470543384552, "Finetune/Loss (Raw)": 1.0430388450622559, "Finetune/Step": 2088, "Finetune/Step Time": 4.94877402856946} +{"Finetune/Learning Rate": 1.3578735527086585e-05, "Finetune/Loss": 0.8887219429016113, "Finetune/Loss (Raw)": 0.9737735390663147, "Finetune/Step": 2089, "Finetune/Step Time": 4.943140292540193} +{"Finetune/Learning Rate": 1.357291780914415e-05, "Finetune/Loss": 0.8876655101776123, "Finetune/Loss (Raw)": 0.2752413749694824, "Finetune/Step": 2090, "Finetune/Step Time": 4.938548622652888} +{"Finetune/Learning Rate": 1.3567098704622064e-05, "Finetune/Loss": 0.8911323547363281, "Finetune/Loss (Raw)": 0.8420756459236145, "Finetune/Step": 2091, "Finetune/Step Time": 4.906069051474333} +{"Finetune/Learning Rate": 1.3561278215778618e-05, "Finetune/Loss": 0.8910300135612488, "Finetune/Loss (Raw)": 0.9982445240020752, "Finetune/Step": 2092, "Finetune/Step Time": 4.893730580806732} +{"Finetune/Learning Rate": 1.355545634487262e-05, "Finetune/Loss": 0.8922038078308105, "Finetune/Loss (Raw)": 0.9664023518562317, "Finetune/Step": 2093, "Finetune/Step Time": 4.891680316999555} +{"Finetune/Learning Rate": 1.354963309416343e-05, "Finetune/Loss": 0.8936050534248352, "Finetune/Loss (Raw)": 1.0547773838043213, "Finetune/Step": 2094, "Finetune/Step Time": 4.892332941293716} +{"Finetune/Learning Rate": 1.3543808465910938e-05, "Finetune/Loss": 0.8945749998092651, "Finetune/Loss (Raw)": 0.9638142585754395, "Finetune/Step": 2095, "Finetune/Step Time": 4.907649869099259} +{"Finetune/Learning Rate": 1.3537982462375566e-05, "Finetune/Loss": 0.8939180374145508, "Finetune/Loss (Raw)": 0.925840437412262, "Finetune/Step": 2096, "Finetune/Step Time": 4.903852576389909} +{"Finetune/Learning Rate": 1.3532155085818276e-05, "Finetune/Loss": 0.898637056350708, "Finetune/Loss (Raw)": 1.052704930305481, "Finetune/Step": 2097, "Finetune/Step Time": 4.867276839911938} +{"Finetune/Learning Rate": 1.3526326338500553e-05, "Finetune/Loss": 0.8985604047775269, "Finetune/Loss (Raw)": 0.894034206867218, "Finetune/Step": 2098, "Finetune/Step Time": 4.876091852784157} +{"Finetune/Learning Rate": 1.3520496222684424e-05, "Finetune/Loss": 0.8982170820236206, "Finetune/Loss (Raw)": 0.36259016394615173, "Finetune/Step": 2099, "Finetune/Step Time": 4.872877592220902} +{"Finetune/Learning Rate": 1.3514664740632446e-05, "Finetune/Loss": 0.8972263336181641, "Finetune/Loss (Raw)": 0.7879853844642639, "Finetune/Step": 2100, "Finetune/Step Time": 4.87908817268908} +{"Finetune/Learning Rate": 1.3508831894607696e-05, "Finetune/Loss": 0.8982506990432739, "Finetune/Loss (Raw)": 1.0056545734405518, "Finetune/Step": 2101, "Finetune/Step Time": 4.843303794041276} +{"Finetune/Learning Rate": 1.3502997686873797e-05, "Finetune/Loss": 0.8968676328659058, "Finetune/Loss (Raw)": 0.8977137207984924, "Finetune/Step": 2102, "Finetune/Step Time": 4.851848870515823} +{"Finetune/Learning Rate": 1.3497162119694883e-05, "Finetune/Loss": 0.8926277756690979, "Finetune/Loss (Raw)": 0.37215760350227356, "Finetune/Step": 2103, "Finetune/Step Time": 4.889305982738733} +{"Finetune/Learning Rate": 1.3491325195335627e-05, "Finetune/Loss": 0.8922544717788696, "Finetune/Loss (Raw)": 0.8961748480796814, "Finetune/Step": 2104, "Finetune/Step Time": 4.896655071526766} +{"Finetune/Learning Rate": 1.3485486916061228e-05, "Finetune/Loss": 0.8917871713638306, "Finetune/Loss (Raw)": 0.9275373220443726, "Finetune/Step": 2105, "Finetune/Step Time": 4.906869055703282} +{"Finetune/Learning Rate": 1.3479647284137398e-05, "Finetune/Loss": 0.8908690810203552, "Finetune/Loss (Raw)": 0.9294920563697815, "Finetune/Step": 2106, "Finetune/Step Time": 4.905225593596697} +{"Finetune/Learning Rate": 1.3473806301830398e-05, "Finetune/Loss": 0.8951631784439087, "Finetune/Loss (Raw)": 0.9293733239173889, "Finetune/Step": 2107, "Finetune/Step Time": 4.869081102311611} +{"Finetune/Learning Rate": 1.346796397140699e-05, "Finetune/Loss": 0.8958034515380859, "Finetune/Loss (Raw)": 0.8905377388000488, "Finetune/Step": 2108, "Finetune/Step Time": 4.85675673186779} +{"Finetune/Learning Rate": 1.3462120295134476e-05, "Finetune/Loss": 0.8953642249107361, "Finetune/Loss (Raw)": 0.9452600479125977, "Finetune/Step": 2109, "Finetune/Step Time": 4.860195890069008} +{"Finetune/Learning Rate": 1.3456275275280661e-05, "Finetune/Loss": 0.8953254222869873, "Finetune/Loss (Raw)": 0.962855875492096, "Finetune/Step": 2110, "Finetune/Step Time": 4.868157049641013} +{"Finetune/Learning Rate": 1.3450428914113892e-05, "Finetune/Loss": 0.8954590559005737, "Finetune/Loss (Raw)": 0.9165306687355042, "Finetune/Step": 2111, "Finetune/Step Time": 4.882664466276765} +{"Finetune/Learning Rate": 1.3444581213903026e-05, "Finetune/Loss": 0.890485405921936, "Finetune/Loss (Raw)": 0.4575220048427582, "Finetune/Step": 2112, "Finetune/Step Time": 4.918228147551417} +{"Finetune/Learning Rate": 1.3438732176917437e-05, "Finetune/Loss": 0.8899515867233276, "Finetune/Loss (Raw)": 0.9392950534820557, "Finetune/Step": 2113, "Finetune/Step Time": 4.916087483987212} +{"Finetune/Learning Rate": 1.3432881805427032e-05, "Finetune/Loss": 0.8893687725067139, "Finetune/Loss (Raw)": 1.1023926734924316, "Finetune/Step": 2114, "Finetune/Step Time": 4.934509884566069} +{"Finetune/Learning Rate": 1.3427030101702215e-05, "Finetune/Loss": 0.8898812532424927, "Finetune/Loss (Raw)": 1.0038217306137085, "Finetune/Step": 2115, "Finetune/Step Time": 4.910297041758895} +{"Finetune/Learning Rate": 1.3421177068013925e-05, "Finetune/Loss": 0.8863465189933777, "Finetune/Loss (Raw)": 0.42169538140296936, "Finetune/Step": 2116, "Finetune/Step Time": 4.942710556089878} +{"Finetune/Learning Rate": 1.3415322706633606e-05, "Finetune/Loss": 0.884732723236084, "Finetune/Loss (Raw)": 0.7590090036392212, "Finetune/Step": 2117, "Finetune/Step Time": 4.959162715822458} +{"Finetune/Learning Rate": 1.3409467019833223e-05, "Finetune/Loss": 0.8837608098983765, "Finetune/Loss (Raw)": 0.8611785173416138, "Finetune/Step": 2118, "Finetune/Step Time": 4.9545382261276245} +{"Finetune/Learning Rate": 1.3403610009885258e-05, "Finetune/Loss": 0.8841802477836609, "Finetune/Loss (Raw)": 0.8683335185050964, "Finetune/Step": 2119, "Finetune/Step Time": 4.967863615602255} +{"Finetune/Learning Rate": 1.3397751679062692e-05, "Finetune/Loss": 0.8792070746421814, "Finetune/Loss (Raw)": 0.3354395925998688, "Finetune/Step": 2120, "Finetune/Step Time": 5.003950580954552} +{"Finetune/Learning Rate": 1.3391892029639037e-05, "Finetune/Loss": 0.8797879219055176, "Finetune/Loss (Raw)": 0.9355584383010864, "Finetune/Step": 2121, "Finetune/Step Time": 4.995651068165898} +{"Finetune/Learning Rate": 1.3386031063888309e-05, "Finetune/Loss": 0.881669819355011, "Finetune/Loss (Raw)": 0.9948840737342834, "Finetune/Step": 2122, "Finetune/Step Time": 4.989567823708057} +{"Finetune/Learning Rate": 1.3380168784085028e-05, "Finetune/Loss": 0.8833036422729492, "Finetune/Loss (Raw)": 0.9998738765716553, "Finetune/Step": 2123, "Finetune/Step Time": 4.98069279268384} +{"Finetune/Learning Rate": 1.337430519250424e-05, "Finetune/Loss": 0.8845459818840027, "Finetune/Loss (Raw)": 0.9920327663421631, "Finetune/Step": 2124, "Finetune/Step Time": 4.977185331285} +{"Finetune/Learning Rate": 1.3368440291421475e-05, "Finetune/Loss": 0.8840285539627075, "Finetune/Loss (Raw)": 0.998268723487854, "Finetune/Step": 2125, "Finetune/Step Time": 4.991486392915249} +{"Finetune/Learning Rate": 1.3362574083112804e-05, "Finetune/Loss": 0.8848479390144348, "Finetune/Loss (Raw)": 1.0092673301696777, "Finetune/Step": 2126, "Finetune/Step Time": 4.988322023302317} +{"Finetune/Learning Rate": 1.3356706569854778e-05, "Finetune/Loss": 0.8889271020889282, "Finetune/Loss (Raw)": 0.9363934993743896, "Finetune/Step": 2127, "Finetune/Step Time": 4.955233005806804} +{"Finetune/Learning Rate": 1.3350837753924466e-05, "Finetune/Loss": 0.884888768196106, "Finetune/Loss (Raw)": 0.3561055660247803, "Finetune/Step": 2128, "Finetune/Step Time": 4.966013366356492} +{"Finetune/Learning Rate": 1.3344967637599445e-05, "Finetune/Loss": 0.879683792591095, "Finetune/Loss (Raw)": 0.2803468406200409, "Finetune/Step": 2129, "Finetune/Step Time": 4.9967716839164495} +{"Finetune/Learning Rate": 1.3339096223157782e-05, "Finetune/Loss": 0.8799200654029846, "Finetune/Loss (Raw)": 0.9626227021217346, "Finetune/Step": 2130, "Finetune/Step Time": 4.995354514569044} +{"Finetune/Learning Rate": 1.3333223512878068e-05, "Finetune/Loss": 0.8808504939079285, "Finetune/Loss (Raw)": 0.9881889820098877, "Finetune/Step": 2131, "Finetune/Step Time": 4.985135098919272} +{"Finetune/Learning Rate": 1.3327349509039387e-05, "Finetune/Loss": 0.8807615041732788, "Finetune/Loss (Raw)": 0.8772623538970947, "Finetune/Step": 2132, "Finetune/Step Time": 4.989715896546841} +{"Finetune/Learning Rate": 1.3321474213921317e-05, "Finetune/Loss": 0.8826492428779602, "Finetune/Loss (Raw)": 1.1486525535583496, "Finetune/Step": 2133, "Finetune/Step Time": 4.986921833828092} +{"Finetune/Learning Rate": 1.3315597629803959e-05, "Finetune/Loss": 0.8822523355484009, "Finetune/Loss (Raw)": 0.8807639479637146, "Finetune/Step": 2134, "Finetune/Step Time": 4.997471569105983} +{"Finetune/Learning Rate": 1.3309719758967885e-05, "Finetune/Loss": 0.8802640438079834, "Finetune/Loss (Raw)": 0.8703752160072327, "Finetune/Step": 2135, "Finetune/Step Time": 4.993257896974683} +{"Finetune/Learning Rate": 1.3303840603694192e-05, "Finetune/Loss": 0.88023442029953, "Finetune/Loss (Raw)": 0.9678179025650024, "Finetune/Step": 2136, "Finetune/Step Time": 4.979504702612758} +{"Finetune/Learning Rate": 1.3297960166264462e-05, "Finetune/Loss": 0.8766424059867859, "Finetune/Loss (Raw)": 0.42710620164871216, "Finetune/Step": 2137, "Finetune/Step Time": 5.016309920698404} +{"Finetune/Learning Rate": 1.3292078448960778e-05, "Finetune/Loss": 0.8758520483970642, "Finetune/Loss (Raw)": 0.920566976070404, "Finetune/Step": 2138, "Finetune/Step Time": 5.014390267431736} +{"Finetune/Learning Rate": 1.3286195454065726e-05, "Finetune/Loss": 0.8741023540496826, "Finetune/Loss (Raw)": 0.8714014291763306, "Finetune/Step": 2139, "Finetune/Step Time": 5.023399407044053} +{"Finetune/Learning Rate": 1.3280311183862371e-05, "Finetune/Loss": 0.8744944930076599, "Finetune/Loss (Raw)": 0.9739458560943604, "Finetune/Step": 2140, "Finetune/Step Time": 5.03180929645896} +{"Finetune/Learning Rate": 1.3274425640634287e-05, "Finetune/Loss": 0.875070333480835, "Finetune/Loss (Raw)": 1.080754041671753, "Finetune/Step": 2141, "Finetune/Step Time": 5.017217021435499} +{"Finetune/Learning Rate": 1.3268538826665544e-05, "Finetune/Loss": 0.8756662607192993, "Finetune/Loss (Raw)": 0.8593980073928833, "Finetune/Step": 2142, "Finetune/Step Time": 5.005981342867017} +{"Finetune/Learning Rate": 1.326265074424069e-05, "Finetune/Loss": 0.8798757791519165, "Finetune/Loss (Raw)": 0.9333605766296387, "Finetune/Step": 2143, "Finetune/Step Time": 4.992359694093466} +{"Finetune/Learning Rate": 1.3256761395644787e-05, "Finetune/Loss": 0.8837752342224121, "Finetune/Loss (Raw)": 0.9033792018890381, "Finetune/Step": 2144, "Finetune/Step Time": 4.974047450348735} +{"Finetune/Learning Rate": 1.3250870783163363e-05, "Finetune/Loss": 0.882627546787262, "Finetune/Loss (Raw)": 0.7000542283058167, "Finetune/Step": 2145, "Finetune/Step Time": 4.9743939861655235} +{"Finetune/Learning Rate": 1.3244978909082457e-05, "Finetune/Loss": 0.8823497891426086, "Finetune/Loss (Raw)": 0.9221930503845215, "Finetune/Step": 2146, "Finetune/Step Time": 4.989438908174634} +{"Finetune/Learning Rate": 1.3239085775688591e-05, "Finetune/Loss": 0.8829127550125122, "Finetune/Loss (Raw)": 0.939643383026123, "Finetune/Step": 2147, "Finetune/Step Time": 4.986055668443441} +{"Finetune/Learning Rate": 1.3233191385268771e-05, "Finetune/Loss": 0.8836470246315002, "Finetune/Loss (Raw)": 1.0643540620803833, "Finetune/Step": 2148, "Finetune/Step Time": 4.9766413401812315} +{"Finetune/Learning Rate": 1.3227295740110502e-05, "Finetune/Loss": 0.8839414119720459, "Finetune/Loss (Raw)": 0.991841733455658, "Finetune/Step": 2149, "Finetune/Step Time": 4.970977753400803} +{"Finetune/Learning Rate": 1.3221398842501758e-05, "Finetune/Loss": 0.8839644193649292, "Finetune/Loss (Raw)": 0.9152838587760925, "Finetune/Step": 2150, "Finetune/Step Time": 4.968911973759532} +{"Finetune/Learning Rate": 1.321550069473102e-05, "Finetune/Loss": 0.8868311643600464, "Finetune/Loss (Raw)": 1.1294231414794922, "Finetune/Step": 2151, "Finetune/Step Time": 4.952845325693488} +{"Finetune/Learning Rate": 1.3209601299087238e-05, "Finetune/Loss": 0.8858861923217773, "Finetune/Loss (Raw)": 0.8087995052337646, "Finetune/Step": 2152, "Finetune/Step Time": 4.959183497354388} +{"Finetune/Learning Rate": 1.3203700657859855e-05, "Finetune/Loss": 0.8858222961425781, "Finetune/Loss (Raw)": 0.8591415882110596, "Finetune/Step": 2153, "Finetune/Step Time": 4.948541199788451} +{"Finetune/Learning Rate": 1.3197798773338793e-05, "Finetune/Loss": 0.8850993514060974, "Finetune/Loss (Raw)": 0.8661218881607056, "Finetune/Step": 2154, "Finetune/Step Time": 4.958378272131085} +{"Finetune/Learning Rate": 1.3191895647814457e-05, "Finetune/Loss": 0.8855814933776855, "Finetune/Loss (Raw)": 0.9173678159713745, "Finetune/Step": 2155, "Finetune/Step Time": 4.965112913399935} +{"Finetune/Learning Rate": 1.318599128357774e-05, "Finetune/Loss": 0.8840648531913757, "Finetune/Loss (Raw)": 0.7643542289733887, "Finetune/Step": 2156, "Finetune/Step Time": 4.955509474501014} +{"Finetune/Learning Rate": 1.3180085682920005e-05, "Finetune/Loss": 0.8828516602516174, "Finetune/Loss (Raw)": 0.8813665509223938, "Finetune/Step": 2157, "Finetune/Step Time": 4.970291098579764} +{"Finetune/Learning Rate": 1.31741788481331e-05, "Finetune/Loss": 0.8876581788063049, "Finetune/Loss (Raw)": 0.9662847518920898, "Finetune/Step": 2158, "Finetune/Step Time": 4.924439864233136} +{"Finetune/Learning Rate": 1.316827078150936e-05, "Finetune/Loss": 0.888352632522583, "Finetune/Loss (Raw)": 0.9820607304573059, "Finetune/Step": 2159, "Finetune/Step Time": 4.941426679491997} +{"Finetune/Learning Rate": 1.316236148534158e-05, "Finetune/Loss": 0.8875781893730164, "Finetune/Loss (Raw)": 0.9391800165176392, "Finetune/Step": 2160, "Finetune/Step Time": 4.9318507462739944} +{"Finetune/Learning Rate": 1.3156450961923051e-05, "Finetune/Loss": 0.886439323425293, "Finetune/Loss (Raw)": 0.8665044903755188, "Finetune/Step": 2161, "Finetune/Step Time": 4.926217310130596} +{"Finetune/Learning Rate": 1.3150539213547528e-05, "Finetune/Loss": 0.885388970375061, "Finetune/Loss (Raw)": 0.9149413704872131, "Finetune/Step": 2162, "Finetune/Step Time": 4.936325380578637} +{"Finetune/Learning Rate": 1.3144626242509243e-05, "Finetune/Loss": 0.8858500719070435, "Finetune/Loss (Raw)": 0.8837360143661499, "Finetune/Step": 2163, "Finetune/Step Time": 4.925921496003866} +{"Finetune/Learning Rate": 1.3138712051102908e-05, "Finetune/Loss": 0.8864002227783203, "Finetune/Loss (Raw)": 1.0473785400390625, "Finetune/Step": 2164, "Finetune/Step Time": 4.912525525316596} +{"Finetune/Learning Rate": 1.3132796641623703e-05, "Finetune/Loss": 0.8862442970275879, "Finetune/Loss (Raw)": 0.9344371557235718, "Finetune/Step": 2165, "Finetune/Step Time": 4.922379080206156} +{"Finetune/Learning Rate": 1.3126880016367287e-05, "Finetune/Loss": 0.88750159740448, "Finetune/Loss (Raw)": 0.9631580114364624, "Finetune/Step": 2166, "Finetune/Step Time": 4.922229044139385} +{"Finetune/Learning Rate": 1.3120962177629783e-05, "Finetune/Loss": 0.8876283764839172, "Finetune/Loss (Raw)": 1.0654624700546265, "Finetune/Step": 2167, "Finetune/Step Time": 4.921869000419974} +{"Finetune/Learning Rate": 1.311504312770779e-05, "Finetune/Loss": 0.8869754076004028, "Finetune/Loss (Raw)": 0.927582859992981, "Finetune/Step": 2168, "Finetune/Step Time": 4.926283340901136} +{"Finetune/Learning Rate": 1.3109122868898374e-05, "Finetune/Loss": 0.8868426084518433, "Finetune/Loss (Raw)": 0.9589468240737915, "Finetune/Step": 2169, "Finetune/Step Time": 4.883038979023695} +{"Finetune/Learning Rate": 1.310320140349907e-05, "Finetune/Loss": 0.8870480060577393, "Finetune/Loss (Raw)": 0.8401274085044861, "Finetune/Step": 2170, "Finetune/Step Time": 4.884399205446243} +{"Finetune/Learning Rate": 1.3097278733807892e-05, "Finetune/Loss": 0.886768102645874, "Finetune/Loss (Raw)": 0.9058115482330322, "Finetune/Step": 2171, "Finetune/Step Time": 4.894046608358622} +{"Finetune/Learning Rate": 1.3091354862123304e-05, "Finetune/Loss": 0.8865280747413635, "Finetune/Loss (Raw)": 0.9059549570083618, "Finetune/Step": 2172, "Finetune/Step Time": 4.894354039803147} +{"Finetune/Learning Rate": 1.3085429790744248e-05, "Finetune/Loss": 0.8858374357223511, "Finetune/Loss (Raw)": 0.9394852519035339, "Finetune/Step": 2173, "Finetune/Step Time": 4.89821121096611} +{"Finetune/Learning Rate": 1.3079503521970127e-05, "Finetune/Loss": 0.8837425112724304, "Finetune/Loss (Raw)": 0.769523561000824, "Finetune/Step": 2174, "Finetune/Step Time": 4.9108600206673145} +{"Finetune/Learning Rate": 1.3073576058100815e-05, "Finetune/Loss": 0.883575439453125, "Finetune/Loss (Raw)": 0.9401656985282898, "Finetune/Step": 2175, "Finetune/Step Time": 4.926674352958798} +{"Finetune/Learning Rate": 1.3067647401436636e-05, "Finetune/Loss": 0.8850036859512329, "Finetune/Loss (Raw)": 0.9887319207191467, "Finetune/Step": 2176, "Finetune/Step Time": 4.92643566429615} +{"Finetune/Learning Rate": 1.3061717554278393e-05, "Finetune/Loss": 0.8878800868988037, "Finetune/Loss (Raw)": 1.0715065002441406, "Finetune/Step": 2177, "Finetune/Step Time": 4.9154576770961285} +{"Finetune/Learning Rate": 1.3055786518927342e-05, "Finetune/Loss": 0.8877139091491699, "Finetune/Loss (Raw)": 0.8022608160972595, "Finetune/Step": 2178, "Finetune/Step Time": 4.903253244236112} +{"Finetune/Learning Rate": 1.3049854297685206e-05, "Finetune/Loss": 0.8867930173873901, "Finetune/Loss (Raw)": 0.9551258683204651, "Finetune/Step": 2179, "Finetune/Step Time": 4.913278188556433} +{"Finetune/Learning Rate": 1.3043920892854155e-05, "Finetune/Loss": 0.8867377042770386, "Finetune/Loss (Raw)": 0.969177782535553, "Finetune/Step": 2180, "Finetune/Step Time": 4.907150560989976} +{"Finetune/Learning Rate": 1.3037986306736839e-05, "Finetune/Loss": 0.8870394229888916, "Finetune/Loss (Raw)": 0.9743929505348206, "Finetune/Step": 2181, "Finetune/Step Time": 4.905074968934059} +{"Finetune/Learning Rate": 1.3032050541636343e-05, "Finetune/Loss": 0.8866682648658752, "Finetune/Loss (Raw)": 0.9944162964820862, "Finetune/Step": 2182, "Finetune/Step Time": 4.908676849678159} +{"Finetune/Learning Rate": 1.3026113599856229e-05, "Finetune/Loss": 0.8849613666534424, "Finetune/Loss (Raw)": 0.7892994284629822, "Finetune/Step": 2183, "Finetune/Step Time": 4.925126811489463} +{"Finetune/Learning Rate": 1.302017548370051e-05, "Finetune/Loss": 0.8834864497184753, "Finetune/Loss (Raw)": 0.8106614351272583, "Finetune/Step": 2184, "Finetune/Step Time": 4.940179757773876} +{"Finetune/Learning Rate": 1.3014236195473649e-05, "Finetune/Loss": 0.8838174343109131, "Finetune/Loss (Raw)": 0.9286640286445618, "Finetune/Step": 2185, "Finetune/Step Time": 4.9262767266482115} +{"Finetune/Learning Rate": 1.3008295737480567e-05, "Finetune/Loss": 0.8838050365447998, "Finetune/Loss (Raw)": 0.9025256633758545, "Finetune/Step": 2186, "Finetune/Step Time": 4.911396153271198} +{"Finetune/Learning Rate": 1.300235411202664e-05, "Finetune/Loss": 0.8831806182861328, "Finetune/Loss (Raw)": 0.7533091306686401, "Finetune/Step": 2187, "Finetune/Step Time": 4.94506979547441} +{"Finetune/Learning Rate": 1.2996411321417699e-05, "Finetune/Loss": 0.8830899000167847, "Finetune/Loss (Raw)": 0.9211665987968445, "Finetune/Step": 2188, "Finetune/Step Time": 4.978846488520503} +{"Finetune/Learning Rate": 1.2990467367960027e-05, "Finetune/Loss": 0.8816666603088379, "Finetune/Loss (Raw)": 0.8422754406929016, "Finetune/Step": 2189, "Finetune/Step Time": 4.986511809751391} +{"Finetune/Learning Rate": 1.298452225396035e-05, "Finetune/Loss": 0.8815146684646606, "Finetune/Loss (Raw)": 0.8451375365257263, "Finetune/Step": 2190, "Finetune/Step Time": 4.996327912434936} +{"Finetune/Learning Rate": 1.2978575981725858e-05, "Finetune/Loss": 0.8811583518981934, "Finetune/Loss (Raw)": 0.955071210861206, "Finetune/Step": 2191, "Finetune/Step Time": 5.0114961713552475} +{"Finetune/Learning Rate": 1.2972628553564178e-05, "Finetune/Loss": 0.8802119493484497, "Finetune/Loss (Raw)": 0.8005444407463074, "Finetune/Step": 2192, "Finetune/Step Time": 4.993133541196585} +{"Finetune/Learning Rate": 1.2966679971783387e-05, "Finetune/Loss": 0.8803659677505493, "Finetune/Loss (Raw)": 0.9901435375213623, "Finetune/Step": 2193, "Finetune/Step Time": 4.9880468510091305} +{"Finetune/Learning Rate": 1.2960730238692023e-05, "Finetune/Loss": 0.8853409886360168, "Finetune/Loss (Raw)": 0.9763408899307251, "Finetune/Step": 2194, "Finetune/Step Time": 4.956365708261728} +{"Finetune/Learning Rate": 1.2954779356599056e-05, "Finetune/Loss": 0.8857972025871277, "Finetune/Loss (Raw)": 0.9941467642784119, "Finetune/Step": 2195, "Finetune/Step Time": 4.954960519447923} +{"Finetune/Learning Rate": 1.2948827327813908e-05, "Finetune/Loss": 0.8852864503860474, "Finetune/Loss (Raw)": 0.8581472039222717, "Finetune/Step": 2196, "Finetune/Step Time": 4.96405516564846} +{"Finetune/Learning Rate": 1.2942874154646447e-05, "Finetune/Loss": 0.885970413684845, "Finetune/Loss (Raw)": 0.9958825707435608, "Finetune/Step": 2197, "Finetune/Step Time": 4.965683661401272} +{"Finetune/Learning Rate": 1.2936919839406978e-05, "Finetune/Loss": 0.8874258399009705, "Finetune/Loss (Raw)": 0.8797194957733154, "Finetune/Step": 2198, "Finetune/Step Time": 4.959594056010246} +{"Finetune/Learning Rate": 1.2930964384406259e-05, "Finetune/Loss": 0.8864701986312866, "Finetune/Loss (Raw)": 0.8521715998649597, "Finetune/Step": 2199, "Finetune/Step Time": 4.951296407729387} +{"Finetune/Learning Rate": 1.2925007791955487e-05, "Finetune/Loss": 0.8847280740737915, "Finetune/Loss (Raw)": 0.8753063678741455, "Finetune/Step": 2200, "Finetune/Step Time": 4.958059802651405} +{"Finetune/Learning Rate": 1.2919050064366294e-05, "Finetune/Loss": 0.88040691614151, "Finetune/Loss (Raw)": 0.3573588728904724, "Finetune/Step": 2201, "Finetune/Step Time": 4.981121299788356} +{"Finetune/Learning Rate": 1.2913091203950764e-05, "Finetune/Loss": 0.8795377016067505, "Finetune/Loss (Raw)": 0.9164067506790161, "Finetune/Step": 2202, "Finetune/Step Time": 4.972445983439684} +{"Finetune/Learning Rate": 1.2907131213021404e-05, "Finetune/Loss": 0.8795146942138672, "Finetune/Loss (Raw)": 0.8757386207580566, "Finetune/Step": 2203, "Finetune/Step Time": 4.973181996494532} +{"Finetune/Learning Rate": 1.2901170093891182e-05, "Finetune/Loss": 0.880106508731842, "Finetune/Loss (Raw)": 0.8526091575622559, "Finetune/Step": 2204, "Finetune/Step Time": 4.97049274854362} +{"Finetune/Learning Rate": 1.2895207848873488e-05, "Finetune/Loss": 0.8766655921936035, "Finetune/Loss (Raw)": 0.4287460744380951, "Finetune/Step": 2205, "Finetune/Step Time": 5.009337961673737} +{"Finetune/Learning Rate": 1.288924448028215e-05, "Finetune/Loss": 0.875045120716095, "Finetune/Loss (Raw)": 0.6775640845298767, "Finetune/Step": 2206, "Finetune/Step Time": 5.0698707066476345} +{"Finetune/Learning Rate": 1.288327999043144e-05, "Finetune/Loss": 0.875351071357727, "Finetune/Loss (Raw)": 0.968134880065918, "Finetune/Step": 2207, "Finetune/Step Time": 5.070639658719301} +{"Finetune/Learning Rate": 1.2877314381636052e-05, "Finetune/Loss": 0.8760632276535034, "Finetune/Loss (Raw)": 0.865800142288208, "Finetune/Step": 2208, "Finetune/Step Time": 5.062931556254625} +{"Finetune/Learning Rate": 1.287134765621113e-05, "Finetune/Loss": 0.8758972883224487, "Finetune/Loss (Raw)": 0.9004564881324768, "Finetune/Step": 2209, "Finetune/Step Time": 5.052843622863293} +{"Finetune/Learning Rate": 1.2865379816472244e-05, "Finetune/Loss": 0.8723244667053223, "Finetune/Loss (Raw)": 0.3936145305633545, "Finetune/Step": 2210, "Finetune/Step Time": 5.0917858220636845} +{"Finetune/Learning Rate": 1.2859410864735389e-05, "Finetune/Loss": 0.8726080656051636, "Finetune/Loss (Raw)": 1.0109946727752686, "Finetune/Step": 2211, "Finetune/Step Time": 5.1208345629274845} +{"Finetune/Learning Rate": 1.2853440803317003e-05, "Finetune/Loss": 0.869530200958252, "Finetune/Loss (Raw)": 0.7100977897644043, "Finetune/Step": 2212, "Finetune/Step Time": 5.143720066174865} +{"Finetune/Learning Rate": 1.2847469634533952e-05, "Finetune/Loss": 0.8733434677124023, "Finetune/Loss (Raw)": 0.8091409206390381, "Finetune/Step": 2213, "Finetune/Step Time": 5.114804744720459} +{"Finetune/Learning Rate": 1.2841497360703528e-05, "Finetune/Loss": 0.8711689114570618, "Finetune/Loss (Raw)": 0.714166522026062, "Finetune/Step": 2214, "Finetune/Step Time": 5.121734298765659} +{"Finetune/Learning Rate": 1.283552398414345e-05, "Finetune/Loss": 0.8704806566238403, "Finetune/Loss (Raw)": 0.9730578064918518, "Finetune/Step": 2215, "Finetune/Step Time": 5.116019940003753} +{"Finetune/Learning Rate": 1.2829549507171877e-05, "Finetune/Loss": 0.8697754144668579, "Finetune/Loss (Raw)": 0.9527573585510254, "Finetune/Step": 2216, "Finetune/Step Time": 5.118234630674124} +{"Finetune/Learning Rate": 1.2823573932107381e-05, "Finetune/Loss": 0.8700976371765137, "Finetune/Loss (Raw)": 1.0150234699249268, "Finetune/Step": 2217, "Finetune/Step Time": 5.119764588773251} +{"Finetune/Learning Rate": 1.2817597261268965e-05, "Finetune/Loss": 0.8737432956695557, "Finetune/Loss (Raw)": 0.7418850660324097, "Finetune/Step": 2218, "Finetune/Step Time": 5.094508484005928} +{"Finetune/Learning Rate": 1.2811619496976066e-05, "Finetune/Loss": 0.873920738697052, "Finetune/Loss (Raw)": 0.8647910952568054, "Finetune/Step": 2219, "Finetune/Step Time": 5.125189462676644} +{"Finetune/Learning Rate": 1.2805640641548529e-05, "Finetune/Loss": 0.8738254308700562, "Finetune/Loss (Raw)": 0.9860423803329468, "Finetune/Step": 2220, "Finetune/Step Time": 5.124885408207774} +{"Finetune/Learning Rate": 1.2799660697306636e-05, "Finetune/Loss": 0.8728605508804321, "Finetune/Loss (Raw)": 0.8429006338119507, "Finetune/Step": 2221, "Finetune/Step Time": 5.141617400571704} +{"Finetune/Learning Rate": 1.2793679666571084e-05, "Finetune/Loss": 0.8722386956214905, "Finetune/Loss (Raw)": 0.9751781821250916, "Finetune/Step": 2222, "Finetune/Step Time": 5.145155416801572} +{"Finetune/Learning Rate": 1.2787697551662994e-05, "Finetune/Loss": 0.8720529079437256, "Finetune/Loss (Raw)": 0.9400410652160645, "Finetune/Step": 2223, "Finetune/Step Time": 5.1382842399179935} +{"Finetune/Learning Rate": 1.2781714354903913e-05, "Finetune/Loss": 0.8719810247421265, "Finetune/Loss (Raw)": 0.9166340231895447, "Finetune/Step": 2224, "Finetune/Step Time": 5.139089170843363} +{"Finetune/Learning Rate": 1.2775730078615798e-05, "Finetune/Loss": 0.8707147836685181, "Finetune/Loss (Raw)": 0.8906232118606567, "Finetune/Step": 2225, "Finetune/Step Time": 5.150676064193249} +{"Finetune/Learning Rate": 1.276974472512103e-05, "Finetune/Loss": 0.8710383176803589, "Finetune/Loss (Raw)": 0.9354546070098877, "Finetune/Step": 2226, "Finetune/Step Time": 5.161378663033247} +{"Finetune/Learning Rate": 1.276375829674241e-05, "Finetune/Loss": 0.875913143157959, "Finetune/Loss (Raw)": 0.9865604043006897, "Finetune/Step": 2227, "Finetune/Step Time": 5.130214419215918} +{"Finetune/Learning Rate": 1.275777079580315e-05, "Finetune/Loss": 0.8767532110214233, "Finetune/Loss (Raw)": 0.8955185413360596, "Finetune/Step": 2228, "Finetune/Step Time": 5.134064922109246} +{"Finetune/Learning Rate": 1.2751782224626891e-05, "Finetune/Loss": 0.876052975654602, "Finetune/Loss (Raw)": 0.9160178303718567, "Finetune/Step": 2229, "Finetune/Step Time": 5.133513528853655} +{"Finetune/Learning Rate": 1.274579258553767e-05, "Finetune/Loss": 0.8757810592651367, "Finetune/Loss (Raw)": 0.8629133105278015, "Finetune/Step": 2230, "Finetune/Step Time": 5.134122781455517} +{"Finetune/Learning Rate": 1.2739801880859954e-05, "Finetune/Loss": 0.8806093335151672, "Finetune/Loss (Raw)": 0.9901777505874634, "Finetune/Step": 2231, "Finetune/Step Time": 5.09781457670033} +{"Finetune/Learning Rate": 1.2733810112918618e-05, "Finetune/Loss": 0.8816733360290527, "Finetune/Loss (Raw)": 1.0323694944381714, "Finetune/Step": 2232, "Finetune/Step Time": 5.089089119806886} +{"Finetune/Learning Rate": 1.2727817284038953e-05, "Finetune/Loss": 0.880652904510498, "Finetune/Loss (Raw)": 0.7969250082969666, "Finetune/Step": 2233, "Finetune/Step Time": 5.0871722381561995} +{"Finetune/Learning Rate": 1.2721823396546655e-05, "Finetune/Loss": 0.8810154795646667, "Finetune/Loss (Raw)": 0.9758946895599365, "Finetune/Step": 2234, "Finetune/Step Time": 5.091196725144982} +{"Finetune/Learning Rate": 1.2715828452767833e-05, "Finetune/Loss": 0.8817943930625916, "Finetune/Loss (Raw)": 1.0290749073028564, "Finetune/Step": 2235, "Finetune/Step Time": 5.102850878611207} +{"Finetune/Learning Rate": 1.2709832455029017e-05, "Finetune/Loss": 0.8826421499252319, "Finetune/Loss (Raw)": 0.9990434646606445, "Finetune/Step": 2236, "Finetune/Step Time": 5.103661756962538} +{"Finetune/Learning Rate": 1.2703835405657124e-05, "Finetune/Loss": 0.8822744488716125, "Finetune/Loss (Raw)": 0.8982025980949402, "Finetune/Step": 2237, "Finetune/Step Time": 5.1041384395211935} +{"Finetune/Learning Rate": 1.2697837306979499e-05, "Finetune/Loss": 0.8807638883590698, "Finetune/Loss (Raw)": 0.7694994211196899, "Finetune/Step": 2238, "Finetune/Step Time": 5.108719268813729} +{"Finetune/Learning Rate": 1.2691838161323888e-05, "Finetune/Loss": 0.8827251195907593, "Finetune/Loss (Raw)": 1.1675682067871094, "Finetune/Step": 2239, "Finetune/Step Time": 5.094629988074303} +{"Finetune/Learning Rate": 1.2685837971018438e-05, "Finetune/Loss": 0.8874223232269287, "Finetune/Loss (Raw)": 1.0587711334228516, "Finetune/Step": 2240, "Finetune/Step Time": 5.061438856646419} +{"Finetune/Learning Rate": 1.2679836738391714e-05, "Finetune/Loss": 0.8878723382949829, "Finetune/Loss (Raw)": 0.9968888163566589, "Finetune/Step": 2241, "Finetune/Step Time": 5.069674804806709} +{"Finetune/Learning Rate": 1.2673834465772662e-05, "Finetune/Loss": 0.8862197399139404, "Finetune/Loss (Raw)": 0.8908588886260986, "Finetune/Step": 2242, "Finetune/Step Time": 5.052254255861044} +{"Finetune/Learning Rate": 1.2667831155490662e-05, "Finetune/Loss": 0.8853370547294617, "Finetune/Loss (Raw)": 0.890845775604248, "Finetune/Step": 2243, "Finetune/Step Time": 5.055084323510528} +{"Finetune/Learning Rate": 1.2661826809875474e-05, "Finetune/Loss": 0.8911311626434326, "Finetune/Loss (Raw)": 1.1633365154266357, "Finetune/Step": 2244, "Finetune/Step Time": 5.017506740987301} +{"Finetune/Learning Rate": 1.265582143125727e-05, "Finetune/Loss": 0.8927444219589233, "Finetune/Loss (Raw)": 0.965509295463562, "Finetune/Step": 2245, "Finetune/Step Time": 4.997720215469599} +{"Finetune/Learning Rate": 1.264981502196662e-05, "Finetune/Loss": 0.8945110440254211, "Finetune/Loss (Raw)": 1.0873075723648071, "Finetune/Step": 2246, "Finetune/Step Time": 5.0006138533353806} +{"Finetune/Learning Rate": 1.2643807584334491e-05, "Finetune/Loss": 0.8944350481033325, "Finetune/Loss (Raw)": 0.8586021065711975, "Finetune/Step": 2247, "Finetune/Step Time": 4.9993024207651615} +{"Finetune/Learning Rate": 1.2637799120692253e-05, "Finetune/Loss": 0.8978438973426819, "Finetune/Loss (Raw)": 0.7717757225036621, "Finetune/Step": 2248, "Finetune/Step Time": 4.959751892834902} +{"Finetune/Learning Rate": 1.2631789633371678e-05, "Finetune/Loss": 0.897350549697876, "Finetune/Loss (Raw)": 0.8724108934402466, "Finetune/Step": 2249, "Finetune/Step Time": 4.959982553496957} +{"Finetune/Learning Rate": 1.2625779124704925e-05, "Finetune/Loss": 0.8969791531562805, "Finetune/Loss (Raw)": 0.947342038154602, "Finetune/Step": 2250, "Finetune/Step Time": 4.981169680133462} +{"Finetune/Learning Rate": 1.2619767597024561e-05, "Finetune/Loss": 0.8966202735900879, "Finetune/Loss (Raw)": 0.9539347887039185, "Finetune/Step": 2251, "Finetune/Step Time": 4.982516303658485} +{"Finetune/Learning Rate": 1.2613755052663536e-05, "Finetune/Loss": 0.8913778066635132, "Finetune/Loss (Raw)": 0.32099637389183044, "Finetune/Step": 2252, "Finetune/Step Time": 5.0131489131599665} +{"Finetune/Learning Rate": 1.2607741493955208e-05, "Finetune/Loss": 0.8916634917259216, "Finetune/Loss (Raw)": 1.0348432064056396, "Finetune/Step": 2253, "Finetune/Step Time": 4.993807645514607} +{"Finetune/Learning Rate": 1.2601726923233315e-05, "Finetune/Loss": 0.8919248580932617, "Finetune/Loss (Raw)": 1.0427179336547852, "Finetune/Step": 2254, "Finetune/Step Time": 5.0002599488943815} +{"Finetune/Learning Rate": 1.2595711342831998e-05, "Finetune/Loss": 0.8920779228210449, "Finetune/Loss (Raw)": 0.9559918642044067, "Finetune/Step": 2255, "Finetune/Step Time": 4.997994087636471} +{"Finetune/Learning Rate": 1.2589694755085789e-05, "Finetune/Loss": 0.8962485790252686, "Finetune/Loss (Raw)": 0.8899482488632202, "Finetune/Step": 2256, "Finetune/Step Time": 4.961563140153885} +{"Finetune/Learning Rate": 1.25836771623296e-05, "Finetune/Loss": 0.9017218351364136, "Finetune/Loss (Raw)": 0.980915904045105, "Finetune/Step": 2257, "Finetune/Step Time": 4.929114794358611} +{"Finetune/Learning Rate": 1.257765856689875e-05, "Finetune/Loss": 0.9000358581542969, "Finetune/Loss (Raw)": 0.7468184232711792, "Finetune/Step": 2258, "Finetune/Step Time": 4.929235331714153} +{"Finetune/Learning Rate": 1.2571638971128933e-05, "Finetune/Loss": 0.8989275693893433, "Finetune/Loss (Raw)": 0.846335232257843, "Finetune/Step": 2259, "Finetune/Step Time": 4.926409650593996} +{"Finetune/Learning Rate": 1.2565618377356237e-05, "Finetune/Loss": 0.8987787365913391, "Finetune/Loss (Raw)": 0.8582108616828918, "Finetune/Step": 2260, "Finetune/Step Time": 4.931741559877992} +{"Finetune/Learning Rate": 1.2559596787917144e-05, "Finetune/Loss": 0.8973022699356079, "Finetune/Loss (Raw)": 0.9596653580665588, "Finetune/Step": 2261, "Finetune/Step Time": 4.93407941237092} +{"Finetune/Learning Rate": 1.2553574205148502e-05, "Finetune/Loss": 0.8986167311668396, "Finetune/Loss (Raw)": 1.0490140914916992, "Finetune/Step": 2262, "Finetune/Step Time": 4.92817866243422} +{"Finetune/Learning Rate": 1.2547550631387571e-05, "Finetune/Loss": 0.8984525799751282, "Finetune/Loss (Raw)": 0.8493618369102478, "Finetune/Step": 2263, "Finetune/Step Time": 4.932633398100734} +{"Finetune/Learning Rate": 1.2541526068971975e-05, "Finetune/Loss": 0.8983656167984009, "Finetune/Loss (Raw)": 0.9566900730133057, "Finetune/Step": 2264, "Finetune/Step Time": 4.935640320181847} +{"Finetune/Learning Rate": 1.2535500520239727e-05, "Finetune/Loss": 0.9021157026290894, "Finetune/Loss (Raw)": 0.9071161150932312, "Finetune/Step": 2265, "Finetune/Step Time": 4.901977960020304} +{"Finetune/Learning Rate": 1.2529473987529235e-05, "Finetune/Loss": 0.9008662104606628, "Finetune/Loss (Raw)": 0.760631263256073, "Finetune/Step": 2266, "Finetune/Step Time": 4.914670038968325} +{"Finetune/Learning Rate": 1.2523446473179267e-05, "Finetune/Loss": 0.9009685516357422, "Finetune/Loss (Raw)": 0.8845043182373047, "Finetune/Step": 2267, "Finetune/Step Time": 4.8943454567343} +{"Finetune/Learning Rate": 1.2517417979528991e-05, "Finetune/Loss": 0.8999478220939636, "Finetune/Loss (Raw)": 0.8432897925376892, "Finetune/Step": 2268, "Finetune/Step Time": 4.886318257078528} +{"Finetune/Learning Rate": 1.2511388508917946e-05, "Finetune/Loss": 0.8988516330718994, "Finetune/Loss (Raw)": 0.9404383301734924, "Finetune/Step": 2269, "Finetune/Step Time": 4.879675466567278} +{"Finetune/Learning Rate": 1.250535806368605e-05, "Finetune/Loss": 0.898245096206665, "Finetune/Loss (Raw)": 0.7817615866661072, "Finetune/Step": 2270, "Finetune/Step Time": 4.88876867480576} +{"Finetune/Learning Rate": 1.2499326646173606e-05, "Finetune/Loss": 0.8984693288803101, "Finetune/Loss (Raw)": 0.9620617628097534, "Finetune/Step": 2271, "Finetune/Step Time": 4.862896732985973} +{"Finetune/Learning Rate": 1.2493294258721282e-05, "Finetune/Loss": 0.899075448513031, "Finetune/Loss (Raw)": 0.980964720249176, "Finetune/Step": 2272, "Finetune/Step Time": 4.851843060925603} +{"Finetune/Learning Rate": 1.2487260903670136e-05, "Finetune/Loss": 0.9008851051330566, "Finetune/Loss (Raw)": 0.9316930174827576, "Finetune/Step": 2273, "Finetune/Step Time": 4.842323863878846} +{"Finetune/Learning Rate": 1.2481226583361592e-05, "Finetune/Loss": 0.9002132415771484, "Finetune/Loss (Raw)": 0.8361989259719849, "Finetune/Step": 2274, "Finetune/Step Time": 4.853598058223724} +{"Finetune/Learning Rate": 1.2475191300137452e-05, "Finetune/Loss": 0.9009482264518738, "Finetune/Loss (Raw)": 1.0337141752243042, "Finetune/Step": 2275, "Finetune/Step Time": 4.84510607086122} +{"Finetune/Learning Rate": 1.2469155056339894e-05, "Finetune/Loss": 0.8999230265617371, "Finetune/Loss (Raw)": 0.9331303834915161, "Finetune/Step": 2276, "Finetune/Step Time": 4.8534603882581} +{"Finetune/Learning Rate": 1.2463117854311463e-05, "Finetune/Loss": 0.8980448246002197, "Finetune/Loss (Raw)": 0.751433789730072, "Finetune/Step": 2277, "Finetune/Step Time": 4.858186107128859} +{"Finetune/Learning Rate": 1.2457079696395082e-05, "Finetune/Loss": 0.8984960317611694, "Finetune/Loss (Raw)": 0.973039984703064, "Finetune/Step": 2278, "Finetune/Step Time": 4.856280852109194} +{"Finetune/Learning Rate": 1.2451040584934042e-05, "Finetune/Loss": 0.8976865410804749, "Finetune/Loss (Raw)": 1.0258080959320068, "Finetune/Step": 2279, "Finetune/Step Time": 4.8453703206032515} +{"Finetune/Learning Rate": 1.2445000522272005e-05, "Finetune/Loss": 0.8983539342880249, "Finetune/Loss (Raw)": 0.8942219018936157, "Finetune/Step": 2280, "Finetune/Step Time": 4.8402791284024715} +{"Finetune/Learning Rate": 1.2438959510752997e-05, "Finetune/Loss": 0.8986837863922119, "Finetune/Loss (Raw)": 0.9013584852218628, "Finetune/Step": 2281, "Finetune/Step Time": 4.841757010668516} +{"Finetune/Learning Rate": 1.243291755272142e-05, "Finetune/Loss": 0.8989426493644714, "Finetune/Loss (Raw)": 0.8992618918418884, "Finetune/Step": 2282, "Finetune/Step Time": 4.835342915728688} +{"Finetune/Learning Rate": 1.2426874650522039e-05, "Finetune/Loss": 0.899834930896759, "Finetune/Loss (Raw)": 1.0315783023834229, "Finetune/Step": 2283, "Finetune/Step Time": 4.822896238416433} +{"Finetune/Learning Rate": 1.2420830806499988e-05, "Finetune/Loss": 0.900797963142395, "Finetune/Loss (Raw)": 0.8876296281814575, "Finetune/Step": 2284, "Finetune/Step Time": 4.841704420745373} +{"Finetune/Learning Rate": 1.2414786023000765e-05, "Finetune/Loss": 0.9013861417770386, "Finetune/Loss (Raw)": 0.9566507339477539, "Finetune/Step": 2285, "Finetune/Step Time": 4.8360351752489805} +{"Finetune/Learning Rate": 1.240874030237023e-05, "Finetune/Loss": 0.9012856483459473, "Finetune/Loss (Raw)": 0.9534196257591248, "Finetune/Step": 2286, "Finetune/Step Time": 4.846775498241186} +{"Finetune/Learning Rate": 1.2402693646954607e-05, "Finetune/Loss": 0.8996226787567139, "Finetune/Loss (Raw)": 0.7691981196403503, "Finetune/Step": 2287, "Finetune/Step Time": 4.8315045889467} +{"Finetune/Learning Rate": 1.2396646059100492e-05, "Finetune/Loss": 0.899726152420044, "Finetune/Loss (Raw)": 0.9524204134941101, "Finetune/Step": 2288, "Finetune/Step Time": 4.841090796515346} +{"Finetune/Learning Rate": 1.2390597541154832e-05, "Finetune/Loss": 0.8987154364585876, "Finetune/Loss (Raw)": 0.7371383905410767, "Finetune/Step": 2289, "Finetune/Step Time": 4.854440398514271} +{"Finetune/Learning Rate": 1.238454809546494e-05, "Finetune/Loss": 0.8994170427322388, "Finetune/Loss (Raw)": 1.0047465562820435, "Finetune/Step": 2290, "Finetune/Step Time": 4.847482958808541} +{"Finetune/Learning Rate": 1.2378497724378484e-05, "Finetune/Loss": 0.8990548849105835, "Finetune/Loss (Raw)": 0.8373805284500122, "Finetune/Step": 2291, "Finetune/Step Time": 4.854600878432393} +{"Finetune/Learning Rate": 1.2372446430243497e-05, "Finetune/Loss": 0.8975688219070435, "Finetune/Loss (Raw)": 0.857164740562439, "Finetune/Step": 2292, "Finetune/Step Time": 4.862696912139654} +{"Finetune/Learning Rate": 1.2366394215408369e-05, "Finetune/Loss": 0.8960444927215576, "Finetune/Loss (Raw)": 0.739321768283844, "Finetune/Step": 2293, "Finetune/Step Time": 4.84751445427537} +{"Finetune/Learning Rate": 1.2360341082221847e-05, "Finetune/Loss": 0.8958325386047363, "Finetune/Loss (Raw)": 0.9360270500183105, "Finetune/Step": 2294, "Finetune/Step Time": 4.84553231112659} +{"Finetune/Learning Rate": 1.2354287033033032e-05, "Finetune/Loss": 0.8947262763977051, "Finetune/Loss (Raw)": 0.923859715461731, "Finetune/Step": 2295, "Finetune/Step Time": 4.863815354183316} +{"Finetune/Learning Rate": 1.2348232070191383e-05, "Finetune/Loss": 0.893437385559082, "Finetune/Loss (Raw)": 0.7626010775566101, "Finetune/Step": 2296, "Finetune/Step Time": 4.859083300456405} +{"Finetune/Learning Rate": 1.2342176196046715e-05, "Finetune/Loss": 0.8931987881660461, "Finetune/Loss (Raw)": 0.9284086227416992, "Finetune/Step": 2297, "Finetune/Step Time": 4.861375451087952} +{"Finetune/Learning Rate": 1.2336119412949193e-05, "Finetune/Loss": 0.8934287428855896, "Finetune/Loss (Raw)": 0.8695618510246277, "Finetune/Step": 2298, "Finetune/Step Time": 4.862792203202844} +{"Finetune/Learning Rate": 1.2330061723249328e-05, "Finetune/Loss": 0.8939772248268127, "Finetune/Loss (Raw)": 0.9760175347328186, "Finetune/Step": 2299, "Finetune/Step Time": 4.8604362066835165} +{"Finetune/Learning Rate": 1.2324003129298006e-05, "Finetune/Loss": 0.8945093154907227, "Finetune/Loss (Raw)": 0.9740585684776306, "Finetune/Step": 2300, "Finetune/Step Time": 4.8714379500597715} +{"Finetune/Learning Rate": 1.2317943633446441e-05, "Finetune/Loss": 0.8937183618545532, "Finetune/Loss (Raw)": 0.8382520079612732, "Finetune/Step": 2301, "Finetune/Step Time": 4.857778515666723} +{"Finetune/Learning Rate": 1.2311883238046204e-05, "Finetune/Loss": 0.8939672708511353, "Finetune/Loss (Raw)": 0.8013842105865479, "Finetune/Step": 2302, "Finetune/Step Time": 4.850322440266609} +{"Finetune/Learning Rate": 1.2305821945449222e-05, "Finetune/Loss": 0.8937877416610718, "Finetune/Loss (Raw)": 0.9171815514564514, "Finetune/Step": 2303, "Finetune/Step Time": 4.832656590268016} +{"Finetune/Learning Rate": 1.2299759758007755e-05, "Finetune/Loss": 0.8917058706283569, "Finetune/Loss (Raw)": 0.7222504615783691, "Finetune/Step": 2304, "Finetune/Step Time": 4.841054296121001} +{"Finetune/Learning Rate": 1.2293696678074428e-05, "Finetune/Loss": 0.8896776437759399, "Finetune/Loss (Raw)": 0.8118933439254761, "Finetune/Step": 2305, "Finetune/Step Time": 4.836844000965357} +{"Finetune/Learning Rate": 1.22876327080022e-05, "Finetune/Loss": 0.8916088342666626, "Finetune/Loss (Raw)": 1.0494489669799805, "Finetune/Step": 2306, "Finetune/Step Time": 4.838871220126748} +{"Finetune/Learning Rate": 1.2281567850144383e-05, "Finetune/Loss": 0.891750693321228, "Finetune/Loss (Raw)": 0.9732900857925415, "Finetune/Step": 2307, "Finetune/Step Time": 4.822863068431616} +{"Finetune/Learning Rate": 1.2275502106854624e-05, "Finetune/Loss": 0.8889592885971069, "Finetune/Loss (Raw)": 0.6118809580802917, "Finetune/Step": 2308, "Finetune/Step Time": 4.825550843030214} +{"Finetune/Learning Rate": 1.2269435480486923e-05, "Finetune/Loss": 0.8887203931808472, "Finetune/Loss (Raw)": 0.9438121914863586, "Finetune/Step": 2309, "Finetune/Step Time": 4.825670709833503} +{"Finetune/Learning Rate": 1.226336797339562e-05, "Finetune/Loss": 0.8884351253509521, "Finetune/Loss (Raw)": 0.9579006433486938, "Finetune/Step": 2310, "Finetune/Step Time": 4.820506576448679} +{"Finetune/Learning Rate": 1.2257299587935394e-05, "Finetune/Loss": 0.8903292417526245, "Finetune/Loss (Raw)": 1.0317490100860596, "Finetune/Step": 2311, "Finetune/Step Time": 4.799901001155376} +{"Finetune/Learning Rate": 1.2251230326461268e-05, "Finetune/Loss": 0.8908260464668274, "Finetune/Loss (Raw)": 0.8742536306381226, "Finetune/Step": 2312, "Finetune/Step Time": 4.808817960321903} +{"Finetune/Learning Rate": 1.2245160191328605e-05, "Finetune/Loss": 0.8909306526184082, "Finetune/Loss (Raw)": 0.942051112651825, "Finetune/Step": 2313, "Finetune/Step Time": 4.814431784674525} +{"Finetune/Learning Rate": 1.2239089184893102e-05, "Finetune/Loss": 0.8905931711196899, "Finetune/Loss (Raw)": 0.8593254685401917, "Finetune/Step": 2314, "Finetune/Step Time": 4.822842938825488} +{"Finetune/Learning Rate": 1.2233017309510804e-05, "Finetune/Loss": 0.8914273977279663, "Finetune/Loss (Raw)": 0.8600975275039673, "Finetune/Step": 2315, "Finetune/Step Time": 4.788791388273239} +{"Finetune/Learning Rate": 1.2226944567538084e-05, "Finetune/Loss": 0.8919076919555664, "Finetune/Loss (Raw)": 0.9826364517211914, "Finetune/Step": 2316, "Finetune/Step Time": 4.753573887050152} +{"Finetune/Learning Rate": 1.2220870961331654e-05, "Finetune/Loss": 0.8920220136642456, "Finetune/Loss (Raw)": 0.8569118976593018, "Finetune/Step": 2317, "Finetune/Step Time": 4.750084713101387} +{"Finetune/Learning Rate": 1.2214796493248564e-05, "Finetune/Loss": 0.8923308849334717, "Finetune/Loss (Raw)": 0.8846710324287415, "Finetune/Step": 2318, "Finetune/Step Time": 4.737744649872184} +{"Finetune/Learning Rate": 1.2208721165646193e-05, "Finetune/Loss": 0.8933281898498535, "Finetune/Loss (Raw)": 1.0827295780181885, "Finetune/Step": 2319, "Finetune/Step Time": 4.7287861332297325} +{"Finetune/Learning Rate": 1.2202644980882258e-05, "Finetune/Loss": 0.8943524956703186, "Finetune/Loss (Raw)": 0.9316505193710327, "Finetune/Step": 2320, "Finetune/Step Time": 4.727223988622427} +{"Finetune/Learning Rate": 1.2196567941314815e-05, "Finetune/Loss": 0.8938885927200317, "Finetune/Loss (Raw)": 0.9307728409767151, "Finetune/Step": 2321, "Finetune/Step Time": 4.734748389571905} +{"Finetune/Learning Rate": 1.2190490049302233e-05, "Finetune/Loss": 0.89398193359375, "Finetune/Loss (Raw)": 0.9882817268371582, "Finetune/Step": 2322, "Finetune/Step Time": 4.734911695122719} +{"Finetune/Learning Rate": 1.218441130720323e-05, "Finetune/Loss": 0.8924540281295776, "Finetune/Loss (Raw)": 0.7985693216323853, "Finetune/Step": 2323, "Finetune/Step Time": 4.749567169696093} +{"Finetune/Learning Rate": 1.2178331717376848e-05, "Finetune/Loss": 0.892456591129303, "Finetune/Loss (Raw)": 0.8584809303283691, "Finetune/Step": 2324, "Finetune/Step Time": 4.739582676440477} +{"Finetune/Learning Rate": 1.217225128218245e-05, "Finetune/Loss": 0.8926133513450623, "Finetune/Loss (Raw)": 1.0159478187561035, "Finetune/Step": 2325, "Finetune/Step Time": 4.746697774156928} +{"Finetune/Learning Rate": 1.2166170003979742e-05, "Finetune/Loss": 0.8891642093658447, "Finetune/Loss (Raw)": 0.43822547793388367, "Finetune/Step": 2326, "Finetune/Step Time": 4.780179228633642} +{"Finetune/Learning Rate": 1.2160087885128745e-05, "Finetune/Loss": 0.8896140456199646, "Finetune/Loss (Raw)": 0.9097543954849243, "Finetune/Step": 2327, "Finetune/Step Time": 4.79591772146523} +{"Finetune/Learning Rate": 1.2154004927989815e-05, "Finetune/Loss": 0.8894355297088623, "Finetune/Loss (Raw)": 0.8524622321128845, "Finetune/Step": 2328, "Finetune/Step Time": 4.789802359417081} +{"Finetune/Learning Rate": 1.2147921134923624e-05, "Finetune/Loss": 0.8943126201629639, "Finetune/Loss (Raw)": 0.9816266894340515, "Finetune/Step": 2329, "Finetune/Step Time": 4.7567502446472645} +{"Finetune/Learning Rate": 1.2141836508291175e-05, "Finetune/Loss": 0.8934884071350098, "Finetune/Loss (Raw)": 0.8109034299850464, "Finetune/Step": 2330, "Finetune/Step Time": 4.769911581650376} +{"Finetune/Learning Rate": 1.2135751050453793e-05, "Finetune/Loss": 0.8939687013626099, "Finetune/Loss (Raw)": 0.9372156262397766, "Finetune/Step": 2331, "Finetune/Step Time": 4.778058554977179} +{"Finetune/Learning Rate": 1.2129664763773125e-05, "Finetune/Loss": 0.8960468173027039, "Finetune/Loss (Raw)": 1.118608832359314, "Finetune/Step": 2332, "Finetune/Step Time": 4.773869454860687} +{"Finetune/Learning Rate": 1.212357765061114e-05, "Finetune/Loss": 0.900496244430542, "Finetune/Loss (Raw)": 0.9982700943946838, "Finetune/Step": 2333, "Finetune/Step Time": 4.752186950296164} +{"Finetune/Learning Rate": 1.211748971333013e-05, "Finetune/Loss": 0.9025970697402954, "Finetune/Loss (Raw)": 0.946472704410553, "Finetune/Step": 2334, "Finetune/Step Time": 4.7112972792238} +{"Finetune/Learning Rate": 1.21114009542927e-05, "Finetune/Loss": 0.9018828868865967, "Finetune/Loss (Raw)": 0.876721978187561, "Finetune/Step": 2335, "Finetune/Step Time": 4.707863427698612} +{"Finetune/Learning Rate": 1.2105311375861786e-05, "Finetune/Loss": 0.9020626544952393, "Finetune/Loss (Raw)": 0.8888022899627686, "Finetune/Step": 2336, "Finetune/Step Time": 4.713153405115008} +{"Finetune/Learning Rate": 1.2099220980400625e-05, "Finetune/Loss": 0.9025870561599731, "Finetune/Loss (Raw)": 0.9675856232643127, "Finetune/Step": 2337, "Finetune/Step Time": 4.717969506978989} +{"Finetune/Learning Rate": 1.2093129770272787e-05, "Finetune/Loss": 0.9071847200393677, "Finetune/Loss (Raw)": 0.9821146726608276, "Finetune/Step": 2338, "Finetune/Step Time": 4.678626526147127} +{"Finetune/Learning Rate": 1.208703774784215e-05, "Finetune/Loss": 0.9058704376220703, "Finetune/Loss (Raw)": 0.8427616953849792, "Finetune/Step": 2339, "Finetune/Step Time": 4.6694524716585875} +{"Finetune/Learning Rate": 1.2080944915472908e-05, "Finetune/Loss": 0.9074429869651794, "Finetune/Loss (Raw)": 0.9113885760307312, "Finetune/Step": 2340, "Finetune/Step Time": 4.6551537327468395} +{"Finetune/Learning Rate": 1.2074851275529575e-05, "Finetune/Loss": 0.9092504978179932, "Finetune/Loss (Raw)": 1.0405032634735107, "Finetune/Step": 2341, "Finetune/Step Time": 4.682685473933816} +{"Finetune/Learning Rate": 1.2068756830376966e-05, "Finetune/Loss": 0.9064497947692871, "Finetune/Loss (Raw)": 0.3556675910949707, "Finetune/Step": 2342, "Finetune/Step Time": 4.708505779504776} +{"Finetune/Learning Rate": 1.2062661582380222e-05, "Finetune/Loss": 0.9065831899642944, "Finetune/Loss (Raw)": 0.9901360869407654, "Finetune/Step": 2343, "Finetune/Step Time": 4.731445988640189} +{"Finetune/Learning Rate": 1.2056565533904787e-05, "Finetune/Loss": 0.9062474966049194, "Finetune/Loss (Raw)": 0.9097930192947388, "Finetune/Step": 2344, "Finetune/Step Time": 4.760745584964752} +{"Finetune/Learning Rate": 1.2050468687316418e-05, "Finetune/Loss": 0.9072697162628174, "Finetune/Loss (Raw)": 1.1458603143692017, "Finetune/Step": 2345, "Finetune/Step Time": 4.75474276393652} +{"Finetune/Learning Rate": 1.2044371044981188e-05, "Finetune/Loss": 0.9085699915885925, "Finetune/Loss (Raw)": 0.9083225727081299, "Finetune/Step": 2346, "Finetune/Step Time": 4.745076643303037} +{"Finetune/Learning Rate": 1.2038272609265464e-05, "Finetune/Loss": 0.9088226556777954, "Finetune/Loss (Raw)": 0.8971302509307861, "Finetune/Step": 2347, "Finetune/Step Time": 4.713023763149977} +{"Finetune/Learning Rate": 1.2032173382535936e-05, "Finetune/Loss": 0.9089454412460327, "Finetune/Loss (Raw)": 1.0017635822296143, "Finetune/Step": 2348, "Finetune/Step Time": 4.7151820641011} +{"Finetune/Learning Rate": 1.2026073367159597e-05, "Finetune/Loss": 0.9095573425292969, "Finetune/Loss (Raw)": 0.9212175607681274, "Finetune/Step": 2349, "Finetune/Step Time": 4.706521317362785} +{"Finetune/Learning Rate": 1.2019972565503733e-05, "Finetune/Loss": 0.9090834856033325, "Finetune/Loss (Raw)": 0.9145249128341675, "Finetune/Step": 2350, "Finetune/Step Time": 4.699429541826248} +{"Finetune/Learning Rate": 1.2013870979935957e-05, "Finetune/Loss": 0.9097903966903687, "Finetune/Loss (Raw)": 1.0305283069610596, "Finetune/Step": 2351, "Finetune/Step Time": 4.696519214659929} +{"Finetune/Learning Rate": 1.200776861282417e-05, "Finetune/Loss": 0.911045491695404, "Finetune/Loss (Raw)": 1.0772861242294312, "Finetune/Step": 2352, "Finetune/Step Time": 4.705071710050106} +{"Finetune/Learning Rate": 1.2001665466536582e-05, "Finetune/Loss": 0.9112498760223389, "Finetune/Loss (Raw)": 0.9167906045913696, "Finetune/Step": 2353, "Finetune/Step Time": 4.7039304450154305} +{"Finetune/Learning Rate": 1.199556154344171e-05, "Finetune/Loss": 0.9111746549606323, "Finetune/Loss (Raw)": 0.9258214235305786, "Finetune/Step": 2354, "Finetune/Step Time": 4.685244966298342} +{"Finetune/Learning Rate": 1.1989456845908356e-05, "Finetune/Loss": 0.9098953604698181, "Finetune/Loss (Raw)": 0.8228118419647217, "Finetune/Step": 2355, "Finetune/Step Time": 4.682672379538417} +{"Finetune/Learning Rate": 1.1983351376305648e-05, "Finetune/Loss": 0.9092427492141724, "Finetune/Loss (Raw)": 0.8119836449623108, "Finetune/Step": 2356, "Finetune/Step Time": 4.677440337836742} +{"Finetune/Learning Rate": 1.197724513700299e-05, "Finetune/Loss": 0.9091901779174805, "Finetune/Loss (Raw)": 0.9092854857444763, "Finetune/Step": 2357, "Finetune/Step Time": 4.6815090123564005} +{"Finetune/Learning Rate": 1.1971138130370097e-05, "Finetune/Loss": 0.9104448556900024, "Finetune/Loss (Raw)": 1.0235168933868408, "Finetune/Step": 2358, "Finetune/Step Time": 4.672875387594104} +{"Finetune/Learning Rate": 1.1965030358776975e-05, "Finetune/Loss": 0.9050015211105347, "Finetune/Loss (Raw)": 0.2934284210205078, "Finetune/Step": 2359, "Finetune/Step Time": 4.713528731837869} +{"Finetune/Learning Rate": 1.1958921824593936e-05, "Finetune/Loss": 0.904119610786438, "Finetune/Loss (Raw)": 0.9194836616516113, "Finetune/Step": 2360, "Finetune/Step Time": 4.7180314641445875} +{"Finetune/Learning Rate": 1.1952812530191586e-05, "Finetune/Loss": 0.9052932858467102, "Finetune/Loss (Raw)": 0.9471576809883118, "Finetune/Step": 2361, "Finetune/Step Time": 4.711725901812315} +{"Finetune/Learning Rate": 1.1946702477940815e-05, "Finetune/Loss": 0.9049892425537109, "Finetune/Loss (Raw)": 0.9369722008705139, "Finetune/Step": 2362, "Finetune/Step Time": 4.704605858772993} +{"Finetune/Learning Rate": 1.194059167021282e-05, "Finetune/Loss": 0.9026422500610352, "Finetune/Loss (Raw)": 0.7286673784255981, "Finetune/Step": 2363, "Finetune/Step Time": 4.710491821169853} +{"Finetune/Learning Rate": 1.1934480109379083e-05, "Finetune/Loss": 0.9014261960983276, "Finetune/Loss (Raw)": 0.8433899283409119, "Finetune/Step": 2364, "Finetune/Step Time": 4.707624267786741} +{"Finetune/Learning Rate": 1.1928367797811384e-05, "Finetune/Loss": 0.9024792909622192, "Finetune/Loss (Raw)": 1.0329936742782593, "Finetune/Step": 2365, "Finetune/Step Time": 4.703604333102703} +{"Finetune/Learning Rate": 1.1922254737881792e-05, "Finetune/Loss": 0.9025236368179321, "Finetune/Loss (Raw)": 0.7751767039299011, "Finetune/Step": 2366, "Finetune/Step Time": 4.699774194508791} +{"Finetune/Learning Rate": 1.1916140931962661e-05, "Finetune/Loss": 0.9009844064712524, "Finetune/Loss (Raw)": 0.970555305480957, "Finetune/Step": 2367, "Finetune/Step Time": 4.705086342990398} +{"Finetune/Learning Rate": 1.191002638242665e-05, "Finetune/Loss": 0.8997859954833984, "Finetune/Loss (Raw)": 0.9053595662117004, "Finetune/Step": 2368, "Finetune/Step Time": 4.705985881388187} +{"Finetune/Learning Rate": 1.1903911091646684e-05, "Finetune/Loss": 0.8997923731803894, "Finetune/Loss (Raw)": 0.9977172613143921, "Finetune/Step": 2369, "Finetune/Step Time": 4.702933523803949} +{"Finetune/Learning Rate": 1.1897795061995998e-05, "Finetune/Loss": 0.899946928024292, "Finetune/Loss (Raw)": 0.9106315970420837, "Finetune/Step": 2370, "Finetune/Step Time": 4.705174900591373} +{"Finetune/Learning Rate": 1.18916782958481e-05, "Finetune/Loss": 0.9008841514587402, "Finetune/Loss (Raw)": 1.010817289352417, "Finetune/Step": 2371, "Finetune/Step Time": 4.703816268593073} +{"Finetune/Learning Rate": 1.1885560795576782e-05, "Finetune/Loss": 0.8987627029418945, "Finetune/Loss (Raw)": 0.8917880654335022, "Finetune/Step": 2372, "Finetune/Step Time": 4.706235935911536} +{"Finetune/Learning Rate": 1.1879442563556141e-05, "Finetune/Loss": 0.8991344571113586, "Finetune/Loss (Raw)": 1.0130993127822876, "Finetune/Step": 2373, "Finetune/Step Time": 4.72839742526412} +{"Finetune/Learning Rate": 1.1873323602160526e-05, "Finetune/Loss": 0.8970876932144165, "Finetune/Loss (Raw)": 0.825320303440094, "Finetune/Step": 2374, "Finetune/Step Time": 4.729772472754121} +{"Finetune/Learning Rate": 1.1867203913764599e-05, "Finetune/Loss": 0.8963544368743896, "Finetune/Loss (Raw)": 0.7647431492805481, "Finetune/Step": 2375, "Finetune/Step Time": 4.732973974198103} +{"Finetune/Learning Rate": 1.1861083500743284e-05, "Finetune/Loss": 0.8977636098861694, "Finetune/Loss (Raw)": 0.9521506428718567, "Finetune/Step": 2376, "Finetune/Step Time": 4.738432379439473} +{"Finetune/Learning Rate": 1.1854962365471798e-05, "Finetune/Loss": 0.8992823362350464, "Finetune/Loss (Raw)": 1.0668046474456787, "Finetune/Step": 2377, "Finetune/Step Time": 4.738259334117174} +{"Finetune/Learning Rate": 1.1848840510325635e-05, "Finetune/Loss": 0.899725079536438, "Finetune/Loss (Raw)": 1.0040149688720703, "Finetune/Step": 2378, "Finetune/Step Time": 4.716340366750956} +{"Finetune/Learning Rate": 1.184271793768056e-05, "Finetune/Loss": 0.899669885635376, "Finetune/Loss (Raw)": 0.9468751549720764, "Finetune/Step": 2379, "Finetune/Step Time": 4.73384877294302} +{"Finetune/Learning Rate": 1.1836594649912636e-05, "Finetune/Loss": 0.9033293724060059, "Finetune/Loss (Raw)": 0.7894084453582764, "Finetune/Step": 2380, "Finetune/Step Time": 4.697748698294163} +{"Finetune/Learning Rate": 1.1830470649398183e-05, "Finetune/Loss": 0.902469277381897, "Finetune/Loss (Raw)": 0.9247536063194275, "Finetune/Step": 2381, "Finetune/Step Time": 4.700949715450406} +{"Finetune/Learning Rate": 1.1824345938513806e-05, "Finetune/Loss": 0.9004285335540771, "Finetune/Loss (Raw)": 0.7814978361129761, "Finetune/Step": 2382, "Finetune/Step Time": 4.712818782776594} +{"Finetune/Learning Rate": 1.181822051963639e-05, "Finetune/Loss": 0.8982757329940796, "Finetune/Loss (Raw)": 0.6804384589195251, "Finetune/Step": 2383, "Finetune/Step Time": 4.7262784745544195} +{"Finetune/Learning Rate": 1.1812094395143086e-05, "Finetune/Loss": 0.8999093770980835, "Finetune/Loss (Raw)": 1.0990452766418457, "Finetune/Step": 2384, "Finetune/Step Time": 4.721869317814708} +{"Finetune/Learning Rate": 1.1805967567411325e-05, "Finetune/Loss": 0.8993600010871887, "Finetune/Loss (Raw)": 0.9106037616729736, "Finetune/Step": 2385, "Finetune/Step Time": 4.727067468687892} +{"Finetune/Learning Rate": 1.1799840038818813e-05, "Finetune/Loss": 0.9015763998031616, "Finetune/Loss (Raw)": 1.030509352684021, "Finetune/Step": 2386, "Finetune/Step Time": 4.733748756349087} +{"Finetune/Learning Rate": 1.1793711811743515e-05, "Finetune/Loss": 0.9018101692199707, "Finetune/Loss (Raw)": 0.8762628436088562, "Finetune/Step": 2387, "Finetune/Step Time": 4.748020734637976} +{"Finetune/Learning Rate": 1.1787582888563689e-05, "Finetune/Loss": 0.9027739763259888, "Finetune/Loss (Raw)": 0.9815786480903625, "Finetune/Step": 2388, "Finetune/Step Time": 4.758100440725684} +{"Finetune/Learning Rate": 1.1781453271657839e-05, "Finetune/Loss": 0.9033486843109131, "Finetune/Loss (Raw)": 1.033229112625122, "Finetune/Step": 2389, "Finetune/Step Time": 4.7564141526818275} +{"Finetune/Learning Rate": 1.1775322963404757e-05, "Finetune/Loss": 0.9032469391822815, "Finetune/Loss (Raw)": 1.035988688468933, "Finetune/Step": 2390, "Finetune/Step Time": 4.7602059710770845} +{"Finetune/Learning Rate": 1.176919196618349e-05, "Finetune/Loss": 0.9043415784835815, "Finetune/Loss (Raw)": 0.9894810318946838, "Finetune/Step": 2391, "Finetune/Step Time": 4.759657878428698} +{"Finetune/Learning Rate": 1.1763060282373364e-05, "Finetune/Loss": 0.9041627049446106, "Finetune/Loss (Raw)": 0.9337888956069946, "Finetune/Step": 2392, "Finetune/Step Time": 4.760075904428959} +{"Finetune/Learning Rate": 1.1756927914353963e-05, "Finetune/Loss": 0.9050690531730652, "Finetune/Loss (Raw)": 1.0231246948242188, "Finetune/Step": 2393, "Finetune/Step Time": 4.773406697437167} +{"Finetune/Learning Rate": 1.175079486450514e-05, "Finetune/Loss": 0.9022286534309387, "Finetune/Loss (Raw)": 0.3970628082752228, "Finetune/Step": 2394, "Finetune/Step Time": 4.803924545645714} +{"Finetune/Learning Rate": 1.1744661135207009e-05, "Finetune/Loss": 0.9026715755462646, "Finetune/Loss (Raw)": 0.9412039518356323, "Finetune/Step": 2395, "Finetune/Step Time": 4.808793714269996} +{"Finetune/Learning Rate": 1.1738526728839956e-05, "Finetune/Loss": 0.9028781056404114, "Finetune/Loss (Raw)": 0.8697177171707153, "Finetune/Step": 2396, "Finetune/Step Time": 4.811608146876097} +{"Finetune/Learning Rate": 1.1732391647784618e-05, "Finetune/Loss": 0.9028282761573792, "Finetune/Loss (Raw)": 0.9340659379959106, "Finetune/Step": 2397, "Finetune/Step Time": 4.809146473184228} +{"Finetune/Learning Rate": 1.1726255894421908e-05, "Finetune/Loss": 0.9051670432090759, "Finetune/Loss (Raw)": 1.0811200141906738, "Finetune/Step": 2398, "Finetune/Step Time": 4.8001354560256} +{"Finetune/Learning Rate": 1.1720119471132985e-05, "Finetune/Loss": 0.9053777456283569, "Finetune/Loss (Raw)": 0.9890288710594177, "Finetune/Step": 2399, "Finetune/Step Time": 4.81234429590404} +{"Finetune/Learning Rate": 1.1713982380299278e-05, "Finetune/Loss": 0.9041276574134827, "Finetune/Loss (Raw)": 0.8209578394889832, "Finetune/Step": 2400, "Finetune/Step Time": 4.804145762696862} +{"Finetune/Learning Rate": 1.1707844624302471e-05, "Finetune/Loss": 0.9042143821716309, "Finetune/Loss (Raw)": 0.9427962303161621, "Finetune/Step": 2401, "Finetune/Step Time": 4.802246727049351} +{"Finetune/Learning Rate": 1.170170620552451e-05, "Finetune/Loss": 0.9040944576263428, "Finetune/Loss (Raw)": 0.8208478093147278, "Finetune/Step": 2402, "Finetune/Step Time": 4.78564291074872} +{"Finetune/Learning Rate": 1.1695567126347593e-05, "Finetune/Loss": 0.9033390879631042, "Finetune/Loss (Raw)": 0.9370255470275879, "Finetune/Step": 2403, "Finetune/Step Time": 4.794496119022369} +{"Finetune/Learning Rate": 1.1689427389154176e-05, "Finetune/Loss": 0.9034674763679504, "Finetune/Loss (Raw)": 0.9495629072189331, "Finetune/Step": 2404, "Finetune/Step Time": 4.787684680894017} +{"Finetune/Learning Rate": 1.1683286996326975e-05, "Finetune/Loss": 0.9049087762832642, "Finetune/Loss (Raw)": 0.9359196424484253, "Finetune/Step": 2405, "Finetune/Step Time": 4.783647894859314} +{"Finetune/Learning Rate": 1.1677145950248954e-05, "Finetune/Loss": 0.9050710201263428, "Finetune/Loss (Raw)": 0.9938045144081116, "Finetune/Step": 2406, "Finetune/Step Time": 4.785025918856263} +{"Finetune/Learning Rate": 1.167100425330333e-05, "Finetune/Loss": 0.9025256633758545, "Finetune/Loss (Raw)": 0.7000030875205994, "Finetune/Step": 2407, "Finetune/Step Time": 4.806473704054952} +{"Finetune/Learning Rate": 1.1664861907873583e-05, "Finetune/Loss": 0.9041483402252197, "Finetune/Loss (Raw)": 1.1019319295883179, "Finetune/Step": 2408, "Finetune/Step Time": 4.814922574907541} +{"Finetune/Learning Rate": 1.165871891634343e-05, "Finetune/Loss": 0.9040669202804565, "Finetune/Loss (Raw)": 0.8909334540367126, "Finetune/Step": 2409, "Finetune/Step Time": 4.81492436490953} +{"Finetune/Learning Rate": 1.165257528109685e-05, "Finetune/Loss": 0.904301643371582, "Finetune/Loss (Raw)": 0.9293025135993958, "Finetune/Step": 2410, "Finetune/Step Time": 4.824252491816878} +{"Finetune/Learning Rate": 1.1646431004518068e-05, "Finetune/Loss": 0.9021785259246826, "Finetune/Loss (Raw)": 0.7598236203193665, "Finetune/Step": 2411, "Finetune/Step Time": 4.831360528245568} +{"Finetune/Learning Rate": 1.1640286088991558e-05, "Finetune/Loss": 0.9022513031959534, "Finetune/Loss (Raw)": 0.8969454765319824, "Finetune/Step": 2412, "Finetune/Step Time": 4.80466216057539} +{"Finetune/Learning Rate": 1.1634140536902039e-05, "Finetune/Loss": 0.9011340141296387, "Finetune/Loss (Raw)": 0.8136382102966309, "Finetune/Step": 2413, "Finetune/Step Time": 4.800552532076836} +{"Finetune/Learning Rate": 1.1627994350634482e-05, "Finetune/Loss": 0.9011172652244568, "Finetune/Loss (Raw)": 0.9512746930122375, "Finetune/Step": 2414, "Finetune/Step Time": 4.786024119704962} +{"Finetune/Learning Rate": 1.1621847532574101e-05, "Finetune/Loss": 0.9020107984542847, "Finetune/Loss (Raw)": 0.8835653066635132, "Finetune/Step": 2415, "Finetune/Step Time": 4.785680711269379} +{"Finetune/Learning Rate": 1.1615700085106357e-05, "Finetune/Loss": 0.9020054340362549, "Finetune/Loss (Raw)": 0.951732873916626, "Finetune/Step": 2416, "Finetune/Step Time": 4.777759779244661} +{"Finetune/Learning Rate": 1.1609552010616953e-05, "Finetune/Loss": 0.9032894372940063, "Finetune/Loss (Raw)": 0.9015013575553894, "Finetune/Step": 2417, "Finetune/Step Time": 4.775857636705041} +{"Finetune/Learning Rate": 1.1603403311491841e-05, "Finetune/Loss": 0.8988915681838989, "Finetune/Loss (Raw)": 0.44181087613105774, "Finetune/Step": 2418, "Finetune/Step Time": 4.804234566166997} +{"Finetune/Learning Rate": 1.1597253990117205e-05, "Finetune/Loss": 0.8993047475814819, "Finetune/Loss (Raw)": 0.890275776386261, "Finetune/Step": 2419, "Finetune/Step Time": 4.811617096886039} +{"Finetune/Learning Rate": 1.1591104048879479e-05, "Finetune/Loss": 0.9010151028633118, "Finetune/Loss (Raw)": 1.076083779335022, "Finetune/Step": 2420, "Finetune/Step Time": 4.801065942272544} +{"Finetune/Learning Rate": 1.1584953490165334e-05, "Finetune/Loss": 0.9012584686279297, "Finetune/Loss (Raw)": 0.7704787850379944, "Finetune/Step": 2421, "Finetune/Step Time": 4.810464233160019} +{"Finetune/Learning Rate": 1.1578802316361685e-05, "Finetune/Loss": 0.9001144170761108, "Finetune/Loss (Raw)": 0.7895786166191101, "Finetune/Step": 2422, "Finetune/Step Time": 4.8206753972917795} +{"Finetune/Learning Rate": 1.157265052985568e-05, "Finetune/Loss": 0.8986179828643799, "Finetune/Loss (Raw)": 0.7323266863822937, "Finetune/Step": 2423, "Finetune/Step Time": 4.819513086229563} +{"Finetune/Learning Rate": 1.156649813303471e-05, "Finetune/Loss": 0.900011420249939, "Finetune/Loss (Raw)": 0.9409523010253906, "Finetune/Step": 2424, "Finetune/Step Time": 4.824457040056586} +{"Finetune/Learning Rate": 1.1560345128286396e-05, "Finetune/Loss": 0.8993128538131714, "Finetune/Loss (Raw)": 0.8389972448348999, "Finetune/Step": 2425, "Finetune/Step Time": 4.8347858134657145} +{"Finetune/Learning Rate": 1.1554191517998598e-05, "Finetune/Loss": 0.8954213857650757, "Finetune/Loss (Raw)": 0.3714466392993927, "Finetune/Step": 2426, "Finetune/Step Time": 4.873759130015969} +{"Finetune/Learning Rate": 1.1548037304559419e-05, "Finetune/Loss": 0.8953644037246704, "Finetune/Loss (Raw)": 0.9687224626541138, "Finetune/Step": 2427, "Finetune/Step Time": 4.864481026306748} +{"Finetune/Learning Rate": 1.1541882490357183e-05, "Finetune/Loss": 0.8939991593360901, "Finetune/Loss (Raw)": 0.7993138432502747, "Finetune/Step": 2428, "Finetune/Step Time": 4.853593053296208} +{"Finetune/Learning Rate": 1.1535727077780456e-05, "Finetune/Loss": 0.895043671131134, "Finetune/Loss (Raw)": 0.9719499349594116, "Finetune/Step": 2429, "Finetune/Step Time": 4.865694360807538} +{"Finetune/Learning Rate": 1.1529571069218033e-05, "Finetune/Loss": 0.8946915864944458, "Finetune/Loss (Raw)": 0.7563185691833496, "Finetune/Step": 2430, "Finetune/Step Time": 4.895485600456595} +{"Finetune/Learning Rate": 1.152341446705894e-05, "Finetune/Loss": 0.8953422904014587, "Finetune/Loss (Raw)": 1.0004745721817017, "Finetune/Step": 2431, "Finetune/Step Time": 4.8933145850896835} +{"Finetune/Learning Rate": 1.1517257273692432e-05, "Finetune/Loss": 0.8968104124069214, "Finetune/Loss (Raw)": 0.9101656079292297, "Finetune/Step": 2432, "Finetune/Step Time": 4.896038541570306} +{"Finetune/Learning Rate": 1.1511099491508003e-05, "Finetune/Loss": 0.8972688913345337, "Finetune/Loss (Raw)": 0.870580792427063, "Finetune/Step": 2433, "Finetune/Step Time": 4.89517080783844} +{"Finetune/Learning Rate": 1.1504941122895361e-05, "Finetune/Loss": 0.8962810039520264, "Finetune/Loss (Raw)": 0.9229989051818848, "Finetune/Step": 2434, "Finetune/Step Time": 4.894327199086547} +{"Finetune/Learning Rate": 1.1498782170244449e-05, "Finetune/Loss": 0.8955796957015991, "Finetune/Loss (Raw)": 0.8835249543190002, "Finetune/Step": 2435, "Finetune/Step Time": 4.90364209190011} +{"Finetune/Learning Rate": 1.1492622635945438e-05, "Finetune/Loss": 0.8984339833259583, "Finetune/Loss (Raw)": 0.9772293567657471, "Finetune/Step": 2436, "Finetune/Step Time": 4.896324949339032} +{"Finetune/Learning Rate": 1.1486462522388724e-05, "Finetune/Loss": 0.8978899717330933, "Finetune/Loss (Raw)": 0.8741713762283325, "Finetune/Step": 2437, "Finetune/Step Time": 4.901606110855937} +{"Finetune/Learning Rate": 1.1480301831964926e-05, "Finetune/Loss": 0.8983062505722046, "Finetune/Loss (Raw)": 1.011188268661499, "Finetune/Step": 2438, "Finetune/Step Time": 4.90569331869483} +{"Finetune/Learning Rate": 1.1474140567064889e-05, "Finetune/Loss": 0.8980087041854858, "Finetune/Loss (Raw)": 0.9936639666557312, "Finetune/Step": 2439, "Finetune/Step Time": 4.9060440715402365} +{"Finetune/Learning Rate": 1.1467978730079677e-05, "Finetune/Loss": 0.8990135192871094, "Finetune/Loss (Raw)": 1.00286865234375, "Finetune/Step": 2440, "Finetune/Step Time": 4.886065548285842} +{"Finetune/Learning Rate": 1.1461816323400579e-05, "Finetune/Loss": 0.899187445640564, "Finetune/Loss (Raw)": 0.9643206000328064, "Finetune/Step": 2441, "Finetune/Step Time": 4.881490835919976} +{"Finetune/Learning Rate": 1.1455653349419105e-05, "Finetune/Loss": 0.8996787071228027, "Finetune/Loss (Raw)": 0.922199547290802, "Finetune/Step": 2442, "Finetune/Step Time": 4.877565737813711} +{"Finetune/Learning Rate": 1.1449489810526985e-05, "Finetune/Loss": 0.9006065130233765, "Finetune/Loss (Raw)": 0.9788616299629211, "Finetune/Step": 2443, "Finetune/Step Time": 4.882912116125226} +{"Finetune/Learning Rate": 1.144332570911617e-05, "Finetune/Loss": 0.9004702568054199, "Finetune/Loss (Raw)": 0.9651892781257629, "Finetune/Step": 2444, "Finetune/Step Time": 4.889585204422474} +{"Finetune/Learning Rate": 1.1437161047578826e-05, "Finetune/Loss": 0.8967922925949097, "Finetune/Loss (Raw)": 0.38613811135292053, "Finetune/Step": 2445, "Finetune/Step Time": 4.921360835433006} +{"Finetune/Learning Rate": 1.1430995828307337e-05, "Finetune/Loss": 0.8971550464630127, "Finetune/Loss (Raw)": 0.931097686290741, "Finetune/Step": 2446, "Finetune/Step Time": 4.923216260969639} +{"Finetune/Learning Rate": 1.1424830053694303e-05, "Finetune/Loss": 0.8957602977752686, "Finetune/Loss (Raw)": 0.9042108654975891, "Finetune/Step": 2447, "Finetune/Step Time": 4.9278241861611605} +{"Finetune/Learning Rate": 1.1418663726132543e-05, "Finetune/Loss": 0.8963238000869751, "Finetune/Loss (Raw)": 1.0037707090377808, "Finetune/Step": 2448, "Finetune/Step Time": 4.929276784881949} +{"Finetune/Learning Rate": 1.141249684801509e-05, "Finetune/Loss": 0.8953888416290283, "Finetune/Loss (Raw)": 0.8111050724983215, "Finetune/Step": 2449, "Finetune/Step Time": 4.921329019591212} +{"Finetune/Learning Rate": 1.140632942173519e-05, "Finetune/Loss": 0.8914744853973389, "Finetune/Loss (Raw)": 0.48724204301834106, "Finetune/Step": 2450, "Finetune/Step Time": 4.9532531797885895} +{"Finetune/Learning Rate": 1.1400161449686293e-05, "Finetune/Loss": 0.8877506256103516, "Finetune/Loss (Raw)": 0.32191529870033264, "Finetune/Step": 2451, "Finetune/Step Time": 4.9805009458214045} +{"Finetune/Learning Rate": 1.1393992934262073e-05, "Finetune/Loss": 0.8885774612426758, "Finetune/Loss (Raw)": 0.9643121957778931, "Finetune/Step": 2452, "Finetune/Step Time": 4.981928175315261} +{"Finetune/Learning Rate": 1.1387823877856413e-05, "Finetune/Loss": 0.8878859877586365, "Finetune/Loss (Raw)": 0.9274402856826782, "Finetune/Step": 2453, "Finetune/Step Time": 4.9994731694459915} +{"Finetune/Learning Rate": 1.1381654282863399e-05, "Finetune/Loss": 0.8916720151901245, "Finetune/Loss (Raw)": 0.9228395819664001, "Finetune/Step": 2454, "Finetune/Step Time": 4.969888621941209} +{"Finetune/Learning Rate": 1.1375484151677334e-05, "Finetune/Loss": 0.8923752307891846, "Finetune/Loss (Raw)": 0.9997644424438477, "Finetune/Step": 2455, "Finetune/Step Time": 4.957064468413591} +{"Finetune/Learning Rate": 1.1369313486692723e-05, "Finetune/Loss": 0.8932251334190369, "Finetune/Loss (Raw)": 0.9612498879432678, "Finetune/Step": 2456, "Finetune/Step Time": 4.946797577664256} +{"Finetune/Learning Rate": 1.136314229030428e-05, "Finetune/Loss": 0.8929908275604248, "Finetune/Loss (Raw)": 0.9516388773918152, "Finetune/Step": 2457, "Finetune/Step Time": 4.943772545084357} +{"Finetune/Learning Rate": 1.1356970564906928e-05, "Finetune/Loss": 0.8941351175308228, "Finetune/Loss (Raw)": 0.957366406917572, "Finetune/Step": 2458, "Finetune/Step Time": 4.949437715113163} +{"Finetune/Learning Rate": 1.1350798312895795e-05, "Finetune/Loss": 0.8936262130737305, "Finetune/Loss (Raw)": 0.8720762133598328, "Finetune/Step": 2459, "Finetune/Step Time": 4.943167207762599} +{"Finetune/Learning Rate": 1.1344625536666208e-05, "Finetune/Loss": 0.89217209815979, "Finetune/Loss (Raw)": 0.9324843883514404, "Finetune/Step": 2460, "Finetune/Step Time": 4.937279835343361} +{"Finetune/Learning Rate": 1.13384522386137e-05, "Finetune/Loss": 0.8927090167999268, "Finetune/Loss (Raw)": 1.0669913291931152, "Finetune/Step": 2461, "Finetune/Step Time": 4.933379918336868} +{"Finetune/Learning Rate": 1.1332278421134007e-05, "Finetune/Loss": 0.8928334712982178, "Finetune/Loss (Raw)": 0.9624052047729492, "Finetune/Step": 2462, "Finetune/Step Time": 4.915015701204538} +{"Finetune/Learning Rate": 1.1326104086623075e-05, "Finetune/Loss": 0.893332839012146, "Finetune/Loss (Raw)": 0.940643846988678, "Finetune/Step": 2463, "Finetune/Step Time": 4.9202341213822365} +{"Finetune/Learning Rate": 1.1319929237477033e-05, "Finetune/Loss": 0.894298791885376, "Finetune/Loss (Raw)": 1.012441873550415, "Finetune/Step": 2464, "Finetune/Step Time": 4.904900122433901} +{"Finetune/Learning Rate": 1.1313753876092223e-05, "Finetune/Loss": 0.8937881588935852, "Finetune/Loss (Raw)": 0.9022254347801208, "Finetune/Step": 2465, "Finetune/Step Time": 4.901888705790043} +{"Finetune/Learning Rate": 1.1307578004865188e-05, "Finetune/Loss": 0.8924616575241089, "Finetune/Loss (Raw)": 0.8123199939727783, "Finetune/Step": 2466, "Finetune/Step Time": 4.908073579892516} +{"Finetune/Learning Rate": 1.130140162619265e-05, "Finetune/Loss": 0.8930355310440063, "Finetune/Loss (Raw)": 0.9162161946296692, "Finetune/Step": 2467, "Finetune/Step Time": 4.88779522664845} +{"Finetune/Learning Rate": 1.1295224742471555e-05, "Finetune/Loss": 0.8946272730827332, "Finetune/Loss (Raw)": 1.115135669708252, "Finetune/Step": 2468, "Finetune/Step Time": 4.892169266939163} +{"Finetune/Learning Rate": 1.128904735609902e-05, "Finetune/Loss": 0.8943148851394653, "Finetune/Loss (Raw)": 1.0005114078521729, "Finetune/Step": 2469, "Finetune/Step Time": 4.863129071891308} +{"Finetune/Learning Rate": 1.1282869469472375e-05, "Finetune/Loss": 0.8981269598007202, "Finetune/Loss (Raw)": 0.8436154127120972, "Finetune/Step": 2470, "Finetune/Step Time": 4.835026556625962} +{"Finetune/Learning Rate": 1.1276691084989135e-05, "Finetune/Loss": 0.8980977535247803, "Finetune/Loss (Raw)": 0.9863985180854797, "Finetune/Step": 2471, "Finetune/Step Time": 4.813667632639408} +{"Finetune/Learning Rate": 1.1270512205047008e-05, "Finetune/Loss": 0.89833003282547, "Finetune/Loss (Raw)": 0.939528226852417, "Finetune/Step": 2472, "Finetune/Step Time": 4.7819560039788485} +{"Finetune/Learning Rate": 1.1264332832043902e-05, "Finetune/Loss": 0.8967340588569641, "Finetune/Loss (Raw)": 0.9415741562843323, "Finetune/Step": 2473, "Finetune/Step Time": 4.793268647044897} +{"Finetune/Learning Rate": 1.1258152968377907e-05, "Finetune/Loss": 0.8968912959098816, "Finetune/Loss (Raw)": 0.9284486770629883, "Finetune/Step": 2474, "Finetune/Step Time": 4.80262971110642} +{"Finetune/Learning Rate": 1.125197261644731e-05, "Finetune/Loss": 0.896773099899292, "Finetune/Loss (Raw)": 0.882002055644989, "Finetune/Step": 2475, "Finetune/Step Time": 4.802461974322796} +{"Finetune/Learning Rate": 1.1245791778650578e-05, "Finetune/Loss": 0.8962806463241577, "Finetune/Loss (Raw)": 0.9387313723564148, "Finetune/Step": 2476, "Finetune/Step Time": 4.800068272277713} +{"Finetune/Learning Rate": 1.1239610457386381e-05, "Finetune/Loss": 0.8963394165039062, "Finetune/Loss (Raw)": 0.9287374019622803, "Finetune/Step": 2477, "Finetune/Step Time": 4.796916546300054} +{"Finetune/Learning Rate": 1.1233428655053572e-05, "Finetune/Loss": 0.896335780620575, "Finetune/Loss (Raw)": 0.9140609502792358, "Finetune/Step": 2478, "Finetune/Step Time": 4.796088550239801} +{"Finetune/Learning Rate": 1.1227246374051175e-05, "Finetune/Loss": 0.8948060274124146, "Finetune/Loss (Raw)": 0.8347213864326477, "Finetune/Step": 2479, "Finetune/Step Time": 4.796792559325695} +{"Finetune/Learning Rate": 1.1221063616778425e-05, "Finetune/Loss": 0.8930155038833618, "Finetune/Loss (Raw)": 0.8480989336967468, "Finetune/Step": 2480, "Finetune/Step Time": 4.795694539323449} +{"Finetune/Learning Rate": 1.1214880385634723e-05, "Finetune/Loss": 0.8887860178947449, "Finetune/Loss (Raw)": 0.37541326880455017, "Finetune/Step": 2481, "Finetune/Step Time": 4.826610889285803} +{"Finetune/Learning Rate": 1.120869668301966e-05, "Finetune/Loss": 0.8900819420814514, "Finetune/Loss (Raw)": 1.0917035341262817, "Finetune/Step": 2482, "Finetune/Step Time": 4.823107708245516} +{"Finetune/Learning Rate": 1.1202512511333017e-05, "Finetune/Loss": 0.8907725811004639, "Finetune/Loss (Raw)": 0.9112090468406677, "Finetune/Step": 2483, "Finetune/Step Time": 4.828547494485974} +{"Finetune/Learning Rate": 1.119632787297474e-05, "Finetune/Loss": 0.8912242650985718, "Finetune/Loss (Raw)": 0.8698071837425232, "Finetune/Step": 2484, "Finetune/Step Time": 4.833891859278083} +{"Finetune/Learning Rate": 1.1190142770344976e-05, "Finetune/Loss": 0.8913894891738892, "Finetune/Loss (Raw)": 0.9304323196411133, "Finetune/Step": 2485, "Finetune/Step Time": 4.834034556522965} +{"Finetune/Learning Rate": 1.1183957205844037e-05, "Finetune/Loss": 0.889829158782959, "Finetune/Loss (Raw)": 0.8237878680229187, "Finetune/Step": 2486, "Finetune/Step Time": 4.844869129359722} +{"Finetune/Learning Rate": 1.1177771181872416e-05, "Finetune/Loss": 0.8899967670440674, "Finetune/Loss (Raw)": 0.31487902998924255, "Finetune/Step": 2487, "Finetune/Step Time": 4.837081773206592} +{"Finetune/Learning Rate": 1.1171584700830803e-05, "Finetune/Loss": 0.8909276127815247, "Finetune/Loss (Raw)": 1.0386419296264648, "Finetune/Step": 2488, "Finetune/Step Time": 4.835385985672474} +{"Finetune/Learning Rate": 1.1165397765120035e-05, "Finetune/Loss": 0.8910121917724609, "Finetune/Loss (Raw)": 0.957983672618866, "Finetune/Step": 2489, "Finetune/Step Time": 4.858420668169856} +{"Finetune/Learning Rate": 1.115921037714115e-05, "Finetune/Loss": 0.886183500289917, "Finetune/Loss (Raw)": 0.31889861822128296, "Finetune/Step": 2490, "Finetune/Step Time": 4.89770383387804} +{"Finetune/Learning Rate": 1.1153022539295345e-05, "Finetune/Loss": 0.8874870538711548, "Finetune/Loss (Raw)": 0.8955235481262207, "Finetune/Step": 2491, "Finetune/Step Time": 4.8904813546687365} +{"Finetune/Learning Rate": 1.1146834253984008e-05, "Finetune/Loss": 0.8871039748191833, "Finetune/Loss (Raw)": 0.7943506240844727, "Finetune/Step": 2492, "Finetune/Step Time": 4.897546570748091} +{"Finetune/Learning Rate": 1.1140645523608684e-05, "Finetune/Loss": 0.8866528272628784, "Finetune/Loss (Raw)": 0.9752518534660339, "Finetune/Step": 2493, "Finetune/Step Time": 4.905583860352635} +{"Finetune/Learning Rate": 1.1134456350571099e-05, "Finetune/Loss": 0.8885838389396667, "Finetune/Loss (Raw)": 1.0223437547683716, "Finetune/Step": 2494, "Finetune/Step Time": 4.899795213714242} +{"Finetune/Learning Rate": 1.112826673727316e-05, "Finetune/Loss": 0.8883077502250671, "Finetune/Loss (Raw)": 0.9352147579193115, "Finetune/Step": 2495, "Finetune/Step Time": 4.892958238720894} +{"Finetune/Learning Rate": 1.112207668611692e-05, "Finetune/Loss": 0.8884228467941284, "Finetune/Loss (Raw)": 0.9200922250747681, "Finetune/Step": 2496, "Finetune/Step Time": 4.909368932247162} +{"Finetune/Learning Rate": 1.1115886199504631e-05, "Finetune/Loss": 0.8869446516036987, "Finetune/Loss (Raw)": 0.808506429195404, "Finetune/Step": 2497, "Finetune/Step Time": 4.908282820135355} +{"Finetune/Learning Rate": 1.110969527983869e-05, "Finetune/Loss": 0.8877555131912231, "Finetune/Loss (Raw)": 1.0144296884536743, "Finetune/Step": 2498, "Finetune/Step Time": 4.908932723104954} +{"Finetune/Learning Rate": 1.1103503929521677e-05, "Finetune/Loss": 0.885045051574707, "Finetune/Loss (Raw)": 0.6638779640197754, "Finetune/Step": 2499, "Finetune/Step Time": 4.921778963878751} +{"Finetune/Learning Rate": 1.1097312150956336e-05, "Finetune/Loss": 0.8854407668113708, "Finetune/Loss (Raw)": 0.9424341320991516, "Finetune/Step": 2500, "Finetune/Step Time": 4.921915745362639} +{"Finetune/Learning Rate": 1.1091119946545568e-05, "Finetune/Loss": 0.8848354816436768, "Finetune/Loss (Raw)": 0.9356285929679871, "Finetune/Step": 2501, "Finetune/Step Time": 4.902552351355553} +{"Finetune/Learning Rate": 1.1084927318692453e-05, "Finetune/Loss": 0.8857628107070923, "Finetune/Loss (Raw)": 0.9440136551856995, "Finetune/Step": 2502, "Finetune/Step Time": 4.903013674542308} +{"Finetune/Learning Rate": 1.1078734269800227e-05, "Finetune/Loss": 0.8866883516311646, "Finetune/Loss (Raw)": 0.883207380771637, "Finetune/Step": 2503, "Finetune/Step Time": 4.891814371570945} +{"Finetune/Learning Rate": 1.1072540802272291e-05, "Finetune/Loss": 0.8862552642822266, "Finetune/Loss (Raw)": 0.8967203497886658, "Finetune/Step": 2504, "Finetune/Step Time": 4.896045906469226} +{"Finetune/Learning Rate": 1.1066346918512211e-05, "Finetune/Loss": 0.8849650621414185, "Finetune/Loss (Raw)": 0.9016609191894531, "Finetune/Step": 2505, "Finetune/Step Time": 4.895744547247887} +{"Finetune/Learning Rate": 1.1060152620923707e-05, "Finetune/Loss": 0.8838580846786499, "Finetune/Loss (Raw)": 0.8623149991035461, "Finetune/Step": 2506, "Finetune/Step Time": 4.899826727807522} +{"Finetune/Learning Rate": 1.105395791191067e-05, "Finetune/Loss": 0.8843057751655579, "Finetune/Loss (Raw)": 1.0041807889938354, "Finetune/Step": 2507, "Finetune/Step Time": 4.886855136603117} +{"Finetune/Learning Rate": 1.1047762793877146e-05, "Finetune/Loss": 0.8847377300262451, "Finetune/Loss (Raw)": 0.8446937203407288, "Finetune/Step": 2508, "Finetune/Step Time": 4.894421981647611} +{"Finetune/Learning Rate": 1.1041567269227336e-05, "Finetune/Loss": 0.8840500116348267, "Finetune/Loss (Raw)": 0.8367287516593933, "Finetune/Step": 2509, "Finetune/Step Time": 4.897416844964027} +{"Finetune/Learning Rate": 1.1035371340365606e-05, "Finetune/Loss": 0.8861310482025146, "Finetune/Loss (Raw)": 1.0478746891021729, "Finetune/Step": 2510, "Finetune/Step Time": 4.88073268905282} +{"Finetune/Learning Rate": 1.1029175009696468e-05, "Finetune/Loss": 0.8884938359260559, "Finetune/Loss (Raw)": 0.9828740358352661, "Finetune/Step": 2511, "Finetune/Step Time": 4.877303605899215} +{"Finetune/Learning Rate": 1.1022978279624606e-05, "Finetune/Loss": 0.8862196803092957, "Finetune/Loss (Raw)": 0.8079554438591003, "Finetune/Step": 2512, "Finetune/Step Time": 4.886672616004944} +{"Finetune/Learning Rate": 1.1016781152554846e-05, "Finetune/Loss": 0.8857376575469971, "Finetune/Loss (Raw)": 0.8489025831222534, "Finetune/Step": 2513, "Finetune/Step Time": 4.885961135849357} +{"Finetune/Learning Rate": 1.1010583630892166e-05, "Finetune/Loss": 0.8856227397918701, "Finetune/Loss (Raw)": 1.0158085823059082, "Finetune/Step": 2514, "Finetune/Step Time": 4.892481194809079} +{"Finetune/Learning Rate": 1.1004385717041716e-05, "Finetune/Loss": 0.8847668766975403, "Finetune/Loss (Raw)": 0.7667040824890137, "Finetune/Step": 2515, "Finetune/Step Time": 4.897505797445774} +{"Finetune/Learning Rate": 1.0998187413408774e-05, "Finetune/Loss": 0.8843610286712646, "Finetune/Loss (Raw)": 0.9296351075172424, "Finetune/Step": 2516, "Finetune/Step Time": 4.875145172700286} +{"Finetune/Learning Rate": 1.0991988722398786e-05, "Finetune/Loss": 0.8835943937301636, "Finetune/Loss (Raw)": 0.935097873210907, "Finetune/Step": 2517, "Finetune/Step Time": 4.878761325031519} +{"Finetune/Learning Rate": 1.0985789646417341e-05, "Finetune/Loss": 0.8821965456008911, "Finetune/Loss (Raw)": 0.8570639491081238, "Finetune/Step": 2518, "Finetune/Step Time": 4.884707225486636} +{"Finetune/Learning Rate": 1.097959018787018e-05, "Finetune/Loss": 0.8813284635543823, "Finetune/Loss (Raw)": 0.8783578872680664, "Finetune/Step": 2519, "Finetune/Step Time": 4.888861773535609} +{"Finetune/Learning Rate": 1.0973390349163195e-05, "Finetune/Loss": 0.8817405700683594, "Finetune/Loss (Raw)": 0.9865478873252869, "Finetune/Step": 2520, "Finetune/Step Time": 4.88851866312325} +{"Finetune/Learning Rate": 1.0967190132702414e-05, "Finetune/Loss": 0.876926064491272, "Finetune/Loss (Raw)": 0.406870573759079, "Finetune/Step": 2521, "Finetune/Step Time": 4.907661400735378} +{"Finetune/Learning Rate": 1.096098954089403e-05, "Finetune/Loss": 0.8812038898468018, "Finetune/Loss (Raw)": 0.9446197152137756, "Finetune/Step": 2522, "Finetune/Step Time": 4.882999964058399} +{"Finetune/Learning Rate": 1.0954788576144365e-05, "Finetune/Loss": 0.8794058561325073, "Finetune/Loss (Raw)": 0.7110548615455627, "Finetune/Step": 2523, "Finetune/Step Time": 4.884757956489921} +{"Finetune/Learning Rate": 1.0948587240859893e-05, "Finetune/Loss": 0.8795175552368164, "Finetune/Loss (Raw)": 0.8840104937553406, "Finetune/Step": 2524, "Finetune/Step Time": 4.878131089732051} +{"Finetune/Learning Rate": 1.0942385537447236e-05, "Finetune/Loss": 0.8788372278213501, "Finetune/Loss (Raw)": 0.8469828367233276, "Finetune/Step": 2525, "Finetune/Step Time": 4.891465561464429} +{"Finetune/Learning Rate": 1.0936183468313147e-05, "Finetune/Loss": 0.8777551651000977, "Finetune/Loss (Raw)": 0.9426182508468628, "Finetune/Step": 2526, "Finetune/Step Time": 4.893997551873326} +{"Finetune/Learning Rate": 1.0929981035864532e-05, "Finetune/Loss": 0.8777462244033813, "Finetune/Loss (Raw)": 0.9878830909729004, "Finetune/Step": 2527, "Finetune/Step Time": 4.899120915681124} +{"Finetune/Learning Rate": 1.0923778242508437e-05, "Finetune/Loss": 0.8791334629058838, "Finetune/Loss (Raw)": 0.9985273480415344, "Finetune/Step": 2528, "Finetune/Step Time": 4.90539924800396} +{"Finetune/Learning Rate": 1.0917575090652038e-05, "Finetune/Loss": 0.8738835453987122, "Finetune/Loss (Raw)": 0.27080652117729187, "Finetune/Step": 2529, "Finetune/Step Time": 4.939219977706671} +{"Finetune/Learning Rate": 1.091137158270266e-05, "Finetune/Loss": 0.8747328519821167, "Finetune/Loss (Raw)": 0.9295666813850403, "Finetune/Step": 2530, "Finetune/Step Time": 4.930630875751376} +{"Finetune/Learning Rate": 1.0905167721067762e-05, "Finetune/Loss": 0.8739228844642639, "Finetune/Loss (Raw)": 0.8333380818367004, "Finetune/Step": 2531, "Finetune/Step Time": 4.929557099938393} +{"Finetune/Learning Rate": 1.0898963508154947e-05, "Finetune/Loss": 0.8742039203643799, "Finetune/Loss (Raw)": 0.9855433106422424, "Finetune/Step": 2532, "Finetune/Step Time": 4.946511248126626} +{"Finetune/Learning Rate": 1.0892758946371943e-05, "Finetune/Loss": 0.8739243745803833, "Finetune/Loss (Raw)": 0.9001362323760986, "Finetune/Step": 2533, "Finetune/Step Time": 4.954268669709563} +{"Finetune/Learning Rate": 1.0886554038126624e-05, "Finetune/Loss": 0.869110643863678, "Finetune/Loss (Raw)": 0.37764325737953186, "Finetune/Step": 2534, "Finetune/Step Time": 4.988833064213395} +{"Finetune/Learning Rate": 1.0880348785826991e-05, "Finetune/Loss": 0.8715834617614746, "Finetune/Loss (Raw)": 1.0165210962295532, "Finetune/Step": 2535, "Finetune/Step Time": 4.972793938592076} +{"Finetune/Learning Rate": 1.087414319188118e-05, "Finetune/Loss": 0.8686835765838623, "Finetune/Loss (Raw)": 0.7307580709457397, "Finetune/Step": 2536, "Finetune/Step Time": 4.980432415381074} +{"Finetune/Learning Rate": 1.0867937258697463e-05, "Finetune/Loss": 0.868713915348053, "Finetune/Loss (Raw)": 0.8948153853416443, "Finetune/Step": 2537, "Finetune/Step Time": 4.987461362034082} +{"Finetune/Learning Rate": 1.086173098868424e-05, "Finetune/Loss": 0.869259238243103, "Finetune/Loss (Raw)": 0.9991057515144348, "Finetune/Step": 2538, "Finetune/Step Time": 4.99634556286037} +{"Finetune/Learning Rate": 1.0855524384250048e-05, "Finetune/Loss": 0.8702113628387451, "Finetune/Loss (Raw)": 0.8816919326782227, "Finetune/Step": 2539, "Finetune/Step Time": 4.994384679943323} +{"Finetune/Learning Rate": 1.0849317447803542e-05, "Finetune/Loss": 0.8710662722587585, "Finetune/Loss (Raw)": 1.0063685178756714, "Finetune/Step": 2540, "Finetune/Step Time": 4.994337342679501} +{"Finetune/Learning Rate": 1.0843110181753513e-05, "Finetune/Loss": 0.8713259696960449, "Finetune/Loss (Raw)": 0.8468779921531677, "Finetune/Step": 2541, "Finetune/Step Time": 5.005589755252004} +{"Finetune/Learning Rate": 1.0836902588508888e-05, "Finetune/Loss": 0.871068000793457, "Finetune/Loss (Raw)": 0.9182526469230652, "Finetune/Step": 2542, "Finetune/Step Time": 5.015201356261969} +{"Finetune/Learning Rate": 1.0830694670478706e-05, "Finetune/Loss": 0.8714248538017273, "Finetune/Loss (Raw)": 0.9292468428611755, "Finetune/Step": 2543, "Finetune/Step Time": 5.010103441774845} +{"Finetune/Learning Rate": 1.082448643007214e-05, "Finetune/Loss": 0.8697360157966614, "Finetune/Loss (Raw)": 0.7355640530586243, "Finetune/Step": 2544, "Finetune/Step Time": 5.020206781104207} +{"Finetune/Learning Rate": 1.0818277869698488e-05, "Finetune/Loss": 0.8695507049560547, "Finetune/Loss (Raw)": 0.8777823448181152, "Finetune/Step": 2545, "Finetune/Step Time": 5.013478694483638} +{"Finetune/Learning Rate": 1.0812068991767171e-05, "Finetune/Loss": 0.8682961463928223, "Finetune/Loss (Raw)": 0.281225323677063, "Finetune/Step": 2546, "Finetune/Step Time": 5.013855246827006} +{"Finetune/Learning Rate": 1.0805859798687727e-05, "Finetune/Loss": 0.8681457042694092, "Finetune/Loss (Raw)": 0.8710177540779114, "Finetune/Step": 2547, "Finetune/Step Time": 4.984867928549647} +{"Finetune/Learning Rate": 1.0799650292869835e-05, "Finetune/Loss": 0.8678221702575684, "Finetune/Loss (Raw)": 1.034670352935791, "Finetune/Step": 2548, "Finetune/Step Time": 4.987388467416167} +{"Finetune/Learning Rate": 1.0793440476723274e-05, "Finetune/Loss": 0.8643326759338379, "Finetune/Loss (Raw)": 0.3238280713558197, "Finetune/Step": 2549, "Finetune/Step Time": 5.002577258273959} +{"Finetune/Learning Rate": 1.0787230352657954e-05, "Finetune/Loss": 0.865439772605896, "Finetune/Loss (Raw)": 0.9312877058982849, "Finetune/Step": 2550, "Finetune/Step Time": 4.992048254236579} +{"Finetune/Learning Rate": 1.0781019923083905e-05, "Finetune/Loss": 0.8671778440475464, "Finetune/Loss (Raw)": 0.9547916054725647, "Finetune/Step": 2551, "Finetune/Step Time": 4.987766159698367} +{"Finetune/Learning Rate": 1.077480919041127e-05, "Finetune/Loss": 0.8683228492736816, "Finetune/Loss (Raw)": 1.0875189304351807, "Finetune/Step": 2552, "Finetune/Step Time": 4.983159244060516} +{"Finetune/Learning Rate": 1.0768598157050313e-05, "Finetune/Loss": 0.8684887290000916, "Finetune/Loss (Raw)": 0.860230565071106, "Finetune/Step": 2553, "Finetune/Step Time": 4.984731826931238} +{"Finetune/Learning Rate": 1.0762386825411422e-05, "Finetune/Loss": 0.8729751110076904, "Finetune/Loss (Raw)": 0.9457066059112549, "Finetune/Step": 2554, "Finetune/Step Time": 4.973669109866023} +{"Finetune/Learning Rate": 1.0756175197905086e-05, "Finetune/Loss": 0.8679908514022827, "Finetune/Loss (Raw)": 0.3307300806045532, "Finetune/Step": 2555, "Finetune/Step Time": 5.0134392734616995} +{"Finetune/Learning Rate": 1.0749963276941919e-05, "Finetune/Loss": 0.8687757849693298, "Finetune/Loss (Raw)": 0.8997829556465149, "Finetune/Step": 2556, "Finetune/Step Time": 5.013409525156021} +{"Finetune/Learning Rate": 1.0743751064932648e-05, "Finetune/Loss": 0.867138147354126, "Finetune/Loss (Raw)": 0.7623387575149536, "Finetune/Step": 2557, "Finetune/Step Time": 4.996600216254592} +{"Finetune/Learning Rate": 1.0737538564288104e-05, "Finetune/Loss": 0.868010938167572, "Finetune/Loss (Raw)": 0.8680351972579956, "Finetune/Step": 2558, "Finetune/Step Time": 4.961199786514044} +{"Finetune/Learning Rate": 1.0731325777419246e-05, "Finetune/Loss": 0.8672217130661011, "Finetune/Loss (Raw)": 0.8994536995887756, "Finetune/Step": 2559, "Finetune/Step Time": 4.966124622151256} +{"Finetune/Learning Rate": 1.0725112706737133e-05, "Finetune/Loss": 0.8670075535774231, "Finetune/Loss (Raw)": 0.8827529549598694, "Finetune/Step": 2560, "Finetune/Step Time": 4.974853541702032} +{"Finetune/Learning Rate": 1.0718899354652931e-05, "Finetune/Loss": 0.8671789169311523, "Finetune/Loss (Raw)": 0.8925222158432007, "Finetune/Step": 2561, "Finetune/Step Time": 4.97830618545413} +{"Finetune/Learning Rate": 1.071268572357793e-05, "Finetune/Loss": 0.8678468465805054, "Finetune/Loss (Raw)": 1.0084928274154663, "Finetune/Step": 2562, "Finetune/Step Time": 4.977644104510546} +{"Finetune/Learning Rate": 1.0706471815923511e-05, "Finetune/Loss": 0.865964412689209, "Finetune/Loss (Raw)": 0.6425696611404419, "Finetune/Step": 2563, "Finetune/Step Time": 4.975696412846446} +{"Finetune/Learning Rate": 1.070025763410117e-05, "Finetune/Loss": 0.8655426502227783, "Finetune/Loss (Raw)": 0.9232407212257385, "Finetune/Step": 2564, "Finetune/Step Time": 4.984221711754799} +{"Finetune/Learning Rate": 1.0694043180522515e-05, "Finetune/Loss": 0.8663798570632935, "Finetune/Loss (Raw)": 0.981336772441864, "Finetune/Step": 2565, "Finetune/Step Time": 4.981849104166031} +{"Finetune/Learning Rate": 1.0687828457599254e-05, "Finetune/Loss": 0.8657094836235046, "Finetune/Loss (Raw)": 0.9253785014152527, "Finetune/Step": 2566, "Finetune/Step Time": 4.984700573608279} +{"Finetune/Learning Rate": 1.0681613467743195e-05, "Finetune/Loss": 0.8642298579216003, "Finetune/Loss (Raw)": 0.8042716979980469, "Finetune/Step": 2567, "Finetune/Step Time": 4.989605505019426} +{"Finetune/Learning Rate": 1.0675398213366257e-05, "Finetune/Loss": 0.862652063369751, "Finetune/Loss (Raw)": 0.8009165525436401, "Finetune/Step": 2568, "Finetune/Step Time": 4.993091996759176} +{"Finetune/Learning Rate": 1.0669182696880459e-05, "Finetune/Loss": 0.8638904094696045, "Finetune/Loss (Raw)": 1.1228265762329102, "Finetune/Step": 2569, "Finetune/Step Time": 4.990951901301742} +{"Finetune/Learning Rate": 1.0662966920697918e-05, "Finetune/Loss": 0.8638851642608643, "Finetune/Loss (Raw)": 0.9215251803398132, "Finetune/Step": 2570, "Finetune/Step Time": 4.995267368853092} +{"Finetune/Learning Rate": 1.0656750887230862e-05, "Finetune/Loss": 0.8645814657211304, "Finetune/Loss (Raw)": 1.0679807662963867, "Finetune/Step": 2571, "Finetune/Step Time": 4.993676573038101} +{"Finetune/Learning Rate": 1.0650534598891608e-05, "Finetune/Loss": 0.8638937473297119, "Finetune/Loss (Raw)": 0.8771668076515198, "Finetune/Step": 2572, "Finetune/Step Time": 4.99080853164196} +{"Finetune/Learning Rate": 1.0644318058092577e-05, "Finetune/Loss": 0.8679282069206238, "Finetune/Loss (Raw)": 0.9025535583496094, "Finetune/Step": 2573, "Finetune/Step Time": 4.962007420137525} +{"Finetune/Learning Rate": 1.0638101267246283e-05, "Finetune/Loss": 0.8675123453140259, "Finetune/Loss (Raw)": 0.8778642416000366, "Finetune/Step": 2574, "Finetune/Step Time": 4.984755601733923} +{"Finetune/Learning Rate": 1.063188422876535e-05, "Finetune/Loss": 0.8669013381004333, "Finetune/Loss (Raw)": 0.8260049819946289, "Finetune/Step": 2575, "Finetune/Step Time": 4.977048944681883} +{"Finetune/Learning Rate": 1.0625666945062484e-05, "Finetune/Loss": 0.8664027452468872, "Finetune/Loss (Raw)": 0.9399452805519104, "Finetune/Step": 2576, "Finetune/Step Time": 4.978248834609985} +{"Finetune/Learning Rate": 1.061944941855049e-05, "Finetune/Loss": 0.8676943778991699, "Finetune/Loss (Raw)": 0.976437509059906, "Finetune/Step": 2577, "Finetune/Step Time": 4.982171718031168} +{"Finetune/Learning Rate": 1.061323165164227e-05, "Finetune/Loss": 0.871853232383728, "Finetune/Loss (Raw)": 1.019568681716919, "Finetune/Step": 2578, "Finetune/Step Time": 4.947607707232237} +{"Finetune/Learning Rate": 1.0607013646750818e-05, "Finetune/Loss": 0.8771778345108032, "Finetune/Loss (Raw)": 1.0034656524658203, "Finetune/Step": 2579, "Finetune/Step Time": 4.9069722183048725} +{"Finetune/Learning Rate": 1.0600795406289221e-05, "Finetune/Loss": 0.8751449584960938, "Finetune/Loss (Raw)": 0.7041035890579224, "Finetune/Step": 2580, "Finetune/Step Time": 4.917698547244072} +{"Finetune/Learning Rate": 1.059457693267066e-05, "Finetune/Loss": 0.876075267791748, "Finetune/Loss (Raw)": 1.0465283393859863, "Finetune/Step": 2581, "Finetune/Step Time": 4.893910715356469} +{"Finetune/Learning Rate": 1.0588358228308395e-05, "Finetune/Loss": 0.8756961822509766, "Finetune/Loss (Raw)": 0.8743129372596741, "Finetune/Step": 2582, "Finetune/Step Time": 4.8831959795206785} +{"Finetune/Learning Rate": 1.0582139295615793e-05, "Finetune/Loss": 0.87425696849823, "Finetune/Loss (Raw)": 0.8155463933944702, "Finetune/Step": 2583, "Finetune/Step Time": 4.8916386645287275} +{"Finetune/Learning Rate": 1.0575920137006288e-05, "Finetune/Loss": 0.8747972249984741, "Finetune/Loss (Raw)": 1.0304023027420044, "Finetune/Step": 2584, "Finetune/Step Time": 4.892285602167249} +{"Finetune/Learning Rate": 1.056970075489343e-05, "Finetune/Loss": 0.8736642599105835, "Finetune/Loss (Raw)": 0.8066220283508301, "Finetune/Step": 2585, "Finetune/Step Time": 4.8945567067712545} +{"Finetune/Learning Rate": 1.0563481151690827e-05, "Finetune/Loss": 0.8737589716911316, "Finetune/Loss (Raw)": 0.9694843292236328, "Finetune/Step": 2586, "Finetune/Step Time": 4.880857339128852} +{"Finetune/Learning Rate": 1.055726132981219e-05, "Finetune/Loss": 0.8743140697479248, "Finetune/Loss (Raw)": 0.9431229829788208, "Finetune/Step": 2587, "Finetune/Step Time": 4.880973771214485} +{"Finetune/Learning Rate": 1.0551041291671311e-05, "Finetune/Loss": 0.873718798160553, "Finetune/Loss (Raw)": 0.856298565864563, "Finetune/Step": 2588, "Finetune/Step Time": 4.889601863920689} +{"Finetune/Learning Rate": 1.0544821039682066e-05, "Finetune/Loss": 0.8731098175048828, "Finetune/Loss (Raw)": 0.9890375733375549, "Finetune/Step": 2589, "Finetune/Step Time": 4.894088448956609} +{"Finetune/Learning Rate": 1.0538600576258413e-05, "Finetune/Loss": 0.8728231191635132, "Finetune/Loss (Raw)": 0.9257147908210754, "Finetune/Step": 2590, "Finetune/Step Time": 4.896971996873617} +{"Finetune/Learning Rate": 1.0532379903814387e-05, "Finetune/Loss": 0.8718467354774475, "Finetune/Loss (Raw)": 0.815662145614624, "Finetune/Step": 2591, "Finetune/Step Time": 4.938146870583296} +{"Finetune/Learning Rate": 1.0526159024764118e-05, "Finetune/Loss": 0.8710533380508423, "Finetune/Loss (Raw)": 0.9108842611312866, "Finetune/Step": 2592, "Finetune/Step Time": 4.94632507674396} +{"Finetune/Learning Rate": 1.0519937941521805e-05, "Finetune/Loss": 0.8704414367675781, "Finetune/Loss (Raw)": 0.8239016532897949, "Finetune/Step": 2593, "Finetune/Step Time": 4.966635026037693} +{"Finetune/Learning Rate": 1.0513716656501723e-05, "Finetune/Loss": 0.870506763458252, "Finetune/Loss (Raw)": 0.8206863403320312, "Finetune/Step": 2594, "Finetune/Step Time": 4.972842192277312} +{"Finetune/Learning Rate": 1.0507495172118242e-05, "Finetune/Loss": 0.87025386095047, "Finetune/Loss (Raw)": 0.8838417530059814, "Finetune/Step": 2595, "Finetune/Step Time": 4.9777405224740505} +{"Finetune/Learning Rate": 1.0501273490785789e-05, "Finetune/Loss": 0.8682910203933716, "Finetune/Loss (Raw)": 0.8638875484466553, "Finetune/Step": 2596, "Finetune/Step Time": 4.975547252222896} +{"Finetune/Learning Rate": 1.049505161491888e-05, "Finetune/Loss": 0.867974042892456, "Finetune/Loss (Raw)": 0.9599487781524658, "Finetune/Step": 2597, "Finetune/Step Time": 4.978599680587649} +{"Finetune/Learning Rate": 1.0488829546932106e-05, "Finetune/Loss": 0.8671261668205261, "Finetune/Loss (Raw)": 0.7350807785987854, "Finetune/Step": 2598, "Finetune/Step Time": 4.990762278437614} +{"Finetune/Learning Rate": 1.0482607289240128e-05, "Finetune/Loss": 0.8668534755706787, "Finetune/Loss (Raw)": 0.9514973759651184, "Finetune/Step": 2599, "Finetune/Step Time": 4.994969263672829} +{"Finetune/Learning Rate": 1.0476384844257686e-05, "Finetune/Loss": 0.8664418458938599, "Finetune/Loss (Raw)": 0.8868362903594971, "Finetune/Step": 2600, "Finetune/Step Time": 4.999482020735741} +{"Finetune/Learning Rate": 1.0470162214399582e-05, "Finetune/Loss": 0.8662512302398682, "Finetune/Loss (Raw)": 0.9171826839447021, "Finetune/Step": 2601, "Finetune/Step Time": 5.000994700938463} +{"Finetune/Learning Rate": 1.0463939402080706e-05, "Finetune/Loss": 0.866985023021698, "Finetune/Loss (Raw)": 1.022365689277649, "Finetune/Step": 2602, "Finetune/Step Time": 4.9931554570794106} +{"Finetune/Learning Rate": 1.0457716409716002e-05, "Finetune/Loss": 0.8678137063980103, "Finetune/Loss (Raw)": 0.9880717992782593, "Finetune/Step": 2603, "Finetune/Step Time": 4.998120257630944} +{"Finetune/Learning Rate": 1.0451493239720496e-05, "Finetune/Loss": 0.8673675656318665, "Finetune/Loss (Raw)": 0.881629228591919, "Finetune/Step": 2604, "Finetune/Step Time": 5.004247236996889} +{"Finetune/Learning Rate": 1.0445269894509283e-05, "Finetune/Loss": 0.8663123846054077, "Finetune/Loss (Raw)": 0.7936748266220093, "Finetune/Step": 2605, "Finetune/Step Time": 5.002802096307278} +{"Finetune/Learning Rate": 1.043904637649751e-05, "Finetune/Loss": 0.865687906742096, "Finetune/Loss (Raw)": 0.8341328501701355, "Finetune/Step": 2606, "Finetune/Step Time": 5.032365093007684} +{"Finetune/Learning Rate": 1.0432822688100414e-05, "Finetune/Loss": 0.8665850758552551, "Finetune/Loss (Raw)": 0.9495536684989929, "Finetune/Step": 2607, "Finetune/Step Time": 5.05281500518322} +{"Finetune/Learning Rate": 1.0426598831733283e-05, "Finetune/Loss": 0.8674159646034241, "Finetune/Loss (Raw)": 0.9544536471366882, "Finetune/Step": 2608, "Finetune/Step Time": 5.044870980083942} +{"Finetune/Learning Rate": 1.0420374809811472e-05, "Finetune/Loss": 0.8717437982559204, "Finetune/Loss (Raw)": 0.9293793439865112, "Finetune/Step": 2609, "Finetune/Step Time": 5.008504351601005} +{"Finetune/Learning Rate": 1.0414150624750408e-05, "Finetune/Loss": 0.8700172305107117, "Finetune/Loss (Raw)": 0.8706982731819153, "Finetune/Step": 2610, "Finetune/Step Time": 5.0310073141008615} +{"Finetune/Learning Rate": 1.0407926278965568e-05, "Finetune/Loss": 0.8684694766998291, "Finetune/Loss (Raw)": 0.7130914926528931, "Finetune/Step": 2611, "Finetune/Step Time": 5.0319711826741695} +{"Finetune/Learning Rate": 1.0401701774872508e-05, "Finetune/Loss": 0.8685906529426575, "Finetune/Loss (Raw)": 0.8853207230567932, "Finetune/Step": 2612, "Finetune/Step Time": 5.020249741151929} +{"Finetune/Learning Rate": 1.0395477114886827e-05, "Finetune/Loss": 0.8692783713340759, "Finetune/Loss (Raw)": 1.0184625387191772, "Finetune/Step": 2613, "Finetune/Step Time": 5.018955606967211} +{"Finetune/Learning Rate": 1.0389252301424197e-05, "Finetune/Loss": 0.8694348931312561, "Finetune/Loss (Raw)": 0.8438239693641663, "Finetune/Step": 2614, "Finetune/Step Time": 5.011005895212293} +{"Finetune/Learning Rate": 1.0383027336900356e-05, "Finetune/Loss": 0.8744685649871826, "Finetune/Loss (Raw)": 0.9591875672340393, "Finetune/Step": 2615, "Finetune/Step Time": 4.98608511313796} +{"Finetune/Learning Rate": 1.0376802223731077e-05, "Finetune/Loss": 0.8737897872924805, "Finetune/Loss (Raw)": 0.9517544507980347, "Finetune/Step": 2616, "Finetune/Step Time": 4.9840869549661875} +{"Finetune/Learning Rate": 1.0370576964332217e-05, "Finetune/Loss": 0.8693928718566895, "Finetune/Loss (Raw)": 0.3951781094074249, "Finetune/Step": 2617, "Finetune/Step Time": 4.995660785585642} +{"Finetune/Learning Rate": 1.0364351561119667e-05, "Finetune/Loss": 0.8731539845466614, "Finetune/Loss (Raw)": 0.8003231287002563, "Finetune/Step": 2618, "Finetune/Step Time": 4.966628290712833} +{"Finetune/Learning Rate": 1.0358126016509392e-05, "Finetune/Loss": 0.8736811876296997, "Finetune/Loss (Raw)": 0.9630060195922852, "Finetune/Step": 2619, "Finetune/Step Time": 4.960825324058533} +{"Finetune/Learning Rate": 1.0351900332917401e-05, "Finetune/Loss": 0.8739486932754517, "Finetune/Loss (Raw)": 0.8285928964614868, "Finetune/Step": 2620, "Finetune/Step Time": 4.951375307515264} +{"Finetune/Learning Rate": 1.034567451275976e-05, "Finetune/Loss": 0.8731859922409058, "Finetune/Loss (Raw)": 0.8776260018348694, "Finetune/Step": 2621, "Finetune/Step Time": 4.953988499939442} +{"Finetune/Learning Rate": 1.0339448558452594e-05, "Finetune/Loss": 0.8725534677505493, "Finetune/Loss (Raw)": 0.9413822293281555, "Finetune/Step": 2622, "Finetune/Step Time": 4.9434446562081575} +{"Finetune/Learning Rate": 1.0333222472412064e-05, "Finetune/Loss": 0.8722887635231018, "Finetune/Loss (Raw)": 0.9013311862945557, "Finetune/Step": 2623, "Finetune/Step Time": 4.963899068534374} +{"Finetune/Learning Rate": 1.03269962570544e-05, "Finetune/Loss": 0.8721818923950195, "Finetune/Loss (Raw)": 0.9064111113548279, "Finetune/Step": 2624, "Finetune/Step Time": 4.9576664343476295} +{"Finetune/Learning Rate": 1.0320769914795874e-05, "Finetune/Loss": 0.8718397617340088, "Finetune/Loss (Raw)": 0.7647229433059692, "Finetune/Step": 2625, "Finetune/Step Time": 4.971353992819786} +{"Finetune/Learning Rate": 1.0314543448052806e-05, "Finetune/Loss": 0.8663368225097656, "Finetune/Loss (Raw)": 0.3100460469722748, "Finetune/Step": 2626, "Finetune/Step Time": 5.012578999623656} +{"Finetune/Learning Rate": 1.0308316859241567e-05, "Finetune/Loss": 0.8680962324142456, "Finetune/Loss (Raw)": 0.889086902141571, "Finetune/Step": 2627, "Finetune/Step Time": 4.995057361200452} +{"Finetune/Learning Rate": 1.0302090150778572e-05, "Finetune/Loss": 0.8684545755386353, "Finetune/Loss (Raw)": 0.9882948398590088, "Finetune/Step": 2628, "Finetune/Step Time": 4.99428172595799} +{"Finetune/Learning Rate": 1.029586332508029e-05, "Finetune/Loss": 0.8661300539970398, "Finetune/Loss (Raw)": 0.6380935907363892, "Finetune/Step": 2629, "Finetune/Step Time": 5.009647268801928} +{"Finetune/Learning Rate": 1.0289636384563226e-05, "Finetune/Loss": 0.8650603294372559, "Finetune/Loss (Raw)": 0.8070935010910034, "Finetune/Step": 2630, "Finetune/Step Time": 5.00958832539618} +{"Finetune/Learning Rate": 1.0283409331643936e-05, "Finetune/Loss": 0.8662476539611816, "Finetune/Loss (Raw)": 1.0351775884628296, "Finetune/Step": 2631, "Finetune/Step Time": 5.00650885142386} +{"Finetune/Learning Rate": 1.0277182168739019e-05, "Finetune/Loss": 0.8654906749725342, "Finetune/Loss (Raw)": 0.7998211979866028, "Finetune/Step": 2632, "Finetune/Step Time": 5.00204150006175} +{"Finetune/Learning Rate": 1.0270954898265113e-05, "Finetune/Loss": 0.8658199310302734, "Finetune/Loss (Raw)": 0.9438114762306213, "Finetune/Step": 2633, "Finetune/Step Time": 5.003998154774308} +{"Finetune/Learning Rate": 1.0264727522638899e-05, "Finetune/Loss": 0.8665071725845337, "Finetune/Loss (Raw)": 0.950284481048584, "Finetune/Step": 2634, "Finetune/Step Time": 5.004591070115566} +{"Finetune/Learning Rate": 1.0258500044277106e-05, "Finetune/Loss": 0.8658875823020935, "Finetune/Loss (Raw)": 0.924872100353241, "Finetune/Step": 2635, "Finetune/Step Time": 4.998906143009663} +{"Finetune/Learning Rate": 1.025227246559649e-05, "Finetune/Loss": 0.8671947717666626, "Finetune/Loss (Raw)": 1.012012004852295, "Finetune/Step": 2636, "Finetune/Step Time": 4.998094253242016} +{"Finetune/Learning Rate": 1.0246044789013855e-05, "Finetune/Loss": 0.8671413660049438, "Finetune/Loss (Raw)": 0.8298954963684082, "Finetune/Step": 2637, "Finetune/Step Time": 5.003116648644209} +{"Finetune/Learning Rate": 1.023981701694604e-05, "Finetune/Loss": 0.8665878772735596, "Finetune/Loss (Raw)": 0.9770249128341675, "Finetune/Step": 2638, "Finetune/Step Time": 5.023409742861986} +{"Finetune/Learning Rate": 1.0233589151809927e-05, "Finetune/Loss": 0.8653405904769897, "Finetune/Loss (Raw)": 0.8232247233390808, "Finetune/Step": 2639, "Finetune/Step Time": 5.014040995389223} +{"Finetune/Learning Rate": 1.0227361196022421e-05, "Finetune/Loss": 0.8617498278617859, "Finetune/Loss (Raw)": 0.34833624958992004, "Finetune/Step": 2640, "Finetune/Step Time": 5.0406352281570435} +{"Finetune/Learning Rate": 1.0221133152000474e-05, "Finetune/Loss": 0.8621799945831299, "Finetune/Loss (Raw)": 0.9039609432220459, "Finetune/Step": 2641, "Finetune/Step Time": 5.032068310305476} +{"Finetune/Learning Rate": 1.021490502216107e-05, "Finetune/Loss": 0.8611934781074524, "Finetune/Loss (Raw)": 0.8895432949066162, "Finetune/Step": 2642, "Finetune/Step Time": 5.021736932918429} +{"Finetune/Learning Rate": 1.020867680892122e-05, "Finetune/Loss": 0.8609803915023804, "Finetune/Loss (Raw)": 0.7394319772720337, "Finetune/Step": 2643, "Finetune/Step Time": 5.018837591633201} +{"Finetune/Learning Rate": 1.0202448514697975e-05, "Finetune/Loss": 0.8596887588500977, "Finetune/Loss (Raw)": 0.7642960548400879, "Finetune/Step": 2644, "Finetune/Step Time": 5.052418410778046} +{"Finetune/Learning Rate": 1.0196220141908417e-05, "Finetune/Loss": 0.8596987128257751, "Finetune/Loss (Raw)": 0.9363784193992615, "Finetune/Step": 2645, "Finetune/Step Time": 5.0557428281754255} +{"Finetune/Learning Rate": 1.018999169296965e-05, "Finetune/Loss": 0.8603542447090149, "Finetune/Loss (Raw)": 0.9409670829772949, "Finetune/Step": 2646, "Finetune/Step Time": 5.041759813204408} +{"Finetune/Learning Rate": 1.0183763170298818e-05, "Finetune/Loss": 0.859933614730835, "Finetune/Loss (Raw)": 0.8245267271995544, "Finetune/Step": 2647, "Finetune/Step Time": 5.047855794429779} +{"Finetune/Learning Rate": 1.0177534576313084e-05, "Finetune/Loss": 0.8596943616867065, "Finetune/Loss (Raw)": 0.9559123516082764, "Finetune/Step": 2648, "Finetune/Step Time": 5.0586852096021175} +{"Finetune/Learning Rate": 1.0171305913429648e-05, "Finetune/Loss": 0.8631601333618164, "Finetune/Loss (Raw)": 0.850496768951416, "Finetune/Step": 2649, "Finetune/Step Time": 5.036636132746935} +{"Finetune/Learning Rate": 1.0165077184065732e-05, "Finetune/Loss": 0.8621066212654114, "Finetune/Loss (Raw)": 0.8097656965255737, "Finetune/Step": 2650, "Finetune/Step Time": 5.019741959869862} +{"Finetune/Learning Rate": 1.0158848390638588e-05, "Finetune/Loss": 0.8631793260574341, "Finetune/Loss (Raw)": 0.8483616709709167, "Finetune/Step": 2651, "Finetune/Step Time": 5.015041716396809} +{"Finetune/Learning Rate": 1.015261953556548e-05, "Finetune/Loss": 0.8641961812973022, "Finetune/Loss (Raw)": 1.0141628980636597, "Finetune/Step": 2652, "Finetune/Step Time": 5.021550960838795} +{"Finetune/Learning Rate": 1.0146390621263709e-05, "Finetune/Loss": 0.8645141124725342, "Finetune/Loss (Raw)": 0.8876878619194031, "Finetune/Step": 2653, "Finetune/Step Time": 5.033078318461776} +{"Finetune/Learning Rate": 1.0140161650150598e-05, "Finetune/Loss": 0.8598194122314453, "Finetune/Loss (Raw)": 0.34169504046440125, "Finetune/Step": 2654, "Finetune/Step Time": 5.061197642236948} +{"Finetune/Learning Rate": 1.0133932624643487e-05, "Finetune/Loss": 0.85929936170578, "Finetune/Loss (Raw)": 0.9213157296180725, "Finetune/Step": 2655, "Finetune/Step Time": 5.059582628309727} +{"Finetune/Learning Rate": 1.012770354715974e-05, "Finetune/Loss": 0.8585144281387329, "Finetune/Loss (Raw)": 0.8980554342269897, "Finetune/Step": 2656, "Finetune/Step Time": 5.056056363508105} +{"Finetune/Learning Rate": 1.012147442011674e-05, "Finetune/Loss": 0.8619794845581055, "Finetune/Loss (Raw)": 0.7143309116363525, "Finetune/Step": 2657, "Finetune/Step Time": 5.035777559503913} +{"Finetune/Learning Rate": 1.0115245245931883e-05, "Finetune/Loss": 0.8601871728897095, "Finetune/Loss (Raw)": 0.7001508474349976, "Finetune/Step": 2658, "Finetune/Step Time": 5.054806526750326} +{"Finetune/Learning Rate": 1.0109016027022602e-05, "Finetune/Loss": 0.8615490198135376, "Finetune/Loss (Raw)": 1.0076524019241333, "Finetune/Step": 2659, "Finetune/Step Time": 5.045758383348584} +{"Finetune/Learning Rate": 1.0102786765806329e-05, "Finetune/Loss": 0.8611217737197876, "Finetune/Loss (Raw)": 0.9308541417121887, "Finetune/Step": 2660, "Finetune/Step Time": 5.02957733720541} +{"Finetune/Learning Rate": 1.0096557464700518e-05, "Finetune/Loss": 0.8626816272735596, "Finetune/Loss (Raw)": 1.0998090505599976, "Finetune/Step": 2661, "Finetune/Step Time": 5.0225626938045025} +{"Finetune/Learning Rate": 1.0090328126122642e-05, "Finetune/Loss": 0.8676435947418213, "Finetune/Loss (Raw)": 1.0127617120742798, "Finetune/Step": 2662, "Finetune/Step Time": 4.984150042757392} +{"Finetune/Learning Rate": 1.0084098752490179e-05, "Finetune/Loss": 0.8673396706581116, "Finetune/Loss (Raw)": 0.9776274561882019, "Finetune/Step": 2663, "Finetune/Step Time": 4.991571569815278} +{"Finetune/Learning Rate": 1.0077869346220635e-05, "Finetune/Loss": 0.8650602698326111, "Finetune/Loss (Raw)": 0.438994824886322, "Finetune/Step": 2664, "Finetune/Step Time": 5.0136224161833525} +{"Finetune/Learning Rate": 1.0071639909731519e-05, "Finetune/Loss": 0.8641535043716431, "Finetune/Loss (Raw)": 0.778753936290741, "Finetune/Step": 2665, "Finetune/Step Time": 5.006286766380072} +{"Finetune/Learning Rate": 1.0065410445440352e-05, "Finetune/Loss": 0.8639081716537476, "Finetune/Loss (Raw)": 0.9676968455314636, "Finetune/Step": 2666, "Finetune/Step Time": 5.003116976469755} +{"Finetune/Learning Rate": 1.0059180955764666e-05, "Finetune/Loss": 0.8632311820983887, "Finetune/Loss (Raw)": 0.7950422167778015, "Finetune/Step": 2667, "Finetune/Step Time": 4.998172085732222} +{"Finetune/Learning Rate": 1.005295144312201e-05, "Finetune/Loss": 0.862031102180481, "Finetune/Loss (Raw)": 0.8527532815933228, "Finetune/Step": 2668, "Finetune/Step Time": 4.994463037699461} +{"Finetune/Learning Rate": 1.0046721909929928e-05, "Finetune/Loss": 0.8632287383079529, "Finetune/Loss (Raw)": 1.0001790523529053, "Finetune/Step": 2669, "Finetune/Step Time": 4.982936857268214} +{"Finetune/Learning Rate": 1.0040492358605989e-05, "Finetune/Loss": 0.863233745098114, "Finetune/Loss (Raw)": 0.918890655040741, "Finetune/Step": 2670, "Finetune/Step Time": 4.972852025181055} +{"Finetune/Learning Rate": 1.0034262791567755e-05, "Finetune/Loss": 0.861990213394165, "Finetune/Loss (Raw)": 0.7700791954994202, "Finetune/Step": 2671, "Finetune/Step Time": 4.979534080252051} +{"Finetune/Learning Rate": 1.0028033211232802e-05, "Finetune/Loss": 0.8615577816963196, "Finetune/Loss (Raw)": 0.6802077293395996, "Finetune/Step": 2672, "Finetune/Step Time": 4.984870342537761} +{"Finetune/Learning Rate": 1.0021803620018706e-05, "Finetune/Loss": 0.8614686131477356, "Finetune/Loss (Raw)": 0.8663730621337891, "Finetune/Step": 2673, "Finetune/Step Time": 4.984930153936148} +{"Finetune/Learning Rate": 1.0015574020343048e-05, "Finetune/Loss": 0.8658332824707031, "Finetune/Loss (Raw)": 0.8399014472961426, "Finetune/Step": 2674, "Finetune/Step Time": 4.9606782998889685} +{"Finetune/Learning Rate": 1.0009344414623418e-05, "Finetune/Loss": 0.8657709360122681, "Finetune/Loss (Raw)": 0.8630293011665344, "Finetune/Step": 2675, "Finetune/Step Time": 4.970747148618102} +{"Finetune/Learning Rate": 1.0003114805277405e-05, "Finetune/Loss": 0.8642204999923706, "Finetune/Loss (Raw)": 0.8362205028533936, "Finetune/Step": 2676, "Finetune/Step Time": 4.974167715758085} +{"Finetune/Learning Rate": 9.9968851947226e-06, "Finetune/Loss": 0.8693469762802124, "Finetune/Loss (Raw)": 0.9800218939781189, "Finetune/Step": 2677, "Finetune/Step Time": 4.936621308326721} +{"Finetune/Learning Rate": 9.990655585376583e-06, "Finetune/Loss": 0.8688930869102478, "Finetune/Loss (Raw)": 0.8731815218925476, "Finetune/Step": 2678, "Finetune/Step Time": 4.9357655961066484} +{"Finetune/Learning Rate": 9.984425979656955e-06, "Finetune/Loss": 0.8693777322769165, "Finetune/Loss (Raw)": 1.016831636428833, "Finetune/Step": 2679, "Finetune/Step Time": 4.92781719379127} +{"Finetune/Learning Rate": 9.978196379981297e-06, "Finetune/Loss": 0.868070125579834, "Finetune/Loss (Raw)": 0.9201393127441406, "Finetune/Step": 2680, "Finetune/Step Time": 4.930840838700533} +{"Finetune/Learning Rate": 9.971966788767198e-06, "Finetune/Loss": 0.8640214204788208, "Finetune/Loss (Raw)": 0.34200534224510193, "Finetune/Step": 2681, "Finetune/Step Time": 4.953353663906455} +{"Finetune/Learning Rate": 9.96573720843225e-06, "Finetune/Loss": 0.8636248111724854, "Finetune/Loss (Raw)": 0.8949390649795532, "Finetune/Step": 2682, "Finetune/Step Time": 4.929069427773356} +{"Finetune/Learning Rate": 9.959507641394015e-06, "Finetune/Loss": 0.8681304454803467, "Finetune/Loss (Raw)": 0.9074508547782898, "Finetune/Step": 2683, "Finetune/Step Time": 4.886771848425269} +{"Finetune/Learning Rate": 9.953278090070075e-06, "Finetune/Loss": 0.8679728507995605, "Finetune/Loss (Raw)": 0.8796113133430481, "Finetune/Step": 2684, "Finetune/Step Time": 4.886710537597537} +{"Finetune/Learning Rate": 9.947048556877992e-06, "Finetune/Loss": 0.8690926432609558, "Finetune/Loss (Raw)": 0.9056700468063354, "Finetune/Step": 2685, "Finetune/Step Time": 4.890600023791194} +{"Finetune/Learning Rate": 9.940819044235335e-06, "Finetune/Loss": 0.8684922456741333, "Finetune/Loss (Raw)": 0.791189432144165, "Finetune/Step": 2686, "Finetune/Step Time": 4.893291408196092} +{"Finetune/Learning Rate": 9.934589554559653e-06, "Finetune/Loss": 0.8684024810791016, "Finetune/Loss (Raw)": 0.8879584074020386, "Finetune/Step": 2687, "Finetune/Step Time": 4.888763511553407} +{"Finetune/Learning Rate": 9.928360090268484e-06, "Finetune/Loss": 0.8687488436698914, "Finetune/Loss (Raw)": 0.927087664604187, "Finetune/Step": 2688, "Finetune/Step Time": 4.880581585690379} +{"Finetune/Learning Rate": 9.922130653779368e-06, "Finetune/Loss": 0.8697352409362793, "Finetune/Loss (Raw)": 1.0187803506851196, "Finetune/Step": 2689, "Finetune/Step Time": 4.8782466147094965} +{"Finetune/Learning Rate": 9.915901247509821e-06, "Finetune/Loss": 0.8647483587265015, "Finetune/Loss (Raw)": 0.3701702654361725, "Finetune/Step": 2690, "Finetune/Step Time": 4.9193859957158566} +{"Finetune/Learning Rate": 9.909671873877363e-06, "Finetune/Loss": 0.8679391145706177, "Finetune/Loss (Raw)": 1.0509809255599976, "Finetune/Step": 2691, "Finetune/Step Time": 4.911768075078726} +{"Finetune/Learning Rate": 9.903442535299487e-06, "Finetune/Loss": 0.8673661947250366, "Finetune/Loss (Raw)": 0.849915087223053, "Finetune/Step": 2692, "Finetune/Step Time": 4.905511166900396} +{"Finetune/Learning Rate": 9.897213234193673e-06, "Finetune/Loss": 0.8671625852584839, "Finetune/Loss (Raw)": 0.9552725553512573, "Finetune/Step": 2693, "Finetune/Step Time": 4.902890976518393} +{"Finetune/Learning Rate": 9.890983972977401e-06, "Finetune/Loss": 0.8669469952583313, "Finetune/Loss (Raw)": 0.8977847695350647, "Finetune/Step": 2694, "Finetune/Step Time": 4.909734999760985} +{"Finetune/Learning Rate": 9.884754754068116e-06, "Finetune/Loss": 0.8672903180122375, "Finetune/Loss (Raw)": 0.8482123613357544, "Finetune/Step": 2695, "Finetune/Step Time": 4.908732151612639} +{"Finetune/Learning Rate": 9.878525579883264e-06, "Finetune/Loss": 0.8685414791107178, "Finetune/Loss (Raw)": 0.9610751271247864, "Finetune/Step": 2696, "Finetune/Step Time": 4.92079252935946} +{"Finetune/Learning Rate": 9.872296452840266e-06, "Finetune/Loss": 0.8665385246276855, "Finetune/Loss (Raw)": 0.866435170173645, "Finetune/Step": 2697, "Finetune/Step Time": 4.923290500417352} +{"Finetune/Learning Rate": 9.866067375356516e-06, "Finetune/Loss": 0.8657066822052002, "Finetune/Loss (Raw)": 0.8150562047958374, "Finetune/Step": 2698, "Finetune/Step Time": 4.918738748878241} +{"Finetune/Learning Rate": 9.859838349849404e-06, "Finetune/Loss": 0.8651705980300903, "Finetune/Loss (Raw)": 0.9993649125099182, "Finetune/Step": 2699, "Finetune/Step Time": 4.9331444669514894} +{"Finetune/Learning Rate": 9.853609378736293e-06, "Finetune/Loss": 0.8650623559951782, "Finetune/Loss (Raw)": 0.8633114099502563, "Finetune/Step": 2700, "Finetune/Step Time": 4.932805335149169} +{"Finetune/Learning Rate": 9.847380464434525e-06, "Finetune/Loss": 0.8648513555526733, "Finetune/Loss (Raw)": 0.8755428791046143, "Finetune/Step": 2701, "Finetune/Step Time": 4.955815073102713} +{"Finetune/Learning Rate": 9.841151609361419e-06, "Finetune/Loss": 0.8651780486106873, "Finetune/Loss (Raw)": 0.9196807742118835, "Finetune/Step": 2702, "Finetune/Step Time": 4.9443314447999} +{"Finetune/Learning Rate": 9.83492281593427e-06, "Finetune/Loss": 0.865179181098938, "Finetune/Loss (Raw)": 0.8261502981185913, "Finetune/Step": 2703, "Finetune/Step Time": 4.9612630400806665} +{"Finetune/Learning Rate": 9.828694086570353e-06, "Finetune/Loss": 0.8640496730804443, "Finetune/Loss (Raw)": 0.7953776121139526, "Finetune/Step": 2704, "Finetune/Step Time": 4.97355142980814} +{"Finetune/Learning Rate": 9.822465423686917e-06, "Finetune/Loss": 0.8631516098976135, "Finetune/Loss (Raw)": 0.8614702820777893, "Finetune/Step": 2705, "Finetune/Step Time": 4.974180644378066} +{"Finetune/Learning Rate": 9.816236829701187e-06, "Finetune/Loss": 0.8628104329109192, "Finetune/Loss (Raw)": 0.9759066700935364, "Finetune/Step": 2706, "Finetune/Step Time": 4.9811097253113985} +{"Finetune/Learning Rate": 9.810008307030356e-06, "Finetune/Loss": 0.8627625703811646, "Finetune/Loss (Raw)": 0.9973306059837341, "Finetune/Step": 2707, "Finetune/Step Time": 4.977618070319295} +{"Finetune/Learning Rate": 9.803779858091586e-06, "Finetune/Loss": 0.8651286363601685, "Finetune/Loss (Raw)": 1.0069749355316162, "Finetune/Step": 2708, "Finetune/Step Time": 4.974937692284584} +{"Finetune/Learning Rate": 9.797551485302027e-06, "Finetune/Loss": 0.8636949062347412, "Finetune/Loss (Raw)": 0.8629968762397766, "Finetune/Step": 2709, "Finetune/Step Time": 4.984888501465321} +{"Finetune/Learning Rate": 9.791323191078781e-06, "Finetune/Loss": 0.8598345518112183, "Finetune/Loss (Raw)": 0.3801940679550171, "Finetune/Step": 2710, "Finetune/Step Time": 5.021918788552284} +{"Finetune/Learning Rate": 9.785094977838935e-06, "Finetune/Loss": 0.860933244228363, "Finetune/Loss (Raw)": 0.9561779499053955, "Finetune/Step": 2711, "Finetune/Step Time": 5.012292785570025} +{"Finetune/Learning Rate": 9.77886684799953e-06, "Finetune/Loss": 0.8601112365722656, "Finetune/Loss (Raw)": 0.9251899719238281, "Finetune/Step": 2712, "Finetune/Step Time": 5.030718374997377} +{"Finetune/Learning Rate": 9.77263880397758e-06, "Finetune/Loss": 0.8606681823730469, "Finetune/Loss (Raw)": 0.8779043555259705, "Finetune/Step": 2713, "Finetune/Step Time": 5.037595434114337} +{"Finetune/Learning Rate": 9.766410848190076e-06, "Finetune/Loss": 0.860515832901001, "Finetune/Loss (Raw)": 0.9499832987785339, "Finetune/Step": 2714, "Finetune/Step Time": 5.037332616746426} +{"Finetune/Learning Rate": 9.76018298305396e-06, "Finetune/Loss": 0.8596082925796509, "Finetune/Loss (Raw)": 0.8269579410552979, "Finetune/Step": 2715, "Finetune/Step Time": 5.030222523957491} +{"Finetune/Learning Rate": 9.75395521098615e-06, "Finetune/Loss": 0.8596081733703613, "Finetune/Loss (Raw)": 0.8562820553779602, "Finetune/Step": 2716, "Finetune/Step Time": 5.022272335365415} +{"Finetune/Learning Rate": 9.747727534403515e-06, "Finetune/Loss": 0.8588948249816895, "Finetune/Loss (Raw)": 0.8977267146110535, "Finetune/Step": 2717, "Finetune/Step Time": 5.003662221133709} +{"Finetune/Learning Rate": 9.741499955722897e-06, "Finetune/Loss": 0.858454704284668, "Finetune/Loss (Raw)": 0.8693864345550537, "Finetune/Step": 2718, "Finetune/Step Time": 5.003470757976174} +{"Finetune/Learning Rate": 9.735272477361103e-06, "Finetune/Loss": 0.8593788743019104, "Finetune/Loss (Raw)": 0.9339589476585388, "Finetune/Step": 2719, "Finetune/Step Time": 4.958637995645404} +{"Finetune/Learning Rate": 9.729045101734889e-06, "Finetune/Loss": 0.8596136569976807, "Finetune/Loss (Raw)": 0.9409390091896057, "Finetune/Step": 2720, "Finetune/Step Time": 4.954262364655733} +{"Finetune/Learning Rate": 9.722817831260985e-06, "Finetune/Loss": 0.859806478023529, "Finetune/Loss (Raw)": 0.8485742211341858, "Finetune/Step": 2721, "Finetune/Step Time": 4.945002572610974} +{"Finetune/Learning Rate": 9.716590668356067e-06, "Finetune/Loss": 0.8603243827819824, "Finetune/Loss (Raw)": 0.8869855403900146, "Finetune/Step": 2722, "Finetune/Step Time": 4.935423946008086} +{"Finetune/Learning Rate": 9.710363615436777e-06, "Finetune/Loss": 0.860083818435669, "Finetune/Loss (Raw)": 0.8530471324920654, "Finetune/Step": 2723, "Finetune/Step Time": 4.930177751928568} +{"Finetune/Learning Rate": 9.704136674919712e-06, "Finetune/Loss": 0.8605250716209412, "Finetune/Loss (Raw)": 0.9203680157661438, "Finetune/Step": 2724, "Finetune/Step Time": 4.927748227491975} +{"Finetune/Learning Rate": 9.697909849221428e-06, "Finetune/Loss": 0.8601241111755371, "Finetune/Loss (Raw)": 0.9086241126060486, "Finetune/Step": 2725, "Finetune/Step Time": 4.921350106596947} +{"Finetune/Learning Rate": 9.691683140758439e-06, "Finetune/Loss": 0.8615124821662903, "Finetune/Loss (Raw)": 0.9127916693687439, "Finetune/Step": 2726, "Finetune/Step Time": 4.918427450582385} +{"Finetune/Learning Rate": 9.685456551947199e-06, "Finetune/Loss": 0.8609963655471802, "Finetune/Loss (Raw)": 0.8854377269744873, "Finetune/Step": 2727, "Finetune/Step Time": 4.921283653005958} +{"Finetune/Learning Rate": 9.67923008520413e-06, "Finetune/Loss": 0.8601759672164917, "Finetune/Loss (Raw)": 0.7818220257759094, "Finetune/Step": 2728, "Finetune/Step Time": 4.920086815953255} +{"Finetune/Learning Rate": 9.673003742945602e-06, "Finetune/Loss": 0.8562992215156555, "Finetune/Loss (Raw)": 0.4209560453891754, "Finetune/Step": 2729, "Finetune/Step Time": 4.9458037707954645} +{"Finetune/Learning Rate": 9.666777527587936e-06, "Finetune/Loss": 0.854354977607727, "Finetune/Loss (Raw)": 0.7735085487365723, "Finetune/Step": 2730, "Finetune/Step Time": 4.9478308372199535} +{"Finetune/Learning Rate": 9.660551441547411e-06, "Finetune/Loss": 0.8536579608917236, "Finetune/Loss (Raw)": 0.898849606513977, "Finetune/Step": 2731, "Finetune/Step Time": 4.944614067673683} +{"Finetune/Learning Rate": 9.654325487240243e-06, "Finetune/Loss": 0.8540704250335693, "Finetune/Loss (Raw)": 0.9344249963760376, "Finetune/Step": 2732, "Finetune/Step Time": 4.9398106057196856} +{"Finetune/Learning Rate": 9.6480996670826e-06, "Finetune/Loss": 0.8548306822776794, "Finetune/Loss (Raw)": 0.8909866809844971, "Finetune/Step": 2733, "Finetune/Step Time": 4.950678223744035} +{"Finetune/Learning Rate": 9.64187398349061e-06, "Finetune/Loss": 0.8557488918304443, "Finetune/Loss (Raw)": 0.9516640901565552, "Finetune/Step": 2734, "Finetune/Step Time": 4.9270640183240175} +{"Finetune/Learning Rate": 9.635648438880334e-06, "Finetune/Loss": 0.855853796005249, "Finetune/Loss (Raw)": 0.9629897475242615, "Finetune/Step": 2735, "Finetune/Step Time": 4.903687033802271} +{"Finetune/Learning Rate": 9.629423035667788e-06, "Finetune/Loss": 0.8519819974899292, "Finetune/Loss (Raw)": 0.458852618932724, "Finetune/Step": 2736, "Finetune/Step Time": 4.937787739560008} +{"Finetune/Learning Rate": 9.623197776268926e-06, "Finetune/Loss": 0.8495076894760132, "Finetune/Loss (Raw)": 0.6126678586006165, "Finetune/Step": 2737, "Finetune/Step Time": 4.942673359066248} +{"Finetune/Learning Rate": 9.616972663099648e-06, "Finetune/Loss": 0.849868655204773, "Finetune/Loss (Raw)": 0.9169014692306519, "Finetune/Step": 2738, "Finetune/Step Time": 4.926896082237363} +{"Finetune/Learning Rate": 9.610747698575801e-06, "Finetune/Loss": 0.8516020774841309, "Finetune/Loss (Raw)": 0.9349759221076965, "Finetune/Step": 2739, "Finetune/Step Time": 4.923802813515067} +{"Finetune/Learning Rate": 9.60452288511318e-06, "Finetune/Loss": 0.8512306213378906, "Finetune/Loss (Raw)": 0.8377634286880493, "Finetune/Step": 2740, "Finetune/Step Time": 4.932639507576823} +{"Finetune/Learning Rate": 9.598298225127499e-06, "Finetune/Loss": 0.8510681390762329, "Finetune/Loss (Raw)": 0.9976696968078613, "Finetune/Step": 2741, "Finetune/Step Time": 4.933254646137357} +{"Finetune/Learning Rate": 9.592073721034435e-06, "Finetune/Loss": 0.851540207862854, "Finetune/Loss (Raw)": 0.904254138469696, "Finetune/Step": 2742, "Finetune/Step Time": 4.938333120197058} +{"Finetune/Learning Rate": 9.585849375249595e-06, "Finetune/Loss": 0.8515408039093018, "Finetune/Loss (Raw)": 0.9592646956443787, "Finetune/Step": 2743, "Finetune/Step Time": 4.9302730448544025} +{"Finetune/Learning Rate": 9.57962519018853e-06, "Finetune/Loss": 0.8513443470001221, "Finetune/Loss (Raw)": 0.9266053438186646, "Finetune/Step": 2744, "Finetune/Step Time": 4.926928531378508} +{"Finetune/Learning Rate": 9.573401168266722e-06, "Finetune/Loss": 0.8559603095054626, "Finetune/Loss (Raw)": 0.9860166907310486, "Finetune/Step": 2745, "Finetune/Step Time": 4.887120615690947} +{"Finetune/Learning Rate": 9.567177311899589e-06, "Finetune/Loss": 0.8567303419113159, "Finetune/Loss (Raw)": 0.8988936543464661, "Finetune/Step": 2746, "Finetune/Step Time": 4.893763434141874} +{"Finetune/Learning Rate": 9.560953623502493e-06, "Finetune/Loss": 0.8560651540756226, "Finetune/Loss (Raw)": 0.8778566122055054, "Finetune/Step": 2747, "Finetune/Step Time": 4.910338085144758} +{"Finetune/Learning Rate": 9.554730105490722e-06, "Finetune/Loss": 0.8575267195701599, "Finetune/Loss (Raw)": 1.015669822692871, "Finetune/Step": 2748, "Finetune/Step Time": 4.9125525914132595} +{"Finetune/Learning Rate": 9.548506760279506e-06, "Finetune/Loss": 0.8580366969108582, "Finetune/Loss (Raw)": 0.942908525466919, "Finetune/Step": 2749, "Finetune/Step Time": 4.903079438954592} +{"Finetune/Learning Rate": 9.542283590284003e-06, "Finetune/Loss": 0.8573594093322754, "Finetune/Loss (Raw)": 0.8546912670135498, "Finetune/Step": 2750, "Finetune/Step Time": 4.907181611284614} +{"Finetune/Learning Rate": 9.5360605979193e-06, "Finetune/Loss": 0.8575989007949829, "Finetune/Loss (Raw)": 0.9319853186607361, "Finetune/Step": 2751, "Finetune/Step Time": 4.896635422483087} +{"Finetune/Learning Rate": 9.52983778560042e-06, "Finetune/Loss": 0.8579025864601135, "Finetune/Loss (Raw)": 0.945277750492096, "Finetune/Step": 2752, "Finetune/Step Time": 4.896053444594145} +{"Finetune/Learning Rate": 9.523615155742317e-06, "Finetune/Loss": 0.8596314191818237, "Finetune/Loss (Raw)": 0.9860109686851501, "Finetune/Step": 2753, "Finetune/Step Time": 4.881889943033457} +{"Finetune/Learning Rate": 9.517392710759872e-06, "Finetune/Loss": 0.8645610213279724, "Finetune/Loss (Raw)": 0.9410416483879089, "Finetune/Step": 2754, "Finetune/Step Time": 4.838146636262536} +{"Finetune/Learning Rate": 9.511170453067899e-06, "Finetune/Loss": 0.8604015111923218, "Finetune/Loss (Raw)": 0.3566739857196808, "Finetune/Step": 2755, "Finetune/Step Time": 4.87726167216897} +{"Finetune/Learning Rate": 9.504948385081121e-06, "Finetune/Loss": 0.8601728677749634, "Finetune/Loss (Raw)": 0.9590224623680115, "Finetune/Step": 2756, "Finetune/Step Time": 4.8990825936198235} +{"Finetune/Learning Rate": 9.498726509214216e-06, "Finetune/Loss": 0.8612563014030457, "Finetune/Loss (Raw)": 0.7767727375030518, "Finetune/Step": 2757, "Finetune/Step Time": 4.919643519446254} +{"Finetune/Learning Rate": 9.492504827881761e-06, "Finetune/Loss": 0.8623479604721069, "Finetune/Loss (Raw)": 0.9468278288841248, "Finetune/Step": 2758, "Finetune/Step Time": 4.91978744789958} +{"Finetune/Learning Rate": 9.486283343498277e-06, "Finetune/Loss": 0.8607690334320068, "Finetune/Loss (Raw)": 0.8330762386322021, "Finetune/Step": 2759, "Finetune/Step Time": 4.920706598088145} +{"Finetune/Learning Rate": 9.480062058478201e-06, "Finetune/Loss": 0.8603149652481079, "Finetune/Loss (Raw)": 0.7416940927505493, "Finetune/Step": 2760, "Finetune/Step Time": 4.920381881296635} +{"Finetune/Learning Rate": 9.473840975235883e-06, "Finetune/Loss": 0.8562228679656982, "Finetune/Loss (Raw)": 0.42002320289611816, "Finetune/Step": 2761, "Finetune/Step Time": 4.9564135149121284} +{"Finetune/Learning Rate": 9.467620096185616e-06, "Finetune/Loss": 0.8553699254989624, "Finetune/Loss (Raw)": 0.8411111235618591, "Finetune/Step": 2762, "Finetune/Step Time": 4.965930735692382} +{"Finetune/Learning Rate": 9.46139942374159e-06, "Finetune/Loss": 0.8550823926925659, "Finetune/Loss (Raw)": 0.8880638480186462, "Finetune/Step": 2763, "Finetune/Step Time": 4.968821434304118} +{"Finetune/Learning Rate": 9.455178960317936e-06, "Finetune/Loss": 0.8539776802062988, "Finetune/Loss (Raw)": 0.8706125020980835, "Finetune/Step": 2764, "Finetune/Step Time": 4.969444150105119} +{"Finetune/Learning Rate": 9.448958708328694e-06, "Finetune/Loss": 0.8502546548843384, "Finetune/Loss (Raw)": 0.3533449172973633, "Finetune/Step": 2765, "Finetune/Step Time": 4.998749420046806} +{"Finetune/Learning Rate": 9.442738670187814e-06, "Finetune/Loss": 0.8493829369544983, "Finetune/Loss (Raw)": 0.8654487729072571, "Finetune/Step": 2766, "Finetune/Step Time": 4.983745604753494} +{"Finetune/Learning Rate": 9.436518848309178e-06, "Finetune/Loss": 0.8507051467895508, "Finetune/Loss (Raw)": 0.9924708604812622, "Finetune/Step": 2767, "Finetune/Step Time": 4.997184982523322} +{"Finetune/Learning Rate": 9.430299245106574e-06, "Finetune/Loss": 0.8554374575614929, "Finetune/Loss (Raw)": 0.9540711641311646, "Finetune/Step": 2768, "Finetune/Step Time": 4.963030634447932} +{"Finetune/Learning Rate": 9.424079862993712e-06, "Finetune/Loss": 0.8556368947029114, "Finetune/Loss (Raw)": 0.9294862747192383, "Finetune/Step": 2769, "Finetune/Step Time": 4.971695803105831} +{"Finetune/Learning Rate": 9.417860704384214e-06, "Finetune/Loss": 0.8539812564849854, "Finetune/Loss (Raw)": 0.6776267290115356, "Finetune/Step": 2770, "Finetune/Step Time": 4.988214889541268} +{"Finetune/Learning Rate": 9.411641771691608e-06, "Finetune/Loss": 0.8552578687667847, "Finetune/Loss (Raw)": 0.9028317332267761, "Finetune/Step": 2771, "Finetune/Step Time": 4.981178307905793} +{"Finetune/Learning Rate": 9.405423067329344e-06, "Finetune/Loss": 0.8564351797103882, "Finetune/Loss (Raw)": 0.9149950742721558, "Finetune/Step": 2772, "Finetune/Step Time": 4.949942942708731} +{"Finetune/Learning Rate": 9.399204593710777e-06, "Finetune/Loss": 0.8565909266471863, "Finetune/Loss (Raw)": 0.9563135504722595, "Finetune/Step": 2773, "Finetune/Step Time": 4.942692464217544} +{"Finetune/Learning Rate": 9.392986353249183e-06, "Finetune/Loss": 0.8518223762512207, "Finetune/Loss (Raw)": 0.33058881759643555, "Finetune/Step": 2774, "Finetune/Step Time": 4.97814479097724} +{"Finetune/Learning Rate": 9.386768348357735e-06, "Finetune/Loss": 0.8525718450546265, "Finetune/Loss (Raw)": 0.920462429523468, "Finetune/Step": 2775, "Finetune/Step Time": 4.967845436185598} +{"Finetune/Learning Rate": 9.380550581449512e-06, "Finetune/Loss": 0.8513739109039307, "Finetune/Loss (Raw)": 0.8025732040405273, "Finetune/Step": 2776, "Finetune/Step Time": 4.960066264495254} +{"Finetune/Learning Rate": 9.37433305493752e-06, "Finetune/Loss": 0.8513672351837158, "Finetune/Loss (Raw)": 0.8496475219726562, "Finetune/Step": 2777, "Finetune/Step Time": 4.946648636832833} +{"Finetune/Learning Rate": 9.368115771234651e-06, "Finetune/Loss": 0.852040708065033, "Finetune/Loss (Raw)": 0.8959674835205078, "Finetune/Step": 2778, "Finetune/Step Time": 4.947017701342702} +{"Finetune/Learning Rate": 9.361898732753715e-06, "Finetune/Loss": 0.8519967794418335, "Finetune/Loss (Raw)": 0.8427348136901855, "Finetune/Step": 2779, "Finetune/Step Time": 4.952911596745253} +{"Finetune/Learning Rate": 9.355681941907428e-06, "Finetune/Loss": 0.8510216474533081, "Finetune/Loss (Raw)": 0.8893497586250305, "Finetune/Step": 2780, "Finetune/Step Time": 4.951487734913826} +{"Finetune/Learning Rate": 9.349465401108393e-06, "Finetune/Loss": 0.8518887758255005, "Finetune/Loss (Raw)": 0.9986838102340698, "Finetune/Step": 2781, "Finetune/Step Time": 4.935704713687301} +{"Finetune/Learning Rate": 9.34324911276914e-06, "Finetune/Loss": 0.8559830784797668, "Finetune/Loss (Raw)": 0.865766704082489, "Finetune/Step": 2782, "Finetune/Step Time": 4.902702920138836} +{"Finetune/Learning Rate": 9.337033079302082e-06, "Finetune/Loss": 0.8569158911705017, "Finetune/Loss (Raw)": 1.0407123565673828, "Finetune/Step": 2783, "Finetune/Step Time": 4.9067413825541735} +{"Finetune/Learning Rate": 9.330817303119545e-06, "Finetune/Loss": 0.8554052114486694, "Finetune/Loss (Raw)": 0.7046905159950256, "Finetune/Step": 2784, "Finetune/Step Time": 4.919559005647898} +{"Finetune/Learning Rate": 9.324601786633747e-06, "Finetune/Loss": 0.8566052913665771, "Finetune/Loss (Raw)": 0.8679405450820923, "Finetune/Step": 2785, "Finetune/Step Time": 4.9080880377441645} +{"Finetune/Learning Rate": 9.318386532256807e-06, "Finetune/Loss": 0.8586859703063965, "Finetune/Loss (Raw)": 0.9664744138717651, "Finetune/Step": 2786, "Finetune/Step Time": 4.885075436905026} +{"Finetune/Learning Rate": 9.312171542400749e-06, "Finetune/Loss": 0.8573267459869385, "Finetune/Loss (Raw)": 0.8336775302886963, "Finetune/Step": 2787, "Finetune/Step Time": 4.885430816560984} +{"Finetune/Learning Rate": 9.305956819477485e-06, "Finetune/Loss": 0.8573031425476074, "Finetune/Loss (Raw)": 0.9278297424316406, "Finetune/Step": 2788, "Finetune/Step Time": 4.895635211840272} +{"Finetune/Learning Rate": 9.299742365898831e-06, "Finetune/Loss": 0.855445384979248, "Finetune/Loss (Raw)": 0.862017810344696, "Finetune/Step": 2789, "Finetune/Step Time": 4.908846143633127} +{"Finetune/Learning Rate": 9.293528184076495e-06, "Finetune/Loss": 0.8546533584594727, "Finetune/Loss (Raw)": 0.911386251449585, "Finetune/Step": 2790, "Finetune/Step Time": 4.911887161433697} +{"Finetune/Learning Rate": 9.287314276422074e-06, "Finetune/Loss": 0.852786660194397, "Finetune/Loss (Raw)": 0.7386887073516846, "Finetune/Step": 2791, "Finetune/Step Time": 4.922016028314829} +{"Finetune/Learning Rate": 9.28110064534707e-06, "Finetune/Loss": 0.8570799827575684, "Finetune/Loss (Raw)": 0.9885362386703491, "Finetune/Step": 2792, "Finetune/Step Time": 4.896691324189305} +{"Finetune/Learning Rate": 9.274887293262869e-06, "Finetune/Loss": 0.858738899230957, "Finetune/Loss (Raw)": 0.9910984635353088, "Finetune/Step": 2793, "Finetune/Step Time": 4.894693415611982} +{"Finetune/Learning Rate": 9.268674222580757e-06, "Finetune/Loss": 0.8576374053955078, "Finetune/Loss (Raw)": 0.8266989588737488, "Finetune/Step": 2794, "Finetune/Step Time": 4.8973681796342134} +{"Finetune/Learning Rate": 9.262461435711898e-06, "Finetune/Loss": 0.8584917187690735, "Finetune/Loss (Raw)": 0.9043999910354614, "Finetune/Step": 2795, "Finetune/Step Time": 4.908250192180276} +{"Finetune/Learning Rate": 9.256248935067356e-06, "Finetune/Loss": 0.858589768409729, "Finetune/Loss (Raw)": 0.8652999401092529, "Finetune/Step": 2796, "Finetune/Step Time": 4.925049370154738} +{"Finetune/Learning Rate": 9.250036723058083e-06, "Finetune/Loss": 0.8567181825637817, "Finetune/Loss (Raw)": 0.7606146931648254, "Finetune/Step": 2797, "Finetune/Step Time": 4.938175685703754} +{"Finetune/Learning Rate": 9.243824802094914e-06, "Finetune/Loss": 0.8525093793869019, "Finetune/Loss (Raw)": 0.3801671862602234, "Finetune/Step": 2798, "Finetune/Step Time": 4.9779535830020905} +{"Finetune/Learning Rate": 9.237613174588583e-06, "Finetune/Loss": 0.8523125648498535, "Finetune/Loss (Raw)": 0.7448813319206238, "Finetune/Step": 2799, "Finetune/Step Time": 4.9751234613358974} +{"Finetune/Learning Rate": 9.231401842949688e-06, "Finetune/Loss": 0.8527435660362244, "Finetune/Loss (Raw)": 0.7353798747062683, "Finetune/Step": 2800, "Finetune/Step Time": 4.974679155275226} +{"Finetune/Learning Rate": 9.225190809588731e-06, "Finetune/Loss": 0.8535853028297424, "Finetune/Loss (Raw)": 0.9741177558898926, "Finetune/Step": 2801, "Finetune/Step Time": 4.989223051816225} +{"Finetune/Learning Rate": 9.218980076916099e-06, "Finetune/Loss": 0.8546603918075562, "Finetune/Loss (Raw)": 0.9775117039680481, "Finetune/Step": 2802, "Finetune/Step Time": 4.98842846788466} +{"Finetune/Learning Rate": 9.212769647342046e-06, "Finetune/Loss": 0.8550655841827393, "Finetune/Loss (Raw)": 0.9148943424224854, "Finetune/Step": 2803, "Finetune/Step Time": 4.982262518256903} +{"Finetune/Learning Rate": 9.206559523276731e-06, "Finetune/Loss": 0.8557037711143494, "Finetune/Loss (Raw)": 0.9179099798202515, "Finetune/Step": 2804, "Finetune/Step Time": 4.976931802928448} +{"Finetune/Learning Rate": 9.200349707130168e-06, "Finetune/Loss": 0.8560492992401123, "Finetune/Loss (Raw)": 1.0242486000061035, "Finetune/Step": 2805, "Finetune/Step Time": 4.975839259102941} +{"Finetune/Learning Rate": 9.194140201312274e-06, "Finetune/Loss": 0.8557916879653931, "Finetune/Loss (Raw)": 0.8402016162872314, "Finetune/Step": 2806, "Finetune/Step Time": 4.982615699991584} +{"Finetune/Learning Rate": 9.187931008232832e-06, "Finetune/Loss": 0.8544085025787354, "Finetune/Loss (Raw)": 0.8397898077964783, "Finetune/Step": 2807, "Finetune/Step Time": 4.980933453887701} +{"Finetune/Learning Rate": 9.181722130301512e-06, "Finetune/Loss": 0.8531941771507263, "Finetune/Loss (Raw)": 0.7647035717964172, "Finetune/Step": 2808, "Finetune/Step Time": 4.991004779934883} +{"Finetune/Learning Rate": 9.175513569927865e-06, "Finetune/Loss": 0.8532697558403015, "Finetune/Loss (Raw)": 0.3516761064529419, "Finetune/Step": 2809, "Finetune/Step Time": 4.991225320845842} +{"Finetune/Learning Rate": 9.169305329521296e-06, "Finetune/Loss": 0.8522361516952515, "Finetune/Loss (Raw)": 0.7626375555992126, "Finetune/Step": 2810, "Finetune/Step Time": 4.989339582622051} +{"Finetune/Learning Rate": 9.163097411491115e-06, "Finetune/Loss": 0.8523116111755371, "Finetune/Loss (Raw)": 0.9171122312545776, "Finetune/Step": 2811, "Finetune/Step Time": 4.993271427229047} +{"Finetune/Learning Rate": 9.156889818246485e-06, "Finetune/Loss": 0.8522509336471558, "Finetune/Loss (Raw)": 0.8718485236167908, "Finetune/Step": 2812, "Finetune/Step Time": 4.992582494392991} +{"Finetune/Learning Rate": 9.150682552196461e-06, "Finetune/Loss": 0.8529508113861084, "Finetune/Loss (Raw)": 0.9952490925788879, "Finetune/Step": 2813, "Finetune/Step Time": 4.991279035806656} +{"Finetune/Learning Rate": 9.144475615749957e-06, "Finetune/Loss": 0.8495258688926697, "Finetune/Loss (Raw)": 0.352794885635376, "Finetune/Step": 2814, "Finetune/Step Time": 5.026149930432439} +{"Finetune/Learning Rate": 9.138269011315761e-06, "Finetune/Loss": 0.8505812883377075, "Finetune/Loss (Raw)": 1.0230482816696167, "Finetune/Step": 2815, "Finetune/Step Time": 5.028287066146731} +{"Finetune/Learning Rate": 9.13206274130254e-06, "Finetune/Loss": 0.8506444096565247, "Finetune/Loss (Raw)": 0.9351761341094971, "Finetune/Step": 2816, "Finetune/Step Time": 5.015562130138278} +{"Finetune/Learning Rate": 9.125856808118822e-06, "Finetune/Loss": 0.8493830561637878, "Finetune/Loss (Raw)": 0.8573302626609802, "Finetune/Step": 2817, "Finetune/Step Time": 5.0222680028527975} +{"Finetune/Learning Rate": 9.119651214173012e-06, "Finetune/Loss": 0.8530296087265015, "Finetune/Loss (Raw)": 0.8369213938713074, "Finetune/Step": 2818, "Finetune/Step Time": 4.986339332535863} +{"Finetune/Learning Rate": 9.11344596187338e-06, "Finetune/Loss": 0.8529332876205444, "Finetune/Loss (Raw)": 1.0386534929275513, "Finetune/Step": 2819, "Finetune/Step Time": 4.989802440628409} +{"Finetune/Learning Rate": 9.107241053628058e-06, "Finetune/Loss": 0.8541461229324341, "Finetune/Loss (Raw)": 1.0051593780517578, "Finetune/Step": 2820, "Finetune/Step Time": 4.989673897624016} +{"Finetune/Learning Rate": 9.101036491845056e-06, "Finetune/Loss": 0.8523054122924805, "Finetune/Loss (Raw)": 0.7196688055992126, "Finetune/Step": 2821, "Finetune/Step Time": 5.013503227382898} +{"Finetune/Learning Rate": 9.094832278932238e-06, "Finetune/Loss": 0.85238116979599, "Finetune/Loss (Raw)": 0.9074732661247253, "Finetune/Step": 2822, "Finetune/Step Time": 5.003777798265219} +{"Finetune/Learning Rate": 9.088628417297346e-06, "Finetune/Loss": 0.8488963842391968, "Finetune/Loss (Raw)": 0.40216025710105896, "Finetune/Step": 2823, "Finetune/Step Time": 5.033356841653585} +{"Finetune/Learning Rate": 9.082424909347969e-06, "Finetune/Loss": 0.8476680517196655, "Finetune/Loss (Raw)": 0.8038535714149475, "Finetune/Step": 2824, "Finetune/Step Time": 5.0190028585493565} +{"Finetune/Learning Rate": 9.076221757491568e-06, "Finetune/Loss": 0.8471547961235046, "Finetune/Loss (Raw)": 0.8007368445396423, "Finetune/Step": 2825, "Finetune/Step Time": 5.017846802249551} +{"Finetune/Learning Rate": 9.070018964135471e-06, "Finetune/Loss": 0.8468334674835205, "Finetune/Loss (Raw)": 0.773925244808197, "Finetune/Step": 2826, "Finetune/Step Time": 5.025868387892842} +{"Finetune/Learning Rate": 9.063816531686855e-06, "Finetune/Loss": 0.8463975191116333, "Finetune/Loss (Raw)": 0.9435649514198303, "Finetune/Step": 2827, "Finetune/Step Time": 5.013795921579003} +{"Finetune/Learning Rate": 9.057614462552769e-06, "Finetune/Loss": 0.8478521108627319, "Finetune/Loss (Raw)": 1.049497365951538, "Finetune/Step": 2828, "Finetune/Step Time": 5.013378964737058} +{"Finetune/Learning Rate": 9.05141275914011e-06, "Finetune/Loss": 0.848246157169342, "Finetune/Loss (Raw)": 0.9259822964668274, "Finetune/Step": 2829, "Finetune/Step Time": 4.995844041928649} +{"Finetune/Learning Rate": 9.045211423855637e-06, "Finetune/Loss": 0.8486038446426392, "Finetune/Loss (Raw)": 0.9654616713523865, "Finetune/Step": 2830, "Finetune/Step Time": 4.987511079758406} +{"Finetune/Learning Rate": 9.039010459105974e-06, "Finetune/Loss": 0.8503973484039307, "Finetune/Loss (Raw)": 1.0557098388671875, "Finetune/Step": 2831, "Finetune/Step Time": 4.970799943432212} +{"Finetune/Learning Rate": 9.032809867297586e-06, "Finetune/Loss": 0.8519659042358398, "Finetune/Loss (Raw)": 0.9961616396903992, "Finetune/Step": 2832, "Finetune/Step Time": 4.96604298427701} +{"Finetune/Learning Rate": 9.02660965083681e-06, "Finetune/Loss": 0.8521670699119568, "Finetune/Loss (Raw)": 0.8872204422950745, "Finetune/Step": 2833, "Finetune/Step Time": 4.9725606590509415} +{"Finetune/Learning Rate": 9.020409812129824e-06, "Finetune/Loss": 0.8473222851753235, "Finetune/Loss (Raw)": 0.3557688295841217, "Finetune/Step": 2834, "Finetune/Step Time": 5.0012190993875265} +{"Finetune/Learning Rate": 9.014210353582662e-06, "Finetune/Loss": 0.8468121290206909, "Finetune/Loss (Raw)": 0.9320353269577026, "Finetune/Step": 2835, "Finetune/Step Time": 5.004086056724191} +{"Finetune/Learning Rate": 9.008011277601217e-06, "Finetune/Loss": 0.8447084426879883, "Finetune/Loss (Raw)": 0.7377008199691772, "Finetune/Step": 2836, "Finetune/Step Time": 4.996105635538697} +{"Finetune/Learning Rate": 9.001812586591228e-06, "Finetune/Loss": 0.8454755544662476, "Finetune/Loss (Raw)": 0.9611932635307312, "Finetune/Step": 2837, "Finetune/Step Time": 4.984721373766661} +{"Finetune/Learning Rate": 8.995614282958289e-06, "Finetune/Loss": 0.8490201234817505, "Finetune/Loss (Raw)": 0.8338989615440369, "Finetune/Step": 2838, "Finetune/Step Time": 4.944209584966302} +{"Finetune/Learning Rate": 8.989416369107837e-06, "Finetune/Loss": 0.8480475544929504, "Finetune/Loss (Raw)": 0.8316827416419983, "Finetune/Step": 2839, "Finetune/Step Time": 4.949497394263744} +{"Finetune/Learning Rate": 8.983218847445157e-06, "Finetune/Loss": 0.8481248021125793, "Finetune/Loss (Raw)": 0.9350800514221191, "Finetune/Step": 2840, "Finetune/Step Time": 4.933249892666936} +{"Finetune/Learning Rate": 8.977021720375395e-06, "Finetune/Loss": 0.8496372103691101, "Finetune/Loss (Raw)": 1.0714960098266602, "Finetune/Step": 2841, "Finetune/Step Time": 4.927304537966847} +{"Finetune/Learning Rate": 8.970824990303532e-06, "Finetune/Loss": 0.8492652177810669, "Finetune/Loss (Raw)": 0.9023588299751282, "Finetune/Step": 2842, "Finetune/Step Time": 4.9296581372618675} +{"Finetune/Learning Rate": 8.9646286596344e-06, "Finetune/Loss": 0.8486984968185425, "Finetune/Loss (Raw)": 0.7544201612472534, "Finetune/Step": 2843, "Finetune/Step Time": 4.934894947335124} +{"Finetune/Learning Rate": 8.958432730772667e-06, "Finetune/Loss": 0.8490704894065857, "Finetune/Loss (Raw)": 0.9039022326469421, "Finetune/Step": 2844, "Finetune/Step Time": 4.93551667034626} +{"Finetune/Learning Rate": 8.952237206122855e-06, "Finetune/Loss": 0.8495258092880249, "Finetune/Loss (Raw)": 0.9560057520866394, "Finetune/Step": 2845, "Finetune/Step Time": 4.9467932395637035} +{"Finetune/Learning Rate": 8.946042088089331e-06, "Finetune/Loss": 0.8494139313697815, "Finetune/Loss (Raw)": 0.8550710082054138, "Finetune/Step": 2846, "Finetune/Step Time": 4.945060722529888} +{"Finetune/Learning Rate": 8.939847379076293e-06, "Finetune/Loss": 0.8473889231681824, "Finetune/Loss (Raw)": 0.6747525334358215, "Finetune/Step": 2847, "Finetune/Step Time": 4.9572686944156885} +{"Finetune/Learning Rate": 8.933653081487794e-06, "Finetune/Loss": 0.847602367401123, "Finetune/Loss (Raw)": 0.9682566523551941, "Finetune/Step": 2848, "Finetune/Step Time": 4.964222082868218} +{"Finetune/Learning Rate": 8.927459197727712e-06, "Finetune/Loss": 0.8477504253387451, "Finetune/Loss (Raw)": 0.8675328493118286, "Finetune/Step": 2849, "Finetune/Step Time": 4.960230942815542} +{"Finetune/Learning Rate": 8.921265730199775e-06, "Finetune/Loss": 0.8476539254188538, "Finetune/Loss (Raw)": 0.8746272325515747, "Finetune/Step": 2850, "Finetune/Step Time": 4.955691611394286} +{"Finetune/Learning Rate": 8.91507268130755e-06, "Finetune/Loss": 0.848278820514679, "Finetune/Loss (Raw)": 0.9330387115478516, "Finetune/Step": 2851, "Finetune/Step Time": 4.952259270474315} +{"Finetune/Learning Rate": 8.908880053454432e-06, "Finetune/Loss": 0.8487653732299805, "Finetune/Loss (Raw)": 0.9826425909996033, "Finetune/Step": 2852, "Finetune/Step Time": 4.952718801796436} +{"Finetune/Learning Rate": 8.90268784904367e-06, "Finetune/Loss": 0.8485063314437866, "Finetune/Loss (Raw)": 0.875470757484436, "Finetune/Step": 2853, "Finetune/Step Time": 4.9528209790587425} +{"Finetune/Learning Rate": 8.896496070478326e-06, "Finetune/Loss": 0.8487665057182312, "Finetune/Loss (Raw)": 0.9460931420326233, "Finetune/Step": 2854, "Finetune/Step Time": 4.9455487839877605} +{"Finetune/Learning Rate": 8.890304720161311e-06, "Finetune/Loss": 0.8492023944854736, "Finetune/Loss (Raw)": 0.9412270188331604, "Finetune/Step": 2855, "Finetune/Step Time": 4.950176769867539} +{"Finetune/Learning Rate": 8.884113800495372e-06, "Finetune/Loss": 0.8504541516304016, "Finetune/Loss (Raw)": 0.9420521259307861, "Finetune/Step": 2856, "Finetune/Step Time": 4.964042140170932} +{"Finetune/Learning Rate": 8.87792331388308e-06, "Finetune/Loss": 0.8548340201377869, "Finetune/Loss (Raw)": 0.9815815091133118, "Finetune/Step": 2857, "Finetune/Step Time": 4.930773204192519} +{"Finetune/Learning Rate": 8.871733262726846e-06, "Finetune/Loss": 0.8550986647605896, "Finetune/Loss (Raw)": 0.8073768615722656, "Finetune/Step": 2858, "Finetune/Step Time": 4.924149136990309} +{"Finetune/Learning Rate": 8.865543649428903e-06, "Finetune/Loss": 0.8551384210586548, "Finetune/Loss (Raw)": 0.903938353061676, "Finetune/Step": 2859, "Finetune/Step Time": 4.925655696541071} +{"Finetune/Learning Rate": 8.859354476391319e-06, "Finetune/Loss": 0.8540186882019043, "Finetune/Loss (Raw)": 0.7911017537117004, "Finetune/Step": 2860, "Finetune/Step Time": 4.936171660199761} +{"Finetune/Learning Rate": 8.853165746015997e-06, "Finetune/Loss": 0.8550851941108704, "Finetune/Loss (Raw)": 1.0274994373321533, "Finetune/Step": 2861, "Finetune/Step Time": 4.925404749810696} +{"Finetune/Learning Rate": 8.846977460704656e-06, "Finetune/Loss": 0.8540413975715637, "Finetune/Loss (Raw)": 0.8180643916130066, "Finetune/Step": 2862, "Finetune/Step Time": 4.941765921190381} +{"Finetune/Learning Rate": 8.840789622858856e-06, "Finetune/Loss": 0.8537184000015259, "Finetune/Loss (Raw)": 0.9216420650482178, "Finetune/Step": 2863, "Finetune/Step Time": 4.9391613230109215} +{"Finetune/Learning Rate": 8.834602234879968e-06, "Finetune/Loss": 0.8567768335342407, "Finetune/Loss (Raw)": 0.8503286838531494, "Finetune/Step": 2864, "Finetune/Step Time": 4.9132382199168205} +{"Finetune/Learning Rate": 8.8284152991692e-06, "Finetune/Loss": 0.8597500920295715, "Finetune/Loss (Raw)": 0.9932523369789124, "Finetune/Step": 2865, "Finetune/Step Time": 4.905337391421199} +{"Finetune/Learning Rate": 8.822228818127584e-06, "Finetune/Loss": 0.8556626439094543, "Finetune/Loss (Raw)": 0.3937002718448639, "Finetune/Step": 2866, "Finetune/Step Time": 4.9368724171072245} +{"Finetune/Learning Rate": 8.81604279415597e-06, "Finetune/Loss": 0.8554220795631409, "Finetune/Loss (Raw)": 0.9041903018951416, "Finetune/Step": 2867, "Finetune/Step Time": 4.939459383487701} +{"Finetune/Learning Rate": 8.809857229655029e-06, "Finetune/Loss": 0.8568728566169739, "Finetune/Loss (Raw)": 1.0234593152999878, "Finetune/Step": 2868, "Finetune/Step Time": 4.934446299448609} +{"Finetune/Learning Rate": 8.803672127025264e-06, "Finetune/Loss": 0.8547217845916748, "Finetune/Loss (Raw)": 0.7223396301269531, "Finetune/Step": 2869, "Finetune/Step Time": 4.9707000236958265} +{"Finetune/Learning Rate": 8.797487488666986e-06, "Finetune/Loss": 0.8506177663803101, "Finetune/Loss (Raw)": 0.3789336085319519, "Finetune/Step": 2870, "Finetune/Step Time": 4.999505698680878} +{"Finetune/Learning Rate": 8.791303316980342e-06, "Finetune/Loss": 0.850999116897583, "Finetune/Loss (Raw)": 1.0080814361572266, "Finetune/Step": 2871, "Finetune/Step Time": 4.997859805822372} +{"Finetune/Learning Rate": 8.785119614365282e-06, "Finetune/Loss": 0.8514130115509033, "Finetune/Loss (Raw)": 0.9795747399330139, "Finetune/Step": 2872, "Finetune/Step Time": 4.997995778918266} +{"Finetune/Learning Rate": 8.77893638322158e-06, "Finetune/Loss": 0.8506331443786621, "Finetune/Loss (Raw)": 0.8862005472183228, "Finetune/Step": 2873, "Finetune/Step Time": 4.999747747555375} +{"Finetune/Learning Rate": 8.772753625948828e-06, "Finetune/Loss": 0.8464215397834778, "Finetune/Loss (Raw)": 0.3598015606403351, "Finetune/Step": 2874, "Finetune/Step Time": 5.02152176015079} +{"Finetune/Learning Rate": 8.766571344946433e-06, "Finetune/Loss": 0.8474043011665344, "Finetune/Loss (Raw)": 1.0036526918411255, "Finetune/Step": 2875, "Finetune/Step Time": 5.000852318480611} +{"Finetune/Learning Rate": 8.760389542613619e-06, "Finetune/Loss": 0.8463270664215088, "Finetune/Loss (Raw)": 0.8777804374694824, "Finetune/Step": 2876, "Finetune/Step Time": 5.0092105735093355} +{"Finetune/Learning Rate": 8.754208221349427e-06, "Finetune/Loss": 0.8455197811126709, "Finetune/Loss (Raw)": 0.8395783305168152, "Finetune/Step": 2877, "Finetune/Step Time": 5.012236125767231} +{"Finetune/Learning Rate": 8.748027383552696e-06, "Finetune/Loss": 0.8460536599159241, "Finetune/Loss (Raw)": 0.9230314493179321, "Finetune/Step": 2878, "Finetune/Step Time": 5.017440881580114} +{"Finetune/Learning Rate": 8.741847031622096e-06, "Finetune/Loss": 0.8448971509933472, "Finetune/Loss (Raw)": 0.7839468717575073, "Finetune/Step": 2879, "Finetune/Step Time": 5.018394887447357} +{"Finetune/Learning Rate": 8.7356671679561e-06, "Finetune/Loss": 0.8452365398406982, "Finetune/Loss (Raw)": 0.9887247085571289, "Finetune/Step": 2880, "Finetune/Step Time": 5.0044848918914795} +{"Finetune/Learning Rate": 8.729487794952992e-06, "Finetune/Loss": 0.8451707363128662, "Finetune/Loss (Raw)": 0.9775920510292053, "Finetune/Step": 2881, "Finetune/Step Time": 5.009256545454264} +{"Finetune/Learning Rate": 8.72330891501087e-06, "Finetune/Loss": 0.8455350399017334, "Finetune/Loss (Raw)": 0.9876646399497986, "Finetune/Step": 2882, "Finetune/Step Time": 5.015486320480704} +{"Finetune/Learning Rate": 8.717130530527626e-06, "Finetune/Loss": 0.8506371974945068, "Finetune/Loss (Raw)": 1.0097578763961792, "Finetune/Step": 2883, "Finetune/Step Time": 4.982500396668911} +{"Finetune/Learning Rate": 8.710952643900983e-06, "Finetune/Loss": 0.8498003482818604, "Finetune/Loss (Raw)": 0.8518975973129272, "Finetune/Step": 2884, "Finetune/Step Time": 4.983901415020227} +{"Finetune/Learning Rate": 8.704775257528446e-06, "Finetune/Loss": 0.8510728478431702, "Finetune/Loss (Raw)": 0.9396579265594482, "Finetune/Step": 2885, "Finetune/Step Time": 4.953064355999231} +{"Finetune/Learning Rate": 8.69859837380735e-06, "Finetune/Loss": 0.8490093350410461, "Finetune/Loss (Raw)": 0.6826953887939453, "Finetune/Step": 2886, "Finetune/Step Time": 4.9765189085155725} +{"Finetune/Learning Rate": 8.692421995134819e-06, "Finetune/Loss": 0.8486552238464355, "Finetune/Loss (Raw)": 0.7877497673034668, "Finetune/Step": 2887, "Finetune/Step Time": 4.981469692662358} +{"Finetune/Learning Rate": 8.686246123907778e-06, "Finetune/Loss": 0.8510818481445312, "Finetune/Loss (Raw)": 1.0523016452789307, "Finetune/Step": 2888, "Finetune/Step Time": 4.9887157417833805} +{"Finetune/Learning Rate": 8.680070762522969e-06, "Finetune/Loss": 0.8503854274749756, "Finetune/Loss (Raw)": 0.3308804929256439, "Finetune/Step": 2889, "Finetune/Step Time": 4.987622320652008} +{"Finetune/Learning Rate": 8.673895913376928e-06, "Finetune/Loss": 0.8501313924789429, "Finetune/Loss (Raw)": 0.8085933327674866, "Finetune/Step": 2890, "Finetune/Step Time": 4.981803556904197} +{"Finetune/Learning Rate": 8.667721578865993e-06, "Finetune/Loss": 0.8501639366149902, "Finetune/Loss (Raw)": 0.8922312259674072, "Finetune/Step": 2891, "Finetune/Step Time": 4.981029883027077} +{"Finetune/Learning Rate": 8.661547761386307e-06, "Finetune/Loss": 0.8519055843353271, "Finetune/Loss (Raw)": 1.0935481786727905, "Finetune/Step": 2892, "Finetune/Step Time": 4.990490550175309} +{"Finetune/Learning Rate": 8.655374463333797e-06, "Finetune/Loss": 0.8560389280319214, "Finetune/Loss (Raw)": 0.882405698299408, "Finetune/Step": 2893, "Finetune/Step Time": 4.952195608988404} +{"Finetune/Learning Rate": 8.649201687104209e-06, "Finetune/Loss": 0.8571804761886597, "Finetune/Loss (Raw)": 1.0115717649459839, "Finetune/Step": 2894, "Finetune/Step Time": 4.951479749754071} +{"Finetune/Learning Rate": 8.643029435093072e-06, "Finetune/Loss": 0.8563543558120728, "Finetune/Loss (Raw)": 0.8867222666740417, "Finetune/Step": 2895, "Finetune/Step Time": 4.938162095844746} +{"Finetune/Learning Rate": 8.63685770969572e-06, "Finetune/Loss": 0.8562301397323608, "Finetune/Loss (Raw)": 0.9381796717643738, "Finetune/Step": 2896, "Finetune/Step Time": 4.940370488911867} +{"Finetune/Learning Rate": 8.630686513307282e-06, "Finetune/Loss": 0.8566218614578247, "Finetune/Loss (Raw)": 0.9796152710914612, "Finetune/Step": 2897, "Finetune/Step Time": 4.934464283287525} +{"Finetune/Learning Rate": 8.62451584832267e-06, "Finetune/Loss": 0.858471691608429, "Finetune/Loss (Raw)": 0.9144130945205688, "Finetune/Step": 2898, "Finetune/Step Time": 4.917091580107808} +{"Finetune/Learning Rate": 8.618345717136604e-06, "Finetune/Loss": 0.858149528503418, "Finetune/Loss (Raw)": 0.8615951538085938, "Finetune/Step": 2899, "Finetune/Step Time": 4.917575132101774} +{"Finetune/Learning Rate": 8.612176122143589e-06, "Finetune/Loss": 0.8571301698684692, "Finetune/Loss (Raw)": 0.7845202684402466, "Finetune/Step": 2900, "Finetune/Step Time": 4.920707771554589} +{"Finetune/Learning Rate": 8.606007065737928e-06, "Finetune/Loss": 0.8558914661407471, "Finetune/Loss (Raw)": 0.7977503538131714, "Finetune/Step": 2901, "Finetune/Step Time": 4.930190337821841} +{"Finetune/Learning Rate": 8.599838550313714e-06, "Finetune/Loss": 0.8605642318725586, "Finetune/Loss (Raw)": 0.9287042021751404, "Finetune/Step": 2902, "Finetune/Step Time": 4.9174696411937475} +{"Finetune/Learning Rate": 8.593670578264814e-06, "Finetune/Loss": 0.8608181476593018, "Finetune/Loss (Raw)": 0.9529604315757751, "Finetune/Step": 2903, "Finetune/Step Time": 4.920286534354091} +{"Finetune/Learning Rate": 8.587503151984911e-06, "Finetune/Loss": 0.8613017797470093, "Finetune/Loss (Raw)": 0.8644824028015137, "Finetune/Step": 2904, "Finetune/Step Time": 4.91873287409544} +{"Finetune/Learning Rate": 8.581336273867455e-06, "Finetune/Loss": 0.8616586327552795, "Finetune/Loss (Raw)": 0.8953259587287903, "Finetune/Step": 2905, "Finetune/Step Time": 4.920355444774032} +{"Finetune/Learning Rate": 8.575169946305699e-06, "Finetune/Loss": 0.861933708190918, "Finetune/Loss (Raw)": 0.9311733245849609, "Finetune/Step": 2906, "Finetune/Step Time": 4.921764779835939} +{"Finetune/Learning Rate": 8.569004171692667e-06, "Finetune/Loss": 0.8620805740356445, "Finetune/Loss (Raw)": 0.8615405559539795, "Finetune/Step": 2907, "Finetune/Step Time": 4.913853768259287} +{"Finetune/Learning Rate": 8.562838952421177e-06, "Finetune/Loss": 0.8624997138977051, "Finetune/Loss (Raw)": 0.9430007934570312, "Finetune/Step": 2908, "Finetune/Step Time": 4.916948696598411} +{"Finetune/Learning Rate": 8.556674290883833e-06, "Finetune/Loss": 0.8631445169448853, "Finetune/Loss (Raw)": 1.0812113285064697, "Finetune/Step": 2909, "Finetune/Step Time": 4.911444088444114} +{"Finetune/Learning Rate": 8.550510189473016e-06, "Finetune/Loss": 0.8640809059143066, "Finetune/Loss (Raw)": 0.9856345057487488, "Finetune/Step": 2910, "Finetune/Step Time": 4.911985328420997} +{"Finetune/Learning Rate": 8.544346650580899e-06, "Finetune/Loss": 0.8641469478607178, "Finetune/Loss (Raw)": 1.04915452003479, "Finetune/Step": 2911, "Finetune/Step Time": 4.897791961207986} +{"Finetune/Learning Rate": 8.538183676599426e-06, "Finetune/Loss": 0.8659971952438354, "Finetune/Loss (Raw)": 0.9415304660797119, "Finetune/Step": 2912, "Finetune/Step Time": 4.88964763097465} +{"Finetune/Learning Rate": 8.532021269920326e-06, "Finetune/Loss": 0.8664961457252502, "Finetune/Loss (Raw)": 0.9317970871925354, "Finetune/Step": 2913, "Finetune/Step Time": 4.882363324984908} +{"Finetune/Learning Rate": 8.525859432935114e-06, "Finetune/Loss": 0.8644167184829712, "Finetune/Loss (Raw)": 0.700313925743103, "Finetune/Step": 2914, "Finetune/Step Time": 4.9091233890503645} +{"Finetune/Learning Rate": 8.519698168035074e-06, "Finetune/Loss": 0.8636751174926758, "Finetune/Loss (Raw)": 0.7387474179267883, "Finetune/Step": 2915, "Finetune/Step Time": 4.921106310561299} +{"Finetune/Learning Rate": 8.513537477611278e-06, "Finetune/Loss": 0.8636731505393982, "Finetune/Loss (Raw)": 0.9275835156440735, "Finetune/Step": 2916, "Finetune/Step Time": 4.907847175374627} +{"Finetune/Learning Rate": 8.507377364054566e-06, "Finetune/Loss": 0.8645327687263489, "Finetune/Loss (Raw)": 0.9720449447631836, "Finetune/Step": 2917, "Finetune/Step Time": 4.891847409307957} +{"Finetune/Learning Rate": 8.501217829755554e-06, "Finetune/Loss": 0.8640210628509521, "Finetune/Loss (Raw)": 0.8458908200263977, "Finetune/Step": 2918, "Finetune/Step Time": 4.897466920316219} +{"Finetune/Learning Rate": 8.495058877104642e-06, "Finetune/Loss": 0.8653784990310669, "Finetune/Loss (Raw)": 0.9124369621276855, "Finetune/Step": 2919, "Finetune/Step Time": 4.875158937647939} +{"Finetune/Learning Rate": 8.488900508491999e-06, "Finetune/Loss": 0.8603521585464478, "Finetune/Loss (Raw)": 0.3451690077781677, "Finetune/Step": 2920, "Finetune/Step Time": 4.896894715726376} +{"Finetune/Learning Rate": 8.48274272630757e-06, "Finetune/Loss": 0.8596075773239136, "Finetune/Loss (Raw)": 0.895794153213501, "Finetune/Step": 2921, "Finetune/Step Time": 4.914464324712753} +{"Finetune/Learning Rate": 8.476585532941064e-06, "Finetune/Loss": 0.8598343133926392, "Finetune/Loss (Raw)": 0.8557177186012268, "Finetune/Step": 2922, "Finetune/Step Time": 4.907071413472295} +{"Finetune/Learning Rate": 8.470428930781969e-06, "Finetune/Loss": 0.8596398830413818, "Finetune/Loss (Raw)": 0.8795127868652344, "Finetune/Step": 2923, "Finetune/Step Time": 4.929424148052931} +{"Finetune/Learning Rate": 8.464272922219547e-06, "Finetune/Loss": 0.8606672286987305, "Finetune/Loss (Raw)": 0.9967983961105347, "Finetune/Step": 2924, "Finetune/Step Time": 4.912654569372535} +{"Finetune/Learning Rate": 8.458117509642817e-06, "Finetune/Loss": 0.8626108169555664, "Finetune/Loss (Raw)": 1.0093913078308105, "Finetune/Step": 2925, "Finetune/Step Time": 4.896106673404574} +{"Finetune/Learning Rate": 8.451962695440586e-06, "Finetune/Loss": 0.8665106892585754, "Finetune/Loss (Raw)": 0.8793580532073975, "Finetune/Step": 2926, "Finetune/Step Time": 4.899292483925819} +{"Finetune/Learning Rate": 8.445808482001405e-06, "Finetune/Loss": 0.86616051197052, "Finetune/Loss (Raw)": 0.7000565528869629, "Finetune/Step": 2927, "Finetune/Step Time": 4.90914998203516} +{"Finetune/Learning Rate": 8.439654871713608e-06, "Finetune/Loss": 0.8674343228340149, "Finetune/Loss (Raw)": 0.898430347442627, "Finetune/Step": 2928, "Finetune/Step Time": 4.913127766922116} +{"Finetune/Learning Rate": 8.433501866965294e-06, "Finetune/Loss": 0.8679688572883606, "Finetune/Loss (Raw)": 1.0425341129302979, "Finetune/Step": 2929, "Finetune/Step Time": 4.892588784918189} +{"Finetune/Learning Rate": 8.42734947014432e-06, "Finetune/Loss": 0.8679083585739136, "Finetune/Loss (Raw)": 0.9697746634483337, "Finetune/Step": 2930, "Finetune/Step Time": 4.884832223877311} +{"Finetune/Learning Rate": 8.42119768363832e-06, "Finetune/Loss": 0.8681026697158813, "Finetune/Loss (Raw)": 0.9397585988044739, "Finetune/Step": 2931, "Finetune/Step Time": 4.887699361890554} +{"Finetune/Learning Rate": 8.415046509834668e-06, "Finetune/Loss": 0.868240237236023, "Finetune/Loss (Raw)": 0.9355263113975525, "Finetune/Step": 2932, "Finetune/Step Time": 4.89658067189157} +{"Finetune/Learning Rate": 8.408895951120526e-06, "Finetune/Loss": 0.8673332333564758, "Finetune/Loss (Raw)": 0.9081461429595947, "Finetune/Step": 2933, "Finetune/Step Time": 4.90774361602962} +{"Finetune/Learning Rate": 8.402746009882798e-06, "Finetune/Loss": 0.8679739832878113, "Finetune/Loss (Raw)": 0.9222173690795898, "Finetune/Step": 2934, "Finetune/Step Time": 4.901265015825629} +{"Finetune/Learning Rate": 8.39659668850816e-06, "Finetune/Loss": 0.8684560060501099, "Finetune/Loss (Raw)": 0.9014859795570374, "Finetune/Step": 2935, "Finetune/Step Time": 4.906110787764192} +{"Finetune/Learning Rate": 8.39044798938305e-06, "Finetune/Loss": 0.8704595565795898, "Finetune/Loss (Raw)": 1.0211645364761353, "Finetune/Step": 2936, "Finetune/Step Time": 4.91247103177011} +{"Finetune/Learning Rate": 8.384299914893645e-06, "Finetune/Loss": 0.875956654548645, "Finetune/Loss (Raw)": 1.0553033351898193, "Finetune/Step": 2937, "Finetune/Step Time": 4.878039626404643} +{"Finetune/Learning Rate": 8.378152467425902e-06, "Finetune/Loss": 0.8773621320724487, "Finetune/Loss (Raw)": 0.9425405859947205, "Finetune/Step": 2938, "Finetune/Step Time": 4.879849350079894} +{"Finetune/Learning Rate": 8.37200564936552e-06, "Finetune/Loss": 0.8772256374359131, "Finetune/Loss (Raw)": 0.8996450304985046, "Finetune/Step": 2939, "Finetune/Step Time": 4.878470664843917} +{"Finetune/Learning Rate": 8.365859463097963e-06, "Finetune/Loss": 0.8760451674461365, "Finetune/Loss (Raw)": 0.7207396626472473, "Finetune/Step": 2940, "Finetune/Step Time": 4.880872407928109} +{"Finetune/Learning Rate": 8.359713911008447e-06, "Finetune/Loss": 0.8735802173614502, "Finetune/Loss (Raw)": 0.6797337532043457, "Finetune/Step": 2941, "Finetune/Step Time": 4.898961892351508} +{"Finetune/Learning Rate": 8.353568995481934e-06, "Finetune/Loss": 0.8788260221481323, "Finetune/Loss (Raw)": 1.024254560470581, "Finetune/Step": 2942, "Finetune/Step Time": 4.867266166955233} +{"Finetune/Learning Rate": 8.347424718903152e-06, "Finetune/Loss": 0.8782010078430176, "Finetune/Loss (Raw)": 0.9430590867996216, "Finetune/Step": 2943, "Finetune/Step Time": 4.872144013643265} +{"Finetune/Learning Rate": 8.341281083656571e-06, "Finetune/Loss": 0.8780292272567749, "Finetune/Loss (Raw)": 0.9131839275360107, "Finetune/Step": 2944, "Finetune/Step Time": 4.879856808111072} +{"Finetune/Learning Rate": 8.33513809212642e-06, "Finetune/Loss": 0.8777862787246704, "Finetune/Loss (Raw)": 0.8262274265289307, "Finetune/Step": 2945, "Finetune/Step Time": 4.888949990272522} +{"Finetune/Learning Rate": 8.328995746696674e-06, "Finetune/Loss": 0.8784072399139404, "Finetune/Loss (Raw)": 0.9164119362831116, "Finetune/Step": 2946, "Finetune/Step Time": 4.880230773240328} +{"Finetune/Learning Rate": 8.32285404975105e-06, "Finetune/Loss": 0.8771066069602966, "Finetune/Loss (Raw)": 0.8721673488616943, "Finetune/Step": 2947, "Finetune/Step Time": 4.896624548360705} +{"Finetune/Learning Rate": 8.316713003673028e-06, "Finetune/Loss": 0.8768047094345093, "Finetune/Loss (Raw)": 0.9665139317512512, "Finetune/Step": 2948, "Finetune/Step Time": 4.9068065546453} +{"Finetune/Learning Rate": 8.310572610845824e-06, "Finetune/Loss": 0.8785898685455322, "Finetune/Loss (Raw)": 0.9481690526008606, "Finetune/Step": 2949, "Finetune/Step Time": 4.8860835656523705} +{"Finetune/Learning Rate": 8.304432873652412e-06, "Finetune/Loss": 0.8785977959632874, "Finetune/Loss (Raw)": 0.9084881544113159, "Finetune/Step": 2950, "Finetune/Step Time": 4.885866992175579} +{"Finetune/Learning Rate": 8.298293794475495e-06, "Finetune/Loss": 0.877833366394043, "Finetune/Loss (Raw)": 0.3043186664581299, "Finetune/Step": 2951, "Finetune/Step Time": 4.8865076545625925} +{"Finetune/Learning Rate": 8.29215537569753e-06, "Finetune/Loss": 0.8790110349655151, "Finetune/Loss (Raw)": 0.9545942544937134, "Finetune/Step": 2952, "Finetune/Step Time": 4.889638833701611} +{"Finetune/Learning Rate": 8.286017619700725e-06, "Finetune/Loss": 0.8804512619972229, "Finetune/Loss (Raw)": 0.9850846529006958, "Finetune/Step": 2953, "Finetune/Step Time": 4.88883538171649} +{"Finetune/Learning Rate": 8.279880528867017e-06, "Finetune/Loss": 0.8806170225143433, "Finetune/Loss (Raw)": 0.7951440215110779, "Finetune/Step": 2954, "Finetune/Step Time": 4.892797755077481} +{"Finetune/Learning Rate": 8.273744105578097e-06, "Finetune/Loss": 0.8814812302589417, "Finetune/Loss (Raw)": 1.0541867017745972, "Finetune/Step": 2955, "Finetune/Step Time": 4.899379067122936} +{"Finetune/Learning Rate": 8.267608352215385e-06, "Finetune/Loss": 0.8806295394897461, "Finetune/Loss (Raw)": 0.9404794573783875, "Finetune/Step": 2956, "Finetune/Step Time": 4.896298211067915} +{"Finetune/Learning Rate": 8.261473271160046e-06, "Finetune/Loss": 0.8804470300674438, "Finetune/Loss (Raw)": 0.9026188850402832, "Finetune/Step": 2957, "Finetune/Step Time": 4.890421960502863} +{"Finetune/Learning Rate": 8.255338864792993e-06, "Finetune/Loss": 0.8800352811813354, "Finetune/Loss (Raw)": 0.9127618670463562, "Finetune/Step": 2958, "Finetune/Step Time": 4.893536739051342} +{"Finetune/Learning Rate": 8.249205135494862e-06, "Finetune/Loss": 0.8742766976356506, "Finetune/Loss (Raw)": 0.3186069130897522, "Finetune/Step": 2959, "Finetune/Step Time": 4.925969559699297} +{"Finetune/Learning Rate": 8.24307208564604e-06, "Finetune/Loss": 0.8735818862915039, "Finetune/Loss (Raw)": 0.9072198867797852, "Finetune/Step": 2960, "Finetune/Step Time": 4.923034997656941} +{"Finetune/Learning Rate": 8.236939717626639e-06, "Finetune/Loss": 0.8732161521911621, "Finetune/Loss (Raw)": 0.8404095768928528, "Finetune/Step": 2961, "Finetune/Step Time": 4.909330295398831} +{"Finetune/Learning Rate": 8.23080803381651e-06, "Finetune/Loss": 0.8788315057754517, "Finetune/Loss (Raw)": 1.0745322704315186, "Finetune/Step": 2962, "Finetune/Step Time": 4.879876269027591} +{"Finetune/Learning Rate": 8.224677036595245e-06, "Finetune/Loss": 0.8792020082473755, "Finetune/Loss (Raw)": 0.9794686436653137, "Finetune/Step": 2963, "Finetune/Step Time": 4.8903679382056} +{"Finetune/Learning Rate": 8.218546728342161e-06, "Finetune/Loss": 0.8811802864074707, "Finetune/Loss (Raw)": 0.9909117817878723, "Finetune/Step": 2964, "Finetune/Step Time": 4.893118923529983} +{"Finetune/Learning Rate": 8.212417111436315e-06, "Finetune/Loss": 0.8811620473861694, "Finetune/Loss (Raw)": 0.9588582515716553, "Finetune/Step": 2965, "Finetune/Step Time": 4.889180736616254} +{"Finetune/Learning Rate": 8.206288188256486e-06, "Finetune/Loss": 0.8810364007949829, "Finetune/Loss (Raw)": 0.8178234100341797, "Finetune/Step": 2966, "Finetune/Step Time": 4.897131506353617} +{"Finetune/Learning Rate": 8.20015996118119e-06, "Finetune/Loss": 0.8806335926055908, "Finetune/Loss (Raw)": 0.780121386051178, "Finetune/Step": 2967, "Finetune/Step Time": 4.908908095210791} +{"Finetune/Learning Rate": 8.194032432588677e-06, "Finetune/Loss": 0.8813252449035645, "Finetune/Loss (Raw)": 1.0236127376556396, "Finetune/Step": 2968, "Finetune/Step Time": 4.917715881019831} +{"Finetune/Learning Rate": 8.187905604856916e-06, "Finetune/Loss": 0.8797099590301514, "Finetune/Loss (Raw)": 0.8647328019142151, "Finetune/Step": 2969, "Finetune/Step Time": 4.9143123254179955} +{"Finetune/Learning Rate": 8.181779480363614e-06, "Finetune/Loss": 0.8797685503959656, "Finetune/Loss (Raw)": 0.9098645448684692, "Finetune/Step": 2970, "Finetune/Step Time": 4.926297212019563} +{"Finetune/Learning Rate": 8.175654061486197e-06, "Finetune/Loss": 0.8808732032775879, "Finetune/Loss (Raw)": 0.8958134651184082, "Finetune/Step": 2971, "Finetune/Step Time": 4.933747408911586} +{"Finetune/Learning Rate": 8.16952935060182e-06, "Finetune/Loss": 0.8810551166534424, "Finetune/Loss (Raw)": 0.927190363407135, "Finetune/Step": 2972, "Finetune/Step Time": 4.929603848606348} +{"Finetune/Learning Rate": 8.163405350087366e-06, "Finetune/Loss": 0.8796817660331726, "Finetune/Loss (Raw)": 0.7802123427391052, "Finetune/Step": 2973, "Finetune/Step Time": 4.93277576752007} +{"Finetune/Learning Rate": 8.157282062319437e-06, "Finetune/Loss": 0.8802873492240906, "Finetune/Loss (Raw)": 0.9325883388519287, "Finetune/Step": 2974, "Finetune/Step Time": 4.933583555743098} +{"Finetune/Learning Rate": 8.15115948967437e-06, "Finetune/Loss": 0.8780723810195923, "Finetune/Loss (Raw)": 0.39124035835266113, "Finetune/Step": 2975, "Finetune/Step Time": 4.957223825156689} +{"Finetune/Learning Rate": 8.145037634528205e-06, "Finetune/Loss": 0.8769621849060059, "Finetune/Loss (Raw)": 0.8261473774909973, "Finetune/Step": 2976, "Finetune/Step Time": 4.951316216960549} +{"Finetune/Learning Rate": 8.138916499256718e-06, "Finetune/Loss": 0.877377450466156, "Finetune/Loss (Raw)": 0.9206852912902832, "Finetune/Step": 2977, "Finetune/Step Time": 4.94881372526288} +{"Finetune/Learning Rate": 8.132796086235405e-06, "Finetune/Loss": 0.8782353401184082, "Finetune/Loss (Raw)": 0.9844311475753784, "Finetune/Step": 2978, "Finetune/Step Time": 4.948973439633846} +{"Finetune/Learning Rate": 8.126676397839475e-06, "Finetune/Loss": 0.8779215812683105, "Finetune/Loss (Raw)": 0.8928871154785156, "Finetune/Step": 2979, "Finetune/Step Time": 4.959618927910924} +{"Finetune/Learning Rate": 8.120557436443866e-06, "Finetune/Loss": 0.8768383264541626, "Finetune/Loss (Raw)": 0.8439767360687256, "Finetune/Step": 2980, "Finetune/Step Time": 4.963093891739845} +{"Finetune/Learning Rate": 8.11443920442322e-06, "Finetune/Loss": 0.8761355876922607, "Finetune/Loss (Raw)": 0.7855294346809387, "Finetune/Step": 2981, "Finetune/Step Time": 4.990156168118119} +{"Finetune/Learning Rate": 8.108321704151903e-06, "Finetune/Loss": 0.8751636743545532, "Finetune/Loss (Raw)": 0.8216897249221802, "Finetune/Step": 2982, "Finetune/Step Time": 4.983604319393635} +{"Finetune/Learning Rate": 8.102204938004004e-06, "Finetune/Loss": 0.875710129737854, "Finetune/Loss (Raw)": 1.0111689567565918, "Finetune/Step": 2983, "Finetune/Step Time": 4.974083652719855} +{"Finetune/Learning Rate": 8.096088908353316e-06, "Finetune/Loss": 0.8759894371032715, "Finetune/Loss (Raw)": 0.9778074026107788, "Finetune/Step": 2984, "Finetune/Step Time": 4.973642973229289} +{"Finetune/Learning Rate": 8.089973617573355e-06, "Finetune/Loss": 0.8751685619354248, "Finetune/Loss (Raw)": 0.8765103220939636, "Finetune/Step": 2985, "Finetune/Step Time": 4.9735877979546785} +{"Finetune/Learning Rate": 8.08385906803734e-06, "Finetune/Loss": 0.8714331984519958, "Finetune/Loss (Raw)": 0.32924970984458923, "Finetune/Step": 2986, "Finetune/Step Time": 5.01096604578197} +{"Finetune/Learning Rate": 8.07774526211821e-06, "Finetune/Loss": 0.8709337711334229, "Finetune/Loss (Raw)": 0.8400119543075562, "Finetune/Step": 2987, "Finetune/Step Time": 5.023948082700372} +{"Finetune/Learning Rate": 8.071632202188618e-06, "Finetune/Loss": 0.8721969127655029, "Finetune/Loss (Raw)": 0.9527815580368042, "Finetune/Step": 2988, "Finetune/Step Time": 5.020464796572924} +{"Finetune/Learning Rate": 8.065519890620922e-06, "Finetune/Loss": 0.8705844879150391, "Finetune/Loss (Raw)": 0.8211010694503784, "Finetune/Step": 2989, "Finetune/Step Time": 5.025006856769323} +{"Finetune/Learning Rate": 8.059408329787184e-06, "Finetune/Loss": 0.872084379196167, "Finetune/Loss (Raw)": 1.0100542306900024, "Finetune/Step": 2990, "Finetune/Step Time": 5.006554566323757} +{"Finetune/Learning Rate": 8.053297522059189e-06, "Finetune/Loss": 0.8723142743110657, "Finetune/Loss (Raw)": 0.9510722756385803, "Finetune/Step": 2991, "Finetune/Step Time": 5.011367971077561} +{"Finetune/Learning Rate": 8.047187469808418e-06, "Finetune/Loss": 0.8718035221099854, "Finetune/Loss (Raw)": 0.7849482297897339, "Finetune/Step": 2992, "Finetune/Step Time": 5.011440631002188} +{"Finetune/Learning Rate": 8.041078175406064e-06, "Finetune/Loss": 0.8718513250350952, "Finetune/Loss (Raw)": 0.9993682503700256, "Finetune/Step": 2993, "Finetune/Step Time": 5.0073084235191345} +{"Finetune/Learning Rate": 8.03496964122303e-06, "Finetune/Loss": 0.8767194747924805, "Finetune/Loss (Raw)": 1.01682710647583, "Finetune/Step": 2994, "Finetune/Step Time": 4.974113404750824} +{"Finetune/Learning Rate": 8.02886186962991e-06, "Finetune/Loss": 0.8766767978668213, "Finetune/Loss (Raw)": 0.8987341523170471, "Finetune/Step": 2995, "Finetune/Step Time": 4.978020766749978} +{"Finetune/Learning Rate": 8.022754862997016e-06, "Finetune/Loss": 0.8755380511283875, "Finetune/Loss (Raw)": 0.8776952624320984, "Finetune/Step": 2996, "Finetune/Step Time": 4.993070004507899} +{"Finetune/Learning Rate": 8.016648623694354e-06, "Finetune/Loss": 0.876393735408783, "Finetune/Loss (Raw)": 0.8318684101104736, "Finetune/Step": 2997, "Finetune/Step Time": 4.958065900951624} +{"Finetune/Learning Rate": 8.010543154091644e-06, "Finetune/Loss": 0.8801721334457397, "Finetune/Loss (Raw)": 0.8625719547271729, "Finetune/Step": 2998, "Finetune/Step Time": 4.931857939809561} +{"Finetune/Learning Rate": 8.004438456558297e-06, "Finetune/Loss": 0.8806165456771851, "Finetune/Loss (Raw)": 1.0649648904800415, "Finetune/Step": 2999, "Finetune/Step Time": 4.938467124477029} +{"Finetune/Learning Rate": 7.99833453346342e-06, "Finetune/Loss": 0.8799483180046082, "Finetune/Loss (Raw)": 0.894034743309021, "Finetune/Step": 3000, "Finetune/Step Time": 4.9426318276673555} +{"Finetune/Learning Rate": 7.992231387175832e-06, "Finetune/Loss": 0.8797334432601929, "Finetune/Loss (Raw)": 0.8587000370025635, "Finetune/Step": 3001, "Finetune/Step Time": 4.945486897602677} +{"Finetune/Learning Rate": 7.986129020064044e-06, "Finetune/Loss": 0.8842210173606873, "Finetune/Loss (Raw)": 0.9342170357704163, "Finetune/Step": 3002, "Finetune/Step Time": 4.912929067388177} +{"Finetune/Learning Rate": 7.980027434496267e-06, "Finetune/Loss": 0.8832699656486511, "Finetune/Loss (Raw)": 0.8819137215614319, "Finetune/Step": 3003, "Finetune/Step Time": 4.919490190222859} +{"Finetune/Learning Rate": 7.97392663284041e-06, "Finetune/Loss": 0.8836963176727295, "Finetune/Loss (Raw)": 0.9323468804359436, "Finetune/Step": 3004, "Finetune/Step Time": 4.912538236007094} +{"Finetune/Learning Rate": 7.967826617464065e-06, "Finetune/Loss": 0.8828334808349609, "Finetune/Loss (Raw)": 0.7291402816772461, "Finetune/Step": 3005, "Finetune/Step Time": 4.93536171503365} +{"Finetune/Learning Rate": 7.961727390734538e-06, "Finetune/Loss": 0.8829459547996521, "Finetune/Loss (Raw)": 0.9374287724494934, "Finetune/Step": 3006, "Finetune/Step Time": 4.927942520007491} +{"Finetune/Learning Rate": 7.955628955018813e-06, "Finetune/Loss": 0.88279128074646, "Finetune/Loss (Raw)": 0.7641525268554688, "Finetune/Step": 3007, "Finetune/Step Time": 4.9190853536129} +{"Finetune/Learning Rate": 7.949531312683582e-06, "Finetune/Loss": 0.8824377059936523, "Finetune/Loss (Raw)": 0.9434667825698853, "Finetune/Step": 3008, "Finetune/Step Time": 4.920655971392989} +{"Finetune/Learning Rate": 7.943434466095216e-06, "Finetune/Loss": 0.8825083374977112, "Finetune/Loss (Raw)": 0.9866278767585754, "Finetune/Step": 3009, "Finetune/Step Time": 4.913560988381505} +{"Finetune/Learning Rate": 7.937338417619781e-06, "Finetune/Loss": 0.8815821409225464, "Finetune/Loss (Raw)": 0.8691176176071167, "Finetune/Step": 3010, "Finetune/Step Time": 4.909046383574605} +{"Finetune/Learning Rate": 7.931243169623037e-06, "Finetune/Loss": 0.8816285133361816, "Finetune/Loss (Raw)": 1.01569402217865, "Finetune/Step": 3011, "Finetune/Step Time": 4.905626954510808} +{"Finetune/Learning Rate": 7.925148724470428e-06, "Finetune/Loss": 0.8823184967041016, "Finetune/Loss (Raw)": 0.9402083158493042, "Finetune/Step": 3012, "Finetune/Step Time": 4.877364404499531} +{"Finetune/Learning Rate": 7.919055084527094e-06, "Finetune/Loss": 0.8819159269332886, "Finetune/Loss (Raw)": 0.8881332278251648, "Finetune/Step": 3013, "Finetune/Step Time": 4.868408773094416} +{"Finetune/Learning Rate": 7.912962252157854e-06, "Finetune/Loss": 0.8831868171691895, "Finetune/Loss (Raw)": 0.845372200012207, "Finetune/Step": 3014, "Finetune/Step Time": 4.857040107250214} +{"Finetune/Learning Rate": 7.906870229727216e-06, "Finetune/Loss": 0.8829857707023621, "Finetune/Loss (Raw)": 0.7620111703872681, "Finetune/Step": 3015, "Finetune/Step Time": 4.865963272750378} +{"Finetune/Learning Rate": 7.900779019599378e-06, "Finetune/Loss": 0.8818154335021973, "Finetune/Loss (Raw)": 0.9024999737739563, "Finetune/Step": 3016, "Finetune/Step Time": 4.864158684387803} +{"Finetune/Learning Rate": 7.894688624138217e-06, "Finetune/Loss": 0.8852531313896179, "Finetune/Loss (Raw)": 0.7709036469459534, "Finetune/Step": 3017, "Finetune/Step Time": 4.834405222907662} +{"Finetune/Learning Rate": 7.8885990457073e-06, "Finetune/Loss": 0.8855142593383789, "Finetune/Loss (Raw)": 0.8420174717903137, "Finetune/Step": 3018, "Finetune/Step Time": 4.829965831711888} +{"Finetune/Learning Rate": 7.882510286669875e-06, "Finetune/Loss": 0.8857381343841553, "Finetune/Loss (Raw)": 0.9208929538726807, "Finetune/Step": 3019, "Finetune/Step Time": 4.831549357622862} +{"Finetune/Learning Rate": 7.876422349388861e-06, "Finetune/Loss": 0.8839810490608215, "Finetune/Loss (Raw)": 0.8686387538909912, "Finetune/Step": 3020, "Finetune/Step Time": 4.813221950083971} +{"Finetune/Learning Rate": 7.870335236226878e-06, "Finetune/Loss": 0.8840214014053345, "Finetune/Loss (Raw)": 0.8875710368156433, "Finetune/Step": 3021, "Finetune/Step Time": 4.822939855977893} +{"Finetune/Learning Rate": 7.864248949546209e-06, "Finetune/Loss": 0.8837257623672485, "Finetune/Loss (Raw)": 0.9737243056297302, "Finetune/Step": 3022, "Finetune/Step Time": 4.822796801105142} +{"Finetune/Learning Rate": 7.858163491708827e-06, "Finetune/Loss": 0.8825509548187256, "Finetune/Loss (Raw)": 0.7363502979278564, "Finetune/Step": 3023, "Finetune/Step Time": 4.825462080538273} +{"Finetune/Learning Rate": 7.85207886507638e-06, "Finetune/Loss": 0.8824523687362671, "Finetune/Loss (Raw)": 0.9255592823028564, "Finetune/Step": 3024, "Finetune/Step Time": 4.830980092287064} +{"Finetune/Learning Rate": 7.845995072010188e-06, "Finetune/Loss": 0.8820704221725464, "Finetune/Loss (Raw)": 0.9307271838188171, "Finetune/Step": 3025, "Finetune/Step Time": 4.830394219607115} +{"Finetune/Learning Rate": 7.839912114871256e-06, "Finetune/Loss": 0.8821996450424194, "Finetune/Loss (Raw)": 0.9309598207473755, "Finetune/Step": 3026, "Finetune/Step Time": 4.834718951955438} +{"Finetune/Learning Rate": 7.83382999602026e-06, "Finetune/Loss": 0.8831526041030884, "Finetune/Loss (Raw)": 0.9835698008537292, "Finetune/Step": 3027, "Finetune/Step Time": 4.8330073822289705} +{"Finetune/Learning Rate": 7.827748717817552e-06, "Finetune/Loss": 0.8840302228927612, "Finetune/Loss (Raw)": 0.8968495726585388, "Finetune/Step": 3028, "Finetune/Step Time": 4.83631244674325} +{"Finetune/Learning Rate": 7.821668282623159e-06, "Finetune/Loss": 0.8860173225402832, "Finetune/Loss (Raw)": 1.0520952939987183, "Finetune/Step": 3029, "Finetune/Step Time": 4.827233392745256} +{"Finetune/Learning Rate": 7.815588692796771e-06, "Finetune/Loss": 0.8856282234191895, "Finetune/Loss (Raw)": 0.8789106011390686, "Finetune/Step": 3030, "Finetune/Step Time": 4.8243911899626255} +{"Finetune/Learning Rate": 7.809509950697769e-06, "Finetune/Loss": 0.8852891325950623, "Finetune/Loss (Raw)": 0.9095510840415955, "Finetune/Step": 3031, "Finetune/Step Time": 4.8216164615005255} +{"Finetune/Learning Rate": 7.803432058685187e-06, "Finetune/Loss": 0.8861544132232666, "Finetune/Loss (Raw)": 0.9752457737922668, "Finetune/Step": 3032, "Finetune/Step Time": 4.82622348330915} +{"Finetune/Learning Rate": 7.797355019117743e-06, "Finetune/Loss": 0.8862389326095581, "Finetune/Loss (Raw)": 0.9061338305473328, "Finetune/Step": 3033, "Finetune/Step Time": 4.823180221021175} +{"Finetune/Learning Rate": 7.79127883435381e-06, "Finetune/Loss": 0.8871124982833862, "Finetune/Loss (Raw)": 1.042994499206543, "Finetune/Step": 3034, "Finetune/Step Time": 4.818991910666227} +{"Finetune/Learning Rate": 7.78520350675144e-06, "Finetune/Loss": 0.8873720169067383, "Finetune/Loss (Raw)": 0.8947610259056091, "Finetune/Step": 3035, "Finetune/Step Time": 4.820451732724905} +{"Finetune/Learning Rate": 7.779129038668347e-06, "Finetune/Loss": 0.8868848085403442, "Finetune/Loss (Raw)": 0.8806445002555847, "Finetune/Step": 3036, "Finetune/Step Time": 4.833770174533129} +{"Finetune/Learning Rate": 7.773055432461916e-06, "Finetune/Loss": 0.8864012956619263, "Finetune/Loss (Raw)": 1.0193136930465698, "Finetune/Step": 3037, "Finetune/Step Time": 4.8362468015402555} +{"Finetune/Learning Rate": 7.766982690489198e-06, "Finetune/Loss": 0.8863869905471802, "Finetune/Loss (Raw)": 0.9838031530380249, "Finetune/Step": 3038, "Finetune/Step Time": 4.835399417206645} +{"Finetune/Learning Rate": 7.7609108151069e-06, "Finetune/Loss": 0.884752631187439, "Finetune/Loss (Raw)": 0.8399562239646912, "Finetune/Step": 3039, "Finetune/Step Time": 4.835371540859342} +{"Finetune/Learning Rate": 7.754839808671397e-06, "Finetune/Loss": 0.8846925497055054, "Finetune/Loss (Raw)": 0.9338465929031372, "Finetune/Step": 3040, "Finetune/Step Time": 4.828319173306227} +{"Finetune/Learning Rate": 7.748769673538734e-06, "Finetune/Loss": 0.8834871649742126, "Finetune/Loss (Raw)": 0.7775000929832458, "Finetune/Step": 3041, "Finetune/Step Time": 4.833169648423791} +{"Finetune/Learning Rate": 7.742700412064608e-06, "Finetune/Loss": 0.8852664232254028, "Finetune/Loss (Raw)": 0.9280689358711243, "Finetune/Step": 3042, "Finetune/Step Time": 4.808675719425082} +{"Finetune/Learning Rate": 7.736632026604386e-06, "Finetune/Loss": 0.8819100856781006, "Finetune/Loss (Raw)": 0.30912843346595764, "Finetune/Step": 3043, "Finetune/Step Time": 4.833573857322335} +{"Finetune/Learning Rate": 7.730564519513082e-06, "Finetune/Loss": 0.8802520632743835, "Finetune/Loss (Raw)": 0.7153602242469788, "Finetune/Step": 3044, "Finetune/Step Time": 4.852535637095571} +{"Finetune/Learning Rate": 7.724497893145379e-06, "Finetune/Loss": 0.879007875919342, "Finetune/Loss (Raw)": 0.8127847909927368, "Finetune/Step": 3045, "Finetune/Step Time": 4.858806071802974} +{"Finetune/Learning Rate": 7.718432149855622e-06, "Finetune/Loss": 0.8804997205734253, "Finetune/Loss (Raw)": 1.0368428230285645, "Finetune/Step": 3046, "Finetune/Step Time": 4.852553475648165} +{"Finetune/Learning Rate": 7.7123672919978e-06, "Finetune/Loss": 0.8800919651985168, "Finetune/Loss (Raw)": 0.8602514266967773, "Finetune/Step": 3047, "Finetune/Step Time": 4.858705747872591} +{"Finetune/Learning Rate": 7.706303321925577e-06, "Finetune/Loss": 0.8855611085891724, "Finetune/Loss (Raw)": 1.0452244281768799, "Finetune/Step": 3048, "Finetune/Step Time": 4.823030499741435} +{"Finetune/Learning Rate": 7.700240241992247e-06, "Finetune/Loss": 0.8854846954345703, "Finetune/Loss (Raw)": 0.8860042095184326, "Finetune/Step": 3049, "Finetune/Step Time": 4.80582132935524} +{"Finetune/Learning Rate": 7.694178054550782e-06, "Finetune/Loss": 0.8865910768508911, "Finetune/Loss (Raw)": 0.9973390698432922, "Finetune/Step": 3050, "Finetune/Step Time": 4.797337641939521} +{"Finetune/Learning Rate": 7.688116761953797e-06, "Finetune/Loss": 0.8877444863319397, "Finetune/Loss (Raw)": 1.0271474123001099, "Finetune/Step": 3051, "Finetune/Step Time": 4.775515053421259} +{"Finetune/Learning Rate": 7.682056366553559e-06, "Finetune/Loss": 0.8867968320846558, "Finetune/Loss (Raw)": 0.8754994869232178, "Finetune/Step": 3052, "Finetune/Step Time": 4.779648477211595} +{"Finetune/Learning Rate": 7.675996870701999e-06, "Finetune/Loss": 0.8865385055541992, "Finetune/Loss (Raw)": 0.9763182401657104, "Finetune/Step": 3053, "Finetune/Step Time": 4.782839512452483} +{"Finetune/Learning Rate": 7.669938276750673e-06, "Finetune/Loss": 0.8875852823257446, "Finetune/Loss (Raw)": 1.0133554935455322, "Finetune/Step": 3054, "Finetune/Step Time": 4.745963469147682} +{"Finetune/Learning Rate": 7.663880587050812e-06, "Finetune/Loss": 0.8893741965293884, "Finetune/Loss (Raw)": 0.9290367364883423, "Finetune/Step": 3055, "Finetune/Step Time": 4.737387320026755} +{"Finetune/Learning Rate": 7.657823803953288e-06, "Finetune/Loss": 0.8899791240692139, "Finetune/Loss (Raw)": 0.9758588671684265, "Finetune/Step": 3056, "Finetune/Step Time": 4.715965623036027} +{"Finetune/Learning Rate": 7.651767929808615e-06, "Finetune/Loss": 0.887996256351471, "Finetune/Loss (Raw)": 0.7887215614318848, "Finetune/Step": 3057, "Finetune/Step Time": 4.7287449687719345} +{"Finetune/Learning Rate": 7.645712966966973e-06, "Finetune/Loss": 0.8879730701446533, "Finetune/Loss (Raw)": 0.966813862323761, "Finetune/Step": 3058, "Finetune/Step Time": 4.725541489198804} +{"Finetune/Learning Rate": 7.639658917778156e-06, "Finetune/Loss": 0.8880078792572021, "Finetune/Loss (Raw)": 0.9442132711410522, "Finetune/Step": 3059, "Finetune/Step Time": 4.722127061337233} +{"Finetune/Learning Rate": 7.633605784591633e-06, "Finetune/Loss": 0.8883190155029297, "Finetune/Loss (Raw)": 0.9753513336181641, "Finetune/Step": 3060, "Finetune/Step Time": 4.726466881111264} +{"Finetune/Learning Rate": 7.627553569756505e-06, "Finetune/Loss": 0.8886516094207764, "Finetune/Loss (Raw)": 0.9507151246070862, "Finetune/Step": 3061, "Finetune/Step Time": 4.723712649196386} +{"Finetune/Learning Rate": 7.621502275621518e-06, "Finetune/Loss": 0.8880534768104553, "Finetune/Loss (Raw)": 0.8456593155860901, "Finetune/Step": 3062, "Finetune/Step Time": 4.756794614717364} +{"Finetune/Learning Rate": 7.615451904535066e-06, "Finetune/Loss": 0.8877705335617065, "Finetune/Loss (Raw)": 0.8652735948562622, "Finetune/Step": 3063, "Finetune/Step Time": 4.748519631102681} +{"Finetune/Learning Rate": 7.609402458845171e-06, "Finetune/Loss": 0.8868871927261353, "Finetune/Loss (Raw)": 0.9080923795700073, "Finetune/Step": 3064, "Finetune/Step Time": 4.729776790365577} +{"Finetune/Learning Rate": 7.60335394089951e-06, "Finetune/Loss": 0.8849149942398071, "Finetune/Loss (Raw)": 0.8028603196144104, "Finetune/Step": 3065, "Finetune/Step Time": 4.746885620057583} +{"Finetune/Learning Rate": 7.597306353045393e-06, "Finetune/Loss": 0.882940411567688, "Finetune/Loss (Raw)": 0.6897900700569153, "Finetune/Step": 3066, "Finetune/Step Time": 4.775071162730455} +{"Finetune/Learning Rate": 7.5912596976297735e-06, "Finetune/Loss": 0.8828149437904358, "Finetune/Loss (Raw)": 0.8835899829864502, "Finetune/Step": 3067, "Finetune/Step Time": 4.7756096832454205} +{"Finetune/Learning Rate": 7.58521397699924e-06, "Finetune/Loss": 0.8840541839599609, "Finetune/Loss (Raw)": 0.8793643116950989, "Finetune/Step": 3068, "Finetune/Step Time": 4.787232480943203} +{"Finetune/Learning Rate": 7.579169193500014e-06, "Finetune/Loss": 0.8849388360977173, "Finetune/Loss (Raw)": 0.7929641604423523, "Finetune/Step": 3069, "Finetune/Step Time": 4.7784889079630375} +{"Finetune/Learning Rate": 7.573125349477964e-06, "Finetune/Loss": 0.8842235803604126, "Finetune/Loss (Raw)": 0.9327035546302795, "Finetune/Step": 3070, "Finetune/Step Time": 4.776272963732481} +{"Finetune/Learning Rate": 7.567082447278582e-06, "Finetune/Loss": 0.88475501537323, "Finetune/Loss (Raw)": 1.011088490486145, "Finetune/Step": 3071, "Finetune/Step Time": 4.768712656572461} +{"Finetune/Learning Rate": 7.561040489247008e-06, "Finetune/Loss": 0.8843600749969482, "Finetune/Loss (Raw)": 0.8626319169998169, "Finetune/Step": 3072, "Finetune/Step Time": 4.7747719287872314} +{"Finetune/Learning Rate": 7.554999477728001e-06, "Finetune/Loss": 0.8848755955696106, "Finetune/Loss (Raw)": 0.8922060132026672, "Finetune/Step": 3073, "Finetune/Step Time": 4.765097040683031} +{"Finetune/Learning Rate": 7.548959415065961e-06, "Finetune/Loss": 0.8844362497329712, "Finetune/Loss (Raw)": 0.8601757287979126, "Finetune/Step": 3074, "Finetune/Step Time": 4.767301635816693} +{"Finetune/Learning Rate": 7.54292030360492e-06, "Finetune/Loss": 0.8799200057983398, "Finetune/Loss (Raw)": 0.2940863370895386, "Finetune/Step": 3075, "Finetune/Step Time": 4.787028877064586} +{"Finetune/Learning Rate": 7.5368821456885375e-06, "Finetune/Loss": 0.8782831430435181, "Finetune/Loss (Raw)": 0.7569936513900757, "Finetune/Step": 3076, "Finetune/Step Time": 4.773089103400707} +{"Finetune/Learning Rate": 7.530844943660111e-06, "Finetune/Loss": 0.8781859874725342, "Finetune/Loss (Raw)": 0.935741126537323, "Finetune/Step": 3077, "Finetune/Step Time": 4.772819751873612} +{"Finetune/Learning Rate": 7.524808699862552e-06, "Finetune/Loss": 0.8774112462997437, "Finetune/Loss (Raw)": 0.8093234896659851, "Finetune/Step": 3078, "Finetune/Step Time": 4.769334120675921} +{"Finetune/Learning Rate": 7.518773416638412e-06, "Finetune/Loss": 0.8809559345245361, "Finetune/Loss (Raw)": 0.758039653301239, "Finetune/Step": 3079, "Finetune/Step Time": 4.756564037874341} +{"Finetune/Learning Rate": 7.512739096329868e-06, "Finetune/Loss": 0.87946617603302, "Finetune/Loss (Raw)": 0.7639036774635315, "Finetune/Step": 3080, "Finetune/Step Time": 4.755602475255728} +{"Finetune/Learning Rate": 7.50670574127872e-06, "Finetune/Loss": 0.878143310546875, "Finetune/Loss (Raw)": 0.8157484531402588, "Finetune/Step": 3081, "Finetune/Step Time": 4.7586696818470955} +{"Finetune/Learning Rate": 7.500673353826399e-06, "Finetune/Loss": 0.8787610530853271, "Finetune/Loss (Raw)": 0.8742174506187439, "Finetune/Step": 3082, "Finetune/Step Time": 4.7485863752663136} +{"Finetune/Learning Rate": 7.4946419363139535e-06, "Finetune/Loss": 0.8779199123382568, "Finetune/Loss (Raw)": 0.9465197324752808, "Finetune/Step": 3083, "Finetune/Step Time": 4.759594729170203} +{"Finetune/Learning Rate": 7.488611491082056e-06, "Finetune/Loss": 0.8769509196281433, "Finetune/Loss (Raw)": 0.8164510726928711, "Finetune/Step": 3084, "Finetune/Step Time": 4.770818375051022} +{"Finetune/Learning Rate": 7.482582020471012e-06, "Finetune/Loss": 0.8782771825790405, "Finetune/Loss (Raw)": 1.0723793506622314, "Finetune/Step": 3085, "Finetune/Step Time": 4.762064844369888} +{"Finetune/Learning Rate": 7.476553526820733e-06, "Finetune/Loss": 0.8787446022033691, "Finetune/Loss (Raw)": 0.9725955128669739, "Finetune/Step": 3086, "Finetune/Step Time": 4.766758099198341} +{"Finetune/Learning Rate": 7.4705260124707705e-06, "Finetune/Loss": 0.8832916021347046, "Finetune/Loss (Raw)": 0.9006264209747314, "Finetune/Step": 3087, "Finetune/Step Time": 4.736222956329584} +{"Finetune/Learning Rate": 7.464499479760275e-06, "Finetune/Loss": 0.8817100524902344, "Finetune/Loss (Raw)": 0.7047750949859619, "Finetune/Step": 3088, "Finetune/Step Time": 4.735076319426298} +{"Finetune/Learning Rate": 7.458473931028029e-06, "Finetune/Loss": 0.8825313448905945, "Finetune/Loss (Raw)": 0.9455384612083435, "Finetune/Step": 3089, "Finetune/Step Time": 4.744367614388466} +{"Finetune/Learning Rate": 7.452449368612433e-06, "Finetune/Loss": 0.8805992007255554, "Finetune/Loss (Raw)": 0.8272097706794739, "Finetune/Step": 3090, "Finetune/Step Time": 4.73809969983995} +{"Finetune/Learning Rate": 7.446425794851498e-06, "Finetune/Loss": 0.8793438673019409, "Finetune/Loss (Raw)": 0.8187834620475769, "Finetune/Step": 3091, "Finetune/Step Time": 4.741987839341164} +{"Finetune/Learning Rate": 7.440403212082863e-06, "Finetune/Loss": 0.8739449977874756, "Finetune/Loss (Raw)": 0.2998633086681366, "Finetune/Step": 3092, "Finetune/Step Time": 4.775053009390831} +{"Finetune/Learning Rate": 7.434381622643766e-06, "Finetune/Loss": 0.8733208179473877, "Finetune/Loss (Raw)": 0.8789626359939575, "Finetune/Step": 3093, "Finetune/Step Time": 4.78196837566793} +{"Finetune/Learning Rate": 7.42836102887107e-06, "Finetune/Loss": 0.8740414381027222, "Finetune/Loss (Raw)": 0.9100605845451355, "Finetune/Step": 3094, "Finetune/Step Time": 4.776381837204099} +{"Finetune/Learning Rate": 7.422341433101254e-06, "Finetune/Loss": 0.8748558163642883, "Finetune/Loss (Raw)": 0.8843667507171631, "Finetune/Step": 3095, "Finetune/Step Time": 4.757683960720897} +{"Finetune/Learning Rate": 7.4163228376704e-06, "Finetune/Loss": 0.8739557266235352, "Finetune/Loss (Raw)": 0.908399224281311, "Finetune/Step": 3096, "Finetune/Step Time": 4.7522192299366} +{"Finetune/Learning Rate": 7.410305244914216e-06, "Finetune/Loss": 0.8695774078369141, "Finetune/Loss (Raw)": 0.30431416630744934, "Finetune/Step": 3097, "Finetune/Step Time": 4.790082989260554} +{"Finetune/Learning Rate": 7.404288657168006e-06, "Finetune/Loss": 0.8680161237716675, "Finetune/Loss (Raw)": 0.7100155353546143, "Finetune/Step": 3098, "Finetune/Step Time": 4.7777882274240255} +{"Finetune/Learning Rate": 7.398273076766687e-06, "Finetune/Loss": 0.8683197498321533, "Finetune/Loss (Raw)": 0.9346745014190674, "Finetune/Step": 3099, "Finetune/Step Time": 4.781077440828085} +{"Finetune/Learning Rate": 7.392258506044796e-06, "Finetune/Loss": 0.8694363832473755, "Finetune/Loss (Raw)": 1.0701255798339844, "Finetune/Step": 3100, "Finetune/Step Time": 4.783267974853516} +{"Finetune/Learning Rate": 7.386244947336463e-06, "Finetune/Loss": 0.8684169054031372, "Finetune/Loss (Raw)": 0.6497142910957336, "Finetune/Step": 3101, "Finetune/Step Time": 4.785531198605895} +{"Finetune/Learning Rate": 7.380232402975443e-06, "Finetune/Loss": 0.8687549829483032, "Finetune/Loss (Raw)": 0.9758588075637817, "Finetune/Step": 3102, "Finetune/Step Time": 4.7930302657186985} +{"Finetune/Learning Rate": 7.3742208752950776e-06, "Finetune/Loss": 0.873673141002655, "Finetune/Loss (Raw)": 1.0207719802856445, "Finetune/Step": 3103, "Finetune/Step Time": 4.76855992898345} +{"Finetune/Learning Rate": 7.3682103666283245e-06, "Finetune/Loss": 0.874406099319458, "Finetune/Loss (Raw)": 0.919958770275116, "Finetune/Step": 3104, "Finetune/Step Time": 4.763402376323938} +{"Finetune/Learning Rate": 7.362200879307748e-06, "Finetune/Loss": 0.874274492263794, "Finetune/Loss (Raw)": 0.9038445949554443, "Finetune/Step": 3105, "Finetune/Step Time": 4.753639155998826} +{"Finetune/Learning Rate": 7.3561924156655105e-06, "Finetune/Loss": 0.8732591867446899, "Finetune/Loss (Raw)": 0.8544729351997375, "Finetune/Step": 3106, "Finetune/Step Time": 4.7540554609149694} +{"Finetune/Learning Rate": 7.350184978033386e-06, "Finetune/Loss": 0.8741658926010132, "Finetune/Loss (Raw)": 1.0089386701583862, "Finetune/Step": 3107, "Finetune/Step Time": 4.747287396341562} +{"Finetune/Learning Rate": 7.344178568742734e-06, "Finetune/Loss": 0.8739581108093262, "Finetune/Loss (Raw)": 0.8173896670341492, "Finetune/Step": 3108, "Finetune/Step Time": 4.738544782623649} +{"Finetune/Learning Rate": 7.338173190124526e-06, "Finetune/Loss": 0.8747817277908325, "Finetune/Loss (Raw)": 0.8909522294998169, "Finetune/Step": 3109, "Finetune/Step Time": 4.7139512822031975} +{"Finetune/Learning Rate": 7.3321688445093395e-06, "Finetune/Loss": 0.8749487400054932, "Finetune/Loss (Raw)": 0.8430600762367249, "Finetune/Step": 3110, "Finetune/Step Time": 4.710877850651741} +{"Finetune/Learning Rate": 7.326165534227341e-06, "Finetune/Loss": 0.8738850355148315, "Finetune/Loss (Raw)": 0.8750163316726685, "Finetune/Step": 3111, "Finetune/Step Time": 4.719888182356954} +{"Finetune/Learning Rate": 7.320163261608293e-06, "Finetune/Loss": 0.8738125562667847, "Finetune/Loss (Raw)": 0.9685357809066772, "Finetune/Step": 3112, "Finetune/Step Time": 4.704533504322171} +{"Finetune/Learning Rate": 7.314162028981564e-06, "Finetune/Loss": 0.8737962245941162, "Finetune/Loss (Raw)": 0.874414324760437, "Finetune/Step": 3113, "Finetune/Step Time": 4.717992627993226} +{"Finetune/Learning Rate": 7.308161838676113e-06, "Finetune/Loss": 0.8785434365272522, "Finetune/Loss (Raw)": 0.9368937611579895, "Finetune/Step": 3114, "Finetune/Step Time": 4.688203996047378} +{"Finetune/Learning Rate": 7.302162693020503e-06, "Finetune/Loss": 0.8786555528640747, "Finetune/Loss (Raw)": 0.8543659448623657, "Finetune/Step": 3115, "Finetune/Step Time": 4.670503435656428} +{"Finetune/Learning Rate": 7.2961645943428804e-06, "Finetune/Loss": 0.8768919706344604, "Finetune/Loss (Raw)": 0.727039635181427, "Finetune/Step": 3116, "Finetune/Step Time": 4.69816386140883} +{"Finetune/Learning Rate": 7.290167544970989e-06, "Finetune/Loss": 0.8774883151054382, "Finetune/Loss (Raw)": 0.8974366784095764, "Finetune/Step": 3117, "Finetune/Step Time": 4.693203611299396} +{"Finetune/Learning Rate": 7.284171547232168e-06, "Finetune/Loss": 0.8733100891113281, "Finetune/Loss (Raw)": 0.47524014115333557, "Finetune/Step": 3118, "Finetune/Step Time": 4.724983893334866} +{"Finetune/Learning Rate": 7.278176603453347e-06, "Finetune/Loss": 0.8726956248283386, "Finetune/Loss (Raw)": 0.8724232912063599, "Finetune/Step": 3119, "Finetune/Step Time": 4.724787646904588} +{"Finetune/Learning Rate": 7.272182715961049e-06, "Finetune/Loss": 0.8736010789871216, "Finetune/Loss (Raw)": 0.9008375406265259, "Finetune/Step": 3120, "Finetune/Step Time": 4.718777427449822} +{"Finetune/Learning Rate": 7.266189887081385e-06, "Finetune/Loss": 0.871542751789093, "Finetune/Loss (Raw)": 0.7359095811843872, "Finetune/Step": 3121, "Finetune/Step Time": 4.730343261733651} +{"Finetune/Learning Rate": 7.2601981191400484e-06, "Finetune/Loss": 0.8705873489379883, "Finetune/Loss (Raw)": 0.8945399522781372, "Finetune/Step": 3122, "Finetune/Step Time": 4.726656634360552} +{"Finetune/Learning Rate": 7.254207414462333e-06, "Finetune/Loss": 0.8701088428497314, "Finetune/Loss (Raw)": 0.8374775648117065, "Finetune/Step": 3123, "Finetune/Step Time": 4.715385185554624} +{"Finetune/Learning Rate": 7.248217775373112e-06, "Finetune/Loss": 0.8709729909896851, "Finetune/Loss (Raw)": 0.9883089065551758, "Finetune/Step": 3124, "Finetune/Step Time": 4.704209798946977} +{"Finetune/Learning Rate": 7.242229204196849e-06, "Finetune/Loss": 0.871652364730835, "Finetune/Loss (Raw)": 0.9188277721405029, "Finetune/Step": 3125, "Finetune/Step Time": 4.709812242537737} +{"Finetune/Learning Rate": 7.236241703257594e-06, "Finetune/Loss": 0.8724753856658936, "Finetune/Loss (Raw)": 0.9679253101348877, "Finetune/Step": 3126, "Finetune/Step Time": 4.699945325031877} +{"Finetune/Learning Rate": 7.230255274878972e-06, "Finetune/Loss": 0.8719800114631653, "Finetune/Loss (Raw)": 1.0015500783920288, "Finetune/Step": 3127, "Finetune/Step Time": 4.6957125924527645} +{"Finetune/Learning Rate": 7.224269921384206e-06, "Finetune/Loss": 0.8704833984375, "Finetune/Loss (Raw)": 0.7024659514427185, "Finetune/Step": 3128, "Finetune/Step Time": 4.737213850021362} +{"Finetune/Learning Rate": 7.218285645096088e-06, "Finetune/Loss": 0.8704221248626709, "Finetune/Loss (Raw)": 0.8508625030517578, "Finetune/Step": 3129, "Finetune/Step Time": 4.73365244269371} +{"Finetune/Learning Rate": 7.2123024483370056e-06, "Finetune/Loss": 0.8694727420806885, "Finetune/Loss (Raw)": 0.8126911520957947, "Finetune/Step": 3130, "Finetune/Step Time": 4.748556803911924} +{"Finetune/Learning Rate": 7.20632033342892e-06, "Finetune/Loss": 0.8692865371704102, "Finetune/Loss (Raw)": 0.858083188533783, "Finetune/Step": 3131, "Finetune/Step Time": 4.750294096767902} +{"Finetune/Learning Rate": 7.200339302693367e-06, "Finetune/Loss": 0.8652418255805969, "Finetune/Loss (Raw)": 0.4146248698234558, "Finetune/Step": 3132, "Finetune/Step Time": 4.786069775000215} +{"Finetune/Learning Rate": 7.194359358451475e-06, "Finetune/Loss": 0.8667831420898438, "Finetune/Loss (Raw)": 0.9264276623725891, "Finetune/Step": 3133, "Finetune/Step Time": 4.761657815426588} +{"Finetune/Learning Rate": 7.188380503023936e-06, "Finetune/Loss": 0.8662201166152954, "Finetune/Loss (Raw)": 0.8653649687767029, "Finetune/Step": 3134, "Finetune/Step Time": 4.776564484462142} +{"Finetune/Learning Rate": 7.182402738731036e-06, "Finetune/Loss": 0.8675087094306946, "Finetune/Loss (Raw)": 0.9290886521339417, "Finetune/Step": 3135, "Finetune/Step Time": 4.778936933726072} +{"Finetune/Learning Rate": 7.176426067892623e-06, "Finetune/Loss": 0.8676075339317322, "Finetune/Loss (Raw)": 0.9561142921447754, "Finetune/Step": 3136, "Finetune/Step Time": 4.786014026030898} +{"Finetune/Learning Rate": 7.170450492828126e-06, "Finetune/Loss": 0.8664092421531677, "Finetune/Loss (Raw)": 0.833250105381012, "Finetune/Step": 3137, "Finetune/Step Time": 4.802316864952445} +{"Finetune/Learning Rate": 7.164476015856552e-06, "Finetune/Loss": 0.8655466437339783, "Finetune/Loss (Raw)": 0.7587026357650757, "Finetune/Step": 3138, "Finetune/Step Time": 4.810299754142761} +{"Finetune/Learning Rate": 7.158502639296475e-06, "Finetune/Loss": 0.8645004034042358, "Finetune/Loss (Raw)": 0.8817791938781738, "Finetune/Step": 3139, "Finetune/Step Time": 4.811838801950216} +{"Finetune/Learning Rate": 7.152530365466051e-06, "Finetune/Loss": 0.8639494180679321, "Finetune/Loss (Raw)": 0.8696752786636353, "Finetune/Step": 3140, "Finetune/Step Time": 4.822796935215592} +{"Finetune/Learning Rate": 7.146559196683001e-06, "Finetune/Loss": 0.8634932041168213, "Finetune/Loss (Raw)": 0.8297445774078369, "Finetune/Step": 3141, "Finetune/Step Time": 4.824917869642377} +{"Finetune/Learning Rate": 7.140589135264614e-06, "Finetune/Loss": 0.8635233640670776, "Finetune/Loss (Raw)": 0.8492294549942017, "Finetune/Step": 3142, "Finetune/Step Time": 4.824123555794358} +{"Finetune/Learning Rate": 7.13462018352776e-06, "Finetune/Loss": 0.8630070686340332, "Finetune/Loss (Raw)": 0.6959249973297119, "Finetune/Step": 3143, "Finetune/Step Time": 4.823634712025523} +{"Finetune/Learning Rate": 7.12865234378887e-06, "Finetune/Loss": 0.8637081384658813, "Finetune/Loss (Raw)": 0.9922350645065308, "Finetune/Step": 3144, "Finetune/Step Time": 4.815484952181578} +{"Finetune/Learning Rate": 7.122685618363948e-06, "Finetune/Loss": 0.8653749227523804, "Finetune/Loss (Raw)": 0.9842530488967896, "Finetune/Step": 3145, "Finetune/Step Time": 4.810264280065894} +{"Finetune/Learning Rate": 7.116720009568565e-06, "Finetune/Loss": 0.8661436438560486, "Finetune/Loss (Raw)": 0.9404115080833435, "Finetune/Step": 3146, "Finetune/Step Time": 4.810806214809418} +{"Finetune/Learning Rate": 7.110755519717851e-06, "Finetune/Loss": 0.8665667176246643, "Finetune/Loss (Raw)": 0.975051760673523, "Finetune/Step": 3147, "Finetune/Step Time": 4.82314701192081} +{"Finetune/Learning Rate": 7.104792151126515e-06, "Finetune/Loss": 0.8679229021072388, "Finetune/Loss (Raw)": 1.0422285795211792, "Finetune/Step": 3148, "Finetune/Step Time": 4.84347521327436} +{"Finetune/Learning Rate": 7.098829906108819e-06, "Finetune/Loss": 0.8642815351486206, "Finetune/Loss (Raw)": 0.4214768409729004, "Finetune/Step": 3149, "Finetune/Step Time": 4.871157322078943} +{"Finetune/Learning Rate": 7.092868786978596e-06, "Finetune/Loss": 0.8636386394500732, "Finetune/Loss (Raw)": 0.8914337754249573, "Finetune/Step": 3150, "Finetune/Step Time": 4.86826035939157} +{"Finetune/Learning Rate": 7.086908796049243e-06, "Finetune/Loss": 0.8648399114608765, "Finetune/Loss (Raw)": 0.8901139497756958, "Finetune/Step": 3151, "Finetune/Step Time": 4.875228906050324} +{"Finetune/Learning Rate": 7.08094993563371e-06, "Finetune/Loss": 0.86402827501297, "Finetune/Loss (Raw)": 0.8216710686683655, "Finetune/Step": 3152, "Finetune/Step Time": 4.869780071079731} +{"Finetune/Learning Rate": 7.074992208044518e-06, "Finetune/Loss": 0.8639269471168518, "Finetune/Loss (Raw)": 0.9177531599998474, "Finetune/Step": 3153, "Finetune/Step Time": 4.869861915707588} +{"Finetune/Learning Rate": 7.069035615593743e-06, "Finetune/Loss": 0.863592267036438, "Finetune/Loss (Raw)": 0.8881210088729858, "Finetune/Step": 3154, "Finetune/Step Time": 4.861869225278497} +{"Finetune/Learning Rate": 7.063080160593026e-06, "Finetune/Loss": 0.863448441028595, "Finetune/Loss (Raw)": 0.9651634693145752, "Finetune/Step": 3155, "Finetune/Step Time": 4.855904804542661} +{"Finetune/Learning Rate": 7.0571258453535585e-06, "Finetune/Loss": 0.862846314907074, "Finetune/Loss (Raw)": 0.8197711110115051, "Finetune/Step": 3156, "Finetune/Step Time": 4.856241215020418} +{"Finetune/Learning Rate": 7.051172672186093e-06, "Finetune/Loss": 0.8625054359436035, "Finetune/Loss (Raw)": 1.008468747138977, "Finetune/Step": 3157, "Finetune/Step Time": 4.861641865223646} +{"Finetune/Learning Rate": 7.045220643400946e-06, "Finetune/Loss": 0.8578691482543945, "Finetune/Loss (Raw)": 0.28546181321144104, "Finetune/Step": 3158, "Finetune/Step Time": 4.8781885132193565} +{"Finetune/Learning Rate": 7.039269761307977e-06, "Finetune/Loss": 0.857425332069397, "Finetune/Loss (Raw)": 0.8527456521987915, "Finetune/Step": 3159, "Finetune/Step Time": 4.881710520014167} +{"Finetune/Learning Rate": 7.0333200282166146e-06, "Finetune/Loss": 0.8566255569458008, "Finetune/Loss (Raw)": 0.872870147228241, "Finetune/Step": 3160, "Finetune/Step Time": 4.871592221781611} +{"Finetune/Learning Rate": 7.027371446435829e-06, "Finetune/Loss": 0.8566174507141113, "Finetune/Loss (Raw)": 0.9050948023796082, "Finetune/Step": 3161, "Finetune/Step Time": 4.891062954440713} +{"Finetune/Learning Rate": 7.0214240182741455e-06, "Finetune/Loss": 0.8516566753387451, "Finetune/Loss (Raw)": 0.408021479845047, "Finetune/Step": 3162, "Finetune/Step Time": 4.92644901201129} +{"Finetune/Learning Rate": 7.015477746039652e-06, "Finetune/Loss": 0.8528112769126892, "Finetune/Loss (Raw)": 1.0425505638122559, "Finetune/Step": 3163, "Finetune/Step Time": 4.93774170614779} +{"Finetune/Learning Rate": 7.009532632039975e-06, "Finetune/Loss": 0.8533030152320862, "Finetune/Loss (Raw)": 0.9435884952545166, "Finetune/Step": 3164, "Finetune/Step Time": 4.918046975508332} +{"Finetune/Learning Rate": 7.003588678582303e-06, "Finetune/Loss": 0.8506959080696106, "Finetune/Loss (Raw)": 0.6856003999710083, "Finetune/Step": 3165, "Finetune/Step Time": 4.920227756723762} +{"Finetune/Learning Rate": 6.9976458879733635e-06, "Finetune/Loss": 0.8494378924369812, "Finetune/Loss (Raw)": 0.8227770924568176, "Finetune/Step": 3166, "Finetune/Step Time": 4.92067763209343} +{"Finetune/Learning Rate": 6.991704262519436e-06, "Finetune/Loss": 0.8500471711158752, "Finetune/Loss (Raw)": 0.9179461598396301, "Finetune/Step": 3167, "Finetune/Step Time": 4.918196827173233} +{"Finetune/Learning Rate": 6.985763804526354e-06, "Finetune/Loss": 0.8506877422332764, "Finetune/Loss (Raw)": 1.0158356428146362, "Finetune/Step": 3168, "Finetune/Step Time": 4.923733597621322} +{"Finetune/Learning Rate": 6.979824516299491e-06, "Finetune/Loss": 0.8470807075500488, "Finetune/Loss (Raw)": 0.31580251455307007, "Finetune/Step": 3169, "Finetune/Step Time": 4.956418566405773} +{"Finetune/Learning Rate": 6.973886400143772e-06, "Finetune/Loss": 0.8459576964378357, "Finetune/Loss (Raw)": 0.784321129322052, "Finetune/Step": 3170, "Finetune/Step Time": 4.956866716966033} +{"Finetune/Learning Rate": 6.9679494583636595e-06, "Finetune/Loss": 0.8499382734298706, "Finetune/Loss (Raw)": 0.8186392784118652, "Finetune/Step": 3171, "Finetune/Step Time": 4.92405517026782} +{"Finetune/Learning Rate": 6.9620136932631655e-06, "Finetune/Loss": 0.8512516021728516, "Finetune/Loss (Raw)": 0.8834669589996338, "Finetune/Step": 3172, "Finetune/Step Time": 4.905718460679054} +{"Finetune/Learning Rate": 6.956079107145845e-06, "Finetune/Loss": 0.851104736328125, "Finetune/Loss (Raw)": 0.7939924001693726, "Finetune/Step": 3173, "Finetune/Step Time": 4.917153948917985} +{"Finetune/Learning Rate": 6.950145702314795e-06, "Finetune/Loss": 0.8511470556259155, "Finetune/Loss (Raw)": 1.042259931564331, "Finetune/Step": 3174, "Finetune/Step Time": 4.922636728733778} +{"Finetune/Learning Rate": 6.9442134810726615e-06, "Finetune/Loss": 0.8513585329055786, "Finetune/Loss (Raw)": 0.8873232007026672, "Finetune/Step": 3175, "Finetune/Step Time": 4.91862721927464} +{"Finetune/Learning Rate": 6.938282445721609e-06, "Finetune/Loss": 0.8512992858886719, "Finetune/Loss (Raw)": 1.0376346111297607, "Finetune/Step": 3176, "Finetune/Step Time": 4.919993406161666} +{"Finetune/Learning Rate": 6.932352598563365e-06, "Finetune/Loss": 0.8510646224021912, "Finetune/Loss (Raw)": 0.8559672832489014, "Finetune/Step": 3177, "Finetune/Step Time": 4.923805586993694} +{"Finetune/Learning Rate": 6.92642394189919e-06, "Finetune/Loss": 0.8499928712844849, "Finetune/Loss (Raw)": 0.8601564764976501, "Finetune/Step": 3178, "Finetune/Step Time": 4.92990699224174} +{"Finetune/Learning Rate": 6.920496478029873e-06, "Finetune/Loss": 0.8503822088241577, "Finetune/Loss (Raw)": 1.0769803524017334, "Finetune/Step": 3179, "Finetune/Step Time": 4.935843795537949} +{"Finetune/Learning Rate": 6.914570209255758e-06, "Finetune/Loss": 0.8499538898468018, "Finetune/Loss (Raw)": 0.8206763863563538, "Finetune/Step": 3180, "Finetune/Step Time": 4.931828984990716} +{"Finetune/Learning Rate": 6.908645137876699e-06, "Finetune/Loss": 0.8497252464294434, "Finetune/Loss (Raw)": 0.9470492005348206, "Finetune/Step": 3181, "Finetune/Step Time": 4.92716315574944} +{"Finetune/Learning Rate": 6.902721266192112e-06, "Finetune/Loss": 0.8489664793014526, "Finetune/Loss (Raw)": 0.9162322282791138, "Finetune/Step": 3182, "Finetune/Step Time": 4.930051993578672} +{"Finetune/Learning Rate": 6.896798596500929e-06, "Finetune/Loss": 0.8483848571777344, "Finetune/Loss (Raw)": 0.8545867204666138, "Finetune/Step": 3183, "Finetune/Step Time": 4.929624479264021} +{"Finetune/Learning Rate": 6.8908771311016276e-06, "Finetune/Loss": 0.8478382229804993, "Finetune/Loss (Raw)": 0.9058927893638611, "Finetune/Step": 3184, "Finetune/Step Time": 4.940422208979726} +{"Finetune/Learning Rate": 6.884956872292216e-06, "Finetune/Loss": 0.8488912582397461, "Finetune/Loss (Raw)": 0.9235100746154785, "Finetune/Step": 3185, "Finetune/Step Time": 4.934578092768788} +{"Finetune/Learning Rate": 6.87903782237022e-06, "Finetune/Loss": 0.8480004072189331, "Finetune/Loss (Raw)": 0.852785050868988, "Finetune/Step": 3186, "Finetune/Step Time": 4.938956681638956} +{"Finetune/Learning Rate": 6.8731199836327155e-06, "Finetune/Loss": 0.8480455875396729, "Finetune/Loss (Raw)": 0.9499968886375427, "Finetune/Step": 3187, "Finetune/Step Time": 4.94763215072453} +{"Finetune/Learning Rate": 6.867203358376295e-06, "Finetune/Loss": 0.8471592664718628, "Finetune/Loss (Raw)": 0.8619001507759094, "Finetune/Step": 3188, "Finetune/Step Time": 4.951178828254342} +{"Finetune/Learning Rate": 6.861287948897091e-06, "Finetune/Loss": 0.8463572263717651, "Finetune/Loss (Raw)": 0.8480592370033264, "Finetune/Step": 3189, "Finetune/Step Time": 4.942000053822994} +{"Finetune/Learning Rate": 6.855373757490761e-06, "Finetune/Loss": 0.846835732460022, "Finetune/Loss (Raw)": 0.9069092869758606, "Finetune/Step": 3190, "Finetune/Step Time": 4.912060668691993} +{"Finetune/Learning Rate": 6.849460786452476e-06, "Finetune/Loss": 0.8475892543792725, "Finetune/Loss (Raw)": 0.9617146849632263, "Finetune/Step": 3191, "Finetune/Step Time": 4.938043151050806} +{"Finetune/Learning Rate": 6.843549038076951e-06, "Finetune/Loss": 0.8470757603645325, "Finetune/Loss (Raw)": 0.8423731327056885, "Finetune/Step": 3192, "Finetune/Step Time": 4.943400505930185} +{"Finetune/Learning Rate": 6.83763851465842e-06, "Finetune/Loss": 0.8475133180618286, "Finetune/Loss (Raw)": 0.8588689565658569, "Finetune/Step": 3193, "Finetune/Step Time": 4.9281849805265665} +{"Finetune/Learning Rate": 6.831729218490645e-06, "Finetune/Loss": 0.8481791615486145, "Finetune/Loss (Raw)": 0.7750130295753479, "Finetune/Step": 3194, "Finetune/Step Time": 4.900023501366377} +{"Finetune/Learning Rate": 6.825821151866903e-06, "Finetune/Loss": 0.8488102555274963, "Finetune/Loss (Raw)": 0.9643696546554565, "Finetune/Step": 3195, "Finetune/Step Time": 4.912135008722544} +{"Finetune/Learning Rate": 6.819914317079999e-06, "Finetune/Loss": 0.8491249680519104, "Finetune/Loss (Raw)": 0.9196471571922302, "Finetune/Step": 3196, "Finetune/Step Time": 4.901249900460243} +{"Finetune/Learning Rate": 6.814008716422265e-06, "Finetune/Loss": 0.8506858944892883, "Finetune/Loss (Raw)": 0.992772102355957, "Finetune/Step": 3197, "Finetune/Step Time": 4.892838954925537} +{"Finetune/Learning Rate": 6.808104352185544e-06, "Finetune/Loss": 0.8495205640792847, "Finetune/Loss (Raw)": 0.7835406064987183, "Finetune/Step": 3198, "Finetune/Step Time": 4.901613486930728} +{"Finetune/Learning Rate": 6.8022012266612116e-06, "Finetune/Loss": 0.8487045764923096, "Finetune/Loss (Raw)": 0.906634509563446, "Finetune/Step": 3199, "Finetune/Step Time": 4.90122614055872} +{"Finetune/Learning Rate": 6.79629934214015e-06, "Finetune/Loss": 0.8503307104110718, "Finetune/Loss (Raw)": 1.0707803964614868, "Finetune/Step": 3200, "Finetune/Step Time": 4.89124801568687} +{"Finetune/Learning Rate": 6.790398700912764e-06, "Finetune/Loss": 0.8495317697525024, "Finetune/Loss (Raw)": 0.7899458408355713, "Finetune/Step": 3201, "Finetune/Step Time": 4.892758449539542} +{"Finetune/Learning Rate": 6.784499305268983e-06, "Finetune/Loss": 0.8494614958763123, "Finetune/Loss (Raw)": 0.8511778116226196, "Finetune/Step": 3202, "Finetune/Step Time": 4.88937509059906} +{"Finetune/Learning Rate": 6.778601157498241e-06, "Finetune/Loss": 0.8537872433662415, "Finetune/Loss (Raw)": 0.8477776646614075, "Finetune/Step": 3203, "Finetune/Step Time": 4.852557931095362} +{"Finetune/Learning Rate": 6.772704259889503e-06, "Finetune/Loss": 0.8544119596481323, "Finetune/Loss (Raw)": 0.8369606137275696, "Finetune/Step": 3204, "Finetune/Step Time": 4.859478782862425} +{"Finetune/Learning Rate": 6.76680861473123e-06, "Finetune/Loss": 0.8535007834434509, "Finetune/Loss (Raw)": 0.8191074132919312, "Finetune/Step": 3205, "Finetune/Step Time": 4.856171999126673} +{"Finetune/Learning Rate": 6.7609142243114115e-06, "Finetune/Loss": 0.8533110618591309, "Finetune/Loss (Raw)": 0.785041332244873, "Finetune/Step": 3206, "Finetune/Step Time": 4.857367482036352} +{"Finetune/Learning Rate": 6.755021090917545e-06, "Finetune/Loss": 0.8545604348182678, "Finetune/Loss (Raw)": 0.9179531335830688, "Finetune/Step": 3207, "Finetune/Step Time": 4.856896191835403} +{"Finetune/Learning Rate": 6.7491292168366386e-06, "Finetune/Loss": 0.8560578227043152, "Finetune/Loss (Raw)": 0.9555767178535461, "Finetune/Step": 3208, "Finetune/Step Time": 4.8505138251930475} +{"Finetune/Learning Rate": 6.743238604355219e-06, "Finetune/Loss": 0.856397807598114, "Finetune/Loss (Raw)": 0.8592674732208252, "Finetune/Step": 3209, "Finetune/Step Time": 4.853186592459679} +{"Finetune/Learning Rate": 6.737349255759312e-06, "Finetune/Loss": 0.8557282090187073, "Finetune/Loss (Raw)": 0.7885051965713501, "Finetune/Step": 3210, "Finetune/Step Time": 4.85757745616138} +{"Finetune/Learning Rate": 6.731461173334459e-06, "Finetune/Loss": 0.8555957078933716, "Finetune/Loss (Raw)": 0.9295599460601807, "Finetune/Step": 3211, "Finetune/Step Time": 4.840768381953239} +{"Finetune/Learning Rate": 6.725574359365714e-06, "Finetune/Loss": 0.8523211479187012, "Finetune/Loss (Raw)": 0.39730629324913025, "Finetune/Step": 3212, "Finetune/Step Time": 4.868722202256322} +{"Finetune/Learning Rate": 6.7196888161376305e-06, "Finetune/Loss": 0.8513834476470947, "Finetune/Loss (Raw)": 0.9523547291755676, "Finetune/Step": 3213, "Finetune/Step Time": 4.866828568279743} +{"Finetune/Learning Rate": 6.71380454593428e-06, "Finetune/Loss": 0.8510228395462036, "Finetune/Loss (Raw)": 0.9264408349990845, "Finetune/Step": 3214, "Finetune/Step Time": 4.879122534766793} +{"Finetune/Learning Rate": 6.707921551039224e-06, "Finetune/Loss": 0.8501948118209839, "Finetune/Loss (Raw)": 0.7946343421936035, "Finetune/Step": 3215, "Finetune/Step Time": 4.887878211215138} +{"Finetune/Learning Rate": 6.70203983373554e-06, "Finetune/Loss": 0.8504009246826172, "Finetune/Loss (Raw)": 0.7311626076698303, "Finetune/Step": 3216, "Finetune/Step Time": 4.90862075611949} +{"Finetune/Learning Rate": 6.69615939630581e-06, "Finetune/Loss": 0.8501718044281006, "Finetune/Loss (Raw)": 0.9162138104438782, "Finetune/Step": 3217, "Finetune/Step Time": 4.895014680922031} +{"Finetune/Learning Rate": 6.690280241032117e-06, "Finetune/Loss": 0.8508819341659546, "Finetune/Loss (Raw)": 0.9181021451950073, "Finetune/Step": 3218, "Finetune/Step Time": 4.892940742895007} +{"Finetune/Learning Rate": 6.684402370196048e-06, "Finetune/Loss": 0.8505199551582336, "Finetune/Loss (Raw)": 0.7724452614784241, "Finetune/Step": 3219, "Finetune/Step Time": 4.8832292556762695} +{"Finetune/Learning Rate": 6.678525786078684e-06, "Finetune/Loss": 0.8565142154693604, "Finetune/Loss (Raw)": 1.0671354532241821, "Finetune/Step": 3220, "Finetune/Step Time": 4.8460875283926725} +{"Finetune/Learning Rate": 6.672650490960617e-06, "Finetune/Loss": 0.8560956716537476, "Finetune/Loss (Raw)": 0.8253816366195679, "Finetune/Step": 3221, "Finetune/Step Time": 4.849884752184153} +{"Finetune/Learning Rate": 6.666776487121934e-06, "Finetune/Loss": 0.8516026735305786, "Finetune/Loss (Raw)": 0.33495911955833435, "Finetune/Step": 3222, "Finetune/Step Time": 4.884173907339573} +{"Finetune/Learning Rate": 6.660903776842218e-06, "Finetune/Loss": 0.8512901663780212, "Finetune/Loss (Raw)": 0.8443612456321716, "Finetune/Step": 3223, "Finetune/Step Time": 4.88965517282486} +{"Finetune/Learning Rate": 6.655032362400562e-06, "Finetune/Loss": 0.8513464331626892, "Finetune/Loss (Raw)": 0.9156094193458557, "Finetune/Step": 3224, "Finetune/Step Time": 4.890719186514616} +{"Finetune/Learning Rate": 6.649162246075538e-06, "Finetune/Loss": 0.855751633644104, "Finetune/Loss (Raw)": 0.8681796193122864, "Finetune/Step": 3225, "Finetune/Step Time": 4.857812885195017} +{"Finetune/Learning Rate": 6.6432934301452235e-06, "Finetune/Loss": 0.857219934463501, "Finetune/Loss (Raw)": 0.8979594707489014, "Finetune/Step": 3226, "Finetune/Step Time": 4.862248158082366} +{"Finetune/Learning Rate": 6.637425916887198e-06, "Finetune/Loss": 0.856441080570221, "Finetune/Loss (Raw)": 0.8349751234054565, "Finetune/Step": 3227, "Finetune/Step Time": 4.862728813663125} +{"Finetune/Learning Rate": 6.631559708578524e-06, "Finetune/Loss": 0.8548409938812256, "Finetune/Loss (Raw)": 0.865318775177002, "Finetune/Step": 3228, "Finetune/Step Time": 4.860315853729844} +{"Finetune/Learning Rate": 6.625694807495766e-06, "Finetune/Loss": 0.8573482036590576, "Finetune/Loss (Raw)": 0.9706328511238098, "Finetune/Step": 3229, "Finetune/Step Time": 4.844365984201431} +{"Finetune/Learning Rate": 6.619831215914974e-06, "Finetune/Loss": 0.8563429117202759, "Finetune/Loss (Raw)": 0.847181499004364, "Finetune/Step": 3230, "Finetune/Step Time": 4.84706262499094} +{"Finetune/Learning Rate": 6.6139689361116945e-06, "Finetune/Loss": 0.8552154302597046, "Finetune/Loss (Raw)": 0.8764509558677673, "Finetune/Step": 3231, "Finetune/Step Time": 4.859909014776349} +{"Finetune/Learning Rate": 6.608107970360964e-06, "Finetune/Loss": 0.8559406995773315, "Finetune/Loss (Raw)": 1.0128004550933838, "Finetune/Step": 3232, "Finetune/Step Time": 4.86196955293417} +{"Finetune/Learning Rate": 6.602248320937308e-06, "Finetune/Loss": 0.857516884803772, "Finetune/Loss (Raw)": 1.1055982112884521, "Finetune/Step": 3233, "Finetune/Step Time": 4.865047065541148} +{"Finetune/Learning Rate": 6.596389990114748e-06, "Finetune/Loss": 0.8569099307060242, "Finetune/Loss (Raw)": 0.7767780423164368, "Finetune/Step": 3234, "Finetune/Step Time": 4.88692875392735} +{"Finetune/Learning Rate": 6.5905329801667796e-06, "Finetune/Loss": 0.8570120334625244, "Finetune/Loss (Raw)": 1.02200448513031, "Finetune/Step": 3235, "Finetune/Step Time": 4.8915832955390215} +{"Finetune/Learning Rate": 6.584677293366396e-06, "Finetune/Loss": 0.8564237952232361, "Finetune/Loss (Raw)": 0.742097020149231, "Finetune/Step": 3236, "Finetune/Step Time": 4.89855838380754} +{"Finetune/Learning Rate": 6.578822931986078e-06, "Finetune/Loss": 0.8570844531059265, "Finetune/Loss (Raw)": 0.9755164384841919, "Finetune/Step": 3237, "Finetune/Step Time": 4.896691530942917} +{"Finetune/Learning Rate": 6.57296989829779e-06, "Finetune/Loss": 0.8536216020584106, "Finetune/Loss (Raw)": 0.3998122811317444, "Finetune/Step": 3238, "Finetune/Step Time": 4.933827616274357} +{"Finetune/Learning Rate": 6.567118194572972e-06, "Finetune/Loss": 0.8548164963722229, "Finetune/Loss (Raw)": 1.0279649496078491, "Finetune/Step": 3239, "Finetune/Step Time": 4.928734378889203} +{"Finetune/Learning Rate": 6.5612678230825646e-06, "Finetune/Loss": 0.8556541204452515, "Finetune/Loss (Raw)": 1.075751781463623, "Finetune/Step": 3240, "Finetune/Step Time": 4.930030969902873} +{"Finetune/Learning Rate": 6.555418786096976e-06, "Finetune/Loss": 0.8563904762268066, "Finetune/Loss (Raw)": 0.9686698317527771, "Finetune/Step": 3241, "Finetune/Step Time": 4.918635765090585} +{"Finetune/Learning Rate": 6.549571085886111e-06, "Finetune/Loss": 0.8567250370979309, "Finetune/Loss (Raw)": 0.979719340801239, "Finetune/Step": 3242, "Finetune/Step Time": 4.915294464677572} +{"Finetune/Learning Rate": 6.5437247247193445e-06, "Finetune/Loss": 0.8574249744415283, "Finetune/Loss (Raw)": 0.9439566731452942, "Finetune/Step": 3243, "Finetune/Step Time": 4.91427774913609} +{"Finetune/Learning Rate": 6.537879704865531e-06, "Finetune/Loss": 0.8592529296875, "Finetune/Loss (Raw)": 0.9610165953636169, "Finetune/Step": 3244, "Finetune/Step Time": 4.878033792600036} +{"Finetune/Learning Rate": 6.532036028593012e-06, "Finetune/Loss": 0.8600180149078369, "Finetune/Loss (Raw)": 0.9953631162643433, "Finetune/Step": 3245, "Finetune/Step Time": 4.882953630760312} +{"Finetune/Learning Rate": 6.526193698169603e-06, "Finetune/Loss": 0.8619906306266785, "Finetune/Loss (Raw)": 0.727742075920105, "Finetune/Step": 3246, "Finetune/Step Time": 4.850249016657472} +{"Finetune/Learning Rate": 6.520352715862601e-06, "Finetune/Loss": 0.8618563413619995, "Finetune/Loss (Raw)": 0.8552358150482178, "Finetune/Step": 3247, "Finetune/Step Time": 4.849687783047557} +{"Finetune/Learning Rate": 6.514513083938779e-06, "Finetune/Loss": 0.8625912666320801, "Finetune/Loss (Raw)": 0.994907021522522, "Finetune/Step": 3248, "Finetune/Step Time": 4.853230517357588} +{"Finetune/Learning Rate": 6.508674804664374e-06, "Finetune/Loss": 0.864645779132843, "Finetune/Loss (Raw)": 0.9988836646080017, "Finetune/Step": 3249, "Finetune/Step Time": 4.8423450738191605} +{"Finetune/Learning Rate": 6.50283788030512e-06, "Finetune/Loss": 0.8642736077308655, "Finetune/Loss (Raw)": 0.8469052314758301, "Finetune/Step": 3250, "Finetune/Step Time": 4.861082414165139} +{"Finetune/Learning Rate": 6.497002313126204e-06, "Finetune/Loss": 0.864418625831604, "Finetune/Loss (Raw)": 0.8560450077056885, "Finetune/Step": 3251, "Finetune/Step Time": 4.894413994625211} +{"Finetune/Learning Rate": 6.491168105392303e-06, "Finetune/Loss": 0.8647385835647583, "Finetune/Loss (Raw)": 1.029264211654663, "Finetune/Step": 3252, "Finetune/Step Time": 4.889026202261448} +{"Finetune/Learning Rate": 6.485335259367558e-06, "Finetune/Loss": 0.8652663230895996, "Finetune/Loss (Raw)": 0.9863674640655518, "Finetune/Step": 3253, "Finetune/Step Time": 4.891614301130176} +{"Finetune/Learning Rate": 6.4795037773155765e-06, "Finetune/Loss": 0.865095853805542, "Finetune/Loss (Raw)": 0.9461074471473694, "Finetune/Step": 3254, "Finetune/Step Time": 4.891510786488652} +{"Finetune/Learning Rate": 6.473673661499451e-06, "Finetune/Loss": 0.864467203617096, "Finetune/Loss (Raw)": 0.9210842251777649, "Finetune/Step": 3255, "Finetune/Step Time": 4.90295503474772} +{"Finetune/Learning Rate": 6.467844914181727e-06, "Finetune/Loss": 0.8667420148849487, "Finetune/Loss (Raw)": 0.9936450123786926, "Finetune/Step": 3256, "Finetune/Step Time": 4.863907918334007} +{"Finetune/Learning Rate": 6.462017537624437e-06, "Finetune/Loss": 0.8661475777626038, "Finetune/Loss (Raw)": 0.7747684121131897, "Finetune/Step": 3257, "Finetune/Step Time": 4.874359454959631} +{"Finetune/Learning Rate": 6.4561915340890665e-06, "Finetune/Loss": 0.8681654930114746, "Finetune/Loss (Raw)": 1.0709902048110962, "Finetune/Step": 3258, "Finetune/Step Time": 4.862850200384855} +{"Finetune/Learning Rate": 6.450366905836573e-06, "Finetune/Loss": 0.8680711388587952, "Finetune/Loss (Raw)": 0.8460068702697754, "Finetune/Step": 3259, "Finetune/Step Time": 4.8666283544152975} +{"Finetune/Learning Rate": 6.444543655127383e-06, "Finetune/Loss": 0.8723205327987671, "Finetune/Loss (Raw)": 0.9585484266281128, "Finetune/Step": 3260, "Finetune/Step Time": 4.852968165650964} +{"Finetune/Learning Rate": 6.438721784221385e-06, "Finetune/Loss": 0.8714359998703003, "Finetune/Loss (Raw)": 0.8132007122039795, "Finetune/Step": 3261, "Finetune/Step Time": 4.853902334347367} +{"Finetune/Learning Rate": 6.432901295377936e-06, "Finetune/Loss": 0.87237149477005, "Finetune/Loss (Raw)": 0.9851140379905701, "Finetune/Step": 3262, "Finetune/Step Time": 4.840378146618605} +{"Finetune/Learning Rate": 6.427082190855855e-06, "Finetune/Loss": 0.8728979229927063, "Finetune/Loss (Raw)": 0.9964691996574402, "Finetune/Step": 3263, "Finetune/Step Time": 4.8383459858596325} +{"Finetune/Learning Rate": 6.4212644729134155e-06, "Finetune/Loss": 0.8720588684082031, "Finetune/Loss (Raw)": 0.8487208485603333, "Finetune/Step": 3264, "Finetune/Step Time": 4.838798390701413} +{"Finetune/Learning Rate": 6.415448143808367e-06, "Finetune/Loss": 0.8739358186721802, "Finetune/Loss (Raw)": 1.0734927654266357, "Finetune/Step": 3265, "Finetune/Step Time": 4.822259869426489} +{"Finetune/Learning Rate": 6.409633205797911e-06, "Finetune/Loss": 0.8749557137489319, "Finetune/Loss (Raw)": 0.8892530798912048, "Finetune/Step": 3266, "Finetune/Step Time": 4.816129211336374} +{"Finetune/Learning Rate": 6.403819661138712e-06, "Finetune/Loss": 0.8751932382583618, "Finetune/Loss (Raw)": 0.9121830463409424, "Finetune/Step": 3267, "Finetune/Step Time": 4.83035066537559} +{"Finetune/Learning Rate": 6.398007512086897e-06, "Finetune/Loss": 0.8759338855743408, "Finetune/Loss (Raw)": 0.9644775390625, "Finetune/Step": 3268, "Finetune/Step Time": 4.824372185394168} +{"Finetune/Learning Rate": 6.3921967608980375e-06, "Finetune/Loss": 0.876948356628418, "Finetune/Loss (Raw)": 0.9595879316329956, "Finetune/Step": 3269, "Finetune/Step Time": 4.82639348693192} +{"Finetune/Learning Rate": 6.386387409827182e-06, "Finetune/Loss": 0.8768941760063171, "Finetune/Loss (Raw)": 0.8423027396202087, "Finetune/Step": 3270, "Finetune/Step Time": 4.81590330414474} +{"Finetune/Learning Rate": 6.38057946112882e-06, "Finetune/Loss": 0.8782272338867188, "Finetune/Loss (Raw)": 0.8665522933006287, "Finetune/Step": 3271, "Finetune/Step Time": 4.8003347851336} +{"Finetune/Learning Rate": 6.374772917056908e-06, "Finetune/Loss": 0.877906084060669, "Finetune/Loss (Raw)": 0.9511269330978394, "Finetune/Step": 3272, "Finetune/Step Time": 4.8299311976879835} +{"Finetune/Learning Rate": 6.368967779864854e-06, "Finetune/Loss": 0.8764776587486267, "Finetune/Loss (Raw)": 0.8014136552810669, "Finetune/Step": 3273, "Finetune/Step Time": 4.840822072699666} +{"Finetune/Learning Rate": 6.3631640518055085e-06, "Finetune/Loss": 0.871660590171814, "Finetune/Loss (Raw)": 0.3238310217857361, "Finetune/Step": 3274, "Finetune/Step Time": 4.87112757563591} +{"Finetune/Learning Rate": 6.3573617351311945e-06, "Finetune/Loss": 0.8701484203338623, "Finetune/Loss (Raw)": 0.7814929485321045, "Finetune/Step": 3275, "Finetune/Step Time": 4.863728618249297} +{"Finetune/Learning Rate": 6.351560832093673e-06, "Finetune/Loss": 0.8690375089645386, "Finetune/Loss (Raw)": 0.9000346660614014, "Finetune/Step": 3276, "Finetune/Step Time": 4.848728746175766} +{"Finetune/Learning Rate": 6.3457613449441654e-06, "Finetune/Loss": 0.87306809425354, "Finetune/Loss (Raw)": 0.9373898506164551, "Finetune/Step": 3277, "Finetune/Step Time": 4.8113417364656925} +{"Finetune/Learning Rate": 6.3399632759333364e-06, "Finetune/Loss": 0.8727686405181885, "Finetune/Loss (Raw)": 0.8531073927879333, "Finetune/Step": 3278, "Finetune/Step Time": 4.8145886566489935} +{"Finetune/Learning Rate": 6.3341666273113e-06, "Finetune/Loss": 0.8733343482017517, "Finetune/Loss (Raw)": 0.9625200629234314, "Finetune/Step": 3279, "Finetune/Step Time": 4.822104075923562} +{"Finetune/Learning Rate": 6.328371401327629e-06, "Finetune/Loss": 0.8746238946914673, "Finetune/Loss (Raw)": 0.9867346882820129, "Finetune/Step": 3280, "Finetune/Step Time": 4.821121739223599} +{"Finetune/Learning Rate": 6.322577600231332e-06, "Finetune/Loss": 0.8753817081451416, "Finetune/Loss (Raw)": 1.0147525072097778, "Finetune/Step": 3281, "Finetune/Step Time": 4.82306625507772} +{"Finetune/Learning Rate": 6.316785226270878e-06, "Finetune/Loss": 0.8751509189605713, "Finetune/Loss (Raw)": 0.8585823178291321, "Finetune/Step": 3282, "Finetune/Step Time": 4.824177453294396} +{"Finetune/Learning Rate": 6.310994281694167e-06, "Finetune/Loss": 0.8704900145530701, "Finetune/Loss (Raw)": 0.3685697019100189, "Finetune/Step": 3283, "Finetune/Step Time": 4.859804226085544} +{"Finetune/Learning Rate": 6.305204768748551e-06, "Finetune/Loss": 0.8708019256591797, "Finetune/Loss (Raw)": 0.8596923351287842, "Finetune/Step": 3284, "Finetune/Step Time": 4.8574615232646465} +{"Finetune/Learning Rate": 6.299416689680831e-06, "Finetune/Loss": 0.8705471754074097, "Finetune/Loss (Raw)": 0.9758627414703369, "Finetune/Step": 3285, "Finetune/Step Time": 4.851289473474026} +{"Finetune/Learning Rate": 6.293630046737245e-06, "Finetune/Loss": 0.8750297427177429, "Finetune/Loss (Raw)": 0.8592272996902466, "Finetune/Step": 3286, "Finetune/Step Time": 4.8216784950345755} +{"Finetune/Learning Rate": 6.28784484216348e-06, "Finetune/Loss": 0.8754519820213318, "Finetune/Loss (Raw)": 0.9067925214767456, "Finetune/Step": 3287, "Finetune/Step Time": 4.834084454923868} +{"Finetune/Learning Rate": 6.282061078204657e-06, "Finetune/Loss": 0.8762903213500977, "Finetune/Loss (Raw)": 0.9801791906356812, "Finetune/Step": 3288, "Finetune/Step Time": 4.839733850210905} +{"Finetune/Learning Rate": 6.276278757105341e-06, "Finetune/Loss": 0.8755264282226562, "Finetune/Loss (Raw)": 0.8073148727416992, "Finetune/Step": 3289, "Finetune/Step Time": 4.821009933948517} +{"Finetune/Learning Rate": 6.270497881109541e-06, "Finetune/Loss": 0.8796617984771729, "Finetune/Loss (Raw)": 0.9373520612716675, "Finetune/Step": 3290, "Finetune/Step Time": 4.786528678610921} +{"Finetune/Learning Rate": 6.264718452460699e-06, "Finetune/Loss": 0.8775362968444824, "Finetune/Loss (Raw)": 0.7704828977584839, "Finetune/Step": 3291, "Finetune/Step Time": 4.787365837022662} +{"Finetune/Learning Rate": 6.258940473401705e-06, "Finetune/Loss": 0.876737654209137, "Finetune/Loss (Raw)": 0.8413594365119934, "Finetune/Step": 3292, "Finetune/Step Time": 4.7883924432098866} +{"Finetune/Learning Rate": 6.253163946174872e-06, "Finetune/Loss": 0.8786327838897705, "Finetune/Loss (Raw)": 0.9281842708587646, "Finetune/Step": 3293, "Finetune/Step Time": 4.783754987642169} +{"Finetune/Learning Rate": 6.247388873021959e-06, "Finetune/Loss": 0.8787235021591187, "Finetune/Loss (Raw)": 0.8343807458877563, "Finetune/Step": 3294, "Finetune/Step Time": 4.810827702283859} +{"Finetune/Learning Rate": 6.2416152561841635e-06, "Finetune/Loss": 0.8780688643455505, "Finetune/Loss (Raw)": 0.8341582417488098, "Finetune/Step": 3295, "Finetune/Step Time": 4.814050396904349} +{"Finetune/Learning Rate": 6.2358430979021075e-06, "Finetune/Loss": 0.8773698806762695, "Finetune/Loss (Raw)": 0.9263603091239929, "Finetune/Step": 3296, "Finetune/Step Time": 4.815511051565409} +{"Finetune/Learning Rate": 6.230072400415863e-06, "Finetune/Loss": 0.8812770247459412, "Finetune/Loss (Raw)": 0.8159208297729492, "Finetune/Step": 3297, "Finetune/Step Time": 4.800218125805259} +{"Finetune/Learning Rate": 6.224303165964913e-06, "Finetune/Loss": 0.8828531503677368, "Finetune/Loss (Raw)": 0.9860689640045166, "Finetune/Step": 3298, "Finetune/Step Time": 4.798508774489164} +{"Finetune/Learning Rate": 6.21853539678819e-06, "Finetune/Loss": 0.8794655799865723, "Finetune/Loss (Raw)": 0.3850238621234894, "Finetune/Step": 3299, "Finetune/Step Time": 4.832105843350291} +{"Finetune/Learning Rate": 6.212769095124055e-06, "Finetune/Loss": 0.8797730207443237, "Finetune/Loss (Raw)": 0.922813892364502, "Finetune/Step": 3300, "Finetune/Step Time": 4.83470213599503} +{"Finetune/Learning Rate": 6.207004263210292e-06, "Finetune/Loss": 0.8793859481811523, "Finetune/Loss (Raw)": 0.744463324546814, "Finetune/Step": 3301, "Finetune/Step Time": 4.823433559387922} +{"Finetune/Learning Rate": 6.2012409032841294e-06, "Finetune/Loss": 0.8788034915924072, "Finetune/Loss (Raw)": 0.967698872089386, "Finetune/Step": 3302, "Finetune/Step Time": 4.8215943686664104} +{"Finetune/Learning Rate": 6.1954790175822e-06, "Finetune/Loss": 0.8786444067955017, "Finetune/Loss (Raw)": 0.8669586777687073, "Finetune/Step": 3303, "Finetune/Step Time": 4.819625150412321} +{"Finetune/Learning Rate": 6.189718608340587e-06, "Finetune/Loss": 0.8770957589149475, "Finetune/Loss (Raw)": 0.8394057154655457, "Finetune/Step": 3304, "Finetune/Step Time": 4.829598650336266} +{"Finetune/Learning Rate": 6.183959677794795e-06, "Finetune/Loss": 0.8781136274337769, "Finetune/Loss (Raw)": 0.9862651228904724, "Finetune/Step": 3305, "Finetune/Step Time": 4.825558392331004} +{"Finetune/Learning Rate": 6.1782022281797485e-06, "Finetune/Loss": 0.8779321312904358, "Finetune/Loss (Raw)": 0.8369134664535522, "Finetune/Step": 3306, "Finetune/Step Time": 4.8172135557979345} +{"Finetune/Learning Rate": 6.17244626172981e-06, "Finetune/Loss": 0.8775031566619873, "Finetune/Loss (Raw)": 1.0220695734024048, "Finetune/Step": 3307, "Finetune/Step Time": 4.8066176269203424} +{"Finetune/Learning Rate": 6.166691780678743e-06, "Finetune/Loss": 0.8787689208984375, "Finetune/Loss (Raw)": 0.982698380947113, "Finetune/Step": 3308, "Finetune/Step Time": 4.8125618901103735} +{"Finetune/Learning Rate": 6.1609387872597595e-06, "Finetune/Loss": 0.8790370225906372, "Finetune/Loss (Raw)": 0.9813693165779114, "Finetune/Step": 3309, "Finetune/Step Time": 4.823537763208151} +{"Finetune/Learning Rate": 6.155187283705479e-06, "Finetune/Loss": 0.878197431564331, "Finetune/Loss (Raw)": 0.8087654709815979, "Finetune/Step": 3310, "Finetune/Step Time": 4.820388430729508} +{"Finetune/Learning Rate": 6.14943727224795e-06, "Finetune/Loss": 0.8777316212654114, "Finetune/Loss (Raw)": 0.7949608564376831, "Finetune/Step": 3311, "Finetune/Step Time": 4.828684279695153} +{"Finetune/Learning Rate": 6.1436887551186466e-06, "Finetune/Loss": 0.8771658539772034, "Finetune/Loss (Raw)": 0.8334725499153137, "Finetune/Step": 3312, "Finetune/Step Time": 4.822669833898544} +{"Finetune/Learning Rate": 6.137941734548443e-06, "Finetune/Loss": 0.873317539691925, "Finetune/Loss (Raw)": 0.4309310019016266, "Finetune/Step": 3313, "Finetune/Step Time": 4.8520432114601135} +{"Finetune/Learning Rate": 6.132196212767653e-06, "Finetune/Loss": 0.8729245662689209, "Finetune/Loss (Raw)": 0.8024781942367554, "Finetune/Step": 3314, "Finetune/Step Time": 4.848504904657602} +{"Finetune/Learning Rate": 6.126452192005999e-06, "Finetune/Loss": 0.8742097616195679, "Finetune/Loss (Raw)": 1.1145011186599731, "Finetune/Step": 3315, "Finetune/Step Time": 4.83782266266644} +{"Finetune/Learning Rate": 6.120709674492627e-06, "Finetune/Loss": 0.8739542961120605, "Finetune/Loss (Raw)": 0.8292019963264465, "Finetune/Step": 3316, "Finetune/Step Time": 4.82887283526361} +{"Finetune/Learning Rate": 6.114968662456093e-06, "Finetune/Loss": 0.8742750883102417, "Finetune/Loss (Raw)": 0.8891292810440063, "Finetune/Step": 3317, "Finetune/Step Time": 4.830443799495697} +{"Finetune/Learning Rate": 6.10922915812437e-06, "Finetune/Loss": 0.8697953820228577, "Finetune/Loss (Raw)": 0.33350199460983276, "Finetune/Step": 3318, "Finetune/Step Time": 4.861180946230888} +{"Finetune/Learning Rate": 6.103491163724853e-06, "Finetune/Loss": 0.8699188828468323, "Finetune/Loss (Raw)": 0.9775168895721436, "Finetune/Step": 3319, "Finetune/Step Time": 4.841544592753053} +{"Finetune/Learning Rate": 6.09775468148434e-06, "Finetune/Loss": 0.8700374364852905, "Finetune/Loss (Raw)": 0.8575564026832581, "Finetune/Step": 3320, "Finetune/Step Time": 4.839086363092065} +{"Finetune/Learning Rate": 6.092019713629053e-06, "Finetune/Loss": 0.8697624206542969, "Finetune/Loss (Raw)": 0.8236707448959351, "Finetune/Step": 3321, "Finetune/Step Time": 4.838749051094055} +{"Finetune/Learning Rate": 6.086286262384617e-06, "Finetune/Loss": 0.8703300952911377, "Finetune/Loss (Raw)": 0.8476700782775879, "Finetune/Step": 3322, "Finetune/Step Time": 4.837938606739044} +{"Finetune/Learning Rate": 6.080554329976071e-06, "Finetune/Loss": 0.8707268238067627, "Finetune/Loss (Raw)": 1.0151469707489014, "Finetune/Step": 3323, "Finetune/Step Time": 4.839011998847127} +{"Finetune/Learning Rate": 6.074823918627872e-06, "Finetune/Loss": 0.8700872659683228, "Finetune/Loss (Raw)": 0.8377824425697327, "Finetune/Step": 3324, "Finetune/Step Time": 4.852686412632465} +{"Finetune/Learning Rate": 6.069095030563877e-06, "Finetune/Loss": 0.8686420321464539, "Finetune/Loss (Raw)": 0.8077853322029114, "Finetune/Step": 3325, "Finetune/Step Time": 4.852282838895917} +{"Finetune/Learning Rate": 6.063367668007356e-06, "Finetune/Loss": 0.8699191808700562, "Finetune/Loss (Raw)": 0.9470219612121582, "Finetune/Step": 3326, "Finetune/Step Time": 4.843464765697718} +{"Finetune/Learning Rate": 6.057641833180987e-06, "Finetune/Loss": 0.868614673614502, "Finetune/Loss (Raw)": 0.7396445870399475, "Finetune/Step": 3327, "Finetune/Step Time": 4.86131221242249} +{"Finetune/Learning Rate": 6.0519175283068534e-06, "Finetune/Loss": 0.868269681930542, "Finetune/Loss (Raw)": 1.0266255140304565, "Finetune/Step": 3328, "Finetune/Step Time": 4.859974039718509} +{"Finetune/Learning Rate": 6.046194755606448e-06, "Finetune/Loss": 0.8700361251831055, "Finetune/Loss (Raw)": 1.0160554647445679, "Finetune/Step": 3329, "Finetune/Step Time": 4.862566916272044} +{"Finetune/Learning Rate": 6.040473517300665e-06, "Finetune/Loss": 0.8695904612541199, "Finetune/Loss (Raw)": 0.7941274642944336, "Finetune/Step": 3330, "Finetune/Step Time": 4.876068813726306} +{"Finetune/Learning Rate": 6.0347538156098104e-06, "Finetune/Loss": 0.8701820373535156, "Finetune/Loss (Raw)": 0.9234967827796936, "Finetune/Step": 3331, "Finetune/Step Time": 4.890355059877038} +{"Finetune/Learning Rate": 6.0290356527535836e-06, "Finetune/Loss": 0.870553731918335, "Finetune/Loss (Raw)": 0.8845348954200745, "Finetune/Step": 3332, "Finetune/Step Time": 4.889535538852215} +{"Finetune/Learning Rate": 6.023319030951091e-06, "Finetune/Loss": 0.8714527487754822, "Finetune/Loss (Raw)": 0.934186577796936, "Finetune/Step": 3333, "Finetune/Step Time": 4.889356374740601} +{"Finetune/Learning Rate": 6.017603952420846e-06, "Finetune/Loss": 0.8722430467605591, "Finetune/Loss (Raw)": 0.8862011432647705, "Finetune/Step": 3334, "Finetune/Step Time": 4.889804372563958} +{"Finetune/Learning Rate": 6.011890419380756e-06, "Finetune/Loss": 0.8716069459915161, "Finetune/Loss (Raw)": 0.8365317583084106, "Finetune/Step": 3335, "Finetune/Step Time": 4.867923894897103} +{"Finetune/Learning Rate": 6.0061784340481366e-06, "Finetune/Loss": 0.8709145784378052, "Finetune/Loss (Raw)": 0.866960883140564, "Finetune/Step": 3336, "Finetune/Step Time": 4.866645408794284} +{"Finetune/Learning Rate": 6.000467998639692e-06, "Finetune/Loss": 0.8702924251556396, "Finetune/Loss (Raw)": 0.7796329855918884, "Finetune/Step": 3337, "Finetune/Step Time": 4.870412509888411} +{"Finetune/Learning Rate": 5.994759115371531e-06, "Finetune/Loss": 0.8710782527923584, "Finetune/Loss (Raw)": 0.8890803456306458, "Finetune/Step": 3338, "Finetune/Step Time": 4.868261197581887} +{"Finetune/Learning Rate": 5.989051786459163e-06, "Finetune/Loss": 0.8712127208709717, "Finetune/Loss (Raw)": 0.9467703104019165, "Finetune/Step": 3339, "Finetune/Step Time": 4.860367463901639} +{"Finetune/Learning Rate": 5.98334601411749e-06, "Finetune/Loss": 0.8751461505889893, "Finetune/Loss (Raw)": 0.900791585445404, "Finetune/Step": 3340, "Finetune/Step Time": 4.8324487041682005} +{"Finetune/Learning Rate": 5.977641800560811e-06, "Finetune/Loss": 0.875800371170044, "Finetune/Loss (Raw)": 1.03609299659729, "Finetune/Step": 3341, "Finetune/Step Time": 4.8386987410485744} +{"Finetune/Learning Rate": 5.971939148002819e-06, "Finetune/Loss": 0.8763599395751953, "Finetune/Loss (Raw)": 0.9980621337890625, "Finetune/Step": 3342, "Finetune/Step Time": 4.819850767031312} +{"Finetune/Learning Rate": 5.966238058656601e-06, "Finetune/Loss": 0.8774096965789795, "Finetune/Loss (Raw)": 0.9289994835853577, "Finetune/Step": 3343, "Finetune/Step Time": 4.807399764657021} +{"Finetune/Learning Rate": 5.960538534734641e-06, "Finetune/Loss": 0.877449095249176, "Finetune/Loss (Raw)": 0.7362114191055298, "Finetune/Step": 3344, "Finetune/Step Time": 4.7876820135861635} +{"Finetune/Learning Rate": 5.9548405784488095e-06, "Finetune/Loss": 0.8730467557907104, "Finetune/Loss (Raw)": 0.3527151942253113, "Finetune/Step": 3345, "Finetune/Step Time": 4.827600764110684} +{"Finetune/Learning Rate": 5.949144192010378e-06, "Finetune/Loss": 0.8738155364990234, "Finetune/Loss (Raw)": 1.0165038108825684, "Finetune/Step": 3346, "Finetune/Step Time": 4.826354207471013} +{"Finetune/Learning Rate": 5.94344937763e-06, "Finetune/Loss": 0.8737295866012573, "Finetune/Loss (Raw)": 0.7614454627037048, "Finetune/Step": 3347, "Finetune/Step Time": 4.821409301832318} +{"Finetune/Learning Rate": 5.9377561375177164e-06, "Finetune/Loss": 0.8726674914360046, "Finetune/Loss (Raw)": 0.9311881065368652, "Finetune/Step": 3348, "Finetune/Step Time": 4.8261886686086655} +{"Finetune/Learning Rate": 5.9320644738829706e-06, "Finetune/Loss": 0.8735747337341309, "Finetune/Loss (Raw)": 0.9415014982223511, "Finetune/Step": 3349, "Finetune/Step Time": 4.822640243917704} +{"Finetune/Learning Rate": 5.926374388934579e-06, "Finetune/Loss": 0.8792781829833984, "Finetune/Loss (Raw)": 1.0650076866149902, "Finetune/Step": 3350, "Finetune/Step Time": 4.793605905026197} +{"Finetune/Learning Rate": 5.920685884880762e-06, "Finetune/Loss": 0.879272997379303, "Finetune/Loss (Raw)": 0.8436947464942932, "Finetune/Step": 3351, "Finetune/Step Time": 4.806260729208589} +{"Finetune/Learning Rate": 5.914998963929109e-06, "Finetune/Loss": 0.8794236183166504, "Finetune/Loss (Raw)": 0.9348865151405334, "Finetune/Step": 3352, "Finetune/Step Time": 4.81011831946671} +{"Finetune/Learning Rate": 5.9093136282866014e-06, "Finetune/Loss": 0.8802388906478882, "Finetune/Loss (Raw)": 0.9725388288497925, "Finetune/Step": 3353, "Finetune/Step Time": 4.8322010058909655} +{"Finetune/Learning Rate": 5.903629880159614e-06, "Finetune/Loss": 0.8804645538330078, "Finetune/Loss (Raw)": 0.9268451929092407, "Finetune/Step": 3354, "Finetune/Step Time": 4.824719486758113} +{"Finetune/Learning Rate": 5.897947721753894e-06, "Finetune/Loss": 0.8814740180969238, "Finetune/Loss (Raw)": 0.9641842246055603, "Finetune/Step": 3355, "Finetune/Step Time": 4.816415289416909} +{"Finetune/Learning Rate": 5.89226715527458e-06, "Finetune/Loss": 0.8820192217826843, "Finetune/Loss (Raw)": 0.935103178024292, "Finetune/Step": 3356, "Finetune/Step Time": 4.8186546210199594} +{"Finetune/Learning Rate": 5.886588182926185e-06, "Finetune/Loss": 0.8812023997306824, "Finetune/Loss (Raw)": 0.8660821914672852, "Finetune/Step": 3357, "Finetune/Step Time": 4.819775119423866} +{"Finetune/Learning Rate": 5.880910806912607e-06, "Finetune/Loss": 0.8809354901313782, "Finetune/Loss (Raw)": 0.8130205273628235, "Finetune/Step": 3358, "Finetune/Step Time": 4.813453923910856} +{"Finetune/Learning Rate": 5.8752350294371274e-06, "Finetune/Loss": 0.8823217153549194, "Finetune/Loss (Raw)": 1.0538840293884277, "Finetune/Step": 3359, "Finetune/Step Time": 4.786735154688358} +{"Finetune/Learning Rate": 5.8695608527024075e-06, "Finetune/Loss": 0.8821431994438171, "Finetune/Loss (Raw)": 0.9899510741233826, "Finetune/Step": 3360, "Finetune/Step Time": 4.787607487291098} +{"Finetune/Learning Rate": 5.863888278910475e-06, "Finetune/Loss": 0.8809176087379456, "Finetune/Loss (Raw)": 0.9487206935882568, "Finetune/Step": 3361, "Finetune/Step Time": 4.801130689680576} +{"Finetune/Learning Rate": 5.8582173102627526e-06, "Finetune/Loss": 0.881279468536377, "Finetune/Loss (Raw)": 0.8230977654457092, "Finetune/Step": 3362, "Finetune/Step Time": 4.783859014511108} +{"Finetune/Learning Rate": 5.852547948960029e-06, "Finetune/Loss": 0.8806092739105225, "Finetune/Loss (Raw)": 0.9362145066261292, "Finetune/Step": 3363, "Finetune/Step Time": 4.783813647925854} +{"Finetune/Learning Rate": 5.8468801972024755e-06, "Finetune/Loss": 0.8823596835136414, "Finetune/Loss (Raw)": 0.9661535620689392, "Finetune/Step": 3364, "Finetune/Step Time": 4.775622697547078} +{"Finetune/Learning Rate": 5.841214057189638e-06, "Finetune/Loss": 0.8815286159515381, "Finetune/Loss (Raw)": 0.8691380023956299, "Finetune/Step": 3365, "Finetune/Step Time": 4.789501130580902} +{"Finetune/Learning Rate": 5.8355495311204266e-06, "Finetune/Loss": 0.8847808837890625, "Finetune/Loss (Raw)": 0.8161075711250305, "Finetune/Step": 3366, "Finetune/Step Time": 4.765394434332848} +{"Finetune/Learning Rate": 5.829886621193141e-06, "Finetune/Loss": 0.8830713033676147, "Finetune/Loss (Raw)": 0.8091322183609009, "Finetune/Step": 3367, "Finetune/Step Time": 4.762662302702665} +{"Finetune/Learning Rate": 5.824225329605442e-06, "Finetune/Loss": 0.8780730962753296, "Finetune/Loss (Raw)": 0.4359801411628723, "Finetune/Step": 3368, "Finetune/Step Time": 4.797015704214573} +{"Finetune/Learning Rate": 5.818565658554364e-06, "Finetune/Loss": 0.8781425356864929, "Finetune/Loss (Raw)": 0.9775675535202026, "Finetune/Step": 3369, "Finetune/Step Time": 4.79574172757566} +{"Finetune/Learning Rate": 5.8129076102363266e-06, "Finetune/Loss": 0.8780360221862793, "Finetune/Loss (Raw)": 0.9660837054252625, "Finetune/Step": 3370, "Finetune/Step Time": 4.7932710107415915} +{"Finetune/Learning Rate": 5.807251186847095e-06, "Finetune/Loss": 0.876038134098053, "Finetune/Loss (Raw)": 0.6882227063179016, "Finetune/Step": 3371, "Finetune/Step Time": 4.815193837508559} +{"Finetune/Learning Rate": 5.801596390581817e-06, "Finetune/Loss": 0.8753038644790649, "Finetune/Loss (Raw)": 0.8670319318771362, "Finetune/Step": 3372, "Finetune/Step Time": 4.826361116021872} +{"Finetune/Learning Rate": 5.795943223635018e-06, "Finetune/Loss": 0.8749434947967529, "Finetune/Loss (Raw)": 0.9492340683937073, "Finetune/Step": 3373, "Finetune/Step Time": 4.832233339548111} +{"Finetune/Learning Rate": 5.7902916882005755e-06, "Finetune/Loss": 0.8762860298156738, "Finetune/Loss (Raw)": 0.8995932340621948, "Finetune/Step": 3374, "Finetune/Step Time": 4.835070982575417} +{"Finetune/Learning Rate": 5.784641786471742e-06, "Finetune/Loss": 0.8784335851669312, "Finetune/Loss (Raw)": 1.1301177740097046, "Finetune/Step": 3375, "Finetune/Step Time": 4.83290315233171} +{"Finetune/Learning Rate": 5.778993520641132e-06, "Finetune/Loss": 0.8785973191261292, "Finetune/Loss (Raw)": 1.0158655643463135, "Finetune/Step": 3376, "Finetune/Step Time": 4.825267147272825} +{"Finetune/Learning Rate": 5.7733468929007294e-06, "Finetune/Loss": 0.8771289587020874, "Finetune/Loss (Raw)": 0.8109347224235535, "Finetune/Step": 3377, "Finetune/Step Time": 4.836295505985618} +{"Finetune/Learning Rate": 5.7677019054418805e-06, "Finetune/Loss": 0.8781535625457764, "Finetune/Loss (Raw)": 0.978055477142334, "Finetune/Step": 3378, "Finetune/Step Time": 4.837748758494854} +{"Finetune/Learning Rate": 5.762058560455287e-06, "Finetune/Loss": 0.8769792318344116, "Finetune/Loss (Raw)": 0.705731213092804, "Finetune/Step": 3379, "Finetune/Step Time": 4.8066097646951675} +{"Finetune/Learning Rate": 5.756416860131036e-06, "Finetune/Loss": 0.8750857710838318, "Finetune/Loss (Raw)": 0.7868977189064026, "Finetune/Step": 3380, "Finetune/Step Time": 4.804972413927317} +{"Finetune/Learning Rate": 5.750776806658549e-06, "Finetune/Loss": 0.8742331862449646, "Finetune/Loss (Raw)": 0.8772372007369995, "Finetune/Step": 3381, "Finetune/Step Time": 4.793337760493159} +{"Finetune/Learning Rate": 5.745138402226621e-06, "Finetune/Loss": 0.8749843835830688, "Finetune/Loss (Raw)": 1.042264699935913, "Finetune/Step": 3382, "Finetune/Step Time": 4.795190995559096} +{"Finetune/Learning Rate": 5.739501649023413e-06, "Finetune/Loss": 0.874422550201416, "Finetune/Loss (Raw)": 0.8491706252098083, "Finetune/Step": 3383, "Finetune/Step Time": 4.782729649916291} +{"Finetune/Learning Rate": 5.7338665492364365e-06, "Finetune/Loss": 0.8740322589874268, "Finetune/Loss (Raw)": 0.9436805844306946, "Finetune/Step": 3384, "Finetune/Step Time": 4.778125641867518} +{"Finetune/Learning Rate": 5.728233105052563e-06, "Finetune/Loss": 0.8740905523300171, "Finetune/Loss (Raw)": 0.7822391390800476, "Finetune/Step": 3385, "Finetune/Step Time": 4.766746399924159} +{"Finetune/Learning Rate": 5.722601318658021e-06, "Finetune/Loss": 0.8726390600204468, "Finetune/Loss (Raw)": 0.8851867914199829, "Finetune/Step": 3386, "Finetune/Step Time": 4.7589196190238} +{"Finetune/Learning Rate": 5.7169711922383984e-06, "Finetune/Loss": 0.8722379207611084, "Finetune/Loss (Raw)": 0.7946642637252808, "Finetune/Step": 3387, "Finetune/Step Time": 4.752844303846359} +{"Finetune/Learning Rate": 5.711342727978637e-06, "Finetune/Loss": 0.871127724647522, "Finetune/Loss (Raw)": 0.8164464235305786, "Finetune/Step": 3388, "Finetune/Step Time": 4.730132417753339} +{"Finetune/Learning Rate": 5.7057159280630316e-06, "Finetune/Loss": 0.872050404548645, "Finetune/Loss (Raw)": 0.931302011013031, "Finetune/Step": 3389, "Finetune/Step Time": 4.734746037051082} +{"Finetune/Learning Rate": 5.700090794675241e-06, "Finetune/Loss": 0.8717861175537109, "Finetune/Loss (Raw)": 0.9512861371040344, "Finetune/Step": 3390, "Finetune/Step Time": 4.739584516733885} +{"Finetune/Learning Rate": 5.694467329998264e-06, "Finetune/Loss": 0.8722864985466003, "Finetune/Loss (Raw)": 1.0605159997940063, "Finetune/Step": 3391, "Finetune/Step Time": 4.73642323538661} +{"Finetune/Learning Rate": 5.68884553621445e-06, "Finetune/Loss": 0.8723076581954956, "Finetune/Loss (Raw)": 0.8514267206192017, "Finetune/Step": 3392, "Finetune/Step Time": 4.7309007085859776} +{"Finetune/Learning Rate": 5.683225415505521e-06, "Finetune/Loss": 0.871674656867981, "Finetune/Loss (Raw)": 0.9924772381782532, "Finetune/Step": 3393, "Finetune/Step Time": 4.727536531165242} +{"Finetune/Learning Rate": 5.67760697005253e-06, "Finetune/Loss": 0.871738612651825, "Finetune/Loss (Raw)": 0.8974348902702332, "Finetune/Step": 3394, "Finetune/Step Time": 4.753215854987502} +{"Finetune/Learning Rate": 5.671990202035886e-06, "Finetune/Loss": 0.8723322153091431, "Finetune/Loss (Raw)": 0.9881662726402283, "Finetune/Step": 3395, "Finetune/Step Time": 4.736234391108155} +{"Finetune/Learning Rate": 5.666375113635347e-06, "Finetune/Loss": 0.8707496523857117, "Finetune/Loss (Raw)": 0.7619045972824097, "Finetune/Step": 3396, "Finetune/Step Time": 4.736023038625717} +{"Finetune/Learning Rate": 5.660761707030022e-06, "Finetune/Loss": 0.8701694011688232, "Finetune/Loss (Raw)": 0.885321855545044, "Finetune/Step": 3397, "Finetune/Step Time": 4.734131045639515} +{"Finetune/Learning Rate": 5.655149984398359e-06, "Finetune/Loss": 0.8704828023910522, "Finetune/Loss (Raw)": 0.8824155926704407, "Finetune/Step": 3398, "Finetune/Step Time": 4.733057634904981} +{"Finetune/Learning Rate": 5.649539947918159e-06, "Finetune/Loss": 0.8701585531234741, "Finetune/Loss (Raw)": 0.825050413608551, "Finetune/Step": 3399, "Finetune/Step Time": 4.741489082574844} +{"Finetune/Learning Rate": 5.643931599766577e-06, "Finetune/Loss": 0.8693364262580872, "Finetune/Loss (Raw)": 0.8458906412124634, "Finetune/Step": 3400, "Finetune/Step Time": 4.714383147656918} +{"Finetune/Learning Rate": 5.638324942120091e-06, "Finetune/Loss": 0.8693821430206299, "Finetune/Loss (Raw)": 0.8072742819786072, "Finetune/Step": 3401, "Finetune/Step Time": 4.706541636958718} +{"Finetune/Learning Rate": 5.632719977154537e-06, "Finetune/Loss": 0.8737432956695557, "Finetune/Loss (Raw)": 0.882053554058075, "Finetune/Step": 3402, "Finetune/Step Time": 4.6703962199389935} +{"Finetune/Learning Rate": 5.6271167070451016e-06, "Finetune/Loss": 0.8740665912628174, "Finetune/Loss (Raw)": 0.8228691220283508, "Finetune/Step": 3403, "Finetune/Step Time": 4.6659828666597605} +{"Finetune/Learning Rate": 5.621515133966301e-06, "Finetune/Loss": 0.874049961566925, "Finetune/Loss (Raw)": 0.8979072570800781, "Finetune/Step": 3404, "Finetune/Step Time": 4.6626746989786625} +{"Finetune/Learning Rate": 5.615915260091987e-06, "Finetune/Loss": 0.872474730014801, "Finetune/Loss (Raw)": 0.7357655167579651, "Finetune/Step": 3405, "Finetune/Step Time": 4.741717459633946} +{"Finetune/Learning Rate": 5.610317087595372e-06, "Finetune/Loss": 0.8729928731918335, "Finetune/Loss (Raw)": 0.9194290041923523, "Finetune/Step": 3406, "Finetune/Step Time": 4.738769628107548} +{"Finetune/Learning Rate": 5.604720618648993e-06, "Finetune/Loss": 0.8725050687789917, "Finetune/Loss (Raw)": 0.9000762701034546, "Finetune/Step": 3407, "Finetune/Step Time": 4.720774473622441} +{"Finetune/Learning Rate": 5.599125855424734e-06, "Finetune/Loss": 0.8721541166305542, "Finetune/Loss (Raw)": 0.9418171048164368, "Finetune/Step": 3408, "Finetune/Step Time": 4.721562480553985} +{"Finetune/Learning Rate": 5.593532800093808e-06, "Finetune/Loss": 0.8711004257202148, "Finetune/Loss (Raw)": 0.8798761963844299, "Finetune/Step": 3409, "Finetune/Step Time": 4.730554912239313} +{"Finetune/Learning Rate": 5.5879414548267754e-06, "Finetune/Loss": 0.8708193302154541, "Finetune/Loss (Raw)": 0.8225978016853333, "Finetune/Step": 3410, "Finetune/Step Time": 4.734380729496479} +{"Finetune/Learning Rate": 5.582351821793525e-06, "Finetune/Loss": 0.8757710456848145, "Finetune/Loss (Raw)": 1.002392292022705, "Finetune/Step": 3411, "Finetune/Step Time": 4.698420567438006} +{"Finetune/Learning Rate": 5.576763903163281e-06, "Finetune/Loss": 0.8764019012451172, "Finetune/Loss (Raw)": 0.9404409527778625, "Finetune/Step": 3412, "Finetune/Step Time": 4.693663761019707} +{"Finetune/Learning Rate": 5.571177701104614e-06, "Finetune/Loss": 0.8767374157905579, "Finetune/Loss (Raw)": 1.018813967704773, "Finetune/Step": 3413, "Finetune/Step Time": 4.706512466073036} +{"Finetune/Learning Rate": 5.56559321778542e-06, "Finetune/Loss": 0.8780531883239746, "Finetune/Loss (Raw)": 1.0276495218276978, "Finetune/Step": 3414, "Finetune/Step Time": 4.704749597236514} +{"Finetune/Learning Rate": 5.560010455372915e-06, "Finetune/Loss": 0.8767740726470947, "Finetune/Loss (Raw)": 0.7430623173713684, "Finetune/Step": 3415, "Finetune/Step Time": 4.696908114477992} +{"Finetune/Learning Rate": 5.5544294160336735e-06, "Finetune/Loss": 0.8724320530891418, "Finetune/Loss (Raw)": 0.4243999123573303, "Finetune/Step": 3416, "Finetune/Step Time": 4.729694150388241} +{"Finetune/Learning Rate": 5.5488501019335824e-06, "Finetune/Loss": 0.87309730052948, "Finetune/Loss (Raw)": 0.8924630880355835, "Finetune/Step": 3417, "Finetune/Step Time": 4.736246809363365} +{"Finetune/Learning Rate": 5.5432725152378675e-06, "Finetune/Loss": 0.8727239966392517, "Finetune/Loss (Raw)": 0.8895703554153442, "Finetune/Step": 3418, "Finetune/Step Time": 4.747612044215202} +{"Finetune/Learning Rate": 5.537696658111077e-06, "Finetune/Loss": 0.8737728595733643, "Finetune/Loss (Raw)": 0.9047358632087708, "Finetune/Step": 3419, "Finetune/Step Time": 4.7344852685928345} +{"Finetune/Learning Rate": 5.532122532717095e-06, "Finetune/Loss": 0.8721964359283447, "Finetune/Loss (Raw)": 0.6395766139030457, "Finetune/Step": 3420, "Finetune/Step Time": 4.759844174608588} +{"Finetune/Learning Rate": 5.52655014121913e-06, "Finetune/Loss": 0.8721187710762024, "Finetune/Loss (Raw)": 0.918250560760498, "Finetune/Step": 3421, "Finetune/Step Time": 4.768215177580714} +{"Finetune/Learning Rate": 5.5209794857797135e-06, "Finetune/Loss": 0.8723429441452026, "Finetune/Loss (Raw)": 0.8630790114402771, "Finetune/Step": 3422, "Finetune/Step Time": 4.761021668091416} +{"Finetune/Learning Rate": 5.51541056856072e-06, "Finetune/Loss": 0.8721979856491089, "Finetune/Loss (Raw)": 0.8155899047851562, "Finetune/Step": 3423, "Finetune/Step Time": 4.759479638189077} +{"Finetune/Learning Rate": 5.509843391723332e-06, "Finetune/Loss": 0.8677094578742981, "Finetune/Loss (Raw)": 0.3518317937850952, "Finetune/Step": 3424, "Finetune/Step Time": 4.7884665206074715} +{"Finetune/Learning Rate": 5.504277957428053e-06, "Finetune/Loss": 0.8682018518447876, "Finetune/Loss (Raw)": 0.8789558410644531, "Finetune/Step": 3425, "Finetune/Step Time": 4.775691552087665} +{"Finetune/Learning Rate": 5.498714267834731e-06, "Finetune/Loss": 0.8663116693496704, "Finetune/Loss (Raw)": 0.7441165447235107, "Finetune/Step": 3426, "Finetune/Step Time": 4.778122089803219} +{"Finetune/Learning Rate": 5.493152325102523e-06, "Finetune/Loss": 0.8709369897842407, "Finetune/Loss (Raw)": 0.977069079875946, "Finetune/Step": 3427, "Finetune/Step Time": 4.7463330794125795} +{"Finetune/Learning Rate": 5.487592131389907e-06, "Finetune/Loss": 0.8707007765769958, "Finetune/Loss (Raw)": 0.8925778865814209, "Finetune/Step": 3428, "Finetune/Step Time": 4.7545668967068195} +{"Finetune/Learning Rate": 5.482033688854689e-06, "Finetune/Loss": 0.8721078634262085, "Finetune/Loss (Raw)": 0.9245675206184387, "Finetune/Step": 3429, "Finetune/Step Time": 4.751101152971387} +{"Finetune/Learning Rate": 5.47647699965399e-06, "Finetune/Loss": 0.8711336255073547, "Finetune/Loss (Raw)": 0.8429999947547913, "Finetune/Step": 3430, "Finetune/Step Time": 4.745384722948074} +{"Finetune/Learning Rate": 5.470922065944253e-06, "Finetune/Loss": 0.8720294237136841, "Finetune/Loss (Raw)": 0.9816159009933472, "Finetune/Step": 3431, "Finetune/Step Time": 4.749730281531811} +{"Finetune/Learning Rate": 5.465368889881235e-06, "Finetune/Loss": 0.8715054988861084, "Finetune/Loss (Raw)": 0.7723519802093506, "Finetune/Step": 3432, "Finetune/Step Time": 4.75199712254107} +{"Finetune/Learning Rate": 5.459817473620024e-06, "Finetune/Loss": 0.8705611824989319, "Finetune/Loss (Raw)": 0.865387499332428, "Finetune/Step": 3433, "Finetune/Step Time": 4.7698357328772545} +{"Finetune/Learning Rate": 5.454267819315016e-06, "Finetune/Loss": 0.8706929683685303, "Finetune/Loss (Raw)": 0.8537854552268982, "Finetune/Step": 3434, "Finetune/Step Time": 4.794623881578445} +{"Finetune/Learning Rate": 5.448719929119916e-06, "Finetune/Loss": 0.8694308996200562, "Finetune/Loss (Raw)": 0.8605260848999023, "Finetune/Step": 3435, "Finetune/Step Time": 4.80827958881855} +{"Finetune/Learning Rate": 5.44317380518775e-06, "Finetune/Loss": 0.867889940738678, "Finetune/Loss (Raw)": 0.785446286201477, "Finetune/Step": 3436, "Finetune/Step Time": 4.802008772268891} +{"Finetune/Learning Rate": 5.43762944967087e-06, "Finetune/Loss": 0.8672075271606445, "Finetune/Loss (Raw)": 0.8940277099609375, "Finetune/Step": 3437, "Finetune/Step Time": 4.790285367518663} +{"Finetune/Learning Rate": 5.432086864720929e-06, "Finetune/Loss": 0.8672996759414673, "Finetune/Loss (Raw)": 0.8205587863922119, "Finetune/Step": 3438, "Finetune/Step Time": 4.788779255002737} +{"Finetune/Learning Rate": 5.426546052488892e-06, "Finetune/Loss": 0.868671178817749, "Finetune/Loss (Raw)": 0.9705087542533875, "Finetune/Step": 3439, "Finetune/Step Time": 4.793562490493059} +{"Finetune/Learning Rate": 5.421007015125046e-06, "Finetune/Loss": 0.8688383102416992, "Finetune/Loss (Raw)": 0.8548650145530701, "Finetune/Step": 3440, "Finetune/Step Time": 4.795344335958362} +{"Finetune/Learning Rate": 5.415469754778979e-06, "Finetune/Loss": 0.8711400628089905, "Finetune/Loss (Raw)": 0.7255628705024719, "Finetune/Step": 3441, "Finetune/Step Time": 4.7826077491045} +{"Finetune/Learning Rate": 5.409934273599594e-06, "Finetune/Loss": 0.8709676265716553, "Finetune/Loss (Raw)": 0.7803995013237, "Finetune/Step": 3442, "Finetune/Step Time": 4.789950849488378} +{"Finetune/Learning Rate": 5.404400573735105e-06, "Finetune/Loss": 0.8686754107475281, "Finetune/Loss (Raw)": 0.8211001753807068, "Finetune/Step": 3443, "Finetune/Step Time": 4.800253385677934} +{"Finetune/Learning Rate": 5.3988686573330316e-06, "Finetune/Loss": 0.869378387928009, "Finetune/Loss (Raw)": 0.9191854000091553, "Finetune/Step": 3444, "Finetune/Step Time": 4.794059233739972} +{"Finetune/Learning Rate": 5.393338526540202e-06, "Finetune/Loss": 0.8680069446563721, "Finetune/Loss (Raw)": 0.7135828733444214, "Finetune/Step": 3445, "Finetune/Step Time": 4.811960827559233} +{"Finetune/Learning Rate": 5.387810183502749e-06, "Finetune/Loss": 0.8720418810844421, "Finetune/Loss (Raw)": 0.8499746918678284, "Finetune/Step": 3446, "Finetune/Step Time": 4.785948883742094} +{"Finetune/Learning Rate": 5.382283630366123e-06, "Finetune/Loss": 0.8722026348114014, "Finetune/Loss (Raw)": 0.9980897307395935, "Finetune/Step": 3447, "Finetune/Step Time": 4.780542679131031} +{"Finetune/Learning Rate": 5.37675886927507e-06, "Finetune/Loss": 0.8723793029785156, "Finetune/Loss (Raw)": 0.8801683783531189, "Finetune/Step": 3448, "Finetune/Step Time": 4.78092503733933} +{"Finetune/Learning Rate": 5.371235902373632e-06, "Finetune/Loss": 0.8736630082130432, "Finetune/Loss (Raw)": 0.9879874587059021, "Finetune/Step": 3449, "Finetune/Step Time": 4.776142694056034} +{"Finetune/Learning Rate": 5.365714731805174e-06, "Finetune/Loss": 0.8741147518157959, "Finetune/Loss (Raw)": 0.9055041670799255, "Finetune/Step": 3450, "Finetune/Step Time": 4.795562708750367} +{"Finetune/Learning Rate": 5.360195359712353e-06, "Finetune/Loss": 0.8727877140045166, "Finetune/Loss (Raw)": 0.8452786207199097, "Finetune/Step": 3451, "Finetune/Step Time": 4.778722692281008} +{"Finetune/Learning Rate": 5.354677788237126e-06, "Finetune/Loss": 0.8738285303115845, "Finetune/Loss (Raw)": 0.9710073471069336, "Finetune/Step": 3452, "Finetune/Step Time": 4.763003943488002} +{"Finetune/Learning Rate": 5.349162019520759e-06, "Finetune/Loss": 0.8740924596786499, "Finetune/Loss (Raw)": 0.8415683507919312, "Finetune/Step": 3453, "Finetune/Step Time": 4.766316328197718} +{"Finetune/Learning Rate": 5.343648055703812e-06, "Finetune/Loss": 0.8731595277786255, "Finetune/Loss (Raw)": 0.8276070356369019, "Finetune/Step": 3454, "Finetune/Step Time": 4.7748127449303865} +{"Finetune/Learning Rate": 5.338135898926147e-06, "Finetune/Loss": 0.8700525760650635, "Finetune/Loss (Raw)": 0.34195512533187866, "Finetune/Step": 3455, "Finetune/Step Time": 4.795150101184845} +{"Finetune/Learning Rate": 5.33262555132692e-06, "Finetune/Loss": 0.869438111782074, "Finetune/Loss (Raw)": 0.9479721188545227, "Finetune/Step": 3456, "Finetune/Step Time": 4.791867831721902} +{"Finetune/Learning Rate": 5.327117015044599e-06, "Finetune/Loss": 0.8684335350990295, "Finetune/Loss (Raw)": 0.8874741792678833, "Finetune/Step": 3457, "Finetune/Step Time": 4.782198745757341} +{"Finetune/Learning Rate": 5.321610292216937e-06, "Finetune/Loss": 0.8692606687545776, "Finetune/Loss (Raw)": 0.9000028967857361, "Finetune/Step": 3458, "Finetune/Step Time": 4.790424084290862} +{"Finetune/Learning Rate": 5.316105384980974e-06, "Finetune/Loss": 0.8687015771865845, "Finetune/Loss (Raw)": 0.8519273996353149, "Finetune/Step": 3459, "Finetune/Step Time": 4.78578607738018} +{"Finetune/Learning Rate": 5.3106022954730704e-06, "Finetune/Loss": 0.8694518208503723, "Finetune/Loss (Raw)": 0.9805625081062317, "Finetune/Step": 3460, "Finetune/Step Time": 4.782104704529047} +{"Finetune/Learning Rate": 5.3051010258288624e-06, "Finetune/Loss": 0.8647674918174744, "Finetune/Loss (Raw)": 0.33459848165512085, "Finetune/Step": 3461, "Finetune/Step Time": 4.816888935863972} +{"Finetune/Learning Rate": 5.299601578183289e-06, "Finetune/Loss": 0.8651690483093262, "Finetune/Loss (Raw)": 0.9375999569892883, "Finetune/Step": 3462, "Finetune/Step Time": 4.81309380941093} +{"Finetune/Learning Rate": 5.294103954670573e-06, "Finetune/Loss": 0.8653894662857056, "Finetune/Loss (Raw)": 0.8647449016571045, "Finetune/Step": 3463, "Finetune/Step Time": 4.815014101564884} +{"Finetune/Learning Rate": 5.288608157424238e-06, "Finetune/Loss": 0.865901529788971, "Finetune/Loss (Raw)": 0.9325030446052551, "Finetune/Step": 3464, "Finetune/Step Time": 4.816368425264955} +{"Finetune/Learning Rate": 5.283114188577094e-06, "Finetune/Loss": 0.8663351535797119, "Finetune/Loss (Raw)": 0.8351315259933472, "Finetune/Step": 3465, "Finetune/Step Time": 4.811467545107007} +{"Finetune/Learning Rate": 5.277622050261242e-06, "Finetune/Loss": 0.865770161151886, "Finetune/Loss (Raw)": 0.8167679309844971, "Finetune/Step": 3466, "Finetune/Step Time": 4.8149756751954556} +{"Finetune/Learning Rate": 5.272131744608078e-06, "Finetune/Loss": 0.8641461730003357, "Finetune/Loss (Raw)": 0.738895058631897, "Finetune/Step": 3467, "Finetune/Step Time": 4.81373949162662} +{"Finetune/Learning Rate": 5.2666432737482844e-06, "Finetune/Loss": 0.8628048896789551, "Finetune/Loss (Raw)": 0.7291097044944763, "Finetune/Step": 3468, "Finetune/Step Time": 4.816789750009775} +{"Finetune/Learning Rate": 5.261156639811817e-06, "Finetune/Loss": 0.8617626428604126, "Finetune/Loss (Raw)": 0.9026899933815002, "Finetune/Step": 3469, "Finetune/Step Time": 4.8257744666188955} +{"Finetune/Learning Rate": 5.255671844927944e-06, "Finetune/Loss": 0.8605359196662903, "Finetune/Loss (Raw)": 0.8410395383834839, "Finetune/Step": 3470, "Finetune/Step Time": 4.8269679974764585} +{"Finetune/Learning Rate": 5.250188891225203e-06, "Finetune/Loss": 0.8606261014938354, "Finetune/Loss (Raw)": 0.9405490159988403, "Finetune/Step": 3471, "Finetune/Step Time": 4.838567763566971} +{"Finetune/Learning Rate": 5.24470778083142e-06, "Finetune/Loss": 0.8611672520637512, "Finetune/Loss (Raw)": 0.8054711222648621, "Finetune/Step": 3472, "Finetune/Step Time": 4.839080024510622} +{"Finetune/Learning Rate": 5.2392285158737085e-06, "Finetune/Loss": 0.8663296699523926, "Finetune/Loss (Raw)": 1.0135021209716797, "Finetune/Step": 3473, "Finetune/Step Time": 4.828442178666592} +{"Finetune/Learning Rate": 5.233751098478465e-06, "Finetune/Loss": 0.8646007180213928, "Finetune/Loss (Raw)": 0.7952020764350891, "Finetune/Step": 3474, "Finetune/Step Time": 4.833854418247938} +{"Finetune/Learning Rate": 5.228275530771366e-06, "Finetune/Loss": 0.8655440807342529, "Finetune/Loss (Raw)": 0.8821973204612732, "Finetune/Step": 3475, "Finetune/Step Time": 4.840254329144955} +{"Finetune/Learning Rate": 5.22280181487737e-06, "Finetune/Loss": 0.8640804290771484, "Finetune/Loss (Raw)": 0.7438376545906067, "Finetune/Step": 3476, "Finetune/Step Time": 4.845545144751668} +{"Finetune/Learning Rate": 5.217329952920727e-06, "Finetune/Loss": 0.8631260991096497, "Finetune/Loss (Raw)": 0.8193492889404297, "Finetune/Step": 3477, "Finetune/Step Time": 4.863906070590019} +{"Finetune/Learning Rate": 5.2118599470249575e-06, "Finetune/Loss": 0.8625898361206055, "Finetune/Loss (Raw)": 0.9963622689247131, "Finetune/Step": 3478, "Finetune/Step Time": 4.856251120567322} +{"Finetune/Learning Rate": 5.206391799312855e-06, "Finetune/Loss": 0.864253044128418, "Finetune/Loss (Raw)": 1.0565930604934692, "Finetune/Step": 3479, "Finetune/Step Time": 4.837605150416493} +{"Finetune/Learning Rate": 5.200925511906513e-06, "Finetune/Loss": 0.8642035126686096, "Finetune/Loss (Raw)": 0.9285393953323364, "Finetune/Step": 3480, "Finetune/Step Time": 4.82943413965404} +{"Finetune/Learning Rate": 5.195461086927284e-06, "Finetune/Loss": 0.8640365600585938, "Finetune/Loss (Raw)": 0.9511662125587463, "Finetune/Step": 3481, "Finetune/Step Time": 4.806435124948621} +{"Finetune/Learning Rate": 5.189998526495808e-06, "Finetune/Loss": 0.8637783527374268, "Finetune/Loss (Raw)": 0.8937998414039612, "Finetune/Step": 3482, "Finetune/Step Time": 4.806321505457163} +{"Finetune/Learning Rate": 5.184537832731996e-06, "Finetune/Loss": 0.8631302118301392, "Finetune/Loss (Raw)": 0.8812175989151001, "Finetune/Step": 3483, "Finetune/Step Time": 4.849427515640855} +{"Finetune/Learning Rate": 5.1790790077550366e-06, "Finetune/Loss": 0.863028883934021, "Finetune/Loss (Raw)": 0.9221363663673401, "Finetune/Step": 3484, "Finetune/Step Time": 4.866004606708884} +{"Finetune/Learning Rate": 5.173622053683395e-06, "Finetune/Loss": 0.8640217781066895, "Finetune/Loss (Raw)": 0.9931734800338745, "Finetune/Step": 3485, "Finetune/Step Time": 4.862830566242337} +{"Finetune/Learning Rate": 5.168166972634803e-06, "Finetune/Loss": 0.8650226593017578, "Finetune/Loss (Raw)": 0.9411303400993347, "Finetune/Step": 3486, "Finetune/Step Time": 4.862234562635422} +{"Finetune/Learning Rate": 5.1627137667262815e-06, "Finetune/Loss": 0.8627392053604126, "Finetune/Loss (Raw)": 0.7616023421287537, "Finetune/Step": 3487, "Finetune/Step Time": 4.860893042758107} +{"Finetune/Learning Rate": 5.1572624380741044e-06, "Finetune/Loss": 0.8609164953231812, "Finetune/Loss (Raw)": 0.7566471695899963, "Finetune/Step": 3488, "Finetune/Step Time": 4.860331356525421} +{"Finetune/Learning Rate": 5.151812988793825e-06, "Finetune/Loss": 0.8611795902252197, "Finetune/Loss (Raw)": 0.9823911190032959, "Finetune/Step": 3489, "Finetune/Step Time": 4.8473469484597445} +{"Finetune/Learning Rate": 5.146365421000275e-06, "Finetune/Loss": 0.8609610795974731, "Finetune/Loss (Raw)": 0.795138418674469, "Finetune/Step": 3490, "Finetune/Step Time": 4.844959454610944} +{"Finetune/Learning Rate": 5.140919736807546e-06, "Finetune/Loss": 0.8613379001617432, "Finetune/Loss (Raw)": 0.9844381213188171, "Finetune/Step": 3491, "Finetune/Step Time": 4.840549428015947} +{"Finetune/Learning Rate": 5.135475938329002e-06, "Finetune/Loss": 0.8608852028846741, "Finetune/Loss (Raw)": 0.9082158207893372, "Finetune/Step": 3492, "Finetune/Step Time": 4.850326988846064} +{"Finetune/Learning Rate": 5.1300340276772756e-06, "Finetune/Loss": 0.8596835732460022, "Finetune/Loss (Raw)": 0.7153252363204956, "Finetune/Step": 3493, "Finetune/Step Time": 4.842545084655285} +{"Finetune/Learning Rate": 5.124594006964263e-06, "Finetune/Loss": 0.8610901832580566, "Finetune/Loss (Raw)": 0.9961591958999634, "Finetune/Step": 3494, "Finetune/Step Time": 4.829877438023686} +{"Finetune/Learning Rate": 5.1191558783011315e-06, "Finetune/Loss": 0.8620530366897583, "Finetune/Loss (Raw)": 0.9323728680610657, "Finetune/Step": 3495, "Finetune/Step Time": 4.829613162204623} +{"Finetune/Learning Rate": 5.113719643798312e-06, "Finetune/Loss": 0.8660885095596313, "Finetune/Loss (Raw)": 0.9525235295295715, "Finetune/Step": 3496, "Finetune/Step Time": 4.802078504115343} +{"Finetune/Learning Rate": 5.108285305565508e-06, "Finetune/Loss": 0.8645225763320923, "Finetune/Loss (Raw)": 0.7771233916282654, "Finetune/Step": 3497, "Finetune/Step Time": 4.81138570420444} +{"Finetune/Learning Rate": 5.1028528657116715e-06, "Finetune/Loss": 0.864081621170044, "Finetune/Loss (Raw)": 0.9096444845199585, "Finetune/Step": 3498, "Finetune/Step Time": 4.815560940653086} +{"Finetune/Learning Rate": 5.097422326345024e-06, "Finetune/Loss": 0.8662256002426147, "Finetune/Loss (Raw)": 0.9626502990722656, "Finetune/Step": 3499, "Finetune/Step Time": 4.800590446218848} +{"Finetune/Learning Rate": 5.091993689573062e-06, "Finetune/Loss": 0.8668577075004578, "Finetune/Loss (Raw)": 0.9479383230209351, "Finetune/Step": 3500, "Finetune/Step Time": 4.791662998497486} +{"Finetune/Learning Rate": 5.086566957502527e-06, "Finetune/Loss": 0.8667494058609009, "Finetune/Loss (Raw)": 0.9353692531585693, "Finetune/Step": 3501, "Finetune/Step Time": 4.777585444971919} +{"Finetune/Learning Rate": 5.081142132239429e-06, "Finetune/Loss": 0.8679957389831543, "Finetune/Loss (Raw)": 1.0591330528259277, "Finetune/Step": 3502, "Finetune/Step Time": 4.772611627355218} +{"Finetune/Learning Rate": 5.075719215889039e-06, "Finetune/Loss": 0.8666781187057495, "Finetune/Loss (Raw)": 0.9614549875259399, "Finetune/Step": 3503, "Finetune/Step Time": 4.772828904911876} +{"Finetune/Learning Rate": 5.070298210555881e-06, "Finetune/Loss": 0.8653630614280701, "Finetune/Loss (Raw)": 0.8475454449653625, "Finetune/Step": 3504, "Finetune/Step Time": 4.774467539042234} +{"Finetune/Learning Rate": 5.064879118343744e-06, "Finetune/Loss": 0.8664212226867676, "Finetune/Loss (Raw)": 0.9463768601417542, "Finetune/Step": 3505, "Finetune/Step Time": 4.770029732957482} +{"Finetune/Learning Rate": 5.059461941355665e-06, "Finetune/Loss": 0.8652613162994385, "Finetune/Loss (Raw)": 0.8295904994010925, "Finetune/Step": 3506, "Finetune/Step Time": 4.7612876407802105} +{"Finetune/Learning Rate": 5.05404668169396e-06, "Finetune/Loss": 0.8649435043334961, "Finetune/Loss (Raw)": 0.6650432348251343, "Finetune/Step": 3507, "Finetune/Step Time": 4.776336831972003} +{"Finetune/Learning Rate": 5.048633341460174e-06, "Finetune/Loss": 0.8656885027885437, "Finetune/Loss (Raw)": 0.8822646141052246, "Finetune/Step": 3508, "Finetune/Step Time": 4.776517994701862} +{"Finetune/Learning Rate": 5.043221922755115e-06, "Finetune/Loss": 0.8664510250091553, "Finetune/Loss (Raw)": 0.9748315215110779, "Finetune/Step": 3509, "Finetune/Step Time": 4.795353282243013} +{"Finetune/Learning Rate": 5.03781242767886e-06, "Finetune/Loss": 0.8645757436752319, "Finetune/Loss (Raw)": 0.8022319078445435, "Finetune/Step": 3510, "Finetune/Step Time": 4.794063974171877} +{"Finetune/Learning Rate": 5.032404858330721e-06, "Finetune/Loss": 0.8647347688674927, "Finetune/Loss (Raw)": 0.8695305585861206, "Finetune/Step": 3511, "Finetune/Step Time": 4.8014968149363995} +{"Finetune/Learning Rate": 5.026999216809275e-06, "Finetune/Loss": 0.8644615411758423, "Finetune/Loss (Raw)": 0.9087073802947998, "Finetune/Step": 3512, "Finetune/Step Time": 4.7997924368828535} +{"Finetune/Learning Rate": 5.021595505212342e-06, "Finetune/Loss": 0.8657130002975464, "Finetune/Loss (Raw)": 0.9424227476119995, "Finetune/Step": 3513, "Finetune/Step Time": 4.800067877396941} +{"Finetune/Learning Rate": 5.016193725637e-06, "Finetune/Loss": 0.8650943040847778, "Finetune/Loss (Raw)": 0.8059915900230408, "Finetune/Step": 3514, "Finetune/Step Time": 4.803984062746167} +{"Finetune/Learning Rate": 5.0107938801795695e-06, "Finetune/Loss": 0.8655557632446289, "Finetune/Loss (Raw)": 0.8537338972091675, "Finetune/Step": 3515, "Finetune/Step Time": 4.804849393665791} +{"Finetune/Learning Rate": 5.005395970935627e-06, "Finetune/Loss": 0.8660203814506531, "Finetune/Loss (Raw)": 0.8759174942970276, "Finetune/Step": 3516, "Finetune/Step Time": 4.806160736829042} +{"Finetune/Learning Rate": 5.000000000000003e-06, "Finetune/Loss": 0.866162896156311, "Finetune/Loss (Raw)": 0.9495377540588379, "Finetune/Step": 3517, "Finetune/Step Time": 4.803708141669631} +{"Finetune/Learning Rate": 4.994605969466758e-06, "Finetune/Loss": 0.8655812740325928, "Finetune/Loss (Raw)": 0.876844584941864, "Finetune/Step": 3518, "Finetune/Step Time": 4.828868040814996} +{"Finetune/Learning Rate": 4.989213881429211e-06, "Finetune/Loss": 0.8635801076889038, "Finetune/Loss (Raw)": 0.8043665885925293, "Finetune/Step": 3519, "Finetune/Step Time": 4.85260440222919} +{"Finetune/Learning Rate": 4.983823737979933e-06, "Finetune/Loss": 0.8641062378883362, "Finetune/Loss (Raw)": 0.9187705516815186, "Finetune/Step": 3520, "Finetune/Step Time": 4.854066219180822} +{"Finetune/Learning Rate": 4.97843554121073e-06, "Finetune/Loss": 0.8643075227737427, "Finetune/Loss (Raw)": 1.0182443857192993, "Finetune/Step": 3521, "Finetune/Step Time": 4.8599911741912365} +{"Finetune/Learning Rate": 4.973049293212655e-06, "Finetune/Loss": 0.863437831401825, "Finetune/Loss (Raw)": 0.7861133217811584, "Finetune/Step": 3522, "Finetune/Step Time": 4.850633915513754} +{"Finetune/Learning Rate": 4.967664996076007e-06, "Finetune/Loss": 0.8618655800819397, "Finetune/Loss (Raw)": 0.7869192361831665, "Finetune/Step": 3523, "Finetune/Step Time": 4.876112690195441} +{"Finetune/Learning Rate": 4.962282651890325e-06, "Finetune/Loss": 0.8620532751083374, "Finetune/Loss (Raw)": 0.7859322428703308, "Finetune/Step": 3524, "Finetune/Step Time": 4.885629061609507} +{"Finetune/Learning Rate": 4.9569022627443944e-06, "Finetune/Loss": 0.8632643222808838, "Finetune/Loss (Raw)": 1.0403276681900024, "Finetune/Step": 3525, "Finetune/Step Time": 4.889072785153985} +{"Finetune/Learning Rate": 4.951523830726236e-06, "Finetune/Loss": 0.8629747629165649, "Finetune/Loss (Raw)": 0.8453546166419983, "Finetune/Step": 3526, "Finetune/Step Time": 4.897205326706171} +{"Finetune/Learning Rate": 4.946147357923116e-06, "Finetune/Loss": 0.8638410568237305, "Finetune/Loss (Raw)": 0.9359402060508728, "Finetune/Step": 3527, "Finetune/Step Time": 4.885621719062328} +{"Finetune/Learning Rate": 4.940772846421538e-06, "Finetune/Loss": 0.8642071485519409, "Finetune/Loss (Raw)": 0.8927508592605591, "Finetune/Step": 3528, "Finetune/Step Time": 4.884922197088599} +{"Finetune/Learning Rate": 4.935400298307241e-06, "Finetune/Loss": 0.8642277121543884, "Finetune/Loss (Raw)": 0.8099018931388855, "Finetune/Step": 3529, "Finetune/Step Time": 4.895048324018717} +{"Finetune/Learning Rate": 4.930029715665215e-06, "Finetune/Loss": 0.8644300699234009, "Finetune/Loss (Raw)": 0.9079576730728149, "Finetune/Step": 3530, "Finetune/Step Time": 4.897021416574717} +{"Finetune/Learning Rate": 4.9246611005796776e-06, "Finetune/Loss": 0.8655331134796143, "Finetune/Loss (Raw)": 0.9640611410140991, "Finetune/Step": 3531, "Finetune/Step Time": 4.8966535571962595} +{"Finetune/Learning Rate": 4.919294455134071e-06, "Finetune/Loss": 0.8649459481239319, "Finetune/Loss (Raw)": 0.8227490782737732, "Finetune/Step": 3532, "Finetune/Step Time": 4.909400774165988} +{"Finetune/Learning Rate": 4.913929781411099e-06, "Finetune/Loss": 0.8617114424705505, "Finetune/Loss (Raw)": 0.32174474000930786, "Finetune/Step": 3533, "Finetune/Step Time": 4.8682176899164915} +{"Finetune/Learning Rate": 4.90856708149268e-06, "Finetune/Loss": 0.8617454767227173, "Finetune/Loss (Raw)": 0.9237871170043945, "Finetune/Step": 3534, "Finetune/Step Time": 4.8874126728624105} +{"Finetune/Learning Rate": 4.903206357459976e-06, "Finetune/Loss": 0.857358455657959, "Finetune/Loss (Raw)": 0.3385308086872101, "Finetune/Step": 3535, "Finetune/Step Time": 4.922626458108425} +{"Finetune/Learning Rate": 4.897847611393378e-06, "Finetune/Loss": 0.8566067814826965, "Finetune/Loss (Raw)": 0.8456077575683594, "Finetune/Step": 3536, "Finetune/Step Time": 4.919799676164985} +{"Finetune/Learning Rate": 4.892490845372511e-06, "Finetune/Loss": 0.857703685760498, "Finetune/Loss (Raw)": 1.020287036895752, "Finetune/Step": 3537, "Finetune/Step Time": 4.916092814877629} +{"Finetune/Learning Rate": 4.8871360614762295e-06, "Finetune/Loss": 0.8591080904006958, "Finetune/Loss (Raw)": 1.0023560523986816, "Finetune/Step": 3538, "Finetune/Step Time": 4.918519489467144} +{"Finetune/Learning Rate": 4.881783261782621e-06, "Finetune/Loss": 0.8586131930351257, "Finetune/Loss (Raw)": 0.9390449523925781, "Finetune/Step": 3539, "Finetune/Step Time": 4.922498112544417} +{"Finetune/Learning Rate": 4.876432448369006e-06, "Finetune/Loss": 0.858161211013794, "Finetune/Loss (Raw)": 0.8825873732566833, "Finetune/Step": 3540, "Finetune/Step Time": 4.927162347361445} +{"Finetune/Learning Rate": 4.871083623311933e-06, "Finetune/Loss": 0.8571239113807678, "Finetune/Loss (Raw)": 0.8860434889793396, "Finetune/Step": 3541, "Finetune/Step Time": 4.912485059350729} +{"Finetune/Learning Rate": 4.865736788687164e-06, "Finetune/Loss": 0.8558660745620728, "Finetune/Loss (Raw)": 0.8666457533836365, "Finetune/Step": 3542, "Finetune/Step Time": 4.922086035832763} +{"Finetune/Learning Rate": 4.860391946569714e-06, "Finetune/Loss": 0.8568831086158752, "Finetune/Loss (Raw)": 0.8732424378395081, "Finetune/Step": 3543, "Finetune/Step Time": 4.934422692283988} +{"Finetune/Learning Rate": 4.855049099033806e-06, "Finetune/Loss": 0.8607507944107056, "Finetune/Loss (Raw)": 0.9194607734680176, "Finetune/Step": 3544, "Finetune/Step Time": 4.898199835792184} +{"Finetune/Learning Rate": 4.849708248152897e-06, "Finetune/Loss": 0.8570486903190613, "Finetune/Loss (Raw)": 0.4185909032821655, "Finetune/Step": 3545, "Finetune/Step Time": 4.928038187325001} +{"Finetune/Learning Rate": 4.8443693959996655e-06, "Finetune/Loss": 0.8565273880958557, "Finetune/Loss (Raw)": 0.8228464126586914, "Finetune/Step": 3546, "Finetune/Step Time": 4.930214621126652} +{"Finetune/Learning Rate": 4.839032544646016e-06, "Finetune/Loss": 0.8573253750801086, "Finetune/Loss (Raw)": 1.0068800449371338, "Finetune/Step": 3547, "Finetune/Step Time": 4.9274577759206295} +{"Finetune/Learning Rate": 4.833697696163077e-06, "Finetune/Loss": 0.8596881628036499, "Finetune/Loss (Raw)": 0.9420123100280762, "Finetune/Step": 3548, "Finetune/Step Time": 4.915204957127571} +{"Finetune/Learning Rate": 4.828364852621195e-06, "Finetune/Loss": 0.8603037595748901, "Finetune/Loss (Raw)": 0.9970488548278809, "Finetune/Step": 3549, "Finetune/Step Time": 4.908661594614387} +{"Finetune/Learning Rate": 4.823034016089949e-06, "Finetune/Loss": 0.8562437295913696, "Finetune/Loss (Raw)": 0.3433944880962372, "Finetune/Step": 3550, "Finetune/Step Time": 4.923201469704509} +{"Finetune/Learning Rate": 4.8177051886381345e-06, "Finetune/Loss": 0.85597163438797, "Finetune/Loss (Raw)": 0.7807563543319702, "Finetune/Step": 3551, "Finetune/Step Time": 4.941272610798478} +{"Finetune/Learning Rate": 4.812378372333751e-06, "Finetune/Loss": 0.8603163957595825, "Finetune/Loss (Raw)": 0.9079678058624268, "Finetune/Step": 3552, "Finetune/Step Time": 4.903463244438171} +{"Finetune/Learning Rate": 4.807053569244047e-06, "Finetune/Loss": 0.8592619895935059, "Finetune/Loss (Raw)": 0.7439888119697571, "Finetune/Step": 3553, "Finetune/Step Time": 4.9167118687182665} +{"Finetune/Learning Rate": 4.801730781435469e-06, "Finetune/Loss": 0.8610109090805054, "Finetune/Loss (Raw)": 0.9679838418960571, "Finetune/Step": 3554, "Finetune/Step Time": 4.919307380914688} +{"Finetune/Learning Rate": 4.796410010973686e-06, "Finetune/Loss": 0.8616774082183838, "Finetune/Loss (Raw)": 1.062379240989685, "Finetune/Step": 3555, "Finetune/Step Time": 4.917449988424778} +{"Finetune/Learning Rate": 4.791091259923587e-06, "Finetune/Loss": 0.8574409484863281, "Finetune/Loss (Raw)": 0.3503088057041168, "Finetune/Step": 3556, "Finetune/Step Time": 4.944555606693029} +{"Finetune/Learning Rate": 4.785774530349275e-06, "Finetune/Loss": 0.8583076596260071, "Finetune/Loss (Raw)": 1.0355085134506226, "Finetune/Step": 3557, "Finetune/Step Time": 4.942159274592996} +{"Finetune/Learning Rate": 4.7804598243140664e-06, "Finetune/Loss": 0.8586315512657166, "Finetune/Loss (Raw)": 0.8844549059867859, "Finetune/Step": 3558, "Finetune/Step Time": 4.9440676383674145} +{"Finetune/Learning Rate": 4.775147143880496e-06, "Finetune/Loss": 0.8584752082824707, "Finetune/Loss (Raw)": 0.9616104960441589, "Finetune/Step": 3559, "Finetune/Step Time": 4.942644773051143} +{"Finetune/Learning Rate": 4.769836491110314e-06, "Finetune/Loss": 0.8602949380874634, "Finetune/Loss (Raw)": 1.0052744150161743, "Finetune/Step": 3560, "Finetune/Step Time": 4.926398567855358} +{"Finetune/Learning Rate": 4.764527868064484e-06, "Finetune/Loss": 0.8616762161254883, "Finetune/Loss (Raw)": 1.0421841144561768, "Finetune/Step": 3561, "Finetune/Step Time": 4.909884560853243} +{"Finetune/Learning Rate": 4.759221276803172e-06, "Finetune/Loss": 0.8607207536697388, "Finetune/Loss (Raw)": 0.7314882278442383, "Finetune/Step": 3562, "Finetune/Step Time": 4.885027728974819} +{"Finetune/Learning Rate": 4.75391671938576e-06, "Finetune/Loss": 0.8600037097930908, "Finetune/Loss (Raw)": 0.7687456011772156, "Finetune/Step": 3563, "Finetune/Step Time": 4.871258899569511} +{"Finetune/Learning Rate": 4.748614197870852e-06, "Finetune/Loss": 0.8602684736251831, "Finetune/Loss (Raw)": 0.8193431496620178, "Finetune/Step": 3564, "Finetune/Step Time": 4.873947784304619} +{"Finetune/Learning Rate": 4.743313714316249e-06, "Finetune/Loss": 0.8598775863647461, "Finetune/Loss (Raw)": 0.8439849615097046, "Finetune/Step": 3565, "Finetune/Step Time": 4.877021646127105} +{"Finetune/Learning Rate": 4.738015270778965e-06, "Finetune/Loss": 0.8613651990890503, "Finetune/Loss (Raw)": 1.0109765529632568, "Finetune/Step": 3566, "Finetune/Step Time": 4.876635042950511} +{"Finetune/Learning Rate": 4.7327188693152235e-06, "Finetune/Loss": 0.8607099056243896, "Finetune/Loss (Raw)": 0.8866250514984131, "Finetune/Step": 3567, "Finetune/Step Time": 4.880594985559583} +{"Finetune/Learning Rate": 4.727424511980453e-06, "Finetune/Loss": 0.8613641858100891, "Finetune/Loss (Raw)": 0.9386213421821594, "Finetune/Step": 3568, "Finetune/Step Time": 4.885745005682111} +{"Finetune/Learning Rate": 4.722132200829291e-06, "Finetune/Loss": 0.8624469637870789, "Finetune/Loss (Raw)": 0.8641563653945923, "Finetune/Step": 3569, "Finetune/Step Time": 4.873114865273237} +{"Finetune/Learning Rate": 4.716841937915582e-06, "Finetune/Loss": 0.8628475666046143, "Finetune/Loss (Raw)": 0.8316724896430969, "Finetune/Step": 3570, "Finetune/Step Time": 4.867601538076997} +{"Finetune/Learning Rate": 4.71155372529237e-06, "Finetune/Loss": 0.8637118339538574, "Finetune/Loss (Raw)": 0.9317291975021362, "Finetune/Step": 3571, "Finetune/Step Time": 4.859588705003262} +{"Finetune/Learning Rate": 4.706267565011909e-06, "Finetune/Loss": 0.8638993501663208, "Finetune/Loss (Raw)": 0.9431862831115723, "Finetune/Step": 3572, "Finetune/Step Time": 4.861441569402814} +{"Finetune/Learning Rate": 4.700983459125652e-06, "Finetune/Loss": 0.8656668663024902, "Finetune/Loss (Raw)": 0.9398257732391357, "Finetune/Step": 3573, "Finetune/Step Time": 4.843341648578644} +{"Finetune/Learning Rate": 4.695701409684264e-06, "Finetune/Loss": 0.8666625022888184, "Finetune/Loss (Raw)": 0.9774190187454224, "Finetune/Step": 3574, "Finetune/Step Time": 4.8412545043975115} +{"Finetune/Learning Rate": 4.690421418737604e-06, "Finetune/Loss": 0.8656280040740967, "Finetune/Loss (Raw)": 0.8656667470932007, "Finetune/Step": 3575, "Finetune/Step Time": 4.842309955507517} +{"Finetune/Learning Rate": 4.685143488334723e-06, "Finetune/Loss": 0.8664199113845825, "Finetune/Loss (Raw)": 0.9815335273742676, "Finetune/Step": 3576, "Finetune/Step Time": 4.839807512238622} +{"Finetune/Learning Rate": 4.679867620523895e-06, "Finetune/Loss": 0.8655722141265869, "Finetune/Loss (Raw)": 0.8794879913330078, "Finetune/Step": 3577, "Finetune/Step Time": 4.85846297070384} +{"Finetune/Learning Rate": 4.674593817352575e-06, "Finetune/Loss": 0.866080641746521, "Finetune/Loss (Raw)": 0.9705899357795715, "Finetune/Step": 3578, "Finetune/Step Time": 4.84333161264658} +{"Finetune/Learning Rate": 4.669322080867427e-06, "Finetune/Loss": 0.8669137954711914, "Finetune/Loss (Raw)": 0.9519146084785461, "Finetune/Step": 3579, "Finetune/Step Time": 4.843942478299141} +{"Finetune/Learning Rate": 4.664052413114305e-06, "Finetune/Loss": 0.8666000366210938, "Finetune/Loss (Raw)": 0.9308433532714844, "Finetune/Step": 3580, "Finetune/Step Time": 4.851711072027683} +{"Finetune/Learning Rate": 4.658784816138266e-06, "Finetune/Loss": 0.8676888942718506, "Finetune/Loss (Raw)": 0.9809387922286987, "Finetune/Step": 3581, "Finetune/Step Time": 4.856065580621362} +{"Finetune/Learning Rate": 4.653519291983561e-06, "Finetune/Loss": 0.8678361177444458, "Finetune/Loss (Raw)": 0.8464553952217102, "Finetune/Step": 3582, "Finetune/Step Time": 4.858017232269049} +{"Finetune/Learning Rate": 4.648255842693635e-06, "Finetune/Loss": 0.8715802431106567, "Finetune/Loss (Raw)": 0.8212071657180786, "Finetune/Step": 3583, "Finetune/Step Time": 4.822528699412942} +{"Finetune/Learning Rate": 4.642994470311136e-06, "Finetune/Loss": 0.8714916706085205, "Finetune/Loss (Raw)": 0.9366281628608704, "Finetune/Step": 3584, "Finetune/Step Time": 4.824470540508628} +{"Finetune/Learning Rate": 4.6377351768779e-06, "Finetune/Loss": 0.872552752494812, "Finetune/Loss (Raw)": 1.0232903957366943, "Finetune/Step": 3585, "Finetune/Step Time": 4.824660709127784} +{"Finetune/Learning Rate": 4.632477964434944e-06, "Finetune/Loss": 0.8714575171470642, "Finetune/Loss (Raw)": 0.7598227858543396, "Finetune/Step": 3586, "Finetune/Step Time": 4.822840241715312} +{"Finetune/Learning Rate": 4.6272228350225026e-06, "Finetune/Loss": 0.8712677955627441, "Finetune/Loss (Raw)": 0.8276434540748596, "Finetune/Step": 3587, "Finetune/Step Time": 4.810672475025058} +{"Finetune/Learning Rate": 4.621969790679984e-06, "Finetune/Loss": 0.869104266166687, "Finetune/Loss (Raw)": 0.7036292552947998, "Finetune/Step": 3588, "Finetune/Step Time": 4.811548940837383} +{"Finetune/Learning Rate": 4.6167188334459936e-06, "Finetune/Loss": 0.8732742667198181, "Finetune/Loss (Raw)": 0.8683587908744812, "Finetune/Step": 3589, "Finetune/Step Time": 4.790847958996892} +{"Finetune/Learning Rate": 4.611469965358323e-06, "Finetune/Loss": 0.8733636140823364, "Finetune/Loss (Raw)": 0.9490381479263306, "Finetune/Step": 3590, "Finetune/Step Time": 4.794341403990984} +{"Finetune/Learning Rate": 4.606223188453956e-06, "Finetune/Loss": 0.8728588819503784, "Finetune/Loss (Raw)": 0.8001309037208557, "Finetune/Step": 3591, "Finetune/Step Time": 4.798894602805376} +{"Finetune/Learning Rate": 4.6009785047690666e-06, "Finetune/Loss": 0.8727997541427612, "Finetune/Loss (Raw)": 0.9249399304389954, "Finetune/Step": 3592, "Finetune/Step Time": 4.803879141807556} +{"Finetune/Learning Rate": 4.595735916339008e-06, "Finetune/Loss": 0.8741611838340759, "Finetune/Loss (Raw)": 1.0093896389007568, "Finetune/Step": 3593, "Finetune/Step Time": 4.800391769036651} +{"Finetune/Learning Rate": 4.590495425198335e-06, "Finetune/Loss": 0.872586190700531, "Finetune/Loss (Raw)": 0.6151747703552246, "Finetune/Step": 3594, "Finetune/Step Time": 4.8022126741707325} +{"Finetune/Learning Rate": 4.585257033380779e-06, "Finetune/Loss": 0.873799204826355, "Finetune/Loss (Raw)": 0.8941608667373657, "Finetune/Step": 3595, "Finetune/Step Time": 4.819975579157472} +{"Finetune/Learning Rate": 4.580020742919247e-06, "Finetune/Loss": 0.8761078119277954, "Finetune/Loss (Raw)": 1.0246038436889648, "Finetune/Step": 3596, "Finetune/Step Time": 4.805477997288108} +{"Finetune/Learning Rate": 4.5747865558458524e-06, "Finetune/Loss": 0.8710479736328125, "Finetune/Loss (Raw)": 0.25503385066986084, "Finetune/Step": 3597, "Finetune/Step Time": 4.829323826357722} +{"Finetune/Learning Rate": 4.569554474191878e-06, "Finetune/Loss": 0.870925784111023, "Finetune/Loss (Raw)": 0.8254085779190063, "Finetune/Step": 3598, "Finetune/Step Time": 4.849026648327708} +{"Finetune/Learning Rate": 4.56432449998779e-06, "Finetune/Loss": 0.8698983192443848, "Finetune/Loss (Raw)": 0.8090219497680664, "Finetune/Step": 3599, "Finetune/Step Time": 4.834906354546547} +{"Finetune/Learning Rate": 4.559096635263242e-06, "Finetune/Loss": 0.8670109510421753, "Finetune/Loss (Raw)": 0.4358876645565033, "Finetune/Step": 3600, "Finetune/Step Time": 4.862198313698173} +{"Finetune/Learning Rate": 4.553870882047064e-06, "Finetune/Loss": 0.8663927316665649, "Finetune/Loss (Raw)": 0.9343740940093994, "Finetune/Step": 3601, "Finetune/Step Time": 4.84442126378417} +{"Finetune/Learning Rate": 4.548647242367271e-06, "Finetune/Loss": 0.8667967915534973, "Finetune/Loss (Raw)": 0.8469251394271851, "Finetune/Step": 3602, "Finetune/Step Time": 4.8394338842481375} +{"Finetune/Learning Rate": 4.543425718251051e-06, "Finetune/Loss": 0.8665534853935242, "Finetune/Loss (Raw)": 0.851050853729248, "Finetune/Step": 3603, "Finetune/Step Time": 4.8379544746130705} +{"Finetune/Learning Rate": 4.538206311724782e-06, "Finetune/Loss": 0.8678049445152283, "Finetune/Loss (Raw)": 0.9040244221687317, "Finetune/Step": 3604, "Finetune/Step Time": 4.834636388346553} +{"Finetune/Learning Rate": 4.532989024814015e-06, "Finetune/Loss": 0.8675254583358765, "Finetune/Loss (Raw)": 0.7835810780525208, "Finetune/Step": 3605, "Finetune/Step Time": 4.820611044764519} +{"Finetune/Learning Rate": 4.527773859543466e-06, "Finetune/Loss": 0.8665515184402466, "Finetune/Loss (Raw)": 0.8716983795166016, "Finetune/Step": 3606, "Finetune/Step Time": 4.838718432933092} +{"Finetune/Learning Rate": 4.52256081793705e-06, "Finetune/Loss": 0.8654369115829468, "Finetune/Loss (Raw)": 0.9139150977134705, "Finetune/Step": 3607, "Finetune/Step Time": 4.842017080634832} +{"Finetune/Learning Rate": 4.517349902017841e-06, "Finetune/Loss": 0.864989161491394, "Finetune/Loss (Raw)": 0.871224045753479, "Finetune/Step": 3608, "Finetune/Step Time": 4.869744285941124} +{"Finetune/Learning Rate": 4.512141113808097e-06, "Finetune/Loss": 0.8646876811981201, "Finetune/Loss (Raw)": 0.912580668926239, "Finetune/Step": 3609, "Finetune/Step Time": 4.864728474989533} +{"Finetune/Learning Rate": 4.5069344553292436e-06, "Finetune/Loss": 0.8639153838157654, "Finetune/Loss (Raw)": 0.7949482202529907, "Finetune/Step": 3610, "Finetune/Step Time": 4.8639025297015905} +{"Finetune/Learning Rate": 4.501729928601887e-06, "Finetune/Loss": 0.8596811294555664, "Finetune/Loss (Raw)": 0.33923256397247314, "Finetune/Step": 3611, "Finetune/Step Time": 4.85244669392705} +{"Finetune/Learning Rate": 4.4965275356458e-06, "Finetune/Loss": 0.8592231869697571, "Finetune/Loss (Raw)": 0.8635150790214539, "Finetune/Step": 3612, "Finetune/Step Time": 4.849403800442815} +{"Finetune/Learning Rate": 4.491327278479924e-06, "Finetune/Loss": 0.8576216697692871, "Finetune/Loss (Raw)": 0.788179337978363, "Finetune/Step": 3613, "Finetune/Step Time": 4.849890077486634} +{"Finetune/Learning Rate": 4.4861291591223934e-06, "Finetune/Loss": 0.8570447564125061, "Finetune/Loss (Raw)": 0.8672913312911987, "Finetune/Step": 3614, "Finetune/Step Time": 4.848610309883952} +{"Finetune/Learning Rate": 4.4809331795904835e-06, "Finetune/Loss": 0.8582296967506409, "Finetune/Loss (Raw)": 0.9132677912712097, "Finetune/Step": 3615, "Finetune/Step Time": 4.853058205917478} +{"Finetune/Learning Rate": 4.47573934190065e-06, "Finetune/Loss": 0.8583834171295166, "Finetune/Loss (Raw)": 0.776332676410675, "Finetune/Step": 3616, "Finetune/Step Time": 4.857129601761699} +{"Finetune/Learning Rate": 4.470547648068532e-06, "Finetune/Loss": 0.8573880791664124, "Finetune/Loss (Raw)": 0.854984700679779, "Finetune/Step": 3617, "Finetune/Step Time": 4.86578381806612} +{"Finetune/Learning Rate": 4.465358100108916e-06, "Finetune/Loss": 0.857717752456665, "Finetune/Loss (Raw)": 0.8373379111289978, "Finetune/Step": 3618, "Finetune/Step Time": 4.873590754345059} +{"Finetune/Learning Rate": 4.460170700035769e-06, "Finetune/Loss": 0.8566486835479736, "Finetune/Loss (Raw)": 0.8475924134254456, "Finetune/Step": 3619, "Finetune/Step Time": 4.871418057009578} +{"Finetune/Learning Rate": 4.454985449862215e-06, "Finetune/Loss": 0.8561251759529114, "Finetune/Loss (Raw)": 0.8412081599235535, "Finetune/Step": 3620, "Finetune/Step Time": 4.861330348998308} +{"Finetune/Learning Rate": 4.449802351600551e-06, "Finetune/Loss": 0.8569211959838867, "Finetune/Loss (Raw)": 0.8172141909599304, "Finetune/Step": 3621, "Finetune/Step Time": 4.8603161461651325} +{"Finetune/Learning Rate": 4.444621407262237e-06, "Finetune/Loss": 0.8561397790908813, "Finetune/Loss (Raw)": 0.896144688129425, "Finetune/Step": 3622, "Finetune/Step Time": 4.859214451164007} +{"Finetune/Learning Rate": 4.439442618857891e-06, "Finetune/Loss": 0.8560094237327576, "Finetune/Loss (Raw)": 0.9156827926635742, "Finetune/Step": 3623, "Finetune/Step Time": 4.859675228595734} +{"Finetune/Learning Rate": 4.434265988397312e-06, "Finetune/Loss": 0.8561965227127075, "Finetune/Loss (Raw)": 0.9764773845672607, "Finetune/Step": 3624, "Finetune/Step Time": 4.853525893762708} +{"Finetune/Learning Rate": 4.4290915178894355e-06, "Finetune/Loss": 0.8575515151023865, "Finetune/Loss (Raw)": 0.9505603313446045, "Finetune/Step": 3625, "Finetune/Step Time": 4.844034783542156} +{"Finetune/Learning Rate": 4.423919209342376e-06, "Finetune/Loss": 0.8566969633102417, "Finetune/Loss (Raw)": 0.800255298614502, "Finetune/Step": 3626, "Finetune/Step Time": 4.840206800028682} +{"Finetune/Learning Rate": 4.41874906476341e-06, "Finetune/Loss": 0.8563122749328613, "Finetune/Loss (Raw)": 0.9134158492088318, "Finetune/Step": 3627, "Finetune/Step Time": 4.855523515492678} +{"Finetune/Learning Rate": 4.413581086158969e-06, "Finetune/Loss": 0.8559550046920776, "Finetune/Loss (Raw)": 0.9021998047828674, "Finetune/Step": 3628, "Finetune/Step Time": 4.861000312492251} +{"Finetune/Learning Rate": 4.408415275534641e-06, "Finetune/Loss": 0.8551111221313477, "Finetune/Loss (Raw)": 0.8273594379425049, "Finetune/Step": 3629, "Finetune/Step Time": 4.867723835632205} +{"Finetune/Learning Rate": 4.4032516348951795e-06, "Finetune/Loss": 0.8535228967666626, "Finetune/Loss (Raw)": 0.855842113494873, "Finetune/Step": 3630, "Finetune/Step Time": 4.867887370288372} +{"Finetune/Learning Rate": 4.3980901662444905e-06, "Finetune/Loss": 0.8516141176223755, "Finetune/Loss (Raw)": 0.717129111289978, "Finetune/Step": 3631, "Finetune/Step Time": 4.882243484258652} +{"Finetune/Learning Rate": 4.39293087158564e-06, "Finetune/Loss": 0.8526862859725952, "Finetune/Loss (Raw)": 0.9847842454910278, "Finetune/Step": 3632, "Finetune/Step Time": 4.8848803378641605} +{"Finetune/Learning Rate": 4.387773752920845e-06, "Finetune/Loss": 0.8530935645103455, "Finetune/Loss (Raw)": 0.99850994348526, "Finetune/Step": 3633, "Finetune/Step Time": 4.879766112193465} +{"Finetune/Learning Rate": 4.382618812251495e-06, "Finetune/Loss": 0.8534659147262573, "Finetune/Loss (Raw)": 0.8772439956665039, "Finetune/Step": 3634, "Finetune/Step Time": 4.872258100658655} +{"Finetune/Learning Rate": 4.377466051578111e-06, "Finetune/Loss": 0.854310154914856, "Finetune/Loss (Raw)": 0.7731081247329712, "Finetune/Step": 3635, "Finetune/Step Time": 4.856440491974354} +{"Finetune/Learning Rate": 4.372315472900377e-06, "Finetune/Loss": 0.8544710278511047, "Finetune/Loss (Raw)": 0.9028559327125549, "Finetune/Step": 3636, "Finetune/Step Time": 4.855449546128511} +{"Finetune/Learning Rate": 4.367167078217141e-06, "Finetune/Loss": 0.8544407486915588, "Finetune/Loss (Raw)": 0.9709571003913879, "Finetune/Step": 3637, "Finetune/Step Time": 4.835954209789634} +{"Finetune/Learning Rate": 4.36202086952639e-06, "Finetune/Loss": 0.8555065393447876, "Finetune/Loss (Raw)": 0.938654899597168, "Finetune/Step": 3638, "Finetune/Step Time": 4.834134794771671} +{"Finetune/Learning Rate": 4.356876848825266e-06, "Finetune/Loss": 0.8556238412857056, "Finetune/Loss (Raw)": 0.8845456838607788, "Finetune/Step": 3639, "Finetune/Step Time": 4.822840919718146} +{"Finetune/Learning Rate": 4.351735018110066e-06, "Finetune/Loss": 0.850939154624939, "Finetune/Loss (Raw)": 0.3090725541114807, "Finetune/Step": 3640, "Finetune/Step Time": 4.858624054118991} +{"Finetune/Learning Rate": 4.346595379376232e-06, "Finetune/Loss": 0.8498452305793762, "Finetune/Loss (Raw)": 0.8023994565010071, "Finetune/Step": 3641, "Finetune/Step Time": 4.8616655599325895} +{"Finetune/Learning Rate": 4.341457934618357e-06, "Finetune/Loss": 0.8509470820426941, "Finetune/Loss (Raw)": 0.9470208287239075, "Finetune/Step": 3642, "Finetune/Step Time": 4.865608463063836} +{"Finetune/Learning Rate": 4.336322685830181e-06, "Finetune/Loss": 0.8515173196792603, "Finetune/Loss (Raw)": 0.9267316460609436, "Finetune/Step": 3643, "Finetune/Step Time": 4.858922652900219} +{"Finetune/Learning Rate": 4.331189635004604e-06, "Finetune/Loss": 0.8512417078018188, "Finetune/Loss (Raw)": 0.8406379818916321, "Finetune/Step": 3644, "Finetune/Step Time": 4.85572805069387} +{"Finetune/Learning Rate": 4.326058784133652e-06, "Finetune/Loss": 0.8503023386001587, "Finetune/Loss (Raw)": 0.8292969465255737, "Finetune/Step": 3645, "Finetune/Step Time": 4.855271788313985} +{"Finetune/Learning Rate": 4.3209301352085075e-06, "Finetune/Loss": 0.8511358499526978, "Finetune/Loss (Raw)": 0.9835374355316162, "Finetune/Step": 3646, "Finetune/Step Time": 4.844405783340335} +{"Finetune/Learning Rate": 4.315803690219507e-06, "Finetune/Loss": 0.8520181179046631, "Finetune/Loss (Raw)": 0.9172961711883545, "Finetune/Step": 3647, "Finetune/Step Time": 4.822470270097256} +{"Finetune/Learning Rate": 4.310679451156122e-06, "Finetune/Loss": 0.8523004055023193, "Finetune/Loss (Raw)": 0.9548975825309753, "Finetune/Step": 3648, "Finetune/Step Time": 4.829699821770191} +{"Finetune/Learning Rate": 4.305557420006962e-06, "Finetune/Loss": 0.8518097400665283, "Finetune/Loss (Raw)": 0.9554436802864075, "Finetune/Step": 3649, "Finetune/Step Time": 4.828175403177738} +{"Finetune/Learning Rate": 4.300437598759794e-06, "Finetune/Loss": 0.8537013530731201, "Finetune/Loss (Raw)": 1.0282353162765503, "Finetune/Step": 3650, "Finetune/Step Time": 4.808145368471742} +{"Finetune/Learning Rate": 4.295319989401522e-06, "Finetune/Loss": 0.8533975481987, "Finetune/Loss (Raw)": 0.7480291724205017, "Finetune/Step": 3651, "Finetune/Step Time": 4.797431254759431} +{"Finetune/Learning Rate": 4.2902045939181855e-06, "Finetune/Loss": 0.8546085357666016, "Finetune/Loss (Raw)": 0.940940797328949, "Finetune/Step": 3652, "Finetune/Step Time": 4.790009584277868} +{"Finetune/Learning Rate": 4.285091414294973e-06, "Finetune/Loss": 0.8539658784866333, "Finetune/Loss (Raw)": 0.9580633044242859, "Finetune/Step": 3653, "Finetune/Step Time": 4.787331845611334} +{"Finetune/Learning Rate": 4.2799804525162085e-06, "Finetune/Loss": 0.8543199896812439, "Finetune/Loss (Raw)": 0.8906832933425903, "Finetune/Step": 3654, "Finetune/Step Time": 4.797083109617233} +{"Finetune/Learning Rate": 4.274871710565356e-06, "Finetune/Loss": 0.8552682399749756, "Finetune/Loss (Raw)": 1.0573234558105469, "Finetune/Step": 3655, "Finetune/Step Time": 4.806645277887583} +{"Finetune/Learning Rate": 4.269765190425015e-06, "Finetune/Loss": 0.8543857932090759, "Finetune/Loss (Raw)": 0.7797946333885193, "Finetune/Step": 3656, "Finetune/Step Time": 4.810613717883825} +{"Finetune/Learning Rate": 4.264660894076934e-06, "Finetune/Loss": 0.8549016714096069, "Finetune/Loss (Raw)": 0.8759337663650513, "Finetune/Step": 3657, "Finetune/Step Time": 4.799693183973432} +{"Finetune/Learning Rate": 4.25955882350199e-06, "Finetune/Loss": 0.8547694087028503, "Finetune/Loss (Raw)": 0.8910270929336548, "Finetune/Step": 3658, "Finetune/Step Time": 4.815261401236057} +{"Finetune/Learning Rate": 4.254458980680188e-06, "Finetune/Loss": 0.8543668389320374, "Finetune/Loss (Raw)": 0.91253262758255, "Finetune/Step": 3659, "Finetune/Step Time": 4.818599158897996} +{"Finetune/Learning Rate": 4.2493613675906865e-06, "Finetune/Loss": 0.8557766675949097, "Finetune/Loss (Raw)": 1.003203272819519, "Finetune/Step": 3660, "Finetune/Step Time": 4.805548934265971} +{"Finetune/Learning Rate": 4.244265986211766e-06, "Finetune/Loss": 0.8601032495498657, "Finetune/Loss (Raw)": 0.8755472898483276, "Finetune/Step": 3661, "Finetune/Step Time": 4.768795354291797} +{"Finetune/Learning Rate": 4.239172838520846e-06, "Finetune/Loss": 0.8605228662490845, "Finetune/Loss (Raw)": 0.977498471736908, "Finetune/Step": 3662, "Finetune/Step Time": 4.748012037947774} +{"Finetune/Learning Rate": 4.2340819264944755e-06, "Finetune/Loss": 0.8661463856697083, "Finetune/Loss (Raw)": 1.0583425760269165, "Finetune/Step": 3663, "Finetune/Step Time": 4.710707750171423} +{"Finetune/Learning Rate": 4.22899325210834e-06, "Finetune/Loss": 0.8659777641296387, "Finetune/Loss (Raw)": 0.8240258693695068, "Finetune/Step": 3664, "Finetune/Step Time": 4.729284435510635} +{"Finetune/Learning Rate": 4.223906817337253e-06, "Finetune/Loss": 0.8651409149169922, "Finetune/Loss (Raw)": 0.9131730198860168, "Finetune/Step": 3665, "Finetune/Step Time": 4.7267914693802595} +{"Finetune/Learning Rate": 4.218822624155159e-06, "Finetune/Loss": 0.8644928336143494, "Finetune/Loss (Raw)": 0.9193997979164124, "Finetune/Step": 3666, "Finetune/Step Time": 4.727154681459069} +{"Finetune/Learning Rate": 4.213740674535141e-06, "Finetune/Loss": 0.8604747653007507, "Finetune/Loss (Raw)": 0.4247276782989502, "Finetune/Step": 3667, "Finetune/Step Time": 4.759859701618552} +{"Finetune/Learning Rate": 4.2086609704494015e-06, "Finetune/Loss": 0.8599990010261536, "Finetune/Loss (Raw)": 0.821692705154419, "Finetune/Step": 3668, "Finetune/Step Time": 4.749921955168247} +{"Finetune/Learning Rate": 4.203583513869267e-06, "Finetune/Loss": 0.859779953956604, "Finetune/Loss (Raw)": 0.8580081462860107, "Finetune/Step": 3669, "Finetune/Step Time": 4.753471599891782} +{"Finetune/Learning Rate": 4.198508306765209e-06, "Finetune/Loss": 0.8601162433624268, "Finetune/Loss (Raw)": 0.9096858501434326, "Finetune/Step": 3670, "Finetune/Step Time": 4.738791797310114} +{"Finetune/Learning Rate": 4.193435351106811e-06, "Finetune/Loss": 0.8609036207199097, "Finetune/Loss (Raw)": 0.9740281701087952, "Finetune/Step": 3671, "Finetune/Step Time": 4.720453603193164} +{"Finetune/Learning Rate": 4.188364648862791e-06, "Finetune/Loss": 0.8603051900863647, "Finetune/Loss (Raw)": 0.8428677916526794, "Finetune/Step": 3672, "Finetune/Step Time": 4.728557042777538} +{"Finetune/Learning Rate": 4.183296202000988e-06, "Finetune/Loss": 0.8622606992721558, "Finetune/Loss (Raw)": 0.6688959002494812, "Finetune/Step": 3673, "Finetune/Step Time": 4.702773779630661} +{"Finetune/Learning Rate": 4.178230012488368e-06, "Finetune/Loss": 0.8624826073646545, "Finetune/Loss (Raw)": 0.8512446880340576, "Finetune/Step": 3674, "Finetune/Step Time": 4.688325399532914} +{"Finetune/Learning Rate": 4.173166082291019e-06, "Finetune/Loss": 0.8575570583343506, "Finetune/Loss (Raw)": 0.3764171004295349, "Finetune/Step": 3675, "Finetune/Step Time": 4.728248901665211} +{"Finetune/Learning Rate": 4.16810441337415e-06, "Finetune/Loss": 0.8576263189315796, "Finetune/Loss (Raw)": 0.950871467590332, "Finetune/Step": 3676, "Finetune/Step Time": 4.731219304725528} +{"Finetune/Learning Rate": 4.163045007702105e-06, "Finetune/Loss": 0.857381284236908, "Finetune/Loss (Raw)": 0.9656850695610046, "Finetune/Step": 3677, "Finetune/Step Time": 4.730552179738879} +{"Finetune/Learning Rate": 4.157987867238338e-06, "Finetune/Loss": 0.8597511053085327, "Finetune/Loss (Raw)": 0.6467251181602478, "Finetune/Step": 3678, "Finetune/Step Time": 4.695860510692} +{"Finetune/Learning Rate": 4.152932993945418e-06, "Finetune/Loss": 0.8613629341125488, "Finetune/Loss (Raw)": 0.9870743155479431, "Finetune/Step": 3679, "Finetune/Step Time": 4.689027851447463} +{"Finetune/Learning Rate": 4.147880389785051e-06, "Finetune/Loss": 0.8613916635513306, "Finetune/Loss (Raw)": 0.9116470813751221, "Finetune/Step": 3680, "Finetune/Step Time": 4.694923287257552} +{"Finetune/Learning Rate": 4.142830056718052e-06, "Finetune/Loss": 0.8631981611251831, "Finetune/Loss (Raw)": 0.9752292633056641, "Finetune/Step": 3681, "Finetune/Step Time": 4.670305864885449} +{"Finetune/Learning Rate": 4.137781996704356e-06, "Finetune/Loss": 0.8641912937164307, "Finetune/Loss (Raw)": 1.0950952768325806, "Finetune/Step": 3682, "Finetune/Step Time": 4.667526612058282} +{"Finetune/Learning Rate": 4.1327362117030174e-06, "Finetune/Loss": 0.8628382682800293, "Finetune/Loss (Raw)": 0.8891907334327698, "Finetune/Step": 3683, "Finetune/Step Time": 4.667781861498952} +{"Finetune/Learning Rate": 4.127692703672207e-06, "Finetune/Loss": 0.8652979135513306, "Finetune/Loss (Raw)": 0.6651433110237122, "Finetune/Step": 3684, "Finetune/Step Time": 4.650680936872959} +{"Finetune/Learning Rate": 4.1226514745692115e-06, "Finetune/Loss": 0.8639118671417236, "Finetune/Loss (Raw)": 0.8580996990203857, "Finetune/Step": 3685, "Finetune/Step Time": 4.647437850013375} +{"Finetune/Learning Rate": 4.117612526350429e-06, "Finetune/Loss": 0.8641344308853149, "Finetune/Loss (Raw)": 0.9129408597946167, "Finetune/Step": 3686, "Finetune/Step Time": 4.646799884736538} +{"Finetune/Learning Rate": 4.112575860971384e-06, "Finetune/Loss": 0.8635084629058838, "Finetune/Loss (Raw)": 0.8814848065376282, "Finetune/Step": 3687, "Finetune/Step Time": 4.645215094089508} +{"Finetune/Learning Rate": 4.107541480386708e-06, "Finetune/Loss": 0.8620644807815552, "Finetune/Loss (Raw)": 0.820451021194458, "Finetune/Step": 3688, "Finetune/Step Time": 4.653621751815081} +{"Finetune/Learning Rate": 4.102509386550141e-06, "Finetune/Loss": 0.8616288900375366, "Finetune/Loss (Raw)": 0.9864259362220764, "Finetune/Step": 3689, "Finetune/Step Time": 4.653481841087341} +{"Finetune/Learning Rate": 4.0974795814145384e-06, "Finetune/Loss": 0.8631852865219116, "Finetune/Loss (Raw)": 0.9307104349136353, "Finetune/Step": 3690, "Finetune/Step Time": 4.6567290257662535} +{"Finetune/Learning Rate": 4.092452066931877e-06, "Finetune/Loss": 0.8596035242080688, "Finetune/Loss (Raw)": 0.3102788031101227, "Finetune/Step": 3691, "Finetune/Step Time": 4.691014206036925} +{"Finetune/Learning Rate": 4.087426845053236e-06, "Finetune/Loss": 0.8609828948974609, "Finetune/Loss (Raw)": 0.9958980083465576, "Finetune/Step": 3692, "Finetune/Step Time": 4.684407062828541} +{"Finetune/Learning Rate": 4.082403917728796e-06, "Finetune/Loss": 0.8613885641098022, "Finetune/Loss (Raw)": 0.8959146738052368, "Finetune/Step": 3693, "Finetune/Step Time": 4.683721112087369} +{"Finetune/Learning Rate": 4.077383286907869e-06, "Finetune/Loss": 0.8595483303070068, "Finetune/Loss (Raw)": 0.7754194140434265, "Finetune/Step": 3694, "Finetune/Step Time": 4.707716656848788} +{"Finetune/Learning Rate": 4.072364954538858e-06, "Finetune/Loss": 0.861323893070221, "Finetune/Loss (Raw)": 1.1138997077941895, "Finetune/Step": 3695, "Finetune/Step Time": 4.686165140941739} +{"Finetune/Learning Rate": 4.067348922569281e-06, "Finetune/Loss": 0.8608464598655701, "Finetune/Loss (Raw)": 0.8775118589401245, "Finetune/Step": 3696, "Finetune/Step Time": 4.678709514439106} +{"Finetune/Learning Rate": 4.062335192945762e-06, "Finetune/Loss": 0.8597126007080078, "Finetune/Loss (Raw)": 0.7190276980400085, "Finetune/Step": 3697, "Finetune/Step Time": 4.69516028277576} +{"Finetune/Learning Rate": 4.057323767614032e-06, "Finetune/Loss": 0.8603814840316772, "Finetune/Loss (Raw)": 0.9172884225845337, "Finetune/Step": 3698, "Finetune/Step Time": 4.701869182288647} +{"Finetune/Learning Rate": 4.052314648518925e-06, "Finetune/Loss": 0.8603059649467468, "Finetune/Loss (Raw)": 0.922058641910553, "Finetune/Step": 3699, "Finetune/Step Time": 4.698777025565505} +{"Finetune/Learning Rate": 4.047307837604383e-06, "Finetune/Loss": 0.860153317451477, "Finetune/Loss (Raw)": 0.9236486554145813, "Finetune/Step": 3700, "Finetune/Step Time": 4.699630273506045} +{"Finetune/Learning Rate": 4.0423033368134545e-06, "Finetune/Loss": 0.8584635853767395, "Finetune/Loss (Raw)": 0.7235413789749146, "Finetune/Step": 3701, "Finetune/Step Time": 4.707043128088117} +{"Finetune/Learning Rate": 4.037301148088292e-06, "Finetune/Loss": 0.8580944538116455, "Finetune/Loss (Raw)": 0.9301650524139404, "Finetune/Step": 3702, "Finetune/Step Time": 4.69618496671319} +{"Finetune/Learning Rate": 4.032301273370132e-06, "Finetune/Loss": 0.8587691187858582, "Finetune/Loss (Raw)": 0.9520259499549866, "Finetune/Step": 3703, "Finetune/Step Time": 4.694239402189851} +{"Finetune/Learning Rate": 4.027303714599345e-06, "Finetune/Loss": 0.858291745185852, "Finetune/Loss (Raw)": 0.9204277992248535, "Finetune/Step": 3704, "Finetune/Step Time": 4.6998355332762} +{"Finetune/Learning Rate": 4.022308473715379e-06, "Finetune/Loss": 0.8570693135261536, "Finetune/Loss (Raw)": 0.7230238914489746, "Finetune/Step": 3705, "Finetune/Step Time": 4.681234270334244} +{"Finetune/Learning Rate": 4.017315552656788e-06, "Finetune/Loss": 0.8562525510787964, "Finetune/Loss (Raw)": 0.866047203540802, "Finetune/Step": 3706, "Finetune/Step Time": 4.678094308823347} +{"Finetune/Learning Rate": 4.01232495336123e-06, "Finetune/Loss": 0.8554808497428894, "Finetune/Loss (Raw)": 0.8531306982040405, "Finetune/Step": 3707, "Finetune/Step Time": 4.70442040450871} +{"Finetune/Learning Rate": 4.0073366777654574e-06, "Finetune/Loss": 0.8542822003364563, "Finetune/Loss (Raw)": 0.7774136066436768, "Finetune/Step": 3708, "Finetune/Step Time": 4.715584550052881} +{"Finetune/Learning Rate": 4.002350727805324e-06, "Finetune/Loss": 0.8545432090759277, "Finetune/Loss (Raw)": 1.0143539905548096, "Finetune/Step": 3709, "Finetune/Step Time": 4.706114316359162} +{"Finetune/Learning Rate": 3.997367105415775e-06, "Finetune/Loss": 0.8556748628616333, "Finetune/Loss (Raw)": 0.9913054704666138, "Finetune/Step": 3710, "Finetune/Step Time": 4.693003153428435} +{"Finetune/Learning Rate": 3.992385812530864e-06, "Finetune/Loss": 0.8553479909896851, "Finetune/Loss (Raw)": 0.7793591618537903, "Finetune/Step": 3711, "Finetune/Step Time": 4.704893911257386} +{"Finetune/Learning Rate": 3.987406851083735e-06, "Finetune/Loss": 0.8559175729751587, "Finetune/Loss (Raw)": 1.0095430612564087, "Finetune/Step": 3712, "Finetune/Step Time": 4.704107625409961} +{"Finetune/Learning Rate": 3.982430223006614e-06, "Finetune/Loss": 0.8501436710357666, "Finetune/Loss (Raw)": 0.28423011302948, "Finetune/Step": 3713, "Finetune/Step Time": 4.738389372825623} +{"Finetune/Learning Rate": 3.977455930230842e-06, "Finetune/Loss": 0.8512578010559082, "Finetune/Loss (Raw)": 0.902428150177002, "Finetune/Step": 3714, "Finetune/Step Time": 4.731147428974509} +{"Finetune/Learning Rate": 3.972483974686845e-06, "Finetune/Loss": 0.8521881103515625, "Finetune/Loss (Raw)": 0.9467201828956604, "Finetune/Step": 3715, "Finetune/Step Time": 4.74078213609755} +{"Finetune/Learning Rate": 3.967514358304139e-06, "Finetune/Loss": 0.8530171513557434, "Finetune/Loss (Raw)": 0.8097509741783142, "Finetune/Step": 3716, "Finetune/Step Time": 4.764252731576562} +{"Finetune/Learning Rate": 3.962547083011338e-06, "Finetune/Loss": 0.8523160219192505, "Finetune/Loss (Raw)": 0.7786214351654053, "Finetune/Step": 3717, "Finetune/Step Time": 4.773332418873906} +{"Finetune/Learning Rate": 3.957582150736141e-06, "Finetune/Loss": 0.851033627986908, "Finetune/Loss (Raw)": 0.7848845720291138, "Finetune/Step": 3718, "Finetune/Step Time": 4.778104014694691} +{"Finetune/Learning Rate": 3.952619563405343e-06, "Finetune/Loss": 0.8501131534576416, "Finetune/Loss (Raw)": 0.6823042631149292, "Finetune/Step": 3719, "Finetune/Step Time": 4.783288417384028} +{"Finetune/Learning Rate": 3.9476593229448245e-06, "Finetune/Loss": 0.8506587743759155, "Finetune/Loss (Raw)": 0.9947859048843384, "Finetune/Step": 3720, "Finetune/Step Time": 4.787551483139396} +{"Finetune/Learning Rate": 3.942701431279564e-06, "Finetune/Loss": 0.8504377007484436, "Finetune/Loss (Raw)": 0.981090247631073, "Finetune/Step": 3721, "Finetune/Step Time": 4.78701912984252} +{"Finetune/Learning Rate": 3.937745890333623e-06, "Finetune/Loss": 0.8528551459312439, "Finetune/Loss (Raw)": 0.9246150851249695, "Finetune/Step": 3722, "Finetune/Step Time": 4.7838054951280355} +{"Finetune/Learning Rate": 3.932792702030139e-06, "Finetune/Loss": 0.853508710861206, "Finetune/Loss (Raw)": 0.9778091907501221, "Finetune/Step": 3723, "Finetune/Step Time": 4.775812452659011} +{"Finetune/Learning Rate": 3.92784186829136e-06, "Finetune/Loss": 0.8519052267074585, "Finetune/Loss (Raw)": 0.8193626403808594, "Finetune/Step": 3724, "Finetune/Step Time": 4.782160686329007} +{"Finetune/Learning Rate": 3.9228933910386014e-06, "Finetune/Loss": 0.8574602603912354, "Finetune/Loss (Raw)": 0.9660769701004028, "Finetune/Step": 3725, "Finetune/Step Time": 4.755458123981953} +{"Finetune/Learning Rate": 3.917947272192273e-06, "Finetune/Loss": 0.8540124893188477, "Finetune/Loss (Raw)": 0.3840915262699127, "Finetune/Step": 3726, "Finetune/Step Time": 4.77255673520267} +{"Finetune/Learning Rate": 3.913003513671866e-06, "Finetune/Loss": 0.8541584014892578, "Finetune/Loss (Raw)": 0.8277007341384888, "Finetune/Step": 3727, "Finetune/Step Time": 4.77700605802238} +{"Finetune/Learning Rate": 3.908062117395956e-06, "Finetune/Loss": 0.8561276793479919, "Finetune/Loss (Raw)": 0.6879537105560303, "Finetune/Step": 3728, "Finetune/Step Time": 4.749258443713188} +{"Finetune/Learning Rate": 3.903123085282204e-06, "Finetune/Loss": 0.8560339212417603, "Finetune/Loss (Raw)": 0.9223758578300476, "Finetune/Step": 3729, "Finetune/Step Time": 4.753045961260796} +{"Finetune/Learning Rate": 3.898186419247347e-06, "Finetune/Loss": 0.8565515279769897, "Finetune/Loss (Raw)": 0.9131730794906616, "Finetune/Step": 3730, "Finetune/Step Time": 4.782009478658438} +{"Finetune/Learning Rate": 3.893252121207221e-06, "Finetune/Loss": 0.857245683670044, "Finetune/Loss (Raw)": 0.939903199672699, "Finetune/Step": 3731, "Finetune/Step Time": 4.781952874734998} +{"Finetune/Learning Rate": 3.888320193076719e-06, "Finetune/Loss": 0.857181966304779, "Finetune/Loss (Raw)": 0.8958695530891418, "Finetune/Step": 3732, "Finetune/Step Time": 4.780260240659118} +{"Finetune/Learning Rate": 3.8833906367698245e-06, "Finetune/Loss": 0.8577314019203186, "Finetune/Loss (Raw)": 0.8539112210273743, "Finetune/Step": 3733, "Finetune/Step Time": 4.787900876253843} +{"Finetune/Learning Rate": 3.878463454199613e-06, "Finetune/Loss": 0.8584743738174438, "Finetune/Loss (Raw)": 0.9668021202087402, "Finetune/Step": 3734, "Finetune/Step Time": 4.775023892521858} +{"Finetune/Learning Rate": 3.87353864727822e-06, "Finetune/Loss": 0.8569135665893555, "Finetune/Loss (Raw)": 0.71413254737854, "Finetune/Step": 3735, "Finetune/Step Time": 4.776031265035272} +{"Finetune/Learning Rate": 3.8686162179168695e-06, "Finetune/Loss": 0.8586152195930481, "Finetune/Loss (Raw)": 1.089028239250183, "Finetune/Step": 3736, "Finetune/Step Time": 4.746177228167653} +{"Finetune/Learning Rate": 3.863696168025859e-06, "Finetune/Loss": 0.8586496710777283, "Finetune/Loss (Raw)": 0.9169904589653015, "Finetune/Step": 3737, "Finetune/Step Time": 4.76486655883491} +{"Finetune/Learning Rate": 3.858778499514565e-06, "Finetune/Loss": 0.8589593172073364, "Finetune/Loss (Raw)": 0.8345921635627747, "Finetune/Step": 3738, "Finetune/Step Time": 4.779440814629197} +{"Finetune/Learning Rate": 3.853863214291439e-06, "Finetune/Loss": 0.8628769516944885, "Finetune/Loss (Raw)": 0.8406774401664734, "Finetune/Step": 3739, "Finetune/Step Time": 4.743597354739904} +{"Finetune/Learning Rate": 3.848950314264001e-06, "Finetune/Loss": 0.8630536794662476, "Finetune/Loss (Raw)": 0.8861379027366638, "Finetune/Step": 3740, "Finetune/Step Time": 4.7273036148399115} +{"Finetune/Learning Rate": 3.8440398013388655e-06, "Finetune/Loss": 0.8644366264343262, "Finetune/Loss (Raw)": 0.9652018547058105, "Finetune/Step": 3741, "Finetune/Step Time": 4.727443540468812} +{"Finetune/Learning Rate": 3.839131677421694e-06, "Finetune/Loss": 0.8644906878471375, "Finetune/Loss (Raw)": 0.8742105960845947, "Finetune/Step": 3742, "Finetune/Step Time": 4.723133334890008} +{"Finetune/Learning Rate": 3.834225944417235e-06, "Finetune/Loss": 0.8645744323730469, "Finetune/Loss (Raw)": 0.9239784479141235, "Finetune/Step": 3743, "Finetune/Step Time": 4.761127043515444} +{"Finetune/Learning Rate": 3.829322604229313e-06, "Finetune/Loss": 0.8613291382789612, "Finetune/Loss (Raw)": 0.36094367504119873, "Finetune/Step": 3744, "Finetune/Step Time": 4.792656723409891} +{"Finetune/Learning Rate": 3.824421658760815e-06, "Finetune/Loss": 0.8571958541870117, "Finetune/Loss (Raw)": 0.325924277305603, "Finetune/Step": 3745, "Finetune/Step Time": 4.81609733402729} +{"Finetune/Learning Rate": 3.819523109913706e-06, "Finetune/Loss": 0.8565040826797485, "Finetune/Loss (Raw)": 0.7487898468971252, "Finetune/Step": 3746, "Finetune/Step Time": 4.812753826379776} +{"Finetune/Learning Rate": 3.814626959589012e-06, "Finetune/Loss": 0.8557339906692505, "Finetune/Loss (Raw)": 0.7490147948265076, "Finetune/Step": 3747, "Finetune/Step Time": 4.833278531208634} +{"Finetune/Learning Rate": 3.8097332096868368e-06, "Finetune/Loss": 0.8565336465835571, "Finetune/Loss (Raw)": 0.9435696601867676, "Finetune/Step": 3748, "Finetune/Step Time": 4.84434319101274} +{"Finetune/Learning Rate": 3.8048418621063475e-06, "Finetune/Loss": 0.8562572598457336, "Finetune/Loss (Raw)": 0.7818318605422974, "Finetune/Step": 3749, "Finetune/Step Time": 4.846177864819765} +{"Finetune/Learning Rate": 3.7999529187457775e-06, "Finetune/Loss": 0.8566417694091797, "Finetune/Loss (Raw)": 0.9453630447387695, "Finetune/Step": 3750, "Finetune/Step Time": 4.850403741002083} +{"Finetune/Learning Rate": 3.7950663815024424e-06, "Finetune/Loss": 0.8574190139770508, "Finetune/Loss (Raw)": 1.015170931816101, "Finetune/Step": 3751, "Finetune/Step Time": 4.857985146343708} +{"Finetune/Learning Rate": 3.790182252272698e-06, "Finetune/Loss": 0.8564134836196899, "Finetune/Loss (Raw)": 0.8477718830108643, "Finetune/Step": 3752, "Finetune/Step Time": 4.8604719415307045} +{"Finetune/Learning Rate": 3.785300532951983e-06, "Finetune/Loss": 0.8557491898536682, "Finetune/Loss (Raw)": 0.8655285835266113, "Finetune/Step": 3753, "Finetune/Step Time": 4.8754656203091145} +{"Finetune/Learning Rate": 3.780421225434803e-06, "Finetune/Loss": 0.8565769195556641, "Finetune/Loss (Raw)": 0.906209409236908, "Finetune/Step": 3754, "Finetune/Step Time": 4.877923030406237} +{"Finetune/Learning Rate": 3.7755443316147177e-06, "Finetune/Loss": 0.8568586111068726, "Finetune/Loss (Raw)": 0.949467658996582, "Finetune/Step": 3755, "Finetune/Step Time": 4.865710416808724} +{"Finetune/Learning Rate": 3.7706698533843556e-06, "Finetune/Loss": 0.8553651571273804, "Finetune/Loss (Raw)": 0.711043119430542, "Finetune/Step": 3756, "Finetune/Step Time": 4.867160305380821} +{"Finetune/Learning Rate": 3.765797792635406e-06, "Finetune/Loss": 0.8563591241836548, "Finetune/Loss (Raw)": 0.9545865654945374, "Finetune/Step": 3757, "Finetune/Step Time": 4.869465980678797} +{"Finetune/Learning Rate": 3.7609281512586203e-06, "Finetune/Loss": 0.8563756942749023, "Finetune/Loss (Raw)": 0.8579639792442322, "Finetune/Step": 3758, "Finetune/Step Time": 4.884688058868051} +{"Finetune/Learning Rate": 3.756060931143811e-06, "Finetune/Loss": 0.8582123517990112, "Finetune/Loss (Raw)": 0.9522132873535156, "Finetune/Step": 3759, "Finetune/Step Time": 4.8692820351570845} +{"Finetune/Learning Rate": 3.7511961341798487e-06, "Finetune/Loss": 0.8570636510848999, "Finetune/Loss (Raw)": 0.8377500176429749, "Finetune/Step": 3760, "Finetune/Step Time": 4.875209882855415} +{"Finetune/Learning Rate": 3.746333762254677e-06, "Finetune/Loss": 0.8551189303398132, "Finetune/Loss (Raw)": 0.7495867013931274, "Finetune/Step": 3761, "Finetune/Step Time": 4.886166112497449} +{"Finetune/Learning Rate": 3.7414738172552744e-06, "Finetune/Loss": 0.8551723957061768, "Finetune/Loss (Raw)": 0.8840896487236023, "Finetune/Step": 3762, "Finetune/Step Time": 4.891435964033008} +{"Finetune/Learning Rate": 3.7366163010676937e-06, "Finetune/Loss": 0.8564516305923462, "Finetune/Loss (Raw)": 0.936850905418396, "Finetune/Step": 3763, "Finetune/Step Time": 4.892622547224164} +{"Finetune/Learning Rate": 3.7317612155770467e-06, "Finetune/Loss": 0.8557500839233398, "Finetune/Loss (Raw)": 0.8130602240562439, "Finetune/Step": 3764, "Finetune/Step Time": 4.892982630059123} +{"Finetune/Learning Rate": 3.726908562667496e-06, "Finetune/Loss": 0.8547784090042114, "Finetune/Loss (Raw)": 0.8465777635574341, "Finetune/Step": 3765, "Finetune/Step Time": 4.902429090812802} +{"Finetune/Learning Rate": 3.72205834422226e-06, "Finetune/Loss": 0.8548283576965332, "Finetune/Loss (Raw)": 0.9450548887252808, "Finetune/Step": 3766, "Finetune/Step Time": 4.905259169638157} +{"Finetune/Learning Rate": 3.717210562123613e-06, "Finetune/Loss": 0.8552941083908081, "Finetune/Loss (Raw)": 0.9441581964492798, "Finetune/Step": 3767, "Finetune/Step Time": 4.909023562446237} +{"Finetune/Learning Rate": 3.712365218252887e-06, "Finetune/Loss": 0.8588337898254395, "Finetune/Loss (Raw)": 0.7621545195579529, "Finetune/Step": 3768, "Finetune/Step Time": 4.88222136721015} +{"Finetune/Learning Rate": 3.7075223144904638e-06, "Finetune/Loss": 0.8545122146606445, "Finetune/Loss (Raw)": 0.249227836728096, "Finetune/Step": 3769, "Finetune/Step Time": 4.915218101814389} +{"Finetune/Learning Rate": 3.702681852715777e-06, "Finetune/Loss": 0.8552440404891968, "Finetune/Loss (Raw)": 1.0406960248947144, "Finetune/Step": 3770, "Finetune/Step Time": 4.90630722977221} +{"Finetune/Learning Rate": 3.6978438348073253e-06, "Finetune/Loss": 0.8563966155052185, "Finetune/Loss (Raw)": 1.0742698907852173, "Finetune/Step": 3771, "Finetune/Step Time": 4.908218489959836} +{"Finetune/Learning Rate": 3.6930082626426377e-06, "Finetune/Loss": 0.8573160171508789, "Finetune/Loss (Raw)": 0.9583236575126648, "Finetune/Step": 3772, "Finetune/Step Time": 4.913832809776068} +{"Finetune/Learning Rate": 3.6881751380983066e-06, "Finetune/Loss": 0.8535323143005371, "Finetune/Loss (Raw)": 0.3449746072292328, "Finetune/Step": 3773, "Finetune/Step Time": 4.94696294143796} +{"Finetune/Learning Rate": 3.6833444630499804e-06, "Finetune/Loss": 0.8530834317207336, "Finetune/Loss (Raw)": 0.9260855317115784, "Finetune/Step": 3774, "Finetune/Step Time": 4.944162309169769} +{"Finetune/Learning Rate": 3.6785162393723482e-06, "Finetune/Loss": 0.8534623384475708, "Finetune/Loss (Raw)": 0.9657983183860779, "Finetune/Step": 3775, "Finetune/Step Time": 4.948590494692326} +{"Finetune/Learning Rate": 3.6736904689391417e-06, "Finetune/Loss": 0.8533475399017334, "Finetune/Loss (Raw)": 0.9401934742927551, "Finetune/Step": 3776, "Finetune/Step Time": 4.946584464982152} +{"Finetune/Learning Rate": 3.6688671536231557e-06, "Finetune/Loss": 0.848013699054718, "Finetune/Loss (Raw)": 0.2727193236351013, "Finetune/Step": 3777, "Finetune/Step Time": 4.982287190854549} +{"Finetune/Learning Rate": 3.6640462952962217e-06, "Finetune/Loss": 0.8471988439559937, "Finetune/Loss (Raw)": 0.9239301085472107, "Finetune/Step": 3778, "Finetune/Step Time": 4.989905470982194} +{"Finetune/Learning Rate": 3.6592278958292225e-06, "Finetune/Loss": 0.8479907512664795, "Finetune/Loss (Raw)": 0.8493962287902832, "Finetune/Step": 3779, "Finetune/Step Time": 4.994893662631512} +{"Finetune/Learning Rate": 3.6544119570920845e-06, "Finetune/Loss": 0.8469387888908386, "Finetune/Loss (Raw)": 0.806296169757843, "Finetune/Step": 3780, "Finetune/Step Time": 5.000803494825959} +{"Finetune/Learning Rate": 3.6495984809537775e-06, "Finetune/Loss": 0.8468678593635559, "Finetune/Loss (Raw)": 0.948980987071991, "Finetune/Step": 3781, "Finetune/Step Time": 4.998532321304083} +{"Finetune/Learning Rate": 3.6447874692823195e-06, "Finetune/Loss": 0.8464857935905457, "Finetune/Loss (Raw)": 0.841778576374054, "Finetune/Step": 3782, "Finetune/Step Time": 4.9809795413166285} +{"Finetune/Learning Rate": 3.639978923944766e-06, "Finetune/Loss": 0.8445030450820923, "Finetune/Loss (Raw)": 0.8035355806350708, "Finetune/Step": 3783, "Finetune/Step Time": 4.978463275358081} +{"Finetune/Learning Rate": 3.6351728468072265e-06, "Finetune/Loss": 0.8447333574295044, "Finetune/Loss (Raw)": 0.8092718124389648, "Finetune/Step": 3784, "Finetune/Step Time": 4.984550308436155} +{"Finetune/Learning Rate": 3.6303692397348455e-06, "Finetune/Loss": 0.844723105430603, "Finetune/Loss (Raw)": 0.8746128082275391, "Finetune/Step": 3785, "Finetune/Step Time": 4.982656162232161} +{"Finetune/Learning Rate": 3.6255681045918e-06, "Finetune/Loss": 0.8454287052154541, "Finetune/Loss (Raw)": 0.9813526272773743, "Finetune/Step": 3786, "Finetune/Step Time": 4.966313129290938} +{"Finetune/Learning Rate": 3.6207694432413255e-06, "Finetune/Loss": 0.8455592393875122, "Finetune/Loss (Raw)": 0.9292417168617249, "Finetune/Step": 3787, "Finetune/Step Time": 4.963208785280585} +{"Finetune/Learning Rate": 3.6159732575456862e-06, "Finetune/Loss": 0.8445290923118591, "Finetune/Loss (Raw)": 0.8713412880897522, "Finetune/Step": 3788, "Finetune/Step Time": 4.968864010646939} +{"Finetune/Learning Rate": 3.6111795493661885e-06, "Finetune/Loss": 0.843864917755127, "Finetune/Loss (Raw)": 0.7905274629592896, "Finetune/Step": 3789, "Finetune/Step Time": 4.993653601035476} +{"Finetune/Learning Rate": 3.606388320563177e-06, "Finetune/Loss": 0.8436266183853149, "Finetune/Loss (Raw)": 0.9469984173774719, "Finetune/Step": 3790, "Finetune/Step Time": 4.994681593030691} +{"Finetune/Learning Rate": 3.6015995729960316e-06, "Finetune/Loss": 0.8421981334686279, "Finetune/Loss (Raw)": 0.8755054473876953, "Finetune/Step": 3791, "Finetune/Step Time": 4.99457036331296} +{"Finetune/Learning Rate": 3.596813308523176e-06, "Finetune/Loss": 0.8384498357772827, "Finetune/Loss (Raw)": 0.3442402780056, "Finetune/Step": 3792, "Finetune/Step Time": 5.022745940834284} +{"Finetune/Learning Rate": 3.592029529002059e-06, "Finetune/Loss": 0.8384693264961243, "Finetune/Loss (Raw)": 0.9156671762466431, "Finetune/Step": 3793, "Finetune/Step Time": 5.035998487845063} +{"Finetune/Learning Rate": 3.5872482362891802e-06, "Finetune/Loss": 0.838504433631897, "Finetune/Loss (Raw)": 0.9238932728767395, "Finetune/Step": 3794, "Finetune/Step Time": 5.029953146353364} +{"Finetune/Learning Rate": 3.5824694322400653e-06, "Finetune/Loss": 0.8410873413085938, "Finetune/Loss (Raw)": 0.75533527135849, "Finetune/Step": 3795, "Finetune/Step Time": 5.007901495322585} +{"Finetune/Learning Rate": 3.5776931187092667e-06, "Finetune/Loss": 0.8417261838912964, "Finetune/Loss (Raw)": 0.9034723043441772, "Finetune/Step": 3796, "Finetune/Step Time": 5.013532891869545} +{"Finetune/Learning Rate": 3.5729192975503856e-06, "Finetune/Loss": 0.8408282399177551, "Finetune/Loss (Raw)": 0.7430686950683594, "Finetune/Step": 3797, "Finetune/Step Time": 5.018256017938256} +{"Finetune/Learning Rate": 3.5681479706160483e-06, "Finetune/Loss": 0.8402200937271118, "Finetune/Loss (Raw)": 0.8318389654159546, "Finetune/Step": 3798, "Finetune/Step Time": 5.01488708332181} +{"Finetune/Learning Rate": 3.563379139757911e-06, "Finetune/Loss": 0.8389148712158203, "Finetune/Loss (Raw)": 0.8069621324539185, "Finetune/Step": 3799, "Finetune/Step Time": 5.024803688749671} +{"Finetune/Learning Rate": 3.558612806826666e-06, "Finetune/Loss": 0.8397554755210876, "Finetune/Loss (Raw)": 0.9504613280296326, "Finetune/Step": 3800, "Finetune/Step Time": 5.020302224904299} +{"Finetune/Learning Rate": 3.553848973672033e-06, "Finetune/Loss": 0.8413288593292236, "Finetune/Loss (Raw)": 0.8702923655509949, "Finetune/Step": 3801, "Finetune/Step Time": 5.021346997469664} +{"Finetune/Learning Rate": 3.549087642142762e-06, "Finetune/Loss": 0.8422197103500366, "Finetune/Loss (Raw)": 0.9652732610702515, "Finetune/Step": 3802, "Finetune/Step Time": 5.022553285583854} +{"Finetune/Learning Rate": 3.544328814086632e-06, "Finetune/Loss": 0.8471220135688782, "Finetune/Loss (Raw)": 1.003913402557373, "Finetune/Step": 3803, "Finetune/Step Time": 4.98288793861866} +{"Finetune/Learning Rate": 3.5395724913504546e-06, "Finetune/Loss": 0.8468469381332397, "Finetune/Loss (Raw)": 0.9156524538993835, "Finetune/Step": 3804, "Finetune/Step Time": 4.967154094949365} +{"Finetune/Learning Rate": 3.5348186757800683e-06, "Finetune/Loss": 0.8459372520446777, "Finetune/Loss (Raw)": 0.8492549657821655, "Finetune/Step": 3805, "Finetune/Step Time": 4.984397727996111} +{"Finetune/Learning Rate": 3.530067369220326e-06, "Finetune/Loss": 0.8478096127510071, "Finetune/Loss (Raw)": 0.8863861560821533, "Finetune/Step": 3806, "Finetune/Step Time": 5.00176271982491} +{"Finetune/Learning Rate": 3.525318573515125e-06, "Finetune/Loss": 0.8466775417327881, "Finetune/Loss (Raw)": 0.8421698808670044, "Finetune/Step": 3807, "Finetune/Step Time": 4.998916774988174} +{"Finetune/Learning Rate": 3.520572290507378e-06, "Finetune/Loss": 0.8456217050552368, "Finetune/Loss (Raw)": 0.77649986743927, "Finetune/Step": 3808, "Finetune/Step Time": 5.005762364715338} +{"Finetune/Learning Rate": 3.5158285220390256e-06, "Finetune/Loss": 0.8446499109268188, "Finetune/Loss (Raw)": 0.8508362174034119, "Finetune/Step": 3809, "Finetune/Step Time": 5.020373368635774} +{"Finetune/Learning Rate": 3.5110872699510313e-06, "Finetune/Loss": 0.8434066772460938, "Finetune/Loss (Raw)": 0.9359643459320068, "Finetune/Step": 3810, "Finetune/Step Time": 5.017270859330893} +{"Finetune/Learning Rate": 3.5063485360833816e-06, "Finetune/Loss": 0.8430322408676147, "Finetune/Loss (Raw)": 0.841253399848938, "Finetune/Step": 3811, "Finetune/Step Time": 5.023540196940303} +{"Finetune/Learning Rate": 3.5016123222750863e-06, "Finetune/Loss": 0.8440881967544556, "Finetune/Loss (Raw)": 0.8003072142601013, "Finetune/Step": 3812, "Finetune/Step Time": 5.017627470195293} +{"Finetune/Learning Rate": 3.4968786303641764e-06, "Finetune/Loss": 0.8442275524139404, "Finetune/Loss (Raw)": 0.8759465217590332, "Finetune/Step": 3813, "Finetune/Step Time": 5.021007591858506} +{"Finetune/Learning Rate": 3.492147462187714e-06, "Finetune/Loss": 0.8436405658721924, "Finetune/Loss (Raw)": 0.8378092646598816, "Finetune/Step": 3814, "Finetune/Step Time": 5.0187626499682665} +{"Finetune/Learning Rate": 3.4874188195817626e-06, "Finetune/Loss": 0.8435122966766357, "Finetune/Loss (Raw)": 0.8650600910186768, "Finetune/Step": 3815, "Finetune/Step Time": 5.037091528996825} +{"Finetune/Learning Rate": 3.4826927043814197e-06, "Finetune/Loss": 0.8439890146255493, "Finetune/Loss (Raw)": 0.8814717531204224, "Finetune/Step": 3816, "Finetune/Step Time": 5.032290795817971} +{"Finetune/Learning Rate": 3.4779691184207954e-06, "Finetune/Loss": 0.8425866365432739, "Finetune/Loss (Raw)": 0.806920051574707, "Finetune/Step": 3817, "Finetune/Step Time": 5.029368149116635} +{"Finetune/Learning Rate": 3.473248063533028e-06, "Finetune/Loss": 0.8417048454284668, "Finetune/Loss (Raw)": 0.8178386092185974, "Finetune/Step": 3818, "Finetune/Step Time": 5.0225512478500605} +{"Finetune/Learning Rate": 3.4685295415502663e-06, "Finetune/Loss": 0.8455610275268555, "Finetune/Loss (Raw)": 0.8038730025291443, "Finetune/Step": 3819, "Finetune/Step Time": 4.988956294953823} +{"Finetune/Learning Rate": 3.4638135543036656e-06, "Finetune/Loss": 0.8433561325073242, "Finetune/Loss (Raw)": 0.7136712670326233, "Finetune/Step": 3820, "Finetune/Step Time": 4.9911282155662775} +{"Finetune/Learning Rate": 3.4591001036234206e-06, "Finetune/Loss": 0.8427316546440125, "Finetune/Loss (Raw)": 0.8159832954406738, "Finetune/Step": 3821, "Finetune/Step Time": 4.996684316545725} +{"Finetune/Learning Rate": 3.4543891913387253e-06, "Finetune/Loss": 0.8427690863609314, "Finetune/Loss (Raw)": 0.780211329460144, "Finetune/Step": 3822, "Finetune/Step Time": 4.975356537848711} +{"Finetune/Learning Rate": 3.449680819277792e-06, "Finetune/Loss": 0.841725766658783, "Finetune/Loss (Raw)": 0.9803489446640015, "Finetune/Step": 3823, "Finetune/Step Time": 4.978379558771849} +{"Finetune/Learning Rate": 3.4449749892678486e-06, "Finetune/Loss": 0.8418641090393066, "Finetune/Loss (Raw)": 0.8952284455299377, "Finetune/Step": 3824, "Finetune/Step Time": 4.977707365527749} +{"Finetune/Learning Rate": 3.4402717031351363e-06, "Finetune/Loss": 0.8421777486801147, "Finetune/Loss (Raw)": 0.7591679096221924, "Finetune/Step": 3825, "Finetune/Step Time": 4.956051342189312} +{"Finetune/Learning Rate": 3.4355709627049073e-06, "Finetune/Loss": 0.8413878679275513, "Finetune/Loss (Raw)": 0.81618332862854, "Finetune/Step": 3826, "Finetune/Step Time": 4.95203173160553} +{"Finetune/Learning Rate": 3.4308727698014243e-06, "Finetune/Loss": 0.8412003517150879, "Finetune/Loss (Raw)": 0.898056149482727, "Finetune/Step": 3827, "Finetune/Step Time": 4.980475835502148} +{"Finetune/Learning Rate": 3.4261771262479724e-06, "Finetune/Loss": 0.8414393663406372, "Finetune/Loss (Raw)": 0.9542396664619446, "Finetune/Step": 3828, "Finetune/Step Time": 4.983620535582304} +{"Finetune/Learning Rate": 3.4214840338668376e-06, "Finetune/Loss": 0.8379707336425781, "Finetune/Loss (Raw)": 0.2795577943325043, "Finetune/Step": 3829, "Finetune/Step Time": 5.013260284438729} +{"Finetune/Learning Rate": 3.416793494479308e-06, "Finetune/Loss": 0.8376567363739014, "Finetune/Loss (Raw)": 0.889981746673584, "Finetune/Step": 3830, "Finetune/Step Time": 5.02242249250412} +{"Finetune/Learning Rate": 3.412105509905701e-06, "Finetune/Loss": 0.836711049079895, "Finetune/Loss (Raw)": 0.8309708833694458, "Finetune/Step": 3831, "Finetune/Step Time": 5.029477939009666} +{"Finetune/Learning Rate": 3.407420081965327e-06, "Finetune/Loss": 0.8376129269599915, "Finetune/Loss (Raw)": 1.0358707904815674, "Finetune/Step": 3832, "Finetune/Step Time": 5.026261169463396} +{"Finetune/Learning Rate": 3.402737212476509e-06, "Finetune/Loss": 0.8395028710365295, "Finetune/Loss (Raw)": 0.9649345874786377, "Finetune/Step": 3833, "Finetune/Step Time": 5.049685664474964} +{"Finetune/Learning Rate": 3.398056903256579e-06, "Finetune/Loss": 0.8403249979019165, "Finetune/Loss (Raw)": 0.9712814688682556, "Finetune/Step": 3834, "Finetune/Step Time": 5.043757261708379} +{"Finetune/Learning Rate": 3.393379156121873e-06, "Finetune/Loss": 0.8410553932189941, "Finetune/Loss (Raw)": 0.9466208219528198, "Finetune/Step": 3835, "Finetune/Step Time": 5.024215238168836} +{"Finetune/Learning Rate": 3.3887039728877336e-06, "Finetune/Loss": 0.8413586616516113, "Finetune/Loss (Raw)": 0.8162333965301514, "Finetune/Step": 3836, "Finetune/Step Time": 5.001934312283993} +{"Finetune/Learning Rate": 3.3840313553685034e-06, "Finetune/Loss": 0.8400909900665283, "Finetune/Loss (Raw)": 0.852096676826477, "Finetune/Step": 3837, "Finetune/Step Time": 5.0075478926301} +{"Finetune/Learning Rate": 3.379361305377542e-06, "Finetune/Loss": 0.8387349247932434, "Finetune/Loss (Raw)": 0.8177230358123779, "Finetune/Step": 3838, "Finetune/Step Time": 5.005412463098764} +{"Finetune/Learning Rate": 3.374693824727204e-06, "Finetune/Loss": 0.8395637273788452, "Finetune/Loss (Raw)": 0.8854429125785828, "Finetune/Step": 3839, "Finetune/Step Time": 5.020342130213976} +{"Finetune/Learning Rate": 3.3700289152288377e-06, "Finetune/Loss": 0.8382554054260254, "Finetune/Loss (Raw)": 0.8420843482017517, "Finetune/Step": 3840, "Finetune/Step Time": 5.020468672737479} +{"Finetune/Learning Rate": 3.3653665786928138e-06, "Finetune/Loss": 0.8438804745674133, "Finetune/Loss (Raw)": 1.0042351484298706, "Finetune/Step": 3841, "Finetune/Step Time": 4.985005639493465} +{"Finetune/Learning Rate": 3.360706816928491e-06, "Finetune/Loss": 0.8431388139724731, "Finetune/Loss (Raw)": 0.8075006008148193, "Finetune/Step": 3842, "Finetune/Step Time": 4.9830790143460035} +{"Finetune/Learning Rate": 3.3560496317442305e-06, "Finetune/Loss": 0.8422696590423584, "Finetune/Loss (Raw)": 0.8354584574699402, "Finetune/Step": 3843, "Finetune/Step Time": 4.985120944678783} +{"Finetune/Learning Rate": 3.351395024947395e-06, "Finetune/Loss": 0.84256911277771, "Finetune/Loss (Raw)": 0.8480799794197083, "Finetune/Step": 3844, "Finetune/Step Time": 4.9658178221434355} +{"Finetune/Learning Rate": 3.3467429983443477e-06, "Finetune/Loss": 0.8430458307266235, "Finetune/Loss (Raw)": 0.8396515250205994, "Finetune/Step": 3845, "Finetune/Step Time": 4.949174098670483} +{"Finetune/Learning Rate": 3.34209355374045e-06, "Finetune/Loss": 0.8423566222190857, "Finetune/Loss (Raw)": 0.6966618895530701, "Finetune/Step": 3846, "Finetune/Step Time": 4.944296253845096} +{"Finetune/Learning Rate": 3.337446692940055e-06, "Finetune/Loss": 0.8447088599205017, "Finetune/Loss (Raw)": 0.9833901524543762, "Finetune/Step": 3847, "Finetune/Step Time": 4.932190315797925} +{"Finetune/Learning Rate": 3.332802417746527e-06, "Finetune/Loss": 0.8395124673843384, "Finetune/Loss (Raw)": 0.32964423298835754, "Finetune/Step": 3848, "Finetune/Step Time": 4.956872662529349} +{"Finetune/Learning Rate": 3.3281607299622175e-06, "Finetune/Loss": 0.8400058150291443, "Finetune/Loss (Raw)": 1.0442463159561157, "Finetune/Step": 3849, "Finetune/Step Time": 4.9725788198411465} +{"Finetune/Learning Rate": 3.3235216313884657e-06, "Finetune/Loss": 0.8376648426055908, "Finetune/Loss (Raw)": 0.6249667406082153, "Finetune/Step": 3850, "Finetune/Step Time": 4.9801642671227455} +{"Finetune/Learning Rate": 3.3188851238256246e-06, "Finetune/Loss": 0.835995078086853, "Finetune/Loss (Raw)": 0.7640799283981323, "Finetune/Step": 3851, "Finetune/Step Time": 4.971603408455849} +{"Finetune/Learning Rate": 3.3142512090730283e-06, "Finetune/Loss": 0.8368030786514282, "Finetune/Loss (Raw)": 0.9227889180183411, "Finetune/Step": 3852, "Finetune/Step Time": 4.966755883768201} +{"Finetune/Learning Rate": 3.309619888929011e-06, "Finetune/Loss": 0.832813560962677, "Finetune/Loss (Raw)": 0.4554162621498108, "Finetune/Step": 3853, "Finetune/Step Time": 4.99756795540452} +{"Finetune/Learning Rate": 3.304991165190895e-06, "Finetune/Loss": 0.8368138074874878, "Finetune/Loss (Raw)": 0.8961214423179626, "Finetune/Step": 3854, "Finetune/Step Time": 4.96107261069119} +{"Finetune/Learning Rate": 3.300365039655e-06, "Finetune/Loss": 0.8354111313819885, "Finetune/Loss (Raw)": 0.6481566429138184, "Finetune/Step": 3855, "Finetune/Step Time": 4.971870606765151} +{"Finetune/Learning Rate": 3.2957415141166327e-06, "Finetune/Loss": 0.8377548456192017, "Finetune/Loss (Raw)": 0.9879522323608398, "Finetune/Step": 3856, "Finetune/Step Time": 4.964546849951148} +{"Finetune/Learning Rate": 3.2911205903700916e-06, "Finetune/Loss": 0.8333833813667297, "Finetune/Loss (Raw)": 0.3628320097923279, "Finetune/Step": 3857, "Finetune/Step Time": 4.988527279347181} +{"Finetune/Learning Rate": 3.2865022702086756e-06, "Finetune/Loss": 0.8288764953613281, "Finetune/Loss (Raw)": 0.336290180683136, "Finetune/Step": 3858, "Finetune/Step Time": 5.003664489835501} +{"Finetune/Learning Rate": 3.281886555424656e-06, "Finetune/Loss": 0.8298314809799194, "Finetune/Loss (Raw)": 1.0621387958526611, "Finetune/Step": 3859, "Finetune/Step Time": 4.996043831110001} +{"Finetune/Learning Rate": 3.277273447809299e-06, "Finetune/Loss": 0.829791784286499, "Finetune/Loss (Raw)": 0.8907878398895264, "Finetune/Step": 3860, "Finetune/Step Time": 4.988762345165014} +{"Finetune/Learning Rate": 3.2726629491528714e-06, "Finetune/Loss": 0.8297237753868103, "Finetune/Loss (Raw)": 0.8452129364013672, "Finetune/Step": 3861, "Finetune/Step Time": 4.979580000042915} +{"Finetune/Learning Rate": 3.268055061244614e-06, "Finetune/Loss": 0.8291188478469849, "Finetune/Loss (Raw)": 0.8893656134605408, "Finetune/Step": 3862, "Finetune/Step Time": 4.977985464036465} +{"Finetune/Learning Rate": 3.2634497858727553e-06, "Finetune/Loss": 0.8300985097885132, "Finetune/Loss (Raw)": 0.839532732963562, "Finetune/Step": 3863, "Finetune/Step Time": 4.9881178215146065} +{"Finetune/Learning Rate": 3.2588471248245157e-06, "Finetune/Loss": 0.8288146257400513, "Finetune/Loss (Raw)": 0.9246843457221985, "Finetune/Step": 3864, "Finetune/Step Time": 4.991021139547229} +{"Finetune/Learning Rate": 3.254247079886097e-06, "Finetune/Loss": 0.8284991383552551, "Finetune/Loss (Raw)": 0.8766121864318848, "Finetune/Step": 3865, "Finetune/Step Time": 5.008602809160948} +{"Finetune/Learning Rate": 3.2496496528426867e-06, "Finetune/Loss": 0.8300501108169556, "Finetune/Loss (Raw)": 1.033115267753601, "Finetune/Step": 3866, "Finetune/Step Time": 4.9949861615896225} +{"Finetune/Learning Rate": 3.245054845478454e-06, "Finetune/Loss": 0.8319336771965027, "Finetune/Loss (Raw)": 1.081773281097412, "Finetune/Step": 3867, "Finetune/Step Time": 5.001017026603222} +{"Finetune/Learning Rate": 3.240462659576562e-06, "Finetune/Loss": 0.8320080041885376, "Finetune/Loss (Raw)": 0.8956562280654907, "Finetune/Step": 3868, "Finetune/Step Time": 5.0042883064597845} +{"Finetune/Learning Rate": 3.23587309691914e-06, "Finetune/Loss": 0.8308175802230835, "Finetune/Loss (Raw)": 0.8128206729888916, "Finetune/Step": 3869, "Finetune/Step Time": 5.011199643835425} +{"Finetune/Learning Rate": 3.231286159287308e-06, "Finetune/Loss": 0.8302507400512695, "Finetune/Loss (Raw)": 0.8016518950462341, "Finetune/Step": 3870, "Finetune/Step Time": 5.035775175318122} +{"Finetune/Learning Rate": 3.2267018484611725e-06, "Finetune/Loss": 0.8298659920692444, "Finetune/Loss (Raw)": 0.8747338652610779, "Finetune/Step": 3871, "Finetune/Step Time": 4.994223916903138} +{"Finetune/Learning Rate": 3.222120166219812e-06, "Finetune/Loss": 0.8310813307762146, "Finetune/Loss (Raw)": 0.5165099501609802, "Finetune/Step": 3872, "Finetune/Step Time": 4.993891818448901} +{"Finetune/Learning Rate": 3.217541114341288e-06, "Finetune/Loss": 0.8343153595924377, "Finetune/Loss (Raw)": 0.7398852109909058, "Finetune/Step": 3873, "Finetune/Step Time": 4.967689596116543} +{"Finetune/Learning Rate": 3.2129646946026407e-06, "Finetune/Loss": 0.8353455662727356, "Finetune/Loss (Raw)": 0.8806554079055786, "Finetune/Step": 3874, "Finetune/Step Time": 4.96207363717258} +{"Finetune/Learning Rate": 3.2083909087798905e-06, "Finetune/Loss": 0.836360514163971, "Finetune/Loss (Raw)": 0.8789235353469849, "Finetune/Step": 3875, "Finetune/Step Time": 4.944157935678959} +{"Finetune/Learning Rate": 3.2038197586480336e-06, "Finetune/Loss": 0.8358970880508423, "Finetune/Loss (Raw)": 0.8842517137527466, "Finetune/Step": 3876, "Finetune/Step Time": 4.930502958595753} +{"Finetune/Learning Rate": 3.199251245981041e-06, "Finetune/Loss": 0.8364838361740112, "Finetune/Loss (Raw)": 0.8569427132606506, "Finetune/Step": 3877, "Finetune/Step Time": 4.938337545841932} +{"Finetune/Learning Rate": 3.1946853725518724e-06, "Finetune/Loss": 0.8327736854553223, "Finetune/Loss (Raw)": 0.4704601764678955, "Finetune/Step": 3878, "Finetune/Step Time": 4.972465423867106} +{"Finetune/Learning Rate": 3.1901221401324446e-06, "Finetune/Loss": 0.8314566612243652, "Finetune/Loss (Raw)": 0.8465884327888489, "Finetune/Step": 3879, "Finetune/Step Time": 4.99797310307622} +{"Finetune/Learning Rate": 3.18556155049366e-06, "Finetune/Loss": 0.8316217660903931, "Finetune/Loss (Raw)": 0.8689053654670715, "Finetune/Step": 3880, "Finetune/Step Time": 4.99445384927094} +{"Finetune/Learning Rate": 3.1810036054054005e-06, "Finetune/Loss": 0.8320380449295044, "Finetune/Loss (Raw)": 0.9188117980957031, "Finetune/Step": 3881, "Finetune/Step Time": 4.987006431445479} +{"Finetune/Learning Rate": 3.1764483066365117e-06, "Finetune/Loss": 0.8323651552200317, "Finetune/Loss (Raw)": 0.948076605796814, "Finetune/Step": 3882, "Finetune/Step Time": 4.996735038235784} +{"Finetune/Learning Rate": 3.1718956559548154e-06, "Finetune/Loss": 0.8326095342636108, "Finetune/Loss (Raw)": 0.9807563424110413, "Finetune/Step": 3883, "Finetune/Step Time": 4.990497509017587} +{"Finetune/Learning Rate": 3.1673456551271086e-06, "Finetune/Loss": 0.8334735631942749, "Finetune/Loss (Raw)": 0.8216292262077332, "Finetune/Step": 3884, "Finetune/Step Time": 4.983434913679957} +{"Finetune/Learning Rate": 3.162798305919157e-06, "Finetune/Loss": 0.8312959671020508, "Finetune/Loss (Raw)": 0.675853431224823, "Finetune/Step": 3885, "Finetune/Step Time": 4.990041317418218} +{"Finetune/Learning Rate": 3.1582536100956973e-06, "Finetune/Loss": 0.8304744958877563, "Finetune/Loss (Raw)": 0.7528259754180908, "Finetune/Step": 3886, "Finetune/Step Time": 4.976257003843784} +{"Finetune/Learning Rate": 3.1537115694204345e-06, "Finetune/Loss": 0.8292725086212158, "Finetune/Loss (Raw)": 0.7983542084693909, "Finetune/Step": 3887, "Finetune/Step Time": 4.9774209670722485} +{"Finetune/Learning Rate": 3.1491721856560555e-06, "Finetune/Loss": 0.8298467397689819, "Finetune/Loss (Raw)": 0.9112516045570374, "Finetune/Step": 3888, "Finetune/Step Time": 4.969185585156083} +{"Finetune/Learning Rate": 3.144635460564197e-06, "Finetune/Loss": 0.8308625221252441, "Finetune/Loss (Raw)": 0.879608154296875, "Finetune/Step": 3889, "Finetune/Step Time": 4.960117941722274} +{"Finetune/Learning Rate": 3.1401013959054726e-06, "Finetune/Loss": 0.8311144113540649, "Finetune/Loss (Raw)": 0.9163339138031006, "Finetune/Step": 3890, "Finetune/Step Time": 4.953744098544121} +{"Finetune/Learning Rate": 3.1355699934394724e-06, "Finetune/Loss": 0.8306161165237427, "Finetune/Loss (Raw)": 0.8730665445327759, "Finetune/Step": 3891, "Finetune/Step Time": 4.968385903164744} +{"Finetune/Learning Rate": 3.1310412549247403e-06, "Finetune/Loss": 0.830951452255249, "Finetune/Loss (Raw)": 0.8559755682945251, "Finetune/Step": 3892, "Finetune/Step Time": 4.971710147336125} +{"Finetune/Learning Rate": 3.1265151821187933e-06, "Finetune/Loss": 0.8272119760513306, "Finetune/Loss (Raw)": 0.3679364323616028, "Finetune/Step": 3893, "Finetune/Step Time": 4.99784673564136} +{"Finetune/Learning Rate": 3.121991776778109e-06, "Finetune/Loss": 0.826910674571991, "Finetune/Loss (Raw)": 0.9064831733703613, "Finetune/Step": 3894, "Finetune/Step Time": 4.996460312977433} +{"Finetune/Learning Rate": 3.1174710406581355e-06, "Finetune/Loss": 0.8267550468444824, "Finetune/Loss (Raw)": 0.9242409467697144, "Finetune/Step": 3895, "Finetune/Step Time": 4.994484752416611} +{"Finetune/Learning Rate": 3.1129529755132815e-06, "Finetune/Loss": 0.8281221389770508, "Finetune/Loss (Raw)": 0.9371461272239685, "Finetune/Step": 3896, "Finetune/Step Time": 4.983209040015936} +{"Finetune/Learning Rate": 3.108437583096918e-06, "Finetune/Loss": 0.8330389261245728, "Finetune/Loss (Raw)": 0.8785737752914429, "Finetune/Step": 3897, "Finetune/Step Time": 4.947286682203412} +{"Finetune/Learning Rate": 3.103924865161383e-06, "Finetune/Loss": 0.8316084742546082, "Finetune/Loss (Raw)": 0.8575934767723083, "Finetune/Step": 3898, "Finetune/Step Time": 4.9639887008816} +{"Finetune/Learning Rate": 3.0994148234579734e-06, "Finetune/Loss": 0.8310381174087524, "Finetune/Loss (Raw)": 1.0012640953063965, "Finetune/Step": 3899, "Finetune/Step Time": 4.967132639139891} +{"Finetune/Learning Rate": 3.0949074597369444e-06, "Finetune/Loss": 0.8302887082099915, "Finetune/Loss (Raw)": 0.8623986840248108, "Finetune/Step": 3900, "Finetune/Step Time": 4.977950153872371} +{"Finetune/Learning Rate": 3.090402775747523e-06, "Finetune/Loss": 0.8348985910415649, "Finetune/Loss (Raw)": 0.9350366592407227, "Finetune/Step": 3901, "Finetune/Step Time": 4.942249517887831} +{"Finetune/Learning Rate": 3.0859007732378897e-06, "Finetune/Loss": 0.8352611064910889, "Finetune/Loss (Raw)": 0.9724917411804199, "Finetune/Step": 3902, "Finetune/Step Time": 4.926172519102693} +{"Finetune/Learning Rate": 3.0814014539551727e-06, "Finetune/Loss": 0.8351792097091675, "Finetune/Loss (Raw)": 0.9553187489509583, "Finetune/Step": 3903, "Finetune/Step Time": 4.921213699504733} +{"Finetune/Learning Rate": 3.076904819645481e-06, "Finetune/Loss": 0.8348251581192017, "Finetune/Loss (Raw)": 0.8948693871498108, "Finetune/Step": 3904, "Finetune/Step Time": 4.924597280099988} +{"Finetune/Learning Rate": 3.072410872053867e-06, "Finetune/Loss": 0.8380831480026245, "Finetune/Loss (Raw)": 0.6897489428520203, "Finetune/Step": 3905, "Finetune/Step Time": 4.902682362124324} +{"Finetune/Learning Rate": 3.067919612924343e-06, "Finetune/Loss": 0.8382004499435425, "Finetune/Loss (Raw)": 0.9389352202415466, "Finetune/Step": 3906, "Finetune/Step Time": 4.941379576921463} +{"Finetune/Learning Rate": 3.0634310439998803e-06, "Finetune/Loss": 0.8405735492706299, "Finetune/Loss (Raw)": 1.1531633138656616, "Finetune/Step": 3907, "Finetune/Step Time": 4.920966096222401} +{"Finetune/Learning Rate": 3.0589451670224037e-06, "Finetune/Loss": 0.8406754732131958, "Finetune/Loss (Raw)": 0.8193365335464478, "Finetune/Step": 3908, "Finetune/Step Time": 4.922942956909537} +{"Finetune/Learning Rate": 3.054461983732795e-06, "Finetune/Loss": 0.839714527130127, "Finetune/Loss (Raw)": 0.8259808421134949, "Finetune/Step": 3909, "Finetune/Step Time": 4.92895114980638} +{"Finetune/Learning Rate": 3.0499814958708883e-06, "Finetune/Loss": 0.8406161665916443, "Finetune/Loss (Raw)": 0.9571882486343384, "Finetune/Step": 3910, "Finetune/Step Time": 4.929652597755194} +{"Finetune/Learning Rate": 3.045503705175478e-06, "Finetune/Loss": 0.842326283454895, "Finetune/Loss (Raw)": 1.0224285125732422, "Finetune/Step": 3911, "Finetune/Step Time": 4.925947789102793} +{"Finetune/Learning Rate": 3.041028613384307e-06, "Finetune/Loss": 0.842393159866333, "Finetune/Loss (Raw)": 0.8178396224975586, "Finetune/Step": 3912, "Finetune/Step Time": 4.929137215018272} +{"Finetune/Learning Rate": 3.036556222234063e-06, "Finetune/Loss": 0.8425494432449341, "Finetune/Loss (Raw)": 0.8946117758750916, "Finetune/Step": 3913, "Finetune/Step Time": 4.932722708210349} +{"Finetune/Learning Rate": 3.032086533460402e-06, "Finetune/Loss": 0.8414115905761719, "Finetune/Loss (Raw)": 0.8357036113739014, "Finetune/Step": 3914, "Finetune/Step Time": 4.94096090272069} +{"Finetune/Learning Rate": 3.027619548797921e-06, "Finetune/Loss": 0.841223955154419, "Finetune/Loss (Raw)": 0.9052249789237976, "Finetune/Step": 3915, "Finetune/Step Time": 4.938296051695943} +{"Finetune/Learning Rate": 3.023155269980169e-06, "Finetune/Loss": 0.841381311416626, "Finetune/Loss (Raw)": 0.8914899230003357, "Finetune/Step": 3916, "Finetune/Step Time": 4.938637860119343} +{"Finetune/Learning Rate": 3.0186936987396442e-06, "Finetune/Loss": 0.8421674966812134, "Finetune/Loss (Raw)": 0.8911517262458801, "Finetune/Step": 3917, "Finetune/Step Time": 4.933308891952038} +{"Finetune/Learning Rate": 3.014234836807798e-06, "Finetune/Loss": 0.8422187566757202, "Finetune/Loss (Raw)": 0.9535669684410095, "Finetune/Step": 3918, "Finetune/Step Time": 4.932923214510083} +{"Finetune/Learning Rate": 3.0097786859150247e-06, "Finetune/Loss": 0.8420151472091675, "Finetune/Loss (Raw)": 0.8494359254837036, "Finetune/Step": 3919, "Finetune/Step Time": 4.948047589510679} +{"Finetune/Learning Rate": 3.005325247790668e-06, "Finetune/Loss": 0.8462307453155518, "Finetune/Loss (Raw)": 0.8838427662849426, "Finetune/Step": 3920, "Finetune/Step Time": 4.937743544578552} +{"Finetune/Learning Rate": 3.0008745241630266e-06, "Finetune/Loss": 0.8443400859832764, "Finetune/Loss (Raw)": 0.6736550331115723, "Finetune/Step": 3921, "Finetune/Step Time": 4.947466125711799} +{"Finetune/Learning Rate": 2.9964265167593386e-06, "Finetune/Loss": 0.844576358795166, "Finetune/Loss (Raw)": 0.9541491866111755, "Finetune/Step": 3922, "Finetune/Step Time": 4.950525743886828} +{"Finetune/Learning Rate": 2.9919812273057815e-06, "Finetune/Loss": 0.844359278678894, "Finetune/Loss (Raw)": 0.727533221244812, "Finetune/Step": 3923, "Finetune/Step Time": 4.955320600420237} +{"Finetune/Learning Rate": 2.987538657527491e-06, "Finetune/Loss": 0.8443747758865356, "Finetune/Loss (Raw)": 0.9054591655731201, "Finetune/Step": 3924, "Finetune/Step Time": 4.958785783499479} +{"Finetune/Learning Rate": 2.9830988091485426e-06, "Finetune/Loss": 0.8456946015357971, "Finetune/Loss (Raw)": 0.9120113849639893, "Finetune/Step": 3925, "Finetune/Step Time": 4.955197906121612} +{"Finetune/Learning Rate": 2.9786616838919515e-06, "Finetune/Loss": 0.8455913662910461, "Finetune/Loss (Raw)": 0.8186231851577759, "Finetune/Step": 3926, "Finetune/Step Time": 4.959476003423333} +{"Finetune/Learning Rate": 2.9742272834796813e-06, "Finetune/Loss": 0.8459861278533936, "Finetune/Loss (Raw)": 0.8574947118759155, "Finetune/Step": 3927, "Finetune/Step Time": 4.966663163155317} +{"Finetune/Learning Rate": 2.969795609632636e-06, "Finetune/Loss": 0.8462287187576294, "Finetune/Loss (Raw)": 0.9815104007720947, "Finetune/Step": 3928, "Finetune/Step Time": 4.9677126333117485} +{"Finetune/Learning Rate": 2.9653666640706614e-06, "Finetune/Loss": 0.8465856313705444, "Finetune/Loss (Raw)": 0.9159790873527527, "Finetune/Step": 3929, "Finetune/Step Time": 4.963847219944} +{"Finetune/Learning Rate": 2.9609404485125414e-06, "Finetune/Loss": 0.8456597328186035, "Finetune/Loss (Raw)": 0.846766471862793, "Finetune/Step": 3930, "Finetune/Step Time": 4.96962926723063} +{"Finetune/Learning Rate": 2.9565169646760108e-06, "Finetune/Loss": 0.8454991579055786, "Finetune/Loss (Raw)": 0.983354926109314, "Finetune/Step": 3931, "Finetune/Step Time": 4.970878081396222} +{"Finetune/Learning Rate": 2.9520962142777367e-06, "Finetune/Loss": 0.8449293971061707, "Finetune/Loss (Raw)": 0.8427185416221619, "Finetune/Step": 3932, "Finetune/Step Time": 4.98569761775434} +{"Finetune/Learning Rate": 2.947678199033317e-06, "Finetune/Loss": 0.8434065580368042, "Finetune/Loss (Raw)": 0.6543309688568115, "Finetune/Step": 3933, "Finetune/Step Time": 4.994014436379075} +{"Finetune/Learning Rate": 2.943262920657306e-06, "Finetune/Loss": 0.8426502346992493, "Finetune/Loss (Raw)": 0.7895742654800415, "Finetune/Step": 3934, "Finetune/Step Time": 4.981318537145853} +{"Finetune/Learning Rate": 2.938850380863184e-06, "Finetune/Loss": 0.842668890953064, "Finetune/Loss (Raw)": 0.8445641994476318, "Finetune/Step": 3935, "Finetune/Step Time": 4.983577059581876} +{"Finetune/Learning Rate": 2.9344405813633724e-06, "Finetune/Loss": 0.843966007232666, "Finetune/Loss (Raw)": 0.9425215125083923, "Finetune/Step": 3936, "Finetune/Step Time": 4.980020549148321} +{"Finetune/Learning Rate": 2.930033523869228e-06, "Finetune/Loss": 0.8443266749382019, "Finetune/Loss (Raw)": 0.8970066905021667, "Finetune/Step": 3937, "Finetune/Step Time": 4.974497428163886} +{"Finetune/Learning Rate": 2.925629210091043e-06, "Finetune/Loss": 0.8437579870223999, "Finetune/Loss (Raw)": 0.8631744384765625, "Finetune/Step": 3938, "Finetune/Step Time": 4.974675619974732} +{"Finetune/Learning Rate": 2.9212276417380468e-06, "Finetune/Loss": 0.8445318937301636, "Finetune/Loss (Raw)": 0.940311849117279, "Finetune/Step": 3939, "Finetune/Step Time": 4.9759154338389635} +{"Finetune/Learning Rate": 2.9168288205183983e-06, "Finetune/Loss": 0.8451101779937744, "Finetune/Loss (Raw)": 0.8743240237236023, "Finetune/Step": 3940, "Finetune/Step Time": 4.963427405804396} +{"Finetune/Learning Rate": 2.9124327481392023e-06, "Finetune/Loss": 0.8457790613174438, "Finetune/Loss (Raw)": 0.9615656137466431, "Finetune/Step": 3941, "Finetune/Step Time": 4.960926581174135} +{"Finetune/Learning Rate": 2.9080394263064815e-06, "Finetune/Loss": 0.8468453288078308, "Finetune/Loss (Raw)": 0.9742911458015442, "Finetune/Step": 3942, "Finetune/Step Time": 4.967528089880943} +{"Finetune/Learning Rate": 2.9036488567252006e-06, "Finetune/Loss": 0.8463166952133179, "Finetune/Loss (Raw)": 0.7974010705947876, "Finetune/Step": 3943, "Finetune/Step Time": 4.973680192604661} +{"Finetune/Learning Rate": 2.8992610410992506e-06, "Finetune/Loss": 0.8465869426727295, "Finetune/Loss (Raw)": 0.916061282157898, "Finetune/Step": 3944, "Finetune/Step Time": 4.9733691569417715} +{"Finetune/Learning Rate": 2.894875981131463e-06, "Finetune/Loss": 0.8487537503242493, "Finetune/Loss (Raw)": 1.0842713117599487, "Finetune/Step": 3945, "Finetune/Step Time": 4.995907586067915} +{"Finetune/Learning Rate": 2.890493678523595e-06, "Finetune/Loss": 0.849236249923706, "Finetune/Loss (Raw)": 0.8796057105064392, "Finetune/Step": 3946, "Finetune/Step Time": 5.0101682133972645} +{"Finetune/Learning Rate": 2.8861141349763224e-06, "Finetune/Loss": 0.8500936031341553, "Finetune/Loss (Raw)": 0.9136040210723877, "Finetune/Step": 3947, "Finetune/Step Time": 5.017622837796807} +{"Finetune/Learning Rate": 2.881737352189271e-06, "Finetune/Loss": 0.8473469614982605, "Finetune/Loss (Raw)": 0.36210304498672485, "Finetune/Step": 3948, "Finetune/Step Time": 5.053768774494529} +{"Finetune/Learning Rate": 2.8773633318609796e-06, "Finetune/Loss": 0.8488350510597229, "Finetune/Loss (Raw)": 1.0064622163772583, "Finetune/Step": 3949, "Finetune/Step Time": 5.049751901999116} +{"Finetune/Learning Rate": 2.872992075688922e-06, "Finetune/Loss": 0.850400984287262, "Finetune/Loss (Raw)": 0.9806448221206665, "Finetune/Step": 3950, "Finetune/Step Time": 5.052890757098794} +{"Finetune/Learning Rate": 2.868623585369497e-06, "Finetune/Loss": 0.8503420948982239, "Finetune/Loss (Raw)": 0.972818911075592, "Finetune/Step": 3951, "Finetune/Step Time": 5.0564635545015335} +{"Finetune/Learning Rate": 2.864257862598029e-06, "Finetune/Loss": 0.8494419455528259, "Finetune/Loss (Raw)": 0.7800014615058899, "Finetune/Step": 3952, "Finetune/Step Time": 5.06541483104229} +{"Finetune/Learning Rate": 2.8598949090687723e-06, "Finetune/Loss": 0.8501262068748474, "Finetune/Loss (Raw)": 0.8467608094215393, "Finetune/Step": 3953, "Finetune/Step Time": 5.062787398695946} +{"Finetune/Learning Rate": 2.8555347264748988e-06, "Finetune/Loss": 0.8501386642456055, "Finetune/Loss (Raw)": 0.8177808523178101, "Finetune/Step": 3954, "Finetune/Step Time": 5.056361587718129} +{"Finetune/Learning Rate": 2.8511773165085164e-06, "Finetune/Loss": 0.8493494987487793, "Finetune/Loss (Raw)": 0.7970376014709473, "Finetune/Step": 3955, "Finetune/Step Time": 5.023800052702427} +{"Finetune/Learning Rate": 2.8468226808606525e-06, "Finetune/Loss": 0.8501254916191101, "Finetune/Loss (Raw)": 1.0535658597946167, "Finetune/Step": 3956, "Finetune/Step Time": 5.0241963397711515} +{"Finetune/Learning Rate": 2.842470821221246e-06, "Finetune/Loss": 0.8549595475196838, "Finetune/Loss (Raw)": 0.8983182907104492, "Finetune/Step": 3957, "Finetune/Step Time": 4.988838752731681} +{"Finetune/Learning Rate": 2.8381217392791784e-06, "Finetune/Loss": 0.8545668125152588, "Finetune/Loss (Raw)": 0.8397051692008972, "Finetune/Step": 3958, "Finetune/Step Time": 4.9854357447475195} +{"Finetune/Learning Rate": 2.833775436722238e-06, "Finetune/Loss": 0.8542619943618774, "Finetune/Loss (Raw)": 0.791962742805481, "Finetune/Step": 3959, "Finetune/Step Time": 4.99848996847868} +{"Finetune/Learning Rate": 2.829431915237144e-06, "Finetune/Loss": 0.8526146411895752, "Finetune/Loss (Raw)": 0.8250070810317993, "Finetune/Step": 3960, "Finetune/Step Time": 5.002285625785589} +{"Finetune/Learning Rate": 2.825091176509529e-06, "Finetune/Loss": 0.8521244525909424, "Finetune/Loss (Raw)": 0.902190625667572, "Finetune/Step": 3961, "Finetune/Step Time": 4.984368979930878} +{"Finetune/Learning Rate": 2.8207532222239497e-06, "Finetune/Loss": 0.8505817651748657, "Finetune/Loss (Raw)": 0.773823082447052, "Finetune/Step": 3962, "Finetune/Step Time": 4.992129344493151} +{"Finetune/Learning Rate": 2.816418054063883e-06, "Finetune/Loss": 0.8503834009170532, "Finetune/Loss (Raw)": 0.9212270379066467, "Finetune/Step": 3963, "Finetune/Step Time": 4.996083060279489} +{"Finetune/Learning Rate": 2.8120856737117187e-06, "Finetune/Loss": 0.8508657217025757, "Finetune/Loss (Raw)": 0.8779643774032593, "Finetune/Step": 3964, "Finetune/Step Time": 5.00522000156343} +{"Finetune/Learning Rate": 2.807756082848775e-06, "Finetune/Loss": 0.8518864512443542, "Finetune/Loss (Raw)": 0.9827563762664795, "Finetune/Step": 3965, "Finetune/Step Time": 5.017799716442823} +{"Finetune/Learning Rate": 2.803429283155281e-06, "Finetune/Loss": 0.8485054969787598, "Finetune/Loss (Raw)": 0.38495221734046936, "Finetune/Step": 3966, "Finetune/Step Time": 5.057286027818918} +{"Finetune/Learning Rate": 2.7991052763103767e-06, "Finetune/Loss": 0.8480302095413208, "Finetune/Loss (Raw)": 0.8246141076087952, "Finetune/Step": 3967, "Finetune/Step Time": 5.029390901327133} +{"Finetune/Learning Rate": 2.7947840639921308e-06, "Finetune/Loss": 0.8487769365310669, "Finetune/Loss (Raw)": 0.9376620054244995, "Finetune/Step": 3968, "Finetune/Step Time": 5.027749074622989} +{"Finetune/Learning Rate": 2.7904656478775196e-06, "Finetune/Loss": 0.8481050729751587, "Finetune/Loss (Raw)": 0.9182433485984802, "Finetune/Step": 3969, "Finetune/Step Time": 5.052856074646115} +{"Finetune/Learning Rate": 2.7861500296424372e-06, "Finetune/Loss": 0.8490889072418213, "Finetune/Loss (Raw)": 0.9334315657615662, "Finetune/Step": 3970, "Finetune/Step Time": 5.047501735389233} +{"Finetune/Learning Rate": 2.781837210961689e-06, "Finetune/Loss": 0.850135087966919, "Finetune/Loss (Raw)": 0.969366192817688, "Finetune/Step": 3971, "Finetune/Step Time": 5.039517119526863} +{"Finetune/Learning Rate": 2.7775271935089964e-06, "Finetune/Loss": 0.8497747182846069, "Finetune/Loss (Raw)": 0.8019436597824097, "Finetune/Step": 3972, "Finetune/Step Time": 5.037258688360453} +{"Finetune/Learning Rate": 2.7732199789569937e-06, "Finetune/Loss": 0.849770188331604, "Finetune/Loss (Raw)": 0.8390747308731079, "Finetune/Step": 3973, "Finetune/Step Time": 5.0288255754858255} +{"Finetune/Learning Rate": 2.768915568977222e-06, "Finetune/Loss": 0.8506801724433899, "Finetune/Loss (Raw)": 0.813141405582428, "Finetune/Step": 3974, "Finetune/Step Time": 5.037616776302457} +{"Finetune/Learning Rate": 2.7646139652401473e-06, "Finetune/Loss": 0.85042804479599, "Finetune/Loss (Raw)": 0.9511220455169678, "Finetune/Step": 3975, "Finetune/Step Time": 5.035547910258174} +{"Finetune/Learning Rate": 2.7603151694151354e-06, "Finetune/Loss": 0.8546237945556641, "Finetune/Loss (Raw)": 0.8667032718658447, "Finetune/Step": 3976, "Finetune/Step Time": 5.012777512893081} +{"Finetune/Learning Rate": 2.7560191831704584e-06, "Finetune/Loss": 0.8523768782615662, "Finetune/Loss (Raw)": 0.7566351294517517, "Finetune/Step": 3977, "Finetune/Step Time": 5.007597396150231} +{"Finetune/Learning Rate": 2.751726008173312e-06, "Finetune/Loss": 0.8525545597076416, "Finetune/Loss (Raw)": 0.6477136611938477, "Finetune/Step": 3978, "Finetune/Step Time": 4.998888146132231} +{"Finetune/Learning Rate": 2.747435646089791e-06, "Finetune/Loss": 0.8534393906593323, "Finetune/Loss (Raw)": 0.877332329750061, "Finetune/Step": 3979, "Finetune/Step Time": 5.002259753644466} +{"Finetune/Learning Rate": 2.7431480985849023e-06, "Finetune/Loss": 0.8513386249542236, "Finetune/Loss (Raw)": 0.6538907885551453, "Finetune/Step": 3980, "Finetune/Step Time": 5.00945813767612} +{"Finetune/Learning Rate": 2.7388633673225586e-06, "Finetune/Loss": 0.8551819920539856, "Finetune/Loss (Raw)": 0.9473720192909241, "Finetune/Step": 3981, "Finetune/Step Time": 4.969444096088409} +{"Finetune/Learning Rate": 2.734581453965579e-06, "Finetune/Loss": 0.8547005653381348, "Finetune/Loss (Raw)": 0.8344889879226685, "Finetune/Step": 3982, "Finetune/Step Time": 4.968807645142078} +{"Finetune/Learning Rate": 2.730302360175693e-06, "Finetune/Loss": 0.8561686277389526, "Finetune/Loss (Raw)": 0.8360793590545654, "Finetune/Step": 3983, "Finetune/Step Time": 4.9525265712291} +{"Finetune/Learning Rate": 2.726026087613528e-06, "Finetune/Loss": 0.8511145114898682, "Finetune/Loss (Raw)": 0.3410177230834961, "Finetune/Step": 3984, "Finetune/Step Time": 4.988072564825416} +{"Finetune/Learning Rate": 2.7217526379386327e-06, "Finetune/Loss": 0.8554264307022095, "Finetune/Loss (Raw)": 0.9147685766220093, "Finetune/Step": 3985, "Finetune/Step Time": 4.960281120613217} +{"Finetune/Learning Rate": 2.7174820128094393e-06, "Finetune/Loss": 0.8611229658126831, "Finetune/Loss (Raw)": 1.0654417276382446, "Finetune/Step": 3986, "Finetune/Step Time": 4.923012707382441} +{"Finetune/Learning Rate": 2.713214213883295e-06, "Finetune/Loss": 0.8589980006217957, "Finetune/Loss (Raw)": 0.7901405692100525, "Finetune/Step": 3987, "Finetune/Step Time": 4.930576831102371} +{"Finetune/Learning Rate": 2.7089492428164556e-06, "Finetune/Loss": 0.8594537973403931, "Finetune/Loss (Raw)": 0.9491331577301025, "Finetune/Step": 3988, "Finetune/Step Time": 4.934263572096825} +{"Finetune/Learning Rate": 2.7046871012640687e-06, "Finetune/Loss": 0.8603684306144714, "Finetune/Loss (Raw)": 0.9622823596000671, "Finetune/Step": 3989, "Finetune/Step Time": 4.92783628217876} +{"Finetune/Learning Rate": 2.7004277908801877e-06, "Finetune/Loss": 0.8605783581733704, "Finetune/Loss (Raw)": 0.9162375926971436, "Finetune/Step": 3990, "Finetune/Step Time": 4.932072868570685} +{"Finetune/Learning Rate": 2.6961713133177705e-06, "Finetune/Loss": 0.8616023659706116, "Finetune/Loss (Raw)": 0.9706087112426758, "Finetune/Step": 3991, "Finetune/Step Time": 4.923635438084602} +{"Finetune/Learning Rate": 2.6919176702286696e-06, "Finetune/Loss": 0.8601905703544617, "Finetune/Loss (Raw)": 0.7439731359481812, "Finetune/Step": 3992, "Finetune/Step Time": 4.941209034994245} +{"Finetune/Learning Rate": 2.687666863263644e-06, "Finetune/Loss": 0.860403835773468, "Finetune/Loss (Raw)": 0.9039084315299988, "Finetune/Step": 3993, "Finetune/Step Time": 4.907501036301255} +{"Finetune/Learning Rate": 2.683418894072345e-06, "Finetune/Loss": 0.8584164381027222, "Finetune/Loss (Raw)": 0.7787295579910278, "Finetune/Step": 3994, "Finetune/Step Time": 4.920274218544364} +{"Finetune/Learning Rate": 2.6791737643033333e-06, "Finetune/Loss": 0.8562818765640259, "Finetune/Loss (Raw)": 0.8085484504699707, "Finetune/Step": 3995, "Finetune/Step Time": 4.914874043315649} +{"Finetune/Learning Rate": 2.6749314756040547e-06, "Finetune/Loss": 0.8555617928504944, "Finetune/Loss (Raw)": 0.8034833073616028, "Finetune/Step": 3996, "Finetune/Step Time": 4.91337001696229} +{"Finetune/Learning Rate": 2.6706920296208573e-06, "Finetune/Loss": 0.8557373285293579, "Finetune/Loss (Raw)": 0.835292637348175, "Finetune/Step": 3997, "Finetune/Step Time": 4.911498909816146} +{"Finetune/Learning Rate": 2.666455427998994e-06, "Finetune/Loss": 0.8574254512786865, "Finetune/Loss (Raw)": 1.0177247524261475, "Finetune/Step": 3998, "Finetune/Step Time": 4.894958088174462} +{"Finetune/Learning Rate": 2.6622216723826035e-06, "Finetune/Loss": 0.8579106330871582, "Finetune/Loss (Raw)": 0.9368470311164856, "Finetune/Step": 3999, "Finetune/Step Time": 4.911381620913744} +{"Finetune/Learning Rate": 2.6579907644147227e-06, "Finetune/Loss": 0.8603546619415283, "Finetune/Loss (Raw)": 0.8293380737304688, "Finetune/Step": 4000, "Finetune/Step Time": 4.886695846915245} +{"Finetune/Learning Rate": 2.653762705737287e-06, "Finetune/Loss": 0.8610914945602417, "Finetune/Loss (Raw)": 0.8342078328132629, "Finetune/Step": 4001, "Finetune/Step Time": 4.902986479923129} +{"Finetune/Learning Rate": 2.6495374979911215e-06, "Finetune/Loss": 0.8604982495307922, "Finetune/Loss (Raw)": 0.8047145009040833, "Finetune/Step": 4002, "Finetune/Step Time": 4.913251496851444} +{"Finetune/Learning Rate": 2.6453151428159472e-06, "Finetune/Loss": 0.86042320728302, "Finetune/Loss (Raw)": 0.8693110942840576, "Finetune/Step": 4003, "Finetune/Step Time": 4.925336027517915} +{"Finetune/Learning Rate": 2.641095641850375e-06, "Finetune/Loss": 0.8606973886489868, "Finetune/Loss (Raw)": 0.919359028339386, "Finetune/Step": 4004, "Finetune/Step Time": 4.929782684892416} +{"Finetune/Learning Rate": 2.6368789967319196e-06, "Finetune/Loss": 0.8608914017677307, "Finetune/Loss (Raw)": 0.8817684650421143, "Finetune/Step": 4005, "Finetune/Step Time": 4.914387898519635} +{"Finetune/Learning Rate": 2.6326652090969697e-06, "Finetune/Loss": 0.8653684258460999, "Finetune/Loss (Raw)": 1.043519377708435, "Finetune/Step": 4006, "Finetune/Step Time": 4.876442393288016} +{"Finetune/Learning Rate": 2.6284542805808144e-06, "Finetune/Loss": 0.8615874648094177, "Finetune/Loss (Raw)": 0.3626282513141632, "Finetune/Step": 4007, "Finetune/Step Time": 4.8760757353156805} +{"Finetune/Learning Rate": 2.624246212817638e-06, "Finetune/Loss": 0.8608512878417969, "Finetune/Loss (Raw)": 0.7746727466583252, "Finetune/Step": 4008, "Finetune/Step Time": 4.8743042927235365} +{"Finetune/Learning Rate": 2.620041007440508e-06, "Finetune/Loss": 0.8602250218391418, "Finetune/Loss (Raw)": 0.8386514782905579, "Finetune/Step": 4009, "Finetune/Step Time": 4.869770569726825} +{"Finetune/Learning Rate": 2.6158386660813807e-06, "Finetune/Loss": 0.8590918779373169, "Finetune/Loss (Raw)": 0.8030333518981934, "Finetune/Step": 4010, "Finetune/Step Time": 4.8727820590138435} +{"Finetune/Learning Rate": 2.6116391903711035e-06, "Finetune/Loss": 0.8590759038925171, "Finetune/Loss (Raw)": 0.978709876537323, "Finetune/Step": 4011, "Finetune/Step Time": 4.8775802701711655} +{"Finetune/Learning Rate": 2.607442581939411e-06, "Finetune/Loss": 0.8579315543174744, "Finetune/Loss (Raw)": 0.6751537322998047, "Finetune/Step": 4012, "Finetune/Step Time": 4.8952565006911755} +{"Finetune/Learning Rate": 2.603248842414925e-06, "Finetune/Loss": 0.8594048023223877, "Finetune/Loss (Raw)": 0.8644273281097412, "Finetune/Step": 4013, "Finetune/Step Time": 4.889492912217975} +{"Finetune/Learning Rate": 2.599057973425151e-06, "Finetune/Loss": 0.8599098920822144, "Finetune/Loss (Raw)": 0.8174790740013123, "Finetune/Step": 4014, "Finetune/Step Time": 4.915049146860838} +{"Finetune/Learning Rate": 2.5948699765964915e-06, "Finetune/Loss": 0.8603488206863403, "Finetune/Loss (Raw)": 0.8545433878898621, "Finetune/Step": 4015, "Finetune/Step Time": 4.91560410708189} +{"Finetune/Learning Rate": 2.5906848535542196e-06, "Finetune/Loss": 0.8602030277252197, "Finetune/Loss (Raw)": 0.8925905823707581, "Finetune/Step": 4016, "Finetune/Step Time": 4.9172383062541485} +{"Finetune/Learning Rate": 2.5865026059224974e-06, "Finetune/Loss": 0.8604738712310791, "Finetune/Loss (Raw)": 0.9142689108848572, "Finetune/Step": 4017, "Finetune/Step Time": 4.919369526207447} +{"Finetune/Learning Rate": 2.58232323532438e-06, "Finetune/Loss": 0.8558603525161743, "Finetune/Loss (Raw)": 0.325804203748703, "Finetune/Step": 4018, "Finetune/Step Time": 4.955735955387354} +{"Finetune/Learning Rate": 2.5781467433817973e-06, "Finetune/Loss": 0.8568873405456543, "Finetune/Loss (Raw)": 1.0045207738876343, "Finetune/Step": 4019, "Finetune/Step Time": 4.94054095633328} +{"Finetune/Learning Rate": 2.573973131715565e-06, "Finetune/Loss": 0.8576350808143616, "Finetune/Loss (Raw)": 0.9516903162002563, "Finetune/Step": 4020, "Finetune/Step Time": 4.938749106600881} +{"Finetune/Learning Rate": 2.569802401945378e-06, "Finetune/Loss": 0.8616249561309814, "Finetune/Loss (Raw)": 0.8786370754241943, "Finetune/Step": 4021, "Finetune/Step Time": 4.913451611995697} +{"Finetune/Learning Rate": 2.565634555689818e-06, "Finetune/Loss": 0.8608405590057373, "Finetune/Loss (Raw)": 0.8060745000839233, "Finetune/Step": 4022, "Finetune/Step Time": 4.916206605732441} +{"Finetune/Learning Rate": 2.5614695945663415e-06, "Finetune/Loss": 0.8607350587844849, "Finetune/Loss (Raw)": 0.9107420444488525, "Finetune/Step": 4023, "Finetune/Step Time": 4.918740587309003} +{"Finetune/Learning Rate": 2.557307520191291e-06, "Finetune/Loss": 0.8609187602996826, "Finetune/Loss (Raw)": 0.9606669545173645, "Finetune/Step": 4024, "Finetune/Step Time": 4.9249005280435085} +{"Finetune/Learning Rate": 2.5531483341798856e-06, "Finetune/Loss": 0.8608140349388123, "Finetune/Loss (Raw)": 0.8651593327522278, "Finetune/Step": 4025, "Finetune/Step Time": 4.924515558406711} +{"Finetune/Learning Rate": 2.548992038146224e-06, "Finetune/Loss": 0.8616986274719238, "Finetune/Loss (Raw)": 0.9708254933357239, "Finetune/Step": 4026, "Finetune/Step Time": 4.911232052370906} +{"Finetune/Learning Rate": 2.5448386337032817e-06, "Finetune/Loss": 0.8617281317710876, "Finetune/Loss (Raw)": 1.0050348043441772, "Finetune/Step": 4027, "Finetune/Step Time": 4.9078256618231535} +{"Finetune/Learning Rate": 2.5406881224629175e-06, "Finetune/Loss": 0.8613762855529785, "Finetune/Loss (Raw)": 0.8173673152923584, "Finetune/Step": 4028, "Finetune/Step Time": 4.892870478332043} +{"Finetune/Learning Rate": 2.5365405060358663e-06, "Finetune/Loss": 0.8611068725585938, "Finetune/Loss (Raw)": 0.9005516171455383, "Finetune/Step": 4029, "Finetune/Step Time": 4.893356716260314} +{"Finetune/Learning Rate": 2.532395786031726e-06, "Finetune/Loss": 0.856592059135437, "Finetune/Loss (Raw)": 0.394599586725235, "Finetune/Step": 4030, "Finetune/Step Time": 4.925862915813923} +{"Finetune/Learning Rate": 2.528253964058992e-06, "Finetune/Loss": 0.8559226393699646, "Finetune/Loss (Raw)": 0.8696354627609253, "Finetune/Step": 4031, "Finetune/Step Time": 4.925724346190691} +{"Finetune/Learning Rate": 2.5241150417250216e-06, "Finetune/Loss": 0.8558215498924255, "Finetune/Loss (Raw)": 0.8819296956062317, "Finetune/Step": 4032, "Finetune/Step Time": 4.919064607471228} +{"Finetune/Learning Rate": 2.5199790206360486e-06, "Finetune/Loss": 0.8577136397361755, "Finetune/Loss (Raw)": 0.9319323897361755, "Finetune/Step": 4033, "Finetune/Step Time": 4.907450817525387} +{"Finetune/Learning Rate": 2.5158459023971826e-06, "Finetune/Loss": 0.8570156693458557, "Finetune/Loss (Raw)": 0.8495913147926331, "Finetune/Step": 4034, "Finetune/Step Time": 4.862669190391898} +{"Finetune/Learning Rate": 2.511715688612405e-06, "Finetune/Loss": 0.8545383214950562, "Finetune/Loss (Raw)": 0.8360676169395447, "Finetune/Step": 4035, "Finetune/Step Time": 4.874951237812638} +{"Finetune/Learning Rate": 2.5075883808845736e-06, "Finetune/Loss": 0.8562256097793579, "Finetune/Loss (Raw)": 1.0353096723556519, "Finetune/Step": 4036, "Finetune/Step Time": 4.86119432002306} +{"Finetune/Learning Rate": 2.5034639808154115e-06, "Finetune/Loss": 0.8559534549713135, "Finetune/Loss (Raw)": 0.7911412119865417, "Finetune/Step": 4037, "Finetune/Step Time": 4.867561541497707} +{"Finetune/Learning Rate": 2.4993424900055228e-06, "Finetune/Loss": 0.8512656688690186, "Finetune/Loss (Raw)": 0.35714662075042725, "Finetune/Step": 4038, "Finetune/Step Time": 4.900953611359} +{"Finetune/Learning Rate": 2.4952239100543808e-06, "Finetune/Loss": 0.8506491184234619, "Finetune/Loss (Raw)": 0.9435113072395325, "Finetune/Step": 4039, "Finetune/Step Time": 4.92104977928102} +{"Finetune/Learning Rate": 2.491108242560313e-06, "Finetune/Loss": 0.8515051007270813, "Finetune/Loss (Raw)": 0.9274109601974487, "Finetune/Step": 4040, "Finetune/Step Time": 4.909833153709769} +{"Finetune/Learning Rate": 2.486995489120542e-06, "Finetune/Loss": 0.850833535194397, "Finetune/Loss (Raw)": 0.8086547255516052, "Finetune/Step": 4041, "Finetune/Step Time": 4.9047778975218534} +{"Finetune/Learning Rate": 2.4828856513311405e-06, "Finetune/Loss": 0.8510251641273499, "Finetune/Loss (Raw)": 0.8602294921875, "Finetune/Step": 4042, "Finetune/Step Time": 4.912712909281254} +{"Finetune/Learning Rate": 2.4787787307870605e-06, "Finetune/Loss": 0.8510434627532959, "Finetune/Loss (Raw)": 0.9075678586959839, "Finetune/Step": 4043, "Finetune/Step Time": 4.915507799014449} +{"Finetune/Learning Rate": 2.474674729082115e-06, "Finetune/Loss": 0.8509724140167236, "Finetune/Loss (Raw)": 0.8823877573013306, "Finetune/Step": 4044, "Finetune/Step Time": 4.907218148931861} +{"Finetune/Learning Rate": 2.470573647808987e-06, "Finetune/Loss": 0.8462925553321838, "Finetune/Loss (Raw)": 0.2921406626701355, "Finetune/Step": 4045, "Finetune/Step Time": 4.922677792608738} +{"Finetune/Learning Rate": 2.466475488559227e-06, "Finetune/Loss": 0.8452447652816772, "Finetune/Loss (Raw)": 0.8194491267204285, "Finetune/Step": 4046, "Finetune/Step Time": 4.925589269027114} +{"Finetune/Learning Rate": 2.4623802529232476e-06, "Finetune/Loss": 0.8448423147201538, "Finetune/Loss (Raw)": 0.7979178428649902, "Finetune/Step": 4047, "Finetune/Step Time": 4.929286325350404} +{"Finetune/Learning Rate": 2.458287942490335e-06, "Finetune/Loss": 0.840847373008728, "Finetune/Loss (Raw)": 0.37248894572257996, "Finetune/Step": 4048, "Finetune/Step Time": 4.928479623049498} +{"Finetune/Learning Rate": 2.454198558848635e-06, "Finetune/Loss": 0.8414930105209351, "Finetune/Loss (Raw)": 0.7563053369522095, "Finetune/Step": 4049, "Finetune/Step Time": 4.911364767700434} +{"Finetune/Learning Rate": 2.4501121035851494e-06, "Finetune/Loss": 0.8405336737632751, "Finetune/Loss (Raw)": 0.8313443064689636, "Finetune/Step": 4050, "Finetune/Step Time": 4.909186629578471} +{"Finetune/Learning Rate": 2.4460285782857606e-06, "Finetune/Loss": 0.8410100340843201, "Finetune/Loss (Raw)": 0.7885133624076843, "Finetune/Step": 4051, "Finetune/Step Time": 4.901033686473966} +{"Finetune/Learning Rate": 2.441947984535201e-06, "Finetune/Loss": 0.8405251502990723, "Finetune/Loss (Raw)": 0.8433890342712402, "Finetune/Step": 4052, "Finetune/Step Time": 4.921856984496117} +{"Finetune/Learning Rate": 2.4378703239170707e-06, "Finetune/Loss": 0.8363250494003296, "Finetune/Loss (Raw)": 0.3744015693664551, "Finetune/Step": 4053, "Finetune/Step Time": 4.9543096497654915} +{"Finetune/Learning Rate": 2.4337955980138293e-06, "Finetune/Loss": 0.8366538286209106, "Finetune/Loss (Raw)": 0.8607028722763062, "Finetune/Step": 4054, "Finetune/Step Time": 4.963983526453376} +{"Finetune/Learning Rate": 2.4297238084067988e-06, "Finetune/Loss": 0.8369669914245605, "Finetune/Loss (Raw)": 0.8975837230682373, "Finetune/Step": 4055, "Finetune/Step Time": 4.945673745125532} +{"Finetune/Learning Rate": 2.425654956676161e-06, "Finetune/Loss": 0.8342029452323914, "Finetune/Loss (Raw)": 0.6277102828025818, "Finetune/Step": 4056, "Finetune/Step Time": 4.96146335080266} +{"Finetune/Learning Rate": 2.421589044400955e-06, "Finetune/Loss": 0.8346960544586182, "Finetune/Loss (Raw)": 0.97910475730896, "Finetune/Step": 4057, "Finetune/Step Time": 4.952688017860055} +{"Finetune/Learning Rate": 2.4175260731590878e-06, "Finetune/Loss": 0.8345723152160645, "Finetune/Loss (Raw)": 0.8309230208396912, "Finetune/Step": 4058, "Finetune/Step Time": 4.944454485550523} +{"Finetune/Learning Rate": 2.413466044527318e-06, "Finetune/Loss": 0.833796501159668, "Finetune/Loss (Raw)": 0.8840539455413818, "Finetune/Step": 4059, "Finetune/Step Time": 4.958721846342087} +{"Finetune/Learning Rate": 2.4094089600812576e-06, "Finetune/Loss": 0.8323252201080322, "Finetune/Loss (Raw)": 0.654384434223175, "Finetune/Step": 4060, "Finetune/Step Time": 4.960683388635516} +{"Finetune/Learning Rate": 2.405354821395388e-06, "Finetune/Loss": 0.8335748910903931, "Finetune/Loss (Raw)": 0.8142988681793213, "Finetune/Step": 4061, "Finetune/Step Time": 4.952965494245291} +{"Finetune/Learning Rate": 2.4013036300430403e-06, "Finetune/Loss": 0.8352841138839722, "Finetune/Loss (Raw)": 1.0083534717559814, "Finetune/Step": 4062, "Finetune/Step Time": 4.950234612450004} +{"Finetune/Learning Rate": 2.3972553875964035e-06, "Finetune/Loss": 0.8314322829246521, "Finetune/Loss (Raw)": 0.35152435302734375, "Finetune/Step": 4063, "Finetune/Step Time": 4.980715695768595} +{"Finetune/Learning Rate": 2.393210095626515e-06, "Finetune/Loss": 0.8270426988601685, "Finetune/Loss (Raw)": 0.3806564211845398, "Finetune/Step": 4064, "Finetune/Step Time": 5.012828454375267} +{"Finetune/Learning Rate": 2.38916775570328e-06, "Finetune/Loss": 0.826168954372406, "Finetune/Loss (Raw)": 0.785169780254364, "Finetune/Step": 4065, "Finetune/Step Time": 5.023314891383052} +{"Finetune/Learning Rate": 2.3851283693954506e-06, "Finetune/Loss": 0.8262342214584351, "Finetune/Loss (Raw)": 0.8715217113494873, "Finetune/Step": 4066, "Finetune/Step Time": 5.02687974832952} +{"Finetune/Learning Rate": 2.3810919382706297e-06, "Finetune/Loss": 0.8265731930732727, "Finetune/Loss (Raw)": 0.9837074279785156, "Finetune/Step": 4067, "Finetune/Step Time": 5.025748774409294} +{"Finetune/Learning Rate": 2.3770584638952874e-06, "Finetune/Loss": 0.8269346952438354, "Finetune/Loss (Raw)": 0.9206023812294006, "Finetune/Step": 4068, "Finetune/Step Time": 5.027210395783186} +{"Finetune/Learning Rate": 2.3730279478347275e-06, "Finetune/Loss": 0.8266249299049377, "Finetune/Loss (Raw)": 0.9219087362289429, "Finetune/Step": 4069, "Finetune/Step Time": 5.040404310449958} +{"Finetune/Learning Rate": 2.3690003916531157e-06, "Finetune/Loss": 0.8254549503326416, "Finetune/Loss (Raw)": 0.8245325088500977, "Finetune/Step": 4070, "Finetune/Step Time": 5.036025352776051} +{"Finetune/Learning Rate": 2.3649757969134667e-06, "Finetune/Loss": 0.8259801268577576, "Finetune/Loss (Raw)": 0.8646216988563538, "Finetune/Step": 4071, "Finetune/Step Time": 5.042556535452604} +{"Finetune/Learning Rate": 2.3609541651776537e-06, "Finetune/Loss": 0.8247947692871094, "Finetune/Loss (Raw)": 0.7643320560455322, "Finetune/Step": 4072, "Finetune/Step Time": 5.052762288600206} +{"Finetune/Learning Rate": 2.356935498006391e-06, "Finetune/Loss": 0.8233609199523926, "Finetune/Loss (Raw)": 0.9007465839385986, "Finetune/Step": 4073, "Finetune/Step Time": 5.035014985129237} +{"Finetune/Learning Rate": 2.3529197969592375e-06, "Finetune/Loss": 0.8241910934448242, "Finetune/Loss (Raw)": 0.9858685731887817, "Finetune/Step": 4074, "Finetune/Step Time": 5.0225546304136515} +{"Finetune/Learning Rate": 2.3489070635946175e-06, "Finetune/Loss": 0.8234959244728088, "Finetune/Loss (Raw)": 0.8246201276779175, "Finetune/Step": 4075, "Finetune/Step Time": 5.011137623339891} +{"Finetune/Learning Rate": 2.3448972994697906e-06, "Finetune/Loss": 0.8258730173110962, "Finetune/Loss (Raw)": 0.6663750410079956, "Finetune/Step": 4076, "Finetune/Step Time": 4.99462616071105} +{"Finetune/Learning Rate": 2.340890506140869e-06, "Finetune/Loss": 0.8246623277664185, "Finetune/Loss (Raw)": 0.8514862656593323, "Finetune/Step": 4077, "Finetune/Step Time": 5.008545633405447} +{"Finetune/Learning Rate": 2.3368866851628102e-06, "Finetune/Loss": 0.823324978351593, "Finetune/Loss (Raw)": 0.8094668984413147, "Finetune/Step": 4078, "Finetune/Step Time": 5.005664885044098} +{"Finetune/Learning Rate": 2.3328858380894194e-06, "Finetune/Loss": 0.823840856552124, "Finetune/Loss (Raw)": 1.0388543605804443, "Finetune/Step": 4079, "Finetune/Step Time": 5.017556166276336} +{"Finetune/Learning Rate": 2.328887966473348e-06, "Finetune/Loss": 0.8238242268562317, "Finetune/Loss (Raw)": 0.7778698205947876, "Finetune/Step": 4080, "Finetune/Step Time": 5.016646917909384} +{"Finetune/Learning Rate": 2.324893071866087e-06, "Finetune/Loss": 0.8239237666130066, "Finetune/Loss (Raw)": 0.859499454498291, "Finetune/Step": 4081, "Finetune/Step Time": 5.039171190932393} +{"Finetune/Learning Rate": 2.320901155817983e-06, "Finetune/Loss": 0.8237905502319336, "Finetune/Loss (Raw)": 0.8007356524467468, "Finetune/Step": 4082, "Finetune/Step Time": 5.052635725587606} +{"Finetune/Learning Rate": 2.316912219878222e-06, "Finetune/Loss": 0.8252273797988892, "Finetune/Loss (Raw)": 0.9809476733207703, "Finetune/Step": 4083, "Finetune/Step Time": 5.054687604308128} +{"Finetune/Learning Rate": 2.312926265594825e-06, "Finetune/Loss": 0.8206405639648438, "Finetune/Loss (Raw)": 0.46645334362983704, "Finetune/Step": 4084, "Finetune/Step Time": 5.084583178162575} +{"Finetune/Learning Rate": 2.308943294514667e-06, "Finetune/Loss": 0.8202946186065674, "Finetune/Loss (Raw)": 0.8540440201759338, "Finetune/Step": 4085, "Finetune/Step Time": 5.096196347847581} +{"Finetune/Learning Rate": 2.3049633081834634e-06, "Finetune/Loss": 0.8206133246421814, "Finetune/Loss (Raw)": 0.8804922699928284, "Finetune/Step": 4086, "Finetune/Step Time": 5.10376456566155} +{"Finetune/Learning Rate": 2.3009863081457663e-06, "Finetune/Loss": 0.8221847414970398, "Finetune/Loss (Raw)": 0.9931069612503052, "Finetune/Step": 4087, "Finetune/Step Time": 5.090742141008377} +{"Finetune/Learning Rate": 2.2970122959449716e-06, "Finetune/Loss": 0.8231325149536133, "Finetune/Loss (Raw)": 0.9463152885437012, "Finetune/Step": 4088, "Finetune/Step Time": 5.086040256544948} +{"Finetune/Learning Rate": 2.2930412731233174e-06, "Finetune/Loss": 0.8226823806762695, "Finetune/Loss (Raw)": 0.8445838093757629, "Finetune/Step": 4089, "Finetune/Step Time": 5.087083321064711} +{"Finetune/Learning Rate": 2.2890732412218795e-06, "Finetune/Loss": 0.8231631517410278, "Finetune/Loss (Raw)": 0.8353567719459534, "Finetune/Step": 4090, "Finetune/Step Time": 5.079782273620367} +{"Finetune/Learning Rate": 2.2851082017805704e-06, "Finetune/Loss": 0.8224533796310425, "Finetune/Loss (Raw)": 0.8303800225257874, "Finetune/Step": 4091, "Finetune/Step Time": 5.093127943575382} +{"Finetune/Learning Rate": 2.2811461563381497e-06, "Finetune/Loss": 0.8221501111984253, "Finetune/Loss (Raw)": 0.8391425013542175, "Finetune/Step": 4092, "Finetune/Step Time": 5.094739707186818} +{"Finetune/Learning Rate": 2.2771871064322117e-06, "Finetune/Loss": 0.8221744894981384, "Finetune/Loss (Raw)": 0.9858742356300354, "Finetune/Step": 4093, "Finetune/Step Time": 5.078855020925403} +{"Finetune/Learning Rate": 2.2732310535991775e-06, "Finetune/Loss": 0.8274624347686768, "Finetune/Loss (Raw)": 1.0618162155151367, "Finetune/Step": 4094, "Finetune/Step Time": 5.044852219521999} +{"Finetune/Learning Rate": 2.2692779993743207e-06, "Finetune/Loss": 0.827045202255249, "Finetune/Loss (Raw)": 0.7712042331695557, "Finetune/Step": 4095, "Finetune/Step Time": 5.0527875404804945} +{"Finetune/Learning Rate": 2.265327945291742e-06, "Finetune/Loss": 0.8277477622032166, "Finetune/Loss (Raw)": 1.0275880098342896, "Finetune/Step": 4096, "Finetune/Step Time": 5.057446653023362} +{"Finetune/Learning Rate": 2.2613808928843827e-06, "Finetune/Loss": 0.8280562162399292, "Finetune/Loss (Raw)": 0.957720160484314, "Finetune/Step": 4097, "Finetune/Step Time": 5.048709783703089} +{"Finetune/Learning Rate": 2.2574368436840144e-06, "Finetune/Loss": 0.8261893391609192, "Finetune/Loss (Raw)": 0.6944785118103027, "Finetune/Step": 4098, "Finetune/Step Time": 5.05707580037415} +{"Finetune/Learning Rate": 2.2534957992212457e-06, "Finetune/Loss": 0.8258692026138306, "Finetune/Loss (Raw)": 0.9283879399299622, "Finetune/Step": 4099, "Finetune/Step Time": 5.059946833178401} +{"Finetune/Learning Rate": 2.2495577610255203e-06, "Finetune/Loss": 0.8270606398582458, "Finetune/Loss (Raw)": 0.9544472694396973, "Finetune/Step": 4100, "Finetune/Step Time": 5.059979191049933} +{"Finetune/Learning Rate": 2.24562273062511e-06, "Finetune/Loss": 0.8274374604225159, "Finetune/Loss (Raw)": 0.8873070478439331, "Finetune/Step": 4101, "Finetune/Step Time": 5.080657627433538} +{"Finetune/Learning Rate": 2.2416907095471296e-06, "Finetune/Loss": 0.8277087807655334, "Finetune/Loss (Raw)": 0.8478702902793884, "Finetune/Step": 4102, "Finetune/Step Time": 5.071573570370674} +{"Finetune/Learning Rate": 2.237761699317519e-06, "Finetune/Loss": 0.8271989822387695, "Finetune/Loss (Raw)": 0.885874330997467, "Finetune/Step": 4103, "Finetune/Step Time": 5.074384642764926} +{"Finetune/Learning Rate": 2.2338357014610425e-06, "Finetune/Loss": 0.8272046446800232, "Finetune/Loss (Raw)": 0.8674235939979553, "Finetune/Step": 4104, "Finetune/Step Time": 5.061263892799616} +{"Finetune/Learning Rate": 2.2299127175013113e-06, "Finetune/Loss": 0.8273704648017883, "Finetune/Loss (Raw)": 0.7778586149215698, "Finetune/Step": 4105, "Finetune/Step Time": 5.055451776832342} +{"Finetune/Learning Rate": 2.2259927489607568e-06, "Finetune/Loss": 0.8287525773048401, "Finetune/Loss (Raw)": 0.8246217966079712, "Finetune/Step": 4106, "Finetune/Step Time": 5.053781162947416} +{"Finetune/Learning Rate": 2.222075797360641e-06, "Finetune/Loss": 0.8284717798233032, "Finetune/Loss (Raw)": 0.8413942456245422, "Finetune/Step": 4107, "Finetune/Step Time": 5.055700620636344} +{"Finetune/Learning Rate": 2.2181618642210577e-06, "Finetune/Loss": 0.8307338953018188, "Finetune/Loss (Raw)": 0.9434366226196289, "Finetune/Step": 4108, "Finetune/Step Time": 5.04974140971899} +{"Finetune/Learning Rate": 2.2142509510609277e-06, "Finetune/Loss": 0.8297497630119324, "Finetune/Loss (Raw)": 0.8214053511619568, "Finetune/Step": 4109, "Finetune/Step Time": 5.057990036904812} +{"Finetune/Learning Rate": 2.210343059397999e-06, "Finetune/Loss": 0.8303570747375488, "Finetune/Loss (Raw)": 0.9122234582901001, "Finetune/Step": 4110, "Finetune/Step Time": 5.065782342106104} +{"Finetune/Learning Rate": 2.206438190748846e-06, "Finetune/Loss": 0.831613302230835, "Finetune/Loss (Raw)": 0.99687659740448, "Finetune/Step": 4111, "Finetune/Step Time": 5.071583056822419} +{"Finetune/Learning Rate": 2.2025363466288796e-06, "Finetune/Loss": 0.8347713351249695, "Finetune/Loss (Raw)": 0.7452515363693237, "Finetune/Step": 4112, "Finetune/Step Time": 5.049746321514249} +{"Finetune/Learning Rate": 2.1986375285523233e-06, "Finetune/Loss": 0.8332201242446899, "Finetune/Loss (Raw)": 0.716205358505249, "Finetune/Step": 4113, "Finetune/Step Time": 5.048182670027018} +{"Finetune/Learning Rate": 2.194741738032229e-06, "Finetune/Loss": 0.8317420482635498, "Finetune/Loss (Raw)": 0.8762504458427429, "Finetune/Step": 4114, "Finetune/Step Time": 5.043966915458441} +{"Finetune/Learning Rate": 2.1908489765804853e-06, "Finetune/Loss": 0.8324040174484253, "Finetune/Loss (Raw)": 0.8748754262924194, "Finetune/Step": 4115, "Finetune/Step Time": 5.0406533014029264} +{"Finetune/Learning Rate": 2.1869592457077924e-06, "Finetune/Loss": 0.8298705816268921, "Finetune/Loss (Raw)": 0.6248459815979004, "Finetune/Step": 4116, "Finetune/Step Time": 5.05588286742568} +{"Finetune/Learning Rate": 2.183072546923679e-06, "Finetune/Loss": 0.8283894658088684, "Finetune/Loss (Raw)": 0.7727015614509583, "Finetune/Step": 4117, "Finetune/Step Time": 5.070142587646842} +{"Finetune/Learning Rate": 2.179188881736498e-06, "Finetune/Loss": 0.8294249773025513, "Finetune/Loss (Raw)": 1.0487937927246094, "Finetune/Step": 4118, "Finetune/Step Time": 5.068882633000612} +{"Finetune/Learning Rate": 2.1753082516534242e-06, "Finetune/Loss": 0.8295251131057739, "Finetune/Loss (Raw)": 0.98341965675354, "Finetune/Step": 4119, "Finetune/Step Time": 5.063736898824573} +{"Finetune/Learning Rate": 2.1714306581804533e-06, "Finetune/Loss": 0.8307214975357056, "Finetune/Loss (Raw)": 0.8971139192581177, "Finetune/Step": 4120, "Finetune/Step Time": 5.059160597622395} +{"Finetune/Learning Rate": 2.1675561028224012e-06, "Finetune/Loss": 0.8305198550224304, "Finetune/Loss (Raw)": 0.878092348575592, "Finetune/Step": 4121, "Finetune/Step Time": 5.0646738186478615} +{"Finetune/Learning Rate": 2.163684587082915e-06, "Finetune/Loss": 0.8312482237815857, "Finetune/Loss (Raw)": 0.8719630241394043, "Finetune/Step": 4122, "Finetune/Step Time": 5.063607037067413} +{"Finetune/Learning Rate": 2.159816112464447e-06, "Finetune/Loss": 0.8328636288642883, "Finetune/Loss (Raw)": 1.0153160095214844, "Finetune/Step": 4123, "Finetune/Step Time": 5.065099747851491} +{"Finetune/Learning Rate": 2.1559506804682762e-06, "Finetune/Loss": 0.8316663503646851, "Finetune/Loss (Raw)": 0.6502375602722168, "Finetune/Step": 4124, "Finetune/Step Time": 5.067038318142295} +{"Finetune/Learning Rate": 2.1520882925945074e-06, "Finetune/Loss": 0.8321966528892517, "Finetune/Loss (Raw)": 0.9031713008880615, "Finetune/Step": 4125, "Finetune/Step Time": 5.0651589054614305} +{"Finetune/Learning Rate": 2.1482289503420537e-06, "Finetune/Loss": 0.8311507701873779, "Finetune/Loss (Raw)": 0.8838571310043335, "Finetune/Step": 4126, "Finetune/Step Time": 5.05971440859139} +{"Finetune/Learning Rate": 2.144372655208653e-06, "Finetune/Loss": 0.8308558464050293, "Finetune/Loss (Raw)": 0.8990908265113831, "Finetune/Step": 4127, "Finetune/Step Time": 5.039456041529775} +{"Finetune/Learning Rate": 2.1405194086908555e-06, "Finetune/Loss": 0.8308219909667969, "Finetune/Loss (Raw)": 0.8250088691711426, "Finetune/Step": 4128, "Finetune/Step Time": 5.034830402582884} +{"Finetune/Learning Rate": 2.1366692122840317e-06, "Finetune/Loss": 0.8307180404663086, "Finetune/Loss (Raw)": 0.8208907842636108, "Finetune/Step": 4129, "Finetune/Step Time": 5.01548195630312} +{"Finetune/Learning Rate": 2.1328220674823686e-06, "Finetune/Loss": 0.8309506773948669, "Finetune/Loss (Raw)": 0.8344941735267639, "Finetune/Step": 4130, "Finetune/Step Time": 5.005360670387745} +{"Finetune/Learning Rate": 2.1289779757788655e-06, "Finetune/Loss": 0.8310701847076416, "Finetune/Loss (Raw)": 0.8846123814582825, "Finetune/Step": 4131, "Finetune/Step Time": 4.998329071328044} +{"Finetune/Learning Rate": 2.1251369386653454e-06, "Finetune/Loss": 0.8308607339859009, "Finetune/Loss (Raw)": 0.8925427794456482, "Finetune/Step": 4132, "Finetune/Step Time": 5.005728535354137} +{"Finetune/Learning Rate": 2.121298957632434e-06, "Finetune/Loss": 0.8305753469467163, "Finetune/Loss (Raw)": 0.845238208770752, "Finetune/Step": 4133, "Finetune/Step Time": 5.013376120477915} +{"Finetune/Learning Rate": 2.117464034169575e-06, "Finetune/Loss": 0.8291750550270081, "Finetune/Loss (Raw)": 0.8642905950546265, "Finetune/Step": 4134, "Finetune/Step Time": 5.029767217114568} +{"Finetune/Learning Rate": 2.113632169765033e-06, "Finetune/Loss": 0.8324964046478271, "Finetune/Loss (Raw)": 0.7877522706985474, "Finetune/Step": 4135, "Finetune/Step Time": 4.994769055396318} +{"Finetune/Learning Rate": 2.1098033659058792e-06, "Finetune/Loss": 0.8330005407333374, "Finetune/Loss (Raw)": 0.8392061591148376, "Finetune/Step": 4136, "Finetune/Step Time": 4.995383327826858} +{"Finetune/Learning Rate": 2.105977624077995e-06, "Finetune/Loss": 0.8326591849327087, "Finetune/Loss (Raw)": 0.794958770275116, "Finetune/Step": 4137, "Finetune/Step Time": 4.992677768692374} +{"Finetune/Learning Rate": 2.102154945766076e-06, "Finetune/Loss": 0.8338391780853271, "Finetune/Loss (Raw)": 0.9540724754333496, "Finetune/Step": 4138, "Finetune/Step Time": 4.97844010591507} +{"Finetune/Learning Rate": 2.0983353324536316e-06, "Finetune/Loss": 0.8323224782943726, "Finetune/Loss (Raw)": 0.7845675349235535, "Finetune/Step": 4139, "Finetune/Step Time": 4.9685147777199745} +{"Finetune/Learning Rate": 2.0945187856229778e-06, "Finetune/Loss": 0.8334226608276367, "Finetune/Loss (Raw)": 0.8159846067428589, "Finetune/Step": 4140, "Finetune/Step Time": 4.957547344267368} +{"Finetune/Learning Rate": 2.0907053067552384e-06, "Finetune/Loss": 0.8296847343444824, "Finetune/Loss (Raw)": 0.38597196340560913, "Finetune/Step": 4141, "Finetune/Step Time": 4.982757795602083} +{"Finetune/Learning Rate": 2.0868948973303593e-06, "Finetune/Loss": 0.8293393850326538, "Finetune/Loss (Raw)": 0.7732792496681213, "Finetune/Step": 4142, "Finetune/Step Time": 4.959237318485975} +{"Finetune/Learning Rate": 2.0830875588270773e-06, "Finetune/Loss": 0.8288175463676453, "Finetune/Loss (Raw)": 0.787738561630249, "Finetune/Step": 4143, "Finetune/Step Time": 4.956386594101787} +{"Finetune/Learning Rate": 2.0792832927229465e-06, "Finetune/Loss": 0.8240993022918701, "Finetune/Loss (Raw)": 0.28865471482276917, "Finetune/Step": 4144, "Finetune/Step Time": 4.992713762447238} +{"Finetune/Learning Rate": 2.0754821004943337e-06, "Finetune/Loss": 0.822736918926239, "Finetune/Loss (Raw)": 0.739887535572052, "Finetune/Step": 4145, "Finetune/Step Time": 5.001242954283953} +{"Finetune/Learning Rate": 2.071683983616406e-06, "Finetune/Loss": 0.8273336887359619, "Finetune/Loss (Raw)": 0.9141944646835327, "Finetune/Step": 4146, "Finetune/Step Time": 4.960628932341933} +{"Finetune/Learning Rate": 2.0678889435631333e-06, "Finetune/Loss": 0.8261460661888123, "Finetune/Loss (Raw)": 0.8525007963180542, "Finetune/Step": 4147, "Finetune/Step Time": 4.961468445137143} +{"Finetune/Learning Rate": 2.064096981807302e-06, "Finetune/Loss": 0.8257204294204712, "Finetune/Loss (Raw)": 0.8972090482711792, "Finetune/Step": 4148, "Finetune/Step Time": 4.962185420095921} +{"Finetune/Learning Rate": 2.060308099820496e-06, "Finetune/Loss": 0.8254200220108032, "Finetune/Loss (Raw)": 0.8401862382888794, "Finetune/Step": 4149, "Finetune/Step Time": 4.951301829889417} +{"Finetune/Learning Rate": 2.056522299073108e-06, "Finetune/Loss": 0.8251438140869141, "Finetune/Loss (Raw)": 0.7707218527793884, "Finetune/Step": 4150, "Finetune/Step Time": 4.976663835346699} +{"Finetune/Learning Rate": 2.052739581034332e-06, "Finetune/Loss": 0.8247851729393005, "Finetune/Loss (Raw)": 0.8648310303688049, "Finetune/Step": 4151, "Finetune/Step Time": 4.981535814702511} +{"Finetune/Learning Rate": 2.0489599471721678e-06, "Finetune/Loss": 0.8239907026290894, "Finetune/Loss (Raw)": 0.8589854836463928, "Finetune/Step": 4152, "Finetune/Step Time": 4.986271942034364} +{"Finetune/Learning Rate": 2.0451833989534165e-06, "Finetune/Loss": 0.8237189054489136, "Finetune/Loss (Raw)": 0.8303678035736084, "Finetune/Step": 4153, "Finetune/Step Time": 4.983528260141611} +{"Finetune/Learning Rate": 2.0414099378436812e-06, "Finetune/Loss": 0.8230535984039307, "Finetune/Loss (Raw)": 0.8856569528579712, "Finetune/Step": 4154, "Finetune/Step Time": 4.987764790654182} +{"Finetune/Learning Rate": 2.0376395653073735e-06, "Finetune/Loss": 0.8215916156768799, "Finetune/Loss (Raw)": 0.8178988695144653, "Finetune/Step": 4155, "Finetune/Step Time": 4.99016292206943} +{"Finetune/Learning Rate": 2.0338722828077007e-06, "Finetune/Loss": 0.8219213485717773, "Finetune/Loss (Raw)": 0.8595801591873169, "Finetune/Step": 4156, "Finetune/Step Time": 4.989313770085573} +{"Finetune/Learning Rate": 2.030108091806664e-06, "Finetune/Loss": 0.8224917650222778, "Finetune/Loss (Raw)": 0.9735619425773621, "Finetune/Step": 4157, "Finetune/Step Time": 4.989708710461855} +{"Finetune/Learning Rate": 2.0263469937650813e-06, "Finetune/Loss": 0.8259118795394897, "Finetune/Loss (Raw)": 0.8323767781257629, "Finetune/Step": 4158, "Finetune/Step Time": 4.9648564755916595} +{"Finetune/Learning Rate": 2.0225889901425565e-06, "Finetune/Loss": 0.8263872861862183, "Finetune/Loss (Raw)": 0.9304890632629395, "Finetune/Step": 4159, "Finetune/Step Time": 4.965011456981301} +{"Finetune/Learning Rate": 2.0188340823975005e-06, "Finetune/Loss": 0.8271433115005493, "Finetune/Loss (Raw)": 0.9787036776542664, "Finetune/Step": 4160, "Finetune/Step Time": 4.957765823230147} +{"Finetune/Learning Rate": 2.0150822719871167e-06, "Finetune/Loss": 0.8269422054290771, "Finetune/Loss (Raw)": 0.9061878323554993, "Finetune/Step": 4161, "Finetune/Step Time": 4.955097936093807} +{"Finetune/Learning Rate": 2.011333560367411e-06, "Finetune/Loss": 0.8277029991149902, "Finetune/Loss (Raw)": 0.9469645023345947, "Finetune/Step": 4162, "Finetune/Step Time": 4.9567147344350815} +{"Finetune/Learning Rate": 2.007587948993185e-06, "Finetune/Loss": 0.8267582654953003, "Finetune/Loss (Raw)": 0.7151455283164978, "Finetune/Step": 4163, "Finetune/Step Time": 4.9624595902860165} +{"Finetune/Learning Rate": 2.003845439318035e-06, "Finetune/Loss": 0.8233511447906494, "Finetune/Loss (Raw)": 0.5991990566253662, "Finetune/Step": 4164, "Finetune/Step Time": 4.975277962163091} +{"Finetune/Learning Rate": 2.000106032794361e-06, "Finetune/Loss": 0.8250510096549988, "Finetune/Loss (Raw)": 1.0087307691574097, "Finetune/Step": 4165, "Finetune/Step Time": 4.97308686375618} +{"Finetune/Learning Rate": 1.996369730873353e-06, "Finetune/Loss": 0.82961106300354, "Finetune/Loss (Raw)": 0.94083172082901, "Finetune/Step": 4166, "Finetune/Step Time": 4.937334520742297} +{"Finetune/Learning Rate": 1.99263653500499e-06, "Finetune/Loss": 0.8283915519714355, "Finetune/Loss (Raw)": 0.7874153256416321, "Finetune/Step": 4167, "Finetune/Step Time": 4.922016076743603} +{"Finetune/Learning Rate": 1.9889064466380593e-06, "Finetune/Loss": 0.826647937297821, "Finetune/Loss (Raw)": 0.7042273283004761, "Finetune/Step": 4168, "Finetune/Step Time": 4.9202739875763655} +{"Finetune/Learning Rate": 1.985179467220133e-06, "Finetune/Loss": 0.8232401609420776, "Finetune/Loss (Raw)": 0.37245896458625793, "Finetune/Step": 4169, "Finetune/Step Time": 4.955638876184821} +{"Finetune/Learning Rate": 1.9814555981975803e-06, "Finetune/Loss": 0.8225089311599731, "Finetune/Loss (Raw)": 0.7666261196136475, "Finetune/Step": 4170, "Finetune/Step Time": 4.946839412674308} +{"Finetune/Learning Rate": 1.9777348410155606e-06, "Finetune/Loss": 0.822490394115448, "Finetune/Loss (Raw)": 0.9051989316940308, "Finetune/Step": 4171, "Finetune/Step Time": 4.949132617563009} +{"Finetune/Learning Rate": 1.9740171971180278e-06, "Finetune/Loss": 0.8226504921913147, "Finetune/Loss (Raw)": 0.9028798937797546, "Finetune/Step": 4172, "Finetune/Step Time": 4.969887474551797} +{"Finetune/Learning Rate": 1.9703026679477253e-06, "Finetune/Loss": 0.8267768621444702, "Finetune/Loss (Raw)": 0.8203136920928955, "Finetune/Step": 4173, "Finetune/Step Time": 4.940496526658535} +{"Finetune/Learning Rate": 1.9665912549461898e-06, "Finetune/Loss": 0.8231663107872009, "Finetune/Loss (Raw)": 0.35729968547821045, "Finetune/Step": 4174, "Finetune/Step Time": 4.970485650002956} +{"Finetune/Learning Rate": 1.962882959553751e-06, "Finetune/Loss": 0.8230292797088623, "Finetune/Loss (Raw)": 0.7803794741630554, "Finetune/Step": 4175, "Finetune/Step Time": 4.954052099958062} +{"Finetune/Learning Rate": 1.959177783209527e-06, "Finetune/Loss": 0.8263863325119019, "Finetune/Loss (Raw)": 0.802191436290741, "Finetune/Step": 4176, "Finetune/Step Time": 4.9314161613583565} +{"Finetune/Learning Rate": 1.955475727351417e-06, "Finetune/Loss": 0.8280129432678223, "Finetune/Loss (Raw)": 0.9645093083381653, "Finetune/Step": 4177, "Finetune/Step Time": 4.947660947218537} +{"Finetune/Learning Rate": 1.9517767934161224e-06, "Finetune/Loss": 0.8286372423171997, "Finetune/Loss (Raw)": 0.911258339881897, "Finetune/Step": 4178, "Finetune/Step Time": 4.956238571554422} +{"Finetune/Learning Rate": 1.9480809828391266e-06, "Finetune/Loss": 0.8293828368186951, "Finetune/Loss (Raw)": 0.8839459419250488, "Finetune/Step": 4179, "Finetune/Step Time": 4.963764680549502} +{"Finetune/Learning Rate": 1.9443882970547003e-06, "Finetune/Loss": 0.8302967548370361, "Finetune/Loss (Raw)": 0.9603686332702637, "Finetune/Step": 4180, "Finetune/Step Time": 4.9435476921498775} +{"Finetune/Learning Rate": 1.940698737495904e-06, "Finetune/Loss": 0.8337380886077881, "Finetune/Loss (Raw)": 0.8148924708366394, "Finetune/Step": 4181, "Finetune/Step Time": 4.9120476227253675} +{"Finetune/Learning Rate": 1.937012305594583e-06, "Finetune/Loss": 0.8323685526847839, "Finetune/Loss (Raw)": 0.6854044795036316, "Finetune/Step": 4182, "Finetune/Step Time": 4.9123643562197685} +{"Finetune/Learning Rate": 1.93332900278137e-06, "Finetune/Loss": 0.8326719999313354, "Finetune/Loss (Raw)": 0.9364317655563354, "Finetune/Step": 4183, "Finetune/Step Time": 4.911897202953696} +{"Finetune/Learning Rate": 1.9296488304856785e-06, "Finetune/Loss": 0.8348462581634521, "Finetune/Loss (Raw)": 0.9060071706771851, "Finetune/Step": 4184, "Finetune/Step Time": 4.894984655082226} +{"Finetune/Learning Rate": 1.92597179013572e-06, "Finetune/Loss": 0.8347750306129456, "Finetune/Loss (Raw)": 0.9699899554252625, "Finetune/Step": 4185, "Finetune/Step Time": 4.897696806117892} +{"Finetune/Learning Rate": 1.9222978831584793e-06, "Finetune/Loss": 0.8356599807739258, "Finetune/Loss (Raw)": 0.9441942572593689, "Finetune/Step": 4186, "Finetune/Step Time": 4.910650078207254} +{"Finetune/Learning Rate": 1.9186271109797215e-06, "Finetune/Loss": 0.8350306749343872, "Finetune/Loss (Raw)": 0.8035053014755249, "Finetune/Step": 4187, "Finetune/Step Time": 4.898338491097093} +{"Finetune/Learning Rate": 1.9149594750240087e-06, "Finetune/Loss": 0.8359492421150208, "Finetune/Loss (Raw)": 0.7719628214836121, "Finetune/Step": 4188, "Finetune/Step Time": 4.88929077796638} +{"Finetune/Learning Rate": 1.911294976714676e-06, "Finetune/Loss": 0.8358849287033081, "Finetune/Loss (Raw)": 0.8060598373413086, "Finetune/Step": 4189, "Finetune/Step Time": 4.871619077399373} +{"Finetune/Learning Rate": 1.9076336174738476e-06, "Finetune/Loss": 0.8333562612533569, "Finetune/Loss (Raw)": 0.6846886873245239, "Finetune/Step": 4190, "Finetune/Step Time": 4.872034892439842} +{"Finetune/Learning Rate": 1.9039753987224164e-06, "Finetune/Loss": 0.8378325700759888, "Finetune/Loss (Raw)": 0.9244899749755859, "Finetune/Step": 4191, "Finetune/Step Time": 4.834616897627711} +{"Finetune/Learning Rate": 1.9003203218800748e-06, "Finetune/Loss": 0.8428099751472473, "Finetune/Loss (Raw)": 1.0177725553512573, "Finetune/Step": 4192, "Finetune/Step Time": 4.796518722549081} +{"Finetune/Learning Rate": 1.8966683883652826e-06, "Finetune/Loss": 0.8390535116195679, "Finetune/Loss (Raw)": 0.30433693528175354, "Finetune/Step": 4193, "Finetune/Step Time": 4.8170689307153225} +{"Finetune/Learning Rate": 1.8930195995952827e-06, "Finetune/Loss": 0.8391972780227661, "Finetune/Loss (Raw)": 0.88992840051651, "Finetune/Step": 4194, "Finetune/Step Time": 4.821978708729148} +{"Finetune/Learning Rate": 1.8893739569861058e-06, "Finetune/Loss": 0.8385601043701172, "Finetune/Loss (Raw)": 0.9021466374397278, "Finetune/Step": 4195, "Finetune/Step Time": 4.8278229385614395} +{"Finetune/Learning Rate": 1.8857314619525469e-06, "Finetune/Loss": 0.838371217250824, "Finetune/Loss (Raw)": 0.8964260220527649, "Finetune/Step": 4196, "Finetune/Step Time": 4.831231955438852} +{"Finetune/Learning Rate": 1.8820921159081896e-06, "Finetune/Loss": 0.8357444405555725, "Finetune/Loss (Raw)": 0.5856754183769226, "Finetune/Step": 4197, "Finetune/Step Time": 4.8405850157141685} +{"Finetune/Learning Rate": 1.8784559202653907e-06, "Finetune/Loss": 0.8368424773216248, "Finetune/Loss (Raw)": 0.9650841951370239, "Finetune/Step": 4198, "Finetune/Step Time": 4.838669614866376} +{"Finetune/Learning Rate": 1.8748228764352915e-06, "Finetune/Loss": 0.8377485275268555, "Finetune/Loss (Raw)": 0.980596125125885, "Finetune/Step": 4199, "Finetune/Step Time": 4.80783512070775} +{"Finetune/Learning Rate": 1.8711929858278066e-06, "Finetune/Loss": 0.8388490676879883, "Finetune/Loss (Raw)": 0.9051979184150696, "Finetune/Step": 4200, "Finetune/Step Time": 4.804244142025709} +{"Finetune/Learning Rate": 1.8675662498516167e-06, "Finetune/Loss": 0.8383845090866089, "Finetune/Loss (Raw)": 0.8412843942642212, "Finetune/Step": 4201, "Finetune/Step Time": 4.814789669588208} +{"Finetune/Learning Rate": 1.863942669914196e-06, "Finetune/Loss": 0.8370836973190308, "Finetune/Loss (Raw)": 0.8193665742874146, "Finetune/Step": 4202, "Finetune/Step Time": 4.817387584596872} +{"Finetune/Learning Rate": 1.8603222474217819e-06, "Finetune/Loss": 0.8367494344711304, "Finetune/Loss (Raw)": 0.7818352580070496, "Finetune/Step": 4203, "Finetune/Step Time": 4.818352764472365} +{"Finetune/Learning Rate": 1.8567049837793894e-06, "Finetune/Loss": 0.8377885222434998, "Finetune/Loss (Raw)": 0.7993770837783813, "Finetune/Step": 4204, "Finetune/Step Time": 4.798812985420227} +{"Finetune/Learning Rate": 1.8530908803908088e-06, "Finetune/Loss": 0.8381071090698242, "Finetune/Loss (Raw)": 0.8922646045684814, "Finetune/Step": 4205, "Finetune/Step Time": 4.780196582898498} +{"Finetune/Learning Rate": 1.8494799386586015e-06, "Finetune/Loss": 0.8382796049118042, "Finetune/Loss (Raw)": 0.8315461874008179, "Finetune/Step": 4206, "Finetune/Step Time": 4.778014501556754} +{"Finetune/Learning Rate": 1.8458721599841056e-06, "Finetune/Loss": 0.8373092412948608, "Finetune/Loss (Raw)": 0.9146471619606018, "Finetune/Step": 4207, "Finetune/Step Time": 4.771651562303305} +{"Finetune/Learning Rate": 1.8422675457674254e-06, "Finetune/Loss": 0.8377034664154053, "Finetune/Loss (Raw)": 0.8283326625823975, "Finetune/Step": 4208, "Finetune/Step Time": 4.7615193370729685} +{"Finetune/Learning Rate": 1.8386660974074466e-06, "Finetune/Loss": 0.8392831683158875, "Finetune/Loss (Raw)": 1.0616997480392456, "Finetune/Step": 4209, "Finetune/Step Time": 4.738929705694318} +{"Finetune/Learning Rate": 1.8350678163018221e-06, "Finetune/Loss": 0.8396046757698059, "Finetune/Loss (Raw)": 0.8418872356414795, "Finetune/Step": 4210, "Finetune/Step Time": 4.734728451818228} +{"Finetune/Learning Rate": 1.831472703846966e-06, "Finetune/Loss": 0.8392126560211182, "Finetune/Loss (Raw)": 0.9307771921157837, "Finetune/Step": 4211, "Finetune/Step Time": 4.740002706646919} +{"Finetune/Learning Rate": 1.827880761438079e-06, "Finetune/Loss": 0.8416428565979004, "Finetune/Loss (Raw)": 0.7775143384933472, "Finetune/Step": 4212, "Finetune/Step Time": 4.704146156087518} +{"Finetune/Learning Rate": 1.8242919904691214e-06, "Finetune/Loss": 0.8405261039733887, "Finetune/Loss (Raw)": 0.7110990881919861, "Finetune/Step": 4213, "Finetune/Step Time": 4.702378038316965} +{"Finetune/Learning Rate": 1.820706392332824e-06, "Finetune/Loss": 0.8406313061714172, "Finetune/Loss (Raw)": 0.893957257270813, "Finetune/Step": 4214, "Finetune/Step Time": 4.692504553124309} +{"Finetune/Learning Rate": 1.8171239684206898e-06, "Finetune/Loss": 0.8396575450897217, "Finetune/Loss (Raw)": 0.868465006351471, "Finetune/Step": 4215, "Finetune/Step Time": 4.718671258538961} +{"Finetune/Learning Rate": 1.8135447201229861e-06, "Finetune/Loss": 0.8390487432479858, "Finetune/Loss (Raw)": 0.8683865666389465, "Finetune/Step": 4216, "Finetune/Step Time": 4.713583797216415} +{"Finetune/Learning Rate": 1.8099686488287483e-06, "Finetune/Loss": 0.8398107290267944, "Finetune/Loss (Raw)": 0.9421186447143555, "Finetune/Step": 4217, "Finetune/Step Time": 4.710592478513718} +{"Finetune/Learning Rate": 1.8063957559257794e-06, "Finetune/Loss": 0.839920163154602, "Finetune/Loss (Raw)": 0.8493693470954895, "Finetune/Step": 4218, "Finetune/Step Time": 4.709611646831036} +{"Finetune/Learning Rate": 1.8028260428006538e-06, "Finetune/Loss": 0.8392564058303833, "Finetune/Loss (Raw)": 0.7454174160957336, "Finetune/Step": 4219, "Finetune/Step Time": 4.692228076979518} +{"Finetune/Learning Rate": 1.7992595108387068e-06, "Finetune/Loss": 0.839527428150177, "Finetune/Loss (Raw)": 0.8738299608230591, "Finetune/Step": 4220, "Finetune/Step Time": 4.681526482105255} +{"Finetune/Learning Rate": 1.795696161424032e-06, "Finetune/Loss": 0.8396289944648743, "Finetune/Loss (Raw)": 0.9988754391670227, "Finetune/Step": 4221, "Finetune/Step Time": 4.683324815705419} +{"Finetune/Learning Rate": 1.7921359959395046e-06, "Finetune/Loss": 0.838837742805481, "Finetune/Loss (Raw)": 0.9605404734611511, "Finetune/Step": 4222, "Finetune/Step Time": 4.686776101589203} +{"Finetune/Learning Rate": 1.7885790157667526e-06, "Finetune/Loss": 0.8403832912445068, "Finetune/Loss (Raw)": 0.9690359830856323, "Finetune/Step": 4223, "Finetune/Step Time": 4.678094461560249} +{"Finetune/Learning Rate": 1.7850252222861697e-06, "Finetune/Loss": 0.8386403322219849, "Finetune/Loss (Raw)": 0.8044837117195129, "Finetune/Step": 4224, "Finetune/Step Time": 4.683250281959772} +{"Finetune/Learning Rate": 1.7814746168769138e-06, "Finetune/Loss": 0.8373515605926514, "Finetune/Loss (Raw)": 0.7927585244178772, "Finetune/Step": 4225, "Finetune/Step Time": 4.670345459133387} +{"Finetune/Learning Rate": 1.7779272009169068e-06, "Finetune/Loss": 0.8396402597427368, "Finetune/Loss (Raw)": 0.9874336123466492, "Finetune/Step": 4226, "Finetune/Step Time": 4.663752179592848} +{"Finetune/Learning Rate": 1.77438297578283e-06, "Finetune/Loss": 0.8406792879104614, "Finetune/Loss (Raw)": 1.0613864660263062, "Finetune/Step": 4227, "Finetune/Step Time": 4.663322493433952} +{"Finetune/Learning Rate": 1.7708419428501256e-06, "Finetune/Loss": 0.8399068117141724, "Finetune/Loss (Raw)": 0.8555634021759033, "Finetune/Step": 4228, "Finetune/Step Time": 4.6653064880520105} +{"Finetune/Learning Rate": 1.7673041034930083e-06, "Finetune/Loss": 0.8400124907493591, "Finetune/Loss (Raw)": 0.900829553604126, "Finetune/Step": 4229, "Finetune/Step Time": 4.648293336853385} +{"Finetune/Learning Rate": 1.7637694590844357e-06, "Finetune/Loss": 0.8354886174201965, "Finetune/Loss (Raw)": 0.2688225209712982, "Finetune/Step": 4230, "Finetune/Step Time": 4.6779138296842575} +{"Finetune/Learning Rate": 1.7602380109961348e-06, "Finetune/Loss": 0.8336787819862366, "Finetune/Loss (Raw)": 0.6542133092880249, "Finetune/Step": 4231, "Finetune/Step Time": 4.728750746697187} +{"Finetune/Learning Rate": 1.7567097605985972e-06, "Finetune/Loss": 0.8338761329650879, "Finetune/Loss (Raw)": 0.8926829695701599, "Finetune/Step": 4232, "Finetune/Step Time": 4.737676125019789} +{"Finetune/Learning Rate": 1.7531847092610644e-06, "Finetune/Loss": 0.8339662551879883, "Finetune/Loss (Raw)": 0.7893926501274109, "Finetune/Step": 4233, "Finetune/Step Time": 4.748709436506033} +{"Finetune/Learning Rate": 1.7496628583515395e-06, "Finetune/Loss": 0.8360686302185059, "Finetune/Loss (Raw)": 1.0937345027923584, "Finetune/Step": 4234, "Finetune/Step Time": 4.751094114035368} +{"Finetune/Learning Rate": 1.7461442092367864e-06, "Finetune/Loss": 0.8364649415016174, "Finetune/Loss (Raw)": 0.892113208770752, "Finetune/Step": 4235, "Finetune/Step Time": 4.7511455826461315} +{"Finetune/Learning Rate": 1.7426287632823213e-06, "Finetune/Loss": 0.8347580432891846, "Finetune/Loss (Raw)": 0.7249601483345032, "Finetune/Step": 4236, "Finetune/Step Time": 4.759873518720269} +{"Finetune/Learning Rate": 1.7391165218524232e-06, "Finetune/Loss": 0.8361561298370361, "Finetune/Loss (Raw)": 1.000360131263733, "Finetune/Step": 4237, "Finetune/Step Time": 4.752122886478901} +{"Finetune/Learning Rate": 1.735607486310119e-06, "Finetune/Loss": 0.836478054523468, "Finetune/Loss (Raw)": 0.953429639339447, "Finetune/Step": 4238, "Finetune/Step Time": 4.739714950323105} +{"Finetune/Learning Rate": 1.7321016580172068e-06, "Finetune/Loss": 0.835808515548706, "Finetune/Loss (Raw)": 0.9111782312393188, "Finetune/Step": 4239, "Finetune/Step Time": 4.736058058217168} +{"Finetune/Learning Rate": 1.728599038334221e-06, "Finetune/Loss": 0.8364827632904053, "Finetune/Loss (Raw)": 0.8315519094467163, "Finetune/Step": 4240, "Finetune/Step Time": 4.723710449412465} +{"Finetune/Learning Rate": 1.7250996286204602e-06, "Finetune/Loss": 0.8375400304794312, "Finetune/Loss (Raw)": 0.8515348434448242, "Finetune/Step": 4241, "Finetune/Step Time": 4.725165084004402} +{"Finetune/Learning Rate": 1.7216034302339824e-06, "Finetune/Loss": 0.8380130529403687, "Finetune/Loss (Raw)": 0.9367907643318176, "Finetune/Step": 4242, "Finetune/Step Time": 4.729767370969057} +{"Finetune/Learning Rate": 1.718110444531591e-06, "Finetune/Loss": 0.8380979895591736, "Finetune/Loss (Raw)": 0.885753333568573, "Finetune/Step": 4243, "Finetune/Step Time": 4.750108011066914} +{"Finetune/Learning Rate": 1.7146206728688465e-06, "Finetune/Loss": 0.8401089906692505, "Finetune/Loss (Raw)": 0.8822494149208069, "Finetune/Step": 4244, "Finetune/Step Time": 4.73350659571588} +{"Finetune/Learning Rate": 1.7111341166000594e-06, "Finetune/Loss": 0.8409498929977417, "Finetune/Loss (Raw)": 0.8803353905677795, "Finetune/Step": 4245, "Finetune/Step Time": 4.735607922077179} +{"Finetune/Learning Rate": 1.7076507770782957e-06, "Finetune/Loss": 0.8394448757171631, "Finetune/Loss (Raw)": 0.856157124042511, "Finetune/Step": 4246, "Finetune/Step Time": 4.766919830814004} +{"Finetune/Learning Rate": 1.7041706556553717e-06, "Finetune/Loss": 0.8385262489318848, "Finetune/Loss (Raw)": 0.8658429384231567, "Finetune/Step": 4247, "Finetune/Step Time": 4.768467154353857} +{"Finetune/Learning Rate": 1.7006937536818503e-06, "Finetune/Loss": 0.8337953090667725, "Finetune/Loss (Raw)": 0.29155078530311584, "Finetune/Step": 4248, "Finetune/Step Time": 4.790490657091141} +{"Finetune/Learning Rate": 1.6972200725070587e-06, "Finetune/Loss": 0.8341736793518066, "Finetune/Loss (Raw)": 0.9265157580375671, "Finetune/Step": 4249, "Finetune/Step Time": 4.78332794085145} +{"Finetune/Learning Rate": 1.6937496134790554e-06, "Finetune/Loss": 0.8330439329147339, "Finetune/Loss (Raw)": 0.7273529171943665, "Finetune/Step": 4250, "Finetune/Step Time": 4.779457489028573} +{"Finetune/Learning Rate": 1.6902823779446586e-06, "Finetune/Loss": 0.8309252858161926, "Finetune/Loss (Raw)": 0.7441331148147583, "Finetune/Step": 4251, "Finetune/Step Time": 4.778640089556575} +{"Finetune/Learning Rate": 1.686818367249441e-06, "Finetune/Loss": 0.8340668678283691, "Finetune/Loss (Raw)": 1.0523667335510254, "Finetune/Step": 4252, "Finetune/Step Time": 4.780519297346473} +{"Finetune/Learning Rate": 1.6833575827377135e-06, "Finetune/Loss": 0.8320817947387695, "Finetune/Loss (Raw)": 0.6490711569786072, "Finetune/Step": 4253, "Finetune/Step Time": 4.801458323374391} +{"Finetune/Learning Rate": 1.6799000257525388e-06, "Finetune/Loss": 0.8304408192634583, "Finetune/Loss (Raw)": 0.67381751537323, "Finetune/Step": 4254, "Finetune/Step Time": 4.811845174059272} +{"Finetune/Learning Rate": 1.6764456976357279e-06, "Finetune/Loss": 0.8294596672058105, "Finetune/Loss (Raw)": 0.7735090255737305, "Finetune/Step": 4255, "Finetune/Step Time": 4.818123975768685} +{"Finetune/Learning Rate": 1.6729945997278386e-06, "Finetune/Loss": 0.8301382660865784, "Finetune/Loss (Raw)": 0.9118613600730896, "Finetune/Step": 4256, "Finetune/Step Time": 4.834431290626526} +{"Finetune/Learning Rate": 1.6695467333681748e-06, "Finetune/Loss": 0.8312580585479736, "Finetune/Loss (Raw)": 0.9642268419265747, "Finetune/Step": 4257, "Finetune/Step Time": 4.83313612267375} +{"Finetune/Learning Rate": 1.6661020998947818e-06, "Finetune/Loss": 0.8300737142562866, "Finetune/Loss (Raw)": 0.6828977465629578, "Finetune/Step": 4258, "Finetune/Step Time": 4.834618737921119} +{"Finetune/Learning Rate": 1.6626607006444628e-06, "Finetune/Loss": 0.8310354948043823, "Finetune/Loss (Raw)": 1.0077224969863892, "Finetune/Step": 4259, "Finetune/Step Time": 4.8385648392140865} +{"Finetune/Learning Rate": 1.6592225369527515e-06, "Finetune/Loss": 0.8306416273117065, "Finetune/Loss (Raw)": 0.8421295881271362, "Finetune/Step": 4260, "Finetune/Step Time": 4.8293375968933105} +{"Finetune/Learning Rate": 1.6557876101539305e-06, "Finetune/Loss": 0.8313127160072327, "Finetune/Loss (Raw)": 0.9311344623565674, "Finetune/Step": 4261, "Finetune/Step Time": 4.823149660602212} +{"Finetune/Learning Rate": 1.6523559215810336e-06, "Finetune/Loss": 0.8311418294906616, "Finetune/Loss (Raw)": 0.8424153327941895, "Finetune/Step": 4262, "Finetune/Step Time": 4.813454534858465} +{"Finetune/Learning Rate": 1.64892747256583e-06, "Finetune/Loss": 0.8330550193786621, "Finetune/Loss (Raw)": 1.0326472520828247, "Finetune/Step": 4263, "Finetune/Step Time": 4.811570284888148} +{"Finetune/Learning Rate": 1.645502264438833e-06, "Finetune/Loss": 0.8326183557510376, "Finetune/Loss (Raw)": 0.7833065986633301, "Finetune/Step": 4264, "Finetune/Step Time": 4.822725621983409} +{"Finetune/Learning Rate": 1.6420802985292995e-06, "Finetune/Loss": 0.8334106206893921, "Finetune/Loss (Raw)": 0.8963707089424133, "Finetune/Step": 4265, "Finetune/Step Time": 4.830459531396627} +{"Finetune/Learning Rate": 1.638661576165227e-06, "Finetune/Loss": 0.8333659768104553, "Finetune/Loss (Raw)": 0.9483532309532166, "Finetune/Step": 4266, "Finetune/Step Time": 4.829526524990797} +{"Finetune/Learning Rate": 1.6352460986733564e-06, "Finetune/Loss": 0.8332616090774536, "Finetune/Loss (Raw)": 0.7712196111679077, "Finetune/Step": 4267, "Finetune/Step Time": 4.843192243948579} +{"Finetune/Learning Rate": 1.6318338673791655e-06, "Finetune/Loss": 0.8331255912780762, "Finetune/Loss (Raw)": 0.7985696792602539, "Finetune/Step": 4268, "Finetune/Step Time": 4.851491888985038} +{"Finetune/Learning Rate": 1.6284248836068772e-06, "Finetune/Loss": 0.8357518911361694, "Finetune/Loss (Raw)": 0.7221354842185974, "Finetune/Step": 4269, "Finetune/Step Time": 4.852494981139898} +{"Finetune/Learning Rate": 1.6250191486794497e-06, "Finetune/Loss": 0.8362657427787781, "Finetune/Loss (Raw)": 0.8390509486198425, "Finetune/Step": 4270, "Finetune/Step Time": 4.8527957163751125} +{"Finetune/Learning Rate": 1.6216166639185804e-06, "Finetune/Loss": 0.8326610326766968, "Finetune/Loss (Raw)": 0.3263438940048218, "Finetune/Step": 4271, "Finetune/Step Time": 4.889867078512907} +{"Finetune/Learning Rate": 1.618217430644713e-06, "Finetune/Loss": 0.8375349640846252, "Finetune/Loss (Raw)": 0.912510871887207, "Finetune/Step": 4272, "Finetune/Step Time": 4.851550852879882} +{"Finetune/Learning Rate": 1.6148214501770221e-06, "Finetune/Loss": 0.8375482559204102, "Finetune/Loss (Raw)": 0.7415867447853088, "Finetune/Step": 4273, "Finetune/Step Time": 4.8391807191073895} +{"Finetune/Learning Rate": 1.6114287238334148e-06, "Finetune/Loss": 0.8371105790138245, "Finetune/Loss (Raw)": 0.858176052570343, "Finetune/Step": 4274, "Finetune/Step Time": 4.844893142580986} +{"Finetune/Learning Rate": 1.6080392529305489e-06, "Finetune/Loss": 0.8368887305259705, "Finetune/Loss (Raw)": 0.8241074681282043, "Finetune/Step": 4275, "Finetune/Step Time": 4.85589193738997} +{"Finetune/Learning Rate": 1.60465303878381e-06, "Finetune/Loss": 0.8370919823646545, "Finetune/Loss (Raw)": 0.9232202172279358, "Finetune/Step": 4276, "Finetune/Step Time": 4.85688771866262} +{"Finetune/Learning Rate": 1.6012700827073213e-06, "Finetune/Loss": 0.8371098637580872, "Finetune/Loss (Raw)": 0.8424772620201111, "Finetune/Step": 4277, "Finetune/Step Time": 4.879153652116656} +{"Finetune/Learning Rate": 1.5978903860139416e-06, "Finetune/Loss": 0.8383231163024902, "Finetune/Loss (Raw)": 0.9260137677192688, "Finetune/Step": 4278, "Finetune/Step Time": 4.8526870254427195} +{"Finetune/Learning Rate": 1.5945139500152662e-06, "Finetune/Loss": 0.8378093242645264, "Finetune/Loss (Raw)": 0.799063503742218, "Finetune/Step": 4279, "Finetune/Step Time": 4.850650954991579} +{"Finetune/Learning Rate": 1.5911407760216235e-06, "Finetune/Loss": 0.8384700417518616, "Finetune/Loss (Raw)": 0.943560004234314, "Finetune/Step": 4280, "Finetune/Step Time": 4.843926340341568} +{"Finetune/Learning Rate": 1.5877708653420742e-06, "Finetune/Loss": 0.839775800704956, "Finetune/Loss (Raw)": 0.9975067973136902, "Finetune/Step": 4281, "Finetune/Step Time": 4.849230080842972} +{"Finetune/Learning Rate": 1.58440421928442e-06, "Finetune/Loss": 0.8402919769287109, "Finetune/Loss (Raw)": 0.9517198801040649, "Finetune/Step": 4282, "Finetune/Step Time": 4.84740174934268} +{"Finetune/Learning Rate": 1.581040839155189e-06, "Finetune/Loss": 0.8407663106918335, "Finetune/Loss (Raw)": 0.8786234259605408, "Finetune/Step": 4283, "Finetune/Step Time": 4.852552888914943} +{"Finetune/Learning Rate": 1.5776807262596371e-06, "Finetune/Loss": 0.8412100076675415, "Finetune/Loss (Raw)": 0.9163694381713867, "Finetune/Step": 4284, "Finetune/Step Time": 4.85102298296988} +{"Finetune/Learning Rate": 1.574323881901767e-06, "Finetune/Loss": 0.8388450145721436, "Finetune/Loss (Raw)": 0.6708480715751648, "Finetune/Step": 4285, "Finetune/Step Time": 4.885799769312143} +{"Finetune/Learning Rate": 1.5709703073843008e-06, "Finetune/Loss": 0.839381217956543, "Finetune/Loss (Raw)": 0.9010094404220581, "Finetune/Step": 4286, "Finetune/Step Time": 4.876845443621278} +{"Finetune/Learning Rate": 1.5676200040086941e-06, "Finetune/Loss": 0.8392176032066345, "Finetune/Loss (Raw)": 0.9095468521118164, "Finetune/Step": 4287, "Finetune/Step Time": 4.879897454753518} +{"Finetune/Learning Rate": 1.564272973075136e-06, "Finetune/Loss": 0.8394056558609009, "Finetune/Loss (Raw)": 1.002770185470581, "Finetune/Step": 4288, "Finetune/Step Time": 4.895064987242222} +{"Finetune/Learning Rate": 1.560929215882544e-06, "Finetune/Loss": 0.8386696577072144, "Finetune/Loss (Raw)": 0.8119786977767944, "Finetune/Step": 4289, "Finetune/Step Time": 4.896575482562184} +{"Finetune/Learning Rate": 1.5575887337285644e-06, "Finetune/Loss": 0.8388847708702087, "Finetune/Loss (Raw)": 0.9745019674301147, "Finetune/Step": 4290, "Finetune/Step Time": 4.897470714524388} +{"Finetune/Learning Rate": 1.5542515279095704e-06, "Finetune/Loss": 0.8405209183692932, "Finetune/Loss (Raw)": 0.9245709180831909, "Finetune/Step": 4291, "Finetune/Step Time": 4.887343754991889} +{"Finetune/Learning Rate": 1.5509175997206705e-06, "Finetune/Loss": 0.8430755138397217, "Finetune/Loss (Raw)": 0.9261924624443054, "Finetune/Step": 4292, "Finetune/Step Time": 4.882555067539215} +{"Finetune/Learning Rate": 1.5475869504556996e-06, "Finetune/Loss": 0.8422874808311462, "Finetune/Loss (Raw)": 0.9078588485717773, "Finetune/Step": 4293, "Finetune/Step Time": 4.88523187674582} +{"Finetune/Learning Rate": 1.5442595814072081e-06, "Finetune/Loss": 0.8417574167251587, "Finetune/Loss (Raw)": 0.8729830384254456, "Finetune/Step": 4294, "Finetune/Step Time": 4.907445337623358} +{"Finetune/Learning Rate": 1.540935493866491e-06, "Finetune/Loss": 0.8379452228546143, "Finetune/Loss (Raw)": 0.2994535267353058, "Finetune/Step": 4295, "Finetune/Step Time": 4.940112786367536} +{"Finetune/Learning Rate": 1.53761468912356e-06, "Finetune/Loss": 0.840129017829895, "Finetune/Loss (Raw)": 0.9837530255317688, "Finetune/Step": 4296, "Finetune/Step Time": 4.936907943338156} +{"Finetune/Learning Rate": 1.5342971684671537e-06, "Finetune/Loss": 0.8442227840423584, "Finetune/Loss (Raw)": 0.896470308303833, "Finetune/Step": 4297, "Finetune/Step Time": 4.904121410101652} +{"Finetune/Learning Rate": 1.530982933184737e-06, "Finetune/Loss": 0.843812108039856, "Finetune/Loss (Raw)": 0.7140560746192932, "Finetune/Step": 4298, "Finetune/Step Time": 4.898565357550979} +{"Finetune/Learning Rate": 1.527671984562501e-06, "Finetune/Loss": 0.844035804271698, "Finetune/Loss (Raw)": 0.9338253140449524, "Finetune/Step": 4299, "Finetune/Step Time": 4.905381791293621} +{"Finetune/Learning Rate": 1.524364323885359e-06, "Finetune/Loss": 0.8443838357925415, "Finetune/Loss (Raw)": 0.9474257230758667, "Finetune/Step": 4300, "Finetune/Step Time": 4.890140440315008} +{"Finetune/Learning Rate": 1.5210599524369484e-06, "Finetune/Loss": 0.840826153755188, "Finetune/Loss (Raw)": 0.3649410009384155, "Finetune/Step": 4301, "Finetune/Step Time": 4.919643001630902} +{"Finetune/Learning Rate": 1.517758871499635e-06, "Finetune/Loss": 0.8455931544303894, "Finetune/Loss (Raw)": 0.9674741625785828, "Finetune/Step": 4302, "Finetune/Step Time": 4.889171160757542} +{"Finetune/Learning Rate": 1.5144610823545035e-06, "Finetune/Loss": 0.84724360704422, "Finetune/Loss (Raw)": 0.9916319847106934, "Finetune/Step": 4303, "Finetune/Step Time": 4.889309300109744} +{"Finetune/Learning Rate": 1.511166586281355e-06, "Finetune/Loss": 0.8436442613601685, "Finetune/Loss (Raw)": 0.3414759039878845, "Finetune/Step": 4304, "Finetune/Step Time": 4.911534458398819} +{"Finetune/Learning Rate": 1.507875384558727e-06, "Finetune/Loss": 0.8421510457992554, "Finetune/Loss (Raw)": 0.7733780741691589, "Finetune/Step": 4305, "Finetune/Step Time": 4.902782369405031} +{"Finetune/Learning Rate": 1.504587478463867e-06, "Finetune/Loss": 0.8415910601615906, "Finetune/Loss (Raw)": 0.8395813703536987, "Finetune/Step": 4306, "Finetune/Step Time": 4.902910614386201} +{"Finetune/Learning Rate": 1.5013028692727483e-06, "Finetune/Loss": 0.8399924635887146, "Finetune/Loss (Raw)": 0.6793204545974731, "Finetune/Step": 4307, "Finetune/Step Time": 4.891123134642839} +{"Finetune/Learning Rate": 1.4980215582600621e-06, "Finetune/Loss": 0.8394261598587036, "Finetune/Loss (Raw)": 0.8878905773162842, "Finetune/Step": 4308, "Finetune/Step Time": 4.891722898930311} +{"Finetune/Learning Rate": 1.4947435466992222e-06, "Finetune/Loss": 0.8397722244262695, "Finetune/Loss (Raw)": 0.8591791987419128, "Finetune/Step": 4309, "Finetune/Step Time": 4.896425884217024} +{"Finetune/Learning Rate": 1.4914688358623608e-06, "Finetune/Loss": 0.8371633887290955, "Finetune/Loss (Raw)": 0.35148361325263977, "Finetune/Step": 4310, "Finetune/Step Time": 4.9213855266571045} +{"Finetune/Learning Rate": 1.488197427020327e-06, "Finetune/Loss": 0.836647629737854, "Finetune/Loss (Raw)": 0.870409369468689, "Finetune/Step": 4311, "Finetune/Step Time": 4.927364306524396} +{"Finetune/Learning Rate": 1.4849293214426975e-06, "Finetune/Loss": 0.8372211456298828, "Finetune/Loss (Raw)": 0.9794126152992249, "Finetune/Step": 4312, "Finetune/Step Time": 4.929605044424534} +{"Finetune/Learning Rate": 1.481664520397753e-06, "Finetune/Loss": 0.8373109698295593, "Finetune/Loss (Raw)": 0.9814946055412292, "Finetune/Step": 4313, "Finetune/Step Time": 4.9404502138495445} +{"Finetune/Learning Rate": 1.4784030251524994e-06, "Finetune/Loss": 0.8371922969818115, "Finetune/Loss (Raw)": 0.9290077090263367, "Finetune/Step": 4314, "Finetune/Step Time": 4.936204085126519} +{"Finetune/Learning Rate": 1.4751448369726652e-06, "Finetune/Loss": 0.8375204801559448, "Finetune/Loss (Raw)": 0.845507800579071, "Finetune/Step": 4315, "Finetune/Step Time": 4.932974513620138} +{"Finetune/Learning Rate": 1.4718899571226842e-06, "Finetune/Loss": 0.8378521800041199, "Finetune/Loss (Raw)": 0.8144196271896362, "Finetune/Step": 4316, "Finetune/Step Time": 4.93638851493597} +{"Finetune/Learning Rate": 1.468638386865716e-06, "Finetune/Loss": 0.8382861018180847, "Finetune/Loss (Raw)": 0.8616029620170593, "Finetune/Step": 4317, "Finetune/Step Time": 4.934381127357483} +{"Finetune/Learning Rate": 1.465390127463625e-06, "Finetune/Loss": 0.8390926122665405, "Finetune/Loss (Raw)": 0.78792405128479, "Finetune/Step": 4318, "Finetune/Step Time": 4.94355434179306} +{"Finetune/Learning Rate": 1.4621451801770038e-06, "Finetune/Loss": 0.8376258611679077, "Finetune/Loss (Raw)": 0.7367397546768188, "Finetune/Step": 4319, "Finetune/Step Time": 4.949722865596414} +{"Finetune/Learning Rate": 1.4589035462651502e-06, "Finetune/Loss": 0.8361039161682129, "Finetune/Loss (Raw)": 0.8229721188545227, "Finetune/Step": 4320, "Finetune/Step Time": 4.953225625678897} +{"Finetune/Learning Rate": 1.4556652269860772e-06, "Finetune/Loss": 0.8392800092697144, "Finetune/Loss (Raw)": 0.7108691930770874, "Finetune/Step": 4321, "Finetune/Step Time": 4.931343227624893} +{"Finetune/Learning Rate": 1.4524302235965203e-06, "Finetune/Loss": 0.8402346968650818, "Finetune/Loss (Raw)": 1.0121313333511353, "Finetune/Step": 4322, "Finetune/Step Time": 4.928859416395426} +{"Finetune/Learning Rate": 1.4491985373519146e-06, "Finetune/Loss": 0.8386805653572083, "Finetune/Loss (Raw)": 0.7032194137573242, "Finetune/Step": 4323, "Finetune/Step Time": 4.920629048720002} +{"Finetune/Learning Rate": 1.4459701695064155e-06, "Finetune/Loss": 0.838042140007019, "Finetune/Loss (Raw)": 0.8147046566009521, "Finetune/Step": 4324, "Finetune/Step Time": 4.914508018642664} +{"Finetune/Learning Rate": 1.4427451213128873e-06, "Finetune/Loss": 0.8386491537094116, "Finetune/Loss (Raw)": 0.6633752584457397, "Finetune/Step": 4325, "Finetune/Step Time": 4.90397509559989} +{"Finetune/Learning Rate": 1.4395233940229147e-06, "Finetune/Loss": 0.8369389176368713, "Finetune/Loss (Raw)": 0.7461718320846558, "Finetune/Step": 4326, "Finetune/Step Time": 4.912475412711501} +{"Finetune/Learning Rate": 1.4363049888867853e-06, "Finetune/Loss": 0.8348535299301147, "Finetune/Loss (Raw)": 0.7136707305908203, "Finetune/Step": 4327, "Finetune/Step Time": 4.934916956350207} +{"Finetune/Learning Rate": 1.4330899071534942e-06, "Finetune/Loss": 0.8345089554786682, "Finetune/Loss (Raw)": 0.8610906600952148, "Finetune/Step": 4328, "Finetune/Step Time": 4.928793586790562} +{"Finetune/Learning Rate": 1.4298781500707558e-06, "Finetune/Loss": 0.8346226215362549, "Finetune/Loss (Raw)": 0.8558369874954224, "Finetune/Step": 4329, "Finetune/Step Time": 4.923378940671682} +{"Finetune/Learning Rate": 1.426669718884991e-06, "Finetune/Loss": 0.8356871604919434, "Finetune/Loss (Raw)": 0.9556199312210083, "Finetune/Step": 4330, "Finetune/Step Time": 4.923101119697094} +{"Finetune/Learning Rate": 1.4234646148413277e-06, "Finetune/Loss": 0.8370020985603333, "Finetune/Loss (Raw)": 0.9501481056213379, "Finetune/Step": 4331, "Finetune/Step Time": 4.922907065600157} +{"Finetune/Learning Rate": 1.420262839183605e-06, "Finetune/Loss": 0.8377429842948914, "Finetune/Loss (Raw)": 0.8942160606384277, "Finetune/Step": 4332, "Finetune/Step Time": 4.925417635589838} +{"Finetune/Learning Rate": 1.4170643931543693e-06, "Finetune/Loss": 0.8382882475852966, "Finetune/Loss (Raw)": 0.9620546102523804, "Finetune/Step": 4333, "Finetune/Step Time": 4.931905595585704} +{"Finetune/Learning Rate": 1.4138692779948749e-06, "Finetune/Loss": 0.8391396999359131, "Finetune/Loss (Raw)": 0.9405342936515808, "Finetune/Step": 4334, "Finetune/Step Time": 4.930538231506944} +{"Finetune/Learning Rate": 1.4106774949450808e-06, "Finetune/Loss": 0.839103639125824, "Finetune/Loss (Raw)": 0.9100303053855896, "Finetune/Step": 4335, "Finetune/Step Time": 4.924986906349659} +{"Finetune/Learning Rate": 1.4074890452436607e-06, "Finetune/Loss": 0.8404746055603027, "Finetune/Loss (Raw)": 1.003815770149231, "Finetune/Step": 4336, "Finetune/Step Time": 4.940878979861736} +{"Finetune/Learning Rate": 1.4043039301279904e-06, "Finetune/Loss": 0.8373416662216187, "Finetune/Loss (Raw)": 0.6606874465942383, "Finetune/Step": 4337, "Finetune/Step Time": 4.963728856295347} +{"Finetune/Learning Rate": 1.4011221508341433e-06, "Finetune/Loss": 0.8375411033630371, "Finetune/Loss (Raw)": 0.8674159049987793, "Finetune/Step": 4338, "Finetune/Step Time": 4.962521439418197} +{"Finetune/Learning Rate": 1.3979437085969127e-06, "Finetune/Loss": 0.8364689350128174, "Finetune/Loss (Raw)": 0.7935348153114319, "Finetune/Step": 4339, "Finetune/Step Time": 4.962901385501027} +{"Finetune/Learning Rate": 1.3947686046497878e-06, "Finetune/Loss": 0.8371161818504333, "Finetune/Loss (Raw)": 0.860358715057373, "Finetune/Step": 4340, "Finetune/Step Time": 4.977356802672148} +{"Finetune/Learning Rate": 1.3915968402249647e-06, "Finetune/Loss": 0.8391116857528687, "Finetune/Loss (Raw)": 0.966530978679657, "Finetune/Step": 4341, "Finetune/Step Time": 4.969032395631075} +{"Finetune/Learning Rate": 1.3884284165533423e-06, "Finetune/Loss": 0.8386731147766113, "Finetune/Loss (Raw)": 0.8378136157989502, "Finetune/Step": 4342, "Finetune/Step Time": 5.015292823314667} +{"Finetune/Learning Rate": 1.3852633348645262e-06, "Finetune/Loss": 0.8381586074829102, "Finetune/Loss (Raw)": 0.8026145100593567, "Finetune/Step": 4343, "Finetune/Step Time": 4.978562893345952} +{"Finetune/Learning Rate": 1.38210159638682e-06, "Finetune/Loss": 0.8384066224098206, "Finetune/Loss (Raw)": 0.900126576423645, "Finetune/Step": 4344, "Finetune/Step Time": 4.987918211147189} +{"Finetune/Learning Rate": 1.3789432023472315e-06, "Finetune/Loss": 0.8377580046653748, "Finetune/Loss (Raw)": 0.859098494052887, "Finetune/Step": 4345, "Finetune/Step Time": 4.987414948642254} +{"Finetune/Learning Rate": 1.3757881539714768e-06, "Finetune/Loss": 0.8379226326942444, "Finetune/Loss (Raw)": 0.8704437017440796, "Finetune/Step": 4346, "Finetune/Step Time": 4.990897800773382} +{"Finetune/Learning Rate": 1.3726364524839687e-06, "Finetune/Loss": 0.8396205902099609, "Finetune/Loss (Raw)": 0.9627503156661987, "Finetune/Step": 4347, "Finetune/Step Time": 4.983728630468249} +{"Finetune/Learning Rate": 1.369488099107814e-06, "Finetune/Loss": 0.8399714827537537, "Finetune/Loss (Raw)": 0.9187493920326233, "Finetune/Step": 4348, "Finetune/Step Time": 4.989781949669123} +{"Finetune/Learning Rate": 1.3663430950648326e-06, "Finetune/Loss": 0.8398138880729675, "Finetune/Loss (Raw)": 0.9787021279335022, "Finetune/Step": 4349, "Finetune/Step Time": 4.986715961247683} +{"Finetune/Learning Rate": 1.363201441575539e-06, "Finetune/Loss": 0.8373379111289978, "Finetune/Loss (Raw)": 0.64361572265625, "Finetune/Step": 4350, "Finetune/Step Time": 4.9986490327864885} +{"Finetune/Learning Rate": 1.360063139859147e-06, "Finetune/Loss": 0.8367905616760254, "Finetune/Loss (Raw)": 0.8989762663841248, "Finetune/Step": 4351, "Finetune/Step Time": 4.998237604275346} +{"Finetune/Learning Rate": 1.3569281911335686e-06, "Finetune/Loss": 0.8372486233711243, "Finetune/Loss (Raw)": 0.8631086945533752, "Finetune/Step": 4352, "Finetune/Step Time": 4.989726388826966} +{"Finetune/Learning Rate": 1.3537965966154177e-06, "Finetune/Loss": 0.8381965160369873, "Finetune/Loss (Raw)": 0.9140966534614563, "Finetune/Step": 4353, "Finetune/Step Time": 4.98753309994936} +{"Finetune/Learning Rate": 1.3506683575200053e-06, "Finetune/Loss": 0.8363449573516846, "Finetune/Loss (Raw)": 0.7504288554191589, "Finetune/Step": 4354, "Finetune/Step Time": 4.986571066081524} +{"Finetune/Learning Rate": 1.347543475061337e-06, "Finetune/Loss": 0.8349747657775879, "Finetune/Loss (Raw)": 0.8860052824020386, "Finetune/Step": 4355, "Finetune/Step Time": 4.98231759481132} +{"Finetune/Learning Rate": 1.344421950452126e-06, "Finetune/Loss": 0.8352464437484741, "Finetune/Loss (Raw)": 0.890339195728302, "Finetune/Step": 4356, "Finetune/Step Time": 4.980474365875125} +{"Finetune/Learning Rate": 1.341303784903767e-06, "Finetune/Loss": 0.8353833556175232, "Finetune/Loss (Raw)": 0.9183517694473267, "Finetune/Step": 4357, "Finetune/Step Time": 4.9805889669805765} +{"Finetune/Learning Rate": 1.3381889796263602e-06, "Finetune/Loss": 0.8403656482696533, "Finetune/Loss (Raw)": 0.9065557718276978, "Finetune/Step": 4358, "Finetune/Step Time": 4.955368675291538} +{"Finetune/Learning Rate": 1.335077535828706e-06, "Finetune/Loss": 0.842176079750061, "Finetune/Loss (Raw)": 0.885941743850708, "Finetune/Step": 4359, "Finetune/Step Time": 4.9042105209082365} +{"Finetune/Learning Rate": 1.3319694547182916e-06, "Finetune/Loss": 0.8415406942367554, "Finetune/Loss (Raw)": 0.8113608956336975, "Finetune/Step": 4360, "Finetune/Step Time": 4.902138950303197} +{"Finetune/Learning Rate": 1.328864737501302e-06, "Finetune/Loss": 0.8436716198921204, "Finetune/Loss (Raw)": 1.0621517896652222, "Finetune/Step": 4361, "Finetune/Step Time": 4.905413029715419} +{"Finetune/Learning Rate": 1.325763385382618e-06, "Finetune/Loss": 0.8414578437805176, "Finetune/Loss (Raw)": 0.8103700876235962, "Finetune/Step": 4362, "Finetune/Step Time": 4.908160153776407} +{"Finetune/Learning Rate": 1.3226653995658144e-06, "Finetune/Loss": 0.8410255312919617, "Finetune/Loss (Raw)": 0.8367758989334106, "Finetune/Step": 4363, "Finetune/Step Time": 4.904944209381938} +{"Finetune/Learning Rate": 1.319570781253159e-06, "Finetune/Loss": 0.8414478302001953, "Finetune/Loss (Raw)": 0.7790176272392273, "Finetune/Step": 4364, "Finetune/Step Time": 4.916900794953108} +{"Finetune/Learning Rate": 1.3164795316456091e-06, "Finetune/Loss": 0.8396601676940918, "Finetune/Loss (Raw)": 0.7715405821800232, "Finetune/Step": 4365, "Finetune/Step Time": 4.932099603116512} +{"Finetune/Learning Rate": 1.313391651942827e-06, "Finetune/Loss": 0.8393166065216064, "Finetune/Loss (Raw)": 0.9094554781913757, "Finetune/Step": 4366, "Finetune/Step Time": 4.931920317932963} +{"Finetune/Learning Rate": 1.3103071433431501e-06, "Finetune/Loss": 0.8383800983428955, "Finetune/Loss (Raw)": 0.7912997007369995, "Finetune/Step": 4367, "Finetune/Step Time": 4.937761323526502} +{"Finetune/Learning Rate": 1.307226007043617e-06, "Finetune/Loss": 0.8390652537345886, "Finetune/Loss (Raw)": 0.9192507863044739, "Finetune/Step": 4368, "Finetune/Step Time": 4.934338349848986} +{"Finetune/Learning Rate": 1.30414824423996e-06, "Finetune/Loss": 0.8388285636901855, "Finetune/Loss (Raw)": 0.8212357759475708, "Finetune/Step": 4369, "Finetune/Step Time": 4.936198655515909} +{"Finetune/Learning Rate": 1.301073856126598e-06, "Finetune/Loss": 0.8390209674835205, "Finetune/Loss (Raw)": 0.9614297747612, "Finetune/Step": 4370, "Finetune/Step Time": 4.932654805481434} +{"Finetune/Learning Rate": 1.2980028438966396e-06, "Finetune/Loss": 0.8398271799087524, "Finetune/Loss (Raw)": 0.9889347553253174, "Finetune/Step": 4371, "Finetune/Step Time": 4.910339469090104} +{"Finetune/Learning Rate": 1.2949352087418864e-06, "Finetune/Loss": 0.8348759412765503, "Finetune/Loss (Raw)": 0.24850083887577057, "Finetune/Step": 4372, "Finetune/Step Time": 4.9468190390616655} +{"Finetune/Learning Rate": 1.291870951852826e-06, "Finetune/Loss": 0.8339101076126099, "Finetune/Loss (Raw)": 0.7567104697227478, "Finetune/Step": 4373, "Finetune/Step Time": 4.938115395605564} +{"Finetune/Learning Rate": 1.288810074418637e-06, "Finetune/Loss": 0.8336871266365051, "Finetune/Loss (Raw)": 0.8276108503341675, "Finetune/Step": 4374, "Finetune/Step Time": 4.908761462196708} +{"Finetune/Learning Rate": 1.2857525776271852e-06, "Finetune/Loss": 0.8331565856933594, "Finetune/Loss (Raw)": 0.7979260087013245, "Finetune/Step": 4375, "Finetune/Step Time": 4.921308688819408} +{"Finetune/Learning Rate": 1.2826984626650297e-06, "Finetune/Loss": 0.8381341695785522, "Finetune/Loss (Raw)": 0.928688108921051, "Finetune/Step": 4376, "Finetune/Step Time": 4.900500947609544} +{"Finetune/Learning Rate": 1.2796477307174082e-06, "Finetune/Loss": 0.8376815319061279, "Finetune/Loss (Raw)": 0.8685796856880188, "Finetune/Step": 4377, "Finetune/Step Time": 4.901678031310439} +{"Finetune/Learning Rate": 1.2766003829682504e-06, "Finetune/Loss": 0.8388378620147705, "Finetune/Loss (Raw)": 0.8753646612167358, "Finetune/Step": 4378, "Finetune/Step Time": 4.911551335826516} +{"Finetune/Learning Rate": 1.273556420600175e-06, "Finetune/Loss": 0.8390147089958191, "Finetune/Loss (Raw)": 0.766765832901001, "Finetune/Step": 4379, "Finetune/Step Time": 4.927917780354619} +{"Finetune/Learning Rate": 1.2705158447944843e-06, "Finetune/Loss": 0.8384377956390381, "Finetune/Loss (Raw)": 0.9785265326499939, "Finetune/Step": 4380, "Finetune/Step Time": 4.9305623807013035} +{"Finetune/Learning Rate": 1.2674786567311648e-06, "Finetune/Loss": 0.840430498123169, "Finetune/Loss (Raw)": 0.9041332602500916, "Finetune/Step": 4381, "Finetune/Step Time": 4.917757468298078} +{"Finetune/Learning Rate": 1.2644448575888913e-06, "Finetune/Loss": 0.8424675464630127, "Finetune/Loss (Raw)": 0.9345661401748657, "Finetune/Step": 4382, "Finetune/Step Time": 4.911927593871951} +{"Finetune/Learning Rate": 1.2614144485450208e-06, "Finetune/Loss": 0.8421127200126648, "Finetune/Loss (Raw)": 0.7280827760696411, "Finetune/Step": 4383, "Finetune/Step Time": 4.926483055576682} +{"Finetune/Learning Rate": 1.2583874307755961e-06, "Finetune/Loss": 0.8417989015579224, "Finetune/Loss (Raw)": 0.8716951012611389, "Finetune/Step": 4384, "Finetune/Step Time": 4.907045604661107} +{"Finetune/Learning Rate": 1.2553638054553418e-06, "Finetune/Loss": 0.840609073638916, "Finetune/Loss (Raw)": 0.811927855014801, "Finetune/Step": 4385, "Finetune/Step Time": 4.900291116908193} +{"Finetune/Learning Rate": 1.252343573757674e-06, "Finetune/Loss": 0.8419064283370972, "Finetune/Loss (Raw)": 0.8489603996276855, "Finetune/Step": 4386, "Finetune/Step Time": 4.917865781113505} +{"Finetune/Learning Rate": 1.2493267368546802e-06, "Finetune/Loss": 0.8421884775161743, "Finetune/Loss (Raw)": 1.0438297986984253, "Finetune/Step": 4387, "Finetune/Step Time": 4.905825149267912} +{"Finetune/Learning Rate": 1.2463132959171343e-06, "Finetune/Loss": 0.8426333069801331, "Finetune/Loss (Raw)": 0.899060845375061, "Finetune/Step": 4388, "Finetune/Step Time": 4.910792130976915} +{"Finetune/Learning Rate": 1.2433032521145006e-06, "Finetune/Loss": 0.8425232172012329, "Finetune/Loss (Raw)": 0.9170433282852173, "Finetune/Step": 4389, "Finetune/Step Time": 4.909756813198328} +{"Finetune/Learning Rate": 1.2402966066149135e-06, "Finetune/Loss": 0.8424327969551086, "Finetune/Loss (Raw)": 0.8308445811271667, "Finetune/Step": 4390, "Finetune/Step Time": 4.914356146007776} +{"Finetune/Learning Rate": 1.2372933605851944e-06, "Finetune/Loss": 0.8409342765808105, "Finetune/Loss (Raw)": 0.8408296704292297, "Finetune/Step": 4391, "Finetune/Step Time": 4.922314755618572} +{"Finetune/Learning Rate": 1.2342935151908453e-06, "Finetune/Loss": 0.8375872373580933, "Finetune/Loss (Raw)": 0.3548985421657562, "Finetune/Step": 4392, "Finetune/Step Time": 4.950684390962124} +{"Finetune/Learning Rate": 1.231297071596046e-06, "Finetune/Loss": 0.8364107608795166, "Finetune/Loss (Raw)": 0.7457754015922546, "Finetune/Step": 4393, "Finetune/Step Time": 4.958456555381417} +{"Finetune/Learning Rate": 1.2283040309636596e-06, "Finetune/Loss": 0.8354179859161377, "Finetune/Loss (Raw)": 0.8212792873382568, "Finetune/Step": 4394, "Finetune/Step Time": 4.968564150854945} +{"Finetune/Learning Rate": 1.2253143944552258e-06, "Finetune/Loss": 0.8360520005226135, "Finetune/Loss (Raw)": 0.8523693084716797, "Finetune/Step": 4395, "Finetune/Step Time": 4.9586485568434} +{"Finetune/Learning Rate": 1.2223281632309636e-06, "Finetune/Loss": 0.8373895883560181, "Finetune/Loss (Raw)": 0.9697819948196411, "Finetune/Step": 4396, "Finetune/Step Time": 4.950426794588566} +{"Finetune/Learning Rate": 1.2193453384497722e-06, "Finetune/Loss": 0.8350323438644409, "Finetune/Loss (Raw)": 0.4204038977622986, "Finetune/Step": 4397, "Finetune/Step Time": 4.951651230454445} +{"Finetune/Learning Rate": 1.2163659212692246e-06, "Finetune/Loss": 0.8354356288909912, "Finetune/Loss (Raw)": 0.8906770348548889, "Finetune/Step": 4398, "Finetune/Step Time": 4.953875955194235} +{"Finetune/Learning Rate": 1.2133899128455796e-06, "Finetune/Loss": 0.8391806483268738, "Finetune/Loss (Raw)": 0.805706799030304, "Finetune/Step": 4399, "Finetune/Step Time": 4.9254971742630005} +{"Finetune/Learning Rate": 1.2104173143337672e-06, "Finetune/Loss": 0.8389821648597717, "Finetune/Loss (Raw)": 0.8871025443077087, "Finetune/Step": 4400, "Finetune/Step Time": 4.933126747608185} +{"Finetune/Learning Rate": 1.2074481268873894e-06, "Finetune/Loss": 0.8357909321784973, "Finetune/Loss (Raw)": 0.33310800790786743, "Finetune/Step": 4401, "Finetune/Step Time": 4.973721198737621} +{"Finetune/Learning Rate": 1.2044823516587345e-06, "Finetune/Loss": 0.8353161215782166, "Finetune/Loss (Raw)": 0.7974040508270264, "Finetune/Step": 4402, "Finetune/Step Time": 4.992052540183067} +{"Finetune/Learning Rate": 1.2015199897987616e-06, "Finetune/Loss": 0.8360808491706848, "Finetune/Loss (Raw)": 0.9219908714294434, "Finetune/Step": 4403, "Finetune/Step Time": 4.982362754642963} +{"Finetune/Learning Rate": 1.198561042457105e-06, "Finetune/Loss": 0.8367727398872375, "Finetune/Loss (Raw)": 1.0117837190628052, "Finetune/Step": 4404, "Finetune/Step Time": 4.9885823633521795} +{"Finetune/Learning Rate": 1.195605510782073e-06, "Finetune/Loss": 0.8366138935089111, "Finetune/Loss (Raw)": 0.8221396207809448, "Finetune/Step": 4405, "Finetune/Step Time": 4.978557083755732} +{"Finetune/Learning Rate": 1.192653395920652e-06, "Finetune/Loss": 0.8352271914482117, "Finetune/Loss (Raw)": 0.7485247254371643, "Finetune/Step": 4406, "Finetune/Step Time": 4.976033575832844} +{"Finetune/Learning Rate": 1.189704699018499e-06, "Finetune/Loss": 0.8350803852081299, "Finetune/Loss (Raw)": 0.7802659273147583, "Finetune/Step": 4407, "Finetune/Step Time": 4.974014526233077} +{"Finetune/Learning Rate": 1.186759421219943e-06, "Finetune/Loss": 0.8356407880783081, "Finetune/Loss (Raw)": 1.0153019428253174, "Finetune/Step": 4408, "Finetune/Step Time": 4.973310889676213} +{"Finetune/Learning Rate": 1.1838175636679938e-06, "Finetune/Loss": 0.8341872692108154, "Finetune/Loss (Raw)": 0.8114482164382935, "Finetune/Step": 4409, "Finetune/Step Time": 4.972731279209256} +{"Finetune/Learning Rate": 1.1808791275043297e-06, "Finetune/Loss": 0.8336502313613892, "Finetune/Loss (Raw)": 0.8829777836799622, "Finetune/Step": 4410, "Finetune/Step Time": 4.9694592989981174} +{"Finetune/Learning Rate": 1.1779441138692926e-06, "Finetune/Loss": 0.8334177136421204, "Finetune/Loss (Raw)": 0.8488608002662659, "Finetune/Step": 4411, "Finetune/Step Time": 4.962320638820529} +{"Finetune/Learning Rate": 1.1750125239019118e-06, "Finetune/Loss": 0.8325991630554199, "Finetune/Loss (Raw)": 0.8115949630737305, "Finetune/Step": 4412, "Finetune/Step Time": 4.964081287384033} +{"Finetune/Learning Rate": 1.1720843587398756e-06, "Finetune/Loss": 0.8358304500579834, "Finetune/Loss (Raw)": 1.0844494104385376, "Finetune/Step": 4413, "Finetune/Step Time": 4.926845813170075} +{"Finetune/Learning Rate": 1.1691596195195498e-06, "Finetune/Loss": 0.8350920677185059, "Finetune/Loss (Raw)": 0.8065025210380554, "Finetune/Step": 4414, "Finetune/Step Time": 4.925993751734495} +{"Finetune/Learning Rate": 1.1662383073759686e-06, "Finetune/Loss": 0.835310697555542, "Finetune/Loss (Raw)": 0.9375265836715698, "Finetune/Step": 4415, "Finetune/Step Time": 4.931125281378627} +{"Finetune/Learning Rate": 1.1633204234428352e-06, "Finetune/Loss": 0.8333998918533325, "Finetune/Loss (Raw)": 0.7581965327262878, "Finetune/Step": 4416, "Finetune/Step Time": 4.932078255340457} +{"Finetune/Learning Rate": 1.1604059688525238e-06, "Finetune/Loss": 0.8345701098442078, "Finetune/Loss (Raw)": 0.9617533087730408, "Finetune/Step": 4417, "Finetune/Step Time": 4.932442374527454} +{"Finetune/Learning Rate": 1.157494944736075e-06, "Finetune/Loss": 0.8335970640182495, "Finetune/Loss (Raw)": 0.8499599099159241, "Finetune/Step": 4418, "Finetune/Step Time": 4.932534297928214} +{"Finetune/Learning Rate": 1.1545873522232055e-06, "Finetune/Loss": 0.8336000442504883, "Finetune/Loss (Raw)": 0.9249474406242371, "Finetune/Step": 4419, "Finetune/Step Time": 4.932081211358309} +{"Finetune/Learning Rate": 1.1516831924422945e-06, "Finetune/Loss": 0.8323115110397339, "Finetune/Loss (Raw)": 0.7612658143043518, "Finetune/Step": 4420, "Finetune/Step Time": 4.944288022816181} +{"Finetune/Learning Rate": 1.1487824665203828e-06, "Finetune/Loss": 0.8317772746086121, "Finetune/Loss (Raw)": 0.8394776582717896, "Finetune/Step": 4421, "Finetune/Step Time": 4.947570336982608} +{"Finetune/Learning Rate": 1.1458851755831934e-06, "Finetune/Loss": 0.8319428563117981, "Finetune/Loss (Raw)": 0.8941825032234192, "Finetune/Step": 4422, "Finetune/Step Time": 4.929913651198149} +{"Finetune/Learning Rate": 1.1429913207551046e-06, "Finetune/Loss": 0.8357943296432495, "Finetune/Loss (Raw)": 0.792433500289917, "Finetune/Step": 4423, "Finetune/Step Time": 4.8996546026319265} +{"Finetune/Learning Rate": 1.1401009031591658e-06, "Finetune/Loss": 0.8306918144226074, "Finetune/Loss (Raw)": 0.330640584230423, "Finetune/Step": 4424, "Finetune/Step Time": 4.9387640953063965} +{"Finetune/Learning Rate": 1.137213923917092e-06, "Finetune/Loss": 0.8300905227661133, "Finetune/Loss (Raw)": 0.8194974064826965, "Finetune/Step": 4425, "Finetune/Step Time": 4.95543497055769} +{"Finetune/Learning Rate": 1.1343303841492626e-06, "Finetune/Loss": 0.8315999507904053, "Finetune/Loss (Raw)": 0.9072632789611816, "Finetune/Step": 4426, "Finetune/Step Time": 4.95157946459949} +{"Finetune/Learning Rate": 1.1314502849747244e-06, "Finetune/Loss": 0.8303830027580261, "Finetune/Loss (Raw)": 0.7780506014823914, "Finetune/Step": 4427, "Finetune/Step Time": 4.960546683520079} +{"Finetune/Learning Rate": 1.1285736275111836e-06, "Finetune/Loss": 0.8299906253814697, "Finetune/Loss (Raw)": 0.897212028503418, "Finetune/Step": 4428, "Finetune/Step Time": 4.965739665552974} +{"Finetune/Learning Rate": 1.1257004128750205e-06, "Finetune/Loss": 0.8339380621910095, "Finetune/Loss (Raw)": 0.8702009320259094, "Finetune/Step": 4429, "Finetune/Step Time": 4.931829167529941} +{"Finetune/Learning Rate": 1.122830642181273e-06, "Finetune/Loss": 0.8317375183105469, "Finetune/Loss (Raw)": 0.6858178973197937, "Finetune/Step": 4430, "Finetune/Step Time": 4.972465494647622} +{"Finetune/Learning Rate": 1.119964316543637e-06, "Finetune/Loss": 0.8307957649230957, "Finetune/Loss (Raw)": 0.8710795044898987, "Finetune/Step": 4431, "Finetune/Step Time": 4.979919549077749} +{"Finetune/Learning Rate": 1.1171014370744837e-06, "Finetune/Loss": 0.8350261449813843, "Finetune/Loss (Raw)": 0.8829753994941711, "Finetune/Step": 4432, "Finetune/Step Time": 4.951061157509685} +{"Finetune/Learning Rate": 1.114242004884839e-06, "Finetune/Loss": 0.8346264362335205, "Finetune/Loss (Raw)": 0.7221944332122803, "Finetune/Step": 4433, "Finetune/Step Time": 4.949464630335569} +{"Finetune/Learning Rate": 1.111386021084394e-06, "Finetune/Loss": 0.8356139659881592, "Finetune/Loss (Raw)": 0.9659944772720337, "Finetune/Step": 4434, "Finetune/Step Time": 4.944144360721111} +{"Finetune/Learning Rate": 1.1085334867814946e-06, "Finetune/Loss": 0.837143063545227, "Finetune/Loss (Raw)": 0.8750385642051697, "Finetune/Step": 4435, "Finetune/Step Time": 4.939312152564526} +{"Finetune/Learning Rate": 1.1056844030831603e-06, "Finetune/Loss": 0.8361557722091675, "Finetune/Loss (Raw)": 0.7615228891372681, "Finetune/Step": 4436, "Finetune/Step Time": 4.951332274824381} +{"Finetune/Learning Rate": 1.1028387710950616e-06, "Finetune/Loss": 0.8358789086341858, "Finetune/Loss (Raw)": 0.8237420320510864, "Finetune/Step": 4437, "Finetune/Step Time": 4.951379042118788} +{"Finetune/Learning Rate": 1.0999965919215327e-06, "Finetune/Loss": 0.8399314880371094, "Finetune/Loss (Raw)": 0.8702211976051331, "Finetune/Step": 4438, "Finetune/Step Time": 4.917282957583666} +{"Finetune/Learning Rate": 1.0971578666655713e-06, "Finetune/Loss": 0.8411694765090942, "Finetune/Loss (Raw)": 1.0288596153259277, "Finetune/Step": 4439, "Finetune/Step Time": 4.9143328461796045} +{"Finetune/Learning Rate": 1.0943225964288272e-06, "Finetune/Loss": 0.8413971066474915, "Finetune/Loss (Raw)": 1.0085586309432983, "Finetune/Step": 4440, "Finetune/Step Time": 4.91215855628252} +{"Finetune/Learning Rate": 1.091490782311614e-06, "Finetune/Loss": 0.8413513898849487, "Finetune/Loss (Raw)": 0.9756330251693726, "Finetune/Step": 4441, "Finetune/Step Time": 4.911357870325446} +{"Finetune/Learning Rate": 1.088662425412903e-06, "Finetune/Loss": 0.8414127230644226, "Finetune/Loss (Raw)": 0.9368652105331421, "Finetune/Step": 4442, "Finetune/Step Time": 4.920563373714685} +{"Finetune/Learning Rate": 1.0858375268303267e-06, "Finetune/Loss": 0.8413865566253662, "Finetune/Loss (Raw)": 0.8421543836593628, "Finetune/Step": 4443, "Finetune/Step Time": 4.921210767701268} +{"Finetune/Learning Rate": 1.0830160876601737e-06, "Finetune/Loss": 0.8413790464401245, "Finetune/Loss (Raw)": 0.8134618401527405, "Finetune/Step": 4444, "Finetune/Step Time": 4.916870500892401} +{"Finetune/Learning Rate": 1.0801981089973844e-06, "Finetune/Loss": 0.8403654098510742, "Finetune/Loss (Raw)": 0.7318525314331055, "Finetune/Step": 4445, "Finetune/Step Time": 4.925121458247304} +{"Finetune/Learning Rate": 1.0773835919355669e-06, "Finetune/Loss": 0.841797947883606, "Finetune/Loss (Raw)": 0.9712902903556824, "Finetune/Step": 4446, "Finetune/Step Time": 4.910989077761769} +{"Finetune/Learning Rate": 1.0745725375669768e-06, "Finetune/Loss": 0.8426105976104736, "Finetune/Loss (Raw)": 0.8407607674598694, "Finetune/Step": 4447, "Finetune/Step Time": 4.900932922959328} +{"Finetune/Learning Rate": 1.0717649469825298e-06, "Finetune/Loss": 0.8430166840553284, "Finetune/Loss (Raw)": 0.8749527335166931, "Finetune/Step": 4448, "Finetune/Step Time": 4.900040358304977} +{"Finetune/Learning Rate": 1.0689608212718005e-06, "Finetune/Loss": 0.8452224731445312, "Finetune/Loss (Raw)": 0.993205189704895, "Finetune/Step": 4449, "Finetune/Step Time": 4.88619769923389} +{"Finetune/Learning Rate": 1.0661601615230122e-06, "Finetune/Loss": 0.8433541059494019, "Finetune/Loss (Raw)": 0.7729931473731995, "Finetune/Step": 4450, "Finetune/Step Time": 4.880289016291499} +{"Finetune/Learning Rate": 1.0633629688230452e-06, "Finetune/Loss": 0.843824028968811, "Finetune/Loss (Raw)": 0.7633655071258545, "Finetune/Step": 4451, "Finetune/Step Time": 4.885414801537991} +{"Finetune/Learning Rate": 1.0605692442574356e-06, "Finetune/Loss": 0.840171217918396, "Finetune/Loss (Raw)": 0.3471379280090332, "Finetune/Step": 4452, "Finetune/Step Time": 4.919943016022444} +{"Finetune/Learning Rate": 1.0577789889103763e-06, "Finetune/Loss": 0.8408997654914856, "Finetune/Loss (Raw)": 0.7566266059875488, "Finetune/Step": 4453, "Finetune/Step Time": 4.916840147227049} +{"Finetune/Learning Rate": 1.054992203864712e-06, "Finetune/Loss": 0.841935396194458, "Finetune/Loss (Raw)": 0.8787314295768738, "Finetune/Step": 4454, "Finetune/Step Time": 4.91506139934063} +{"Finetune/Learning Rate": 1.0522088902019324e-06, "Finetune/Loss": 0.8421064019203186, "Finetune/Loss (Raw)": 0.7355643510818481, "Finetune/Step": 4455, "Finetune/Step Time": 4.906870508566499} +{"Finetune/Learning Rate": 1.049429049002193e-06, "Finetune/Loss": 0.8417367935180664, "Finetune/Loss (Raw)": 0.8137856721878052, "Finetune/Step": 4456, "Finetune/Step Time": 4.907752903178334} +{"Finetune/Learning Rate": 1.0466526813442957e-06, "Finetune/Loss": 0.841195285320282, "Finetune/Loss (Raw)": 0.7865182161331177, "Finetune/Step": 4457, "Finetune/Step Time": 4.9138668943196535} +{"Finetune/Learning Rate": 1.0438797883056927e-06, "Finetune/Loss": 0.8403319716453552, "Finetune/Loss (Raw)": 0.845120906829834, "Finetune/Step": 4458, "Finetune/Step Time": 4.927831515669823} +{"Finetune/Learning Rate": 1.0411103709624903e-06, "Finetune/Loss": 0.8394955992698669, "Finetune/Loss (Raw)": 0.843087375164032, "Finetune/Step": 4459, "Finetune/Step Time": 4.938502494245768} +{"Finetune/Learning Rate": 1.0383444303894453e-06, "Finetune/Loss": 0.8399907946586609, "Finetune/Loss (Raw)": 0.9576027989387512, "Finetune/Step": 4460, "Finetune/Step Time": 4.939161146059632} +{"Finetune/Learning Rate": 1.0355819676599655e-06, "Finetune/Loss": 0.8386715650558472, "Finetune/Loss (Raw)": 0.7931957244873047, "Finetune/Step": 4461, "Finetune/Step Time": 4.933369489386678} +{"Finetune/Learning Rate": 1.0328229838461069e-06, "Finetune/Loss": 0.8384273052215576, "Finetune/Loss (Raw)": 0.9092680215835571, "Finetune/Step": 4462, "Finetune/Step Time": 4.946082821115851} +{"Finetune/Learning Rate": 1.0300674800185795e-06, "Finetune/Loss": 0.8365213871002197, "Finetune/Loss (Raw)": 0.6660731434822083, "Finetune/Step": 4463, "Finetune/Step Time": 4.956054840236902} +{"Finetune/Learning Rate": 1.0273154572467436e-06, "Finetune/Loss": 0.8353844881057739, "Finetune/Loss (Raw)": 0.858291745185852, "Finetune/Step": 4464, "Finetune/Step Time": 4.959005298092961} +{"Finetune/Learning Rate": 1.0245669165985973e-06, "Finetune/Loss": 0.8364472985267639, "Finetune/Loss (Raw)": 0.7967243790626526, "Finetune/Step": 4465, "Finetune/Step Time": 4.941323922947049} +{"Finetune/Learning Rate": 1.0218218591408013e-06, "Finetune/Loss": 0.8361597061157227, "Finetune/Loss (Raw)": 0.8306066393852234, "Finetune/Step": 4466, "Finetune/Step Time": 4.935263665392995} +{"Finetune/Learning Rate": 1.0190802859386585e-06, "Finetune/Loss": 0.8370808959007263, "Finetune/Loss (Raw)": 0.9114472270011902, "Finetune/Step": 4467, "Finetune/Step Time": 4.929504781961441} +{"Finetune/Learning Rate": 1.0163421980561184e-06, "Finetune/Loss": 0.8369960784912109, "Finetune/Loss (Raw)": 0.8494950532913208, "Finetune/Step": 4468, "Finetune/Step Time": 4.921278575435281} +{"Finetune/Learning Rate": 1.0136075965557812e-06, "Finetune/Loss": 0.8360314965248108, "Finetune/Loss (Raw)": 0.8430669903755188, "Finetune/Step": 4469, "Finetune/Step Time": 4.921038536354899} +{"Finetune/Learning Rate": 1.0108764824988903e-06, "Finetune/Loss": 0.8360140323638916, "Finetune/Loss (Raw)": 0.8355834484100342, "Finetune/Step": 4470, "Finetune/Step Time": 4.876942399889231} +{"Finetune/Learning Rate": 1.008148856945339e-06, "Finetune/Loss": 0.8358656764030457, "Finetune/Loss (Raw)": 0.7836257815361023, "Finetune/Step": 4471, "Finetune/Step Time": 4.8902769684791565} +{"Finetune/Learning Rate": 1.0054247209536628e-06, "Finetune/Loss": 0.8356367349624634, "Finetune/Loss (Raw)": 0.8708224892616272, "Finetune/Step": 4472, "Finetune/Step Time": 4.880860805511475} +{"Finetune/Learning Rate": 1.0027040755810513e-06, "Finetune/Loss": 0.8370444178581238, "Finetune/Loss (Raw)": 1.0392805337905884, "Finetune/Step": 4473, "Finetune/Step Time": 4.878982800990343} +{"Finetune/Learning Rate": 9.999869218833313e-07, "Finetune/Loss": 0.8373056650161743, "Finetune/Loss (Raw)": 0.9038851261138916, "Finetune/Step": 4474, "Finetune/Step Time": 4.877416957169771} +{"Finetune/Learning Rate": 9.972732609149738e-07, "Finetune/Loss": 0.8358696103096008, "Finetune/Loss (Raw)": 0.7789298295974731, "Finetune/Step": 4475, "Finetune/Step Time": 4.898688208311796} +{"Finetune/Learning Rate": 9.945630937291006e-07, "Finetune/Loss": 0.8359359502792358, "Finetune/Loss (Raw)": 0.9272403120994568, "Finetune/Step": 4476, "Finetune/Step Time": 4.899481482803822} +{"Finetune/Learning Rate": 9.918564213774762e-07, "Finetune/Loss": 0.8344882726669312, "Finetune/Loss (Raw)": 0.7934092879295349, "Finetune/Step": 4477, "Finetune/Step Time": 4.900085974484682} +{"Finetune/Learning Rate": 9.891532449105045e-07, "Finetune/Loss": 0.8368115425109863, "Finetune/Loss (Raw)": 0.9409942030906677, "Finetune/Step": 4478, "Finetune/Step Time": 4.881825543940067} +{"Finetune/Learning Rate": 9.864535653772367e-07, "Finetune/Loss": 0.8366804122924805, "Finetune/Loss (Raw)": 0.8821874856948853, "Finetune/Step": 4479, "Finetune/Step Time": 4.891251936554909} +{"Finetune/Learning Rate": 9.837573838253655e-07, "Finetune/Loss": 0.836456298828125, "Finetune/Loss (Raw)": 0.8344138264656067, "Finetune/Step": 4480, "Finetune/Step Time": 4.895718418061733} +{"Finetune/Learning Rate": 9.810647013012265e-07, "Finetune/Loss": 0.8358808159828186, "Finetune/Loss (Raw)": 0.8404371738433838, "Finetune/Step": 4481, "Finetune/Step Time": 4.897490603849292} +{"Finetune/Learning Rate": 9.783755188497944e-07, "Finetune/Loss": 0.8363332748413086, "Finetune/Loss (Raw)": 0.8083428144454956, "Finetune/Step": 4482, "Finetune/Step Time": 4.911518439650536} +{"Finetune/Learning Rate": 9.75689837514696e-07, "Finetune/Loss": 0.8361240029335022, "Finetune/Loss (Raw)": 0.8592227101325989, "Finetune/Step": 4483, "Finetune/Step Time": 4.909248990938067} +{"Finetune/Learning Rate": 9.730076583381841e-07, "Finetune/Loss": 0.8362619280815125, "Finetune/Loss (Raw)": 0.9079915285110474, "Finetune/Step": 4484, "Finetune/Step Time": 4.924074079841375} +{"Finetune/Learning Rate": 9.703289823611606e-07, "Finetune/Loss": 0.8357440829277039, "Finetune/Loss (Raw)": 0.8520691394805908, "Finetune/Step": 4485, "Finetune/Step Time": 4.9243601355701685} +{"Finetune/Learning Rate": 9.676538106231725e-07, "Finetune/Loss": 0.8353832960128784, "Finetune/Loss (Raw)": 0.8603782653808594, "Finetune/Step": 4486, "Finetune/Step Time": 4.921684240922332} +{"Finetune/Learning Rate": 9.649821441623986e-07, "Finetune/Loss": 0.8309967517852783, "Finetune/Loss (Raw)": 0.324457585811615, "Finetune/Step": 4487, "Finetune/Step Time": 4.9563183560967445} +{"Finetune/Learning Rate": 9.623139840156604e-07, "Finetune/Loss": 0.832028865814209, "Finetune/Loss (Raw)": 0.9434758424758911, "Finetune/Step": 4488, "Finetune/Step Time": 4.95690437965095} +{"Finetune/Learning Rate": 9.596493312184197e-07, "Finetune/Loss": 0.8289554715156555, "Finetune/Loss (Raw)": 0.668755054473877, "Finetune/Step": 4489, "Finetune/Step Time": 4.946977224200964} +{"Finetune/Learning Rate": 9.569881868047747e-07, "Finetune/Loss": 0.8284211158752441, "Finetune/Loss (Raw)": 0.7419745922088623, "Finetune/Step": 4490, "Finetune/Step Time": 4.9513398464769125} +{"Finetune/Learning Rate": 9.543305518074652e-07, "Finetune/Loss": 0.8280275464057922, "Finetune/Loss (Raw)": 0.7863987684249878, "Finetune/Step": 4491, "Finetune/Step Time": 4.9560529086738825} +{"Finetune/Learning Rate": 9.516764272578638e-07, "Finetune/Loss": 0.8290247321128845, "Finetune/Loss (Raw)": 0.9066598415374756, "Finetune/Step": 4492, "Finetune/Step Time": 4.937866503372788} +{"Finetune/Learning Rate": 9.49025814185992e-07, "Finetune/Loss": 0.829375147819519, "Finetune/Loss (Raw)": 0.816392183303833, "Finetune/Step": 4493, "Finetune/Step Time": 4.933345274999738} +{"Finetune/Learning Rate": 9.463787136204938e-07, "Finetune/Loss": 0.8277044296264648, "Finetune/Loss (Raw)": 0.6956112384796143, "Finetune/Step": 4494, "Finetune/Step Time": 4.938322480767965} +{"Finetune/Learning Rate": 9.437351265886585e-07, "Finetune/Loss": 0.8285365700721741, "Finetune/Loss (Raw)": 0.8978031873703003, "Finetune/Step": 4495, "Finetune/Step Time": 4.937985397875309} +{"Finetune/Learning Rate": 9.410950541164143e-07, "Finetune/Loss": 0.8273829221725464, "Finetune/Loss (Raw)": 0.7715871334075928, "Finetune/Step": 4496, "Finetune/Step Time": 4.9453483838588} +{"Finetune/Learning Rate": 9.384584972283184e-07, "Finetune/Loss": 0.8283321261405945, "Finetune/Loss (Raw)": 0.9427331686019897, "Finetune/Step": 4497, "Finetune/Step Time": 4.948184357956052} +{"Finetune/Learning Rate": 9.358254569475689e-07, "Finetune/Loss": 0.8267920613288879, "Finetune/Loss (Raw)": 0.7642977237701416, "Finetune/Step": 4498, "Finetune/Step Time": 4.9529410395771265} +{"Finetune/Learning Rate": 9.331959342959951e-07, "Finetune/Loss": 0.8264572024345398, "Finetune/Loss (Raw)": 0.9460776448249817, "Finetune/Step": 4499, "Finetune/Step Time": 4.952993435785174} +{"Finetune/Learning Rate": 9.305699302940662e-07, "Finetune/Loss": 0.831165075302124, "Finetune/Loss (Raw)": 0.8511123657226562, "Finetune/Step": 4500, "Finetune/Step Time": 4.92455449514091} +{"Finetune/Learning Rate": 9.279474459608806e-07, "Finetune/Loss": 0.832629919052124, "Finetune/Loss (Raw)": 0.9442014694213867, "Finetune/Step": 4501, "Finetune/Step Time": 4.923399914056063} +{"Finetune/Learning Rate": 9.253284823141728e-07, "Finetune/Loss": 0.8318125009536743, "Finetune/Loss (Raw)": 0.7229873538017273, "Finetune/Step": 4502, "Finetune/Step Time": 4.922473195940256} +{"Finetune/Learning Rate": 9.227130403703178e-07, "Finetune/Loss": 0.8329746127128601, "Finetune/Loss (Raw)": 0.9466713666915894, "Finetune/Step": 4503, "Finetune/Step Time": 4.907909782603383} +{"Finetune/Learning Rate": 9.201011211443101e-07, "Finetune/Loss": 0.8323218822479248, "Finetune/Loss (Raw)": 0.8451456427574158, "Finetune/Step": 4504, "Finetune/Step Time": 4.897478647530079} +{"Finetune/Learning Rate": 9.174927256497846e-07, "Finetune/Loss": 0.8319477438926697, "Finetune/Loss (Raw)": 0.8206817507743835, "Finetune/Step": 4505, "Finetune/Step Time": 4.903063595294952} +{"Finetune/Learning Rate": 9.148878548990137e-07, "Finetune/Loss": 0.8329671621322632, "Finetune/Loss (Raw)": 1.005857229232788, "Finetune/Step": 4506, "Finetune/Step Time": 4.882008921355009} +{"Finetune/Learning Rate": 9.122865099028944e-07, "Finetune/Loss": 0.8295378088951111, "Finetune/Loss (Raw)": 0.3278028070926666, "Finetune/Step": 4507, "Finetune/Step Time": 4.898363104090095} +{"Finetune/Learning Rate": 9.09688691670958e-07, "Finetune/Loss": 0.8286947011947632, "Finetune/Loss (Raw)": 0.870613157749176, "Finetune/Step": 4508, "Finetune/Step Time": 4.891600808128715} +{"Finetune/Learning Rate": 9.070944012113669e-07, "Finetune/Loss": 0.8277685046195984, "Finetune/Loss (Raw)": 0.7855757474899292, "Finetune/Step": 4509, "Finetune/Step Time": 4.888813277706504} +{"Finetune/Learning Rate": 9.045036395309148e-07, "Finetune/Loss": 0.8279444575309753, "Finetune/Loss (Raw)": 0.957095742225647, "Finetune/Step": 4510, "Finetune/Step Time": 4.8847019243985415} +{"Finetune/Learning Rate": 9.019164076350262e-07, "Finetune/Loss": 0.8301712274551392, "Finetune/Loss (Raw)": 1.0131027698516846, "Finetune/Step": 4511, "Finetune/Step Time": 4.882371570914984} +{"Finetune/Learning Rate": 8.993327065277524e-07, "Finetune/Loss": 0.8261805772781372, "Finetune/Loss (Raw)": 0.360887348651886, "Finetune/Step": 4512, "Finetune/Step Time": 4.912855198606849} +{"Finetune/Learning Rate": 8.967525372117825e-07, "Finetune/Loss": 0.8263640403747559, "Finetune/Loss (Raw)": 0.8354214429855347, "Finetune/Step": 4513, "Finetune/Step Time": 4.917562484741211} +{"Finetune/Learning Rate": 8.941759006884265e-07, "Finetune/Loss": 0.8256107568740845, "Finetune/Loss (Raw)": 0.7525333762168884, "Finetune/Step": 4514, "Finetune/Step Time": 4.899107916280627} +{"Finetune/Learning Rate": 8.916027979576247e-07, "Finetune/Loss": 0.8240141868591309, "Finetune/Loss (Raw)": 0.8394715189933777, "Finetune/Step": 4515, "Finetune/Step Time": 4.902183450758457} +{"Finetune/Learning Rate": 8.890332300179516e-07, "Finetune/Loss": 0.8229890465736389, "Finetune/Loss (Raw)": 0.7678428888320923, "Finetune/Step": 4516, "Finetune/Step Time": 4.896409312263131} +{"Finetune/Learning Rate": 8.864671978666084e-07, "Finetune/Loss": 0.8216276168823242, "Finetune/Loss (Raw)": 0.7427759170532227, "Finetune/Step": 4517, "Finetune/Step Time": 4.898001683875918} +{"Finetune/Learning Rate": 8.839047024994152e-07, "Finetune/Loss": 0.8229838609695435, "Finetune/Loss (Raw)": 1.004450798034668, "Finetune/Step": 4518, "Finetune/Step Time": 4.888120487332344} +{"Finetune/Learning Rate": 8.813457449108309e-07, "Finetune/Loss": 0.8237873315811157, "Finetune/Loss (Raw)": 0.9436657428741455, "Finetune/Step": 4519, "Finetune/Step Time": 4.8813145607709885} +{"Finetune/Learning Rate": 8.787903260939368e-07, "Finetune/Loss": 0.8283075094223022, "Finetune/Loss (Raw)": 0.9334874153137207, "Finetune/Step": 4520, "Finetune/Step Time": 4.843253554776311} +{"Finetune/Learning Rate": 8.762384470404417e-07, "Finetune/Loss": 0.8302220106124878, "Finetune/Loss (Raw)": 0.9908260107040405, "Finetune/Step": 4521, "Finetune/Step Time": 4.830862732604146} +{"Finetune/Learning Rate": 8.736901087406779e-07, "Finetune/Loss": 0.8303424119949341, "Finetune/Loss (Raw)": 0.8366958498954773, "Finetune/Step": 4522, "Finetune/Step Time": 4.8330447897315025} +{"Finetune/Learning Rate": 8.711453121836066e-07, "Finetune/Loss": 0.8304620981216431, "Finetune/Loss (Raw)": 0.8676841259002686, "Finetune/Step": 4523, "Finetune/Step Time": 4.842752480879426} +{"Finetune/Learning Rate": 8.686040583568134e-07, "Finetune/Loss": 0.8295478820800781, "Finetune/Loss (Raw)": 0.8527681231498718, "Finetune/Step": 4524, "Finetune/Step Time": 4.836536040529609} +{"Finetune/Learning Rate": 8.660663482465093e-07, "Finetune/Loss": 0.8327613472938538, "Finetune/Loss (Raw)": 0.8317269086837769, "Finetune/Step": 4525, "Finetune/Step Time": 4.803611570969224} +{"Finetune/Learning Rate": 8.63532182837531e-07, "Finetune/Loss": 0.8310154676437378, "Finetune/Loss (Raw)": 0.6672107577323914, "Finetune/Step": 4526, "Finetune/Step Time": 4.79958906210959} +{"Finetune/Learning Rate": 8.610015631133395e-07, "Finetune/Loss": 0.8289350271224976, "Finetune/Loss (Raw)": 0.5394033789634705, "Finetune/Step": 4527, "Finetune/Step Time": 4.81313450820744} +{"Finetune/Learning Rate": 8.584744900560149e-07, "Finetune/Loss": 0.8285160064697266, "Finetune/Loss (Raw)": 0.8334690928459167, "Finetune/Step": 4528, "Finetune/Step Time": 4.8035115990787745} +{"Finetune/Learning Rate": 8.559509646462693e-07, "Finetune/Loss": 0.8330354690551758, "Finetune/Loss (Raw)": 0.911591112613678, "Finetune/Step": 4529, "Finetune/Step Time": 4.767596837133169} +{"Finetune/Learning Rate": 8.534309878634317e-07, "Finetune/Loss": 0.8343151211738586, "Finetune/Loss (Raw)": 0.9612070322036743, "Finetune/Step": 4530, "Finetune/Step Time": 4.750949101522565} +{"Finetune/Learning Rate": 8.509145606854552e-07, "Finetune/Loss": 0.8348665237426758, "Finetune/Loss (Raw)": 0.992572546005249, "Finetune/Step": 4531, "Finetune/Step Time": 4.752399632707238} +{"Finetune/Learning Rate": 8.484016840889176e-07, "Finetune/Loss": 0.8320995569229126, "Finetune/Loss (Raw)": 0.6576069593429565, "Finetune/Step": 4532, "Finetune/Step Time": 4.757104981690645} +{"Finetune/Learning Rate": 8.458923590490165e-07, "Finetune/Loss": 0.8325862288475037, "Finetune/Loss (Raw)": 0.8844375014305115, "Finetune/Step": 4533, "Finetune/Step Time": 4.7545464374125} +{"Finetune/Learning Rate": 8.433865865395718e-07, "Finetune/Loss": 0.8293721675872803, "Finetune/Loss (Raw)": 0.3371242880821228, "Finetune/Step": 4534, "Finetune/Step Time": 4.791149104014039} +{"Finetune/Learning Rate": 8.408843675330225e-07, "Finetune/Loss": 0.8290409445762634, "Finetune/Loss (Raw)": 0.7378706336021423, "Finetune/Step": 4535, "Finetune/Step Time": 4.799568396061659} +{"Finetune/Learning Rate": 8.383857030004361e-07, "Finetune/Loss": 0.8278621435165405, "Finetune/Loss (Raw)": 0.8644106984138489, "Finetune/Step": 4536, "Finetune/Step Time": 4.81459604203701} +{"Finetune/Learning Rate": 8.358905939114948e-07, "Finetune/Loss": 0.8283332586288452, "Finetune/Loss (Raw)": 0.8717551827430725, "Finetune/Step": 4537, "Finetune/Step Time": 4.811969552189112} +{"Finetune/Learning Rate": 8.333990412344961e-07, "Finetune/Loss": 0.82843416929245, "Finetune/Loss (Raw)": 0.8958868384361267, "Finetune/Step": 4538, "Finetune/Step Time": 4.811370091512799} +{"Finetune/Learning Rate": 8.30911045936369e-07, "Finetune/Loss": 0.8278247117996216, "Finetune/Loss (Raw)": 0.7708598375320435, "Finetune/Step": 4539, "Finetune/Step Time": 4.818961137905717} +{"Finetune/Learning Rate": 8.284266089826531e-07, "Finetune/Loss": 0.8238998651504517, "Finetune/Loss (Raw)": 0.30921417474746704, "Finetune/Step": 4540, "Finetune/Step Time": 4.855416227132082} +{"Finetune/Learning Rate": 8.259457313375097e-07, "Finetune/Loss": 0.8214068412780762, "Finetune/Loss (Raw)": 0.7653375864028931, "Finetune/Step": 4541, "Finetune/Step Time": 4.87784374691546} +{"Finetune/Learning Rate": 8.234684139637205e-07, "Finetune/Loss": 0.82093346118927, "Finetune/Loss (Raw)": 0.7459091544151306, "Finetune/Step": 4542, "Finetune/Step Time": 4.879192376509309} +{"Finetune/Learning Rate": 8.209946578226836e-07, "Finetune/Loss": 0.8201844692230225, "Finetune/Loss (Raw)": 0.8416603207588196, "Finetune/Step": 4543, "Finetune/Step Time": 4.8732643984258175} +{"Finetune/Learning Rate": 8.185244638744139e-07, "Finetune/Loss": 0.8213924169540405, "Finetune/Loss (Raw)": 0.9128074049949646, "Finetune/Step": 4544, "Finetune/Step Time": 4.864344799891114} +{"Finetune/Learning Rate": 8.160578330775459e-07, "Finetune/Loss": 0.8210583925247192, "Finetune/Loss (Raw)": 0.9189952611923218, "Finetune/Step": 4545, "Finetune/Step Time": 4.876744316890836} +{"Finetune/Learning Rate": 8.135947663893329e-07, "Finetune/Loss": 0.8218011856079102, "Finetune/Loss (Raw)": 0.9450507760047913, "Finetune/Step": 4546, "Finetune/Step Time": 4.875835977494717} +{"Finetune/Learning Rate": 8.111352647656423e-07, "Finetune/Loss": 0.8217949867248535, "Finetune/Loss (Raw)": 0.9241540431976318, "Finetune/Step": 4547, "Finetune/Step Time": 4.869394965469837} +{"Finetune/Learning Rate": 8.086793291609562e-07, "Finetune/Loss": 0.8203223347663879, "Finetune/Loss (Raw)": 0.5727581977844238, "Finetune/Step": 4548, "Finetune/Step Time": 4.926031969487667} +{"Finetune/Learning Rate": 8.062269605283779e-07, "Finetune/Loss": 0.8193897604942322, "Finetune/Loss (Raw)": 0.7201112508773804, "Finetune/Step": 4549, "Finetune/Step Time": 4.921058870851994} +{"Finetune/Learning Rate": 8.037781598196225e-07, "Finetune/Loss": 0.8200795650482178, "Finetune/Loss (Raw)": 0.9824821352958679, "Finetune/Step": 4550, "Finetune/Step Time": 4.918254712596536} +{"Finetune/Learning Rate": 8.013329279850235e-07, "Finetune/Loss": 0.8209955096244812, "Finetune/Loss (Raw)": 0.9096724987030029, "Finetune/Step": 4551, "Finetune/Step Time": 4.915394723415375} +{"Finetune/Learning Rate": 7.988912659735249e-07, "Finetune/Loss": 0.8237854242324829, "Finetune/Loss (Raw)": 0.6877493858337402, "Finetune/Step": 4552, "Finetune/Step Time": 4.891426058486104} +{"Finetune/Learning Rate": 7.964531747326898e-07, "Finetune/Loss": 0.8246015906333923, "Finetune/Loss (Raw)": 0.9239628314971924, "Finetune/Step": 4553, "Finetune/Step Time": 4.871518988162279} +{"Finetune/Learning Rate": 7.940186552086937e-07, "Finetune/Loss": 0.825602650642395, "Finetune/Loss (Raw)": 1.0354009866714478, "Finetune/Step": 4554, "Finetune/Step Time": 4.870188431814313} +{"Finetune/Learning Rate": 7.915877083463242e-07, "Finetune/Loss": 0.8263324499130249, "Finetune/Loss (Raw)": 0.8714577555656433, "Finetune/Step": 4555, "Finetune/Step Time": 4.871542872861028} +{"Finetune/Learning Rate": 7.891603350889888e-07, "Finetune/Loss": 0.8276276588439941, "Finetune/Loss (Raw)": 1.0630062818527222, "Finetune/Step": 4556, "Finetune/Step Time": 4.867996968328953} +{"Finetune/Learning Rate": 7.867365363787005e-07, "Finetune/Loss": 0.8268997669219971, "Finetune/Loss (Raw)": 0.7770326733589172, "Finetune/Step": 4557, "Finetune/Step Time": 4.870627801865339} +{"Finetune/Learning Rate": 7.843163131560871e-07, "Finetune/Loss": 0.826691746711731, "Finetune/Loss (Raw)": 0.6591843366622925, "Finetune/Step": 4558, "Finetune/Step Time": 4.843128489330411} +{"Finetune/Learning Rate": 7.818996663603917e-07, "Finetune/Loss": 0.8274619579315186, "Finetune/Loss (Raw)": 0.9696745276451111, "Finetune/Step": 4559, "Finetune/Step Time": 4.843740655109286} +{"Finetune/Learning Rate": 7.794865969294685e-07, "Finetune/Loss": 0.8270924091339111, "Finetune/Loss (Raw)": 0.8356670141220093, "Finetune/Step": 4560, "Finetune/Step Time": 4.844990214332938} +{"Finetune/Learning Rate": 7.770771057997828e-07, "Finetune/Loss": 0.8284747004508972, "Finetune/Loss (Raw)": 0.899128794670105, "Finetune/Step": 4561, "Finetune/Step Time": 4.8339171558618546} +{"Finetune/Learning Rate": 7.746711939064066e-07, "Finetune/Loss": 0.8282955288887024, "Finetune/Loss (Raw)": 0.9430618286132812, "Finetune/Step": 4562, "Finetune/Step Time": 4.834115212783217} +{"Finetune/Learning Rate": 7.722688621830333e-07, "Finetune/Loss": 0.8238438367843628, "Finetune/Loss (Raw)": 0.30522316694259644, "Finetune/Step": 4563, "Finetune/Step Time": 4.874708343297243} +{"Finetune/Learning Rate": 7.698701115619578e-07, "Finetune/Loss": 0.8245924711227417, "Finetune/Loss (Raw)": 0.8573448061943054, "Finetune/Step": 4564, "Finetune/Step Time": 4.857913579791784} +{"Finetune/Learning Rate": 7.674749429740868e-07, "Finetune/Loss": 0.8243417739868164, "Finetune/Loss (Raw)": 0.791656494140625, "Finetune/Step": 4565, "Finetune/Step Time": 4.852681934833527} +{"Finetune/Learning Rate": 7.650833573489436e-07, "Finetune/Loss": 0.8240714073181152, "Finetune/Loss (Raw)": 0.8356127738952637, "Finetune/Step": 4566, "Finetune/Step Time": 4.855595236644149} +{"Finetune/Learning Rate": 7.626953556146521e-07, "Finetune/Loss": 0.8235836029052734, "Finetune/Loss (Raw)": 0.9664145708084106, "Finetune/Step": 4567, "Finetune/Step Time": 4.853130295872688} +{"Finetune/Learning Rate": 7.603109386979501e-07, "Finetune/Loss": 0.8229192495346069, "Finetune/Loss (Raw)": 0.9235265254974365, "Finetune/Step": 4568, "Finetune/Step Time": 4.850859969854355} +{"Finetune/Learning Rate": 7.579301075241808e-07, "Finetune/Loss": 0.8231931924819946, "Finetune/Loss (Raw)": 1.0106903314590454, "Finetune/Step": 4569, "Finetune/Step Time": 4.843162775039673} +{"Finetune/Learning Rate": 7.555528630173014e-07, "Finetune/Loss": 0.823159396648407, "Finetune/Loss (Raw)": 0.9325470328330994, "Finetune/Step": 4570, "Finetune/Step Time": 4.829495847225189} +{"Finetune/Learning Rate": 7.531792060998766e-07, "Finetune/Loss": 0.8231379389762878, "Finetune/Loss (Raw)": 0.8394045233726501, "Finetune/Step": 4571, "Finetune/Step Time": 4.852577658370137} +{"Finetune/Learning Rate": 7.508091376930682e-07, "Finetune/Loss": 0.8234150409698486, "Finetune/Loss (Raw)": 0.8489276170730591, "Finetune/Step": 4572, "Finetune/Step Time": 4.857720151543617} +{"Finetune/Learning Rate": 7.484426587166615e-07, "Finetune/Loss": 0.823859453201294, "Finetune/Loss (Raw)": 0.7887431383132935, "Finetune/Step": 4573, "Finetune/Step Time": 4.867759380489588} +{"Finetune/Learning Rate": 7.460797700890365e-07, "Finetune/Loss": 0.8232709765434265, "Finetune/Loss (Raw)": 0.8959658741950989, "Finetune/Step": 4574, "Finetune/Step Time": 4.867069909349084} +{"Finetune/Learning Rate": 7.437204727271852e-07, "Finetune/Loss": 0.823019802570343, "Finetune/Loss (Raw)": 0.8086086511611938, "Finetune/Step": 4575, "Finetune/Step Time": 4.881047988310456} +{"Finetune/Learning Rate": 7.41364767546705e-07, "Finetune/Loss": 0.8223651647567749, "Finetune/Loss (Raw)": 0.7911542654037476, "Finetune/Step": 4576, "Finetune/Step Time": 4.8882766757160425} +{"Finetune/Learning Rate": 7.390126554617982e-07, "Finetune/Loss": 0.8223702907562256, "Finetune/Loss (Raw)": 0.9938704967498779, "Finetune/Step": 4577, "Finetune/Step Time": 4.891123114153743} +{"Finetune/Learning Rate": 7.366641373852745e-07, "Finetune/Loss": 0.8237773180007935, "Finetune/Loss (Raw)": 0.9530841112136841, "Finetune/Step": 4578, "Finetune/Step Time": 4.894827920943499} +{"Finetune/Learning Rate": 7.343192142285471e-07, "Finetune/Loss": 0.8210586309432983, "Finetune/Loss (Raw)": 0.4153718948364258, "Finetune/Step": 4579, "Finetune/Step Time": 4.91862652823329} +{"Finetune/Learning Rate": 7.319778869016358e-07, "Finetune/Loss": 0.8204328417778015, "Finetune/Loss (Raw)": 0.26704171299934387, "Finetune/Step": 4580, "Finetune/Step Time": 4.918883599340916} +{"Finetune/Learning Rate": 7.296401563131661e-07, "Finetune/Loss": 0.8210710287094116, "Finetune/Loss (Raw)": 0.8383147120475769, "Finetune/Step": 4581, "Finetune/Step Time": 4.927834695205092} +{"Finetune/Learning Rate": 7.273060233703599e-07, "Finetune/Loss": 0.821224570274353, "Finetune/Loss (Raw)": 0.8983814120292664, "Finetune/Step": 4582, "Finetune/Step Time": 4.930951364338398} +{"Finetune/Learning Rate": 7.249754889790539e-07, "Finetune/Loss": 0.8230091333389282, "Finetune/Loss (Raw)": 0.963984489440918, "Finetune/Step": 4583, "Finetune/Step Time": 4.917234232649207} +{"Finetune/Learning Rate": 7.2264855404368e-07, "Finetune/Loss": 0.8227455615997314, "Finetune/Loss (Raw)": 0.7800514101982117, "Finetune/Step": 4584, "Finetune/Step Time": 4.9183703772723675} +{"Finetune/Learning Rate": 7.203252194672783e-07, "Finetune/Loss": 0.8224003314971924, "Finetune/Loss (Raw)": 0.7423315048217773, "Finetune/Step": 4585, "Finetune/Step Time": 4.906293749809265} +{"Finetune/Learning Rate": 7.180054861514885e-07, "Finetune/Loss": 0.8216159343719482, "Finetune/Loss (Raw)": 0.7447229623794556, "Finetune/Step": 4586, "Finetune/Step Time": 4.897420698776841} +{"Finetune/Learning Rate": 7.156893549965538e-07, "Finetune/Loss": 0.8221263885498047, "Finetune/Loss (Raw)": 0.9084181785583496, "Finetune/Step": 4587, "Finetune/Step Time": 4.893497506156564} +{"Finetune/Learning Rate": 7.133768269013186e-07, "Finetune/Loss": 0.8227747678756714, "Finetune/Loss (Raw)": 1.0405986309051514, "Finetune/Step": 4588, "Finetune/Step Time": 4.917059194296598} +{"Finetune/Learning Rate": 7.110679027632295e-07, "Finetune/Loss": 0.8231417536735535, "Finetune/Loss (Raw)": 0.8401752710342407, "Finetune/Step": 4589, "Finetune/Step Time": 4.922289567068219} +{"Finetune/Learning Rate": 7.087625834783385e-07, "Finetune/Loss": 0.8223599195480347, "Finetune/Loss (Raw)": 0.8091979026794434, "Finetune/Step": 4590, "Finetune/Step Time": 4.911890681833029} +{"Finetune/Learning Rate": 7.064608699412944e-07, "Finetune/Loss": 0.8242481350898743, "Finetune/Loss (Raw)": 0.907758355140686, "Finetune/Step": 4591, "Finetune/Step Time": 4.895944004878402} +{"Finetune/Learning Rate": 7.041627630453418e-07, "Finetune/Loss": 0.8198074698448181, "Finetune/Loss (Raw)": 0.289882093667984, "Finetune/Step": 4592, "Finetune/Step Time": 4.912823719903827} +{"Finetune/Learning Rate": 7.018682636823371e-07, "Finetune/Loss": 0.8207172155380249, "Finetune/Loss (Raw)": 0.9131677746772766, "Finetune/Step": 4593, "Finetune/Step Time": 4.91234527528286} +{"Finetune/Learning Rate": 6.99577372742728e-07, "Finetune/Loss": 0.8174397349357605, "Finetune/Loss (Raw)": 0.411097913980484, "Finetune/Step": 4594, "Finetune/Step Time": 4.947274720296264} +{"Finetune/Learning Rate": 6.972900911155656e-07, "Finetune/Loss": 0.8175156116485596, "Finetune/Loss (Raw)": 0.9211551547050476, "Finetune/Step": 4595, "Finetune/Step Time": 4.949599610641599} +{"Finetune/Learning Rate": 6.950064196884987e-07, "Finetune/Loss": 0.8182359933853149, "Finetune/Loss (Raw)": 0.9417026042938232, "Finetune/Step": 4596, "Finetune/Step Time": 4.951488872990012} +{"Finetune/Learning Rate": 6.927263593477751e-07, "Finetune/Loss": 0.8175621032714844, "Finetune/Loss (Raw)": 0.756811797618866, "Finetune/Step": 4597, "Finetune/Step Time": 4.953233161941171} +{"Finetune/Learning Rate": 6.904499109782426e-07, "Finetune/Loss": 0.8177986145019531, "Finetune/Loss (Raw)": 0.8658587336540222, "Finetune/Step": 4598, "Finetune/Step Time": 4.950728103518486} +{"Finetune/Learning Rate": 6.881770754633444e-07, "Finetune/Loss": 0.8179193735122681, "Finetune/Loss (Raw)": 0.7990819215774536, "Finetune/Step": 4599, "Finetune/Step Time": 4.944540113210678} +{"Finetune/Learning Rate": 6.859078536851283e-07, "Finetune/Loss": 0.817588210105896, "Finetune/Loss (Raw)": 0.8284299373626709, "Finetune/Step": 4600, "Finetune/Step Time": 4.948508812114596} +{"Finetune/Learning Rate": 6.836422465242298e-07, "Finetune/Loss": 0.8151212334632874, "Finetune/Loss (Raw)": 0.7235108017921448, "Finetune/Step": 4601, "Finetune/Step Time": 4.959107864648104} +{"Finetune/Learning Rate": 6.81380254859888e-07, "Finetune/Loss": 0.8160771131515503, "Finetune/Loss (Raw)": 1.0262439250946045, "Finetune/Step": 4602, "Finetune/Step Time": 4.960886266082525} +{"Finetune/Learning Rate": 6.791218795699406e-07, "Finetune/Loss": 0.8170995712280273, "Finetune/Loss (Raw)": 0.9097911715507507, "Finetune/Step": 4603, "Finetune/Step Time": 4.953139340505004} +{"Finetune/Learning Rate": 6.768671215308165e-07, "Finetune/Loss": 0.8164476156234741, "Finetune/Loss (Raw)": 0.8438006043434143, "Finetune/Step": 4604, "Finetune/Step Time": 4.953052105382085} +{"Finetune/Learning Rate": 6.746159816175435e-07, "Finetune/Loss": 0.8128361701965332, "Finetune/Loss (Raw)": 0.33113670349121094, "Finetune/Step": 4605, "Finetune/Step Time": 4.9873936623334885} +{"Finetune/Learning Rate": 6.723684607037451e-07, "Finetune/Loss": 0.8124051690101624, "Finetune/Loss (Raw)": 0.885834813117981, "Finetune/Step": 4606, "Finetune/Step Time": 4.99519576318562} +{"Finetune/Learning Rate": 6.701245596616402e-07, "Finetune/Loss": 0.8118879199028015, "Finetune/Loss (Raw)": 0.8159814476966858, "Finetune/Step": 4607, "Finetune/Step Time": 4.986922299489379} +{"Finetune/Learning Rate": 6.678842793620433e-07, "Finetune/Loss": 0.8120473623275757, "Finetune/Loss (Raw)": 0.8548216819763184, "Finetune/Step": 4608, "Finetune/Step Time": 4.982886301353574} +{"Finetune/Learning Rate": 6.656476206743623e-07, "Finetune/Loss": 0.8136342167854309, "Finetune/Loss (Raw)": 1.0435447692871094, "Finetune/Step": 4609, "Finetune/Step Time": 4.981246495619416} +{"Finetune/Learning Rate": 6.634145844666051e-07, "Finetune/Loss": 0.8137211203575134, "Finetune/Loss (Raw)": 0.819475531578064, "Finetune/Step": 4610, "Finetune/Step Time": 4.963639222085476} +{"Finetune/Learning Rate": 6.611851716053641e-07, "Finetune/Loss": 0.8092396259307861, "Finetune/Loss (Raw)": 0.28558075428009033, "Finetune/Step": 4611, "Finetune/Step Time": 4.998444451019168} +{"Finetune/Learning Rate": 6.589593829558316e-07, "Finetune/Loss": 0.8090396523475647, "Finetune/Loss (Raw)": 0.8824000954627991, "Finetune/Step": 4612, "Finetune/Step Time": 4.979856912046671} +{"Finetune/Learning Rate": 6.567372193817967e-07, "Finetune/Loss": 0.8087856769561768, "Finetune/Loss (Raw)": 0.8195569515228271, "Finetune/Step": 4613, "Finetune/Step Time": 5.0045038387179375} +{"Finetune/Learning Rate": 6.545186817456339e-07, "Finetune/Loss": 0.8080184459686279, "Finetune/Loss (Raw)": 0.7621760368347168, "Finetune/Step": 4614, "Finetune/Step Time": 5.011000180616975} +{"Finetune/Learning Rate": 6.523037709083158e-07, "Finetune/Loss": 0.8129394054412842, "Finetune/Loss (Raw)": 0.9543479681015015, "Finetune/Step": 4615, "Finetune/Step Time": 4.9759578593075275} +{"Finetune/Learning Rate": 6.500924877294057e-07, "Finetune/Loss": 0.8120081424713135, "Finetune/Loss (Raw)": 0.8242660164833069, "Finetune/Step": 4616, "Finetune/Step Time": 4.967791737988591} +{"Finetune/Learning Rate": 6.478848330670607e-07, "Finetune/Loss": 0.8128131628036499, "Finetune/Loss (Raw)": 0.7717978954315186, "Finetune/Step": 4617, "Finetune/Step Time": 4.959619035944343} +{"Finetune/Learning Rate": 6.456808077780263e-07, "Finetune/Loss": 0.8138744831085205, "Finetune/Loss (Raw)": 0.877824068069458, "Finetune/Step": 4618, "Finetune/Step Time": 4.951025960966945} +{"Finetune/Learning Rate": 6.434804127176409e-07, "Finetune/Loss": 0.8156038522720337, "Finetune/Loss (Raw)": 1.0077567100524902, "Finetune/Step": 4619, "Finetune/Step Time": 4.947347020730376} +{"Finetune/Learning Rate": 6.4128364873984e-07, "Finetune/Loss": 0.8149951696395874, "Finetune/Loss (Raw)": 0.8287513256072998, "Finetune/Step": 4620, "Finetune/Step Time": 4.951601076871157} +{"Finetune/Learning Rate": 6.390905166971406e-07, "Finetune/Loss": 0.8163808584213257, "Finetune/Loss (Raw)": 0.9937636256217957, "Finetune/Step": 4621, "Finetune/Step Time": 4.941062590107322} +{"Finetune/Learning Rate": 6.369010174406531e-07, "Finetune/Loss": 0.8177991509437561, "Finetune/Loss (Raw)": 0.8771480321884155, "Finetune/Step": 4622, "Finetune/Step Time": 4.942978378385305} +{"Finetune/Learning Rate": 6.347151518200834e-07, "Finetune/Loss": 0.8183664083480835, "Finetune/Loss (Raw)": 0.970416247844696, "Finetune/Step": 4623, "Finetune/Step Time": 4.940511075779796} +{"Finetune/Learning Rate": 6.325329206837217e-07, "Finetune/Loss": 0.8188950419425964, "Finetune/Loss (Raw)": 0.8392507433891296, "Finetune/Step": 4624, "Finetune/Step Time": 4.93347985483706} +{"Finetune/Learning Rate": 6.303543248784483e-07, "Finetune/Loss": 0.8185291290283203, "Finetune/Loss (Raw)": 0.8959006667137146, "Finetune/Step": 4625, "Finetune/Step Time": 4.933954980224371} +{"Finetune/Learning Rate": 6.281793652497359e-07, "Finetune/Loss": 0.8152612447738647, "Finetune/Loss (Raw)": 0.3460025191307068, "Finetune/Step": 4626, "Finetune/Step Time": 4.964027415961027} +{"Finetune/Learning Rate": 6.260080426416415e-07, "Finetune/Loss": 0.8147342801094055, "Finetune/Loss (Raw)": 0.8786219358444214, "Finetune/Step": 4627, "Finetune/Step Time": 4.973921539261937} +{"Finetune/Learning Rate": 6.238403578968144e-07, "Finetune/Loss": 0.814245343208313, "Finetune/Loss (Raw)": 0.7885275483131409, "Finetune/Step": 4628, "Finetune/Step Time": 4.977810362353921} +{"Finetune/Learning Rate": 6.216763118564883e-07, "Finetune/Loss": 0.813525915145874, "Finetune/Loss (Raw)": 0.8521195650100708, "Finetune/Step": 4629, "Finetune/Step Time": 4.975862644612789} +{"Finetune/Learning Rate": 6.195159053604926e-07, "Finetune/Loss": 0.8140794038772583, "Finetune/Loss (Raw)": 0.7938368320465088, "Finetune/Step": 4630, "Finetune/Step Time": 4.9839654471725225} +{"Finetune/Learning Rate": 6.173591392472334e-07, "Finetune/Loss": 0.8123024702072144, "Finetune/Loss (Raw)": 0.7192159295082092, "Finetune/Step": 4631, "Finetune/Step Time": 5.002835087478161} +{"Finetune/Learning Rate": 6.152060143537109e-07, "Finetune/Loss": 0.8078015446662903, "Finetune/Loss (Raw)": 0.26903268694877625, "Finetune/Step": 4632, "Finetune/Step Time": 5.032976727932692} +{"Finetune/Learning Rate": 6.130565315155124e-07, "Finetune/Loss": 0.8078182339668274, "Finetune/Loss (Raw)": 0.822819173336029, "Finetune/Step": 4633, "Finetune/Step Time": 5.039488298818469} +{"Finetune/Learning Rate": 6.109106915668106e-07, "Finetune/Loss": 0.8070659637451172, "Finetune/Loss (Raw)": 0.909567654132843, "Finetune/Step": 4634, "Finetune/Step Time": 5.046572936698794} +{"Finetune/Learning Rate": 6.087684953403616e-07, "Finetune/Loss": 0.8098822236061096, "Finetune/Loss (Raw)": 0.6882813572883606, "Finetune/Step": 4635, "Finetune/Step Time": 5.0312467738986015} +{"Finetune/Learning Rate": 6.066299436675117e-07, "Finetune/Loss": 0.8097896575927734, "Finetune/Loss (Raw)": 0.8587682843208313, "Finetune/Step": 4636, "Finetune/Step Time": 5.039094902575016} +{"Finetune/Learning Rate": 6.04495037378191e-07, "Finetune/Loss": 0.8106775879859924, "Finetune/Loss (Raw)": 0.8992313742637634, "Finetune/Step": 4637, "Finetune/Step Time": 5.033253666013479} +{"Finetune/Learning Rate": 6.023637773009139e-07, "Finetune/Loss": 0.8117330074310303, "Finetune/Loss (Raw)": 1.0921872854232788, "Finetune/Step": 4638, "Finetune/Step Time": 5.036974493414164} +{"Finetune/Learning Rate": 6.002361642627807e-07, "Finetune/Loss": 0.8093736171722412, "Finetune/Loss (Raw)": 0.7111003994941711, "Finetune/Step": 4639, "Finetune/Step Time": 5.050834145396948} +{"Finetune/Learning Rate": 5.981121990894789e-07, "Finetune/Loss": 0.8140906095504761, "Finetune/Loss (Raw)": 0.9646593332290649, "Finetune/Step": 4640, "Finetune/Step Time": 5.0215969607234} +{"Finetune/Learning Rate": 5.959918826052757e-07, "Finetune/Loss": 0.8143562078475952, "Finetune/Loss (Raw)": 0.8694232106208801, "Finetune/Step": 4641, "Finetune/Step Time": 5.025452649220824} +{"Finetune/Learning Rate": 5.938752156330241e-07, "Finetune/Loss": 0.8153254985809326, "Finetune/Loss (Raw)": 0.876603364944458, "Finetune/Step": 4642, "Finetune/Step Time": 5.026077013462782} +{"Finetune/Learning Rate": 5.917621989941624e-07, "Finetune/Loss": 0.8161022663116455, "Finetune/Loss (Raw)": 0.9388906359672546, "Finetune/Step": 4643, "Finetune/Step Time": 5.023237349465489} +{"Finetune/Learning Rate": 5.896528335087137e-07, "Finetune/Loss": 0.8130077123641968, "Finetune/Loss (Raw)": 0.37173742055892944, "Finetune/Step": 4644, "Finetune/Step Time": 5.0602286253124475} +{"Finetune/Learning Rate": 5.875471199952754e-07, "Finetune/Loss": 0.814997673034668, "Finetune/Loss (Raw)": 0.997497022151947, "Finetune/Step": 4645, "Finetune/Step Time": 5.063871910795569} +{"Finetune/Learning Rate": 5.854450592710382e-07, "Finetune/Loss": 0.8130706548690796, "Finetune/Loss (Raw)": 0.7577963471412659, "Finetune/Step": 4646, "Finetune/Step Time": 5.07686155103147} +{"Finetune/Learning Rate": 5.833466521517695e-07, "Finetune/Loss": 0.8126571774482727, "Finetune/Loss (Raw)": 0.8907336592674255, "Finetune/Step": 4647, "Finetune/Step Time": 5.081230981275439} +{"Finetune/Learning Rate": 5.812518994518213e-07, "Finetune/Loss": 0.8124608397483826, "Finetune/Loss (Raw)": 0.9083592891693115, "Finetune/Step": 4648, "Finetune/Step Time": 5.0864083636552095} +{"Finetune/Learning Rate": 5.791608019841244e-07, "Finetune/Loss": 0.8121325969696045, "Finetune/Loss (Raw)": 0.9488080143928528, "Finetune/Step": 4649, "Finetune/Step Time": 5.082728698849678} +{"Finetune/Learning Rate": 5.770733605601953e-07, "Finetune/Loss": 0.8117591738700867, "Finetune/Loss (Raw)": 0.7889019846916199, "Finetune/Step": 4650, "Finetune/Step Time": 5.085317451506853} +{"Finetune/Learning Rate": 5.74989575990127e-07, "Finetune/Loss": 0.8117961287498474, "Finetune/Loss (Raw)": 0.8724086880683899, "Finetune/Step": 4651, "Finetune/Step Time": 5.072904707863927} +{"Finetune/Learning Rate": 5.729094490825948e-07, "Finetune/Loss": 0.8117005228996277, "Finetune/Loss (Raw)": 0.8405371308326721, "Finetune/Step": 4652, "Finetune/Step Time": 5.071436103433371} +{"Finetune/Learning Rate": 5.708329806448587e-07, "Finetune/Loss": 0.807556688785553, "Finetune/Loss (Raw)": 0.30131518840789795, "Finetune/Step": 4653, "Finetune/Step Time": 5.104327181354165} +{"Finetune/Learning Rate": 5.687601714827551e-07, "Finetune/Loss": 0.8091399669647217, "Finetune/Loss (Raw)": 0.869872510433197, "Finetune/Step": 4654, "Finetune/Step Time": 5.109088405966759} +{"Finetune/Learning Rate": 5.666910224006971e-07, "Finetune/Loss": 0.8104207515716553, "Finetune/Loss (Raw)": 0.7033364176750183, "Finetune/Step": 4655, "Finetune/Step Time": 5.098304716870189} +{"Finetune/Learning Rate": 5.646255342016848e-07, "Finetune/Loss": 0.8073041439056396, "Finetune/Loss (Raw)": 0.4345455467700958, "Finetune/Step": 4656, "Finetune/Step Time": 5.141648001968861} +{"Finetune/Learning Rate": 5.625637076872914e-07, "Finetune/Loss": 0.8067221641540527, "Finetune/Loss (Raw)": 0.8371002078056335, "Finetune/Step": 4657, "Finetune/Step Time": 5.143921993672848} +{"Finetune/Learning Rate": 5.605055436576746e-07, "Finetune/Loss": 0.8064849376678467, "Finetune/Loss (Raw)": 0.9308421015739441, "Finetune/Step": 4658, "Finetune/Step Time": 5.139685735106468} +{"Finetune/Learning Rate": 5.584510429115641e-07, "Finetune/Loss": 0.8059637546539307, "Finetune/Loss (Raw)": 0.9258612394332886, "Finetune/Step": 4659, "Finetune/Step Time": 5.136325461789966} +{"Finetune/Learning Rate": 5.564002062462737e-07, "Finetune/Loss": 0.8081620335578918, "Finetune/Loss (Raw)": 0.9389833211898804, "Finetune/Step": 4660, "Finetune/Step Time": 5.143215084448457} +{"Finetune/Learning Rate": 5.54353034457693e-07, "Finetune/Loss": 0.8074665069580078, "Finetune/Loss (Raw)": 0.7954077124595642, "Finetune/Step": 4661, "Finetune/Step Time": 5.139436462894082} +{"Finetune/Learning Rate": 5.523095283402868e-07, "Finetune/Loss": 0.8118058443069458, "Finetune/Loss (Raw)": 0.8925703763961792, "Finetune/Step": 4662, "Finetune/Step Time": 5.1032149735838175} +{"Finetune/Learning Rate": 5.502696886871039e-07, "Finetune/Loss": 0.8132152557373047, "Finetune/Loss (Raw)": 0.9182628393173218, "Finetune/Step": 4663, "Finetune/Step Time": 5.092168390750885} +{"Finetune/Learning Rate": 5.482335162897657e-07, "Finetune/Loss": 0.8132020235061646, "Finetune/Loss (Raw)": 0.8627182245254517, "Finetune/Step": 4664, "Finetune/Step Time": 5.075543100014329} +{"Finetune/Learning Rate": 5.462010119384665e-07, "Finetune/Loss": 0.81258225440979, "Finetune/Loss (Raw)": 0.7924197316169739, "Finetune/Step": 4665, "Finetune/Step Time": 5.075622696429491} +{"Finetune/Learning Rate": 5.441721764219853e-07, "Finetune/Loss": 0.8112478256225586, "Finetune/Loss (Raw)": 0.7250927686691284, "Finetune/Step": 4666, "Finetune/Step Time": 5.095515292137861} +{"Finetune/Learning Rate": 5.42147010527675e-07, "Finetune/Loss": 0.8128154277801514, "Finetune/Loss (Raw)": 0.9715017080307007, "Finetune/Step": 4667, "Finetune/Step Time": 5.0823410004377365} +{"Finetune/Learning Rate": 5.401255150414598e-07, "Finetune/Loss": 0.8169809579849243, "Finetune/Loss (Raw)": 0.8424119353294373, "Finetune/Step": 4668, "Finetune/Step Time": 5.057219248265028} +{"Finetune/Learning Rate": 5.381076907478433e-07, "Finetune/Loss": 0.8137341737747192, "Finetune/Loss (Raw)": 0.3497425317764282, "Finetune/Step": 4669, "Finetune/Step Time": 5.072305491194129} +{"Finetune/Learning Rate": 5.360935384299049e-07, "Finetune/Loss": 0.8149056434631348, "Finetune/Loss (Raw)": 0.8958601355552673, "Finetune/Step": 4670, "Finetune/Step Time": 5.082783257588744} +{"Finetune/Learning Rate": 5.34083058869297e-07, "Finetune/Loss": 0.8146665096282959, "Finetune/Loss (Raw)": 0.8110478520393372, "Finetune/Step": 4671, "Finetune/Step Time": 5.077930683270097} +{"Finetune/Learning Rate": 5.320762528462453e-07, "Finetune/Loss": 0.8144465684890747, "Finetune/Loss (Raw)": 0.8846542239189148, "Finetune/Step": 4672, "Finetune/Step Time": 5.092221261933446} +{"Finetune/Learning Rate": 5.30073121139556e-07, "Finetune/Loss": 0.8138127326965332, "Finetune/Loss (Raw)": 0.8378731608390808, "Finetune/Step": 4673, "Finetune/Step Time": 5.0808375254273415} +{"Finetune/Learning Rate": 5.28073664526606e-07, "Finetune/Loss": 0.8137578964233398, "Finetune/Loss (Raw)": 0.9380273222923279, "Finetune/Step": 4674, "Finetune/Step Time": 5.104687413200736} +{"Finetune/Learning Rate": 5.260778837833391e-07, "Finetune/Loss": 0.8134728670120239, "Finetune/Loss (Raw)": 0.887672483921051, "Finetune/Step": 4675, "Finetune/Step Time": 5.106368446722627} +{"Finetune/Learning Rate": 5.240857796842846e-07, "Finetune/Loss": 0.8155527710914612, "Finetune/Loss (Raw)": 0.8389822840690613, "Finetune/Step": 4676, "Finetune/Step Time": 5.033005157485604} +{"Finetune/Learning Rate": 5.220973530025375e-07, "Finetune/Loss": 0.8170362710952759, "Finetune/Loss (Raw)": 0.910007655620575, "Finetune/Step": 4677, "Finetune/Step Time": 5.0227635987102985} +{"Finetune/Learning Rate": 5.201126045097671e-07, "Finetune/Loss": 0.8165983557701111, "Finetune/Loss (Raw)": 0.9264240860939026, "Finetune/Step": 4678, "Finetune/Step Time": 5.048535650596023} +{"Finetune/Learning Rate": 5.181315349762162e-07, "Finetune/Loss": 0.8152611255645752, "Finetune/Loss (Raw)": 0.73851078748703, "Finetune/Step": 4679, "Finetune/Step Time": 5.0480186101049185} +{"Finetune/Learning Rate": 5.161541451706975e-07, "Finetune/Loss": 0.8177568912506104, "Finetune/Loss (Raw)": 1.0071980953216553, "Finetune/Step": 4680, "Finetune/Step Time": 5.039549369364977} +{"Finetune/Learning Rate": 5.141804358605995e-07, "Finetune/Loss": 0.8174080848693848, "Finetune/Loss (Raw)": 0.879318118095398, "Finetune/Step": 4681, "Finetune/Step Time": 5.05666109547019} +{"Finetune/Learning Rate": 5.122104078118773e-07, "Finetune/Loss": 0.8163521885871887, "Finetune/Loss (Raw)": 0.9002479910850525, "Finetune/Step": 4682, "Finetune/Step Time": 5.067845579236746} +{"Finetune/Learning Rate": 5.102440617890658e-07, "Finetune/Loss": 0.8166817426681519, "Finetune/Loss (Raw)": 0.9136412739753723, "Finetune/Step": 4683, "Finetune/Step Time": 5.051526768133044} +{"Finetune/Learning Rate": 5.082813985552604e-07, "Finetune/Loss": 0.8157449960708618, "Finetune/Loss (Raw)": 0.9431030750274658, "Finetune/Step": 4684, "Finetune/Step Time": 5.055433865636587} +{"Finetune/Learning Rate": 5.063224188721327e-07, "Finetune/Loss": 0.816855788230896, "Finetune/Loss (Raw)": 0.9192084074020386, "Finetune/Step": 4685, "Finetune/Step Time": 5.057705437764525} +{"Finetune/Learning Rate": 5.043671234999271e-07, "Finetune/Loss": 0.8175458908081055, "Finetune/Loss (Raw)": 0.7475242018699646, "Finetune/Step": 4686, "Finetune/Step Time": 5.0560711491853} +{"Finetune/Learning Rate": 5.02415513197454e-07, "Finetune/Loss": 0.8168463706970215, "Finetune/Loss (Raw)": 0.8801314234733582, "Finetune/Step": 4687, "Finetune/Step Time": 5.047652849927545} +{"Finetune/Learning Rate": 5.004675887220978e-07, "Finetune/Loss": 0.8188536167144775, "Finetune/Loss (Raw)": 1.0925991535186768, "Finetune/Step": 4688, "Finetune/Step Time": 5.039285371080041} +{"Finetune/Learning Rate": 4.985233508298037e-07, "Finetune/Loss": 0.8195489048957825, "Finetune/Loss (Raw)": 0.9881218671798706, "Finetune/Step": 4689, "Finetune/Step Time": 5.030855434015393} +{"Finetune/Learning Rate": 4.965828002750994e-07, "Finetune/Loss": 0.8183267116546631, "Finetune/Loss (Raw)": 0.7866243124008179, "Finetune/Step": 4690, "Finetune/Step Time": 5.045179883018136} +{"Finetune/Learning Rate": 4.946459378110701e-07, "Finetune/Loss": 0.8235458135604858, "Finetune/Loss (Raw)": 0.9732657074928284, "Finetune/Step": 4691, "Finetune/Step Time": 5.004592252895236} +{"Finetune/Learning Rate": 4.927127641893759e-07, "Finetune/Loss": 0.8248001933097839, "Finetune/Loss (Raw)": 1.0179064273834229, "Finetune/Step": 4692, "Finetune/Step Time": 5.005277331918478} +{"Finetune/Learning Rate": 4.907832801602452e-07, "Finetune/Loss": 0.8256161212921143, "Finetune/Loss (Raw)": 0.8961018919944763, "Finetune/Step": 4693, "Finetune/Step Time": 5.017289726063609} +{"Finetune/Learning Rate": 4.888574864724715e-07, "Finetune/Loss": 0.8268068432807922, "Finetune/Loss (Raw)": 0.98802250623703, "Finetune/Step": 4694, "Finetune/Step Time": 5.010993834584951} +{"Finetune/Learning Rate": 4.869353838734169e-07, "Finetune/Loss": 0.8269697427749634, "Finetune/Loss (Raw)": 0.9872617721557617, "Finetune/Step": 4695, "Finetune/Step Time": 5.015634020790458} +{"Finetune/Learning Rate": 4.850169731090127e-07, "Finetune/Loss": 0.8259482979774475, "Finetune/Loss (Raw)": 0.7927836179733276, "Finetune/Step": 4696, "Finetune/Step Time": 5.026924770325422} +{"Finetune/Learning Rate": 4.831022549237574e-07, "Finetune/Loss": 0.8249121904373169, "Finetune/Loss (Raw)": 0.8780711889266968, "Finetune/Step": 4697, "Finetune/Step Time": 5.021550428122282} +{"Finetune/Learning Rate": 4.811912300607169e-07, "Finetune/Loss": 0.82486891746521, "Finetune/Loss (Raw)": 0.9270091652870178, "Finetune/Step": 4698, "Finetune/Step Time": 5.025622913613915} +{"Finetune/Learning Rate": 4.792838992615179e-07, "Finetune/Loss": 0.8258401155471802, "Finetune/Loss (Raw)": 0.9637138247489929, "Finetune/Step": 4699, "Finetune/Step Time": 5.015694031491876} +{"Finetune/Learning Rate": 4.773802632663627e-07, "Finetune/Loss": 0.8261310458183289, "Finetune/Loss (Raw)": 0.8861656188964844, "Finetune/Step": 4700, "Finetune/Step Time": 5.005854941904545} +{"Finetune/Learning Rate": 4.7548032281401367e-07, "Finetune/Loss": 0.8271106481552124, "Finetune/Loss (Raw)": 0.9141291975975037, "Finetune/Step": 4701, "Finetune/Step Time": 4.99433327652514} +{"Finetune/Learning Rate": 4.735840786418011e-07, "Finetune/Loss": 0.8267182111740112, "Finetune/Loss (Raw)": 0.8457427024841309, "Finetune/Step": 4702, "Finetune/Step Time": 4.996351186186075} +{"Finetune/Learning Rate": 4.716915314856196e-07, "Finetune/Loss": 0.8279467821121216, "Finetune/Loss (Raw)": 0.9658641815185547, "Finetune/Step": 4703, "Finetune/Step Time": 4.979163121432066} +{"Finetune/Learning Rate": 4.698026820799306e-07, "Finetune/Loss": 0.827582597732544, "Finetune/Loss (Raw)": 0.7445366382598877, "Finetune/Step": 4704, "Finetune/Step Time": 4.978594524785876} +{"Finetune/Learning Rate": 4.6791753115775884e-07, "Finetune/Loss": 0.8272924423217773, "Finetune/Loss (Raw)": 0.9567237496376038, "Finetune/Step": 4705, "Finetune/Step Time": 4.979327838867903} +{"Finetune/Learning Rate": 4.660360794506946e-07, "Finetune/Loss": 0.8262796401977539, "Finetune/Loss (Raw)": 0.8234519362449646, "Finetune/Step": 4706, "Finetune/Step Time": 4.972723063081503} +{"Finetune/Learning Rate": 4.641583276888939e-07, "Finetune/Loss": 0.8289121389389038, "Finetune/Loss (Raw)": 0.752332866191864, "Finetune/Step": 4707, "Finetune/Step Time": 4.939603252336383} +{"Finetune/Learning Rate": 4.622842766010749e-07, "Finetune/Loss": 0.8328328132629395, "Finetune/Loss (Raw)": 0.7688870429992676, "Finetune/Step": 4708, "Finetune/Step Time": 4.910091718658805} +{"Finetune/Learning Rate": 4.604139269145191e-07, "Finetune/Loss": 0.8320236206054688, "Finetune/Loss (Raw)": 0.734731137752533, "Finetune/Step": 4709, "Finetune/Step Time": 4.909265719354153} +{"Finetune/Learning Rate": 4.585472793550738e-07, "Finetune/Loss": 0.8328248858451843, "Finetune/Loss (Raw)": 1.0009516477584839, "Finetune/Step": 4710, "Finetune/Step Time": 4.9000175558030605} +{"Finetune/Learning Rate": 4.5668433464714835e-07, "Finetune/Loss": 0.8319995403289795, "Finetune/Loss (Raw)": 0.8583380579948425, "Finetune/Step": 4711, "Finetune/Step Time": 4.905796896666288} +{"Finetune/Learning Rate": 4.548250935137144e-07, "Finetune/Loss": 0.8321390151977539, "Finetune/Loss (Raw)": 0.7979016900062561, "Finetune/Step": 4712, "Finetune/Step Time": 4.918714454397559} +{"Finetune/Learning Rate": 4.5296955667630926e-07, "Finetune/Loss": 0.8337910175323486, "Finetune/Loss (Raw)": 0.9537851810455322, "Finetune/Step": 4713, "Finetune/Step Time": 4.920993460342288} +{"Finetune/Learning Rate": 4.51117724855028e-07, "Finetune/Loss": 0.8344361186027527, "Finetune/Loss (Raw)": 0.8273000717163086, "Finetune/Step": 4714, "Finetune/Step Time": 4.9244050066918135} +{"Finetune/Learning Rate": 4.492695987685314e-07, "Finetune/Loss": 0.8345166444778442, "Finetune/Loss (Raw)": 0.918729305267334, "Finetune/Step": 4715, "Finetune/Step Time": 4.922750484198332} +{"Finetune/Learning Rate": 4.474251791340412e-07, "Finetune/Loss": 0.8330277800559998, "Finetune/Loss (Raw)": 0.8500239849090576, "Finetune/Step": 4716, "Finetune/Step Time": 4.898555520921946} +{"Finetune/Learning Rate": 4.4558446666734147e-07, "Finetune/Loss": 0.8334499597549438, "Finetune/Loss (Raw)": 0.8942153453826904, "Finetune/Step": 4717, "Finetune/Step Time": 4.89611129462719} +{"Finetune/Learning Rate": 4.437474620827764e-07, "Finetune/Loss": 0.832195520401001, "Finetune/Loss (Raw)": 0.6486288905143738, "Finetune/Step": 4718, "Finetune/Step Time": 4.910271104425192} +{"Finetune/Learning Rate": 4.419141660932491e-07, "Finetune/Loss": 0.8310018181800842, "Finetune/Loss (Raw)": 0.7549614310264587, "Finetune/Step": 4719, "Finetune/Step Time": 4.909700330346823} +{"Finetune/Learning Rate": 4.4008457941022933e-07, "Finetune/Loss": 0.8333324790000916, "Finetune/Loss (Raw)": 0.5882062911987305, "Finetune/Step": 4720, "Finetune/Step Time": 4.915028870105743} +{"Finetune/Learning Rate": 4.382587027437435e-07, "Finetune/Loss": 0.8324244022369385, "Finetune/Loss (Raw)": 0.7969292998313904, "Finetune/Step": 4721, "Finetune/Step Time": 4.9195691011846066} +{"Finetune/Learning Rate": 4.364365368023771e-07, "Finetune/Loss": 0.8357654809951782, "Finetune/Loss (Raw)": 0.8387593626976013, "Finetune/Step": 4722, "Finetune/Step Time": 4.885660348460078} +{"Finetune/Learning Rate": 4.3461808229327773e-07, "Finetune/Loss": 0.8354294300079346, "Finetune/Loss (Raw)": 0.8781391978263855, "Finetune/Step": 4723, "Finetune/Step Time": 4.884491430595517} +{"Finetune/Learning Rate": 4.3280333992215206e-07, "Finetune/Loss": 0.834503173828125, "Finetune/Loss (Raw)": 0.8231483101844788, "Finetune/Step": 4724, "Finetune/Step Time": 4.906576449051499} +{"Finetune/Learning Rate": 4.3099231039326785e-07, "Finetune/Loss": 0.8352560997009277, "Finetune/Loss (Raw)": 0.8531769514083862, "Finetune/Step": 4725, "Finetune/Step Time": 4.909116558730602} +{"Finetune/Learning Rate": 4.291849944094462e-07, "Finetune/Loss": 0.8343361616134644, "Finetune/Loss (Raw)": 0.7481100559234619, "Finetune/Step": 4726, "Finetune/Step Time": 4.911751117557287} +{"Finetune/Learning Rate": 4.273813926720771e-07, "Finetune/Loss": 0.8345540165901184, "Finetune/Loss (Raw)": 0.8269693851470947, "Finetune/Step": 4727, "Finetune/Step Time": 4.908118242397904} +{"Finetune/Learning Rate": 4.2558150588109747e-07, "Finetune/Loss": 0.8350603580474854, "Finetune/Loss (Raw)": 0.8932424187660217, "Finetune/Step": 4728, "Finetune/Step Time": 4.9108651373535395} +{"Finetune/Learning Rate": 4.237853347350107e-07, "Finetune/Loss": 0.837639331817627, "Finetune/Loss (Raw)": 1.053619623184204, "Finetune/Step": 4729, "Finetune/Step Time": 4.912908924743533} +{"Finetune/Learning Rate": 4.219928799308759e-07, "Finetune/Loss": 0.8365445137023926, "Finetune/Loss (Raw)": 0.8861038684844971, "Finetune/Step": 4730, "Finetune/Step Time": 4.907803043723106} +{"Finetune/Learning Rate": 4.2020414216431106e-07, "Finetune/Loss": 0.835604190826416, "Finetune/Loss (Raw)": 0.7894336581230164, "Finetune/Step": 4731, "Finetune/Step Time": 4.90630161575973} +{"Finetune/Learning Rate": 4.184191221294875e-07, "Finetune/Loss": 0.8356475830078125, "Finetune/Loss (Raw)": 0.8493552207946777, "Finetune/Step": 4732, "Finetune/Step Time": 4.900574907660484} +{"Finetune/Learning Rate": 4.16637820519139e-07, "Finetune/Loss": 0.8402325510978699, "Finetune/Loss (Raw)": 0.9180071353912354, "Finetune/Step": 4733, "Finetune/Step Time": 4.866137636825442} +{"Finetune/Learning Rate": 4.1486023802455253e-07, "Finetune/Loss": 0.8394800424575806, "Finetune/Loss (Raw)": 0.7895117998123169, "Finetune/Step": 4734, "Finetune/Step Time": 4.873114293441176} +{"Finetune/Learning Rate": 4.1308637533557294e-07, "Finetune/Loss": 0.8405575752258301, "Finetune/Loss (Raw)": 0.953913152217865, "Finetune/Step": 4735, "Finetune/Step Time": 4.8734515849500895} +{"Finetune/Learning Rate": 4.1131623314060175e-07, "Finetune/Loss": 0.8394433856010437, "Finetune/Loss (Raw)": 0.7122047543525696, "Finetune/Step": 4736, "Finetune/Step Time": 4.883476452901959} +{"Finetune/Learning Rate": 4.095498121265995e-07, "Finetune/Loss": 0.8374091386795044, "Finetune/Loss (Raw)": 0.7831550240516663, "Finetune/Step": 4737, "Finetune/Step Time": 4.893029611557722} +{"Finetune/Learning Rate": 4.077871129790767e-07, "Finetune/Loss": 0.8386566638946533, "Finetune/Loss (Raw)": 0.9791575074195862, "Finetune/Step": 4738, "Finetune/Step Time": 4.902418300509453} +{"Finetune/Learning Rate": 4.060281363821017e-07, "Finetune/Loss": 0.8425335884094238, "Finetune/Loss (Raw)": 0.7818276882171631, "Finetune/Step": 4739, "Finetune/Step Time": 4.870480824261904} +{"Finetune/Learning Rate": 4.042728830183018e-07, "Finetune/Loss": 0.8420709371566772, "Finetune/Loss (Raw)": 0.8231845498085022, "Finetune/Step": 4740, "Finetune/Step Time": 4.874362535774708} +{"Finetune/Learning Rate": 4.025213535688544e-07, "Finetune/Loss": 0.8410568237304688, "Finetune/Loss (Raw)": 0.6897458434104919, "Finetune/Step": 4741, "Finetune/Step Time": 4.856900056824088} +{"Finetune/Learning Rate": 4.007735487134956e-07, "Finetune/Loss": 0.8411079049110413, "Finetune/Loss (Raw)": 0.7687185406684875, "Finetune/Step": 4742, "Finetune/Step Time": 4.867504131048918} +{"Finetune/Learning Rate": 3.99029469130513e-07, "Finetune/Loss": 0.8393567204475403, "Finetune/Loss (Raw)": 0.7301955819129944, "Finetune/Step": 4743, "Finetune/Step Time": 4.883115839213133} +{"Finetune/Learning Rate": 3.972891154967506e-07, "Finetune/Loss": 0.8407520055770874, "Finetune/Loss (Raw)": 1.0028653144836426, "Finetune/Step": 4744, "Finetune/Step Time": 4.8844212256371975} +{"Finetune/Learning Rate": 3.95552488487605e-07, "Finetune/Loss": 0.8407727479934692, "Finetune/Loss (Raw)": 0.7744505405426025, "Finetune/Step": 4745, "Finetune/Step Time": 4.90373944118619} +{"Finetune/Learning Rate": 3.9381958877702684e-07, "Finetune/Loss": 0.8400962352752686, "Finetune/Loss (Raw)": 0.7912338376045227, "Finetune/Step": 4746, "Finetune/Step Time": 4.927895246073604} +{"Finetune/Learning Rate": 3.920904170375239e-07, "Finetune/Loss": 0.8402222394943237, "Finetune/Loss (Raw)": 1.0238841772079468, "Finetune/Step": 4747, "Finetune/Step Time": 4.927669202908874} +{"Finetune/Learning Rate": 3.903649739401494e-07, "Finetune/Loss": 0.8410390615463257, "Finetune/Loss (Raw)": 0.9333004951477051, "Finetune/Step": 4748, "Finetune/Step Time": 4.923057436943054} +{"Finetune/Learning Rate": 3.8864326015451536e-07, "Finetune/Loss": 0.8398044109344482, "Finetune/Loss (Raw)": 0.8357318639755249, "Finetune/Step": 4749, "Finetune/Step Time": 4.9196880385279655} +{"Finetune/Learning Rate": 3.8692527634878853e-07, "Finetune/Loss": 0.8396895527839661, "Finetune/Loss (Raw)": 0.8624449372291565, "Finetune/Step": 4750, "Finetune/Step Time": 4.916898628696799} +{"Finetune/Learning Rate": 3.8521102318968106e-07, "Finetune/Loss": 0.8390184640884399, "Finetune/Loss (Raw)": 0.8845194578170776, "Finetune/Step": 4751, "Finetune/Step Time": 4.938093958422542} +{"Finetune/Learning Rate": 3.835005013424631e-07, "Finetune/Loss": 0.8394749164581299, "Finetune/Loss (Raw)": 0.8976770043373108, "Finetune/Step": 4752, "Finetune/Step Time": 4.9412883799523115} +{"Finetune/Learning Rate": 3.8179371147095354e-07, "Finetune/Loss": 0.8384573459625244, "Finetune/Loss (Raw)": 0.7656552791595459, "Finetune/Step": 4753, "Finetune/Step Time": 4.9328222051262856} +{"Finetune/Learning Rate": 3.80090654237526e-07, "Finetune/Loss": 0.8429349660873413, "Finetune/Loss (Raw)": 0.9191296696662903, "Finetune/Step": 4754, "Finetune/Step Time": 4.902964621782303} +{"Finetune/Learning Rate": 3.7839133030310174e-07, "Finetune/Loss": 0.843309223651886, "Finetune/Loss (Raw)": 0.9265303015708923, "Finetune/Step": 4755, "Finetune/Step Time": 4.902950564399362} +{"Finetune/Learning Rate": 3.766957403271554e-07, "Finetune/Loss": 0.8439755439758301, "Finetune/Loss (Raw)": 0.8738206624984741, "Finetune/Step": 4756, "Finetune/Step Time": 4.895578306168318} +{"Finetune/Learning Rate": 3.750038849677162e-07, "Finetune/Loss": 0.8441097140312195, "Finetune/Loss (Raw)": 0.8692870736122131, "Finetune/Step": 4757, "Finetune/Step Time": 4.8950465470552444} +{"Finetune/Learning Rate": 3.7331576488135456e-07, "Finetune/Loss": 0.8445522785186768, "Finetune/Loss (Raw)": 0.8504887819290161, "Finetune/Step": 4758, "Finetune/Step Time": 4.884998042136431} +{"Finetune/Learning Rate": 3.716313807231986e-07, "Finetune/Loss": 0.845439612865448, "Finetune/Loss (Raw)": 0.8327922821044922, "Finetune/Step": 4759, "Finetune/Step Time": 4.8705896232277155} +{"Finetune/Learning Rate": 3.699507331469276e-07, "Finetune/Loss": 0.8500868678092957, "Finetune/Loss (Raw)": 0.8638838529586792, "Finetune/Step": 4760, "Finetune/Step Time": 4.8378904070705175} +{"Finetune/Learning Rate": 3.682738228047655e-07, "Finetune/Loss": 0.8495542407035828, "Finetune/Loss (Raw)": 0.7546401619911194, "Finetune/Step": 4761, "Finetune/Step Time": 4.841014806181192} +{"Finetune/Learning Rate": 3.666006503474895e-07, "Finetune/Loss": 0.8486034274101257, "Finetune/Loss (Raw)": 0.7878658175468445, "Finetune/Step": 4762, "Finetune/Step Time": 4.839609945192933} +{"Finetune/Learning Rate": 3.6493121642442454e-07, "Finetune/Loss": 0.850048303604126, "Finetune/Loss (Raw)": 0.8732265830039978, "Finetune/Step": 4763, "Finetune/Step Time": 4.829390920698643} +{"Finetune/Learning Rate": 3.6326552168344574e-07, "Finetune/Loss": 0.8499146699905396, "Finetune/Loss (Raw)": 0.8416581153869629, "Finetune/Step": 4764, "Finetune/Step Time": 4.8279134929180145} +{"Finetune/Learning Rate": 3.616035667709761e-07, "Finetune/Loss": 0.848914384841919, "Finetune/Loss (Raw)": 0.7711940407752991, "Finetune/Step": 4765, "Finetune/Step Time": 4.82789371162653} +{"Finetune/Learning Rate": 3.5994535233198844e-07, "Finetune/Loss": 0.84836745262146, "Finetune/Loss (Raw)": 1.022179365158081, "Finetune/Step": 4766, "Finetune/Step Time": 4.829734003171325} +{"Finetune/Learning Rate": 3.582908790100037e-07, "Finetune/Loss": 0.8497037887573242, "Finetune/Loss (Raw)": 0.8821521997451782, "Finetune/Step": 4767, "Finetune/Step Time": 4.805037409067154} +{"Finetune/Learning Rate": 3.566401474470904e-07, "Finetune/Loss": 0.848449170589447, "Finetune/Loss (Raw)": 0.8040764331817627, "Finetune/Step": 4768, "Finetune/Step Time": 4.8095412738621235} +{"Finetune/Learning Rate": 3.5499315828386393e-07, "Finetune/Loss": 0.8480973839759827, "Finetune/Loss (Raw)": 0.8243855834007263, "Finetune/Step": 4769, "Finetune/Step Time": 4.802868466824293} +{"Finetune/Learning Rate": 3.533499121594919e-07, "Finetune/Loss": 0.8461974859237671, "Finetune/Loss (Raw)": 0.6334134936332703, "Finetune/Step": 4770, "Finetune/Step Time": 4.806699696928263} +{"Finetune/Learning Rate": 3.5171040971168414e-07, "Finetune/Loss": 0.8465984463691711, "Finetune/Loss (Raw)": 0.9902241826057434, "Finetune/Step": 4771, "Finetune/Step Time": 4.80990001372993} +{"Finetune/Learning Rate": 3.500746515766973e-07, "Finetune/Loss": 0.8496698141098022, "Finetune/Loss (Raw)": 0.7648667097091675, "Finetune/Step": 4772, "Finetune/Step Time": 4.786827752366662} +{"Finetune/Learning Rate": 3.4844263838934023e-07, "Finetune/Loss": 0.8467369079589844, "Finetune/Loss (Raw)": 0.6220876574516296, "Finetune/Step": 4773, "Finetune/Step Time": 4.78116768412292} +{"Finetune/Learning Rate": 3.468143707829641e-07, "Finetune/Loss": 0.847682774066925, "Finetune/Loss (Raw)": 0.8788682222366333, "Finetune/Step": 4774, "Finetune/Step Time": 4.771472441032529} +{"Finetune/Learning Rate": 3.45189849389469e-07, "Finetune/Loss": 0.8482365012168884, "Finetune/Loss (Raw)": 0.9616091847419739, "Finetune/Step": 4775, "Finetune/Step Time": 4.774330146610737} +{"Finetune/Learning Rate": 3.4356907483929835e-07, "Finetune/Loss": 0.8473882079124451, "Finetune/Loss (Raw)": 0.7997769117355347, "Finetune/Step": 4776, "Finetune/Step Time": 4.786702433601022} +{"Finetune/Learning Rate": 3.419520477614435e-07, "Finetune/Loss": 0.8468841314315796, "Finetune/Loss (Raw)": 0.8842912316322327, "Finetune/Step": 4777, "Finetune/Step Time": 4.787807315587997} +{"Finetune/Learning Rate": 3.403387687834425e-07, "Finetune/Loss": 0.842972993850708, "Finetune/Loss (Raw)": 0.28826773166656494, "Finetune/Step": 4778, "Finetune/Step Time": 4.807914920151234} +{"Finetune/Learning Rate": 3.3872923853137453e-07, "Finetune/Loss": 0.8440496921539307, "Finetune/Loss (Raw)": 1.010229229927063, "Finetune/Step": 4779, "Finetune/Step Time": 4.809308907017112} +{"Finetune/Learning Rate": 3.371234576298699e-07, "Finetune/Loss": 0.8439735174179077, "Finetune/Loss (Raw)": 0.8307837247848511, "Finetune/Step": 4780, "Finetune/Step Time": 4.81433486007154} +{"Finetune/Learning Rate": 3.355214267021012e-07, "Finetune/Loss": 0.8478716015815735, "Finetune/Loss (Raw)": 0.8002726435661316, "Finetune/Step": 4781, "Finetune/Step Time": 4.792401779443026} +{"Finetune/Learning Rate": 3.3392314636978117e-07, "Finetune/Loss": 0.8475774526596069, "Finetune/Loss (Raw)": 0.8322203755378723, "Finetune/Step": 4782, "Finetune/Step Time": 4.794226346537471} +{"Finetune/Learning Rate": 3.323286172531748e-07, "Finetune/Loss": 0.8449434041976929, "Finetune/Loss (Raw)": 0.3661831021308899, "Finetune/Step": 4783, "Finetune/Step Time": 4.819669516757131} +{"Finetune/Learning Rate": 3.30737839971087e-07, "Finetune/Loss": 0.8484365344047546, "Finetune/Loss (Raw)": 0.8816654086112976, "Finetune/Step": 4784, "Finetune/Step Time": 4.777087107300758} +{"Finetune/Learning Rate": 3.2915081514086624e-07, "Finetune/Loss": 0.8500145673751831, "Finetune/Loss (Raw)": 1.0390820503234863, "Finetune/Step": 4785, "Finetune/Step Time": 4.773607052862644} +{"Finetune/Learning Rate": 3.2756754337840757e-07, "Finetune/Loss": 0.8497117757797241, "Finetune/Loss (Raw)": 0.8920795321464539, "Finetune/Step": 4786, "Finetune/Step Time": 4.775898220017552} +{"Finetune/Learning Rate": 3.2598802529814734e-07, "Finetune/Loss": 0.8491102457046509, "Finetune/Loss (Raw)": 0.8488742709159851, "Finetune/Step": 4787, "Finetune/Step Time": 4.771421929821372} +{"Finetune/Learning Rate": 3.2441226151306403e-07, "Finetune/Loss": 0.8481794595718384, "Finetune/Loss (Raw)": 0.8198447823524475, "Finetune/Step": 4788, "Finetune/Step Time": 4.780927957966924} +{"Finetune/Learning Rate": 3.228402526346819e-07, "Finetune/Loss": 0.8475247025489807, "Finetune/Loss (Raw)": 0.7115979194641113, "Finetune/Step": 4789, "Finetune/Step Time": 4.7811091132462025} +{"Finetune/Learning Rate": 3.212719992730684e-07, "Finetune/Loss": 0.8479161858558655, "Finetune/Loss (Raw)": 0.9426764845848083, "Finetune/Step": 4790, "Finetune/Step Time": 4.786672111600637} +{"Finetune/Learning Rate": 3.197075020368312e-07, "Finetune/Loss": 0.8431013822555542, "Finetune/Loss (Raw)": 0.30196985602378845, "Finetune/Step": 4791, "Finetune/Step Time": 4.822749335318804} +{"Finetune/Learning Rate": 3.1814676153311907e-07, "Finetune/Loss": 0.8442943096160889, "Finetune/Loss (Raw)": 1.0154157876968384, "Finetune/Step": 4792, "Finetune/Step Time": 4.82637538947165} +{"Finetune/Learning Rate": 3.165897783676275e-07, "Finetune/Loss": 0.8452625274658203, "Finetune/Loss (Raw)": 0.9163480997085571, "Finetune/Step": 4793, "Finetune/Step Time": 4.824849082157016} +{"Finetune/Learning Rate": 3.1503655314458983e-07, "Finetune/Loss": 0.8459568023681641, "Finetune/Loss (Raw)": 0.8139623999595642, "Finetune/Step": 4794, "Finetune/Step Time": 4.811017895117402} +{"Finetune/Learning Rate": 3.134870864667827e-07, "Finetune/Loss": 0.844538152217865, "Finetune/Loss (Raw)": 0.7899149060249329, "Finetune/Step": 4795, "Finetune/Step Time": 4.8433955274522305} +{"Finetune/Learning Rate": 3.1194137893552523e-07, "Finetune/Loss": 0.845324695110321, "Finetune/Loss (Raw)": 0.9430878162384033, "Finetune/Step": 4796, "Finetune/Step Time": 4.842009246349335} +{"Finetune/Learning Rate": 3.1039943115067417e-07, "Finetune/Loss": 0.849707841873169, "Finetune/Loss (Raw)": 0.9107863306999207, "Finetune/Step": 4797, "Finetune/Step Time": 4.812701353803277} +{"Finetune/Learning Rate": 3.088612437106309e-07, "Finetune/Loss": 0.8481804132461548, "Finetune/Loss (Raw)": 0.700350821018219, "Finetune/Step": 4798, "Finetune/Step Time": 4.818200374022126} +{"Finetune/Learning Rate": 3.0732681721233446e-07, "Finetune/Loss": 0.8486791849136353, "Finetune/Loss (Raw)": 0.8748857975006104, "Finetune/Step": 4799, "Finetune/Step Time": 4.823176991194487} +{"Finetune/Learning Rate": 3.057961522512676e-07, "Finetune/Loss": 0.8500004410743713, "Finetune/Loss (Raw)": 1.0537810325622559, "Finetune/Step": 4800, "Finetune/Step Time": 4.806623289361596} +{"Finetune/Learning Rate": 3.0426924942145166e-07, "Finetune/Loss": 0.8506575226783752, "Finetune/Loss (Raw)": 0.9219716191291809, "Finetune/Step": 4801, "Finetune/Step Time": 4.799640258774161} +{"Finetune/Learning Rate": 3.027461093154449e-07, "Finetune/Loss": 0.85025554895401, "Finetune/Loss (Raw)": 0.8865765929222107, "Finetune/Step": 4802, "Finetune/Step Time": 4.773288244381547} +{"Finetune/Learning Rate": 3.012267325243512e-07, "Finetune/Loss": 0.8504694700241089, "Finetune/Loss (Raw)": 0.9150592088699341, "Finetune/Step": 4803, "Finetune/Step Time": 4.778269324451685} +{"Finetune/Learning Rate": 2.9971111963780994e-07, "Finetune/Loss": 0.8510017395019531, "Finetune/Loss (Raw)": 0.9071081280708313, "Finetune/Step": 4804, "Finetune/Step Time": 4.781124047935009} +{"Finetune/Learning Rate": 2.981992712440007e-07, "Finetune/Loss": 0.8512706756591797, "Finetune/Loss (Raw)": 0.9444305896759033, "Finetune/Step": 4805, "Finetune/Step Time": 4.787637000903487} +{"Finetune/Learning Rate": 2.96691187929643e-07, "Finetune/Loss": 0.8510831594467163, "Finetune/Loss (Raw)": 0.9024195671081543, "Finetune/Step": 4806, "Finetune/Step Time": 4.768342584371567} +{"Finetune/Learning Rate": 2.9518687027999427e-07, "Finetune/Loss": 0.8518825769424438, "Finetune/Loss (Raw)": 0.8408434391021729, "Finetune/Step": 4807, "Finetune/Step Time": 4.769447827711701} +{"Finetune/Learning Rate": 2.936863188788497e-07, "Finetune/Loss": 0.8515042066574097, "Finetune/Loss (Raw)": 0.9587659239768982, "Finetune/Step": 4808, "Finetune/Step Time": 4.765683135017753} +{"Finetune/Learning Rate": 2.921895343085446e-07, "Finetune/Loss": 0.8498265743255615, "Finetune/Loss (Raw)": 0.6645724773406982, "Finetune/Step": 4809, "Finetune/Step Time": 4.754377420991659} +{"Finetune/Learning Rate": 2.9069651714995426e-07, "Finetune/Loss": 0.8492597341537476, "Finetune/Loss (Raw)": 0.8276954889297485, "Finetune/Step": 4810, "Finetune/Step Time": 4.745293382555246} +{"Finetune/Learning Rate": 2.8920726798248645e-07, "Finetune/Loss": 0.8491171598434448, "Finetune/Loss (Raw)": 0.8953972458839417, "Finetune/Step": 4811, "Finetune/Step Time": 4.761457057669759} +{"Finetune/Learning Rate": 2.8772178738408985e-07, "Finetune/Loss": 0.8486229181289673, "Finetune/Loss (Raw)": 0.87984299659729, "Finetune/Step": 4812, "Finetune/Step Time": 4.76151336170733} +{"Finetune/Learning Rate": 2.862400759312522e-07, "Finetune/Loss": 0.8451642990112305, "Finetune/Loss (Raw)": 0.4765048325061798, "Finetune/Step": 4813, "Finetune/Step Time": 4.792190508916974} +{"Finetune/Learning Rate": 2.847621341989959e-07, "Finetune/Loss": 0.8461968898773193, "Finetune/Loss (Raw)": 0.8796913623809814, "Finetune/Step": 4814, "Finetune/Step Time": 4.782107533887029} +{"Finetune/Learning Rate": 2.832879627608831e-07, "Finetune/Loss": 0.8466131687164307, "Finetune/Loss (Raw)": 0.9334160685539246, "Finetune/Step": 4815, "Finetune/Step Time": 4.779860781505704} +{"Finetune/Learning Rate": 2.8181756218900626e-07, "Finetune/Loss": 0.8444291949272156, "Finetune/Loss (Raw)": 0.8130545616149902, "Finetune/Step": 4816, "Finetune/Step Time": 4.796356404200196} +{"Finetune/Learning Rate": 2.803509330540033e-07, "Finetune/Loss": 0.8436682224273682, "Finetune/Loss (Raw)": 0.8907043933868408, "Finetune/Step": 4817, "Finetune/Step Time": 4.80098363943398} +{"Finetune/Learning Rate": 2.788880759250423e-07, "Finetune/Loss": 0.8450492024421692, "Finetune/Loss (Raw)": 0.9633992314338684, "Finetune/Step": 4818, "Finetune/Step Time": 4.79124041646719} +{"Finetune/Learning Rate": 2.7742899136983026e-07, "Finetune/Loss": 0.8448835611343384, "Finetune/Loss (Raw)": 0.9520576596260071, "Finetune/Step": 4819, "Finetune/Step Time": 4.790782315656543} +{"Finetune/Learning Rate": 2.759736799546109e-07, "Finetune/Loss": 0.8440561294555664, "Finetune/Loss (Raw)": 0.9120036363601685, "Finetune/Step": 4820, "Finetune/Step Time": 4.793278964236379} +{"Finetune/Learning Rate": 2.745221422441602e-07, "Finetune/Loss": 0.8444299697875977, "Finetune/Loss (Raw)": 0.9439547061920166, "Finetune/Step": 4821, "Finetune/Step Time": 4.78039113432169} +{"Finetune/Learning Rate": 2.73074378801792e-07, "Finetune/Loss": 0.8435961008071899, "Finetune/Loss (Raw)": 0.8812826871871948, "Finetune/Step": 4822, "Finetune/Step Time": 4.788500975817442} +{"Finetune/Learning Rate": 2.7163039018935357e-07, "Finetune/Loss": 0.8432259559631348, "Finetune/Loss (Raw)": 0.9398894309997559, "Finetune/Step": 4823, "Finetune/Step Time": 4.787309106439352} +{"Finetune/Learning Rate": 2.701901769672322e-07, "Finetune/Loss": 0.8393287658691406, "Finetune/Loss (Raw)": 0.2939407527446747, "Finetune/Step": 4824, "Finetune/Step Time": 4.807494226843119} +{"Finetune/Learning Rate": 2.687537396943451e-07, "Finetune/Loss": 0.8392331004142761, "Finetune/Loss (Raw)": 0.8658227324485779, "Finetune/Step": 4825, "Finetune/Step Time": 4.812216106802225} +{"Finetune/Learning Rate": 2.673210789281433e-07, "Finetune/Loss": 0.839074969291687, "Finetune/Loss (Raw)": 0.9067657589912415, "Finetune/Step": 4826, "Finetune/Step Time": 4.811075320467353} +{"Finetune/Learning Rate": 2.658921952246163e-07, "Finetune/Loss": 0.8388718962669373, "Finetune/Loss (Raw)": 0.9377264380455017, "Finetune/Step": 4827, "Finetune/Step Time": 4.806154567748308} +{"Finetune/Learning Rate": 2.6446708913828725e-07, "Finetune/Loss": 0.8393199443817139, "Finetune/Loss (Raw)": 0.9435099959373474, "Finetune/Step": 4828, "Finetune/Step Time": 4.813294414430857} +{"Finetune/Learning Rate": 2.6304576122221035e-07, "Finetune/Loss": 0.8387490510940552, "Finetune/Loss (Raw)": 0.8410552144050598, "Finetune/Step": 4829, "Finetune/Step Time": 4.819415856152773} +{"Finetune/Learning Rate": 2.616282120279745e-07, "Finetune/Loss": 0.8390179872512817, "Finetune/Loss (Raw)": 0.8801717162132263, "Finetune/Step": 4830, "Finetune/Step Time": 4.822874829173088} +{"Finetune/Learning Rate": 2.602144421057051e-07, "Finetune/Loss": 0.834237813949585, "Finetune/Loss (Raw)": 0.3540022373199463, "Finetune/Step": 4831, "Finetune/Step Time": 4.861696677282453} +{"Finetune/Learning Rate": 2.5880445200405667e-07, "Finetune/Loss": 0.8343411684036255, "Finetune/Loss (Raw)": 0.7577666640281677, "Finetune/Step": 4832, "Finetune/Step Time": 4.879999002441764} +{"Finetune/Learning Rate": 2.573982422702193e-07, "Finetune/Loss": 0.8334649801254272, "Finetune/Loss (Raw)": 0.8445597290992737, "Finetune/Step": 4833, "Finetune/Step Time": 4.883053008466959} +{"Finetune/Learning Rate": 2.559958134499163e-07, "Finetune/Loss": 0.8347803950309753, "Finetune/Loss (Raw)": 0.99183589220047, "Finetune/Step": 4834, "Finetune/Step Time": 4.887482114136219} +{"Finetune/Learning Rate": 2.545971660874036e-07, "Finetune/Loss": 0.8362325429916382, "Finetune/Loss (Raw)": 0.9382043480873108, "Finetune/Step": 4835, "Finetune/Step Time": 4.884351301938295} +{"Finetune/Learning Rate": 2.532023007254658e-07, "Finetune/Loss": 0.8377550840377808, "Finetune/Loss (Raw)": 0.9637786149978638, "Finetune/Step": 4836, "Finetune/Step Time": 4.87702870555222} +{"Finetune/Learning Rate": 2.518112179054266e-07, "Finetune/Loss": 0.8387273550033569, "Finetune/Loss (Raw)": 0.8591769933700562, "Finetune/Step": 4837, "Finetune/Step Time": 4.858868733048439} +{"Finetune/Learning Rate": 2.504239181671353e-07, "Finetune/Loss": 0.8384182453155518, "Finetune/Loss (Raw)": 0.9613917469978333, "Finetune/Step": 4838, "Finetune/Step Time": 4.860065512359142} +{"Finetune/Learning Rate": 2.490404020489767e-07, "Finetune/Loss": 0.8380553722381592, "Finetune/Loss (Raw)": 0.8118887543678284, "Finetune/Step": 4839, "Finetune/Step Time": 4.855855202302337} +{"Finetune/Learning Rate": 2.476606700878681e-07, "Finetune/Loss": 0.8388557434082031, "Finetune/Loss (Raw)": 0.9003382921218872, "Finetune/Step": 4840, "Finetune/Step Time": 4.851729642599821} +{"Finetune/Learning Rate": 2.4628472281925443e-07, "Finetune/Loss": 0.8377782702445984, "Finetune/Loss (Raw)": 0.8158773183822632, "Finetune/Step": 4841, "Finetune/Step Time": 4.858011361211538} +{"Finetune/Learning Rate": 2.449125607771141e-07, "Finetune/Loss": 0.8380019664764404, "Finetune/Loss (Raw)": 0.8559321165084839, "Finetune/Step": 4842, "Finetune/Step Time": 4.868370410054922} +{"Finetune/Learning Rate": 2.435441844939557e-07, "Finetune/Loss": 0.8390058279037476, "Finetune/Loss (Raw)": 1.0472255945205688, "Finetune/Step": 4843, "Finetune/Step Time": 4.864267455413938} +{"Finetune/Learning Rate": 2.4217959450082207e-07, "Finetune/Loss": 0.8388136029243469, "Finetune/Loss (Raw)": 0.8254164457321167, "Finetune/Step": 4844, "Finetune/Step Time": 4.88425668142736} +{"Finetune/Learning Rate": 2.408187913272841e-07, "Finetune/Loss": 0.839708685874939, "Finetune/Loss (Raw)": 1.0087817907333374, "Finetune/Step": 4845, "Finetune/Step Time": 4.8809564635157585} +{"Finetune/Learning Rate": 2.3946177550143835e-07, "Finetune/Loss": 0.8420189619064331, "Finetune/Loss (Raw)": 0.9443495273590088, "Finetune/Step": 4846, "Finetune/Step Time": 4.892227889969945} +{"Finetune/Learning Rate": 2.3810854754992008e-07, "Finetune/Loss": 0.8386421203613281, "Finetune/Loss (Raw)": 0.3227277100086212, "Finetune/Step": 4847, "Finetune/Step Time": 4.928423447534442} +{"Finetune/Learning Rate": 2.3675910799789036e-07, "Finetune/Loss": 0.8400242328643799, "Finetune/Loss (Raw)": 0.7651132941246033, "Finetune/Step": 4848, "Finetune/Step Time": 4.885956922546029} +{"Finetune/Learning Rate": 2.354134573690392e-07, "Finetune/Loss": 0.8397133350372314, "Finetune/Loss (Raw)": 0.7571350932121277, "Finetune/Step": 4849, "Finetune/Step Time": 4.901184985414147} +{"Finetune/Learning Rate": 2.3407159618558772e-07, "Finetune/Loss": 0.8406267762184143, "Finetune/Loss (Raw)": 0.9556826949119568, "Finetune/Step": 4850, "Finetune/Step Time": 4.909692311659455} +{"Finetune/Learning Rate": 2.3273352496828605e-07, "Finetune/Loss": 0.8408934473991394, "Finetune/Loss (Raw)": 0.9122725129127502, "Finetune/Step": 4851, "Finetune/Step Time": 4.9144557397812605} +{"Finetune/Learning Rate": 2.3139924423641325e-07, "Finetune/Loss": 0.8409807085990906, "Finetune/Loss (Raw)": 0.8343189358711243, "Finetune/Step": 4852, "Finetune/Step Time": 4.906957598403096} +{"Finetune/Learning Rate": 2.3006875450777733e-07, "Finetune/Loss": 0.8422342538833618, "Finetune/Loss (Raw)": 1.0136224031448364, "Finetune/Step": 4853, "Finetune/Step Time": 4.918715478852391} +{"Finetune/Learning Rate": 2.287420562987175e-07, "Finetune/Loss": 0.8412041664123535, "Finetune/Loss (Raw)": 0.6162577271461487, "Finetune/Step": 4854, "Finetune/Step Time": 4.921646900475025} +{"Finetune/Learning Rate": 2.2741915012409744e-07, "Finetune/Loss": 0.8417904376983643, "Finetune/Loss (Raw)": 0.9020212888717651, "Finetune/Step": 4855, "Finetune/Step Time": 4.923089234158397} +{"Finetune/Learning Rate": 2.2610003649731093e-07, "Finetune/Loss": 0.841820240020752, "Finetune/Loss (Raw)": 0.8970518112182617, "Finetune/Step": 4856, "Finetune/Step Time": 4.917435757815838} +{"Finetune/Learning Rate": 2.2478471593028073e-07, "Finetune/Loss": 0.8410919308662415, "Finetune/Loss (Raw)": 0.9603956937789917, "Finetune/Step": 4857, "Finetune/Step Time": 4.904820436611772} +{"Finetune/Learning Rate": 2.234731889334585e-07, "Finetune/Loss": 0.8419645428657532, "Finetune/Loss (Raw)": 0.9978011846542358, "Finetune/Step": 4858, "Finetune/Step Time": 4.914865892380476} +{"Finetune/Learning Rate": 2.221654560158204e-07, "Finetune/Loss": 0.8426997661590576, "Finetune/Loss (Raw)": 0.8835407495498657, "Finetune/Step": 4859, "Finetune/Step Time": 4.906860485672951} +{"Finetune/Learning Rate": 2.2086151768487273e-07, "Finetune/Loss": 0.8422311544418335, "Finetune/Loss (Raw)": 0.7893705368041992, "Finetune/Step": 4860, "Finetune/Step Time": 4.90561393275857} +{"Finetune/Learning Rate": 2.195613744466485e-07, "Finetune/Loss": 0.8417410850524902, "Finetune/Loss (Raw)": 0.8552775979042053, "Finetune/Step": 4861, "Finetune/Step Time": 4.914911825209856} +{"Finetune/Learning Rate": 2.182650268057085e-07, "Finetune/Loss": 0.8430662155151367, "Finetune/Loss (Raw)": 0.9591363072395325, "Finetune/Step": 4862, "Finetune/Step Time": 4.899638148024678} +{"Finetune/Learning Rate": 2.1697247526513698e-07, "Finetune/Loss": 0.8433306813240051, "Finetune/Loss (Raw)": 0.9877641797065735, "Finetune/Step": 4863, "Finetune/Step Time": 4.904148081317544} +{"Finetune/Learning Rate": 2.1568372032655384e-07, "Finetune/Loss": 0.844989538192749, "Finetune/Loss (Raw)": 0.9245314598083496, "Finetune/Step": 4864, "Finetune/Step Time": 4.909575786441565} +{"Finetune/Learning Rate": 2.143987624900945e-07, "Finetune/Loss": 0.8459471464157104, "Finetune/Loss (Raw)": 0.9057343602180481, "Finetune/Step": 4865, "Finetune/Step Time": 4.909605456516147} +{"Finetune/Learning Rate": 2.1311760225442678e-07, "Finetune/Loss": 0.844327449798584, "Finetune/Loss (Raw)": 0.7718374133110046, "Finetune/Step": 4866, "Finetune/Step Time": 4.901334851980209} +{"Finetune/Learning Rate": 2.118402401167452e-07, "Finetune/Loss": 0.8409651517868042, "Finetune/Loss (Raw)": 0.35145097970962524, "Finetune/Step": 4867, "Finetune/Step Time": 4.937026541680098} +{"Finetune/Learning Rate": 2.1056667657276876e-07, "Finetune/Loss": 0.8404731154441833, "Finetune/Loss (Raw)": 0.7602031826972961, "Finetune/Step": 4868, "Finetune/Step Time": 4.940808845683932} +{"Finetune/Learning Rate": 2.0929691211674319e-07, "Finetune/Loss": 0.8411499261856079, "Finetune/Loss (Raw)": 0.7763731479644775, "Finetune/Step": 4869, "Finetune/Step Time": 4.944989969953895} +{"Finetune/Learning Rate": 2.0803094724143879e-07, "Finetune/Loss": 0.8412763476371765, "Finetune/Loss (Raw)": 0.7849031090736389, "Finetune/Step": 4870, "Finetune/Step Time": 4.930383559316397} +{"Finetune/Learning Rate": 2.0676878243815145e-07, "Finetune/Loss": 0.8428807258605957, "Finetune/Loss (Raw)": 0.9355561137199402, "Finetune/Step": 4871, "Finetune/Step Time": 4.924852458760142} +{"Finetune/Learning Rate": 2.055104181967027e-07, "Finetune/Loss": 0.841576337814331, "Finetune/Loss (Raw)": 0.8359075784683228, "Finetune/Step": 4872, "Finetune/Step Time": 4.924833094701171} +{"Finetune/Learning Rate": 2.0425585500543854e-07, "Finetune/Loss": 0.8418182134628296, "Finetune/Loss (Raw)": 0.805409848690033, "Finetune/Step": 4873, "Finetune/Step Time": 4.921904729679227} +{"Finetune/Learning Rate": 2.0300509335123286e-07, "Finetune/Loss": 0.8408883213996887, "Finetune/Loss (Raw)": 0.6722062230110168, "Finetune/Step": 4874, "Finetune/Step Time": 4.905777966603637} +{"Finetune/Learning Rate": 2.0175813371948072e-07, "Finetune/Loss": 0.8393633365631104, "Finetune/Loss (Raw)": 0.8286867141723633, "Finetune/Step": 4875, "Finetune/Step Time": 4.914417557418346} +{"Finetune/Learning Rate": 2.0051497659409947e-07, "Finetune/Loss": 0.839301586151123, "Finetune/Loss (Raw)": 0.9253931045532227, "Finetune/Step": 4876, "Finetune/Step Time": 4.907890746369958} +{"Finetune/Learning Rate": 1.9927562245753874e-07, "Finetune/Loss": 0.8404276371002197, "Finetune/Loss (Raw)": 0.9798634052276611, "Finetune/Step": 4877, "Finetune/Step Time": 4.909682458266616} +{"Finetune/Learning Rate": 1.9804007179076602e-07, "Finetune/Loss": 0.8402661085128784, "Finetune/Loss (Raw)": 0.8417729735374451, "Finetune/Step": 4878, "Finetune/Step Time": 4.911347810178995} +{"Finetune/Learning Rate": 1.968083250732744e-07, "Finetune/Loss": 0.8405107259750366, "Finetune/Loss (Raw)": 0.9158327579498291, "Finetune/Step": 4879, "Finetune/Step Time": 4.887165879830718} +{"Finetune/Learning Rate": 1.9558038278308044e-07, "Finetune/Loss": 0.8409168124198914, "Finetune/Loss (Raw)": 0.9496511816978455, "Finetune/Step": 4880, "Finetune/Step Time": 4.883761854842305} +{"Finetune/Learning Rate": 1.9435624539672515e-07, "Finetune/Loss": 0.8432143330574036, "Finetune/Loss (Raw)": 1.059740424156189, "Finetune/Step": 4881, "Finetune/Step Time": 4.8838759288191795} +{"Finetune/Learning Rate": 1.9313591338927185e-07, "Finetune/Loss": 0.8420524597167969, "Finetune/Loss (Raw)": 0.7704172730445862, "Finetune/Step": 4882, "Finetune/Step Time": 4.8863756619393826} +{"Finetune/Learning Rate": 1.9191938723430614e-07, "Finetune/Loss": 0.8406524062156677, "Finetune/Loss (Raw)": 0.7473194599151611, "Finetune/Step": 4883, "Finetune/Step Time": 4.877019176259637} +{"Finetune/Learning Rate": 1.9070666740394151e-07, "Finetune/Loss": 0.841477632522583, "Finetune/Loss (Raw)": 0.9794409275054932, "Finetune/Step": 4884, "Finetune/Step Time": 4.879436714574695} +{"Finetune/Learning Rate": 1.8949775436880924e-07, "Finetune/Loss": 0.841278612613678, "Finetune/Loss (Raw)": 0.843818724155426, "Finetune/Step": 4885, "Finetune/Step Time": 4.882666846737266} +{"Finetune/Learning Rate": 1.8829264859806296e-07, "Finetune/Loss": 0.8422213196754456, "Finetune/Loss (Raw)": 0.9711530804634094, "Finetune/Step": 4886, "Finetune/Step Time": 4.88960999250412} +{"Finetune/Learning Rate": 1.8709135055938298e-07, "Finetune/Loss": 0.8426387906074524, "Finetune/Loss (Raw)": 0.8862269520759583, "Finetune/Step": 4887, "Finetune/Step Time": 4.896205309778452} +{"Finetune/Learning Rate": 1.8589386071896975e-07, "Finetune/Loss": 0.8431805372238159, "Finetune/Loss (Raw)": 0.9332351684570312, "Finetune/Step": 4888, "Finetune/Step Time": 4.9063990116119385} +{"Finetune/Learning Rate": 1.8470017954154262e-07, "Finetune/Loss": 0.8440993428230286, "Finetune/Loss (Raw)": 0.8722402453422546, "Finetune/Step": 4889, "Finetune/Step Time": 4.894388223066926} +{"Finetune/Learning Rate": 1.8351030749034992e-07, "Finetune/Loss": 0.8446104526519775, "Finetune/Loss (Raw)": 0.8532950282096863, "Finetune/Step": 4890, "Finetune/Step Time": 4.9139538537710905} +{"Finetune/Learning Rate": 1.8232424502715672e-07, "Finetune/Loss": 0.8439019322395325, "Finetune/Loss (Raw)": 0.7825355529785156, "Finetune/Step": 4891, "Finetune/Step Time": 4.9109125174582005} +{"Finetune/Learning Rate": 1.811419926122493e-07, "Finetune/Loss": 0.8441911339759827, "Finetune/Loss (Raw)": 0.8786705136299133, "Finetune/Step": 4892, "Finetune/Step Time": 4.906461054459214} +{"Finetune/Learning Rate": 1.7996355070443727e-07, "Finetune/Loss": 0.8459527492523193, "Finetune/Loss (Raw)": 0.996676504611969, "Finetune/Step": 4893, "Finetune/Step Time": 4.906333917751908} +{"Finetune/Learning Rate": 1.787889197610515e-07, "Finetune/Loss": 0.8444094657897949, "Finetune/Loss (Raw)": 0.824647068977356, "Finetune/Step": 4894, "Finetune/Step Time": 4.918218202888966} +{"Finetune/Learning Rate": 1.776181002379429e-07, "Finetune/Loss": 0.8436229825019836, "Finetune/Loss (Raw)": 0.7814785242080688, "Finetune/Step": 4895, "Finetune/Step Time": 4.914659280329943} +{"Finetune/Learning Rate": 1.7645109258948245e-07, "Finetune/Loss": 0.8445875644683838, "Finetune/Loss (Raw)": 0.9275416731834412, "Finetune/Step": 4896, "Finetune/Step Time": 4.907288769260049} +{"Finetune/Learning Rate": 1.7528789726856566e-07, "Finetune/Loss": 0.8435889482498169, "Finetune/Loss (Raw)": 0.6965675354003906, "Finetune/Step": 4897, "Finetune/Step Time": 4.913602966815233} +{"Finetune/Learning Rate": 1.7412851472660585e-07, "Finetune/Loss": 0.8445960879325867, "Finetune/Loss (Raw)": 0.7623305320739746, "Finetune/Step": 4898, "Finetune/Step Time": 4.915417006239295} +{"Finetune/Learning Rate": 1.7297294541353317e-07, "Finetune/Loss": 0.8444512486457825, "Finetune/Loss (Raw)": 0.9716810584068298, "Finetune/Step": 4899, "Finetune/Step Time": 4.912825424224138} +{"Finetune/Learning Rate": 1.7182118977780548e-07, "Finetune/Loss": 0.8451453447341919, "Finetune/Loss (Raw)": 0.853712260723114, "Finetune/Step": 4900, "Finetune/Step Time": 4.899753790348768} +{"Finetune/Learning Rate": 1.706732482663942e-07, "Finetune/Loss": 0.846679151058197, "Finetune/Loss (Raw)": 0.8184102773666382, "Finetune/Step": 4901, "Finetune/Step Time": 4.89824490621686} +{"Finetune/Learning Rate": 1.6952912132479404e-07, "Finetune/Loss": 0.8462114930152893, "Finetune/Loss (Raw)": 0.8190053105354309, "Finetune/Step": 4902, "Finetune/Step Time": 4.897560715675354} +{"Finetune/Learning Rate": 1.6838880939701875e-07, "Finetune/Loss": 0.8452140092849731, "Finetune/Loss (Raw)": 0.8339377045631409, "Finetune/Step": 4903, "Finetune/Step Time": 4.89143948443234} +{"Finetune/Learning Rate": 1.6725231292559985e-07, "Finetune/Loss": 0.8456275463104248, "Finetune/Loss (Raw)": 0.8526997566223145, "Finetune/Step": 4904, "Finetune/Step Time": 4.876591389998794} +{"Finetune/Learning Rate": 1.661196323515901e-07, "Finetune/Loss": 0.8459694385528564, "Finetune/Loss (Raw)": 0.9280633330345154, "Finetune/Step": 4905, "Finetune/Step Time": 4.880381762981415} +{"Finetune/Learning Rate": 1.6499076811456015e-07, "Finetune/Loss": 0.850291907787323, "Finetune/Loss (Raw)": 0.8415433168411255, "Finetune/Step": 4906, "Finetune/Step Time": 4.8471838887780905} +{"Finetune/Learning Rate": 1.6386572065260065e-07, "Finetune/Loss": 0.8483929634094238, "Finetune/Loss (Raw)": 0.7671661376953125, "Finetune/Step": 4907, "Finetune/Step Time": 4.854869801551104} +{"Finetune/Learning Rate": 1.6274449040232233e-07, "Finetune/Loss": 0.8488296270370483, "Finetune/Loss (Raw)": 0.8866784572601318, "Finetune/Step": 4908, "Finetune/Step Time": 4.872415205463767} +{"Finetune/Learning Rate": 1.6162707779884933e-07, "Finetune/Loss": 0.8492417335510254, "Finetune/Loss (Raw)": 0.8530145883560181, "Finetune/Step": 4909, "Finetune/Step Time": 4.863438222557306} +{"Finetune/Learning Rate": 1.6051348327583038e-07, "Finetune/Loss": 0.8497462272644043, "Finetune/Loss (Raw)": 0.896795392036438, "Finetune/Step": 4910, "Finetune/Step Time": 4.85285340808332} +{"Finetune/Learning Rate": 1.5940370726542864e-07, "Finetune/Loss": 0.8534594178199768, "Finetune/Loss (Raw)": 0.8414753675460815, "Finetune/Step": 4911, "Finetune/Step Time": 4.816542167216539} +{"Finetune/Learning Rate": 1.5829775019832628e-07, "Finetune/Loss": 0.8535953760147095, "Finetune/Loss (Raw)": 0.8990688920021057, "Finetune/Step": 4912, "Finetune/Step Time": 4.823989879339933} +{"Finetune/Learning Rate": 1.571956125037244e-07, "Finetune/Loss": 0.8527388572692871, "Finetune/Loss (Raw)": 0.9294449090957642, "Finetune/Step": 4913, "Finetune/Step Time": 4.832406433299184} +{"Finetune/Learning Rate": 1.5609729460934087e-07, "Finetune/Loss": 0.8489605784416199, "Finetune/Loss (Raw)": 0.4084619879722595, "Finetune/Step": 4914, "Finetune/Step Time": 4.865135518833995} +{"Finetune/Learning Rate": 1.550027969414114e-07, "Finetune/Loss": 0.8481502532958984, "Finetune/Loss (Raw)": 0.7451555728912354, "Finetune/Step": 4915, "Finetune/Step Time": 4.8681483548134565} +{"Finetune/Learning Rate": 1.5391211992468736e-07, "Finetune/Loss": 0.8483955264091492, "Finetune/Loss (Raw)": 0.8512395024299622, "Finetune/Step": 4916, "Finetune/Step Time": 4.841332985088229} +{"Finetune/Learning Rate": 1.5282526398244236e-07, "Finetune/Loss": 0.8460450172424316, "Finetune/Loss (Raw)": 0.41073188185691833, "Finetune/Step": 4917, "Finetune/Step Time": 4.872576218098402} +{"Finetune/Learning Rate": 1.5174222953646344e-07, "Finetune/Loss": 0.8455825448036194, "Finetune/Loss (Raw)": 0.8834749460220337, "Finetune/Step": 4918, "Finetune/Step Time": 4.866067750379443} +{"Finetune/Learning Rate": 1.5066301700705334e-07, "Finetune/Loss": 0.8499492406845093, "Finetune/Loss (Raw)": 0.8609114289283752, "Finetune/Step": 4919, "Finetune/Step Time": 4.838003987446427} +{"Finetune/Learning Rate": 1.4958762681303362e-07, "Finetune/Loss": 0.8490983247756958, "Finetune/Loss (Raw)": 0.9064944982528687, "Finetune/Step": 4920, "Finetune/Step Time": 4.834317812696099} +{"Finetune/Learning Rate": 1.4851605937174384e-07, "Finetune/Loss": 0.849089503288269, "Finetune/Loss (Raw)": 0.9152248501777649, "Finetune/Step": 4921, "Finetune/Step Time": 4.84771934337914} +{"Finetune/Learning Rate": 1.474483150990369e-07, "Finetune/Loss": 0.8486385345458984, "Finetune/Loss (Raw)": 0.7562368512153625, "Finetune/Step": 4922, "Finetune/Step Time": 4.848717857152224} +{"Finetune/Learning Rate": 1.4638439440928354e-07, "Finetune/Loss": 0.8487887382507324, "Finetune/Loss (Raw)": 0.8091385364532471, "Finetune/Step": 4923, "Finetune/Step Time": 4.822745382785797} +{"Finetune/Learning Rate": 1.4532429771537237e-07, "Finetune/Loss": 0.8481659293174744, "Finetune/Loss (Raw)": 0.8633679747581482, "Finetune/Step": 4924, "Finetune/Step Time": 4.819152273237705} +{"Finetune/Learning Rate": 1.4426802542870433e-07, "Finetune/Loss": 0.8474988341331482, "Finetune/Loss (Raw)": 0.8254004716873169, "Finetune/Step": 4925, "Finetune/Step Time": 4.813250759616494} +{"Finetune/Learning Rate": 1.4321557795919705e-07, "Finetune/Loss": 0.8483558297157288, "Finetune/Loss (Raw)": 0.8100433349609375, "Finetune/Step": 4926, "Finetune/Step Time": 4.797311395406723} +{"Finetune/Learning Rate": 1.4216695571528828e-07, "Finetune/Loss": 0.8478410243988037, "Finetune/Loss (Raw)": 0.8089953064918518, "Finetune/Step": 4927, "Finetune/Step Time": 4.795484023168683} +{"Finetune/Learning Rate": 1.411221591039269e-07, "Finetune/Loss": 0.8447344899177551, "Finetune/Loss (Raw)": 0.6561461091041565, "Finetune/Step": 4928, "Finetune/Step Time": 4.8186781238764524} +{"Finetune/Learning Rate": 1.4008118853057529e-07, "Finetune/Loss": 0.8440278768539429, "Finetune/Loss (Raw)": 0.8315247893333435, "Finetune/Step": 4929, "Finetune/Step Time": 4.82839747518301} +{"Finetune/Learning Rate": 1.3904404439921582e-07, "Finetune/Loss": 0.8439834117889404, "Finetune/Loss (Raw)": 0.8808832764625549, "Finetune/Step": 4930, "Finetune/Step Time": 4.828195048496127} +{"Finetune/Learning Rate": 1.3801072711234541e-07, "Finetune/Loss": 0.844200849533081, "Finetune/Loss (Raw)": 0.942886471748352, "Finetune/Step": 4931, "Finetune/Step Time": 4.826158115640283} +{"Finetune/Learning Rate": 1.369812370709711e-07, "Finetune/Loss": 0.844168484210968, "Finetune/Loss (Raw)": 0.9029730558395386, "Finetune/Step": 4932, "Finetune/Step Time": 4.823556985706091} +{"Finetune/Learning Rate": 1.3595557467461994e-07, "Finetune/Loss": 0.842985212802887, "Finetune/Loss (Raw)": 0.792969286441803, "Finetune/Step": 4933, "Finetune/Step Time": 4.834225362166762} +{"Finetune/Learning Rate": 1.3493374032133134e-07, "Finetune/Loss": 0.8431320190429688, "Finetune/Loss (Raw)": 0.9212055206298828, "Finetune/Step": 4934, "Finetune/Step Time": 4.829238783568144} +{"Finetune/Learning Rate": 1.3391573440765915e-07, "Finetune/Loss": 0.8438436388969421, "Finetune/Loss (Raw)": 0.9319336414337158, "Finetune/Step": 4935, "Finetune/Step Time": 4.828638885170221} +{"Finetune/Learning Rate": 1.329015573286696e-07, "Finetune/Loss": 0.8425019979476929, "Finetune/Loss (Raw)": 0.7870387434959412, "Finetune/Step": 4936, "Finetune/Step Time": 4.844267008826137} +{"Finetune/Learning Rate": 1.3189120947794897e-07, "Finetune/Loss": 0.8451829552650452, "Finetune/Loss (Raw)": 1.0077342987060547, "Finetune/Step": 4937, "Finetune/Step Time": 4.838034616783261} +{"Finetune/Learning Rate": 1.308846912475914e-07, "Finetune/Loss": 0.845862627029419, "Finetune/Loss (Raw)": 0.9146870970726013, "Finetune/Step": 4938, "Finetune/Step Time": 4.849709527567029} +{"Finetune/Learning Rate": 1.2988200302820552e-07, "Finetune/Loss": 0.841521143913269, "Finetune/Loss (Raw)": 0.3396911323070526, "Finetune/Step": 4939, "Finetune/Step Time": 4.8633845541626215} +{"Finetune/Learning Rate": 1.2888314520891676e-07, "Finetune/Loss": 0.8400178551673889, "Finetune/Loss (Raw)": 0.6874189972877502, "Finetune/Step": 4940, "Finetune/Step Time": 4.857460057362914} +{"Finetune/Learning Rate": 1.2788811817736168e-07, "Finetune/Loss": 0.8421614170074463, "Finetune/Loss (Raw)": 0.7508869171142578, "Finetune/Step": 4941, "Finetune/Step Time": 4.822989866137505} +{"Finetune/Learning Rate": 1.2689692231969364e-07, "Finetune/Loss": 0.8416463136672974, "Finetune/Loss (Raw)": 0.8137527108192444, "Finetune/Step": 4942, "Finetune/Step Time": 4.829828914254904} +{"Finetune/Learning Rate": 1.2590955802057047e-07, "Finetune/Loss": 0.8404860496520996, "Finetune/Loss (Raw)": 0.7849022746086121, "Finetune/Step": 4943, "Finetune/Step Time": 4.841133506968617} +{"Finetune/Learning Rate": 1.2492602566317457e-07, "Finetune/Loss": 0.841311514377594, "Finetune/Loss (Raw)": 0.9187221527099609, "Finetune/Step": 4944, "Finetune/Step Time": 4.827329970896244} +{"Finetune/Learning Rate": 1.2394632562919283e-07, "Finetune/Loss": 0.8417060971260071, "Finetune/Loss (Raw)": 0.941204309463501, "Finetune/Step": 4945, "Finetune/Step Time": 4.826954290270805} +{"Finetune/Learning Rate": 1.2297045829882892e-07, "Finetune/Loss": 0.8408824801445007, "Finetune/Loss (Raw)": 0.8579803705215454, "Finetune/Step": 4946, "Finetune/Step Time": 4.827613085508347} +{"Finetune/Learning Rate": 1.2199842405079766e-07, "Finetune/Loss": 0.8406227827072144, "Finetune/Loss (Raw)": 0.9188138842582703, "Finetune/Step": 4947, "Finetune/Step Time": 4.8292396161705256} +{"Finetune/Learning Rate": 1.2103022326232505e-07, "Finetune/Loss": 0.8398611545562744, "Finetune/Loss (Raw)": 0.8145220875740051, "Finetune/Step": 4948, "Finetune/Step Time": 4.826615899801254} +{"Finetune/Learning Rate": 1.2006585630915391e-07, "Finetune/Loss": 0.8387851715087891, "Finetune/Loss (Raw)": 0.806221067905426, "Finetune/Step": 4949, "Finetune/Step Time": 4.830132752656937} +{"Finetune/Learning Rate": 1.1910532356553261e-07, "Finetune/Loss": 0.8382178544998169, "Finetune/Loss (Raw)": 0.8086633682250977, "Finetune/Step": 4950, "Finetune/Step Time": 4.837089205160737} +{"Finetune/Learning Rate": 1.1814862540422744e-07, "Finetune/Loss": 0.8379461169242859, "Finetune/Loss (Raw)": 0.9051085710525513, "Finetune/Step": 4951, "Finetune/Step Time": 4.835787415504456} +{"Finetune/Learning Rate": 1.1719576219651585e-07, "Finetune/Loss": 0.8428287506103516, "Finetune/Loss (Raw)": 0.918927013874054, "Finetune/Step": 4952, "Finetune/Step Time": 4.8106234688311815} +{"Finetune/Learning Rate": 1.1624673431218092e-07, "Finetune/Loss": 0.8424749374389648, "Finetune/Loss (Raw)": 0.8205239176750183, "Finetune/Step": 4953, "Finetune/Step Time": 4.812008475884795} +{"Finetune/Learning Rate": 1.1530154211952583e-07, "Finetune/Loss": 0.8425605297088623, "Finetune/Loss (Raw)": 0.9177311658859253, "Finetune/Step": 4954, "Finetune/Step Time": 4.867906503379345} +{"Finetune/Learning Rate": 1.1436018598535937e-07, "Finetune/Loss": 0.8419845104217529, "Finetune/Loss (Raw)": 0.8639822006225586, "Finetune/Step": 4955, "Finetune/Step Time": 4.865169018507004} +{"Finetune/Learning Rate": 1.1342266627500375e-07, "Finetune/Loss": 0.8419415950775146, "Finetune/Loss (Raw)": 0.9380223155021667, "Finetune/Step": 4956, "Finetune/Step Time": 4.8528345208615065} +{"Finetune/Learning Rate": 1.1248898335229241e-07, "Finetune/Loss": 0.8429654240608215, "Finetune/Loss (Raw)": 0.9721085429191589, "Finetune/Step": 4957, "Finetune/Step Time": 4.842448893934488} +{"Finetune/Learning Rate": 1.1155913757956881e-07, "Finetune/Loss": 0.8424246311187744, "Finetune/Loss (Raw)": 0.8109446167945862, "Finetune/Step": 4958, "Finetune/Step Time": 4.854025132954121} +{"Finetune/Learning Rate": 1.1063312931768877e-07, "Finetune/Loss": 0.8430981636047363, "Finetune/Loss (Raw)": 0.4402236342430115, "Finetune/Step": 4959, "Finetune/Step Time": 4.860108535736799} +{"Finetune/Learning Rate": 1.0971095892601702e-07, "Finetune/Loss": 0.8436555862426758, "Finetune/Loss (Raw)": 0.8291190266609192, "Finetune/Step": 4960, "Finetune/Step Time": 4.833483027294278} +{"Finetune/Learning Rate": 1.0879262676243174e-07, "Finetune/Loss": 0.8433959484100342, "Finetune/Loss (Raw)": 0.81131511926651, "Finetune/Step": 4961, "Finetune/Step Time": 4.8368960209190845} +{"Finetune/Learning Rate": 1.0787813318331786e-07, "Finetune/Loss": 0.8428099155426025, "Finetune/Loss (Raw)": 0.9168363213539124, "Finetune/Step": 4962, "Finetune/Step Time": 4.8359390161931515} +{"Finetune/Learning Rate": 1.0696747854357259e-07, "Finetune/Loss": 0.8420889377593994, "Finetune/Loss (Raw)": 0.8459046483039856, "Finetune/Step": 4963, "Finetune/Step Time": 4.839966665953398} +{"Finetune/Learning Rate": 1.0606066319660435e-07, "Finetune/Loss": 0.8419338464736938, "Finetune/Loss (Raw)": 0.9439323544502258, "Finetune/Step": 4964, "Finetune/Step Time": 4.843946237117052} +{"Finetune/Learning Rate": 1.0515768749432942e-07, "Finetune/Loss": 0.8428228497505188, "Finetune/Loss (Raw)": 0.9729712009429932, "Finetune/Step": 4965, "Finetune/Step Time": 4.844734197482467} +{"Finetune/Learning Rate": 1.0425855178717636e-07, "Finetune/Loss": 0.8415469527244568, "Finetune/Loss (Raw)": 0.7980768084526062, "Finetune/Step": 4966, "Finetune/Step Time": 4.847995040938258} +{"Finetune/Learning Rate": 1.0336325642408052e-07, "Finetune/Loss": 0.8424758911132812, "Finetune/Loss (Raw)": 0.9307905435562134, "Finetune/Step": 4967, "Finetune/Step Time": 4.845500148832798} +{"Finetune/Learning Rate": 1.024718017524895e-07, "Finetune/Loss": 0.8423923254013062, "Finetune/Loss (Raw)": 0.889644205570221, "Finetune/Step": 4968, "Finetune/Step Time": 4.835257329046726} +{"Finetune/Learning Rate": 1.015841881183588e-07, "Finetune/Loss": 0.8428378701210022, "Finetune/Loss (Raw)": 0.8729032874107361, "Finetune/Step": 4969, "Finetune/Step Time": 4.838734025135636} +{"Finetune/Learning Rate": 1.0070041586615398e-07, "Finetune/Loss": 0.8434929847717285, "Finetune/Loss (Raw)": 0.939789891242981, "Finetune/Step": 4970, "Finetune/Step Time": 4.820344403386116} +{"Finetune/Learning Rate": 9.98204853388518e-08, "Finetune/Loss": 0.8431764841079712, "Finetune/Loss (Raw)": 1.0067061185836792, "Finetune/Step": 4971, "Finetune/Step Time": 4.832572769373655} +{"Finetune/Learning Rate": 9.894439687793466e-08, "Finetune/Loss": 0.8434773683547974, "Finetune/Loss (Raw)": 0.8639330267906189, "Finetune/Step": 4972, "Finetune/Step Time": 4.826573112979531} +{"Finetune/Learning Rate": 9.807215082339395e-08, "Finetune/Loss": 0.84336256980896, "Finetune/Loss (Raw)": 0.9940927624702454, "Finetune/Step": 4973, "Finetune/Step Time": 4.827003952115774} +{"Finetune/Learning Rate": 9.720374751373329e-08, "Finetune/Loss": 0.8383679389953613, "Finetune/Loss (Raw)": 0.305032879114151, "Finetune/Step": 4974, "Finetune/Step Time": 4.835049238055944} +{"Finetune/Learning Rate": 9.633918728596203e-08, "Finetune/Loss": 0.8421410322189331, "Finetune/Loss (Raw)": 0.8056808114051819, "Finetune/Step": 4975, "Finetune/Step Time": 4.803762432187796} +{"Finetune/Learning Rate": 9.547847047559955e-08, "Finetune/Loss": 0.8438829183578491, "Finetune/Loss (Raw)": 0.9880865216255188, "Finetune/Step": 4976, "Finetune/Step Time": 4.806799445301294} +{"Finetune/Learning Rate": 9.462159741667309e-08, "Finetune/Loss": 0.8438114523887634, "Finetune/Loss (Raw)": 0.7479786276817322, "Finetune/Step": 4977, "Finetune/Step Time": 4.806607414036989} +{"Finetune/Learning Rate": 9.376856844171889e-08, "Finetune/Loss": 0.8426206111907959, "Finetune/Loss (Raw)": 0.8032589554786682, "Finetune/Step": 4978, "Finetune/Step Time": 4.802543582394719} +{"Finetune/Learning Rate": 9.291938388178102e-08, "Finetune/Loss": 0.841862678527832, "Finetune/Loss (Raw)": 0.8152480721473694, "Finetune/Step": 4979, "Finetune/Step Time": 4.805025365203619} +{"Finetune/Learning Rate": 9.207404406640919e-08, "Finetune/Loss": 0.8408651351928711, "Finetune/Loss (Raw)": 0.7066450715065002, "Finetune/Step": 4980, "Finetune/Step Time": 4.798279361799359} +{"Finetune/Learning Rate": 9.123254932366655e-08, "Finetune/Loss": 0.8389225006103516, "Finetune/Loss (Raw)": 0.7649553418159485, "Finetune/Step": 4981, "Finetune/Step Time": 4.784148653969169} +{"Finetune/Learning Rate": 9.039489998011853e-08, "Finetune/Loss": 0.8416081666946411, "Finetune/Loss (Raw)": 0.960033118724823, "Finetune/Step": 4982, "Finetune/Step Time": 4.7793628461658955} +{"Finetune/Learning Rate": 8.956109636084065e-08, "Finetune/Loss": 0.8377816677093506, "Finetune/Loss (Raw)": 0.4122254550457001, "Finetune/Step": 4983, "Finetune/Step Time": 4.812930906191468} +{"Finetune/Learning Rate": 8.873113878941631e-08, "Finetune/Loss": 0.8368957042694092, "Finetune/Loss (Raw)": 0.7836428880691528, "Finetune/Step": 4984, "Finetune/Step Time": 4.829481475055218} +{"Finetune/Learning Rate": 8.790502758793562e-08, "Finetune/Loss": 0.8373768925666809, "Finetune/Loss (Raw)": 1.0219941139221191, "Finetune/Step": 4985, "Finetune/Step Time": 4.826478378847241} +{"Finetune/Learning Rate": 8.70827630769977e-08, "Finetune/Loss": 0.8350632190704346, "Finetune/Loss (Raw)": 0.7016519904136658, "Finetune/Step": 4986, "Finetune/Step Time": 4.818296853452921} +{"Finetune/Learning Rate": 8.626434557570507e-08, "Finetune/Loss": 0.8353307247161865, "Finetune/Loss (Raw)": 0.9177780747413635, "Finetune/Step": 4987, "Finetune/Step Time": 4.819794733077288} +{"Finetune/Learning Rate": 8.544977540167032e-08, "Finetune/Loss": 0.8365484476089478, "Finetune/Loss (Raw)": 0.9452410340309143, "Finetune/Step": 4988, "Finetune/Step Time": 4.82773164100945} +{"Finetune/Learning Rate": 8.463905287101282e-08, "Finetune/Loss": 0.8370752334594727, "Finetune/Loss (Raw)": 0.9226983785629272, "Finetune/Step": 4989, "Finetune/Step Time": 4.818934228271246} +{"Finetune/Learning Rate": 8.383217829835755e-08, "Finetune/Loss": 0.8363022804260254, "Finetune/Loss (Raw)": 0.8602105379104614, "Finetune/Step": 4990, "Finetune/Step Time": 4.827371422201395} +{"Finetune/Learning Rate": 8.302915199683737e-08, "Finetune/Loss": 0.8350658416748047, "Finetune/Loss (Raw)": 0.8294918537139893, "Finetune/Step": 4991, "Finetune/Step Time": 4.8216714058071375} +{"Finetune/Learning Rate": 8.222997427809076e-08, "Finetune/Loss": 0.8355424404144287, "Finetune/Loss (Raw)": 0.9855374097824097, "Finetune/Step": 4992, "Finetune/Step Time": 4.826852133497596} +{"Finetune/Learning Rate": 8.143464545226298e-08, "Finetune/Loss": 0.8331472873687744, "Finetune/Loss (Raw)": 0.5991545915603638, "Finetune/Step": 4993, "Finetune/Step Time": 4.834843747317791} +{"Finetune/Learning Rate": 8.064316582800491e-08, "Finetune/Loss": 0.8336549401283264, "Finetune/Loss (Raw)": 0.8368199467658997, "Finetune/Step": 4994, "Finetune/Step Time": 4.841583659872413} +{"Finetune/Learning Rate": 7.98555357124764e-08, "Finetune/Loss": 0.8357481360435486, "Finetune/Loss (Raw)": 0.6193757653236389, "Finetune/Step": 4995, "Finetune/Step Time": 4.809810657054186} +{"Finetune/Learning Rate": 7.907175541133849e-08, "Finetune/Loss": 0.8376878499984741, "Finetune/Loss (Raw)": 1.0084896087646484, "Finetune/Step": 4996, "Finetune/Step Time": 4.806400841102004} +{"Finetune/Learning Rate": 7.829182522876344e-08, "Finetune/Loss": 0.8388494253158569, "Finetune/Loss (Raw)": 0.9250568747520447, "Finetune/Step": 4997, "Finetune/Step Time": 4.791225243359804} +{"Finetune/Learning Rate": 7.75157454674258e-08, "Finetune/Loss": 0.8412907123565674, "Finetune/Loss (Raw)": 1.0973833799362183, "Finetune/Step": 4998, "Finetune/Step Time": 4.789041306823492} +{"Finetune/Learning Rate": 7.674351642850797e-08, "Finetune/Loss": 0.8417144417762756, "Finetune/Loss (Raw)": 0.9897943139076233, "Finetune/Step": 4999, "Finetune/Step Time": 4.784534079954028} +{"Finetune/Learning Rate": 7.597513841169469e-08, "Finetune/Loss": 0.8379404544830322, "Finetune/Loss (Raw)": 0.3528382480144501, "Finetune/Step": 5000, "Finetune/Step Time": 4.8209173660725355} +{"Finetune/Learning Rate": 7.521061171518184e-08, "Finetune/Loss": 0.8390239477157593, "Finetune/Loss (Raw)": 0.9440978765487671, "Finetune/Step": 5001, "Finetune/Step Time": 4.802852865308523} +{"Finetune/Learning Rate": 7.444993663566325e-08, "Finetune/Loss": 0.8390745520591736, "Finetune/Loss (Raw)": 0.6786860823631287, "Finetune/Step": 5002, "Finetune/Step Time": 4.8045689929276705} +{"Finetune/Learning Rate": 7.369311346834496e-08, "Finetune/Loss": 0.8388504981994629, "Finetune/Loss (Raw)": 0.8000093102455139, "Finetune/Step": 5003, "Finetune/Step Time": 4.792878206819296} +{"Finetune/Learning Rate": 7.294014250693538e-08, "Finetune/Loss": 0.837441623210907, "Finetune/Loss (Raw)": 0.7450559139251709, "Finetune/Step": 5004, "Finetune/Step Time": 4.806147096678615} +{"Finetune/Learning Rate": 7.21910240436463e-08, "Finetune/Loss": 0.8365132808685303, "Finetune/Loss (Raw)": 0.8610295057296753, "Finetune/Step": 5005, "Finetune/Step Time": 4.80806490778923} +{"Finetune/Learning Rate": 7.144575836919632e-08, "Finetune/Loss": 0.8368351459503174, "Finetune/Loss (Raw)": 0.8829753398895264, "Finetune/Step": 5006, "Finetune/Step Time": 4.799945110455155} +{"Finetune/Learning Rate": 7.07043457728096e-08, "Finetune/Loss": 0.8357691764831543, "Finetune/Loss (Raw)": 0.7793852090835571, "Finetune/Step": 5007, "Finetune/Step Time": 4.809979815036058} +{"Finetune/Learning Rate": 6.996678654221267e-08, "Finetune/Loss": 0.8344855308532715, "Finetune/Loss (Raw)": 0.7853472232818604, "Finetune/Step": 5008, "Finetune/Step Time": 4.816720586270094} +{"Finetune/Learning Rate": 6.923308096363879e-08, "Finetune/Loss": 0.834050714969635, "Finetune/Loss (Raw)": 1.0040847063064575, "Finetune/Step": 5009, "Finetune/Step Time": 4.81362190656364} +{"Finetune/Learning Rate": 6.850322932182352e-08, "Finetune/Loss": 0.8354121446609497, "Finetune/Loss (Raw)": 0.9446853399276733, "Finetune/Step": 5010, "Finetune/Step Time": 4.809928888455033} +{"Finetune/Learning Rate": 6.777723190000918e-08, "Finetune/Loss": 0.83632493019104, "Finetune/Loss (Raw)": 0.8641506433486938, "Finetune/Step": 5011, "Finetune/Step Time": 4.811120731756091} +{"Finetune/Learning Rate": 6.705508897994151e-08, "Finetune/Loss": 0.8352834582328796, "Finetune/Loss (Raw)": 0.8461281657218933, "Finetune/Step": 5012, "Finetune/Step Time": 4.809415824711323} +{"Finetune/Learning Rate": 6.633680084186856e-08, "Finetune/Loss": 0.8344074487686157, "Finetune/Loss (Raw)": 0.7317008972167969, "Finetune/Step": 5013, "Finetune/Step Time": 4.808107495307922} +{"Finetune/Learning Rate": 6.562236776454512e-08, "Finetune/Loss": 0.8330252170562744, "Finetune/Loss (Raw)": 0.7942229509353638, "Finetune/Step": 5014, "Finetune/Step Time": 4.797626344487071} +{"Finetune/Learning Rate": 6.491179002522829e-08, "Finetune/Loss": 0.8321482539176941, "Finetune/Loss (Raw)": 0.7739764451980591, "Finetune/Step": 5015, "Finetune/Step Time": 4.786537969484925} +{"Finetune/Learning Rate": 6.420506789967973e-08, "Finetune/Loss": 0.83165442943573, "Finetune/Loss (Raw)": 0.8700197339057922, "Finetune/Step": 5016, "Finetune/Step Time": 4.8016720674932} +{"Finetune/Learning Rate": 6.350220166216337e-08, "Finetune/Loss": 0.8309663534164429, "Finetune/Loss (Raw)": 0.7841726541519165, "Finetune/Step": 5017, "Finetune/Step Time": 4.802340850234032} +{"Finetune/Learning Rate": 6.280319158544989e-08, "Finetune/Loss": 0.8301593065261841, "Finetune/Loss (Raw)": 0.7499880790710449, "Finetune/Step": 5018, "Finetune/Step Time": 4.795770548284054} +{"Finetune/Learning Rate": 6.21080379408101e-08, "Finetune/Loss": 0.8301602005958557, "Finetune/Loss (Raw)": 0.7826526761054993, "Finetune/Step": 5019, "Finetune/Step Time": 4.789362238720059} +{"Finetune/Learning Rate": 6.14167409980193e-08, "Finetune/Loss": 0.8259986639022827, "Finetune/Loss (Raw)": 0.3459908962249756, "Finetune/Step": 5020, "Finetune/Step Time": 4.823128128424287} +{"Finetune/Learning Rate": 6.072930102535734e-08, "Finetune/Loss": 0.8253905773162842, "Finetune/Loss (Raw)": 0.9188417196273804, "Finetune/Step": 5021, "Finetune/Step Time": 4.822399843484163} +{"Finetune/Learning Rate": 6.004571828960525e-08, "Finetune/Loss": 0.8258711099624634, "Finetune/Loss (Raw)": 0.8861552476882935, "Finetune/Step": 5022, "Finetune/Step Time": 4.799063781276345} +{"Finetune/Learning Rate": 5.936599305604862e-08, "Finetune/Loss": 0.8265502452850342, "Finetune/Loss (Raw)": 0.8684077858924866, "Finetune/Step": 5023, "Finetune/Step Time": 4.806009575724602} +{"Finetune/Learning Rate": 5.869012558847642e-08, "Finetune/Loss": 0.8257352113723755, "Finetune/Loss (Raw)": 0.8232142925262451, "Finetune/Step": 5024, "Finetune/Step Time": 4.809893775731325} +{"Finetune/Learning Rate": 5.801811614917885e-08, "Finetune/Loss": 0.8267781138420105, "Finetune/Loss (Raw)": 0.8300628662109375, "Finetune/Step": 5025, "Finetune/Step Time": 4.8145744148641825} +{"Finetune/Learning Rate": 5.73499649989484e-08, "Finetune/Loss": 0.8280566930770874, "Finetune/Loss (Raw)": 0.9259883761405945, "Finetune/Step": 5026, "Finetune/Step Time": 4.806370256468654} +{"Finetune/Learning Rate": 5.6685672397083226e-08, "Finetune/Loss": 0.8276426196098328, "Finetune/Loss (Raw)": 0.9186786413192749, "Finetune/Step": 5027, "Finetune/Step Time": 4.80912902019918} +{"Finetune/Learning Rate": 5.602523860138154e-08, "Finetune/Loss": 0.8284916877746582, "Finetune/Loss (Raw)": 0.9623938798904419, "Finetune/Step": 5028, "Finetune/Step Time": 4.811743790283799} +{"Finetune/Learning Rate": 5.536866386814499e-08, "Finetune/Loss": 0.8290155529975891, "Finetune/Loss (Raw)": 0.8854674100875854, "Finetune/Step": 5029, "Finetune/Step Time": 4.813376720994711} +{"Finetune/Learning Rate": 5.471594845217865e-08, "Finetune/Loss": 0.8290054202079773, "Finetune/Loss (Raw)": 0.817710280418396, "Finetune/Step": 5030, "Finetune/Step Time": 4.816792227327824} +{"Finetune/Learning Rate": 5.4067092606785445e-08, "Finetune/Loss": 0.8294986486434937, "Finetune/Loss (Raw)": 0.8970669507980347, "Finetune/Step": 5031, "Finetune/Step Time": 4.818631809204817} +{"Finetune/Learning Rate": 5.342209658377728e-08, "Finetune/Loss": 0.8302080035209656, "Finetune/Loss (Raw)": 0.9434922933578491, "Finetune/Step": 5032, "Finetune/Step Time": 4.816620817407966} +{"Finetune/Learning Rate": 5.278096063346172e-08, "Finetune/Loss": 0.8296278715133667, "Finetune/Loss (Raw)": 0.8538132309913635, "Finetune/Step": 5033, "Finetune/Step Time": 4.815629245713353} +{"Finetune/Learning Rate": 5.214368500465305e-08, "Finetune/Loss": 0.8301728367805481, "Finetune/Loss (Raw)": 0.9112953543663025, "Finetune/Step": 5034, "Finetune/Step Time": 4.818669019266963} +{"Finetune/Learning Rate": 5.151026994466457e-08, "Finetune/Loss": 0.8303372859954834, "Finetune/Loss (Raw)": 0.7882218956947327, "Finetune/Step": 5035, "Finetune/Step Time": 4.816009383648634} +{"Finetune/Learning Rate": 5.088071569931185e-08, "Finetune/Loss": 0.8298318386077881, "Finetune/Loss (Raw)": 0.8219795227050781, "Finetune/Step": 5036, "Finetune/Step Time": 4.790253844112158} +{"Finetune/Learning Rate": 5.025502251291392e-08, "Finetune/Loss": 0.8307642936706543, "Finetune/Loss (Raw)": 0.9723657369613647, "Finetune/Step": 5037, "Finetune/Step Time": 4.785342995077372} +{"Finetune/Learning Rate": 4.963319062828875e-08, "Finetune/Loss": 0.8313121795654297, "Finetune/Loss (Raw)": 0.9669339060783386, "Finetune/Step": 5038, "Finetune/Step Time": 4.793077228590846} +{"Finetune/Learning Rate": 4.901522028675776e-08, "Finetune/Loss": 0.8330804705619812, "Finetune/Loss (Raw)": 1.0678074359893799, "Finetune/Step": 5039, "Finetune/Step Time": 4.798354081809521} +{"Finetune/Learning Rate": 4.8401111728143545e-08, "Finetune/Loss": 0.8317860960960388, "Finetune/Loss (Raw)": 0.7333892583847046, "Finetune/Step": 5040, "Finetune/Step Time": 4.802610833197832} +{"Finetune/Learning Rate": 4.779086519076992e-08, "Finetune/Loss": 0.8322473764419556, "Finetune/Loss (Raw)": 0.9884894490242004, "Finetune/Step": 5041, "Finetune/Step Time": 4.790742924436927} +{"Finetune/Learning Rate": 4.7184480911459665e-08, "Finetune/Loss": 0.836579442024231, "Finetune/Loss (Raw)": 0.9629727602005005, "Finetune/Step": 5042, "Finetune/Step Time": 4.769001809880137} +{"Finetune/Learning Rate": 4.65819591255412e-08, "Finetune/Loss": 0.8375270366668701, "Finetune/Loss (Raw)": 0.8664419651031494, "Finetune/Step": 5043, "Finetune/Step Time": 4.771167166531086} +{"Finetune/Learning Rate": 4.5983300066839706e-08, "Finetune/Loss": 0.8374990820884705, "Finetune/Loss (Raw)": 0.8476608395576477, "Finetune/Step": 5044, "Finetune/Step Time": 4.779867850244045} +{"Finetune/Learning Rate": 4.538850396768379e-08, "Finetune/Loss": 0.8365504741668701, "Finetune/Loss (Raw)": 0.28930073976516724, "Finetune/Step": 5045, "Finetune/Step Time": 4.7839704398065805} +{"Finetune/Learning Rate": 4.479757105890104e-08, "Finetune/Loss": 0.8322383165359497, "Finetune/Loss (Raw)": 0.33153122663497925, "Finetune/Step": 5046, "Finetune/Step Time": 4.81917523778975} +{"Finetune/Learning Rate": 4.4210501569823536e-08, "Finetune/Loss": 0.8324822187423706, "Finetune/Loss (Raw)": 0.8921236991882324, "Finetune/Step": 5047, "Finetune/Step Time": 4.813086682930589} +{"Finetune/Learning Rate": 4.362729572827906e-08, "Finetune/Loss": 0.8329460024833679, "Finetune/Loss (Raw)": 0.9658591151237488, "Finetune/Step": 5048, "Finetune/Step Time": 4.813647087663412} +{"Finetune/Learning Rate": 4.304795376059989e-08, "Finetune/Loss": 0.8324309587478638, "Finetune/Loss (Raw)": 0.8493009805679321, "Finetune/Step": 5049, "Finetune/Step Time": 4.805643483996391} +{"Finetune/Learning Rate": 4.2472475891617295e-08, "Finetune/Loss": 0.8330565690994263, "Finetune/Loss (Raw)": 0.8363143801689148, "Finetune/Step": 5050, "Finetune/Step Time": 4.8119301572442055} +{"Finetune/Learning Rate": 4.1900862344661506e-08, "Finetune/Loss": 0.8327944278717041, "Finetune/Loss (Raw)": 0.7755812406539917, "Finetune/Step": 5051, "Finetune/Step Time": 4.826154371723533} +{"Finetune/Learning Rate": 4.133311334156509e-08, "Finetune/Loss": 0.8289445638656616, "Finetune/Loss (Raw)": 0.37058529257774353, "Finetune/Step": 5052, "Finetune/Step Time": 4.856022177264094} +{"Finetune/Learning Rate": 4.076922910266179e-08, "Finetune/Loss": 0.8286744952201843, "Finetune/Loss (Raw)": 0.7908310890197754, "Finetune/Step": 5053, "Finetune/Step Time": 4.8695220071822405} +{"Finetune/Learning Rate": 4.020920984678323e-08, "Finetune/Loss": 0.830238401889801, "Finetune/Loss (Raw)": 1.0102320909500122, "Finetune/Step": 5054, "Finetune/Step Time": 4.875557688996196} +{"Finetune/Learning Rate": 3.965305579126111e-08, "Finetune/Loss": 0.8303688764572144, "Finetune/Loss (Raw)": 0.8256922364234924, "Finetune/Step": 5055, "Finetune/Step Time": 4.875673850998282} +{"Finetune/Learning Rate": 3.9100767151928345e-08, "Finetune/Loss": 0.8321502208709717, "Finetune/Loss (Raw)": 0.8841609358787537, "Finetune/Step": 5056, "Finetune/Step Time": 4.850148683413863} +{"Finetune/Learning Rate": 3.855234414311904e-08, "Finetune/Loss": 0.8325199484825134, "Finetune/Loss (Raw)": 0.8788420557975769, "Finetune/Step": 5057, "Finetune/Step Time": 4.848241673782468} +{"Finetune/Learning Rate": 3.800778697766294e-08, "Finetune/Loss": 0.8324853181838989, "Finetune/Loss (Raw)": 0.8764575719833374, "Finetune/Step": 5058, "Finetune/Step Time": 4.851806923747063} +{"Finetune/Learning Rate": 3.746709586689323e-08, "Finetune/Loss": 0.8325527310371399, "Finetune/Loss (Raw)": 0.9515122771263123, "Finetune/Step": 5059, "Finetune/Step Time": 4.849315065890551} +{"Finetune/Learning Rate": 3.693027102064206e-08, "Finetune/Loss": 0.833111047744751, "Finetune/Loss (Raw)": 0.9744353294372559, "Finetune/Step": 5060, "Finetune/Step Time": 4.84940149076283} +{"Finetune/Learning Rate": 3.639731264724056e-08, "Finetune/Loss": 0.83403480052948, "Finetune/Loss (Raw)": 0.9112091064453125, "Finetune/Step": 5061, "Finetune/Step Time": 4.837412567809224} +{"Finetune/Learning Rate": 3.586822095351772e-08, "Finetune/Loss": 0.8329787254333496, "Finetune/Loss (Raw)": 0.7860310673713684, "Finetune/Step": 5062, "Finetune/Step Time": 4.847407732158899} +{"Finetune/Learning Rate": 3.5342996144805966e-08, "Finetune/Loss": 0.8330638408660889, "Finetune/Loss (Raw)": 0.9428264498710632, "Finetune/Step": 5063, "Finetune/Step Time": 4.844356061890721} +{"Finetune/Learning Rate": 3.482163842493447e-08, "Finetune/Loss": 0.8328605890274048, "Finetune/Loss (Raw)": 0.7610249519348145, "Finetune/Step": 5064, "Finetune/Step Time": 4.834628161042929} +{"Finetune/Learning Rate": 3.4304147996230275e-08, "Finetune/Loss": 0.8325214385986328, "Finetune/Loss (Raw)": 0.9643229246139526, "Finetune/Step": 5065, "Finetune/Step Time": 4.8407435938715935} +{"Finetune/Learning Rate": 3.379052505952274e-08, "Finetune/Loss": 0.8326700925827026, "Finetune/Loss (Raw)": 0.9337111711502075, "Finetune/Step": 5066, "Finetune/Step Time": 4.831648359075189} +{"Finetune/Learning Rate": 3.328076981413908e-08, "Finetune/Loss": 0.8364450335502625, "Finetune/Loss (Raw)": 0.8228870630264282, "Finetune/Step": 5067, "Finetune/Step Time": 4.80043613165617} +{"Finetune/Learning Rate": 3.2774882457904386e-08, "Finetune/Loss": 0.8381239175796509, "Finetune/Loss (Raw)": 0.9023180603981018, "Finetune/Step": 5068, "Finetune/Step Time": 4.793688874691725} +{"Finetune/Learning Rate": 3.227286318714384e-08, "Finetune/Loss": 0.838355541229248, "Finetune/Loss (Raw)": 0.7805261015892029, "Finetune/Step": 5069, "Finetune/Step Time": 4.79536490701139} +{"Finetune/Learning Rate": 3.177471219668271e-08, "Finetune/Loss": 0.8383063077926636, "Finetune/Loss (Raw)": 0.8074634671211243, "Finetune/Step": 5070, "Finetune/Step Time": 4.792568493634462} +{"Finetune/Learning Rate": 3.1280429679841905e-08, "Finetune/Loss": 0.8386528491973877, "Finetune/Loss (Raw)": 0.8292566537857056, "Finetune/Step": 5071, "Finetune/Step Time": 4.7993680741637945} +{"Finetune/Learning Rate": 3.079001582844354e-08, "Finetune/Loss": 0.8398821353912354, "Finetune/Loss (Raw)": 1.0760717391967773, "Finetune/Step": 5072, "Finetune/Step Time": 4.802319122478366} +{"Finetune/Learning Rate": 3.0303470832806494e-08, "Finetune/Loss": 0.8395662307739258, "Finetune/Loss (Raw)": 0.9007607698440552, "Finetune/Step": 5073, "Finetune/Step Time": 4.806552371010184} +{"Finetune/Learning Rate": 2.9820794881751934e-08, "Finetune/Loss": 0.8402142524719238, "Finetune/Loss (Raw)": 0.9409362077713013, "Finetune/Step": 5074, "Finetune/Step Time": 4.802340833470225} +{"Finetune/Learning Rate": 2.9341988162595593e-08, "Finetune/Loss": 0.8358931541442871, "Finetune/Loss (Raw)": 0.365714967250824, "Finetune/Step": 5075, "Finetune/Step Time": 4.841596582904458} +{"Finetune/Learning Rate": 2.886705086115216e-08, "Finetune/Loss": 0.8368053436279297, "Finetune/Loss (Raw)": 0.9312708377838135, "Finetune/Step": 5076, "Finetune/Step Time": 4.8400785103440285} +{"Finetune/Learning Rate": 2.8395983161736418e-08, "Finetune/Loss": 0.8371409177780151, "Finetune/Loss (Raw)": 0.8491748571395874, "Finetune/Step": 5077, "Finetune/Step Time": 4.85637092590332} +{"Finetune/Learning Rate": 2.7928785247161027e-08, "Finetune/Loss": 0.8375519514083862, "Finetune/Loss (Raw)": 0.8612828254699707, "Finetune/Step": 5078, "Finetune/Step Time": 4.839817443862557} +{"Finetune/Learning Rate": 2.7465457298736508e-08, "Finetune/Loss": 0.838376522064209, "Finetune/Loss (Raw)": 1.0106512308120728, "Finetune/Step": 5079, "Finetune/Step Time": 4.840014439076185} +{"Finetune/Learning Rate": 2.700599949626903e-08, "Finetune/Loss": 0.8387008905410767, "Finetune/Loss (Raw)": 0.9604445695877075, "Finetune/Step": 5080, "Finetune/Step Time": 4.832041224464774} +{"Finetune/Learning Rate": 2.6550412018067072e-08, "Finetune/Loss": 0.8379029035568237, "Finetune/Loss (Raw)": 0.7183752655982971, "Finetune/Step": 5081, "Finetune/Step Time": 4.827829286456108} +{"Finetune/Learning Rate": 2.6098695040935874e-08, "Finetune/Loss": 0.8334074020385742, "Finetune/Loss (Raw)": 0.34231460094451904, "Finetune/Step": 5082, "Finetune/Step Time": 4.803080940619111} +{"Finetune/Learning Rate": 2.5650848740177425e-08, "Finetune/Loss": 0.8333300948143005, "Finetune/Loss (Raw)": 0.8540900349617004, "Finetune/Step": 5083, "Finetune/Step Time": 4.8131047915667295} +{"Finetune/Learning Rate": 2.5206873289590484e-08, "Finetune/Loss": 0.8319660425186157, "Finetune/Loss (Raw)": 0.763427197933197, "Finetune/Step": 5084, "Finetune/Step Time": 4.823483010753989} +{"Finetune/Learning Rate": 2.4766768861475e-08, "Finetune/Loss": 0.8311244249343872, "Finetune/Loss (Raw)": 0.8643735647201538, "Finetune/Step": 5085, "Finetune/Step Time": 4.826897997409105} +{"Finetune/Learning Rate": 2.4330535626627684e-08, "Finetune/Loss": 0.8325166702270508, "Finetune/Loss (Raw)": 0.9891518950462341, "Finetune/Step": 5086, "Finetune/Step Time": 4.813778875395656} +{"Finetune/Learning Rate": 2.389817375433978e-08, "Finetune/Loss": 0.8361393213272095, "Finetune/Loss (Raw)": 0.9039228558540344, "Finetune/Step": 5087, "Finetune/Step Time": 4.795287923887372} +{"Finetune/Learning Rate": 2.3469683412403745e-08, "Finetune/Loss": 0.8375358581542969, "Finetune/Loss (Raw)": 1.007874608039856, "Finetune/Step": 5088, "Finetune/Step Time": 4.7949773743748665} +{"Finetune/Learning Rate": 2.304506476710877e-08, "Finetune/Loss": 0.8384631276130676, "Finetune/Loss (Raw)": 0.9300059676170349, "Finetune/Step": 5089, "Finetune/Step Time": 4.7889025174081326} +{"Finetune/Learning Rate": 2.262431798323972e-08, "Finetune/Loss": 0.8375838398933411, "Finetune/Loss (Raw)": 0.804291307926178, "Finetune/Step": 5090, "Finetune/Step Time": 4.790357541292906} +{"Finetune/Learning Rate": 2.2207443224081528e-08, "Finetune/Loss": 0.8383474946022034, "Finetune/Loss (Raw)": 0.9436531066894531, "Finetune/Step": 5091, "Finetune/Step Time": 4.798585491254926} +{"Finetune/Learning Rate": 2.1794440651414782e-08, "Finetune/Loss": 0.8335112929344177, "Finetune/Loss (Raw)": 0.32489848136901855, "Finetune/Step": 5092, "Finetune/Step Time": 4.830002101138234} +{"Finetune/Learning Rate": 2.1385310425517945e-08, "Finetune/Loss": 0.8329159021377563, "Finetune/Loss (Raw)": 0.8967623114585876, "Finetune/Step": 5093, "Finetune/Step Time": 4.836522031575441} +{"Finetune/Learning Rate": 2.0980052705165123e-08, "Finetune/Loss": 0.8330532312393188, "Finetune/Loss (Raw)": 0.8156557083129883, "Finetune/Step": 5094, "Finetune/Step Time": 4.845775756984949} +{"Finetune/Learning Rate": 2.05786676476305e-08, "Finetune/Loss": 0.8312749862670898, "Finetune/Loss (Raw)": 0.703177809715271, "Finetune/Step": 5095, "Finetune/Step Time": 4.8535680044442415} +{"Finetune/Learning Rate": 2.0181155408682816e-08, "Finetune/Loss": 0.8305242657661438, "Finetune/Loss (Raw)": 0.7935423254966736, "Finetune/Step": 5096, "Finetune/Step Time": 4.881622791290283} +{"Finetune/Learning Rate": 1.978751614258978e-08, "Finetune/Loss": 0.8287298679351807, "Finetune/Loss (Raw)": 0.643218457698822, "Finetune/Step": 5097, "Finetune/Step Time": 4.871941395103931} +{"Finetune/Learning Rate": 1.9397750002113634e-08, "Finetune/Loss": 0.8270359635353088, "Finetune/Loss (Raw)": 0.7229744791984558, "Finetune/Step": 5098, "Finetune/Step Time": 4.873625153675675} +{"Finetune/Learning Rate": 1.901185713851672e-08, "Finetune/Loss": 0.8252672553062439, "Finetune/Loss (Raw)": 0.7803168892860413, "Finetune/Step": 5099, "Finetune/Step Time": 4.8607397470623255} +{"Finetune/Learning Rate": 1.8629837701554797e-08, "Finetune/Loss": 0.8245384693145752, "Finetune/Loss (Raw)": 0.7706459164619446, "Finetune/Step": 5100, "Finetune/Step Time": 4.846161792054772} +{"Finetune/Learning Rate": 1.825169183948372e-08, "Finetune/Loss": 0.8229436874389648, "Finetune/Loss (Raw)": 0.789965808391571, "Finetune/Step": 5101, "Finetune/Step Time": 4.847510229796171} +{"Finetune/Learning Rate": 1.7877419699053876e-08, "Finetune/Loss": 0.8274896144866943, "Finetune/Loss (Raw)": 0.8869055509567261, "Finetune/Step": 5102, "Finetune/Step Time": 4.81451266258955} +{"Finetune/Learning Rate": 1.7507021425512417e-08, "Finetune/Loss": 0.8286443948745728, "Finetune/Loss (Raw)": 0.9534950256347656, "Finetune/Step": 5103, "Finetune/Step Time": 4.809912094846368} +{"Finetune/Learning Rate": 1.7140497162605464e-08, "Finetune/Loss": 0.8259322643280029, "Finetune/Loss (Raw)": 0.6409338712692261, "Finetune/Step": 5104, "Finetune/Step Time": 4.81244619935751} +{"Finetune/Learning Rate": 1.677784705257146e-08, "Finetune/Loss": 0.8269518613815308, "Finetune/Loss (Raw)": 0.8784868121147156, "Finetune/Step": 5105, "Finetune/Step Time": 4.790389787405729} +{"Finetune/Learning Rate": 1.6419071236148943e-08, "Finetune/Loss": 0.8283804655075073, "Finetune/Loss (Raw)": 0.9861185550689697, "Finetune/Step": 5106, "Finetune/Step Time": 4.791090257465839} +{"Finetune/Learning Rate": 1.6064169852572088e-08, "Finetune/Loss": 0.8294738531112671, "Finetune/Loss (Raw)": 0.955199658870697, "Finetune/Step": 5107, "Finetune/Step Time": 4.7958378084003925} +{"Finetune/Learning Rate": 1.5713143039571833e-08, "Finetune/Loss": 0.830599308013916, "Finetune/Loss (Raw)": 0.8507005572319031, "Finetune/Step": 5108, "Finetune/Step Time": 4.779028492048383} +{"Finetune/Learning Rate": 1.5365990933373653e-08, "Finetune/Loss": 0.8321051001548767, "Finetune/Loss (Raw)": 0.9577021598815918, "Finetune/Step": 5109, "Finetune/Step Time": 4.781749878078699} +{"Finetune/Learning Rate": 1.5022713668699785e-08, "Finetune/Loss": 0.831462025642395, "Finetune/Loss (Raw)": 0.8777186274528503, "Finetune/Step": 5110, "Finetune/Step Time": 4.793611031025648} +{"Finetune/Learning Rate": 1.4683311378771437e-08, "Finetune/Loss": 0.8341312408447266, "Finetune/Loss (Raw)": 0.7538881301879883, "Finetune/Step": 5111, "Finetune/Step Time": 4.788410564884543} +{"Finetune/Learning Rate": 1.434778419530325e-08, "Finetune/Loss": 0.8307232856750488, "Finetune/Loss (Raw)": 0.3474183976650238, "Finetune/Step": 5112, "Finetune/Step Time": 4.807928783819079} +{"Finetune/Learning Rate": 1.401613224850551e-08, "Finetune/Loss": 0.8249936103820801, "Finetune/Loss (Raw)": 0.2886020243167877, "Finetune/Step": 5113, "Finetune/Step Time": 4.847943292930722} +{"Finetune/Learning Rate": 1.3688355667087483e-08, "Finetune/Loss": 0.8261796236038208, "Finetune/Loss (Raw)": 0.8534579277038574, "Finetune/Step": 5114, "Finetune/Step Time": 4.849336137995124} +{"Finetune/Learning Rate": 1.3364454578252972e-08, "Finetune/Loss": 0.825069785118103, "Finetune/Loss (Raw)": 0.7757169604301453, "Finetune/Step": 5115, "Finetune/Step Time": 4.8413883578032255} +{"Finetune/Learning Rate": 1.3044429107700319e-08, "Finetune/Loss": 0.8196921944618225, "Finetune/Loss (Raw)": 0.2569124698638916, "Finetune/Step": 5116, "Finetune/Step Time": 4.870422085747123} +{"Finetune/Learning Rate": 1.272827937962573e-08, "Finetune/Loss": 0.8206139802932739, "Finetune/Loss (Raw)": 1.0406819581985474, "Finetune/Step": 5117, "Finetune/Step Time": 4.871653724461794} +{"Finetune/Learning Rate": 1.2416005516722174e-08, "Finetune/Loss": 0.8215917348861694, "Finetune/Loss (Raw)": 0.9853711128234863, "Finetune/Step": 5118, "Finetune/Step Time": 4.869033807888627} +{"Finetune/Learning Rate": 1.2107607640174934e-08, "Finetune/Loss": 0.8231160044670105, "Finetune/Loss (Raw)": 1.0245980024337769, "Finetune/Step": 5119, "Finetune/Step Time": 4.869129151105881} +{"Finetune/Learning Rate": 1.1803085869667164e-08, "Finetune/Loss": 0.8180969953536987, "Finetune/Loss (Raw)": 0.3430970013141632, "Finetune/Step": 5120, "Finetune/Step Time": 4.888712866231799} +{"Finetune/Learning Rate": 1.1502440323379882e-08, "Finetune/Loss": 0.8197787404060364, "Finetune/Loss (Raw)": 0.8144214153289795, "Finetune/Step": 5121, "Finetune/Step Time": 4.874863855540752} +{"Finetune/Learning Rate": 1.1205671117985318e-08, "Finetune/Loss": 0.8193615674972534, "Finetune/Loss (Raw)": 0.783424973487854, "Finetune/Step": 5122, "Finetune/Step Time": 4.866559728980064} +{"Finetune/Learning Rate": 1.0912778368655786e-08, "Finetune/Loss": 0.8215122222900391, "Finetune/Loss (Raw)": 0.8946583867073059, "Finetune/Step": 5123, "Finetune/Step Time": 4.86866157874465} +{"Finetune/Learning Rate": 1.0623762189057029e-08, "Finetune/Loss": 0.8209702968597412, "Finetune/Loss (Raw)": 0.9391171932220459, "Finetune/Step": 5124, "Finetune/Step Time": 4.869839318096638} +{"Finetune/Learning Rate": 1.0338622691349332e-08, "Finetune/Loss": 0.82145756483078, "Finetune/Loss (Raw)": 0.9874323010444641, "Finetune/Step": 5125, "Finetune/Step Time": 4.874644502997398} +{"Finetune/Learning Rate": 1.005735998619084e-08, "Finetune/Loss": 0.8159216642379761, "Finetune/Loss (Raw)": 0.3887878954410553, "Finetune/Step": 5126, "Finetune/Step Time": 4.908812064677477} +{"Finetune/Learning Rate": 9.779974182733131e-09, "Finetune/Loss": 0.8148694634437561, "Finetune/Loss (Raw)": 0.8551149964332581, "Finetune/Step": 5127, "Finetune/Step Time": 4.917130637913942} +{"Finetune/Learning Rate": 9.506465388624541e-09, "Finetune/Loss": 0.8188363313674927, "Finetune/Loss (Raw)": 0.8605961203575134, "Finetune/Step": 5128, "Finetune/Step Time": 4.890396919101477} +{"Finetune/Learning Rate": 9.236833710010163e-09, "Finetune/Loss": 0.8137458562850952, "Finetune/Loss (Raw)": 0.2925206422805786, "Finetune/Step": 5129, "Finetune/Step Time": 4.926790218800306} +{"Finetune/Learning Rate": 8.971079251526293e-09, "Finetune/Loss": 0.8149867057800293, "Finetune/Loss (Raw)": 0.8375087976455688, "Finetune/Step": 5130, "Finetune/Step Time": 4.921344496309757} +{"Finetune/Learning Rate": 8.709202116308214e-09, "Finetune/Loss": 0.8144460916519165, "Finetune/Loss (Raw)": 0.730812668800354, "Finetune/Step": 5131, "Finetune/Step Time": 4.932669436559081} +{"Finetune/Learning Rate": 8.451202405985736e-09, "Finetune/Loss": 0.8145163059234619, "Finetune/Loss (Raw)": 0.7540353536605835, "Finetune/Step": 5132, "Finetune/Step Time": 4.927082011476159} +{"Finetune/Learning Rate": 8.197080220683217e-09, "Finetune/Loss": 0.8146548271179199, "Finetune/Loss (Raw)": 0.8787657022476196, "Finetune/Step": 5133, "Finetune/Step Time": 4.92757273465395} +{"Finetune/Learning Rate": 7.946835659020657e-09, "Finetune/Loss": 0.8147258162498474, "Finetune/Loss (Raw)": 0.8920671343803406, "Finetune/Step": 5134, "Finetune/Step Time": 4.931336492300034} +{"Finetune/Learning Rate": 7.700468818111483e-09, "Finetune/Loss": 0.8143226504325867, "Finetune/Loss (Raw)": 0.7277765870094299, "Finetune/Step": 5135, "Finetune/Step Time": 4.9249227643013} +{"Finetune/Learning Rate": 7.457979793568105e-09, "Finetune/Loss": 0.816257655620575, "Finetune/Loss (Raw)": 1.0330263376235962, "Finetune/Step": 5136, "Finetune/Step Time": 4.929360369220376} +{"Finetune/Learning Rate": 7.219368679495242e-09, "Finetune/Loss": 0.8148143291473389, "Finetune/Loss (Raw)": 0.8193364143371582, "Finetune/Step": 5137, "Finetune/Step Time": 4.932413438335061} +{"Finetune/Learning Rate": 6.984635568492159e-09, "Finetune/Loss": 0.8139104247093201, "Finetune/Loss (Raw)": 0.8289955258369446, "Finetune/Step": 5138, "Finetune/Step Time": 4.9408628810197115} +{"Finetune/Learning Rate": 6.753780551654876e-09, "Finetune/Loss": 0.8098213076591492, "Finetune/Loss (Raw)": 0.3407409191131592, "Finetune/Step": 5139, "Finetune/Step Time": 4.974783588200808} +{"Finetune/Learning Rate": 6.526803718572838e-09, "Finetune/Loss": 0.810387134552002, "Finetune/Loss (Raw)": 0.9185548424720764, "Finetune/Step": 5140, "Finetune/Step Time": 4.96943493373692} +{"Finetune/Learning Rate": 6.3037051573333615e-09, "Finetune/Loss": 0.812269389629364, "Finetune/Loss (Raw)": 0.9726241827011108, "Finetune/Step": 5141, "Finetune/Step Time": 4.966787548735738} +{"Finetune/Learning Rate": 6.0844849545149685e-09, "Finetune/Loss": 0.8124929666519165, "Finetune/Loss (Raw)": 0.8228446841239929, "Finetune/Step": 5142, "Finetune/Step Time": 4.976268358528614} +{"Finetune/Learning Rate": 5.869143195192939e-09, "Finetune/Loss": 0.8138210773468018, "Finetune/Loss (Raw)": 0.9439717531204224, "Finetune/Step": 5143, "Finetune/Step Time": 4.9792655277997255} +{"Finetune/Learning Rate": 5.657679962938201e-09, "Finetune/Loss": 0.8122427463531494, "Finetune/Loss (Raw)": 0.6679890751838684, "Finetune/Step": 5144, "Finetune/Step Time": 4.98478795774281} +{"Finetune/Learning Rate": 5.4500953398139985e-09, "Finetune/Loss": 0.8124933242797852, "Finetune/Loss (Raw)": 0.8162534236907959, "Finetune/Step": 5145, "Finetune/Step Time": 4.985312050208449} +{"Finetune/Learning Rate": 5.246389406381447e-09, "Finetune/Loss": 0.8145287036895752, "Finetune/Loss (Raw)": 1.0105154514312744, "Finetune/Step": 5146, "Finetune/Step Time": 4.974951792508364} +{"Finetune/Learning Rate": 5.046562241693975e-09, "Finetune/Loss": 0.8151305913925171, "Finetune/Loss (Raw)": 0.8596908450126648, "Finetune/Step": 5147, "Finetune/Step Time": 4.976945640519261} +{"Finetune/Learning Rate": 4.850613923300662e-09, "Finetune/Loss": 0.8176780939102173, "Finetune/Loss (Raw)": 0.672070324420929, "Finetune/Step": 5148, "Finetune/Step Time": 4.947776064276695} +{"Finetune/Learning Rate": 4.658544527244013e-09, "Finetune/Loss": 0.8169090747833252, "Finetune/Loss (Raw)": 0.8204112648963928, "Finetune/Step": 5149, "Finetune/Step Time": 4.954403920099139} +{"Finetune/Learning Rate": 4.470354128065513e-09, "Finetune/Loss": 0.8164197206497192, "Finetune/Loss (Raw)": 0.82352215051651, "Finetune/Step": 5150, "Finetune/Step Time": 4.961417373269796} +{"Finetune/Learning Rate": 4.286042798794521e-09, "Finetune/Loss": 0.815879225730896, "Finetune/Loss (Raw)": 0.7992231249809265, "Finetune/Step": 5151, "Finetune/Step Time": 4.962919307872653} +{"Finetune/Learning Rate": 4.105610610961597e-09, "Finetune/Loss": 0.8155115246772766, "Finetune/Loss (Raw)": 0.7761447429656982, "Finetune/Step": 5152, "Finetune/Step Time": 4.964779200032353} +{"Finetune/Learning Rate": 3.929057634587397e-09, "Finetune/Loss": 0.8123028874397278, "Finetune/Loss (Raw)": 0.41935819387435913, "Finetune/Step": 5153, "Finetune/Step Time": 4.987906021997333} +{"Finetune/Learning Rate": 3.756383938189334e-09, "Finetune/Loss": 0.8079530000686646, "Finetune/Loss (Raw)": 0.36920076608657837, "Finetune/Step": 5154, "Finetune/Step Time": 5.023510862141848} +{"Finetune/Learning Rate": 3.587589588778251e-09, "Finetune/Loss": 0.8067469596862793, "Finetune/Loss (Raw)": 0.7643089890480042, "Finetune/Step": 5155, "Finetune/Step Time": 5.024543264880776} +{"Finetune/Learning Rate": 3.4226746518606356e-09, "Finetune/Loss": 0.8066893219947815, "Finetune/Loss (Raw)": 0.9550170302391052, "Finetune/Step": 5156, "Finetune/Step Time": 5.0372246243059635} +{"Finetune/Learning Rate": 3.2616391914364056e-09, "Finetune/Loss": 0.8064864873886108, "Finetune/Loss (Raw)": 0.8595026731491089, "Finetune/Step": 5157, "Finetune/Step Time": 5.042294712737203} +{"Finetune/Learning Rate": 3.1044832699989035e-09, "Finetune/Loss": 0.8067260384559631, "Finetune/Loss (Raw)": 0.8483759760856628, "Finetune/Step": 5158, "Finetune/Step Time": 5.039144145324826} +{"Finetune/Learning Rate": 2.9512069485393426e-09, "Finetune/Loss": 0.8071362972259521, "Finetune/Loss (Raw)": 0.9495737552642822, "Finetune/Step": 5159, "Finetune/Step Time": 5.041628964245319} +{"Finetune/Learning Rate": 2.801810286540141e-09, "Finetune/Loss": 0.8075667023658752, "Finetune/Loss (Raw)": 0.9985854029655457, "Finetune/Step": 5160, "Finetune/Step Time": 5.040393799543381} +{"Finetune/Learning Rate": 2.656293341980476e-09, "Finetune/Loss": 0.8080989122390747, "Finetune/Loss (Raw)": 0.9219316840171814, "Finetune/Step": 5161, "Finetune/Step Time": 5.052286205813289} +{"Finetune/Learning Rate": 2.5146561713307315e-09, "Finetune/Loss": 0.8071997165679932, "Finetune/Loss (Raw)": 0.7962052822113037, "Finetune/Step": 5162, "Finetune/Step Time": 5.051618121564388} +{"Finetune/Learning Rate": 2.3768988295580497e-09, "Finetune/Loss": 0.8080973625183105, "Finetune/Loss (Raw)": 0.9031187295913696, "Finetune/Step": 5163, "Finetune/Step Time": 5.047843998298049} +{"Finetune/Learning Rate": 2.2430213701252203e-09, "Finetune/Loss": 0.8092795014381409, "Finetune/Loss (Raw)": 0.9732913374900818, "Finetune/Step": 5164, "Finetune/Step Time": 5.061527650803328} +{"Finetune/Learning Rate": 2.1130238449851292e-09, "Finetune/Loss": 0.8072271943092346, "Finetune/Loss (Raw)": 0.7096728086471558, "Finetune/Step": 5165, "Finetune/Step Time": 5.071608109399676} +{"Finetune/Learning Rate": 1.986906304588532e-09, "Finetune/Loss": 0.8067941069602966, "Finetune/Loss (Raw)": 0.9114971160888672, "Finetune/Step": 5166, "Finetune/Step Time": 5.0790640488266945} +{"Finetune/Learning Rate": 1.8646687978784992e-09, "Finetune/Loss": 0.8046170473098755, "Finetune/Loss (Raw)": 0.7891467809677124, "Finetune/Step": 5167, "Finetune/Step Time": 5.071830468252301} +{"Finetune/Learning Rate": 1.7463113722937518e-09, "Finetune/Loss": 0.8061733245849609, "Finetune/Loss (Raw)": 0.9325897693634033, "Finetune/Step": 5168, "Finetune/Step Time": 5.062062809243798} +{"Finetune/Learning Rate": 1.631834073765326e-09, "Finetune/Loss": 0.8056713342666626, "Finetune/Loss (Raw)": 0.924235463142395, "Finetune/Step": 5169, "Finetune/Step Time": 5.068945860490203} +{"Finetune/Learning Rate": 1.5212369467210164e-09, "Finetune/Loss": 0.8037799596786499, "Finetune/Loss (Raw)": 0.7208753824234009, "Finetune/Step": 5170, "Finetune/Step Time": 5.065945198759437} +{"Finetune/Learning Rate": 1.4145200340809352e-09, "Finetune/Loss": 0.804793655872345, "Finetune/Loss (Raw)": 0.9961950182914734, "Finetune/Step": 5171, "Finetune/Step Time": 5.061420325189829} +{"Finetune/Learning Rate": 1.3116833772597315e-09, "Finetune/Loss": 0.8048408031463623, "Finetune/Loss (Raw)": 0.8536991477012634, "Finetune/Step": 5172, "Finetune/Step Time": 5.053058929741383} +{"Finetune/Learning Rate": 1.2127270161654824e-09, "Finetune/Loss": 0.8087893128395081, "Finetune/Loss (Raw)": 0.7947049140930176, "Finetune/Step": 5173, "Finetune/Step Time": 5.012405255809426} +{"Finetune/Learning Rate": 1.1176509892030229e-09, "Finetune/Loss": 0.8147159814834595, "Finetune/Loss (Raw)": 1.0901471376419067, "Finetune/Step": 5174, "Finetune/Step Time": 4.977362575009465} +{"Finetune/Learning Rate": 1.0264553332683946e-09, "Finetune/Loss": 0.8146103620529175, "Finetune/Loss (Raw)": 0.8786033391952515, "Finetune/Step": 5175, "Finetune/Step Time": 4.984036177396774} +{"Finetune/Learning Rate": 9.39140083752177e-10, "Finetune/Loss": 0.8096184134483337, "Finetune/Loss (Raw)": 0.32689008116722107, "Finetune/Step": 5176, "Finetune/Step Time": 5.01726002804935} +{"Finetune/Learning Rate": 8.557052745405969e-10, "Finetune/Loss": 0.8106874823570251, "Finetune/Loss (Raw)": 0.9861417412757874, "Finetune/Step": 5177, "Finetune/Step Time": 5.010916259139776} +{"Finetune/Learning Rate": 7.761509380133092e-10, "Finetune/Loss": 0.810053825378418, "Finetune/Loss (Raw)": 0.7552119493484497, "Finetune/Step": 5178, "Finetune/Step Time": 4.995194060727954} +{"Finetune/Learning Rate": 7.004771050433956e-10, "Finetune/Loss": 0.8101279735565186, "Finetune/Loss (Raw)": 0.7850688695907593, "Finetune/Step": 5179, "Finetune/Step Time": 4.990924127399921} +{"Finetune/Learning Rate": 6.286838049984755e-10, "Finetune/Loss": 0.8133422136306763, "Finetune/Loss (Raw)": 0.7820011377334595, "Finetune/Step": 5180, "Finetune/Step Time": 4.964262975379825} +{"Finetune/Learning Rate": 5.607710657407061e-10, "Finetune/Loss": 0.814130961894989, "Finetune/Loss (Raw)": 0.8917935490608215, "Finetune/Step": 5181, "Finetune/Step Time": 4.9594445414841175} +{"Finetune/Learning Rate": 4.967389136245615e-10, "Finetune/Loss": 0.8117714524269104, "Finetune/Loss (Raw)": 0.7082160115242004, "Finetune/Step": 5182, "Finetune/Step Time": 4.96286803111434} +{"Finetune/Learning Rate": 4.3658737350016354e-10, "Finetune/Loss": 0.8123238682746887, "Finetune/Loss (Raw)": 0.8963989019393921, "Finetune/Step": 5183, "Finetune/Step Time": 4.967486418783665} +{"Finetune/Learning Rate": 3.8031646871106166e-10, "Finetune/Loss": 0.8120250701904297, "Finetune/Loss (Raw)": 0.8459231853485107, "Finetune/Step": 5184, "Finetune/Step Time": 4.977923410013318} +{"Finetune/Learning Rate": 3.279262210953427e-10, "Finetune/Loss": 0.8128448128700256, "Finetune/Loss (Raw)": 0.983766496181488, "Finetune/Step": 5185, "Finetune/Step Time": 4.98961110599339} +{"Finetune/Learning Rate": 2.794166509845209e-10, "Finetune/Loss": 0.8089600801467896, "Finetune/Loss (Raw)": 0.3792068660259247, "Finetune/Step": 5186, "Finetune/Step Time": 5.022502563893795} +{"Finetune/Learning Rate": 2.3478777720353785e-10, "Finetune/Loss": 0.8073256015777588, "Finetune/Loss (Raw)": 0.7422970533370972, "Finetune/Step": 5187, "Finetune/Step Time": 5.045783279463649} +{"Finetune/Learning Rate": 1.940396170718728e-10, "Finetune/Loss": 0.8060377836227417, "Finetune/Loss (Raw)": 0.809597909450531, "Finetune/Step": 5188, "Finetune/Step Time": 5.047076128423214} +{"Finetune/Learning Rate": 1.5717218640465272e-10, "Finetune/Loss": 0.804905116558075, "Finetune/Loss (Raw)": 0.7662240266799927, "Finetune/Step": 5189, "Finetune/Step Time": 5.054418221116066} +{"Finetune/Learning Rate": 1.2418549950821145e-10, "Finetune/Loss": 0.8045731782913208, "Finetune/Loss (Raw)": 0.7435504794120789, "Finetune/Step": 5190, "Finetune/Step Time": 5.047549642622471} +{"Finetune/Learning Rate": 9.507956918453076e-11, "Finetune/Loss": 0.8028793334960938, "Finetune/Loss (Raw)": 0.7260117530822754, "Finetune/Step": 5191, "Finetune/Step Time": 5.060203155502677} +{"Finetune/Learning Rate": 6.985440672790945e-11, "Finetune/Loss": 0.803505539894104, "Finetune/Loss (Raw)": 0.8411819338798523, "Finetune/Step": 5192, "Finetune/Step Time": 5.055814154446125} +{"Finetune/Learning Rate": 4.851002192940435e-11, "Finetune/Loss": 0.8016762733459473, "Finetune/Loss (Raw)": 0.7301809191703796, "Finetune/Step": 5193, "Finetune/Step Time": 5.06763113103807} +{"Finetune/Learning Rate": 3.1046423071279254e-11, "Finetune/Loss": 0.7969841957092285, "Finetune/Loss (Raw)": 0.3331233859062195, "Finetune/Step": 5194, "Finetune/Step Time": 5.1023509204387665} +{"Finetune/Learning Rate": 1.7463616930335493e-11, "Finetune/Loss": 0.7972069978713989, "Finetune/Loss (Raw)": 0.8513973951339722, "Finetune/Step": 5195, "Finetune/Step Time": 5.100198227912188} +{"Finetune/Learning Rate": 7.761608780132434e-12, "Finetune/Loss": 0.7963719367980957, "Finetune/Loss (Raw)": 0.7954277396202087, "Finetune/Step": 5196, "Finetune/Step Time": 5.106055384501815} +{"Finetune/Learning Rate": 1.9404023832159115e-12, "Finetune/Loss": 0.7971075177192688, "Finetune/Loss (Raw)": 0.8746828436851501, "Finetune/Step": 5197, "Finetune/Step Time": 5.104403056204319} +{"Finetune/Learning Rate": 0.0, "Finetune/Loss": 0.7975542545318604, "Finetune/Loss (Raw)": 0.8646546006202698, "Finetune/Step": 5198, "Finetune/Step Time": 5.1002165004611015} diff --git a/qformer2_128/run-metrics.jsonl b/qformer2_128/run-metrics.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..f3827ebd3b1843ff670ab3466e75adbfcc01b526 --- /dev/null +++ b/qformer2_128/run-metrics.jsonl @@ -0,0 +1 @@ +{"hparams": {"dataset": {"align_stage_components": ["download/llava-laion-cc-sbu-558k/chat.json", "download/llava-laion-cc-sbu-558k"], "dataset_id": "llava-v15", "dataset_root_dir": "data", "finetune_stage_components": ["download/llava-v1.5-instruct/llava_v1_5_mix665k.json", "download/llava-v1.5-instruct"], "type": "llava-v15"}, "hf_token": ".hf_token", "max_length": 4096, "model": {"align_epochs": 1, "align_global_batch_size": 256, "align_learning_rate": 0.001, "align_lr_scheduler_type": "linear-warmup+cosine-decay", "align_max_grad_norm": 1.0, "align_max_steps": null, "align_per_device_batch_size": 16, "align_train_strategy": "fsdp-shard-grad-op", "align_warmup_ratio": 0.03, "align_weight_decay": 0.0, "arch_specifier": "qformer2_128", "enable_gradient_checkpointing": true, "enable_mixed_precision_training": true, "finetune_epochs": 1, "finetune_global_batch_size": 128, "finetune_learning_rate": 2e-05, "finetune_lr_scheduler_type": "linear-warmup+cosine-decay", "finetune_max_grad_norm": 1.0, "finetune_max_steps": null, "finetune_per_device_batch_size": 16, "finetune_train_strategy": "fsdp-full-shard", "finetune_warmup_ratio": 0.03, "finetune_weight_decay": 0.1, "image_resize_strategy": "letterbox", "llm_backbone_id": "vicuna-v15-7b", "llm_max_length": 2048, "model_id": "one-stage+7b_qformer2_128", "reduce_in_full_precision": false, "type": "one-stage+7b", "vision_backbone_id": "clip-vit-l-336px"}, "pretrained_checkpoint": null, "run_id": "qformer2_128", "run_root_dir": "runs", "seed": 7, "stage": "finetune", "trackers": ["jsonl", "wandb"], "wandb_entity": "lilei_stones", "wandb_project": "hf-vlms"}, "run_id": "qformer2_128"} diff --git a/qformer2_128/wandb/debug-internal.log b/qformer2_128/wandb/debug-internal.log new file mode 120000 index 0000000000000000000000000000000000000000..fd139932ccc3908b3a8f08cf1fb2a48a77cd02dd --- /dev/null +++ b/qformer2_128/wandb/debug-internal.log @@ -0,0 +1 @@ +run-20240312_142853-zo3nkbji/logs/debug-internal.log \ No newline at end of file diff --git a/qformer2_128/wandb/debug.log b/qformer2_128/wandb/debug.log new file mode 120000 index 0000000000000000000000000000000000000000..f28162ea6f3ec08461005487c95239aba3c9136c --- /dev/null +++ b/qformer2_128/wandb/debug.log @@ -0,0 +1 @@ +run-20240312_142853-zo3nkbji/logs/debug.log \ No newline at end of file diff --git a/qformer2_128/wandb/latest-run b/qformer2_128/wandb/latest-run new file mode 120000 index 0000000000000000000000000000000000000000..29a3983a2bfb96a8cab2f3f837c00ae74601193e --- /dev/null +++ b/qformer2_128/wandb/latest-run @@ -0,0 +1 @@ +run-20240312_142853-zo3nkbji \ No newline at end of file diff --git a/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/conda-environment.yaml b/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/conda-environment.yaml new file mode 100644 index 0000000000000000000000000000000000000000..799a6f3a49522423b58a3ae238b95584ecfe64c1 --- /dev/null +++ b/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/conda-environment.yaml @@ -0,0 +1,194 @@ +name: pvlm +channels: + - pytorch + - nvidia + - defaults +dependencies: + - _libgcc_mutex=0.1=main + - _openmp_mutex=5.1=1_gnu + - blas=1.0=mkl + - bzip2=1.0.8=h7b6447c_0 + - ca-certificates=2023.12.12=h06a4308_0 + - certifi=2024.2.2=py310h06a4308_0 + - charset-normalizer=2.0.4=pyhd3eb1b0_0 + - cuda-cudart=11.8.89=0 + - cuda-cupti=11.8.87=0 + - cuda-libraries=11.8.0=0 + - cuda-nvrtc=11.8.89=0 + - cuda-nvtx=11.8.86=0 + - cuda-runtime=11.8.0=0 + - ffmpeg=4.3=hf484d3e_0 + - filelock=3.13.1=py310h06a4308_0 + - freetype=2.12.1=h4a9f257_0 + - gmp=6.2.1=h295c915_3 + - gmpy2=2.1.2=py310heeb90bb_0 + - gnutls=3.6.15=he1e5248_0 + - idna=3.4=py310h06a4308_0 + - intel-openmp=2023.1.0=hdb19cb5_46306 + - jinja2=3.1.3=py310h06a4308_0 + - jpeg=9e=h5eee18b_1 + - lame=3.100=h7b6447c_0 + - lcms2=2.12=h3be6417_0 + - ld_impl_linux-64=2.38=h1181459_1 + - lerc=3.0=h295c915_0 + - libcublas=11.11.3.6=0 + - libcufft=10.9.0.58=0 + - libcufile=1.8.1.2=0 + - libcurand=10.3.4.107=0 + - libcusolver=11.4.1.48=0 + - libcusparse=11.7.5.86=0 + - libdeflate=1.17=h5eee18b_1 + - libffi=3.4.4=h6a678d5_0 + - libgcc-ng=11.2.0=h1234567_1 + - libgomp=11.2.0=h1234567_1 + - libiconv=1.16=h7f8727e_2 + - libidn2=2.3.4=h5eee18b_0 + - libjpeg-turbo=2.0.0=h9bf148f_0 + - libnpp=11.8.0.86=0 + - libnvjpeg=11.9.0.86=0 + - libpng=1.6.39=h5eee18b_0 + - libstdcxx-ng=11.2.0=h1234567_1 + - libtasn1=4.19.0=h5eee18b_0 + - libtiff=4.5.1=h6a678d5_0 + - libunistring=0.9.10=h27cfd23_0 + - libuuid=1.41.5=h5eee18b_0 + - libwebp-base=1.3.2=h5eee18b_0 + - llvm-openmp=14.0.6=h9e868ea_0 + - lz4-c=1.9.4=h6a678d5_0 + - markupsafe=2.1.3=py310h5eee18b_0 + - mkl=2023.1.0=h213fc3f_46344 + - mkl-service=2.4.0=py310h5eee18b_1 + - mkl_fft=1.3.8=py310h5eee18b_0 + - mkl_random=1.2.4=py310hdb19cb5_0 + - mpc=1.1.0=h10f8cd9_1 + - mpfr=4.0.2=hb69a4c5_1 + - mpmath=1.3.0=py310h06a4308_0 + - ncurses=6.4=h6a678d5_0 + - nettle=3.7.3=hbbd107a_1 + - networkx=3.1=py310h06a4308_0 + - numpy=1.26.4=py310h5f9d8c6_0 + - numpy-base=1.26.4=py310hb5e798b_0 + - openh264=2.1.1=h4ff587b_0 + - openjpeg=2.4.0=h3ad879b_0 + - openssl=3.0.13=h7f8727e_0 + - pillow=10.2.0=py310h5eee18b_0 + - pip=23.3.1=py310h06a4308_0 + - python=3.10.13=h955ad1f_0 + - pytorch=2.2.1=py3.10_cuda11.8_cudnn8.7.0_0 + - pytorch-cuda=11.8=h7e8668a_5 + - pytorch-mutex=1.0=cuda + - pyyaml=6.0.1=py310h5eee18b_0 + - readline=8.2=h5eee18b_0 + - requests=2.31.0=py310h06a4308_1 + - setuptools=68.2.2=py310h06a4308_0 + - sqlite=3.41.2=h5eee18b_0 + - sympy=1.12=py310h06a4308_0 + - tbb=2021.8.0=hdb19cb5_0 + - tk=8.6.12=h1ccaba5_0 + - torchaudio=2.2.1=py310_cu118 + - torchtriton=2.2.0=py310 + - torchvision=0.17.1=py310_cu118 + - typing_extensions=4.9.0=py310h06a4308_1 + - urllib3=2.1.0=py310h06a4308_0 + - wheel=0.41.2=py310h06a4308_0 + - xz=5.4.5=h5eee18b_0 + - yaml=0.2.5=h7b6447c_0 + - zlib=1.2.13=h5eee18b_0 + - zstd=1.5.5=hc292b87_0 + - pip: + - accelerate==0.27.2 + - aiohttp==3.9.3 + - aiosignal==1.3.1 + - annotated-types==0.6.0 + - appdirs==1.4.4 + - async-timeout==4.0.3 + - attrs==23.2.0 + - beautifulsoup4==4.12.3 + - blis==0.7.11 + - catalogue==2.0.10 + - click==8.1.7 + - cloudpathlib==0.16.0 + - confection==0.1.4 + - contourpy==1.2.0 + - cycler==0.12.1 + - cymem==2.0.8 + - datasets==2.18.0 + - deepspeed==0.13.4 + - dill==0.3.8 + - docker-pycreds==0.4.0 + - draccus==0.7.1 + - einops==0.7.0 + - flash-attn==2.5.5 + - fonttools==4.49.0 + - frozenlist==1.4.1 + - fsspec==2024.2.0 + - gdown==5.1.0 + - gitdb==4.0.11 + - gitpython==3.1.42 + - hjson==3.1.0 + - huggingface-hub==0.20.3 + - install==1.3.5 + - jsonlines==4.0.0 + - kiwisolver==1.4.5 + - langcodes==3.3.0 + - markdown-it-py==3.0.0 + - matplotlib==3.8.3 + - mdurl==0.1.2 + - mergedeep==1.3.4 + - multidict==6.0.5 + - multiprocess==0.70.16 + - murmurhash==1.0.10 + - mypy-extensions==1.0.0 + - ninja==1.11.1.1 + - packaging==23.2 + - pandas==2.2.1 + - preshed==3.0.9 + - prismatic==0.0.1 + - protobuf==4.25.3 + - psutil==5.9.8 + - py-cpuinfo==9.0.0 + - pyarrow==15.0.0 + - pyarrow-hotfix==0.6 + - pycocoevalcap==1.2 + - pycocotools==2.0.7 + - pydantic==2.6.3 + - pydantic-core==2.16.3 + - pygments==2.17.2 + - pynvml==11.5.0 + - pyparsing==3.1.1 + - pysocks==1.7.1 + - pythainlp==5.0.1 + - python-dateutil==2.9.0.post0 + - pytz==2024.1 + - pyyaml-include==1.3.2 + - regex==2023.12.25 + - rich==13.7.0 + - safetensors==0.4.2 + - sentencepiece==0.2.0 + - sentry-sdk==1.40.5 + - setproctitle==1.3.3 + - six==1.16.0 + - smart-open==6.4.0 + - smmap==5.0.1 + - soupsieve==2.5 + - spacy==3.7.4 + - spacy-legacy==3.0.12 + - spacy-loggers==1.0.5 + - srsly==2.4.8 + - sudachidict-core==20240109 + - sudachipy==0.6.8 + - thinc==8.2.3 + - timm==0.9.16 + - tokenizers==0.15.2 + - toml==0.10.2 + - tqdm==4.66.2 + - transformers==4.38.1 + - typer==0.9.0 + - typing-inspect==0.9.0 + - tzdata==2024.1 + - wandb==0.16.3 + - wasabi==1.1.2 + - weasel==0.3.4 + - xxhash==3.4.1 + - yarl==1.9.4 +prefix: /home/lilei/anaconda3/envs/pvlm diff --git a/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/config.yaml b/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ec83a886b4bae77c04f9eb9c7d4d265de8ca7aa3 --- /dev/null +++ b/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/config.yaml @@ -0,0 +1,119 @@ +wandb_version: 1 + +model: + desc: null + value: + type: one-stage+7b + model_id: one-stage+7b_qformer2_128 + arch_specifier: qformer2_128 + vision_backbone_id: clip-vit-l-336px + llm_backbone_id: vicuna-v15-7b + image_resize_strategy: letterbox + llm_max_length: 2048 + align_epochs: 1 + align_max_steps: null + align_global_batch_size: 256 + align_per_device_batch_size: 16 + align_learning_rate: 0.001 + align_weight_decay: 0.0 + align_max_grad_norm: 1.0 + align_lr_scheduler_type: linear-warmup+cosine-decay + align_warmup_ratio: 0.03 + align_train_strategy: fsdp-shard-grad-op + finetune_epochs: 1 + finetune_max_steps: null + finetune_global_batch_size: 128 + finetune_per_device_batch_size: 16 + finetune_learning_rate: 2.0e-05 + finetune_weight_decay: 0.1 + finetune_max_grad_norm: 1.0 + finetune_lr_scheduler_type: linear-warmup+cosine-decay + finetune_warmup_ratio: 0.03 + finetune_train_strategy: fsdp-full-shard + enable_gradient_checkpointing: true + enable_mixed_precision_training: true + reduce_in_full_precision: false +dataset: + desc: null + value: + type: llava-v15 + dataset_id: llava-v15 + align_stage_components: + - download/llava-laion-cc-sbu-558k/chat.json + - download/llava-laion-cc-sbu-558k + finetune_stage_components: + - download/llava-v1.5-instruct/llava_v1_5_mix665k.json + - download/llava-v1.5-instruct + dataset_root_dir: data +stage: + desc: null + value: finetune +pretrained_checkpoint: + desc: null + value: null +run_id: + desc: null + value: qformer2_128 +run_root_dir: + desc: null + value: runs +seed: + desc: null + value: 7 +hf_token: + desc: null + value: .hf_token +trackers: + desc: null + value: + - jsonl + - wandb +wandb_project: + desc: null + value: hf-vlms +wandb_entity: + desc: null + value: lilei_stones +max_length: + desc: null + value: 4096 +_wandb: + desc: null + value: + python_version: 3.10.13 + cli_version: 0.16.3 + framework: huggingface + huggingface_version: 4.38.1 + is_jupyter_run: false + is_kaggle_kernel: false + start_time: 1710253733.688548 + t: + 1: + - 1 + - 11 + - 41 + - 49 + - 51 + - 55 + - 63 + - 71 + 2: + - 1 + - 11 + - 41 + - 49 + - 51 + - 55 + - 63 + - 71 + 3: + - 2 + - 13 + - 16 + - 23 + 4: 3.10.13 + 5: 0.16.3 + 6: 4.38.1 + 8: + - 5 + 13: linux-x86_64 diff --git a/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log b/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..9f173b428c225e5fcd923bc1c6e32777073806e6 --- /dev/null +++ b/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log @@ -0,0 +1,5203 @@ + +03/12 [14:28:59] INFO  | >> [*] Starting Training Loop ]8;id=72103;file:///home/lilei/prismatic-vlms/scripts/pretrain.py\pretrain.py]8;;\:]8;id=880770;file:///home/lilei/prismatic-vlms/scripts/pretrain.py#226\226]8;;\ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +=>> [Global Step] 005198 =>> LR :: 0.000000 -- Loss :: 0.7976: 100%|█████████████████████████████████████████████| 5198/5198 [7:06:02<00:00, 4.68s/it] +03/12 [21:39:13] INFO  | >> [*] Done with Training =>> Finalizing Metrics ]8;id=696414;file:///home/lilei/prismatic-vlms/scripts/pretrain.py\pretrain.py]8;;\:]8;id=68157;file:///home/lilei/prismatic-vlms/scripts/pretrain.py#230\230]8;;\ \ No newline at end of file diff --git a/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/requirements.txt b/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..52215be56573f749927d1265d23e43a5fe36d5b1 --- /dev/null +++ b/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/requirements.txt @@ -0,0 +1,122 @@ +accelerate==0.27.2 +aiohttp==3.9.3 +aiosignal==1.3.1 +annotated-types==0.6.0 +appdirs==1.4.4 +async-timeout==4.0.3 +attrs==23.2.0 +beautifulsoup4==4.12.3 +blis==0.7.11 +catalogue==2.0.10 +certifi==2024.2.2 +charset-normalizer==2.0.4 +click==8.1.7 +cloudpathlib==0.16.0 +confection==0.1.4 +contourpy==1.2.0 +cycler==0.12.1 +cymem==2.0.8 +datasets==2.18.0 +deepspeed==0.13.4 +dill==0.3.8 +docker-pycreds==0.4.0 +draccus==0.7.1 +einops==0.7.0 +filelock==3.13.1 +flash-attn==2.5.5 +fonttools==4.49.0 +frozenlist==1.4.1 +fsspec==2024.2.0 +gdown==5.1.0 +gitdb==4.0.11 +gitpython==3.1.42 +gmpy2==2.1.2 +hjson==3.1.0 +huggingface-hub==0.20.3 +idna==3.4 +install==1.3.5 +jinja2==3.1.3 +jsonlines==4.0.0 +kiwisolver==1.4.5 +langcodes==3.3.0 +markdown-it-py==3.0.0 +markupsafe==2.1.3 +matplotlib==3.8.3 +mdurl==0.1.2 +mergedeep==1.3.4 +mkl-fft==1.3.8 +mkl-random==1.2.4 +mkl-service==2.4.0 +mpmath==1.3.0 +multidict==6.0.5 +multiprocess==0.70.16 +murmurhash==1.0.10 +mypy-extensions==1.0.0 +networkx==3.1 +ninja==1.11.1.1 +numpy==1.26.4 +ocifs==1.3.1 +packaging==23.2 +pandas==2.2.1 +pillow==10.2.0 +pip==23.3.1 +preshed==3.0.9 +prismatic==0.0.1 +protobuf==4.25.3 +psutil==5.9.8 +py-cpuinfo==9.0.0 +pyarrow-hotfix==0.6 +pyarrow==15.0.0 +pycocoevalcap==1.2 +pycocotools==2.0.7 +pydantic-core==2.16.3 +pydantic==2.6.3 +pygments==2.17.2 +pynvml==11.5.0 +pyparsing==3.1.1 +pysocks==1.7.1 +pythainlp==5.0.1 +python-dateutil==2.9.0.post0 +pytz==2024.1 +pyyaml-include==1.3.2 +pyyaml==6.0.1 +regex==2023.12.25 +requests==2.31.0 +rich==13.7.0 +safetensors==0.4.2 +sentencepiece==0.2.0 +sentry-sdk==1.40.5 +setproctitle==1.3.3 +setuptools==68.2.2 +six==1.16.0 +smart-open==6.4.0 +smmap==5.0.1 +soupsieve==2.5 +spacy-legacy==3.0.12 +spacy-loggers==1.0.5 +spacy==3.7.4 +srsly==2.4.8 +sudachidict-core==20240109 +sudachipy==0.6.8 +sympy==1.12 +thinc==8.2.3 +timm==0.9.16 +tokenizers==0.15.2 +toml==0.10.2 +torch==2.2.1 +torchaudio==2.2.1 +torchvision==0.17.1 +tqdm==4.66.2 +transformers==4.38.1 +triton==2.2.0 +typer==0.9.0 +typing-extensions==4.9.0 +typing-inspect==0.9.0 +tzdata==2024.1 +urllib3==2.1.0 +wandb==0.16.3 +wasabi==1.1.2 +weasel==0.3.4 +wheel==0.41.2 +xxhash==3.4.1 +yarl==1.9.4 \ No newline at end of file diff --git a/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-metadata.json b/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..25e3ed4dc7766b51f87cc2e1008e7498c3c01ee6 --- /dev/null +++ b/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-metadata.json @@ -0,0 +1,1360 @@ +{ + "os": "Linux-5.15.0-1027-oracle-x86_64-with-glibc2.31", + "python": "3.10.13", + "heartbeatAt": "2024-03-12T14:28:54.394423", + "startedAt": "2024-03-12T14:28:53.639298", + "docker": null, + "cuda": null, + "args": [ + "--model.type", + "one-stage+7b", + "--model.model_id", + "one-stage+7b_qformer2_128", + "--run_id", + "qformer2_128", + "--model.arch_specifier", + "qformer2_128" + ], + "state": "running", + "program": "/home/lilei/prismatic-vlms/scripts/pretrain.py", + "codePathLocal": "scripts/pretrain.py", + "codePath": "scripts/pretrain.py", + "git": { + "remote": "https://github.com/TRI-ML/prismatic-vlms.git", + "commit": "b1c33c284c6e76d908c29e5545b00f5b8b4c601f" + }, + "email": "tobiaslee@qq.com", + "root": "/home/lilei/prismatic-vlms", + "host": "compute-permanent-node-815", + "username": "lilei", + "executable": "/home/lilei/anaconda3/envs/pvlm/bin/python", + "cpu_count": 128, + "cpu_count_logical": 255, + "cpu_freq": { + "current": 2571.1995686274513, + "min": 1500.0, + "max": 2550.0 + }, + "cpu_freq_per_core": [ + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 3248.918, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 3248.707, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 3249.201, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 3249.098, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 3248.2, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 3245.15, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 3248.386, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 3248.237, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 3248.479, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + } + ], + "disk": { + "/": { + "total": 242.1086883544922, + "used": 55.302242279052734 + } + }, + "gpu": "NVIDIA A100-SXM4-80GB", + "gpu_count": 8, + "gpu_devices": [ + { + "name": "NVIDIA A100-SXM4-80GB", + "memory_total": 85899345920 + }, + { + "name": "NVIDIA A100-SXM4-80GB", + "memory_total": 85899345920 + }, + { + "name": "NVIDIA A100-SXM4-80GB", + "memory_total": 85899345920 + }, + { + "name": "NVIDIA A100-SXM4-80GB", + "memory_total": 85899345920 + }, + { + "name": "NVIDIA A100-SXM4-80GB", + "memory_total": 85899345920 + }, + { + "name": "NVIDIA A100-SXM4-80GB", + "memory_total": 85899345920 + }, + { + "name": "NVIDIA A100-SXM4-80GB", + "memory_total": 85899345920 + }, + { + "name": "NVIDIA A100-SXM4-80GB", + "memory_total": 85899345920 + } + ], + "memory": { + "total": 2015.682991027832 + } +} diff --git a/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json b/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json new file mode 100644 index 0000000000000000000000000000000000000000..0b62c46c4295ce14a349b610dd0cff69059c0517 --- /dev/null +++ b/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json @@ -0,0 +1 @@ +{"Finetune/Step": 5198, "Finetune/Loss": 0.7975542545318604, "Finetune/Loss (Raw)": 0.8646546006202698, "Finetune/Learning Rate": 0.0, "Finetune/Step Time": 5.1002165004611015, "_timestamp": 1710279305.2482264, "_runtime": 25571.559678316116, "_step": 5198, "_wandb": {"runtime": 25819}} \ No newline at end of file diff --git a/qformer2_128/wandb/run-20240312_142853-zo3nkbji/logs/debug-internal.log b/qformer2_128/wandb/run-20240312_142853-zo3nkbji/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..25d27cb1f1b738da5df5c4bc18a2f0ed5a35dde8 --- /dev/null +++ b/qformer2_128/wandb/run-20240312_142853-zo3nkbji/logs/debug-internal.log @@ -0,0 +1,42131 @@ +2024-03-12 14:28:53,689 INFO StreamThr :1827008 [internal.py:wandb_internal():86] W&B internal server running at pid: 1827008, started at: 2024-03-12 14:28:53.687253 +2024-03-12 14:28:53,690 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status +2024-03-12 14:28:53,701 INFO WriterThread:1827008 [datastore.py:open_for_write():87] open: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/run-zo3nkbji.wandb +2024-03-12 14:28:53,703 DEBUG SenderThread:1827008 [sender.py:send():382] send: header +2024-03-12 14:28:53,720 DEBUG SenderThread:1827008 [sender.py:send():382] send: run +2024-03-12 14:28:54,216 INFO SenderThread:1827008 [dir_watcher.py:__init__():211] watching files in: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files +2024-03-12 14:28:54,216 INFO SenderThread:1827008 [sender.py:_start_run_threads():1136] run started: zo3nkbji with start time 1710253733.688548 +2024-03-12 14:28:54,230 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: check_version +2024-03-12 14:28:54,230 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: check_version +2024-03-12 14:28:54,280 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: run_start +2024-03-12 14:28:54,290 DEBUG HandlerThread:1827008 [system_info.py:__init__():27] System info init +2024-03-12 14:28:54,290 DEBUG HandlerThread:1827008 [system_info.py:__init__():42] System info init done +2024-03-12 14:28:54,290 INFO HandlerThread:1827008 [system_monitor.py:start():194] Starting system monitor +2024-03-12 14:28:54,290 INFO SystemMonitor:1827008 [system_monitor.py:_start():158] Starting system asset monitoring threads +2024-03-12 14:28:54,291 INFO HandlerThread:1827008 [system_monitor.py:probe():214] Collecting system info +2024-03-12 14:28:54,291 INFO SystemMonitor:1827008 [interfaces.py:start():190] Started cpu monitoring +2024-03-12 14:28:54,292 INFO SystemMonitor:1827008 [interfaces.py:start():190] Started disk monitoring +2024-03-12 14:28:54,292 INFO SystemMonitor:1827008 [interfaces.py:start():190] Started gpu monitoring +2024-03-12 14:28:54,293 INFO SystemMonitor:1827008 [interfaces.py:start():190] Started memory monitoring +2024-03-12 14:28:54,294 INFO SystemMonitor:1827008 [interfaces.py:start():190] Started network monitoring +2024-03-12 14:28:54,394 DEBUG HandlerThread:1827008 [system_info.py:probe():151] Probing system +2024-03-12 14:28:54,398 DEBUG HandlerThread:1827008 [system_info.py:_probe_git():136] Probing git +2024-03-12 14:28:54,419 DEBUG HandlerThread:1827008 [system_info.py:_probe_git():144] Probing git done +2024-03-12 14:28:54,419 DEBUG HandlerThread:1827008 [system_info.py:probe():199] Probing system done +2024-03-12 14:28:54,419 DEBUG HandlerThread:1827008 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-1027-oracle-x86_64-with-glibc2.31', 'python': '3.10.13', 'heartbeatAt': '2024-03-12T14:28:54.394423', 'startedAt': '2024-03-12T14:28:53.639298', 'docker': None, 'cuda': None, 'args': ('--model.type', 'one-stage+7b', '--model.model_id', 'one-stage+7b_qformer2_128', '--run_id', 'qformer2_128', '--model.arch_specifier', 'qformer2_128'), 'state': 'running', 'program': '/home/lilei/prismatic-vlms/scripts/pretrain.py', 'codePathLocal': 'scripts/pretrain.py', 'codePath': 'scripts/pretrain.py', 'git': {'remote': 'https://github.com/TRI-ML/prismatic-vlms.git', 'commit': 'b1c33c284c6e76d908c29e5545b00f5b8b4c601f'}, 'email': 'tobiaslee@qq.com', 'root': '/home/lilei/prismatic-vlms', 'host': 'compute-permanent-node-815', 'username': 'lilei', 'executable': '/home/lilei/anaconda3/envs/pvlm/bin/python', 'cpu_count': 128, 'cpu_count_logical': 255, 'cpu_freq': {'current': 2571.1995686274513, 'min': 1500.0, 'max': 2550.0}, 'cpu_freq_per_core': [{'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 3248.918, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 3248.707, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 3249.201, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 3249.098, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 3248.2, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 3245.15, 'min': 1500.0, 'max': 2550.0}, {'current': 3248.386, 'min': 1500.0, 'max': 2550.0}, {'current': 3248.237, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 3248.479, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}], 'disk': {'/': {'total': 242.1086883544922, 'used': 55.302242279052734}}, 'gpu': 'NVIDIA A100-SXM4-80GB', 'gpu_count': 8, 'gpu_devices': [{'name': 'NVIDIA A100-SXM4-80GB', 'memory_total': 85899345920}, {'name': 'NVIDIA A100-SXM4-80GB', 'memory_total': 85899345920}, {'name': 'NVIDIA A100-SXM4-80GB', 'memory_total': 85899345920}, {'name': 'NVIDIA A100-SXM4-80GB', 'memory_total': 85899345920}, {'name': 'NVIDIA A100-SXM4-80GB', 'memory_total': 85899345920}, {'name': 'NVIDIA A100-SXM4-80GB', 'memory_total': 85899345920}, {'name': 'NVIDIA A100-SXM4-80GB', 'memory_total': 85899345920}, {'name': 'NVIDIA A100-SXM4-80GB', 'memory_total': 85899345920}], 'memory': {'total': 2015.682991027832}} +2024-03-12 14:28:54,420 INFO HandlerThread:1827008 [system_monitor.py:probe():224] Finished collecting system info +2024-03-12 14:28:54,420 INFO HandlerThread:1827008 [system_monitor.py:probe():227] Publishing system info +2024-03-12 14:28:54,420 DEBUG HandlerThread:1827008 [system_info.py:_save_conda():208] Saving list of conda packages installed into the current environment +2024-03-12 14:28:55,219 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_created():271] file/dir created: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/conda-environment.yaml +2024-03-12 14:28:59,025 DEBUG HandlerThread:1827008 [system_info.py:_save_conda():220] Saving conda packages done +2024-03-12 14:28:59,034 INFO HandlerThread:1827008 [system_monitor.py:probe():229] Finished publishing system info +2024-03-12 14:28:59,040 DEBUG SenderThread:1827008 [sender.py:send():382] send: files +2024-03-12 14:28:59,040 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-metadata.json with policy now +2024-03-12 14:28:59,041 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:28:59,051 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: python_packages +2024-03-12 14:28:59,051 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: python_packages +2024-03-12 14:28:59,052 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:28:59,053 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:28:59,055 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:28:59,222 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/conda-environment.yaml +2024-03-12 14:28:59,223 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_created():271] file/dir created: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-metadata.json +2024-03-12 14:28:59,223 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_created():271] file/dir created: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/requirements.txt +2024-03-12 14:28:59,296 DEBUG SenderThread:1827008 [sender.py:send():382] send: telemetry +2024-03-12 14:29:00,132 INFO wandb-upload_0:1827008 [upload_job.py:push():131] Uploaded file /tmp/tmpeemnmoqdwandb/r7dv6dtt-wandb-metadata.json +2024-03-12 14:29:00,224 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_created():271] file/dir created: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:29:02,227 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:29:04,454 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:29:05,231 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:29:09,455 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:29:14,052 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:29:14,052 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:29:14,052 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:29:15,253 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:29:20,253 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:29:24,996 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:29:26,044 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:29:27,250 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/config.yaml +2024-03-12 14:29:27,250 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:29:29,052 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:29:29,052 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:29:29,054 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:29:31,319 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:29:32,162 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:29:32,163 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:29:32,164 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:29:32,167 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:29:32,255 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_created():271] file/dir created: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:29:35,259 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:29:35,809 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:29:35,810 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:29:35,810 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:29:35,815 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:29:36,259 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:29:36,815 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:29:37,262 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:29:39,599 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:29:39,600 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:29:39,601 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:29:39,606 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:29:40,264 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:29:41,266 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:29:42,607 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:29:44,052 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:29:44,052 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:29:44,054 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:29:44,457 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:29:44,457 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:29:44,458 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:29:44,463 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:29:45,268 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:29:47,272 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:29:48,501 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:29:48,804 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:29:48,805 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:29:48,806 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:29:48,810 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:29:49,273 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:29:51,277 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:29:53,137 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:29:53,138 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:29:53,138 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:29:53,143 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:29:53,278 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:29:54,143 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:29:54,294 DEBUG SystemMonitor:1827008 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2024-03-12 14:29:54,297 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 14:29:55,281 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:29:58,740 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:29:58,740 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:29:58,741 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:29:58,746 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:29:59,052 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:29:59,052 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:29:59,054 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:29:59,280 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:29:59,285 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:30:01,288 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:30:03,134 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:30:03,135 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:30:03,135 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:30:03,140 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:30:03,289 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:30:05,177 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:30:05,292 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:30:10,014 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:30:10,014 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:30:10,015 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:30:10,019 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:30:10,295 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:30:11,020 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:30:11,298 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:30:14,052 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:30:14,052 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:30:14,054 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:30:14,085 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:30:14,348 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:30:14,349 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:30:14,354 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:30:15,300 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:30:16,354 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:30:17,303 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:30:18,297 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:30:18,298 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:30:18,298 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:30:18,303 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:30:18,304 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:30:21,307 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:30:22,303 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:30:22,475 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:30:22,476 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:30:22,476 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:30:22,481 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:30:23,309 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:30:24,300 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 14:30:25,312 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:30:26,253 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:30:26,254 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:30:26,254 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:30:26,259 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:30:26,313 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:30:28,260 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:30:29,052 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:30:29,052 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:30:29,054 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:30:29,316 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:30:31,537 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:30:31,538 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:30:31,538 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:30:31,543 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:30:32,318 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:30:33,321 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:30:33,589 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:30:37,001 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:30:37,002 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:30:37,003 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:30:37,008 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:30:37,323 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:30:39,009 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:30:39,326 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:30:41,384 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:30:41,385 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:30:41,385 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:30:41,390 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:30:42,328 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:30:43,331 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:30:44,052 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:30:44,053 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:30:44,054 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:30:44,358 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:30:46,228 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:30:46,229 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:30:46,229 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:30:46,234 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:30:46,333 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:30:49,337 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:30:50,101 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:30:50,102 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:30:50,103 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:30:50,103 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:30:50,108 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:30:50,338 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:30:53,341 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:30:53,702 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:30:53,703 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:30:53,703 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:30:53,708 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:30:54,303 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 14:30:54,342 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:30:55,304 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:30:55,344 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:30:59,052 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:30:59,053 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:30:59,053 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:30:59,186 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:30:59,346 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:30:59,346 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:30:59,351 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:30:59,351 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:31:00,352 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:31:01,354 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:31:03,527 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:31:03,528 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:31:03,528 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:31:03,533 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:31:04,356 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:31:05,359 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:31:05,533 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:31:10,534 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:31:12,028 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:31:12,028 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:31:12,029 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:31:12,034 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:31:12,363 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:31:13,365 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:31:14,052 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:31:14,053 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:31:14,054 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:31:15,743 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:31:15,744 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:31:15,744 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:31:15,745 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:31:15,750 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:31:16,367 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:31:17,369 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:31:20,095 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:31:20,096 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:31:20,096 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:31:20,101 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:31:20,372 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:31:21,101 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:31:21,372 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:31:24,306 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 14:31:26,307 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:31:29,052 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:31:29,053 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:31:29,054 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:31:29,394 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:31:29,395 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:31:29,395 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:31:29,400 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:31:30,384 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:31:31,386 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:31:31,437 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:31:32,821 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:31:32,822 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:31:32,822 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:31:32,827 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:31:33,388 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:31:35,391 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:31:36,570 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:31:36,571 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:31:36,571 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:31:36,571 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:31:36,576 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:31:37,392 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:31:39,395 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:31:41,577 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:31:41,812 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:31:41,813 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:31:41,813 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:31:41,818 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:31:42,397 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:31:43,399 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:31:44,053 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:31:44,053 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:31:44,054 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:31:47,109 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:31:47,110 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:31:47,110 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:31:47,111 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:31:47,115 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:31:47,403 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:31:49,404 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:31:51,588 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:31:51,589 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:31:51,590 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:31:51,595 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:31:52,406 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:31:52,595 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:31:53,408 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:31:54,317 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 14:31:57,892 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:31:57,893 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:31:57,893 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:31:57,893 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:31:57,898 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:31:58,411 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:31:59,053 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:31:59,053 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:31:59,055 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:31:59,414 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:32:02,641 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:32:02,642 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:32:02,642 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:32:02,647 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:32:03,416 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:32:03,685 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:32:05,419 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:32:06,779 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:32:06,780 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:32:06,780 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:32:06,785 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:32:07,421 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:32:08,786 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:32:09,424 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:32:11,233 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:32:11,234 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:32:11,235 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:32:11,240 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:32:11,425 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:32:13,428 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:32:14,053 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:32:14,053 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:32:14,054 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:32:14,338 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:32:16,983 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:32:16,984 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:32:16,984 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:32:16,989 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:32:17,430 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:32:19,431 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:32:20,029 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:32:24,280 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:32:24,281 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:32:24,281 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:32:24,286 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:32:24,311 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 14:32:24,435 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:32:25,312 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:32:27,439 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:32:28,044 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:32:28,044 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:32:28,045 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:32:28,050 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:32:28,440 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:32:29,053 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:32:29,053 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:32:29,055 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:32:29,443 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:32:30,321 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:32:31,786 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:32:31,786 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:32:31,787 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:32:31,792 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:32:32,444 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:32:33,447 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:32:35,829 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:32:37,421 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:32:37,422 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:32:37,423 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:32:37,428 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:32:37,449 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:32:39,452 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:32:40,617 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:32:40,618 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:32:40,618 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:32:40,623 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:32:41,454 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:32:41,624 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:32:43,456 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:32:44,053 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:32:44,053 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:32:44,055 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:32:44,354 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:32:44,354 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:32:44,354 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:32:44,359 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:32:44,458 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:32:47,397 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:32:47,462 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:32:49,566 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:32:49,567 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:32:49,567 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:32:49,572 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:32:50,464 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:32:51,466 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:32:52,573 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:32:54,319 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 14:32:54,489 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:32:54,490 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:32:54,490 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:32:54,495 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:32:55,468 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:32:57,471 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:32:58,496 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:32:59,053 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:32:59,053 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:32:59,055 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:33:02,361 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:33:02,362 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:33:02,362 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:33:02,368 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:33:02,474 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:33:04,405 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:33:05,478 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:33:09,405 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:33:12,626 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:33:12,626 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:33:12,627 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:33:12,632 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:33:13,483 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:33:14,053 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:33:14,053 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:33:14,055 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:33:15,356 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:33:15,486 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:33:17,527 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:33:17,528 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:33:17,528 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:33:17,533 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:33:18,488 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:33:19,490 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:33:20,573 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:33:23,245 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:33:23,246 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:33:23,246 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:33:23,251 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:33:23,493 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:33:24,315 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 14:33:25,496 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:33:26,316 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:33:27,459 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:33:27,460 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:33:27,460 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:33:27,465 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:33:27,498 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:33:29,053 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:33:29,054 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:33:29,055 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:33:29,502 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:33:31,812 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:33:31,813 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:33:31,813 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:33:31,813 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:33:31,818 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:33:32,503 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:33:33,506 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:33:35,805 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:33:35,806 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:33:35,807 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:33:35,812 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:33:36,508 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:33:37,508 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:33:37,812 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:33:40,703 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:33:40,704 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:33:40,704 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:33:40,709 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:33:41,511 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:33:43,514 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:33:43,710 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:33:44,053 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:33:44,054 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:33:44,055 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:33:46,768 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:33:46,769 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:33:46,769 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:33:46,774 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:33:47,518 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:33:48,813 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:33:49,521 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:33:51,835 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:33:51,836 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:33:51,836 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:33:51,841 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:33:52,523 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:33:53,525 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:33:53,841 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:33:54,318 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 14:33:57,280 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:33:57,281 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:33:57,281 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:33:57,286 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:33:57,527 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:33:59,053 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:33:59,054 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:33:59,055 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:33:59,356 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:33:59,529 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:34:03,195 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:34:03,201 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:34:03,202 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:34:03,206 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:34:03,531 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:34:05,237 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:34:05,534 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:34:07,262 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:34:07,263 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:34:07,263 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:34:07,268 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:34:07,535 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:34:09,538 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:34:10,268 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:34:14,053 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:34:14,054 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:34:14,055 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:34:14,169 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:34:14,256 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:34:14,256 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:34:14,261 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:34:14,541 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:34:15,542 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:34:16,261 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:34:19,996 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:34:19,997 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:34:19,997 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:34:20,002 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:34:20,545 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:34:21,547 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:34:22,003 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:34:23,870 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:34:23,871 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:34:23,871 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:34:23,876 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:34:24,320 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 14:34:24,550 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:34:25,552 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:34:27,321 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:34:28,291 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:34:28,292 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:34:28,292 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:34:28,297 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:34:28,555 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:34:29,054 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:34:29,054 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:34:29,056 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:34:29,558 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:34:32,068 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:34:32,069 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:34:32,069 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:34:32,074 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:34:32,559 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:34:33,113 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:34:33,562 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:34:36,120 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:34:36,121 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:34:36,121 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:34:36,126 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:34:36,564 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:34:37,566 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:34:38,126 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:34:43,127 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:34:43,678 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:34:43,679 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:34:43,679 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:34:43,684 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:34:44,054 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:34:44,054 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:34:44,055 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:34:44,571 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:34:45,574 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:34:48,368 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:34:48,644 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:34:48,645 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:34:48,645 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:34:48,650 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:34:49,576 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:34:51,579 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:34:53,062 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:34:53,063 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:34:53,063 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:34:53,068 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:34:53,580 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:34:54,068 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:34:54,323 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 14:34:55,581 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:34:56,732 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:34:56,733 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:34:56,733 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:34:56,738 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:34:57,582 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:34:59,054 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:34:59,054 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:34:59,056 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:34:59,276 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:34:59,586 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:35:01,842 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:35:01,843 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:35:01,843 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:35:01,848 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:35:02,588 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:35:03,590 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:35:04,885 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:35:05,520 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:35:05,521 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:35:05,521 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:35:05,526 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:35:05,592 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:35:07,593 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:35:09,435 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:35:09,436 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:35:09,436 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:35:09,441 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:35:09,594 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:35:10,441 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:35:11,597 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:35:13,471 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:35:13,472 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:35:13,472 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:35:13,478 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:35:13,598 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:35:14,054 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:35:14,054 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:35:14,055 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:35:15,601 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:35:16,367 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:35:17,034 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:35:17,035 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:35:17,035 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:35:17,040 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:35:17,602 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:35:19,606 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:35:20,909 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:35:20,910 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:35:20,910 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:35:20,916 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:35:21,607 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:35:21,916 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:35:23,610 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:35:24,332 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 14:35:24,889 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:35:24,890 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:35:24,890 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:35:24,895 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:35:25,612 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:35:27,616 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:35:27,896 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:35:29,054 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:35:29,054 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:35:29,056 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:35:29,637 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:35:29,637 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:35:29,638 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:35:29,642 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:35:30,617 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:35:31,620 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:35:33,681 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:35:34,335 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:35:34,336 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:35:34,337 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:35:34,342 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:35:34,621 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:35:35,624 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:35:37,875 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:35:37,876 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:35:37,876 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:35:37,881 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:35:38,626 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:35:38,881 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:35:39,629 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:35:41,455 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:35:41,456 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:35:41,457 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:35:41,461 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:35:41,630 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:35:43,633 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:35:44,054 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:35:44,054 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:35:44,056 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:35:44,294 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:35:45,391 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:35:45,392 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:35:45,392 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:35:45,397 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:35:45,635 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:35:47,638 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:35:49,433 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:35:50,999 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:35:51,000 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:35:51,001 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:35:51,005 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:35:51,641 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:35:53,644 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:35:54,330 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 14:35:55,331 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:35:55,934 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:35:55,935 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:35:55,935 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:35:55,940 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:35:56,646 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:35:57,648 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:35:59,054 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:35:59,054 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:35:59,056 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:35:59,733 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:35:59,733 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:35:59,734 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:35:59,739 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:36:00,650 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:36:00,777 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:36:01,653 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:36:05,539 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:36:05,540 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:36:05,541 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:36:05,545 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:36:05,655 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:36:06,546 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:36:07,658 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:36:10,831 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:36:10,832 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:36:10,832 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:36:10,837 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:36:11,661 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:36:11,838 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:36:13,664 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:36:14,054 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:36:14,054 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:36:14,056 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:36:14,791 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:36:14,791 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:36:14,792 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:36:14,796 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:36:15,665 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:36:17,668 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:36:17,833 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:36:22,834 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:36:24,105 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:36:24,106 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:36:24,106 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:36:24,111 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:36:24,337 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 14:36:24,672 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:36:25,675 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:36:27,989 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:36:27,989 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:36:27,990 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:36:27,990 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:36:27,995 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:36:28,677 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:36:29,054 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:36:29,055 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:36:29,056 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:36:29,680 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:36:33,359 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:36:33,828 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:36:33,829 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:36:33,829 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:36:33,834 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:36:34,683 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:36:35,686 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:36:37,791 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:36:37,792 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:36:37,792 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:36:37,797 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:36:38,688 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:36:38,797 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:36:39,690 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:36:43,662 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:36:43,663 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:36:43,663 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:36:43,668 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:36:43,692 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:36:44,055 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:36:44,055 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:36:44,055 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:36:44,057 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:36:45,697 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:36:49,067 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:36:49,068 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:36:49,068 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:36:49,068 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:36:49,073 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:36:49,699 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:36:51,702 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:36:53,197 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:36:53,198 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:36:53,198 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:36:53,203 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:36:53,704 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:36:54,204 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:36:54,334 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 14:36:55,707 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:36:57,592 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:36:57,593 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:36:57,593 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:36:57,598 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:36:57,708 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:36:59,054 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:36:59,055 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:36:59,056 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:36:59,294 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:36:59,710 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:37:01,128 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:37:01,129 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:37:01,129 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:37:01,134 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:37:01,711 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:37:03,714 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:37:05,119 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:37:05,120 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:37:05,120 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:37:05,120 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:37:05,125 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:37:05,715 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:37:07,718 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:37:10,126 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:37:10,506 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:37:10,507 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:37:10,507 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:37:10,512 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:37:10,720 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:37:13,724 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:37:14,054 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:37:14,055 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:37:14,056 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:37:15,123 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:37:15,124 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:37:15,124 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:37:15,129 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:37:15,129 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:37:15,725 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:37:17,728 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:37:19,059 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:37:19,060 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:37:19,060 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:37:19,065 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:37:19,730 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:37:21,066 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:37:21,732 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:37:24,336 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 14:37:26,337 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:37:27,338 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:37:27,344 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:37:27,345 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:37:27,350 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:37:27,737 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:37:29,055 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:37:29,055 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:37:29,057 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:37:29,740 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:37:31,733 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:37:31,733 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:37:31,734 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:37:31,734 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:37:31,739 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:37:31,741 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:37:33,745 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:37:36,777 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:37:36,979 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:37:36,980 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:37:36,980 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:37:36,985 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:37:37,747 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:37:39,750 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:37:40,367 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:37:40,368 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:37:40,368 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:37:40,373 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:37:40,751 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:37:41,753 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:37:42,373 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:37:44,055 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:37:44,055 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:37:44,057 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:37:48,323 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:37:48,796 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:37:48,797 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:37:48,797 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:37:48,802 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:37:49,760 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:37:51,762 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:37:52,548 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:37:52,549 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:37:52,549 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:37:52,554 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:37:52,763 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:37:53,554 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:37:54,338 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 14:37:55,767 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:37:55,889 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:37:55,890 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:37:55,890 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:37:55,895 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:37:56,767 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:37:57,770 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:37:58,896 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:37:59,055 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:37:59,055 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:37:59,056 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:37:59,649 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:37:59,650 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:37:59,650 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:37:59,655 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:37:59,771 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:38:01,774 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:38:03,492 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:38:03,493 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:38:03,493 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:38:03,498 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:38:03,775 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:38:04,499 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:38:05,778 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:38:07,627 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:38:07,628 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:38:07,628 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:38:07,633 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:38:07,779 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:38:09,633 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:38:09,781 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:38:12,161 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:38:12,162 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:38:12,163 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:38:12,168 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:38:12,782 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:38:13,785 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:38:14,055 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:38:14,055 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:38:14,057 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:38:15,271 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:38:15,868 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:38:15,869 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:38:15,869 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:38:15,874 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:38:16,787 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:38:17,790 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:38:20,913 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:38:21,531 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:38:21,532 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:38:21,532 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:38:21,537 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:38:21,792 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:38:23,796 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:38:24,340 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 14:38:26,342 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:38:26,715 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:38:26,716 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:38:26,716 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:38:26,721 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:38:26,799 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:38:29,055 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:38:29,056 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:38:29,057 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:38:29,803 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:38:32,085 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:38:32,086 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:38:32,086 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:38:32,086 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:38:32,091 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:38:32,805 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:38:33,807 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:38:37,130 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:38:41,807 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:38:41,808 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:38:41,808 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:38:41,813 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:38:41,814 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:38:42,814 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:38:43,817 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:38:44,055 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:38:44,056 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:38:44,057 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:38:46,402 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:38:46,402 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:38:46,403 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:38:46,407 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:38:46,819 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:38:47,822 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:38:48,445 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:38:52,772 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:38:52,773 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:38:52,773 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:38:52,778 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:38:52,826 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:38:53,779 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:38:54,348 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 14:38:55,829 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:38:57,239 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:38:57,240 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:38:57,240 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:38:57,245 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:38:57,830 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:38:59,055 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:38:59,056 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:38:59,056 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:38:59,057 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:38:59,834 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:39:01,074 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:39:01,074 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:39:01,074 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:39:01,079 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:39:01,835 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:39:03,838 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:39:04,117 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:39:05,101 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:39:05,102 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:39:05,102 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:39:05,108 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:39:05,840 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:39:07,843 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:39:09,657 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:39:09,658 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:39:09,658 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:39:09,658 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:39:09,663 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:39:09,844 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:39:11,848 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:39:14,055 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:39:14,056 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:39:14,057 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:39:15,350 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:39:15,703 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:39:15,704 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:39:15,704 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:39:15,709 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:39:15,851 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:39:17,854 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:39:19,511 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:39:19,512 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:39:19,512 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:39:19,518 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:39:19,856 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:39:20,518 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:39:21,860 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:39:23,359 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:39:23,360 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:39:23,360 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:39:23,365 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:39:23,862 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:39:24,344 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 14:39:25,865 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:39:26,345 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:39:27,548 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:39:27,549 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:39:27,550 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:39:27,555 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:39:27,868 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:39:29,055 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:39:29,056 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:39:29,057 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:39:29,871 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:39:31,703 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:39:31,703 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:39:31,704 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:39:31,704 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:39:31,709 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:39:31,873 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:39:33,876 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:39:36,745 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:39:38,172 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:39:38,173 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:39:38,173 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:39:38,178 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:39:38,879 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:39:39,883 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:39:41,913 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:39:41,914 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:39:41,914 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:39:41,914 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:39:41,920 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:39:42,885 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:39:43,888 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:39:44,055 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:39:44,056 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:39:44,057 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:39:46,061 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:39:46,061 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:39:46,062 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:39:46,067 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:39:46,891 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:39:47,105 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:39:47,894 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:39:49,965 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:39:49,966 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:39:49,966 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:39:49,971 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:39:50,896 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:39:51,898 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:39:52,972 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:39:54,353 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 14:39:54,984 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:39:54,985 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:39:54,985 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:39:54,990 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:39:55,901 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:39:57,904 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:39:57,991 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:39:58,596 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:39:58,597 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:39:58,597 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:39:58,602 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:39:58,905 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:39:59,056 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:39:59,056 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:39:59,057 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:39:59,908 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:40:02,427 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:40:02,428 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:40:02,428 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:40:02,434 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:40:02,910 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:40:03,473 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:40:03,911 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:40:08,473 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:40:08,485 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:40:08,485 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:40:08,486 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:40:08,491 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:40:08,914 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:40:09,917 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:40:12,371 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:40:12,371 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:40:12,372 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:40:12,377 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:40:12,919 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:40:13,921 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:40:14,055 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:40:14,056 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:40:14,057 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:40:14,315 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:40:16,912 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:40:16,912 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:40:16,913 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:40:16,917 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:40:16,923 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:40:19,925 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:40:19,957 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:40:20,618 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:40:20,619 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:40:20,619 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:40:20,624 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:40:20,925 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:40:21,928 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:40:24,348 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 14:40:25,311 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:40:25,312 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:40:25,312 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:40:25,312 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:40:25,317 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:40:25,930 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:40:27,934 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:40:28,567 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:40:28,567 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:40:28,568 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:40:28,572 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:40:28,935 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:40:29,056 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:40:29,056 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:40:29,057 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:40:29,937 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:40:31,304 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:40:32,802 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:40:32,803 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:40:32,803 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:40:32,809 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:40:32,940 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:40:35,955 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:40:36,231 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:40:36,232 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:40:36,232 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:40:36,239 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:40:36,956 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:40:37,240 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:40:37,962 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:40:39,611 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:40:39,611 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:40:39,612 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:40:39,620 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:40:39,963 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:40:41,968 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:40:42,621 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:40:44,056 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:40:44,056 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:40:44,058 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:40:48,318 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:40:48,955 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:40:48,956 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:40:48,956 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:40:48,964 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:40:48,975 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:40:51,984 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:40:54,001 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:40:54,357 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 14:40:56,116 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:40:56,117 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:40:56,117 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:40:56,125 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:40:56,987 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:40:57,993 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:40:59,056 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:40:59,056 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:40:59,058 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:40:59,277 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:41:04,278 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:41:04,696 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:41:04,697 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:41:04,697 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:41:04,706 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:41:04,999 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:41:06,006 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:41:09,741 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:41:13,582 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:41:13,583 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:41:13,583 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:41:13,590 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:41:14,011 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:41:14,056 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:41:14,056 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:41:14,057 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:41:15,261 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:41:16,016 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:41:17,172 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:41:17,173 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:41:17,173 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:41:17,186 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:41:18,018 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:41:20,022 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:41:21,217 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:41:21,847 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:41:21,848 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:41:21,848 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:41:21,857 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:41:22,024 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:41:24,029 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:41:24,353 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 14:41:25,816 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:41:25,817 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:41:25,817 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:41:25,826 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:41:26,031 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:41:26,827 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:41:28,038 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:41:29,056 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:41:29,056 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:41:29,058 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:41:29,956 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:41:29,956 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:41:29,957 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:41:29,964 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:41:30,040 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:41:32,001 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:41:32,049 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:41:34,799 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:41:34,800 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:41:34,800 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:41:34,809 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:41:35,052 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:41:37,810 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:41:38,062 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:41:39,301 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:41:39,301 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:41:39,302 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:41:39,312 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:41:40,063 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:41:42,071 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:41:43,313 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:41:43,704 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:41:43,705 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:41:43,705 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:41:43,715 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:41:44,056 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:41:44,057 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:41:44,058 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:41:44,072 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:41:46,086 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:41:49,250 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:41:49,251 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:41:49,251 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:41:49,252 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:41:49,258 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:41:50,089 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:41:52,100 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:41:53,291 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:41:53,292 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:41:53,292 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:41:53,297 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:41:54,102 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:41:54,298 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:41:54,355 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 14:41:56,117 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:41:57,766 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:41:57,767 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:41:57,767 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:41:57,780 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:41:58,118 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:41:59,056 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:41:59,057 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:41:59,058 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:42:00,125 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:42:00,266 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:42:05,267 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:42:06,302 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:42:06,303 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:42:06,303 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:42:06,310 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:42:07,131 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:42:08,136 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:42:10,266 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:42:10,267 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:42:10,267 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:42:10,267 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:42:10,276 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:42:11,138 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:42:12,144 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:42:14,056 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:42:14,057 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:42:14,058 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:42:14,429 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:42:14,430 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:42:14,430 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:42:14,438 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:42:15,148 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:42:15,473 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:42:16,151 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:42:17,952 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:42:17,953 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:42:17,953 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:42:17,963 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:42:18,152 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:42:20,159 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:42:20,963 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:42:22,871 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:42:22,872 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:42:22,872 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:42:22,878 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:42:23,163 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:42:24,357 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 14:42:26,171 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:42:26,358 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:42:27,143 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:42:27,144 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:42:27,144 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:42:27,151 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:42:27,172 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:42:29,056 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:42:29,057 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:42:29,058 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:42:30,177 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:42:31,086 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:42:31,087 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:42:31,087 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:42:31,093 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:42:31,179 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:42:32,129 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:42:34,181 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:42:35,656 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:42:35,657 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:42:35,657 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:42:35,665 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:42:36,182 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:42:37,666 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:42:38,188 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:42:41,152 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:42:41,153 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:42:41,154 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:42:41,162 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:42:41,191 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:42:43,163 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:42:44,056 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:42:44,057 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:42:44,059 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:42:44,197 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:42:45,492 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:42:45,493 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:42:45,493 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:42:45,501 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:42:46,202 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:42:48,211 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:42:48,537 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:42:49,502 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:42:49,503 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:42:49,503 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:42:49,513 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:42:50,213 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:42:52,221 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:42:53,448 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:42:53,449 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:42:53,449 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:42:53,457 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:42:54,223 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:42:54,359 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 14:42:54,360 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:42:56,229 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:42:57,619 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:42:57,620 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:42:57,620 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:42:57,627 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:42:58,231 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:42:59,057 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:42:59,057 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:42:59,058 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:43:00,237 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:43:00,327 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:43:01,446 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:43:01,447 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:43:01,447 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:43:01,455 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:43:02,239 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:43:04,244 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:43:05,489 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:43:09,582 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:43:09,583 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:43:09,583 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:43:09,588 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:43:10,250 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:43:10,589 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:43:12,257 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:43:14,057 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:43:14,057 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:43:14,058 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:43:14,522 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:43:14,522 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:43:14,522 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:43:14,528 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:43:15,260 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:43:16,264 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:43:16,565 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:43:19,774 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:43:19,775 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:43:19,775 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:43:19,786 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:43:20,268 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:43:21,786 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:43:22,273 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:43:23,564 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:43:23,565 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:43:23,565 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:43:23,577 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:43:24,276 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:43:24,370 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 14:43:26,280 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:43:27,372 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:43:28,148 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:43:28,148 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:43:28,149 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:43:28,159 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:43:28,284 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:43:29,057 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:43:29,057 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:43:29,059 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:43:30,294 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:43:31,905 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:43:31,906 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:43:31,906 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:43:31,916 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:43:32,502 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:43:32,949 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:43:33,508 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:43:37,949 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:43:41,029 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:43:41,030 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:43:41,031 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:43:41,038 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:43:41,515 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:43:43,038 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:43:43,521 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:43:44,057 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:43:44,057 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:43:44,059 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:43:45,603 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:43:45,603 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:43:45,603 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:43:45,611 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:43:46,527 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:43:47,533 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:43:48,645 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:43:50,236 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:43:50,237 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:43:50,237 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:43:50,247 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:43:50,536 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:43:51,541 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:43:54,248 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:43:54,363 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 14:43:54,816 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:43:54,817 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:43:54,817 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:43:54,827 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:43:55,548 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:43:55,548 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:43:59,057 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:43:59,057 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:43:59,059 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:43:59,280 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:43:59,956 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:43:59,956 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:43:59,957 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:43:59,963 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:44:00,554 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:44:01,558 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:44:04,207 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:44:04,208 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:44:04,208 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:44:04,215 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:44:04,561 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:44:05,215 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:44:05,567 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:44:09,854 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:44:09,854 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:44:09,855 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:44:09,861 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:44:10,571 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:44:10,862 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:44:11,574 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:44:13,098 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:44:13,099 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:44:13,099 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:44:13,106 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:44:13,576 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:44:14,057 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:44:14,057 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:44:14,059 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:44:15,581 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:44:16,347 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:44:17,306 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:44:17,307 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:44:17,307 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:44:17,315 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:44:17,583 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:44:19,587 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:44:21,104 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:44:21,105 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:44:21,105 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:44:21,113 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:44:21,589 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:44:22,113 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:44:23,595 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:44:24,372 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 14:44:26,773 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:44:26,774 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:44:26,774 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:44:26,782 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:44:27,604 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:44:27,604 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:44:27,782 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:44:29,057 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:44:29,057 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:44:29,059 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:44:30,760 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:44:30,760 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:44:30,761 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:44:30,767 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:44:31,614 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:44:31,615 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:44:32,805 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:44:34,470 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:44:34,471 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:44:34,471 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:44:34,477 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:44:34,617 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:44:35,621 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:44:38,478 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:44:40,279 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:44:40,279 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:44:40,280 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:44:40,286 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:44:40,625 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:44:41,629 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:44:44,057 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:44:44,057 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:44:44,059 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:44:44,296 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:44:44,836 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:44:44,836 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:44:44,837 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:44:44,844 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:44:45,639 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:44:45,640 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:44:49,530 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:44:49,531 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:44:49,531 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:44:49,532 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:44:49,538 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:44:49,643 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:44:51,648 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:44:54,367 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 14:44:54,466 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:44:54,467 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:44:54,467 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:44:54,475 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:44:54,651 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:44:55,476 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:44:55,657 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:44:58,521 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:44:58,522 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:44:58,522 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:44:58,529 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:44:58,660 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:44:59,057 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:44:59,058 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:44:59,059 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:44:59,663 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:45:01,305 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:45:02,733 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:45:02,733 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:45:02,733 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:45:02,743 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:45:03,670 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:45:03,670 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:45:06,777 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:45:07,282 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:45:07,283 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:45:07,283 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:45:07,289 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:45:07,673 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:45:09,677 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:45:11,841 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:45:11,842 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:45:11,842 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:45:11,843 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:45:11,847 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:45:12,679 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:45:13,683 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:45:14,057 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:45:14,058 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:45:14,059 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:45:16,418 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:45:16,419 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:45:16,419 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:45:16,425 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:45:16,685 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:45:17,461 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:45:17,688 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:45:20,182 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:45:20,183 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:45:20,183 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:45:20,188 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:45:20,691 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:45:21,698 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:45:23,189 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:45:24,369 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 14:45:24,787 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:45:24,788 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:45:24,788 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:45:24,793 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:45:25,704 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:45:25,704 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:45:28,794 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:45:28,798 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:45:28,798 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:45:28,799 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:45:28,804 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:45:29,057 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:45:29,058 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:45:29,059 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:45:29,710 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:45:29,711 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:45:33,343 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:45:33,344 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:45:33,344 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:45:33,350 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:45:33,714 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:45:34,389 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:45:35,717 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:45:37,559 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:45:37,560 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:45:37,561 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:45:37,566 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:45:37,719 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:45:39,568 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:45:39,722 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:45:41,688 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:45:41,689 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:45:41,689 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:45:41,694 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:45:41,724 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:45:43,727 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:45:44,057 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:45:44,058 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:45:44,060 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:45:45,299 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:45:45,591 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:45:45,592 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:45:45,592 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:45:45,597 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:45:45,730 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:45:47,734 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:45:50,637 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:45:54,371 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 14:45:54,492 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:45:54,493 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:45:54,493 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:45:54,498 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:45:54,738 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:45:55,741 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:45:56,499 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:45:59,058 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:45:59,058 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:45:59,059 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:45:59,290 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:45:59,364 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:45:59,364 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:45:59,369 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:45:59,743 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:46:01,745 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:46:02,370 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:46:06,050 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:46:06,051 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:46:06,051 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:46:06,056 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:46:06,748 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:46:07,751 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:46:08,093 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:46:09,774 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:46:09,775 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:46:09,775 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:46:09,780 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:46:10,753 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:46:11,755 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:46:13,781 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:46:13,958 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:46:13,959 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:46:13,959 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:46:13,964 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:46:14,057 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:46:14,058 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:46:14,059 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:46:14,757 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:46:15,760 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:46:19,274 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:46:22,992 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:46:22,993 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:46:22,993 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:46:22,998 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:46:23,767 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:46:23,768 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:46:24,373 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 14:46:24,375 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:46:27,805 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:46:27,806 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:46:27,806 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:46:27,811 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:46:28,772 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:46:29,058 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:46:29,058 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:46:29,060 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:46:29,774 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:46:30,330 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:46:32,799 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:46:32,800 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:46:32,800 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:46:32,805 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:46:33,779 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:46:33,779 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:46:35,845 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:46:37,158 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:46:37,159 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:46:37,159 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:46:37,164 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:46:37,781 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:46:39,784 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:46:41,165 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:46:43,334 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:46:43,335 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:46:43,335 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:46:43,340 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:46:43,789 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:46:44,058 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:46:44,058 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:46:44,060 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:46:45,793 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:46:46,353 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:46:47,256 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:46:47,256 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:46:47,257 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:46:47,261 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:46:47,795 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:46:49,798 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:46:52,301 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:46:52,882 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:46:52,883 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:46:52,883 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:46:52,888 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:46:53,802 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:46:53,803 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:46:54,381 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 14:46:56,582 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:46:56,583 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:46:56,583 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:46:56,588 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:46:56,804 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:46:57,588 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:46:57,807 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:46:59,058 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:46:59,058 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:46:59,060 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:47:01,087 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:47:01,087 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:47:01,087 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:47:01,092 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:47:01,812 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:47:01,812 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:47:03,129 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:47:04,950 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:47:04,950 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:47:04,951 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:47:04,956 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:47:05,817 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:47:05,817 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:47:08,694 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:47:08,694 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:47:08,695 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:47:08,696 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:47:08,700 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:47:08,819 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:47:09,821 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:47:13,422 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:47:13,423 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:47:13,423 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:47:13,428 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:47:13,824 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:47:14,058 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:47:14,058 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:47:14,060 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:47:14,347 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:47:15,827 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:47:17,807 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:47:17,807 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:47:17,808 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:47:17,813 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:47:17,828 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:47:19,833 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:47:19,849 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:47:21,943 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:47:21,943 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:47:21,944 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:47:21,949 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:47:22,835 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:47:23,837 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:47:24,377 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 14:47:25,379 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:47:29,058 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:47:29,058 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:47:29,060 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:47:30,432 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:47:30,433 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:47:30,433 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:47:30,433 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:47:30,438 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:47:30,843 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:47:31,845 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:47:35,117 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:47:35,118 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:47:35,118 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:47:35,123 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:47:35,850 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:47:35,850 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:47:36,123 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:47:38,657 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:47:38,658 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:47:38,658 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:47:38,663 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:47:38,851 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:47:39,854 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:47:41,663 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:47:42,418 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:47:42,419 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:47:42,419 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:47:42,424 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:47:42,856 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:47:43,858 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:47:44,058 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:47:44,058 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:47:44,060 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:47:46,109 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:47:46,110 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:47:46,110 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:47:46,115 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:47:46,862 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:47:47,153 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:47:47,864 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:47:50,544 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:47:50,545 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:47:50,545 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:47:50,550 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:47:50,866 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:47:51,869 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:47:52,550 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:47:54,277 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:47:54,278 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:47:54,278 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:47:54,287 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:47:54,387 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 14:47:54,871 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:47:55,874 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:47:58,365 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:47:58,366 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:47:58,367 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:47:58,367 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:47:58,372 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:47:58,876 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:47:59,058 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:47:59,058 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:47:59,060 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:47:59,878 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:48:02,497 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:48:02,498 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:48:02,498 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:48:02,503 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:48:02,881 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:48:03,541 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:48:03,883 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:48:08,541 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:48:10,839 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:48:10,839 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:48:10,840 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:48:10,844 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:48:10,887 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:48:11,890 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:48:13,845 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:48:14,058 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:48:14,058 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:48:14,060 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:48:17,066 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:48:17,067 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:48:17,067 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:48:17,072 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:48:17,896 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:48:17,896 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:48:19,109 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:48:24,109 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:48:24,383 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 14:48:26,136 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:48:26,137 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:48:26,137 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:48:26,142 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:48:26,903 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:48:27,906 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:48:29,058 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:48:29,059 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:48:29,060 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:48:29,313 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:48:30,304 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:48:30,305 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:48:30,305 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:48:30,310 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:48:30,908 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:48:31,911 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:48:34,349 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:48:38,473 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:48:38,474 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:48:38,474 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:48:38,479 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:48:38,916 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:48:39,480 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:48:39,918 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:48:42,657 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:48:42,657 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:48:42,658 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:48:42,663 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:48:42,920 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:48:43,923 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:48:44,058 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:48:44,059 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:48:44,060 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:48:45,286 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:48:46,280 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:48:46,281 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:48:46,281 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:48:46,286 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:48:46,926 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:48:47,928 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:48:50,325 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:48:50,566 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:48:50,567 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:48:50,568 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:48:50,572 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:48:50,930 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:48:51,933 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:48:54,300 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:48:54,301 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:48:54,301 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:48:54,306 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:48:54,385 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 14:48:54,935 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:48:55,386 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:48:55,937 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:48:58,538 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:48:58,539 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:48:58,540 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:48:58,544 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:48:58,939 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:48:59,058 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:48:59,059 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:48:59,060 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:48:59,942 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:49:01,310 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:49:06,310 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:49:06,672 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:49:06,676 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:49:06,676 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:49:06,681 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:49:06,946 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:49:07,949 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:49:10,402 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:49:10,403 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:49:10,404 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:49:10,409 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:49:10,951 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:49:11,410 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:49:11,951 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:49:13,906 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:49:13,907 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:49:13,907 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:49:13,912 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:49:13,953 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:49:14,058 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:49:14,059 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:49:14,060 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:49:15,956 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:49:17,350 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:49:18,908 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:49:18,909 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:49:18,909 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:49:18,914 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:49:18,958 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:49:19,960 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:49:22,953 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:49:23,005 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:49:23,005 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:49:23,006 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:49:23,010 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:49:23,965 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:49:23,966 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:49:24,387 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 14:49:28,388 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:49:28,599 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:49:28,600 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:49:28,600 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:49:28,605 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:49:28,970 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:49:29,058 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:49:29,059 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:49:29,061 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:49:29,972 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:49:32,173 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:49:32,174 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:49:32,174 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:49:32,179 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:49:32,974 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:49:33,977 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:49:34,217 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:49:35,983 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:49:35,984 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:49:35,984 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:49:35,989 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:49:36,978 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:49:37,981 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:49:39,989 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:49:41,673 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:49:41,674 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:49:41,674 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:49:41,679 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:49:41,983 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:49:43,986 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:49:44,058 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:49:44,059 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:49:44,061 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:49:45,382 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:49:45,696 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:49:45,697 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:49:45,697 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:49:45,702 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:49:45,989 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:49:47,992 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:49:49,242 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:49:49,242 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:49:49,243 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:49:49,247 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:49:49,995 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:49:49,995 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:49:51,248 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:49:53,241 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:49:53,242 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:49:53,242 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:49:53,247 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:49:53,999 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:49:53,999 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:49:54,389 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 14:49:56,390 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:49:59,059 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:49:59,059 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:49:59,061 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:50:01,467 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:50:01,468 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:50:01,468 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:50:01,468 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:50:01,473 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:50:02,005 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:50:04,008 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:50:05,630 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:50:05,631 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:50:05,631 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:50:05,636 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:50:06,009 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:50:06,636 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:50:08,012 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:50:09,488 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:50:09,488 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:50:09,489 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:50:09,494 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:50:10,013 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:50:12,016 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:50:12,495 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:50:14,059 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:50:14,059 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:50:14,060 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:50:14,499 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:50:14,500 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:50:14,500 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:50:14,505 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:50:15,018 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:50:16,020 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:50:17,545 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:50:18,432 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:50:18,433 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:50:18,433 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:50:18,438 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:50:19,023 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:50:20,025 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:50:23,439 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:50:24,394 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 14:50:24,864 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:50:24,865 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:50:24,865 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:50:24,870 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:50:25,028 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:50:26,031 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:50:28,768 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:50:28,769 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:50:28,769 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:50:28,769 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:50:28,775 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:50:29,034 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:50:29,059 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:50:29,059 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:50:29,061 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:50:30,037 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:50:34,046 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:50:34,047 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:50:34,047 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:50:34,047 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:50:34,052 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:50:35,040 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:50:36,042 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:50:38,381 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:50:38,381 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:50:38,382 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:50:38,386 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:50:39,044 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:50:39,387 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:50:40,047 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:50:43,922 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:50:43,923 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:50:43,923 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:50:43,928 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:50:44,049 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:50:44,059 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:50:44,059 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:50:44,061 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:50:45,305 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:50:46,054 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:50:48,164 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:50:48,165 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:50:48,165 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:50:48,170 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:50:49,055 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:50:50,058 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:50:51,209 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:50:52,880 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:50:52,880 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:50:52,881 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:50:52,886 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:50:53,060 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:50:54,062 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:50:54,393 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 14:50:56,394 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:50:56,836 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:50:56,836 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:50:56,837 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:50:56,841 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:50:57,064 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:50:58,066 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:50:59,059 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:50:59,059 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:50:59,061 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:51:00,769 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:51:00,770 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:51:00,770 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:51:00,775 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:51:01,069 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:51:01,813 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:51:02,072 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:51:04,974 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:51:04,974 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:51:04,975 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:51:04,980 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:51:05,074 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:51:06,076 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:51:06,981 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:51:09,460 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:51:09,460 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:51:09,461 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:51:09,465 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:51:10,081 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:51:10,081 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:51:12,466 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:51:14,059 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:51:14,059 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:51:14,061 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:51:15,427 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:51:15,428 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:51:15,428 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:51:15,433 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:51:16,086 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:51:16,086 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:51:17,473 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:51:19,818 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:51:19,819 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:51:19,819 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:51:19,824 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:51:20,088 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:51:22,091 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:51:22,825 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:51:24,135 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:51:24,136 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:51:24,136 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:51:24,142 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:51:24,398 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 14:51:25,094 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:51:26,096 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:51:28,400 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:51:28,642 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:51:28,643 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:51:28,643 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:51:28,648 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:51:29,059 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:51:29,059 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:51:29,061 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:51:29,099 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:51:30,102 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:51:32,876 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:51:32,877 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:51:32,877 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:51:32,882 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:51:33,103 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:51:33,921 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:51:34,106 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:51:38,921 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:51:41,069 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:51:41,070 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:51:41,070 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:51:41,075 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:51:41,110 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:51:42,113 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:51:44,059 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:51:44,059 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:51:44,061 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:51:44,323 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:51:45,083 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:51:45,083 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:51:45,084 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:51:45,089 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:51:45,114 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:51:46,118 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:51:48,974 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:51:48,975 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:51:48,975 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:51:48,980 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:51:49,120 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:51:49,980 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:51:50,122 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:51:54,200 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:51:54,201 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:51:54,201 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:51:54,206 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:51:54,397 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 14:51:55,125 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:51:55,399 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:51:56,128 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:51:58,533 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:51:58,534 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:51:58,534 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:51:58,539 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:51:59,059 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:51:59,060 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:51:59,061 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:51:59,130 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:52:00,132 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:52:01,361 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:52:02,179 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:52:02,179 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:52:02,180 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:52:02,185 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:52:03,134 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:52:04,137 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:52:06,136 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:52:06,137 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:52:06,137 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:52:06,142 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:52:07,139 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:52:07,142 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:52:08,142 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:52:10,323 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:52:10,324 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:52:10,324 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:52:10,329 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:52:11,144 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:52:12,146 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:52:12,330 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:52:14,006 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:52:14,007 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:52:14,007 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:52:14,012 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:52:14,059 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:52:14,060 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:52:14,061 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:52:14,147 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:52:16,150 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:52:18,329 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:52:20,242 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:52:20,242 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:52:20,242 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:52:20,249 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:52:21,153 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:52:22,156 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:52:24,285 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:52:24,402 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 14:52:26,299 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:52:26,300 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:52:26,300 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:52:26,305 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:52:27,159 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:52:28,163 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:52:29,059 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:52:29,060 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:52:29,062 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:52:29,294 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:52:30,849 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:52:30,850 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:52:30,850 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:52:30,855 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:52:31,165 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:52:32,167 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:52:34,893 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:52:36,709 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:52:36,710 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:52:36,710 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:52:36,715 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:52:37,170 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:52:38,172 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:52:40,716 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:52:41,514 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:52:41,515 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:52:41,515 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:52:41,520 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:52:42,176 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:52:42,177 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:52:44,059 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:52:44,060 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:52:44,062 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:52:46,348 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:52:46,936 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:52:46,937 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:52:46,937 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:52:46,942 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:52:47,181 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:52:48,183 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:52:50,706 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:52:50,707 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:52:50,707 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:52:50,712 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:52:51,185 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:52:51,712 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:52:52,188 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:52:54,401 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 14:52:54,717 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:52:54,718 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:52:54,718 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:52:54,723 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:52:55,190 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:52:56,192 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:52:56,723 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:52:58,509 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:52:58,510 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:52:58,511 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:52:58,516 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:52:59,060 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:52:59,060 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:52:59,061 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:52:59,194 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:53:00,197 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:53:02,269 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:53:03,304 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:53:03,305 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:53:03,305 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:53:03,310 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:53:04,202 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:53:04,202 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:53:07,349 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:53:08,077 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:53:08,078 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:53:08,078 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:53:08,083 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:53:08,204 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:53:10,207 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:53:12,785 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:53:12,786 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:53:12,786 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:53:12,786 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:53:12,791 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:53:13,208 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:53:14,060 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:53:14,060 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:53:14,061 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:53:14,211 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:53:17,001 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:53:17,002 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:53:17,002 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:53:17,007 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:53:17,213 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:53:18,045 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:53:18,215 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:53:23,045 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:53:23,939 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:53:23,939 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:53:23,940 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:53:23,945 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:53:24,219 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:53:24,404 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 14:53:26,222 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:53:28,405 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:53:29,060 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:53:29,060 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:53:29,062 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:53:32,845 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:53:32,845 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:53:32,846 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:53:32,850 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:53:33,228 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:53:33,889 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:53:34,231 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:53:38,889 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:53:41,116 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:53:41,117 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:53:41,117 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:53:41,122 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:53:41,235 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:53:42,237 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:53:44,060 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:53:44,060 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:53:44,062 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:53:44,324 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:53:49,325 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:53:49,541 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:53:49,541 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:53:49,541 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:53:49,546 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:53:50,245 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:53:50,245 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:53:54,409 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 14:53:54,410 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:53:55,854 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:53:55,854 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:53:55,855 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:53:55,860 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:53:56,248 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:53:58,251 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:53:59,060 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:53:59,060 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:53:59,062 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:54:00,288 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:54:04,182 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:54:04,183 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:54:04,183 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:54:04,188 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:54:04,255 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:54:06,225 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:54:06,258 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:54:08,222 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:54:08,223 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:54:08,223 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:54:08,228 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:54:08,259 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:54:10,262 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:54:11,228 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:54:12,459 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:54:12,460 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:54:12,460 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:54:12,465 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:54:13,264 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:54:14,060 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:54:14,060 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:54:14,062 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:54:14,267 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:54:16,367 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:54:17,240 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:54:17,241 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:54:17,241 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:54:17,246 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:54:17,268 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:54:18,271 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:54:22,285 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:54:22,775 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:54:22,775 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:54:22,776 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:54:22,781 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:54:23,274 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:54:24,277 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:54:24,408 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 14:54:27,339 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:54:27,339 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:54:27,340 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:54:27,340 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:54:27,346 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:54:28,282 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:54:28,282 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:54:29,060 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:54:29,060 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:54:29,062 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:54:33,290 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:54:33,401 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:54:33,401 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:54:33,401 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:54:33,406 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:54:34,288 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:54:34,288 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:54:37,600 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:54:37,601 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:54:37,602 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:54:37,606 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:54:38,292 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:54:38,292 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:54:38,607 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:54:42,382 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:54:42,383 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:54:42,383 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:54:42,389 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:54:43,295 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:54:44,060 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:54:44,060 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:54:44,062 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:54:44,297 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:54:44,319 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:54:46,496 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:54:46,497 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:54:46,497 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:54:46,502 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:54:47,301 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:54:48,303 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:54:49,541 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:54:51,351 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:54:51,351 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:54:51,352 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:54:51,357 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:54:52,308 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:54:52,308 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:54:54,413 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 14:54:54,820 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:54:54,821 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:54:54,821 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:54:54,821 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:54:54,826 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:54:55,310 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:54:56,312 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:54:59,060 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:54:59,060 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:54:59,062 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:54:59,569 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:54:59,570 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:54:59,570 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:54:59,575 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:55:00,315 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:55:00,315 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:55:00,613 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:55:05,613 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:55:07,947 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:55:07,948 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:55:07,948 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:55:07,953 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:55:08,320 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:55:10,323 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:55:10,954 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:55:12,518 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:55:12,518 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:55:12,519 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:55:12,524 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:55:13,325 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:55:14,060 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:55:14,061 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:55:14,062 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:55:14,327 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:55:16,093 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:55:16,093 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:55:16,094 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:55:16,094 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:55:16,099 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:55:16,329 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:55:18,332 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:55:21,137 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:55:21,735 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:55:21,736 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:55:21,736 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:55:21,741 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:55:22,336 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:55:22,336 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:55:24,412 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 14:55:26,157 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:55:26,158 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:55:26,159 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:55:26,159 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:55:26,163 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:55:26,339 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:55:28,344 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:55:29,060 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:55:29,061 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:55:29,062 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:55:31,361 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:55:35,680 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:55:35,681 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:55:35,681 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:55:35,686 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:55:36,351 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:55:36,351 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:55:36,725 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:55:40,038 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:55:40,039 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:55:40,039 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:55:40,044 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:55:40,353 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:55:42,045 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:55:42,356 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:55:43,873 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:55:43,874 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:55:43,874 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:55:43,879 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:55:44,060 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:55:44,061 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:55:44,062 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:55:44,357 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:55:46,362 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:55:47,340 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:55:48,431 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:55:48,432 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:55:48,432 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:55:48,437 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:55:49,364 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:55:50,366 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:55:52,453 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:55:52,453 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:55:52,454 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:55:52,454 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:55:52,459 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:55:53,368 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:55:54,370 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:55:54,414 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 14:55:56,359 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:55:56,360 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:55:56,360 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:55:56,366 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:55:56,371 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:55:58,366 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:55:58,374 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:55:59,060 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:55:59,061 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:55:59,062 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:56:00,157 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:56:00,157 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:56:00,158 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:56:00,162 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:56:00,376 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:56:02,379 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:56:04,201 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:56:05,248 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:56:05,252 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:56:05,252 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:56:05,257 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:56:05,381 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:56:06,383 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:56:09,258 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:56:11,720 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:56:11,721 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:56:11,721 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:56:11,726 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:56:12,389 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:56:12,389 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:56:14,061 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:56:14,061 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:56:14,062 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:56:14,317 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:56:16,676 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:56:16,676 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:56:16,677 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:56:16,681 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:56:17,392 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:56:18,394 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:56:19,721 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:56:21,272 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:56:21,272 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:56:21,273 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:56:21,278 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:56:21,396 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:56:22,398 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:56:24,416 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 14:56:25,418 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:56:27,206 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:56:27,206 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:56:27,207 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:56:27,213 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:56:27,402 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:56:28,405 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:56:29,061 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:56:29,061 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:56:29,063 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:56:31,307 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:56:31,809 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:56:31,810 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:56:31,810 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:56:31,815 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:56:32,407 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:56:34,410 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:56:35,830 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:56:35,831 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:56:35,831 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:56:35,836 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:56:36,412 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:56:36,412 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:56:36,837 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:56:41,110 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:56:41,111 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:56:41,111 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:56:41,116 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:56:41,414 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:56:42,117 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:56:42,417 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:56:44,061 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:56:44,061 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:56:44,063 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:56:45,234 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:56:45,235 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:56:45,235 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:56:45,240 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:56:45,419 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:56:46,422 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:56:47,277 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:56:48,806 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:56:48,806 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:56:48,807 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:56:48,812 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:56:49,424 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:56:50,426 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:56:52,812 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:56:53,757 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:56:53,758 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:56:53,758 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:56:53,763 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:56:54,418 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 14:56:54,430 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:56:54,431 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:56:58,420 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:56:59,014 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:56:59,015 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:56:59,015 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:56:59,021 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:56:59,061 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:56:59,061 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:56:59,063 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:56:59,433 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:57:00,436 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:57:03,801 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:57:03,802 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:57:03,802 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:57:03,802 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:57:03,807 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:57:04,441 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:57:04,441 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:57:08,279 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:57:08,279 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:57:08,280 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:57:08,284 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:57:08,443 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:57:09,285 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:57:10,446 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:57:12,119 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:57:12,120 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:57:12,120 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:57:12,125 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:57:12,447 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:57:14,061 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:57:14,061 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:57:14,063 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:57:14,306 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:57:14,451 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:57:15,809 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:57:15,810 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:57:15,810 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:57:15,815 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:57:16,454 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:57:16,454 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:57:19,571 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:57:19,572 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:57:19,572 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:57:19,572 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:57:19,577 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:57:20,458 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:57:20,458 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:57:23,588 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:57:23,588 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:57:23,589 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:57:23,594 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:57:24,420 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 14:57:24,463 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:57:24,463 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:57:25,422 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:57:28,948 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:57:28,949 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:57:28,949 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:57:28,954 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:57:29,061 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:57:29,061 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:57:29,063 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:57:29,467 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:57:30,470 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:57:31,325 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:57:33,361 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:57:33,361 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:57:33,361 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:57:33,367 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:57:33,471 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:57:34,474 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:57:36,405 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:57:39,613 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:57:39,614 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:57:39,614 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:57:39,619 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:57:40,480 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:57:40,480 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:57:41,620 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:57:44,061 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:57:44,062 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:57:44,064 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:57:45,472 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:57:45,473 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:57:45,473 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:57:45,478 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:57:45,483 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:57:46,487 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:57:47,517 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:57:51,768 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:57:51,769 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:57:51,769 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:57:51,774 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:57:52,493 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:57:52,493 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:57:52,775 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:57:54,423 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 14:57:57,640 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:57:57,640 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:57:57,641 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:57:57,646 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:57:58,499 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:57:58,499 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:57:58,646 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:57:59,061 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:57:59,062 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:57:59,063 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:58:02,858 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:58:02,859 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:58:02,859 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:58:02,865 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:58:03,502 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:58:03,901 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:58:04,505 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:58:07,351 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:58:07,352 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:58:07,352 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:58:07,357 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:58:07,507 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:58:08,509 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:58:09,357 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:58:11,903 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:58:11,904 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:58:11,904 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:58:11,909 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:58:12,514 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:58:12,514 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:58:14,061 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:58:14,062 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:58:14,063 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:58:15,267 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:58:16,833 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:58:16,834 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:58:16,834 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:58:16,839 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:58:17,517 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:58:18,519 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:58:20,877 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:58:21,511 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:58:21,511 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:58:21,512 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:58:21,517 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:58:21,521 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:58:22,523 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:58:24,429 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 14:58:26,430 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:58:27,746 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:58:27,746 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:58:27,747 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:58:27,751 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:58:28,530 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:58:28,530 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:58:29,061 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:58:29,062 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:58:29,063 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:58:32,323 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:58:35,907 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:58:35,907 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:58:35,907 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:58:35,913 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:58:36,537 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:58:36,537 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:58:37,949 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:58:40,077 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:58:40,078 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:58:40,078 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:58:40,083 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:58:40,539 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:58:42,542 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:58:43,084 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:58:44,062 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:58:44,062 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:58:44,064 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:58:44,259 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:58:44,287 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:58:44,287 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:58:44,292 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:58:44,543 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:58:46,548 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:58:48,043 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:58:48,043 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:58:48,044 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:58:48,049 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:58:48,085 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:58:48,549 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:58:50,552 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:58:53,085 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:58:53,299 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:58:53,299 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:58:53,300 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:58:53,304 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:58:53,554 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:58:54,427 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 14:58:54,556 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:58:57,078 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:58:57,079 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:58:57,079 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:58:57,084 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:58:57,558 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:58:58,561 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:58:59,062 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:58:59,062 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:58:59,064 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:58:59,271 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:59:01,083 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:59:01,083 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:59:01,084 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:59:01,089 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:59:01,563 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:59:02,565 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:59:05,125 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:59:06,013 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:59:06,013 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:59:06,014 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:59:06,020 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:59:06,568 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:59:08,571 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:59:10,862 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:59:10,863 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:59:10,863 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:59:10,863 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:59:10,868 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:59:11,573 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:59:12,575 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:59:14,062 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:59:14,062 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:59:14,063 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:59:15,186 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:59:15,186 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:59:15,187 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:59:15,192 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:59:15,577 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:59:16,229 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:59:16,579 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:59:19,432 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:59:19,433 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:59:19,433 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:59:19,438 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:59:19,581 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:59:20,584 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:59:21,439 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:59:23,220 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:59:23,221 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:59:23,221 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:59:23,226 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:59:23,585 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:59:24,432 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 14:59:24,586 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:59:27,434 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:59:28,106 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:59:28,107 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:59:28,107 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:59:28,112 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:59:28,590 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:59:29,062 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:59:29,062 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:59:29,064 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:59:30,593 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:59:32,184 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:59:32,185 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:59:32,185 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:59:32,190 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:59:32,594 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:59:33,229 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:59:34,598 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:59:37,099 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:59:37,100 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:59:37,100 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:59:37,106 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:59:37,600 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:59:38,602 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:59:39,107 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:59:44,062 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:59:44,062 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:59:44,064 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:59:44,326 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:59:45,489 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:59:45,489 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:59:45,489 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:59:45,494 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:59:45,606 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:59:46,610 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:59:49,533 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:59:53,837 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:59:53,838 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:59:53,838 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:59:53,843 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:59:54,431 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 14:59:54,616 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:59:54,616 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:59:55,432 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:59:57,512 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:59:57,512 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 14:59:57,513 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:59:57,518 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:59:57,618 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 14:59:58,621 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 14:59:59,062 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:59:59,063 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:59:59,064 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:00:01,351 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:00:03,028 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:00:03,028 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:00:03,028 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:00:03,033 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:00:03,624 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:00:04,627 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:00:07,073 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:00:07,484 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:00:07,485 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:00:07,485 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:00:07,491 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:00:07,628 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:00:08,631 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:00:11,168 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:00:11,169 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:00:11,169 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:00:11,174 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:00:11,633 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:00:12,175 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:00:12,635 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:00:14,063 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:00:14,063 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:00:14,064 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:00:15,287 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:00:15,288 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:00:15,288 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:00:15,293 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:00:15,637 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:00:16,639 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:00:17,333 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:00:19,037 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:00:19,038 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:00:19,038 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:00:19,043 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:00:19,641 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:00:20,644 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:00:23,044 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:00:23,646 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:00:23,647 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:00:23,647 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:00:23,652 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:00:24,433 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 15:00:24,648 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:00:24,649 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:00:28,434 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:00:29,063 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:00:29,063 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:00:29,065 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:00:34,246 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:00:34,246 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:00:34,247 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:00:34,247 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:00:34,258 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:00:34,655 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:00:36,659 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:00:39,289 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:00:42,282 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:00:42,282 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:00:42,283 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:00:42,288 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:00:42,662 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:00:44,063 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:00:44,063 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:00:44,065 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:00:44,319 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:00:44,665 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:00:49,320 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:00:50,519 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:00:50,519 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:00:50,519 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:00:50,524 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:00:50,670 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:00:52,673 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:00:54,061 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:00:54,062 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:00:54,063 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:00:54,068 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:00:54,438 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 15:00:54,439 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:00:54,676 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:00:54,676 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:00:59,063 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:00:59,063 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:00:59,065 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:00:59,660 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:00:59,661 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:00:59,661 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:00:59,661 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:00:59,666 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:00:59,679 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:01:00,681 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:01:03,822 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:01:03,823 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:01:03,823 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:01:03,828 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:01:04,686 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:01:04,686 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:01:04,829 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:01:07,404 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:01:07,404 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:01:07,405 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:01:07,410 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:01:07,688 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:01:08,691 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:01:10,410 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:01:11,217 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:01:11,218 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:01:11,218 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:01:11,223 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:01:11,692 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:01:12,695 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:01:14,063 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:01:14,063 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:01:14,065 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:01:14,820 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:01:14,821 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:01:14,821 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:01:14,826 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:01:15,697 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:01:15,865 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:01:16,700 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:01:18,524 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:01:18,525 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:01:18,525 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:01:18,530 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:01:18,701 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:01:20,704 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:01:21,531 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:01:23,697 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:01:23,698 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:01:23,698 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:01:23,703 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:01:23,706 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:01:24,437 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 15:01:24,709 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:01:27,439 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:01:29,063 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:01:29,063 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:01:29,065 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:01:29,538 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:01:29,539 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:01:29,539 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:01:29,544 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:01:29,713 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:01:30,715 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:01:32,581 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:01:36,212 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:01:36,213 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:01:36,213 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:01:36,218 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:01:36,719 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:01:38,218 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:01:38,722 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:01:40,030 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:01:40,031 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:01:40,031 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:01:40,036 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:01:40,725 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:01:40,725 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:01:44,016 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:01:44,017 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:01:44,017 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:01:44,018 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:01:44,023 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:01:44,063 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:01:44,064 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:01:44,065 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:01:44,729 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:01:44,730 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:01:49,352 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:01:50,147 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:01:50,148 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:01:50,148 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:01:50,153 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:01:50,736 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:01:50,736 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:01:53,801 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:01:53,801 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:01:53,802 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:01:53,807 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:01:54,442 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 15:01:54,444 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:01:54,739 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:01:54,739 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:01:58,216 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:01:58,217 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:01:58,217 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:01:58,223 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:01:58,741 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:01:59,063 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:01:59,064 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:01:59,065 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:02:00,270 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:02:00,744 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:02:02,268 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:02:02,268 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:02:02,269 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:02:02,274 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:02:02,746 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:02:04,749 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:02:05,313 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:02:07,004 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:02:07,005 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:02:07,005 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:02:07,010 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:02:07,751 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:02:08,754 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:02:11,011 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:02:12,892 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:02:12,892 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:02:12,893 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:02:12,898 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:02:13,757 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:02:14,063 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:02:14,064 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:02:14,065 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:02:14,759 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:02:16,344 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:02:17,138 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:02:17,138 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:02:17,138 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:02:17,143 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:02:17,761 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:02:18,763 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:02:21,135 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:02:21,136 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:02:21,136 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:02:21,142 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:02:21,765 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:02:22,142 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:02:22,768 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:02:24,441 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 15:02:26,718 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:02:26,719 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:02:26,719 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:02:26,724 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:02:26,770 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:02:27,725 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:02:28,774 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:02:29,064 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:02:29,064 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:02:29,066 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:02:33,359 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:02:35,215 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:02:35,216 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:02:35,216 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:02:35,221 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:02:35,778 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:02:36,781 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:02:39,261 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:02:43,452 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:02:43,453 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:02:43,453 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:02:43,459 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:02:43,786 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:02:44,064 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:02:44,064 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:02:44,066 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:02:44,284 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:02:44,788 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:02:47,757 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:02:47,758 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:02:47,758 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:02:47,763 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:02:47,791 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:02:48,794 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:02:49,801 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:02:51,532 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:02:51,533 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:02:51,533 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:02:51,538 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:02:51,796 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:02:52,798 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:02:54,443 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 15:02:55,210 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:02:55,211 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:02:55,211 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:02:55,211 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:02:55,216 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:02:55,800 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:02:56,803 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:02:59,064 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:02:59,064 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:02:59,065 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:02:59,787 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:02:59,788 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:02:59,788 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:02:59,793 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:02:59,804 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:03:00,807 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:03:00,833 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:03:05,833 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:03:08,700 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:03:08,700 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:03:08,701 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:03:08,706 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:03:08,812 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:03:10,815 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:03:11,706 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:03:12,031 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:03:12,031 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:03:12,032 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:03:12,037 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:03:12,819 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:03:12,819 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:03:14,064 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:03:14,064 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:03:14,066 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:03:16,181 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:03:16,181 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:03:16,182 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:03:16,187 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:03:16,821 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:03:16,822 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:03:17,225 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:03:20,248 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:03:20,248 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:03:20,249 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:03:20,254 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:03:20,825 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:03:20,826 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:03:22,255 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:03:24,298 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:03:24,298 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:03:24,299 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:03:24,304 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:03:24,446 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 15:03:24,830 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:03:24,830 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:03:27,448 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:03:28,458 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:03:28,458 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:03:28,459 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:03:28,464 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:03:28,834 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:03:29,064 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:03:29,064 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:03:29,066 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:03:30,837 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:03:32,224 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:03:32,225 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:03:32,225 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:03:32,230 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:03:32,840 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:03:32,840 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:03:33,269 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:03:38,269 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:03:38,270 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:03:38,270 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:03:38,271 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:03:38,275 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:03:38,846 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:03:38,846 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:03:43,276 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:03:44,064 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:03:44,064 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:03:44,066 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:03:46,685 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:03:46,685 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:03:46,686 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:03:46,691 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:03:46,852 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:03:48,729 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:03:48,855 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:03:50,420 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:03:50,421 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:03:50,421 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:03:50,427 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:03:50,856 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:03:52,859 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:03:54,428 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:03:54,449 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 15:03:55,956 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:03:55,956 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:03:55,957 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:03:55,962 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:03:56,864 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:03:56,864 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:03:59,065 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:03:59,065 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:03:59,066 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:03:59,852 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:03:59,853 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:03:59,853 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:03:59,853 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:03:59,858 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:03:59,865 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:04:00,868 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:04:04,507 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:04:04,508 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:04:04,508 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:04:04,514 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:04:04,871 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:04:05,514 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:04:06,874 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:04:10,515 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:04:13,306 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:04:13,307 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:04:13,307 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:04:13,312 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:04:13,878 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:04:14,064 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:04:14,064 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:04:14,066 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:04:14,881 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:04:16,359 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:04:17,625 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:04:17,626 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:04:17,626 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:04:17,632 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:04:17,883 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:04:18,886 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:04:21,669 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:04:22,578 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:04:22,579 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:04:22,579 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:04:22,584 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:04:22,888 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:04:24,451 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 15:04:24,892 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:04:26,293 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:04:26,294 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:04:26,295 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:04:26,300 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:04:26,893 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:04:26,894 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:04:27,300 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:04:29,064 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:04:29,065 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:04:29,066 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:04:30,708 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:04:30,709 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:04:30,709 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:04:30,715 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:04:30,898 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:04:32,753 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:04:32,901 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:04:35,625 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:04:35,625 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:04:35,626 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:04:35,631 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:04:35,903 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:04:36,906 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:04:38,632 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:04:40,532 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:04:40,532 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:04:40,533 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:04:40,538 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:04:40,909 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:04:42,912 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:04:44,064 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:04:44,065 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:04:44,066 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:04:44,284 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:04:45,496 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:04:45,496 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:04:45,497 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:04:45,502 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:04:45,915 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:04:46,919 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:04:49,363 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:04:49,364 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:04:49,365 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:04:49,365 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:04:49,370 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:04:49,921 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:04:50,923 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:04:54,371 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:04:54,453 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 15:04:54,914 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:04:54,915 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:04:54,915 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:04:54,921 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:04:54,926 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:04:56,929 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:04:58,510 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:04:58,511 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:04:58,511 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:04:58,516 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:04:58,931 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:04:59,064 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:04:59,065 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:04:59,066 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:05:00,273 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:05:00,934 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:05:02,648 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:05:02,649 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:05:02,649 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:05:02,654 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:05:02,935 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:05:04,938 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:05:05,693 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:05:07,564 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:05:07,565 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:05:07,565 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:05:07,570 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:05:07,940 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:05:08,943 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:05:11,517 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:05:11,518 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:05:11,519 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:05:11,519 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:05:11,524 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:05:11,945 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:05:12,947 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:05:14,065 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:05:14,065 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:05:14,066 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:05:16,340 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:05:16,341 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:05:16,341 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:05:16,346 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:05:16,952 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:05:16,952 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:05:17,385 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:05:21,195 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:05:21,196 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:05:21,196 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:05:21,207 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:05:21,956 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:05:22,966 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:05:23,208 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:05:24,464 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 15:05:25,619 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:05:25,619 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:05:25,620 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:05:25,637 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:05:25,969 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:05:26,973 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:05:28,638 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:05:29,065 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:05:29,065 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:05:29,066 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:05:33,929 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:05:33,930 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:05:33,930 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:05:33,930 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:05:33,940 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:05:33,979 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:05:34,987 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:05:38,973 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:05:39,315 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:05:39,315 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:05:39,316 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:05:39,326 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:05:39,991 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:05:40,997 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:05:43,113 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:05:43,114 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:05:43,114 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:05:43,123 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:05:43,999 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:05:44,065 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:05:44,065 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:05:44,065 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:05:44,067 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:05:45,006 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:05:47,086 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:05:47,087 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:05:47,087 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:05:47,093 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:05:48,010 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:05:49,015 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:05:49,129 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:05:51,076 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:05:51,077 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:05:51,077 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:05:51,086 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:05:52,017 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:05:53,025 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:05:54,457 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 15:05:54,458 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:05:55,455 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:05:55,455 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:05:55,456 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:05:55,468 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:05:56,027 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:05:57,030 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:05:59,065 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:05:59,065 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:05:59,066 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:06:00,349 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:06:01,282 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:06:01,283 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:06:01,283 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:06:01,288 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:06:02,034 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:06:03,041 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:06:06,325 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:06:06,426 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:06:06,426 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:06:06,427 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:06:06,438 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:06:07,050 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:06:07,050 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:06:10,686 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:06:10,687 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:06:10,687 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:06:10,694 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:06:11,053 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:06:11,695 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:06:13,057 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:06:14,065 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:06:14,065 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:06:14,067 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:06:15,012 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:06:15,013 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:06:15,013 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:06:15,022 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:06:15,058 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:06:17,057 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:06:17,065 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:06:18,640 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:06:18,641 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:06:18,641 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:06:18,647 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:06:19,067 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:06:21,074 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:06:22,162 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:06:22,162 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:06:22,163 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:06:22,163 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:06:22,170 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:06:23,077 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:06:23,078 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:06:24,469 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 15:06:27,471 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:06:27,887 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:06:27,888 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:06:27,888 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:06:27,896 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:06:28,081 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:06:29,065 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:06:29,065 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:06:29,067 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:06:29,087 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:06:31,513 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:06:31,513 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:06:31,513 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:06:31,521 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:06:32,089 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:06:32,557 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:06:33,100 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:06:37,557 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:06:39,448 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:06:39,449 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:06:39,450 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:06:39,455 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:06:40,105 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:06:41,109 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:06:43,456 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:06:43,514 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:06:43,515 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:06:43,515 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:06:43,521 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:06:44,065 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:06:44,065 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:06:44,067 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:06:44,112 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:06:45,117 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:06:48,032 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:06:48,033 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:06:48,033 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:06:48,040 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:06:48,120 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:06:49,077 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:06:49,126 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:06:52,977 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:06:52,978 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:06:52,978 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:06:52,987 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:06:53,129 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:06:54,461 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 15:06:54,462 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:06:55,135 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:06:58,895 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:06:58,896 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:06:58,897 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:06:58,906 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:06:59,065 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:06:59,066 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:06:59,067 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:06:59,138 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:07:00,348 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:07:01,147 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:07:03,314 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:07:03,315 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:07:03,315 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:07:03,322 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:07:04,149 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:07:05,154 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:07:05,357 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:07:07,316 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:07:07,317 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:07:07,317 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:07:07,326 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:07:08,156 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:07:09,158 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:07:11,326 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:07:11,884 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:07:11,885 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:07:11,885 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:07:11,896 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:07:12,159 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:07:13,164 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:07:14,065 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:07:14,066 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:07:14,067 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:07:17,289 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:07:20,245 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:07:20,246 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:07:20,246 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:07:20,254 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:07:21,173 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:07:21,174 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:07:22,289 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:07:24,463 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 15:07:24,834 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:07:24,835 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:07:24,835 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:07:24,844 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:07:25,177 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:07:27,181 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:07:27,844 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:07:29,065 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:07:29,066 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:07:29,067 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:07:30,323 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:07:30,323 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:07:30,324 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:07:30,329 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:07:31,187 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:07:31,188 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:07:33,365 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:07:35,569 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:07:35,570 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:07:35,570 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:07:35,576 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:07:36,191 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:07:37,195 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:07:38,577 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:07:41,871 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:07:41,872 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:07:41,872 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:07:41,877 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:07:42,198 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:07:43,200 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:07:43,878 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:07:44,065 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:07:44,066 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:07:44,067 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:07:49,366 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:07:50,324 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:07:50,325 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:07:50,325 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:07:50,331 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:07:51,210 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:07:51,210 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:07:54,134 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:07:54,138 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:07:54,139 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:07:54,146 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:07:54,212 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:07:54,465 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 15:07:54,467 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:07:55,215 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:07:59,065 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:07:59,066 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:07:59,067 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:07:59,758 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:07:59,758 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:07:59,759 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:07:59,759 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:07:59,765 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:08:00,218 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:08:01,221 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:08:03,718 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:08:03,718 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:08:03,719 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:08:03,726 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:08:04,224 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:08:05,225 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:08:05,727 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:08:07,474 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:08:07,474 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:08:07,475 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:08:07,480 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:08:08,227 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:08:09,229 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:08:11,481 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:08:12,325 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:08:12,326 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:08:12,327 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:08:12,332 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:08:13,235 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:08:13,235 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:08:14,066 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:08:14,066 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:08:14,068 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:08:16,596 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:08:16,596 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:08:16,597 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:08:16,597 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:08:16,602 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:08:17,239 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:08:17,239 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:08:20,916 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:08:20,917 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:08:20,917 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:08:20,922 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:08:21,242 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:08:21,923 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:08:23,245 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:08:24,467 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 15:08:27,429 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:08:27,430 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:08:27,430 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:08:27,431 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:08:27,436 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:08:28,248 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:08:29,066 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:08:29,066 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:08:29,068 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:08:29,252 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:08:30,710 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:08:30,711 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:08:30,711 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:08:30,716 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:08:31,254 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:08:32,753 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:08:33,257 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:08:34,583 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:08:34,584 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:08:34,584 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:08:34,590 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:08:35,260 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:08:35,260 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:08:38,591 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:08:38,781 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:08:38,782 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:08:38,782 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:08:38,787 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:08:39,262 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:08:41,265 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:08:43,788 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:08:43,839 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:08:43,839 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:08:43,840 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:08:43,845 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:08:44,066 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:08:44,066 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:08:44,068 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:08:44,267 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:08:45,270 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:08:49,346 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:08:49,495 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:08:49,495 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:08:49,495 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:08:49,501 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:08:50,275 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:08:51,277 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:08:54,473 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 15:08:54,475 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:08:55,838 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:08:55,839 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:08:55,839 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:08:55,844 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:08:56,280 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:08:57,283 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:08:59,066 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:08:59,066 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:08:59,068 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:09:00,358 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:09:01,038 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:09:01,039 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:09:01,039 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:09:01,044 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:09:01,285 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:09:03,289 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:09:06,081 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:09:07,767 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:09:07,768 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:09:07,769 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:09:07,774 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:09:08,292 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:09:09,294 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:09:11,774 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:09:13,139 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:09:13,139 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:09:13,140 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:09:13,145 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:09:13,296 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:09:14,066 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:09:14,066 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:09:14,068 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:09:15,300 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:09:17,174 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:09:17,174 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:09:17,175 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:09:17,175 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:09:17,180 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:09:17,301 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:09:19,304 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:09:20,949 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:09:20,950 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:09:20,951 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:09:20,956 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:09:21,306 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:09:22,956 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:09:23,309 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:09:24,471 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 15:09:25,358 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:09:25,358 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:09:25,359 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:09:25,364 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:09:26,311 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:09:27,314 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:09:28,364 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:09:29,066 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:09:29,066 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:09:29,068 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:09:31,405 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:09:31,405 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:09:31,406 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:09:31,411 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:09:32,318 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:09:33,320 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:09:33,449 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:09:38,449 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:09:38,814 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:09:38,815 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:09:38,815 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:09:38,821 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:09:39,324 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:09:41,327 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:09:43,512 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:09:43,512 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:09:43,513 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:09:43,513 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:09:43,518 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:09:44,066 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:09:44,067 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:09:44,068 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:09:44,329 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:09:45,331 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:09:48,039 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:09:48,039 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:09:48,040 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:09:48,045 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:09:48,334 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:09:49,081 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:09:49,335 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:09:54,081 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:09:54,298 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:09:54,298 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:09:54,299 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:09:54,303 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:09:54,338 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:09:54,474 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 15:09:55,340 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:09:58,653 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:09:58,657 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:09:58,657 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:09:58,662 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:09:59,066 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:09:59,067 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:09:59,068 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:09:59,341 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:09:59,345 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:09:59,345 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:10:04,341 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:10:04,637 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:10:04,637 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:10:04,638 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:10:04,643 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:10:05,351 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:10:05,351 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:10:08,687 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:10:08,687 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:10:08,688 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:10:08,693 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:10:09,355 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:10:09,355 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:10:09,693 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:10:12,657 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:10:12,658 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:10:12,658 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:10:12,663 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:10:13,359 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:10:13,359 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:10:14,066 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:10:14,067 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:10:14,068 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:10:15,313 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:10:16,882 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:10:16,883 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:10:16,883 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:10:16,888 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:10:17,361 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:10:19,364 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:10:20,925 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:10:22,311 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:10:22,312 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:10:22,312 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:10:22,317 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:10:22,366 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:10:23,369 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:10:24,477 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 15:10:26,478 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:10:29,032 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:10:29,033 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:10:29,033 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:10:29,039 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:10:29,067 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:10:29,067 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:10:29,107 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:10:29,374 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:10:31,377 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:10:32,342 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:10:35,205 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:10:35,206 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:10:35,206 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:10:35,211 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:10:35,380 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:10:37,383 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:10:38,249 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:10:41,649 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:10:41,650 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:10:41,651 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:10:41,656 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:10:42,386 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:10:43,388 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:10:43,656 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:10:44,066 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:10:44,067 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:10:44,068 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:10:46,014 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:10:46,015 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:10:46,015 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:10:46,020 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:10:46,390 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:10:47,394 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:10:49,057 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:10:49,802 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:10:49,802 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:10:49,803 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:10:49,807 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:10:50,396 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:10:51,398 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:10:54,479 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 15:10:54,480 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:10:55,051 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:10:55,051 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:10:55,052 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:10:55,057 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:10:55,400 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:10:57,403 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:10:58,522 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:10:58,523 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:10:58,523 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:10:58,528 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:10:59,067 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:10:59,067 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:10:59,068 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:10:59,406 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:10:59,407 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:11:00,280 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:11:02,292 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:11:02,293 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:11:02,293 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:11:02,299 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:11:02,409 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:11:03,411 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:11:05,337 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:11:07,405 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:11:07,406 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:11:07,406 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:11:07,411 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:11:07,414 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:11:09,418 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:11:10,412 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:11:12,123 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:11:12,124 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:11:12,125 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:11:12,129 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:11:12,420 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:11:13,422 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:11:14,067 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:11:14,067 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:11:14,069 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:11:16,300 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:11:16,475 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:11:16,476 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:11:16,476 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:11:16,481 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:11:17,427 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:11:17,427 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:11:21,125 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:11:21,129 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:11:21,129 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:11:21,134 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:11:21,429 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:11:22,135 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:11:23,432 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:11:24,481 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 15:11:25,123 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:11:25,124 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:11:25,124 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:11:25,129 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:11:25,434 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:11:27,437 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:11:28,130 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:11:29,067 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:11:29,067 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:11:29,069 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:11:31,772 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:11:31,773 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:11:31,773 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:11:31,779 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:11:32,441 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:11:33,443 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:11:33,817 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:11:38,817 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:11:40,181 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:11:40,181 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:11:40,182 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:11:40,187 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:11:40,448 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:11:41,450 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:11:44,067 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:11:44,067 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:11:44,068 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:11:44,069 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:11:44,164 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:11:44,340 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:11:44,340 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:11:44,345 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:11:44,452 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:11:45,454 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:11:47,836 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:11:47,837 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:11:47,837 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:11:47,842 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:11:48,457 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:11:49,458 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:11:49,843 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:11:51,769 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:11:51,769 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:11:51,770 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:11:51,775 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:11:52,460 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:11:53,462 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:11:54,483 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 15:11:55,484 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:11:57,360 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:11:57,361 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:11:57,361 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:11:57,366 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:11:57,464 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:11:59,067 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:11:59,068 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:11:59,069 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:11:59,468 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:12:01,348 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:12:02,470 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:12:02,470 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:12:02,470 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:12:02,475 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:12:03,473 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:12:03,474 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:12:06,513 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:12:07,242 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:12:07,243 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:12:07,243 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:12:07,248 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:12:07,476 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:12:09,479 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:12:12,175 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:12:12,176 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:12:12,177 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:12:12,177 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:12:12,182 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:12:12,481 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:12:13,484 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:12:14,067 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:12:14,067 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:12:14,069 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:12:16,830 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:12:16,831 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:12:16,831 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:12:16,836 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:12:17,486 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:12:17,486 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:12:17,873 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:12:20,365 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:12:20,366 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:12:20,366 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:12:20,371 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:12:20,488 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:12:21,490 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:12:23,372 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:12:24,381 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:12:24,382 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:12:24,382 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:12:24,387 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:12:24,489 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 15:12:24,492 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:12:25,495 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:12:28,491 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:12:29,067 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:12:29,067 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:12:29,069 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:12:29,682 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:12:29,683 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:12:29,683 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:12:29,688 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:12:30,499 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:12:31,502 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:12:33,312 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:12:33,313 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:12:33,313 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:12:33,319 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:12:33,503 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:12:34,319 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:12:35,506 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:12:36,735 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:12:36,736 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:12:36,736 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:12:36,741 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:12:37,509 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:12:37,509 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:12:39,741 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:12:41,351 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:12:41,352 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:12:41,352 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:12:41,358 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:12:41,512 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:12:43,515 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:12:44,067 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:12:44,068 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:12:44,069 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:12:45,333 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:12:45,334 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:12:45,334 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:12:45,334 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:12:45,339 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:12:45,516 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:12:47,520 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:12:49,088 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:12:49,089 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:12:49,089 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:12:49,094 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:12:49,521 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:12:51,095 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:12:51,524 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:12:52,830 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:12:52,831 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:12:52,831 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:12:52,837 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:12:53,527 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:12:53,528 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:12:54,487 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 15:12:56,489 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:12:57,929 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:12:57,930 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:12:57,930 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:12:57,935 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:12:58,530 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:12:59,068 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:12:59,068 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:12:59,069 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:12:59,533 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:13:02,357 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:13:02,867 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:13:02,868 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:13:02,868 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:13:02,873 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:13:03,537 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:13:03,538 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:13:07,913 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:13:08,153 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:13:08,153 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:13:08,154 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:13:08,159 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:13:08,540 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:13:09,543 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:13:13,160 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:13:14,067 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:13:14,068 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:13:14,069 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:13:15,286 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:13:15,287 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:13:15,287 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:13:15,292 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:13:15,547 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:13:17,550 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:13:18,329 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:13:19,285 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:13:19,286 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:13:19,286 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:13:19,292 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:13:19,551 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:13:21,554 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:13:24,224 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:13:24,225 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:13:24,225 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:13:24,226 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:13:24,231 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:13:24,493 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 15:13:24,556 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:13:25,559 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:13:27,920 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:13:27,921 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:13:27,921 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:13:27,926 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:13:28,560 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:13:29,068 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:13:29,068 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:13:29,070 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:13:29,329 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:13:29,562 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:13:31,658 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:13:31,659 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:13:31,659 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:13:31,664 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:13:32,564 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:13:33,566 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:13:34,701 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:13:35,981 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:13:35,982 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:13:35,982 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:13:35,987 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:13:36,568 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:13:37,571 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:13:39,966 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:13:39,967 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:13:39,967 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:13:39,967 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:13:39,972 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:13:40,573 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:13:41,575 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:13:44,036 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:13:44,036 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:13:44,037 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:13:44,042 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:13:44,068 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:13:44,068 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:13:44,070 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:13:44,577 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:13:45,313 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:13:45,577 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:13:48,221 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:13:48,222 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:13:48,222 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:13:48,227 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:13:48,581 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:13:49,583 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:13:51,265 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:13:51,992 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:13:51,993 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:13:51,993 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:13:51,999 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:13:52,585 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:13:53,587 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:13:54,492 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 15:13:55,957 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:13:55,958 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:13:55,958 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:13:55,963 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:13:56,589 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:13:56,963 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:13:57,592 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:13:59,068 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:13:59,068 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:13:59,069 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:14:00,872 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:14:00,873 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:14:00,873 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:14:00,878 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:14:01,597 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:14:01,597 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:14:02,917 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:14:06,584 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:14:06,585 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:14:06,585 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:14:06,590 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:14:06,600 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:14:07,602 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:14:08,591 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:14:11,524 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:14:11,524 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:14:11,525 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:14:11,530 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:14:11,605 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:14:13,606 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:14:14,068 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:14:14,068 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:14:14,069 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:14:14,368 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:14:14,812 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:14:14,812 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:14:14,813 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:14:14,818 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:14:15,609 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:14:15,609 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:14:18,886 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:14:18,887 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:14:18,887 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:14:18,893 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:14:19,613 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:14:19,613 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:14:19,893 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:14:23,011 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:14:23,012 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:14:23,012 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:14:23,017 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:14:23,618 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:14:23,618 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:14:24,497 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 15:14:25,498 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:14:27,076 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:14:27,076 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:14:27,077 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:14:27,081 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:14:27,620 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:14:29,068 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:14:29,068 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:14:29,070 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:14:29,625 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:14:31,059 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:14:31,060 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:14:31,060 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:14:31,060 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:14:31,064 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:14:31,626 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:14:33,629 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:14:36,101 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:14:37,772 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:14:37,773 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:14:37,773 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:14:37,778 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:14:38,632 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:14:39,634 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:14:41,778 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:14:43,135 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:14:43,135 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:14:43,136 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:14:43,141 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:14:43,637 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:14:44,068 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:14:44,068 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:14:44,070 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:14:45,640 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:14:47,098 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:14:47,098 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:14:47,099 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:14:47,099 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:14:47,103 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:14:47,642 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:14:49,645 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:14:52,141 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:14:54,496 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 15:14:55,572 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:14:55,573 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:14:55,573 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:14:55,578 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:14:55,649 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:14:57,578 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:14:57,652 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:14:59,068 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:14:59,068 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:14:59,070 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:14:59,677 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:14:59,678 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:14:59,678 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:14:59,683 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:15:00,654 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:15:01,656 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:15:02,721 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:15:03,903 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:15:03,904 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:15:03,904 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:15:03,909 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:15:04,658 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:15:05,661 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:15:07,910 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:15:08,738 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:15:08,739 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:15:08,739 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:15:08,744 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:15:09,665 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:15:09,665 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:15:13,228 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:15:13,228 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:15:13,229 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:15:13,230 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:15:13,234 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:15:13,668 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:15:14,068 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:15:14,068 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:15:14,070 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:15:15,671 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:15:16,977 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:15:16,978 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:15:16,978 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:15:16,983 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:15:17,674 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:15:17,674 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:15:19,021 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:15:22,382 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:15:22,383 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:15:22,383 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:15:22,388 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:15:22,677 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:15:23,680 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:15:24,389 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:15:24,498 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 15:15:27,856 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:15:27,857 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:15:27,857 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:15:27,862 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:15:28,683 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:15:29,068 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:15:29,069 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:15:29,070 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:15:29,685 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:15:30,277 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:15:32,020 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:15:32,021 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:15:32,021 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:15:32,026 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:15:32,686 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:15:33,689 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:15:36,065 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:15:37,102 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:15:37,103 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:15:37,103 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:15:37,108 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:15:37,693 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:15:37,694 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:15:41,108 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:15:44,068 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:15:44,069 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:15:44,070 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:15:45,268 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:15:45,269 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:15:45,269 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:15:45,275 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:15:45,698 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:15:46,313 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:15:47,702 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:15:50,477 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:15:50,478 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:15:50,478 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:15:50,484 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:15:50,704 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:15:51,484 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:15:51,707 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:15:54,506 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 15:15:55,627 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:15:55,628 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:15:55,629 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:15:55,633 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:15:55,709 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:15:56,634 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:15:57,712 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:15:59,068 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:15:59,069 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:15:59,070 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:16:01,676 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:16:01,676 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:16:01,677 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:16:01,677 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:16:01,682 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:16:01,714 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:16:03,718 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:16:05,445 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:16:05,445 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:16:05,446 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:16:05,451 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:16:05,719 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:16:07,451 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:16:07,722 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:16:10,078 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:16:10,079 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:16:10,079 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:16:10,084 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:16:10,724 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:16:11,726 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:16:13,085 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:16:14,068 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:16:14,069 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:16:14,070 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:16:14,390 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:16:14,391 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:16:14,391 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:16:14,396 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:16:14,728 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:16:15,731 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:16:18,433 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:16:21,108 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:16:21,109 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:16:21,109 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:16:21,114 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:16:21,736 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:16:21,736 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:16:24,114 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:16:24,502 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 15:16:26,406 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:16:26,407 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:16:26,407 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:16:26,412 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:16:26,739 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:16:27,742 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:16:29,068 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:16:29,069 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:16:29,071 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:16:29,365 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:16:30,298 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:16:30,299 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:16:30,299 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:16:30,304 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:16:30,745 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:16:31,747 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:16:34,011 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:16:34,011 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:16:34,012 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:16:34,017 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:16:34,749 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:16:35,017 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:16:35,752 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:16:40,018 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:16:40,049 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:16:40,050 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:16:40,050 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:16:40,055 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:16:40,755 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:16:41,757 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:16:44,069 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:16:44,069 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:16:44,071 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:16:44,647 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:16:44,648 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:16:44,648 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:16:44,653 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:16:44,759 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:16:45,689 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:16:45,762 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:16:48,691 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:16:48,692 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:16:48,693 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:16:48,697 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:16:48,764 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:16:49,767 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:16:50,698 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:16:54,508 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 15:16:54,583 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:16:54,584 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:16:54,584 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:16:54,589 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:16:54,770 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:16:55,773 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:16:56,590 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:16:58,889 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:16:58,890 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:16:58,890 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:16:58,895 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:16:59,069 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:16:59,069 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:16:59,071 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:16:59,777 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:16:59,777 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:17:02,291 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:17:04,344 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:17:04,345 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:17:04,345 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:17:04,350 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:17:04,781 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:17:05,783 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:17:07,389 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:17:08,690 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:17:08,691 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:17:08,691 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:17:08,696 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:17:08,785 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:17:09,787 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:17:12,697 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:17:14,069 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:17:14,069 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:17:14,071 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:17:17,215 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:17:17,215 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:17:17,216 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:17:17,221 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:17:17,794 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:17:17,794 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:17:18,257 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:17:20,952 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:17:20,953 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:17:20,954 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:17:20,959 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:17:21,798 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:17:21,798 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:17:23,959 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:17:24,507 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 15:17:27,879 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:17:27,880 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:17:27,880 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:17:27,885 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:17:28,803 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:17:29,069 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:17:29,069 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:17:29,071 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:17:29,356 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:17:29,806 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:17:31,676 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:17:31,677 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:17:31,677 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:17:31,682 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:17:31,807 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:17:33,810 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:17:34,721 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:17:35,849 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:17:35,850 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:17:35,850 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:17:35,855 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:17:36,812 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:17:37,815 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:17:39,856 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:17:40,636 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:17:40,637 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:17:40,637 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:17:40,642 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:17:40,817 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:17:41,820 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:17:44,069 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:17:44,069 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:17:44,071 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:17:44,480 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:17:44,481 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:17:44,481 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:17:44,486 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:17:44,821 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:17:45,525 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:17:45,824 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:17:50,525 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:17:50,827 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:17:50,827 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:17:50,828 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:17:50,833 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:17:51,832 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:17:51,832 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:17:54,509 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 15:17:54,766 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:17:54,770 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:17:54,771 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:17:54,776 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:17:54,833 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:17:55,776 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:17:55,834 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:17:58,722 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:17:58,723 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:17:58,723 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:17:58,728 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:17:58,836 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:17:59,069 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:17:59,070 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:17:59,071 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:17:59,838 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:18:01,363 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:18:02,754 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:18:02,755 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:18:02,755 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:18:02,760 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:18:02,840 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:18:03,843 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:18:06,797 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:18:07,092 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:18:07,093 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:18:07,093 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:18:07,098 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:18:07,847 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:18:07,847 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:18:12,099 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:18:13,737 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:18:13,737 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:18:13,738 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:18:13,743 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:18:13,850 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:18:14,069 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:18:14,069 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:18:14,071 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:18:15,854 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:18:17,064 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:18:17,064 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:18:17,065 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:18:17,070 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:18:17,109 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:18:17,857 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:18:17,857 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:18:21,551 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:18:21,552 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:18:21,552 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:18:21,557 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:18:21,859 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:18:22,557 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:18:23,862 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:18:24,511 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 15:18:28,398 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:18:28,399 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:18:28,399 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:18:28,400 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:18:28,404 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:18:28,865 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:18:29,069 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:18:29,070 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:18:29,071 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:18:29,869 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:18:32,352 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:18:32,352 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:18:32,353 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:18:32,357 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:18:32,871 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:18:33,874 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:18:34,397 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:18:37,218 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:18:37,219 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:18:37,219 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:18:37,224 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:18:37,878 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:18:37,878 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:18:40,225 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:18:41,441 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:18:41,442 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:18:41,442 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:18:41,447 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:18:41,880 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:18:43,883 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:18:44,069 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:18:44,070 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:18:44,071 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:18:45,318 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:18:47,360 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:18:47,360 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:18:47,361 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:18:47,366 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:18:47,887 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:18:49,890 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:18:50,405 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:18:52,831 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:18:52,832 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:18:52,832 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:18:52,837 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:18:52,892 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:18:53,895 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:18:54,513 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 15:18:55,514 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:18:59,069 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:18:59,070 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:18:59,071 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:19:00,591 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:19:00,591 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:19:00,592 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:19:00,592 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:19:00,597 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:19:00,899 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:19:01,901 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:19:04,423 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:19:04,424 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:19:04,424 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:19:04,429 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:19:04,904 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:19:05,906 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:19:06,429 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:19:08,885 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:19:08,885 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:19:08,886 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:19:08,891 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:19:08,908 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:19:09,908 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:19:11,891 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:19:13,337 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:19:13,338 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:19:13,339 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:19:13,343 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:19:13,913 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:19:13,913 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:19:14,069 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:19:14,070 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:19:14,071 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:19:16,878 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:19:16,879 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:19:16,879 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:19:16,884 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:19:16,915 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:19:16,924 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:19:17,917 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:19:21,811 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:19:21,812 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:19:21,812 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:19:21,817 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:19:21,920 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:19:22,818 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:19:23,923 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:19:24,515 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 15:19:27,459 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:19:27,460 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:19:27,460 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:19:27,465 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:19:27,926 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:19:28,465 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:19:29,070 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:19:29,070 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:19:29,072 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:19:29,930 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:19:30,976 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:19:30,977 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:19:30,977 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:19:30,982 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:19:31,934 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:19:31,934 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:19:34,021 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:19:35,845 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:19:35,846 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:19:35,846 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:19:35,851 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:19:35,936 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:19:37,939 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:19:39,851 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:19:39,909 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:19:39,909 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:19:39,910 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:19:39,915 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:19:39,940 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:19:41,943 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:19:43,305 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:19:43,306 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:19:43,306 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:19:43,311 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:19:43,946 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:19:43,946 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:19:44,070 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:19:44,070 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:19:44,072 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:19:45,337 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:19:48,598 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:19:48,599 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:19:48,599 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:19:48,604 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:19:48,950 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:19:49,953 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:19:50,641 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:19:54,517 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 15:19:55,325 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:19:55,325 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:19:55,326 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:19:55,331 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:19:55,956 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:19:55,956 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:19:56,331 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:19:58,857 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:19:58,858 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:19:58,858 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:19:58,863 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:19:58,958 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:19:59,070 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:19:59,070 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:19:59,072 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:19:59,961 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:20:02,315 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:20:05,806 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:20:05,806 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:20:05,806 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:20:05,812 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:20:05,964 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:20:07,849 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:20:07,968 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:20:10,480 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:20:10,481 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:20:10,482 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:20:10,486 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:20:10,970 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:20:11,972 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:20:13,487 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:20:14,070 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:20:14,070 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:20:14,072 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:20:15,037 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:20:15,038 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:20:15,038 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:20:15,043 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:20:15,976 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:20:15,977 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:20:19,081 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:20:19,206 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:20:19,207 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:20:19,207 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:20:19,212 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:20:19,980 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:20:19,981 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:20:23,350 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:20:23,351 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:20:23,351 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:20:23,356 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:20:23,985 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:20:23,985 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:20:24,357 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:20:24,522 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 15:20:27,009 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:20:27,009 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:20:27,010 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:20:27,015 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:20:27,989 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:20:27,990 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:20:29,070 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:20:29,070 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:20:29,072 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:20:30,328 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:20:31,570 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:20:31,571 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:20:31,571 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:20:31,576 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:20:31,993 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:20:33,996 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:20:35,613 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:20:35,778 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:20:35,779 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:20:35,779 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:20:35,784 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:20:35,998 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:20:38,001 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:20:40,491 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:20:40,492 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:20:40,493 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:20:40,498 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:20:41,002 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:20:41,498 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:20:42,005 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:20:44,070 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:20:44,070 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:20:44,072 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:20:44,296 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:20:44,317 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:20:44,317 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:20:44,322 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:20:45,007 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:20:46,009 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:20:47,323 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:20:48,279 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:20:48,280 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:20:48,280 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:20:48,285 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:20:49,012 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:20:50,015 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:20:52,325 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:20:53,055 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:20:53,056 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:20:53,056 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:20:53,061 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:20:54,019 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:20:54,019 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:20:54,521 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 15:20:57,100 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:20:57,101 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:20:57,102 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:20:57,107 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:20:58,023 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:20:58,024 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:20:58,107 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:20:59,070 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:20:59,070 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:20:59,072 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:21:01,071 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:21:01,072 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:21:01,072 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:21:01,077 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:21:02,028 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:21:02,028 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:21:03,117 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:21:05,088 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:21:05,089 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:21:05,089 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:21:05,094 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:21:06,033 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:21:06,033 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:21:09,094 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:21:12,018 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:21:12,019 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:21:12,019 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:21:12,024 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:21:12,037 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:21:14,040 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:21:14,070 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:21:14,071 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:21:14,072 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:21:14,365 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:21:15,706 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:21:15,707 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:21:15,707 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:21:15,712 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:21:16,041 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:21:18,044 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:21:19,749 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:21:19,853 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:21:19,854 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:21:19,854 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:21:19,859 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:21:20,045 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:21:22,048 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:21:23,633 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:21:23,634 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:21:23,634 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:21:23,639 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:21:24,049 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:21:24,527 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 15:21:25,528 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:21:26,053 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:21:29,024 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:21:29,024 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:21:29,025 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:21:29,030 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:21:29,055 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:21:29,070 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:21:29,071 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:21:29,072 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:21:30,058 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:21:31,300 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:21:34,043 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:21:34,043 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:21:34,043 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:21:34,048 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:21:34,061 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:21:36,064 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:21:37,085 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:21:42,086 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:21:43,150 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:21:43,150 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:21:43,151 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:21:43,155 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:21:44,070 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:21:44,070 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:21:44,070 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:21:44,071 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:21:44,073 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:21:47,365 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:21:47,743 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:21:47,744 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:21:47,744 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:21:47,749 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:21:48,074 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:21:50,077 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:21:52,353 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:21:52,353 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:21:52,354 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:21:52,358 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:21:53,078 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:21:53,359 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:21:54,081 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:21:54,525 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 15:21:55,830 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:21:55,831 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:21:55,831 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:21:55,836 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:21:56,082 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:21:58,085 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:21:58,837 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:21:59,070 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:21:59,071 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:21:59,073 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:22:01,164 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:22:01,165 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:22:01,165 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:22:01,170 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:22:02,089 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:22:02,089 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:22:04,209 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:22:06,686 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:22:06,687 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:22:06,687 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:22:06,692 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:22:07,093 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:22:08,096 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:22:09,693 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:22:12,263 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:22:12,263 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:22:12,264 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:22:12,269 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:22:13,099 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:22:14,071 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:22:14,071 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:22:14,073 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:22:14,102 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:22:15,376 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:22:16,409 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:22:16,410 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:22:16,410 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:22:16,415 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:22:17,103 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:22:18,106 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:22:20,453 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:22:22,446 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:22:22,446 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:22:22,447 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:22:22,452 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:22:23,109 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:22:24,112 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:22:24,527 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 15:22:25,529 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:22:28,234 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:22:28,235 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:22:28,235 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:22:28,240 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:22:29,071 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:22:29,071 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:22:29,073 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:22:29,116 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:22:30,120 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:22:31,310 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:22:32,288 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:22:32,288 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:22:32,288 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:22:32,294 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:22:33,122 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:22:34,125 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:22:36,333 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:22:36,641 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:22:36,646 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:22:36,646 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:22:36,652 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:22:37,127 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:22:38,130 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:22:40,852 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:22:40,853 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:22:40,853 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:22:40,858 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:22:41,132 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:22:41,859 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:22:42,135 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:22:44,071 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:22:44,071 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:22:44,073 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:22:45,655 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:22:45,655 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:22:45,656 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:22:45,661 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:22:46,137 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:22:47,697 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:22:48,141 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:22:49,509 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:22:49,510 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:22:49,510 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:22:49,515 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:22:50,145 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:22:50,145 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:22:53,516 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:22:54,529 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 15:22:56,713 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:22:56,714 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:22:56,714 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:22:56,719 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:22:57,149 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:22:58,151 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:22:58,719 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:22:59,071 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:22:59,071 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:22:59,073 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:23:00,957 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:23:00,958 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:23:00,958 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:23:00,964 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:23:01,153 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:23:02,156 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:23:04,001 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:23:07,490 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:23:07,491 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:23:07,491 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:23:07,496 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:23:08,161 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:23:08,162 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:23:09,497 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:23:11,887 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:23:11,888 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:23:11,888 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:23:11,894 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:23:12,164 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:23:14,071 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:23:14,071 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:23:14,073 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:23:14,168 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:23:15,366 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:23:15,925 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:23:15,926 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:23:15,926 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:23:15,934 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:23:16,169 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:23:18,179 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:23:19,462 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:23:19,463 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:23:19,463 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:23:19,469 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:23:20,185 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:23:20,185 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:23:20,470 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:23:23,294 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:23:23,295 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:23:23,295 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:23:23,303 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:23:24,193 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:23:24,193 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:23:24,531 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 15:23:25,533 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:23:28,774 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:23:28,775 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:23:28,775 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:23:28,782 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:23:29,071 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:23:29,071 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:23:29,073 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:23:29,197 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:23:30,202 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:23:31,278 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:23:33,830 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:23:33,831 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:23:33,831 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:23:33,837 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:23:34,205 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:23:36,208 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:23:36,873 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:23:37,747 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:23:37,748 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:23:37,748 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:23:37,754 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:23:38,209 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:23:40,215 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:23:42,755 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:23:43,227 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:23:43,228 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:23:43,228 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:23:43,236 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:23:44,071 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:23:44,071 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:23:44,073 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:23:44,221 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:23:44,221 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:23:47,619 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:23:47,620 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:23:47,620 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:23:47,626 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:23:48,231 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:23:48,231 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:23:48,661 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:23:51,175 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:23:51,176 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:23:51,177 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:23:51,182 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:23:51,233 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:23:52,237 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:23:54,183 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:23:54,538 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 15:23:55,513 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:23:55,514 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:23:55,514 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:23:55,521 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:23:56,243 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:23:56,243 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:23:59,071 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:23:59,072 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:23:59,073 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:23:59,348 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:23:59,375 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:23:59,375 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:23:59,375 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:23:59,381 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:24:00,249 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:24:00,249 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:24:03,081 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:24:03,082 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:24:03,082 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:24:03,088 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:24:03,251 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:24:05,125 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:24:06,104 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:24:07,044 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:24:07,045 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:24:07,045 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:24:07,051 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:24:07,105 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:24:08,108 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:24:11,000 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:24:11,001 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:24:11,001 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:24:11,001 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:24:11,007 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:24:11,110 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:24:12,114 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:24:14,071 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:24:14,072 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:24:14,073 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:24:16,198 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:24:16,199 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:24:16,199 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:24:16,199 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:24:16,206 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:24:17,117 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:24:18,122 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:24:21,241 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:24:21,288 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:24:21,289 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:24:21,289 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:24:21,295 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:24:22,127 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:24:22,127 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:24:24,536 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 15:24:26,537 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:24:29,071 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:24:29,072 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:24:29,073 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:24:30,280 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:24:30,281 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:24:30,281 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:24:30,289 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:24:31,133 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:24:32,136 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:24:32,325 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:24:35,166 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:24:35,167 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:24:35,167 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:24:35,172 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:24:36,140 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:24:36,141 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:24:38,173 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:24:39,858 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:24:39,859 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:24:39,859 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:24:39,865 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:24:40,143 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:24:42,148 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:24:43,695 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:24:43,696 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:24:43,696 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:24:43,697 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:24:43,702 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:24:44,072 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:24:44,072 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:24:44,074 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:24:44,151 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:24:44,152 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:24:47,639 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:24:47,639 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:24:47,639 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:24:47,645 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:24:48,156 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:24:48,157 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:24:49,681 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:24:51,191 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:24:51,192 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:24:51,192 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:24:51,198 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:24:52,162 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:24:52,163 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:24:54,545 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 15:24:55,509 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:24:55,510 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:24:55,510 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:24:55,510 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:24:55,515 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:24:56,167 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:24:56,167 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:24:59,072 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:24:59,072 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:24:59,073 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:24:59,330 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:24:59,366 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:24:59,366 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:24:59,371 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:25:00,172 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:25:00,172 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:25:01,372 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:25:05,243 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:25:05,244 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:25:05,244 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:25:05,250 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:25:06,178 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:25:06,178 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:25:07,285 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:25:10,951 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:25:10,952 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:25:10,952 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:25:10,957 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:25:11,181 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:25:12,183 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:25:12,958 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:25:14,072 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:25:14,072 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:25:14,074 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:25:14,746 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:25:14,747 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:25:14,747 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:25:14,753 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:25:15,185 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:25:16,186 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:25:18,789 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:25:18,875 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:25:18,875 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:25:18,876 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:25:18,881 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:25:19,188 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:25:20,190 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:25:23,881 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:25:24,540 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 15:25:24,599 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:25:24,600 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:25:24,600 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:25:24,605 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:25:25,194 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:25:26,196 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:25:29,072 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:25:29,072 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:25:29,074 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:25:29,335 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:25:34,335 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:25:34,971 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:25:34,972 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:25:34,972 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:25:34,977 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:25:35,203 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:25:36,205 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:25:38,900 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:25:38,900 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:25:38,901 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:25:38,906 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:25:39,207 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:25:39,906 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:25:40,210 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:25:44,072 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:25:44,072 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:25:44,074 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:25:45,373 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:25:46,598 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:25:46,598 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:25:46,599 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:25:46,604 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:25:47,214 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:25:48,216 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:25:50,641 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:25:51,973 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:25:51,974 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:25:51,974 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:25:51,979 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:25:52,220 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:25:54,223 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:25:54,542 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 15:25:56,543 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:25:59,072 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:25:59,072 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:25:59,074 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:26:00,439 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:26:00,440 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:26:00,440 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:26:00,445 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:26:01,227 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:26:02,230 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:26:02,485 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:26:06,358 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:26:06,362 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:26:06,362 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:26:06,367 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:26:07,233 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:26:08,235 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:26:08,368 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:26:10,767 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:26:10,768 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:26:10,768 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:26:10,773 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:26:11,237 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:26:12,240 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:26:13,774 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:26:14,072 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:26:14,072 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:26:14,074 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:26:16,769 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:26:16,770 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:26:16,770 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:26:16,776 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:26:17,242 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:26:18,245 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:26:18,813 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:26:20,327 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:26:20,327 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:26:20,328 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:26:20,332 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:26:21,247 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:26:22,250 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:26:24,333 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:26:24,544 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 15:26:25,955 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:26:25,956 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:26:25,956 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:26:25,961 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:26:26,252 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:26:28,255 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:26:29,072 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:26:29,073 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:26:29,074 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:26:30,309 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:26:32,261 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:26:32,262 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:26:32,262 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:26:32,267 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:26:33,259 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:26:34,262 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:26:36,305 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:26:41,306 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:26:42,604 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:26:42,605 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:26:42,605 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:26:42,611 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:26:43,268 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:26:44,072 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:26:44,073 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:26:44,074 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:26:44,270 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:26:46,373 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:26:47,249 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:26:47,249 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:26:47,250 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:26:47,255 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:26:47,272 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:26:48,275 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:26:52,293 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:26:52,697 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:26:52,697 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:26:52,698 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:26:52,702 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:26:53,279 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:26:54,281 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:26:54,546 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 15:26:57,547 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:26:59,072 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:26:59,073 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:26:59,074 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:27:00,193 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:27:00,194 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:27:00,194 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:27:00,199 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:27:00,285 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:27:02,288 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:27:03,237 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:27:04,748 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:27:04,749 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:27:04,749 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:27:04,754 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:27:05,290 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:27:06,293 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:27:08,755 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:27:09,475 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:27:09,476 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:27:09,476 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:27:09,481 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:27:10,297 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:27:10,297 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:27:13,582 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:27:13,583 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:27:13,583 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:27:13,588 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:27:14,072 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:27:14,073 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:27:14,074 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:27:14,301 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:27:14,301 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:27:14,381 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:27:18,189 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:27:18,190 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:27:18,190 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:27:18,196 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:27:18,303 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:27:20,233 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:27:20,306 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:27:22,318 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:27:22,319 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:27:22,319 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:27:22,324 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:27:23,308 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:27:24,311 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:27:24,551 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 15:27:25,553 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:27:26,051 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:27:26,052 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:27:26,052 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:27:26,057 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:27:26,312 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:27:28,315 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:27:29,072 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:27:29,073 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:27:29,075 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:27:31,288 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:27:32,166 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:27:32,166 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:27:32,166 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:27:32,171 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:27:32,319 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:27:34,320 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:27:37,209 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:27:38,268 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:27:38,269 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:27:38,269 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:27:38,274 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:27:38,322 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:27:40,326 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:27:42,275 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:27:43,341 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:27:43,342 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:27:43,342 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:27:43,347 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:27:44,073 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:27:44,073 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:27:44,075 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:27:44,330 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:27:44,330 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:27:47,374 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:27:47,962 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:27:47,962 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:27:47,963 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:27:47,968 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:27:48,332 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:27:50,337 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:27:51,709 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:27:51,709 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:27:51,710 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:27:51,715 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:27:52,340 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:27:52,340 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:27:52,715 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:27:54,550 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 15:27:55,441 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:27:55,442 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:27:55,442 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:27:55,447 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:27:56,344 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:27:56,344 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:27:58,447 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:27:59,073 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:27:59,073 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:27:59,075 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:28:00,835 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:28:00,836 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:28:00,836 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:28:00,841 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:28:01,347 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:28:02,350 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:28:03,881 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:28:05,496 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:28:05,497 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:28:05,497 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:28:05,502 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:28:06,354 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:28:06,355 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:28:09,503 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:28:10,341 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:28:10,341 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:28:10,342 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:28:10,347 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:28:10,357 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:28:12,360 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:28:14,073 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:28:14,073 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:28:14,075 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:28:14,498 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:28:14,499 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:28:14,499 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:28:14,504 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:28:14,504 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:28:15,362 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:28:16,364 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:28:19,541 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:28:21,364 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:28:21,364 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:28:21,365 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:28:21,371 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:28:22,370 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:28:22,370 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:28:24,555 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 15:28:24,557 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:28:28,732 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:28:28,733 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:28:28,733 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:28:28,738 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:28:29,073 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:28:29,073 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:28:29,075 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:28:29,375 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:28:30,355 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:28:30,377 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:28:32,324 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:28:32,324 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:28:32,325 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:28:32,329 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:28:32,380 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:28:34,383 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:28:35,369 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:28:38,363 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:28:38,363 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:28:38,364 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:28:38,369 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:28:38,385 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:28:40,370 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:28:40,388 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:28:42,516 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:28:42,517 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:28:42,517 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:28:42,522 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:28:43,390 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:28:44,073 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:28:44,073 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:28:44,075 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:28:44,393 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:28:45,387 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:28:47,598 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:28:47,599 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:28:47,599 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:28:47,604 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:28:48,397 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:28:48,397 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:28:50,641 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:28:51,782 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:28:51,783 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:28:51,783 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:28:51,789 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:28:52,402 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:28:52,403 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:28:54,554 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 15:28:56,226 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:28:56,227 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:28:56,227 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:28:56,228 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:28:56,233 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:28:56,405 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:28:58,408 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:28:59,073 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:28:59,073 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:28:59,075 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:29:01,383 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:29:01,683 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:29:01,684 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:29:01,684 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:29:01,689 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:29:02,412 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:29:02,413 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:29:06,155 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:29:06,156 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:29:06,157 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:29:06,161 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:29:06,415 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:29:07,162 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:29:08,418 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:29:10,545 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:29:10,546 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:29:10,546 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:29:10,551 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:29:11,420 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:29:12,421 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:29:12,552 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:29:14,073 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:29:14,073 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:29:14,075 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:29:18,352 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:29:18,785 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:29:18,785 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:29:18,786 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:29:18,791 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:29:19,425 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:29:20,427 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:29:23,829 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:29:24,486 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:29:24,487 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:29:24,487 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:29:24,492 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:29:24,556 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 15:29:25,430 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:29:26,433 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:29:28,412 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:29:28,416 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:29:28,416 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:29:28,421 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:29:28,435 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:29:29,073 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:29:29,074 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:29:29,075 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:29:29,305 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:29:30,438 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:29:33,207 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:29:33,207 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:29:33,208 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:29:33,213 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:29:33,440 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:29:34,441 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:29:35,249 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:29:40,249 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:29:41,662 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:29:41,662 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:29:41,663 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:29:41,668 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:29:42,448 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:29:42,448 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:29:44,073 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:29:44,074 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:29:44,075 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:29:45,357 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:29:45,591 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:29:45,592 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:29:45,592 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:29:45,597 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:29:46,452 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:29:46,452 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:29:49,590 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:29:49,591 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:29:49,591 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:29:49,596 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:29:50,458 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:29:50,458 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:29:50,596 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:29:54,559 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 15:29:56,560 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:29:58,198 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:29:58,199 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:29:58,199 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:29:58,204 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:29:58,462 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:29:59,073 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:29:59,074 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:29:59,075 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:30:00,465 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:30:02,332 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:30:03,293 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:30:03,293 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:30:03,294 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:30:03,299 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:30:03,467 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:30:04,470 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:30:07,337 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:30:11,349 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:30:11,350 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:30:11,350 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:30:11,355 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:30:11,474 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:30:12,356 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:30:12,476 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:30:14,074 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:30:14,074 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:30:14,076 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:30:15,512 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:30:15,512 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:30:15,513 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:30:15,518 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:30:16,481 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:30:16,481 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:30:17,557 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:30:19,634 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:30:19,634 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:30:19,635 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:30:19,640 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:30:20,485 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:30:20,485 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:30:22,640 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:30:23,838 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:30:23,839 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:30:23,839 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:30:23,844 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:30:24,488 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:30:24,561 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 15:30:26,491 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:30:28,562 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:30:29,074 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:30:29,074 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:30:29,076 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:30:29,534 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:30:29,535 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:30:29,535 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:30:29,540 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:30:30,496 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:30:30,496 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:30:33,577 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:30:33,587 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:30:33,588 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:30:33,588 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:30:33,593 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:30:34,502 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:30:34,502 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:30:37,720 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:30:37,721 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:30:37,721 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:30:37,726 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:30:38,504 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:30:38,726 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:30:40,507 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:30:41,475 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:30:41,476 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:30:41,476 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:30:41,481 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:30:41,508 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:30:42,510 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:30:44,074 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:30:44,074 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:30:44,076 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:30:44,291 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:30:47,086 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:30:47,087 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:30:47,087 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:30:47,092 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:30:47,513 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:30:48,516 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:30:50,129 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:30:52,500 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:30:52,500 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:30:52,501 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:30:52,506 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:30:52,520 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:30:54,523 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:30:54,566 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 15:30:55,567 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:30:57,357 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:30:57,358 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:30:57,358 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:30:57,363 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:30:57,525 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:30:58,527 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:30:59,074 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:30:59,074 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:30:59,076 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:31:01,338 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:31:01,710 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:31:01,711 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:31:01,711 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:31:01,716 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:31:02,530 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:31:04,533 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:31:06,753 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:31:10,017 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:31:10,017 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:31:10,018 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:31:10,023 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:31:10,537 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:31:12,023 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:31:12,540 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:31:13,649 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:31:13,650 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:31:13,650 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:31:13,655 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:31:14,074 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:31:14,074 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:31:14,076 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:31:14,543 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:31:14,543 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:31:17,332 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:31:21,005 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:31:21,006 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:31:21,006 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:31:21,011 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:31:21,547 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:31:22,548 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:31:23,049 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:31:24,565 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 15:31:24,769 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:31:24,770 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:31:24,770 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:31:24,776 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:31:25,549 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:31:26,552 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:31:28,777 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:31:29,074 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:31:29,074 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:31:29,076 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:31:30,315 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:31:30,316 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:31:30,316 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:31:30,321 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:31:30,555 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:31:32,560 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:31:34,361 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:31:35,223 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:31:35,223 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:31:35,224 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:31:35,229 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:31:35,561 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:31:36,564 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:31:39,081 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:31:39,082 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:31:39,082 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:31:39,088 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:31:39,566 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:31:40,089 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:31:40,568 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:31:42,274 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:31:42,274 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:31:42,275 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:31:42,280 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:31:42,569 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:31:44,074 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:31:44,074 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:31:44,076 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:31:44,572 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:31:45,333 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:31:47,505 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:31:47,505 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:31:47,506 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:31:47,511 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:31:47,574 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:31:48,577 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:31:50,549 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:31:54,570 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 15:31:55,571 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:31:56,388 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:31:56,389 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:31:56,390 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:31:56,395 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:31:56,582 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:31:58,585 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:31:59,074 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:31:59,075 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:31:59,076 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:32:00,847 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:32:00,848 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:32:00,848 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:32:00,849 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:32:00,853 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:32:01,587 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:32:02,590 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:32:04,952 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:32:04,953 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:32:04,953 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:32:04,958 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:32:05,592 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:32:05,959 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:32:06,595 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:32:09,781 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:32:09,782 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:32:09,782 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:32:09,787 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:32:10,599 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:32:10,599 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:32:11,788 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:32:14,074 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:32:14,075 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:32:14,076 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:32:16,227 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:32:16,228 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:32:16,228 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:32:16,233 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:32:16,603 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:32:17,269 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:32:18,606 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:32:21,929 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:32:21,929 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:32:21,930 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:32:21,935 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:32:22,608 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:32:22,935 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:32:24,569 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 15:32:24,612 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:32:27,178 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:32:27,179 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:32:27,179 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:32:27,184 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:32:27,614 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:32:28,184 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:32:28,616 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:32:29,074 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:32:29,075 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:32:29,076 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:32:33,177 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:32:33,178 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:32:33,178 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:32:33,183 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:32:33,221 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:32:33,620 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:32:34,623 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:32:38,221 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:32:39,471 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:32:39,471 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:32:39,472 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:32:39,477 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:32:39,626 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:32:40,629 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:32:43,478 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:32:44,052 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:32:44,053 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:32:44,053 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:32:44,058 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:32:44,074 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:32:44,075 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:32:44,076 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:32:44,631 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:32:46,634 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:32:47,840 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:32:47,840 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:32:47,841 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:32:47,846 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:32:48,635 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:32:48,885 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:32:50,640 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:32:52,630 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:32:52,631 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:32:52,631 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:32:52,637 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:32:52,641 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:32:54,571 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 15:32:54,572 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:32:54,644 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:32:59,075 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:32:59,075 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:32:59,077 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:32:59,296 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:32:59,374 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:32:59,374 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:32:59,379 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:32:59,647 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:33:00,380 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:33:00,650 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:33:05,380 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:33:05,906 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:33:05,907 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:33:05,907 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:33:05,912 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:33:06,653 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:33:08,657 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:33:10,949 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:33:10,988 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:33:10,992 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:33:10,992 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:33:10,997 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:33:11,658 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:33:12,661 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:33:14,075 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:33:14,075 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:33:14,077 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:33:16,343 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:33:17,299 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:33:17,300 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:33:17,300 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:33:17,305 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:33:17,664 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:33:18,666 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:33:21,361 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:33:21,362 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:33:21,362 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:33:21,362 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:33:21,367 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:33:21,668 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:33:22,671 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:33:24,573 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 15:33:26,575 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:33:26,736 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:33:26,736 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:33:26,737 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:33:26,742 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:33:27,674 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:33:28,677 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:33:29,075 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:33:29,075 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:33:29,077 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:33:32,311 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:33:35,222 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:33:35,222 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:33:35,223 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:33:35,227 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:33:35,682 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:33:36,684 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:33:38,265 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:33:39,607 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:33:39,607 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:33:39,608 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:33:39,613 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:33:39,686 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:33:40,687 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:33:43,614 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:33:44,075 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:33:44,075 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:33:44,077 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:33:47,728 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:33:47,729 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:33:47,729 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:33:47,734 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:33:48,693 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:33:48,693 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:33:48,773 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:33:53,773 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:33:54,575 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 15:33:58,549 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:33:58,550 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:33:58,550 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:33:58,555 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:33:58,700 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:33:59,075 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:33:59,075 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:33:59,077 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:33:59,321 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:34:00,703 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:34:04,321 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:34:04,623 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:34:04,623 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:34:04,624 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:34:04,630 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:34:04,706 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:34:06,710 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:34:09,665 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:34:11,277 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:34:11,278 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:34:11,278 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:34:11,283 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:34:11,712 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:34:12,715 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:34:14,075 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:34:14,075 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:34:14,077 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:34:15,145 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:34:15,145 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:34:15,146 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:34:15,146 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:34:15,151 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:34:15,717 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:34:16,720 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:34:20,189 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:34:22,314 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:34:22,314 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:34:22,315 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:34:22,320 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:34:22,723 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:34:24,583 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 15:34:24,726 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:34:25,584 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:34:26,230 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:34:26,231 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:34:26,231 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:34:26,236 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:34:26,728 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:34:28,731 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:34:29,075 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:34:29,075 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:34:29,077 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:34:30,123 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:34:30,124 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:34:30,124 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:34:30,129 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:34:30,732 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:34:31,169 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:34:32,736 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:34:36,169 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:34:37,552 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:34:37,552 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:34:37,553 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:34:37,558 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:34:37,739 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:34:38,742 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:34:41,559 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:34:41,574 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:34:41,575 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:34:41,575 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:34:41,581 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:34:41,744 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:34:42,746 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:34:44,075 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:34:44,076 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:34:44,077 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:34:45,919 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:34:45,920 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:34:45,920 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:34:45,927 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:34:46,750 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:34:46,751 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:34:46,965 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:34:51,293 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:34:51,294 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:34:51,294 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:34:51,299 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:34:51,755 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:34:52,300 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:34:52,757 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:34:54,579 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 15:34:55,495 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:34:55,496 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:34:55,496 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:34:55,501 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:34:55,759 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:34:56,762 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:34:57,502 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:34:59,075 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:34:59,076 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:34:59,077 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:35:01,405 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:35:01,406 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:35:01,406 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:35:01,411 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:35:01,764 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:35:02,767 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:35:03,449 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:35:05,279 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:35:05,279 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:35:05,280 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:35:05,285 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:35:05,769 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:35:06,772 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:35:09,286 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:35:09,689 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:35:09,689 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:35:09,690 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:35:09,695 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:35:09,774 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:35:10,776 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:35:14,076 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:35:14,076 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:35:14,078 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:35:14,158 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:35:14,317 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:35:14,318 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:35:14,318 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:35:14,323 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:35:14,778 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:35:16,781 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:35:19,052 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:35:19,053 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:35:19,053 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:35:19,058 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:35:19,783 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:35:20,059 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:35:20,786 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:35:23,447 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:35:23,447 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:35:23,448 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:35:23,453 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:35:23,788 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:35:24,585 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 15:35:24,790 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:35:25,586 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:35:29,076 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:35:29,076 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:35:29,078 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:35:30,521 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:35:30,521 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:35:30,521 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:35:30,526 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:35:30,794 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:35:31,565 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:35:32,798 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:35:34,999 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:35:35,000 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:35:35,000 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:35:35,005 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:35:35,800 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:35:36,802 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:35:37,005 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:35:40,924 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:35:40,925 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:35:40,925 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:35:40,932 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:35:41,805 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:35:42,808 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:35:42,932 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:35:44,076 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:35:44,076 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:35:44,078 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:35:44,877 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:35:44,878 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:35:44,878 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:35:44,883 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:35:45,810 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:35:46,813 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:35:48,921 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:35:50,079 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:35:50,079 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:35:50,079 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:35:50,084 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:35:50,816 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:35:52,819 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:35:54,085 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:35:54,583 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 15:35:56,693 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:35:56,693 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:35:56,693 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:35:56,698 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:35:56,821 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:35:58,825 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:35:59,076 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:35:59,077 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:35:59,078 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:35:59,311 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:36:00,384 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:36:00,385 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:36:00,385 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:36:00,390 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:36:00,826 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:36:02,829 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:36:04,429 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:36:06,842 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:36:06,843 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:36:06,843 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:36:06,848 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:36:07,832 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:36:08,835 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:36:09,848 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:36:11,251 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:36:11,252 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:36:11,252 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:36:11,256 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:36:11,837 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:36:12,840 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:36:14,076 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:36:14,077 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:36:14,078 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:36:15,248 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:36:15,248 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:36:15,249 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:36:15,249 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:36:15,253 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:36:15,841 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:36:16,844 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:36:19,146 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:36:19,147 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:36:19,147 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:36:19,153 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:36:19,846 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:36:20,848 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:36:21,153 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:36:22,730 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:36:22,731 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:36:22,731 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:36:22,736 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:36:22,849 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:36:24,586 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 15:36:24,853 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:36:26,587 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:36:27,064 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:36:27,065 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:36:27,065 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:36:27,070 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:36:27,855 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:36:28,858 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:36:29,076 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:36:29,077 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:36:29,078 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:36:31,310 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:36:31,310 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:36:31,310 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:36:31,315 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:36:31,861 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:36:32,353 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:36:32,863 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:36:35,672 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:36:35,672 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:36:35,673 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:36:35,678 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:36:35,865 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:36:36,868 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:36:37,678 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:36:39,685 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:36:39,686 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:36:39,686 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:36:39,691 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:36:39,869 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:36:40,872 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:36:42,692 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:36:44,073 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:36:44,074 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:36:44,074 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:36:44,077 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:36:44,078 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:36:44,079 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:36:44,079 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:36:44,876 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:36:44,876 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:36:48,283 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:36:52,471 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:36:52,471 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:36:52,471 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:36:52,476 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:36:52,882 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:36:53,513 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:36:54,588 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 15:36:54,885 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:36:56,405 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:36:56,405 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:36:56,406 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:36:56,410 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:36:56,886 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:36:58,889 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:36:59,077 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:36:59,077 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:36:59,077 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:36:59,078 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:37:01,629 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:37:01,629 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:37:01,629 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:37:01,634 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:37:01,891 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:37:02,894 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:37:04,673 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:37:07,056 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:37:07,057 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:37:07,057 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:37:07,062 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:37:07,897 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:37:08,900 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:37:10,063 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:37:11,512 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:37:11,513 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:37:11,513 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:37:11,518 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:37:11,902 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:37:12,904 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:37:14,077 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:37:14,077 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:37:14,078 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:37:15,340 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:37:15,640 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:37:15,641 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:37:15,641 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:37:15,646 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:37:15,906 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:37:16,908 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:37:19,783 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:37:19,783 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:37:19,784 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:37:19,789 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:37:19,910 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:37:20,789 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:37:20,913 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:37:24,590 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 15:37:24,634 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:37:24,634 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:37:24,634 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:37:24,640 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:37:24,916 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:37:26,640 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:37:26,919 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:37:28,883 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:37:28,884 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:37:28,884 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:37:28,889 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:37:28,920 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:37:29,077 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:37:29,077 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:37:29,079 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:37:30,924 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:37:32,359 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:37:32,826 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:37:32,827 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:37:32,827 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:37:32,832 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:37:32,926 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:37:34,929 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:37:37,869 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:37:41,665 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:37:41,665 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:37:41,665 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:37:41,671 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:37:41,933 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:37:42,934 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:37:43,671 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:37:44,077 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:37:44,077 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:37:44,079 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:37:47,311 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:37:47,312 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:37:47,312 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:37:47,317 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:37:47,936 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:37:48,939 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:37:49,357 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:37:51,634 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:37:51,635 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:37:51,635 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:37:51,640 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:37:51,942 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:37:52,945 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:37:54,595 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 15:37:54,597 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:37:59,071 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:37:59,072 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:37:59,072 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:37:59,077 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:37:59,077 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:37:59,077 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:37:59,079 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:37:59,949 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:38:00,328 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:38:00,951 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:38:03,564 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:38:03,564 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:38:03,565 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:38:03,570 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:38:03,953 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:38:04,956 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:38:05,609 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:38:10,609 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:38:12,044 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:38:12,044 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:38:12,045 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:38:12,050 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:38:12,962 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:38:12,963 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:38:14,077 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:38:14,078 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:38:14,079 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:38:16,302 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:38:17,123 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:38:17,123 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:38:17,123 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:38:17,128 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:38:17,965 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:38:18,968 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:38:21,933 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:38:21,934 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:38:21,934 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:38:21,934 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:38:21,939 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:38:21,970 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:38:22,972 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:38:24,594 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 15:38:25,751 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:38:25,752 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:38:25,752 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:38:25,758 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:38:25,974 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:38:26,977 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:38:27,758 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:38:29,077 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:38:29,077 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:38:29,079 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:38:31,821 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:38:31,821 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:38:31,822 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:38:31,827 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:38:31,981 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:38:32,865 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:38:32,983 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:38:36,155 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:38:36,156 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:38:36,156 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:38:36,161 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:38:36,986 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:38:36,986 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:38:38,161 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:38:41,238 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:38:41,239 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:38:41,239 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:38:41,244 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:38:41,989 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:38:42,991 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:38:43,245 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:38:44,077 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:38:44,077 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:38:44,079 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:38:46,547 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:38:46,547 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:38:46,548 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:38:46,553 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:38:46,994 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:38:48,589 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:38:48,997 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:38:51,407 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:38:51,408 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:38:51,408 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:38:51,414 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:38:52,000 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:38:53,002 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:38:54,414 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:38:54,599 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 15:38:56,014 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:38:56,014 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:38:56,015 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:38:56,020 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:38:57,007 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:38:57,007 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:38:59,077 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:38:59,078 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:38:59,079 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:39:00,356 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:39:02,158 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:39:02,158 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:39:02,158 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:39:02,163 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:39:03,012 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:39:03,013 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:39:06,095 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:39:06,096 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:39:06,096 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:39:06,096 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:39:06,101 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:39:07,017 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:39:07,017 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:39:09,681 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:39:09,682 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:39:09,682 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:39:09,687 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:39:10,019 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:39:11,022 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:39:11,688 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:39:13,292 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:39:13,292 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:39:13,293 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:39:13,297 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:39:14,023 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:39:14,077 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:39:14,078 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:39:14,079 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:39:15,026 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:39:17,343 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:39:22,285 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:39:22,285 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:39:22,285 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:39:22,290 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:39:23,031 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:39:23,329 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:39:24,598 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 15:39:25,034 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:39:27,507 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:39:27,507 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:39:27,507 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:39:27,512 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:39:28,036 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:39:28,513 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:39:29,039 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:39:29,077 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:39:29,078 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:39:29,079 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:39:31,477 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:39:31,478 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:39:31,478 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:39:31,483 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:39:32,042 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:39:33,045 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:39:33,521 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:39:36,793 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:39:36,793 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:39:36,794 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:39:36,799 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:39:37,047 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:39:38,799 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:39:39,050 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:39:41,300 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:39:41,301 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:39:41,301 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:39:41,306 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:39:42,052 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:39:43,055 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:39:44,078 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:39:44,078 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:39:44,078 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:39:44,080 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:39:45,261 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:39:45,261 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:39:45,262 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:39:45,267 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:39:46,056 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:39:47,059 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:39:48,958 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:39:48,959 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:39:48,959 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:39:48,964 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:39:49,060 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:39:49,964 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:39:51,065 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:39:54,368 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:39:54,369 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:39:54,369 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:39:54,375 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:39:54,600 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 15:39:55,067 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:39:55,602 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:39:57,071 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:39:59,078 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:39:59,078 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:39:59,080 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:40:01,341 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:40:03,183 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:40:03,184 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:40:03,184 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:40:03,189 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:40:04,075 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:40:05,078 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:40:07,229 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:40:07,321 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:40:07,322 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:40:07,322 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:40:07,327 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:40:08,080 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:40:09,082 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:40:11,492 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:40:11,493 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:40:11,493 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:40:11,498 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:40:12,084 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:40:12,499 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:40:13,086 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:40:14,078 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:40:14,078 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:40:14,080 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:40:15,528 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:40:15,529 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:40:15,529 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:40:15,534 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:40:16,088 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:40:17,091 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:40:17,573 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:40:21,782 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:40:21,782 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:40:21,783 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:40:21,788 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:40:22,094 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:40:22,788 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:40:23,097 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:40:24,602 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 15:40:26,050 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:40:26,051 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:40:26,051 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:40:26,056 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:40:26,098 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:40:27,101 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:40:28,057 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:40:29,078 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:40:29,078 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:40:29,080 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:40:30,416 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:40:30,416 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:40:30,417 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:40:30,422 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:40:31,104 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:40:33,108 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:40:33,461 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:40:34,688 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:40:34,688 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:40:34,689 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:40:34,694 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:40:35,109 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:40:37,112 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:40:38,694 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:40:40,190 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:40:40,190 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:40:40,191 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:40:40,196 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:40:41,118 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:40:41,118 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:40:44,078 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:40:44,079 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:40:44,079 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:40:44,080 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:40:44,415 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:40:44,416 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:40:44,416 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:40:44,421 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:40:45,120 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:40:47,123 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:40:49,461 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:40:51,160 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:40:51,161 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:40:51,161 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:40:51,166 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:40:52,127 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:40:53,130 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:40:54,604 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 15:40:54,607 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:40:55,368 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:40:55,368 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:40:55,369 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:40:55,374 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:40:56,132 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:40:57,134 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:40:59,078 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:40:59,078 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:40:59,080 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:40:59,347 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:40:59,348 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:40:59,348 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:40:59,353 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:41:00,136 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:41:00,393 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:41:01,138 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:41:03,535 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:41:03,536 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:41:03,536 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:41:03,541 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:41:04,141 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:41:05,144 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:41:05,542 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:41:10,412 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:41:10,413 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:41:10,413 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:41:10,418 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:41:11,147 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:41:11,419 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:41:13,150 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:41:14,078 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:41:14,079 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:41:14,080 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:41:15,034 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:41:15,034 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:41:15,035 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:41:15,039 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:41:15,152 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:41:17,077 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:41:17,155 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:41:19,510 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:41:19,511 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:41:19,511 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:41:19,516 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:41:20,157 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:41:21,159 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:41:22,517 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:41:23,796 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:41:23,797 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:41:23,797 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:41:23,802 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:41:24,161 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:41:24,609 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 15:41:25,164 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:41:27,333 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:41:27,334 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:41:27,334 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:41:27,339 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:41:28,166 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:41:28,339 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:41:29,078 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:41:29,079 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:41:29,080 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:41:29,168 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:41:32,079 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:41:32,080 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:41:32,080 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:41:32,085 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:41:32,172 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:41:33,174 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:41:34,125 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:41:37,637 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:41:37,638 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:41:37,638 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:41:37,643 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:41:38,177 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:41:39,178 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:41:39,644 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:41:41,557 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:41:41,557 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:41:41,558 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:41:41,562 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:41:42,180 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:41:43,182 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:41:44,078 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:41:44,079 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:41:44,080 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:41:45,325 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:41:46,961 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:41:46,961 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:41:46,962 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:41:46,966 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:41:47,185 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:41:49,188 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:41:51,005 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:41:51,498 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:41:51,499 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:41:51,499 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:41:51,505 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:41:52,191 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:41:53,193 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:41:54,608 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 15:41:56,610 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:41:59,008 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:41:59,009 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:41:59,009 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:41:59,014 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:41:59,079 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:41:59,079 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:41:59,081 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:41:59,197 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:42:01,200 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:42:02,272 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:42:03,828 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:42:03,828 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:42:03,829 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:42:03,834 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:42:04,202 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:42:05,205 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:42:07,873 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:42:09,497 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:42:09,498 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:42:09,498 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:42:09,503 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:42:10,208 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:42:11,210 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:42:13,115 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:42:13,115 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:42:13,116 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:42:13,116 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:42:13,120 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:42:13,211 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:42:14,079 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:42:14,079 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:42:14,081 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:42:15,215 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:42:16,794 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:42:16,795 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:42:16,795 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:42:16,800 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:42:17,216 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:42:18,837 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:42:19,219 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:42:20,603 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:42:20,603 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:42:20,603 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:42:20,608 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:42:21,220 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:42:23,223 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:42:24,610 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:42:24,614 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 15:42:29,024 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:42:29,024 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:42:29,025 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:42:29,029 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:42:29,079 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:42:29,079 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:42:29,080 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:42:29,227 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:42:30,303 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:42:31,230 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:42:34,320 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:42:34,321 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:42:34,321 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:42:34,325 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:42:35,236 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:42:35,236 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:42:35,365 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:42:40,365 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:42:43,109 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:42:43,110 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:42:43,110 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:42:43,116 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:42:43,240 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:42:44,079 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:42:44,079 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:42:44,081 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:42:45,243 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:42:46,364 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:42:47,014 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:42:47,015 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:42:47,015 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:42:47,020 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:42:47,244 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:42:49,247 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:42:52,057 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:42:52,125 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:42:52,126 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:42:52,126 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:42:52,132 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:42:52,251 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:42:53,253 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:42:54,612 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 15:42:56,323 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:42:56,324 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:42:56,324 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:42:56,329 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:42:57,255 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:42:57,256 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:42:57,330 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:42:59,079 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:42:59,079 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:42:59,081 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:43:01,326 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:43:01,326 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:43:01,327 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:43:01,332 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:43:02,258 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:43:02,369 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:43:03,261 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:43:05,890 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:43:05,891 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:43:05,891 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:43:05,897 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:43:06,263 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:43:07,266 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:43:07,898 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:43:09,528 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:43:09,528 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:43:09,529 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:43:09,535 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:43:10,267 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:43:11,270 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:43:13,535 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:43:14,079 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:43:14,079 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:43:14,081 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:43:14,738 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:43:14,738 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:43:14,738 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:43:14,743 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:43:15,272 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:43:17,276 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:43:18,781 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:43:21,226 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:43:21,227 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:43:21,227 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:43:21,232 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:43:21,278 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:43:23,281 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:43:24,233 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:43:24,615 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 15:43:26,140 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:43:26,141 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:43:26,141 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:43:26,146 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:43:26,283 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:43:27,286 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:43:29,079 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:43:29,079 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:43:29,081 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:43:29,351 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:43:32,599 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:43:32,600 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:43:32,600 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:43:32,605 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:43:33,291 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:43:34,645 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:43:35,294 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:43:39,645 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:43:41,627 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:43:41,627 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:43:41,627 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:43:41,634 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:43:42,298 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:43:43,301 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:43:44,079 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:43:44,079 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:43:44,081 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:43:45,327 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:43:50,327 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:43:54,617 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 15:43:55,618 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:43:56,190 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:43:56,190 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:43:56,190 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:43:56,196 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:43:56,310 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:43:57,313 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:43:59,079 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:43:59,079 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:43:59,081 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:43:59,935 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:43:59,935 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:43:59,936 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:43:59,941 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:44:00,315 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:44:00,977 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:44:01,317 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:44:03,988 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:44:03,989 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:44:03,989 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:44:03,994 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:44:04,319 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:44:05,320 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:44:05,995 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:44:09,119 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:44:09,120 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:44:09,120 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:44:09,125 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:44:09,322 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:44:11,126 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:44:11,326 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:44:14,079 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:44:14,080 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:44:14,081 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:44:16,291 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:44:16,302 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:44:16,302 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:44:16,303 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:44:16,307 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:44:16,329 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:44:17,331 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:44:21,345 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:44:22,291 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:44:22,292 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:44:22,292 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:44:22,297 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:44:22,334 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:44:23,337 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:44:24,619 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 15:44:25,989 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:44:25,990 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:44:25,990 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:44:25,995 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:44:26,338 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:44:26,995 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:44:27,341 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:44:29,079 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:44:29,080 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:44:29,081 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:44:29,736 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:44:29,736 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:44:29,736 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:44:29,741 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:44:30,343 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:44:31,346 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:44:32,781 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:44:33,481 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:44:33,481 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:44:33,482 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:44:33,487 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:44:34,349 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:44:35,351 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:44:37,508 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:44:37,509 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:44:37,509 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:44:37,514 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:44:38,353 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:44:38,514 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:44:39,355 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:44:42,431 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:44:42,432 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:44:42,432 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:44:42,437 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:44:43,360 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:44:43,360 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:44:44,079 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:44:44,080 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:44:44,081 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:44:44,332 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:44:46,876 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:44:46,877 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:44:46,877 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:44:46,882 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:44:47,363 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:44:49,366 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:44:49,921 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:44:51,061 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:44:51,061 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:44:51,062 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:44:51,067 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:44:51,368 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:44:53,371 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:44:54,624 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 15:44:55,626 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:44:56,062 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:44:56,062 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:44:56,063 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:44:56,067 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:44:56,373 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:44:57,376 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:44:59,079 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:44:59,080 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:44:59,081 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:45:01,342 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:45:02,579 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:45:02,580 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:45:02,580 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:45:02,585 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:45:03,381 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:45:03,382 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:45:06,625 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:45:07,007 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:45:07,008 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:45:07,008 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:45:07,014 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:45:07,384 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:45:09,387 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:45:10,600 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:45:10,601 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:45:10,601 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:45:10,606 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:45:11,391 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:45:11,391 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:45:12,607 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:45:14,080 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:45:14,080 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:45:14,081 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:45:15,373 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:45:15,373 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:45:15,374 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:45:15,379 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:45:15,394 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:45:17,397 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:45:18,417 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:45:23,418 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:45:24,623 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 15:45:24,673 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:45:24,674 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:45:24,674 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:45:24,679 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:45:25,404 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:45:25,404 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:45:28,680 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:45:29,080 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:45:29,080 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:45:29,081 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:45:29,102 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:45:29,362 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:45:29,362 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:45:29,368 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:45:29,407 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:45:31,410 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:45:34,368 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:45:34,969 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:45:34,969 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:45:34,970 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:45:34,975 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:45:35,414 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:45:37,417 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:45:39,975 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:45:40,174 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:45:40,175 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:45:40,175 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:45:40,180 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:45:40,419 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:45:41,422 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:45:44,080 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:45:44,080 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:45:44,082 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:45:44,149 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:45:44,318 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:45:44,318 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:45:44,323 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:45:44,423 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:45:45,324 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:45:45,426 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:45:48,246 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:45:48,247 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:45:48,247 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:45:48,252 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:45:48,428 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:45:49,428 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:45:51,252 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:45:53,062 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:45:53,063 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:45:53,063 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:45:53,068 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:45:53,432 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:45:54,628 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 15:45:55,435 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:45:56,630 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:45:58,072 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:45:58,073 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:45:58,073 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:45:58,079 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:45:58,437 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:45:59,080 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:45:59,080 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:45:59,082 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:45:59,439 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:46:02,320 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:46:04,944 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:46:04,945 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:46:04,945 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:46:04,950 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:46:05,443 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:46:07,446 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:46:07,989 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:46:10,031 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:46:10,032 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:46:10,032 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:46:10,037 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:46:10,449 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:46:11,451 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:46:13,038 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:46:13,951 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:46:13,952 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:46:13,952 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:46:13,957 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:46:14,080 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:46:14,080 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:46:14,082 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:46:14,453 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:46:15,456 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:46:17,317 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:46:17,318 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:46:17,318 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:46:17,323 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:46:17,457 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:46:18,361 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:46:19,460 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:46:21,415 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:46:21,416 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:46:21,416 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:46:21,421 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:46:21,461 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:46:23,422 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:46:23,465 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:46:24,627 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 15:46:26,432 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:46:26,433 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:46:26,433 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:46:26,439 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:46:26,466 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:46:27,469 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:46:28,439 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:46:29,080 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:46:29,080 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:46:29,082 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:46:29,969 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:46:29,970 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:46:29,970 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:46:29,975 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:46:30,471 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:46:31,474 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:46:34,013 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:46:34,674 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:46:34,675 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:46:34,675 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:46:34,681 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:46:35,479 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:46:35,480 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:46:39,682 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:46:41,141 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:46:41,142 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:46:41,142 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:46:41,147 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:46:41,483 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:46:43,486 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:46:44,080 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:46:44,081 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:46:44,082 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:46:45,341 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:46:46,124 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:46:46,125 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:46:46,125 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:46:46,130 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:46:46,488 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:46:47,490 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:46:49,812 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:46:49,813 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:46:49,813 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:46:49,818 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:46:50,493 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:46:50,818 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:46:51,494 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:46:54,629 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 15:46:55,254 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:46:55,255 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:46:55,255 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:46:55,260 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:46:55,496 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:46:56,260 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:46:57,499 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:46:59,031 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:46:59,031 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:46:59,032 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:46:59,037 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:46:59,080 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:46:59,081 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:46:59,082 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:46:59,501 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:47:01,331 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:47:01,504 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:47:03,560 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:47:03,561 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:47:03,561 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:47:03,566 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:47:04,506 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:47:05,508 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:47:06,605 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:47:07,534 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:47:07,534 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:47:07,534 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:47:07,539 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:47:08,510 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:47:09,513 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:47:11,437 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:47:11,437 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:47:11,437 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:47:11,443 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:47:11,514 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:47:12,443 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:47:13,517 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:47:14,080 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:47:14,081 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:47:14,082 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:47:15,580 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:47:15,581 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:47:15,581 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:47:15,586 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:47:16,519 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:47:17,521 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:47:17,625 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:47:20,674 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:47:20,675 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:47:20,675 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:47:20,680 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:47:21,526 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:47:21,526 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:47:22,681 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:47:24,631 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 15:47:24,919 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:47:24,920 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:47:24,920 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:47:24,925 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:47:25,528 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:47:27,531 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:47:27,925 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:47:28,315 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:47:28,316 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:47:28,316 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:47:28,321 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:47:28,532 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:47:29,080 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:47:29,081 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:47:29,082 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:47:29,535 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:47:33,328 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:47:33,972 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:47:33,973 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:47:33,973 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:47:33,978 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:47:34,540 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:47:35,543 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:47:39,018 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:47:40,082 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:47:40,082 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:47:40,083 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:47:40,088 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:47:40,546 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:47:41,548 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:47:44,080 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:47:44,081 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:47:44,082 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:47:44,329 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:47:46,507 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:47:46,507 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:47:46,508 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:47:46,512 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:47:46,551 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:47:47,554 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:47:49,549 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:47:54,549 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:47:54,633 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 15:47:54,904 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:47:54,905 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:47:54,905 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:47:54,910 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:47:55,559 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:47:57,562 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:47:58,436 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:47:58,437 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:47:58,437 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:47:58,442 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:47:58,563 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:47:59,081 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:47:59,081 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:47:59,082 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:47:59,566 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:48:00,376 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:48:02,612 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:48:02,612 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:48:02,612 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:48:02,617 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:48:03,570 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:48:03,570 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:48:05,657 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:48:05,879 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:48:05,880 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:48:05,880 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:48:05,884 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:48:06,572 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:48:07,575 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:48:10,686 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:48:10,686 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:48:10,687 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:48:10,687 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:48:10,692 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:48:11,579 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:48:11,579 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:48:14,081 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:48:14,081 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:48:14,083 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:48:14,527 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:48:14,528 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:48:14,528 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:48:14,533 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:48:14,581 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:48:15,584 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:48:16,573 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:48:18,470 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:48:18,471 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:48:18,471 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:48:18,477 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:48:18,585 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:48:19,588 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:48:22,478 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:48:23,000 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:48:23,001 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:48:23,001 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:48:23,006 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:48:23,590 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:48:24,638 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 15:48:25,593 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:48:26,974 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:48:26,974 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:48:26,974 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:48:26,979 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:48:27,595 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:48:27,980 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:48:29,081 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:48:29,081 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:48:29,083 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:48:29,598 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:48:32,251 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:48:32,252 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:48:32,252 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:48:32,257 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:48:32,601 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:48:33,297 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:48:33,604 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:48:37,521 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:48:37,522 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:48:37,522 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:48:37,527 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:48:37,606 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:48:38,528 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:48:39,609 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:48:41,301 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:48:41,302 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:48:41,302 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:48:41,307 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:48:41,610 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:48:43,613 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:48:44,081 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:48:44,081 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:48:44,083 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:48:44,317 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:48:45,341 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:48:45,342 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:48:45,342 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:48:45,347 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:48:45,614 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:48:47,618 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:48:49,385 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:48:50,123 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:48:50,123 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:48:50,124 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:48:50,128 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:48:50,621 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:48:51,623 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:48:54,637 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 15:48:54,639 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:48:54,777 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:48:54,778 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:48:54,778 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:48:54,783 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:48:55,627 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:48:55,628 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:48:58,688 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:48:58,689 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:48:58,689 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:48:58,694 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:48:59,081 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:48:59,081 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:48:59,083 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:48:59,632 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:48:59,632 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:49:00,338 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:49:02,431 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:49:02,432 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:49:02,432 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:49:02,437 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:49:02,633 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:49:03,636 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:49:05,473 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:49:07,793 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:49:07,794 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:49:07,794 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:49:07,799 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:49:08,640 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:49:09,642 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:49:10,800 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:49:12,324 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:49:12,324 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:49:12,325 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:49:12,330 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:49:12,644 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:49:13,647 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:49:14,081 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:49:14,081 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:49:14,083 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:49:16,336 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:49:18,663 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:49:18,663 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:49:18,664 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:49:18,669 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:49:19,652 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:49:19,652 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:49:21,705 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:49:23,085 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:49:23,086 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:49:23,086 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:49:23,092 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:49:23,654 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:49:24,643 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 15:49:25,657 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:49:27,644 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:49:28,961 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:49:28,962 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:49:28,962 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:49:28,967 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:49:29,081 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:49:29,081 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:49:29,084 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:49:29,660 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:49:31,663 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:49:33,384 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:49:35,717 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:49:35,718 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:49:35,718 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:49:35,723 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:49:36,667 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:49:37,669 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:49:38,761 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:49:42,942 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:49:42,942 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:49:42,943 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:49:42,948 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:49:43,675 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:49:43,675 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:49:43,948 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:49:44,081 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:49:44,081 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:49:44,083 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:49:49,358 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:49:51,206 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:49:51,207 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:49:51,207 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:49:51,212 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:49:51,681 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:49:53,684 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:49:54,642 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 15:49:54,643 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:49:55,467 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:49:55,467 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:49:55,467 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:49:55,473 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:49:55,685 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:49:57,688 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:49:59,081 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:49:59,082 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:49:59,083 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:50:00,299 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:50:01,821 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:50:01,822 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:50:01,822 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:50:01,827 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:50:02,691 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:50:03,693 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:50:05,865 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:50:08,189 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:50:08,189 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:50:08,190 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:50:08,194 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:50:08,699 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:50:09,701 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:50:11,195 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:50:12,914 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:50:12,914 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:50:12,915 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:50:12,920 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:50:13,706 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:50:13,706 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:50:14,081 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:50:14,082 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:50:14,084 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:50:16,344 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:50:17,804 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:50:17,805 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:50:17,805 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:50:17,810 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:50:18,709 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:50:19,712 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:50:21,099 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:50:21,100 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:50:21,100 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:50:21,105 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:50:21,713 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:50:22,106 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:50:23,716 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:50:24,647 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 15:50:25,663 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:50:25,664 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:50:25,664 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:50:25,669 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:50:25,717 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:50:27,669 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:50:27,720 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:50:29,081 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:50:29,082 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:50:29,084 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:50:29,452 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:50:29,452 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:50:29,452 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:50:29,457 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:50:29,721 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:50:31,725 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:50:33,240 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:50:33,241 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:50:33,241 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:50:33,241 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:50:33,246 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:50:33,727 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:50:35,730 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:50:37,108 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:50:37,109 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:50:37,109 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:50:37,114 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:50:37,731 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:50:39,115 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:50:39,734 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:50:41,226 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:50:41,226 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:50:41,227 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:50:41,232 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:50:41,736 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:50:43,739 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:50:44,082 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:50:44,082 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:50:44,084 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:50:44,360 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:50:46,688 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:50:46,689 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:50:46,689 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:50:46,694 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:50:46,741 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:50:47,743 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:50:49,733 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:50:50,691 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:50:50,692 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:50:50,692 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:50:50,697 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:50:50,746 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:50:51,749 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:50:54,646 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 15:50:55,248 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:50:55,248 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:50:55,249 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:50:55,249 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:50:55,253 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:50:55,751 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:50:57,755 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:50:59,082 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:50:59,082 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:50:59,084 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:51:00,334 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:51:01,132 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:51:01,133 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:51:01,133 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:51:01,138 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:51:01,757 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:51:03,760 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:51:05,704 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:51:05,705 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:51:05,705 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:51:05,705 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:51:05,710 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:51:05,762 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:51:07,766 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:51:10,629 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:51:10,630 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:51:10,630 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:51:10,635 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:51:10,768 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:51:11,636 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:51:11,771 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:51:14,082 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:51:14,082 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:51:14,084 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:51:15,461 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:51:15,462 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:51:15,462 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:51:15,467 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:51:15,773 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:51:17,505 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:51:17,776 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:51:19,844 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:51:19,845 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:51:19,845 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:51:19,850 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:51:20,778 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:51:21,780 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:51:22,851 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:51:23,784 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:51:23,785 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:51:23,785 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:51:23,790 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:51:24,648 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 15:51:24,783 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:51:25,785 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:51:28,649 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:51:29,082 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:51:29,082 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:51:29,084 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:51:31,918 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:51:31,919 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:51:31,919 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:51:31,924 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:51:32,791 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:51:33,794 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:51:33,961 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:51:37,174 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:51:37,175 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:51:37,175 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:51:37,180 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:51:37,796 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:51:39,181 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:51:39,799 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:51:40,972 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:51:40,972 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:51:40,973 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:51:40,977 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:51:41,803 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:51:41,803 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:51:44,082 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:51:44,082 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:51:44,084 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:51:44,379 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:51:45,725 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:51:45,726 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:51:45,726 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:51:45,731 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:51:45,805 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:51:47,808 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:51:49,769 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:51:49,973 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:51:49,974 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:51:49,974 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:51:49,979 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:51:50,811 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:51:51,814 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:51:54,653 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 15:51:55,654 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:51:56,271 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:51:56,272 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:51:56,273 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:51:56,278 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:51:56,817 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:51:57,820 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:51:59,082 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:51:59,083 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:51:59,084 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:51:59,961 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:51:59,962 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:51:59,962 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:51:59,967 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:52:00,821 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:52:01,005 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:52:01,824 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:52:03,320 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:52:03,321 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:52:03,321 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:52:03,327 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:52:03,825 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:52:05,829 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:52:06,327 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:52:11,328 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:52:11,838 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:52:11,839 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:52:11,839 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:52:11,844 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:52:12,833 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:52:13,835 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:52:14,082 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:52:14,083 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:52:14,084 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:52:16,342 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:52:16,646 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:52:16,646 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:52:16,647 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:52:16,652 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:52:16,837 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:52:17,840 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:52:21,689 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:52:22,631 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:52:22,632 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:52:22,632 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:52:22,638 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:52:22,843 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:52:23,846 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:52:24,652 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 15:52:26,493 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:52:26,494 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:52:26,494 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:52:26,501 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:52:26,847 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:52:27,501 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:52:27,850 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:52:29,082 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:52:29,083 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:52:29,084 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:52:31,003 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:52:31,004 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:52:31,004 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:52:31,010 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:52:31,855 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:52:31,855 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:52:33,049 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:52:37,460 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:52:37,461 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:52:37,461 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:52:37,466 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:52:37,860 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:52:38,467 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:52:39,863 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:52:42,553 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:52:42,554 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:52:42,554 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:52:42,559 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:52:42,865 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:52:43,560 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:52:43,867 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:52:44,082 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:52:44,083 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:52:44,084 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:52:47,290 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:52:47,291 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:52:47,291 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:52:47,296 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:52:47,870 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:52:49,333 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:52:49,873 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:52:54,333 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:52:54,659 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 15:52:55,725 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:52:55,726 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:52:55,726 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:52:55,731 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:52:55,878 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:52:57,881 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:52:59,082 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:52:59,083 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:52:59,085 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:52:59,400 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:52:59,593 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:52:59,594 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:52:59,594 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:52:59,599 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:52:59,882 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:53:01,886 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:53:03,137 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:53:03,138 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:53:03,138 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:53:03,143 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:53:03,889 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:53:03,889 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:53:05,144 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:53:07,724 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:53:07,725 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:53:07,725 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:53:07,730 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:53:07,892 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:53:09,895 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:53:10,731 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:53:11,498 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:53:11,499 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:53:11,499 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:53:11,504 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:53:11,896 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:53:13,899 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:53:14,083 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:53:14,083 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:53:14,085 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:53:16,234 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:53:16,234 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:53:16,235 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:53:16,235 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:53:16,239 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:53:16,901 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:53:17,903 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:53:21,277 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:53:24,656 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 15:53:24,866 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:53:24,867 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:53:24,867 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:53:24,872 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:53:24,907 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:53:25,911 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:53:26,872 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:53:28,654 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:53:28,654 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:53:28,655 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:53:28,661 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:53:28,913 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:53:29,083 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:53:29,083 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:53:29,085 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:53:29,913 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:53:32,324 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:53:32,690 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:53:32,690 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:53:32,691 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:53:32,696 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:53:32,916 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:53:33,919 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:53:37,733 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:53:42,518 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:53:42,519 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:53:42,519 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:53:42,524 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:53:42,924 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:53:43,525 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:53:43,927 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:53:44,083 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:53:44,083 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:53:44,085 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:53:47,146 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:53:47,147 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:53:47,147 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:53:47,152 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:53:47,931 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:53:47,932 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:53:49,189 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:53:51,147 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:53:51,148 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:53:51,148 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:53:51,153 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:53:51,937 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:53:51,937 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:53:54,661 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 15:53:54,662 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:53:56,377 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:53:56,378 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:53:56,378 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:53:56,383 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:53:56,940 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:53:57,943 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:53:59,083 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:53:59,083 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:53:59,085 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:54:00,296 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:54:01,610 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:54:01,611 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:54:01,611 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:54:01,616 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:54:01,945 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:54:03,948 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:54:05,653 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:54:07,934 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:54:07,934 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:54:07,935 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:54:07,939 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:54:07,951 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:54:09,954 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:54:10,940 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:54:13,101 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:54:13,102 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:54:13,102 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:54:13,107 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:54:13,958 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:54:13,958 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:54:14,083 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:54:14,083 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:54:14,085 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:54:16,342 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:54:17,633 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:54:17,633 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:54:17,634 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:54:17,639 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:54:17,960 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:54:19,964 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:54:21,677 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:54:22,555 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:54:22,555 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:54:22,556 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:54:22,561 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:54:22,965 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:54:23,968 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:54:24,660 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 15:54:27,661 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:54:29,083 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:54:29,083 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:54:29,085 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:54:31,097 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:54:31,098 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:54:31,098 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:54:31,103 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:54:31,975 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:54:31,975 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:54:33,141 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:54:35,294 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:54:35,295 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:54:35,295 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:54:35,300 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:54:35,980 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:54:35,980 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:54:38,300 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:54:40,357 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:54:40,357 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:54:40,358 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:54:40,362 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:54:40,983 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:54:41,986 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:54:43,363 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:54:44,083 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:54:44,084 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:54:44,085 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:54:45,347 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:54:45,348 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:54:45,348 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:54:45,353 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:54:45,988 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:54:47,991 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:54:48,389 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:54:50,060 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:54:50,061 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:54:50,061 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:54:50,066 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:54:50,994 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:54:51,997 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:54:54,067 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:54:54,662 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 15:54:59,083 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:54:59,084 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:54:59,085 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:54:59,350 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:54:59,825 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:54:59,825 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:54:59,826 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:54:59,831 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:55:00,001 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:55:02,004 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:55:04,869 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:55:07,553 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:55:07,554 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:55:07,554 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:55:07,559 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:55:08,009 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:55:10,010 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:55:10,559 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:55:12,988 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:55:12,989 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:55:12,989 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:55:12,994 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:55:13,012 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:55:14,015 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:55:14,083 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:55:14,084 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:55:14,085 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:55:16,366 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:55:18,291 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:55:18,292 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:55:18,292 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:55:18,297 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:55:19,018 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:55:20,020 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:55:22,337 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:55:23,578 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:55:23,579 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:55:23,579 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:55:23,584 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:55:24,023 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:55:24,664 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 15:55:26,026 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:55:27,666 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:55:29,083 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:55:29,084 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:55:29,085 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:55:29,544 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:55:29,545 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:55:29,545 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:55:29,550 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:55:30,029 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:55:32,032 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:55:33,147 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:55:33,148 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:55:33,148 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:55:33,148 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:55:33,153 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:55:34,036 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:55:34,036 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:55:37,196 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:55:37,197 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:55:37,197 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:55:37,202 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:55:38,041 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:55:38,041 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:55:38,202 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:55:41,602 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:55:41,602 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:55:41,603 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:55:41,608 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:55:42,043 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:55:43,608 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:55:44,046 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:55:44,083 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:55:44,084 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:55:44,085 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:55:45,224 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:55:45,225 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:55:45,225 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:55:45,230 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:55:46,049 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:55:46,049 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:55:49,269 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:55:49,712 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:55:49,713 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:55:49,713 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:55:49,721 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:55:50,051 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:55:52,056 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:55:54,666 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 15:55:54,667 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:55:58,458 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:55:58,458 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:55:58,458 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:55:58,463 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:55:59,060 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:55:59,084 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:55:59,084 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:55:59,086 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:56:00,060 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:56:00,343 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:56:03,203 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:56:03,204 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:56:03,204 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:56:03,209 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:56:04,065 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:56:04,065 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:56:06,245 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:56:09,497 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:56:09,498 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:56:09,498 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:56:09,503 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:56:10,069 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:56:11,503 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:56:12,072 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:56:14,084 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:56:14,084 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:56:14,086 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:56:14,307 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:56:14,311 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:56:14,312 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:56:14,317 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:56:15,074 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:56:16,076 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:56:17,317 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:56:19,144 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:56:19,144 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:56:19,145 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:56:19,149 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:56:20,080 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:56:20,081 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:56:23,168 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:56:23,169 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:56:23,169 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:56:23,169 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:56:23,174 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:56:24,085 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:56:24,085 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:56:24,671 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 15:56:26,856 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:56:26,857 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:56:26,857 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:56:26,862 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:56:27,087 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:56:28,089 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:56:28,863 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:56:29,084 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:56:29,084 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:56:29,086 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:56:32,466 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:56:32,467 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:56:32,467 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:56:32,472 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:56:33,094 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:56:34,094 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:56:34,509 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:56:37,854 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:56:37,855 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:56:37,855 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:56:37,860 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:56:38,097 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:56:39,861 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:56:40,100 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:56:42,637 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:56:42,638 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:56:42,638 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:56:42,643 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:56:43,102 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:56:44,084 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:56:44,084 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:56:44,086 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:56:44,104 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:56:45,303 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:56:47,881 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:56:47,882 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:56:47,882 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:56:47,887 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:56:48,107 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:56:50,110 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:56:50,925 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:56:52,467 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:56:52,467 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:56:52,468 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:56:52,473 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:56:53,113 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:56:54,115 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:56:54,670 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 15:56:56,339 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:56:56,340 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:56:56,340 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:56:56,340 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:56:56,346 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:56:57,117 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:56:58,120 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:56:59,084 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:56:59,084 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:56:59,086 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:56:59,934 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:56:59,935 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:56:59,935 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:56:59,940 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:57:00,121 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:57:01,977 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:57:02,124 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:57:03,255 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:57:03,256 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:57:03,257 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:57:03,261 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:57:04,127 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:57:04,127 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:57:07,262 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:57:07,393 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:57:07,394 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:57:07,394 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:57:07,400 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:57:08,132 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:57:08,132 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:57:12,401 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:57:13,269 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:57:13,270 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:57:13,270 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:57:13,276 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:57:14,084 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:57:14,084 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:57:14,086 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:57:14,137 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:57:14,138 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:57:17,739 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:57:17,739 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:57:17,740 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:57:17,740 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:57:17,745 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:57:18,140 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:57:20,143 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:57:21,730 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:57:21,731 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:57:21,731 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:57:21,736 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:57:22,144 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:57:23,737 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:57:24,147 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:57:24,675 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 15:57:26,020 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:57:26,021 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:57:26,021 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:57:26,026 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:57:26,148 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:57:28,151 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:57:29,027 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:57:29,084 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:57:29,084 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:57:29,086 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:57:30,792 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:57:30,793 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:57:30,793 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:57:30,798 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:57:31,153 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:57:32,156 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:57:34,622 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:57:34,623 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:57:34,623 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:57:34,623 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:57:34,628 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:57:35,159 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:57:36,161 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:57:38,467 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:57:38,468 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:57:38,469 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:57:38,473 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:57:39,163 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:57:40,165 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:57:40,474 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:57:42,217 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:57:42,217 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:57:42,218 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:57:42,223 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:57:43,167 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:57:44,084 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:57:44,085 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:57:44,087 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:57:44,170 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:57:46,321 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:57:47,215 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:57:47,216 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:57:47,216 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:57:47,221 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:57:48,174 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:57:48,174 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:57:51,610 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:57:51,611 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:57:51,611 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:57:51,611 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:57:51,616 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:57:52,178 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:57:54,181 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:57:54,674 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 15:57:56,676 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:57:59,084 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:57:59,085 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:57:59,086 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:57:59,158 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:57:59,322 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:57:59,322 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:57:59,326 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:58:00,186 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:58:00,186 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:58:02,327 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:58:02,905 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:58:02,906 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:58:02,907 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:58:02,912 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:58:03,188 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:58:04,190 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:58:06,792 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:58:06,793 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:58:06,793 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:58:06,798 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:58:07,193 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:58:07,798 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:58:08,195 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:58:11,470 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:58:11,470 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:58:11,471 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:58:11,476 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:58:12,198 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:58:12,198 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:58:13,476 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:58:14,084 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:58:14,085 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:58:14,086 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:58:14,917 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:58:14,918 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:58:14,918 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:58:14,923 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:58:15,199 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:58:16,202 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:58:18,961 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:58:19,134 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:58:19,135 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:58:19,135 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:58:19,140 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:58:19,203 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:58:20,206 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:58:24,141 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:58:24,676 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 15:58:24,855 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:58:24,856 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:58:24,856 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:58:24,861 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:58:25,209 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:58:26,211 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:58:28,848 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:58:28,848 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:58:28,849 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:58:28,854 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:58:29,085 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:58:29,085 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:58:29,087 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:58:29,214 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:58:29,370 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:58:30,216 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:58:32,922 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:58:32,923 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:58:32,923 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:58:32,928 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:58:33,225 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:58:34,227 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:58:34,965 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:58:37,449 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:58:37,453 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:58:37,453 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:58:37,459 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:58:38,232 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:58:38,232 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:58:40,460 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:58:41,145 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:58:41,146 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:58:41,146 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:58:41,151 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:58:41,233 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:58:42,236 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:58:44,085 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:58:44,085 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:58:44,087 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:58:45,936 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:58:45,937 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:58:45,937 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:58:45,937 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:58:45,942 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:58:46,238 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:58:48,241 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:58:50,328 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:58:50,329 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:58:50,330 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:58:50,334 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:58:51,244 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:58:51,335 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:58:52,247 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:58:54,678 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 15:58:56,680 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:58:56,824 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:58:56,825 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:58:56,825 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:58:56,830 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:58:57,249 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:58:58,252 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:58:59,085 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:58:59,085 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:58:59,087 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:59:00,489 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:59:00,490 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:59:00,490 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:59:00,495 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:59:01,254 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:59:02,256 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:59:02,533 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:59:06,362 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:59:06,363 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:59:06,363 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:59:06,367 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:59:07,259 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:59:08,262 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:59:08,368 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:59:10,334 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:59:10,336 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:59:10,336 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:59:10,341 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:59:11,264 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:59:12,266 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:59:13,940 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:59:13,941 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:59:13,941 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:59:13,942 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:59:13,946 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:59:14,085 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:59:14,085 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:59:14,087 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:59:14,267 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:59:16,270 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:59:18,100 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:59:18,100 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:59:18,101 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:59:18,105 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:59:18,271 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:59:19,145 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:59:20,275 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:59:23,080 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:59:23,081 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:59:23,081 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:59:23,086 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:59:23,276 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:59:24,279 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:59:24,680 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 15:59:24,682 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:59:26,680 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:59:26,681 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:59:26,682 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:59:26,687 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:59:27,281 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:59:28,283 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:59:29,085 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:59:29,085 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:59:29,087 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:59:30,354 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:59:32,590 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:59:32,590 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:59:32,591 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:59:32,596 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:59:33,288 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:59:34,290 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:59:35,633 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:59:37,039 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:59:37,040 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:59:37,040 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:59:37,045 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:59:37,292 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:59:38,295 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:59:40,651 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:59:40,651 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:59:40,652 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:59:40,652 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:59:40,657 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:59:41,297 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:59:42,299 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:59:44,085 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:59:44,085 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:59:44,087 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:59:45,740 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:59:45,741 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:59:45,741 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:59:45,741 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:59:45,746 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:59:46,302 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:59:48,305 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:59:50,145 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:59:50,146 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:59:50,146 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:59:50,151 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:59:50,306 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:59:51,152 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:59:52,310 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:59:53,677 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:59:53,680 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:59:53,681 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:59:53,686 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:59:54,311 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:59:54,683 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 15:59:56,314 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:59:56,684 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:59:57,550 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:59:57,551 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 15:59:57,551 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:59:57,556 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:59:58,318 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 15:59:58,318 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 15:59:59,085 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:59:59,086 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:59:59,087 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:00:02,443 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:00:03,457 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:00:03,457 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:00:03,458 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:00:03,463 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:00:04,323 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:00:04,323 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:00:07,501 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:00:10,567 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:00:10,568 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:00:10,569 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:00:10,574 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:00:11,328 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:00:12,330 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:00:12,575 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:00:14,085 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:00:14,086 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:00:14,087 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:00:15,031 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:00:15,031 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:00:15,032 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:00:15,037 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:00:15,332 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:00:16,335 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:00:18,073 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:00:19,440 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:00:19,441 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:00:19,441 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:00:19,446 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:00:20,340 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:00:20,340 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:00:23,447 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:00:23,706 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:00:23,707 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:00:23,707 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:00:23,712 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:00:24,342 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:00:24,685 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 16:00:26,345 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:00:27,615 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:00:27,616 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:00:27,616 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:00:27,621 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:00:28,348 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:00:28,349 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:00:28,621 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:00:29,085 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:00:29,086 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:00:29,087 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:00:33,045 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:00:33,045 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:00:33,046 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:00:33,050 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:00:33,353 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:00:34,089 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:00:34,353 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:00:38,263 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:00:38,264 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:00:38,264 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:00:38,269 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:00:38,356 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:00:39,269 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:00:40,359 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:00:43,546 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:00:43,546 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:00:43,547 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:00:43,553 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:00:44,085 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:00:44,086 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:00:44,087 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:00:44,341 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:00:44,363 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:00:44,363 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:00:49,205 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:00:49,206 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:00:49,206 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:00:49,211 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:00:49,366 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:00:50,249 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:00:50,369 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:00:53,042 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:00:53,043 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:00:53,043 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:00:53,048 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:00:53,372 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:00:54,372 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:00:54,690 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 16:00:55,691 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:00:56,951 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:00:56,952 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:00:56,952 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:00:56,957 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:00:57,374 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:00:58,377 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:00:59,085 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:00:59,086 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:00:59,087 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:01:01,343 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:01:02,883 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:01:02,884 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:01:02,884 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:01:02,889 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:01:03,380 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:01:04,384 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:01:06,603 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:01:06,604 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:01:06,604 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:01:06,604 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:01:06,609 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:01:07,388 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:01:08,392 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:01:10,161 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:01:10,162 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:01:10,162 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:01:10,167 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:01:10,393 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:01:12,168 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:01:12,398 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:01:14,086 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:01:14,086 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:01:14,088 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:01:14,120 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:01:14,303 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:01:14,304 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:01:14,309 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:01:14,399 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:01:16,404 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:01:17,309 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:01:17,663 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:01:17,664 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:01:17,664 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:01:17,669 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:01:18,407 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:01:18,407 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:01:22,670 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:01:24,689 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 16:01:25,207 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:01:25,207 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:01:25,208 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:01:25,213 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:01:25,414 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:01:26,418 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:01:28,213 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:01:29,086 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:01:29,086 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:01:29,088 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:01:30,696 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:01:30,696 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:01:30,697 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:01:30,702 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:01:31,422 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:01:32,425 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:01:33,741 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:01:34,607 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:01:34,608 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:01:34,608 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:01:34,613 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:01:35,429 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:01:36,433 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:01:39,286 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:01:39,287 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:01:39,287 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:01:39,287 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:01:39,292 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:01:39,435 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:01:40,439 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:01:42,542 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:01:42,543 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:01:42,543 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:01:42,548 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:01:43,441 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:01:44,086 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:01:44,086 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:01:44,088 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:01:44,315 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:01:44,443 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:01:48,191 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:01:48,192 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:01:48,192 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:01:48,197 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:01:48,448 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:01:50,237 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:01:50,453 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:01:52,498 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:01:52,499 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:01:52,499 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:01:52,504 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:01:53,457 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:01:54,459 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:01:54,697 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 16:01:55,698 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:01:56,622 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:01:56,622 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:01:56,623 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:01:56,628 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:01:57,461 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:01:58,465 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:01:59,086 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:01:59,086 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:01:59,088 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:02:01,297 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:02:02,674 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:02:02,674 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:02:02,675 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:02:02,680 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:02:03,468 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:02:04,472 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:02:06,717 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:02:08,788 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:02:08,789 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:02:08,789 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:02:08,794 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:02:09,479 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:02:10,483 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:02:11,795 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:02:13,150 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:02:13,151 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:02:13,151 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:02:13,156 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:02:13,485 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:02:14,086 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:02:14,086 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:02:14,088 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:02:14,489 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:02:16,887 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:02:16,888 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:02:16,888 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:02:16,889 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:02:16,893 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:02:17,491 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:02:18,493 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:02:20,840 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:02:20,841 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:02:20,841 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:02:20,847 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:02:21,497 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:02:22,501 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:02:22,847 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:02:24,414 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:02:24,415 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:02:24,415 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:02:24,420 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:02:24,502 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:02:24,693 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 16:02:26,507 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:02:28,694 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:02:29,086 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:02:29,086 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:02:29,088 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:02:29,391 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:02:29,392 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:02:29,392 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:02:29,397 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:02:29,509 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:02:30,513 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:02:32,680 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:02:32,681 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:02:32,681 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:02:32,686 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:02:33,516 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:02:34,519 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:02:34,687 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:02:38,556 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:02:38,556 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:02:38,557 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:02:38,561 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:02:39,523 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:02:40,528 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:02:40,562 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:02:42,281 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:02:42,282 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:02:42,282 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:02:42,287 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:02:42,529 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:02:44,086 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:02:44,087 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:02:44,088 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:02:44,533 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:02:46,336 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:02:46,370 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:02:46,370 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:02:46,370 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:02:46,376 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:02:46,535 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:02:48,540 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:02:50,173 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:02:50,174 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:02:50,174 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:02:50,179 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:02:50,541 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:02:52,180 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:02:52,545 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:02:54,321 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:02:54,321 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:02:54,322 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:02:54,327 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:02:54,547 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:02:54,695 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 16:02:56,551 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:02:57,696 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:02:58,790 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:02:58,791 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:02:58,791 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:02:58,796 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:02:59,086 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:02:59,087 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:02:59,088 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:02:59,553 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:03:00,557 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:03:03,180 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:03:03,181 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:03:03,181 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:03:03,181 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:03:03,186 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:03:03,560 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:03:04,564 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:03:07,163 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:03:07,164 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:03:07,165 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:03:07,170 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:03:07,566 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:03:08,568 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:03:09,170 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:03:12,264 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:03:12,264 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:03:12,265 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:03:12,270 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:03:12,573 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:03:14,086 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:03:14,087 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:03:14,088 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:03:14,336 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:03:14,578 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:03:17,580 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:03:17,581 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:03:17,581 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:03:17,586 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:03:18,584 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:03:18,585 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:03:19,625 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:03:21,084 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:03:21,085 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:03:21,086 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:03:21,091 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:03:21,586 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:03:22,590 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:03:24,697 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 16:03:24,698 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:03:25,702 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:03:25,702 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:03:25,703 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:03:25,708 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:03:26,596 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:03:26,596 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:03:29,087 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:03:29,087 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:03:29,089 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:03:30,370 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:03:34,125 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:03:34,125 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:03:34,126 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:03:34,131 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:03:34,603 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:03:36,169 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:03:36,608 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:03:39,756 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:03:39,757 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:03:39,758 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:03:39,762 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:03:40,614 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:03:40,614 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:03:41,763 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:03:44,057 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:03:44,058 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:03:44,058 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:03:44,063 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:03:44,087 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:03:44,087 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:03:44,089 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:03:44,620 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:03:46,625 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:03:47,376 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:03:48,180 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:03:48,180 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:03:48,181 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:03:48,185 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:03:48,626 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:03:50,631 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:03:52,851 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:03:52,852 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:03:52,852 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:03:52,853 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:03:52,858 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:03:53,634 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:03:54,637 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:03:54,699 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 16:03:57,837 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:03:57,838 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:03:57,838 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:03:57,843 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:03:58,643 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:03:58,643 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:03:58,844 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:03:59,087 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:03:59,087 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:03:59,088 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:04:04,381 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:04:06,998 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:04:06,999 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:04:06,999 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:04:07,004 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:04:07,653 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:04:08,657 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:04:10,041 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:04:12,213 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:04:12,214 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:04:12,214 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:04:12,220 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:04:12,661 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:04:14,087 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:04:14,087 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:04:14,089 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:04:14,666 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:04:15,340 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:04:16,218 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:04:16,219 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:04:16,219 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:04:16,224 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:04:16,667 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:04:18,672 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:04:20,578 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:04:20,579 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:04:20,579 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:04:20,579 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:04:20,585 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:04:20,673 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:04:22,678 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:04:24,705 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 16:04:25,706 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:04:27,256 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:04:27,257 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:04:27,258 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:04:27,263 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:04:27,682 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:04:28,686 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:04:29,087 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:04:29,087 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:04:29,089 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:04:31,308 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:04:33,545 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:04:33,546 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:04:33,546 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:04:33,551 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:04:33,692 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:04:34,695 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:04:36,589 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:04:41,589 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:04:42,025 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:04:42,026 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:04:42,026 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:04:42,032 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:04:42,705 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:04:44,087 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:04:44,087 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:04:44,089 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:04:44,710 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:04:47,352 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:04:50,362 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:04:50,363 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:04:50,363 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:04:50,368 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:04:50,716 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:04:52,405 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:04:52,719 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:04:54,703 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 16:04:54,818 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:04:54,819 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:04:54,819 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:04:54,825 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:04:55,723 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:04:56,727 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:04:57,825 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:04:58,807 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:04:58,808 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:04:58,808 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:04:58,813 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:04:59,087 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:04:59,088 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:04:59,089 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:04:59,729 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:05:00,732 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:05:03,378 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:05:04,021 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:05:04,022 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:05:04,022 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:05:04,027 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:05:04,737 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:05:06,743 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:05:08,007 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:05:08,007 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:05:08,008 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:05:08,013 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:05:08,746 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:05:08,746 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:05:09,013 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:05:14,014 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:05:14,087 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:05:14,088 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:05:14,089 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:05:14,906 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:05:14,907 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:05:14,907 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:05:14,912 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:05:15,752 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:05:16,756 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:05:19,949 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:05:21,045 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:05:21,045 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:05:21,046 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:05:21,051 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:05:21,761 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:05:22,765 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:05:24,708 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 16:05:25,438 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:05:25,439 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:05:25,439 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:05:25,440 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:05:25,445 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:05:25,767 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:05:26,771 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:05:29,087 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:05:29,088 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:05:29,089 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:05:31,358 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:05:32,422 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:05:32,422 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:05:32,423 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:05:32,428 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:05:32,775 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:05:34,780 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:05:36,465 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:05:36,833 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:05:36,834 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:05:36,834 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:05:36,839 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:05:37,782 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:05:38,786 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:05:41,840 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:05:44,087 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:05:44,088 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:05:44,089 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:05:45,173 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:05:45,173 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:05:45,173 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:05:45,178 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:05:45,789 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:05:46,791 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:05:47,217 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:05:50,654 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:05:50,654 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:05:50,655 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:05:50,660 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:05:50,794 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:05:52,660 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:05:52,798 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:05:54,708 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 16:05:56,676 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:05:56,677 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:05:56,677 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:05:56,683 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:05:56,801 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:05:57,683 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:05:58,804 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:05:59,087 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:05:59,088 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:05:59,089 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:06:00,911 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:06:00,911 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:06:00,912 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:06:00,916 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:06:01,806 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:06:02,808 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:06:02,953 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:06:05,491 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:06:05,491 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:06:05,492 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:06:05,496 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:06:05,811 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:06:06,813 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:06:08,497 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:06:13,497 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:06:13,943 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:06:13,944 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:06:13,944 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:06:13,949 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:06:14,087 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:06:14,088 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:06:14,089 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:06:14,820 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:06:14,820 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:06:18,194 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:06:18,195 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:06:18,195 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:06:18,200 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:06:18,822 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:06:19,237 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:06:20,825 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:06:22,249 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:06:22,250 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:06:22,250 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:06:22,255 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:06:22,827 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:06:24,256 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:06:24,710 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 16:06:24,828 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:06:26,429 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:06:26,430 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:06:26,431 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:06:26,435 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:06:26,829 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:06:28,832 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:06:29,088 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:06:29,088 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:06:29,090 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:06:29,407 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:06:34,408 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:06:34,651 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:06:34,652 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:06:34,652 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:06:34,657 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:06:34,838 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:06:36,841 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:06:39,700 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:06:40,560 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:06:40,561 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:06:40,561 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:06:40,566 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:06:40,844 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:06:42,847 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:06:44,088 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:06:44,088 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:06:44,090 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:06:44,669 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:06:44,670 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:06:44,670 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:06:44,675 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:06:44,713 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:06:44,848 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:06:46,852 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:06:49,713 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:06:53,203 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:06:53,204 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:06:53,205 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:06:53,210 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:06:53,857 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:06:54,712 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 16:06:54,859 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:06:55,713 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:06:59,088 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:06:59,088 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:06:59,090 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:07:01,289 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:07:01,496 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:07:01,496 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:07:01,497 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:07:01,503 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:07:01,864 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:07:02,866 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:07:05,927 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:07:05,928 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:07:05,928 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:07:05,933 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:07:06,871 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:07:06,871 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:07:06,934 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:07:11,934 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:07:12,704 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:07:12,705 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:07:12,705 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:07:12,710 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:07:12,875 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:07:14,088 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:07:14,088 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:07:14,090 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:07:14,878 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:07:16,338 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:07:16,338 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:07:16,339 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:07:16,344 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:07:16,879 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:07:17,381 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:07:18,882 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:07:21,082 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:07:21,083 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:07:21,084 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:07:21,089 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:07:21,884 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:07:22,884 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:07:23,089 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:07:24,714 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 16:07:25,255 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:07:25,255 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:07:25,256 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:07:25,261 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:07:25,886 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:07:26,888 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:07:28,262 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:07:29,088 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:07:29,088 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:07:29,090 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:07:32,641 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:07:32,641 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:07:32,642 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:07:32,647 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:07:32,892 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:07:33,685 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:07:34,897 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:07:36,378 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:07:36,378 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:07:36,379 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:07:36,383 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:07:36,898 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:07:38,899 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:07:39,384 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:07:40,554 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:07:40,555 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:07:40,555 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:07:40,560 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:07:40,900 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:07:42,903 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:07:44,088 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:07:44,089 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:07:44,090 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:07:45,349 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:07:49,175 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:07:49,176 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:07:49,176 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:07:49,181 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:07:49,907 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:07:50,908 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:07:51,221 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:07:53,561 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:07:53,561 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:07:53,562 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:07:53,566 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:07:53,911 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:07:54,719 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 16:07:54,913 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:07:56,720 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:07:58,204 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:07:58,205 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:07:58,205 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:07:58,210 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:07:58,917 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:07:58,918 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:07:59,088 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:07:59,089 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:07:59,090 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:08:02,149 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:08:02,150 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:08:02,150 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:08:02,150 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:08:02,155 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:08:02,922 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:08:02,922 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:08:06,120 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:08:06,121 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:08:06,121 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:08:06,125 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:08:06,927 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:08:06,927 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:08:08,126 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:08:10,961 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:08:10,962 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:08:10,962 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:08:10,967 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:08:11,930 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:08:12,932 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:08:13,967 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:08:14,088 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:08:14,089 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:08:14,090 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:08:14,719 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:08:14,720 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:08:14,720 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:08:14,725 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:08:14,934 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:08:16,937 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:08:19,765 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:08:20,146 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:08:20,147 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:08:20,147 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:08:20,152 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:08:20,941 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:08:20,941 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:08:23,769 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:08:23,770 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:08:23,770 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:08:23,775 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:08:23,943 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:08:24,718 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 16:08:24,943 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:08:25,719 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:08:27,832 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:08:27,832 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:08:27,833 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:08:27,838 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:08:27,945 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:08:28,948 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:08:29,088 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:08:29,089 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:08:29,090 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:08:31,131 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:08:31,132 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:08:31,132 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:08:31,132 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:08:31,137 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:08:31,950 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:08:32,952 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:08:35,556 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:08:35,556 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:08:35,557 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:08:35,562 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:08:35,955 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:08:36,562 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:08:36,958 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:08:38,952 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:08:38,953 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:08:38,953 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:08:38,958 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:08:38,959 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:08:40,962 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:08:41,959 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:08:43,189 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:08:43,190 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:08:43,190 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:08:43,195 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:08:43,964 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:08:44,088 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:08:44,089 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:08:44,090 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:08:44,966 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:08:47,363 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:08:48,359 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:08:48,359 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:08:48,359 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:08:48,365 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:08:48,968 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:08:50,972 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:08:52,401 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:08:53,145 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:08:53,146 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:08:53,146 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:08:53,150 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:08:53,975 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:08:54,724 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 16:08:54,977 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:08:57,572 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:08:57,572 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:08:57,573 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:08:57,573 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:08:57,578 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:08:57,979 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:08:58,981 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:08:59,089 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:08:59,089 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:08:59,090 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:09:02,587 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:09:02,588 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:09:02,588 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:09:02,588 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:09:02,593 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:09:02,984 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:09:04,987 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:09:07,633 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:09:08,882 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:09:08,883 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:09:08,883 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:09:08,889 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:09:08,990 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:09:10,993 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:09:12,394 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:09:12,395 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:09:12,395 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:09:12,400 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:09:12,994 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:09:13,400 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:09:14,089 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:09:14,089 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:09:14,090 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:09:14,997 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:09:17,104 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:09:17,104 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:09:17,104 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:09:17,109 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:09:17,998 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:09:19,001 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:09:19,149 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:09:21,954 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:09:21,955 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:09:21,955 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:09:21,960 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:09:22,003 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:09:23,005 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:09:24,722 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 16:09:24,723 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:09:26,032 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:09:26,032 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:09:26,033 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:09:26,037 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:09:27,009 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:09:27,010 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:09:29,089 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:09:29,089 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:09:29,091 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:09:29,767 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:09:29,767 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:09:29,767 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:09:29,768 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:09:29,773 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:09:30,012 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:09:31,014 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:09:33,807 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:09:33,808 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:09:33,808 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:09:33,813 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:09:34,017 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:09:34,814 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:09:35,019 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:09:38,108 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:09:38,109 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:09:38,109 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:09:38,114 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:09:39,024 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:09:39,024 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:09:40,114 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:09:41,979 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:09:41,980 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:09:41,980 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:09:41,986 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:09:42,025 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:09:43,028 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:09:44,089 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:09:44,089 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:09:44,091 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:09:45,359 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:09:47,415 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:09:47,416 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:09:47,416 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:09:47,420 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:09:48,031 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:09:49,034 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:09:50,457 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:09:51,593 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:09:51,593 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:09:51,594 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:09:51,599 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:09:52,038 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:09:53,040 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:09:54,727 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 16:09:55,729 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:09:59,089 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:09:59,089 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:09:59,091 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:09:59,940 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:09:59,941 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:09:59,941 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:09:59,946 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:10:00,044 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:10:00,985 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:10:01,047 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:10:03,915 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:10:03,915 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:10:03,915 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:10:03,920 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:10:04,049 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:10:05,052 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:10:06,921 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:10:07,913 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:10:07,914 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:10:07,914 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:10:07,919 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:10:08,054 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:10:09,057 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:10:12,920 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:10:14,089 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:10:14,089 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:10:14,091 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:10:16,379 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:10:16,380 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:10:16,380 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:10:16,385 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:10:17,061 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:10:18,425 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:10:19,065 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:10:19,876 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:10:19,877 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:10:19,877 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:10:19,883 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:10:20,065 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:10:21,068 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:10:23,883 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:10:24,726 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 16:10:25,551 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:10:25,552 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:10:25,552 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:10:25,557 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:10:26,071 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:10:27,074 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:10:29,089 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:10:29,090 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:10:29,091 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:10:29,385 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:10:30,615 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:10:30,615 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:10:30,616 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:10:30,620 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:10:31,076 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:10:33,080 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:10:34,313 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:10:34,313 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:10:34,314 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:10:34,319 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:10:35,085 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:10:35,085 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:10:35,320 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:10:39,010 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:10:39,011 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:10:39,011 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:10:39,017 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:10:39,087 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:10:41,017 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:10:41,090 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:10:43,693 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:10:43,694 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:10:43,694 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:10:43,699 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:10:44,089 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:10:44,090 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:10:44,091 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:10:44,092 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:10:45,095 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:10:46,298 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:10:48,892 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:10:48,893 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:10:48,893 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:10:48,898 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:10:49,097 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:10:51,100 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:10:51,937 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:10:54,461 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:10:54,462 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:10:54,462 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:10:54,467 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:10:54,728 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 16:10:55,104 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:10:57,107 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:10:57,730 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:10:59,089 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:10:59,090 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:10:59,091 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:10:59,656 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:10:59,657 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:10:59,657 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:10:59,662 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:11:00,108 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:11:01,111 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:11:03,434 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:11:03,435 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:11:03,435 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:11:03,435 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:11:03,440 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:11:04,113 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:11:05,115 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:11:08,441 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:11:12,223 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:11:12,224 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:11:12,224 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:11:12,229 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:11:13,122 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:11:13,122 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:11:14,089 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:11:14,090 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:11:14,091 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:11:14,313 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:11:16,767 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:11:16,767 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:11:16,768 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:11:16,772 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:11:17,124 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:11:19,128 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:11:19,809 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:11:21,207 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:11:21,208 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:11:21,208 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:11:21,213 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:11:22,130 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:11:23,132 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:11:24,731 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 16:11:24,739 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:11:24,740 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:11:24,740 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:11:24,745 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:11:25,134 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:11:25,745 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:11:27,137 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:11:29,090 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:11:29,090 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:11:29,091 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:11:30,837 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:11:30,838 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:11:30,838 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:11:30,838 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:11:30,843 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:11:31,140 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:11:33,145 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:11:34,487 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:11:34,488 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:11:34,488 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:11:34,493 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:11:35,147 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:11:36,494 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:11:37,150 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:11:37,903 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:11:37,904 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:11:37,904 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:11:37,909 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:11:38,151 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:11:39,153 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:11:41,910 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:11:43,442 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:11:43,443 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:11:43,443 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:11:43,448 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:11:44,090 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:11:44,090 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:11:44,091 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:11:44,156 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:11:45,159 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:11:47,390 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:11:47,569 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:11:47,569 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:11:47,570 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:11:47,574 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:11:48,161 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:11:49,163 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:11:52,613 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:11:52,831 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:11:52,832 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:11:52,832 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:11:52,837 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:11:53,167 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:11:54,732 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 16:11:55,170 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:11:56,555 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:11:56,556 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:11:56,557 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:11:56,562 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:11:57,171 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:11:58,562 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:11:59,090 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:11:59,090 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:11:59,091 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:11:59,175 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:12:01,361 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:12:01,362 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:12:01,362 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:12:01,367 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:12:02,176 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:12:03,179 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:12:04,405 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:12:05,507 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:12:05,508 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:12:05,508 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:12:05,513 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:12:06,181 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:12:07,184 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:12:09,514 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:12:10,228 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:12:10,229 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:12:10,229 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:12:10,234 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:12:11,188 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:12:11,188 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:12:14,090 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:12:14,090 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:12:14,092 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:12:15,161 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:12:15,162 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:12:15,162 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:12:15,162 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:12:15,167 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:12:15,190 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:12:17,193 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:12:18,628 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:12:18,629 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:12:18,629 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:12:18,634 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:12:19,195 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:12:20,634 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:12:21,198 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:12:24,729 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:12:24,730 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:12:24,730 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:12:24,736 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:12:24,741 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 16:12:25,200 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:12:25,742 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:12:27,203 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:12:28,514 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:12:28,515 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:12:28,515 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:12:28,520 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:12:29,090 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:12:29,090 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:12:29,092 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:12:29,206 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:12:29,207 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:12:31,320 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:12:33,497 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:12:33,498 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:12:33,498 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:12:33,503 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:12:34,211 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:12:35,213 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:12:36,541 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:12:37,655 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:12:37,655 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:12:37,656 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:12:37,661 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:12:38,215 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:12:39,217 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:12:41,166 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:12:41,167 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:12:41,167 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:12:41,172 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:12:41,219 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:12:42,172 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:12:43,222 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:12:44,090 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:12:44,090 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:12:44,092 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:12:45,539 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:12:45,540 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:12:45,540 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:12:45,545 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:12:46,223 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:12:47,224 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:12:47,585 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:12:49,254 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:12:49,255 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:12:49,255 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:12:49,260 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:12:50,226 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:12:51,229 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:12:53,261 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:12:54,738 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 16:12:55,405 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:12:55,405 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:12:55,406 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:12:55,413 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:12:56,233 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:12:57,235 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:12:58,414 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:12:59,090 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:12:59,090 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:12:59,092 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:12:59,837 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:12:59,838 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:12:59,838 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:12:59,843 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:13:00,237 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:13:01,240 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:13:03,659 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:13:03,659 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:13:03,660 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:13:03,660 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:13:03,665 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:13:04,241 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:13:05,244 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:13:08,083 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:13:08,084 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:13:08,084 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:13:08,089 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:13:08,246 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:13:09,089 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:13:09,248 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:13:14,090 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:13:14,090 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:13:14,090 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:13:14,092 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:13:14,696 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:13:14,697 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:13:14,697 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:13:14,702 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:13:15,252 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:13:17,255 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:13:19,095 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:13:19,096 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:13:19,096 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:13:19,096 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:13:19,101 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:13:19,256 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:13:21,259 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:13:24,102 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:13:24,224 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:13:24,224 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:13:24,225 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:13:24,229 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:13:24,261 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:13:24,743 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 16:13:25,263 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:13:28,615 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:13:28,616 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:13:28,616 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:13:28,621 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:13:29,090 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:13:29,091 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:13:29,092 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:13:29,265 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:13:29,367 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:13:31,269 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:13:32,363 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:13:32,363 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:13:32,364 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:13:32,368 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:13:33,272 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:13:33,272 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:13:34,405 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:13:35,968 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:13:35,969 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:13:35,969 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:13:35,974 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:13:36,275 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:13:37,277 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:13:39,975 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:13:40,017 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:13:40,018 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:13:40,018 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:13:40,023 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:13:40,279 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:13:41,280 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:13:44,090 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:13:44,091 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:13:44,092 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:13:44,190 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:13:44,317 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:13:44,317 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:13:44,322 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:13:45,284 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:13:45,284 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:13:45,323 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:13:50,323 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:13:52,617 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:13:52,618 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:13:52,618 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:13:52,623 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:13:53,292 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:13:53,293 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:13:54,743 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 16:13:55,744 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:13:58,424 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:13:58,424 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:13:58,425 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:13:58,429 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:13:59,090 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:13:59,091 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:13:59,092 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:13:59,298 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:13:59,298 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:14:01,293 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:14:01,823 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:14:01,824 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:14:01,824 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:14:01,829 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:14:02,299 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:14:03,302 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:14:05,883 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:14:05,884 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:14:05,884 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:14:05,889 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:14:06,304 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:14:06,889 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:14:07,307 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:14:09,433 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:14:09,433 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:14:09,434 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:14:09,438 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:14:10,308 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:14:11,311 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:14:12,439 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:14:14,091 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:14:14,091 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:14:14,092 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:14:14,516 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:14:14,516 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:14:14,517 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:14:14,522 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:14:15,315 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:14:15,315 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:14:17,561 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:14:18,290 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:14:18,291 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:14:18,291 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:14:18,296 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:14:18,317 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:14:19,319 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:14:23,297 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:14:23,715 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:14:23,715 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:14:23,716 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:14:23,721 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:14:24,322 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:14:24,745 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 16:14:25,325 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:14:28,747 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:14:29,076 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:14:29,080 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:14:29,080 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:14:29,085 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:14:29,091 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:14:29,091 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:14:29,093 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:14:29,327 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:14:31,331 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:14:32,638 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:14:32,639 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:14:32,639 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:14:32,644 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:14:33,332 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:14:34,681 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:14:35,336 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:14:36,582 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:14:36,583 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:14:36,583 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:14:36,588 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:14:37,339 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:14:37,339 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:14:40,589 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:14:40,748 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:14:40,749 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:14:40,749 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:14:40,754 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:14:41,341 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:14:43,344 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:14:44,091 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:14:44,091 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:14:44,093 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:14:46,310 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:14:46,414 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:14:46,414 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:14:46,415 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:14:46,419 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:14:47,349 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:14:47,349 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:14:51,241 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:14:51,242 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:14:51,242 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:14:51,247 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:14:51,351 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:14:52,247 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:14:53,355 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:14:54,749 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 16:14:55,810 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:14:55,811 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:14:55,811 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:14:55,816 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:14:56,357 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:14:57,358 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:14:57,817 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:14:59,091 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:14:59,091 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:14:59,093 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:14:59,818 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:14:59,819 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:14:59,819 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:14:59,824 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:15:00,359 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:15:01,362 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:15:02,861 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:15:04,068 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:15:04,069 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:15:04,069 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:15:04,074 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:15:04,364 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:15:05,366 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:15:08,074 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:15:08,142 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:15:08,143 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:15:08,143 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:15:08,148 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:15:08,369 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:15:09,371 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:15:13,148 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:15:13,495 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:15:13,496 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:15:13,496 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:15:13,501 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:15:14,091 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:15:14,091 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:15:14,093 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:15:14,374 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:15:15,377 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:15:17,636 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:15:17,637 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:15:17,637 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:15:17,642 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:15:18,378 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:15:18,681 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:15:19,381 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:15:21,625 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:15:21,626 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:15:21,626 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:15:21,632 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:15:22,382 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:15:23,385 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:15:24,632 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:15:24,752 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 16:15:27,336 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:15:27,337 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:15:27,337 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:15:27,346 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:15:27,387 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:15:29,091 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:15:29,091 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:15:29,093 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:15:29,390 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:15:30,380 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:15:31,233 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:15:31,234 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:15:31,234 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:15:31,239 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:15:31,392 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:15:33,395 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:15:35,684 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:15:35,685 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:15:35,685 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:15:35,685 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:15:35,693 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:15:36,398 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:15:37,400 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:15:39,753 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:15:39,754 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:15:39,754 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:15:39,759 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:15:40,402 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:15:40,759 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:15:41,404 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:15:43,360 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:15:43,361 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:15:43,361 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:15:43,367 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:15:43,405 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:15:44,308 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:15:44,308 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:15:44,308 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:15:45,409 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:15:46,526 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:15:50,099 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:15:50,100 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:15:50,100 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:15:50,105 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:15:50,411 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:15:51,414 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:15:52,141 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:15:54,454 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:15:54,455 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:15:54,455 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:15:54,460 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:15:54,754 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 16:15:55,420 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:15:55,420 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:15:57,756 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:15:58,969 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:15:58,970 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:15:58,970 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:15:58,975 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:15:59,307 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:15:59,307 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:15:59,308 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:15:59,422 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:16:01,425 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:16:02,849 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:16:04,115 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:16:04,116 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:16:04,116 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:16:04,121 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:16:04,427 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:16:05,430 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:16:08,157 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:16:09,834 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:16:09,835 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:16:09,835 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:16:09,840 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:16:10,433 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:16:11,436 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:16:13,841 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:16:14,307 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:16:14,307 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:16:14,309 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:16:14,647 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:16:14,647 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:16:14,648 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:16:14,653 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:16:15,440 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:16:15,440 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:16:19,689 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:16:20,338 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:16:20,339 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:16:20,340 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:16:20,344 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:16:20,443 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:16:21,445 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:16:24,523 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:16:24,524 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:16:24,524 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:16:24,530 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:16:24,756 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 16:16:24,758 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:16:25,450 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:16:25,450 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:16:29,307 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:16:29,307 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:16:29,308 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:16:29,329 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:16:29,531 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:16:29,531 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:16:29,536 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:16:30,453 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:16:30,536 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:16:31,456 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:16:33,793 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:16:33,794 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:16:33,794 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:16:33,799 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:16:34,459 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:16:35,461 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:16:35,800 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:16:40,279 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:16:40,280 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:16:40,280 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:16:40,285 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:16:40,465 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:16:41,286 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:16:41,467 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:16:44,307 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:16:44,308 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:16:44,308 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:16:46,613 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:16:47,366 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:16:47,366 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:16:47,367 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:16:47,371 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:16:47,471 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:16:49,474 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:16:51,166 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:16:51,167 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:16:51,167 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:16:51,172 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:16:51,475 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:16:52,172 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:16:53,480 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:16:54,762 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 16:16:55,349 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:16:55,350 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:16:55,351 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:16:55,356 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:16:55,481 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:16:57,357 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:16:57,482 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:16:59,308 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:16:59,308 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:16:59,310 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:16:59,870 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:16:59,870 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:16:59,871 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:16:59,875 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:17:00,484 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:17:01,486 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:17:02,913 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:17:06,399 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:17:06,400 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:17:06,400 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:17:06,405 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:17:06,489 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:17:07,492 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:17:08,406 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:17:10,113 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:17:10,114 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:17:10,114 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:17:10,119 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:17:10,494 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:17:11,496 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:17:14,120 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:17:14,307 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:17:14,308 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:17:14,308 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:17:14,731 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:17:14,731 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:17:14,732 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:17:14,737 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:17:15,501 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:17:15,501 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:17:19,773 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:17:20,712 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:17:20,713 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:17:20,714 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:17:20,719 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:17:21,506 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:17:21,506 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:17:24,762 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 16:17:25,184 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:17:25,185 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:17:25,185 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:17:25,186 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:17:25,190 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:17:25,508 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:17:27,512 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:17:29,307 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:17:29,308 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:17:29,308 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:17:29,703 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:17:29,704 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:17:29,704 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:17:29,709 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:17:30,514 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:17:30,749 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:17:31,517 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:17:35,749 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:17:36,334 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:17:36,335 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:17:36,335 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:17:36,340 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:17:36,520 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:17:37,523 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:17:40,932 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:17:40,933 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:17:40,933 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:17:40,934 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:17:40,938 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:17:41,525 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:17:43,528 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:17:44,307 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:17:44,308 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:17:44,308 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:17:44,742 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:17:44,743 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:17:44,743 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:17:44,748 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:17:45,531 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:17:45,532 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:17:46,785 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:17:49,068 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:17:49,068 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:17:49,069 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:17:49,074 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:17:49,534 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:17:51,537 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:17:52,074 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:17:54,072 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:17:54,072 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:17:54,073 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:17:54,078 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:17:54,540 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:17:54,767 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 16:17:55,543 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:17:57,767 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:17:57,768 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:17:57,768 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:17:57,769 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:17:57,774 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:17:58,544 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:17:59,308 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:17:59,308 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:17:59,308 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:17:59,547 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:18:01,821 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:18:01,822 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:18:01,822 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:18:01,827 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:18:02,549 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:18:02,865 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:18:03,551 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:18:05,473 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:18:05,473 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:18:05,474 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:18:05,479 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:18:05,552 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:18:07,556 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:18:08,479 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:18:10,446 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:18:10,447 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:18:10,447 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:18:10,452 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:18:10,557 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:18:11,560 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:18:14,308 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:18:14,308 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:18:14,308 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:18:14,602 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:18:14,618 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:18:14,618 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:18:14,618 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:18:14,623 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:18:15,564 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:18:15,565 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:18:18,760 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:18:18,761 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:18:18,761 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:18:18,767 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:18:19,567 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:18:19,567 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:18:19,767 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:18:22,505 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:18:22,506 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:18:22,506 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:18:22,511 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:18:22,569 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:18:23,571 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:18:24,768 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 16:18:24,769 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:18:26,864 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:18:26,865 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:18:26,865 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:18:26,870 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:18:27,576 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:18:27,576 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:18:29,308 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:18:29,308 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:18:29,308 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:18:30,592 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:18:32,379 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:18:32,379 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:18:32,380 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:18:32,385 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:18:32,579 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:18:33,581 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:18:35,922 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:18:35,923 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:18:35,923 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:18:35,923 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:18:35,928 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:18:36,584 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:18:37,587 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:18:40,929 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:18:44,119 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:18:44,119 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:18:44,120 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:18:44,125 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:18:44,308 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:18:44,308 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:18:44,309 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:18:44,592 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:18:45,594 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:18:46,527 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:18:47,825 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:18:47,826 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:18:47,826 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:18:47,831 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:18:48,596 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:18:49,597 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:18:51,869 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:18:52,031 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:18:52,032 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:18:52,032 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:18:52,037 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:18:52,600 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:18:53,602 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:18:54,771 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 16:18:55,823 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:18:55,824 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:18:55,824 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:18:55,829 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:18:56,604 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:18:57,607 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:18:57,829 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:18:59,308 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:18:59,308 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:18:59,308 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:18:59,552 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:18:59,586 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:18:59,586 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:18:59,591 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:18:59,608 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:19:01,611 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:19:03,592 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:19:04,992 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:19:04,993 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:19:04,993 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:19:04,999 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:19:05,613 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:19:07,617 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:19:09,037 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:19:09,562 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:19:09,563 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:19:09,563 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:19:09,568 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:19:09,618 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:19:11,621 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:19:13,555 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:19:13,556 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:19:13,556 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:19:13,561 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:19:13,622 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:19:14,308 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:19:14,309 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:19:14,309 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:19:14,555 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:19:15,625 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:19:17,235 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:19:17,236 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:19:17,236 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:19:17,241 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:19:17,626 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:19:19,629 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:19:20,277 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:19:21,552 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:19:21,552 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:19:21,553 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:19:21,558 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:19:21,631 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:19:23,634 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:19:24,774 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 16:19:25,776 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:19:25,943 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:19:25,943 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:19:25,944 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:19:25,949 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:19:26,636 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:19:27,638 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:19:29,308 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:19:29,309 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:19:29,309 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:19:31,552 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:19:34,430 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:19:34,431 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:19:34,431 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:19:34,436 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:19:34,644 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:19:35,646 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:19:37,473 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:19:39,212 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:19:39,213 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:19:39,213 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:19:39,218 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:19:39,649 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:19:41,652 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:19:43,219 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:19:44,308 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:19:44,309 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:19:44,309 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:19:44,321 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:19:44,511 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:19:44,511 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:19:44,516 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:19:44,654 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:19:45,656 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:19:48,358 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:19:48,359 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:19:48,359 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:19:48,360 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:19:48,367 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:19:48,658 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:19:49,660 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:19:53,368 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:19:53,438 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:19:53,439 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:19:53,439 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:19:53,444 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:19:53,664 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:19:54,777 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 16:19:55,667 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:19:58,778 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:19:59,308 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:19:59,308 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:19:59,310 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:20:00,479 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:20:00,480 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:20:00,480 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:20:00,485 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:20:00,670 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:20:01,672 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:20:04,525 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:20:05,091 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:20:05,091 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:20:05,092 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:20:05,096 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:20:05,675 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:20:07,678 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:20:09,100 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:20:09,101 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:20:09,101 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:20:09,106 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:20:09,680 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:20:10,106 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:20:11,683 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:20:13,358 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:20:13,359 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:20:13,359 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:20:13,365 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:20:13,684 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:20:14,308 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:20:14,309 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:20:14,309 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:20:15,575 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:20:15,687 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:20:19,499 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:20:19,505 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:20:19,506 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:20:19,511 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:20:19,689 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:20:21,541 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:20:21,693 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:20:24,780 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 16:20:26,781 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:20:27,868 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:20:27,868 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:20:27,869 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:20:27,874 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:20:28,697 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:20:29,308 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:20:29,309 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:20:29,309 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:20:29,700 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:20:31,045 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:20:31,046 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:20:31,046 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:20:31,051 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:20:31,701 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:20:32,089 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:20:33,704 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:20:35,899 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:20:35,900 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:20:35,900 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:20:35,905 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:20:36,707 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:20:37,707 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:20:37,906 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:20:40,057 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:20:40,057 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:20:40,058 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:20:40,063 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:20:40,709 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:20:41,712 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:20:43,063 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:20:44,309 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:20:44,309 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:20:44,309 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:20:48,566 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:20:49,264 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:20:49,265 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:20:49,265 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:20:49,270 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:20:49,716 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:20:51,720 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:20:53,275 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:20:53,276 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:20:53,276 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:20:53,281 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:20:53,722 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:20:54,282 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:20:54,783 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 16:20:55,725 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:20:58,077 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:20:58,078 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:20:58,078 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:20:58,083 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:20:58,727 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:20:59,309 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:20:59,309 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:20:59,309 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:20:59,605 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:20:59,729 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:21:02,754 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:21:02,755 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:21:02,755 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:21:02,761 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:21:03,733 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:21:03,734 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:21:04,797 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:21:07,140 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:21:07,141 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:21:07,141 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:21:07,146 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:21:07,736 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:21:09,739 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:21:10,147 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:21:12,541 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:21:12,542 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:21:12,542 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:21:12,547 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:21:12,741 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:21:13,744 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:21:14,309 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:21:14,309 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:21:14,309 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:21:15,598 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:21:16,426 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:21:16,426 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:21:16,426 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:21:16,432 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:21:16,746 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:21:17,748 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:21:21,469 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:21:22,762 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:21:22,762 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:21:22,763 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:21:22,768 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:21:23,754 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:21:23,754 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:21:24,789 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 16:21:26,791 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:21:26,882 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:21:26,883 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:21:26,883 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:21:26,888 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:21:27,758 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:21:27,758 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:21:29,309 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:21:29,310 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:21:29,310 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:21:30,512 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:21:30,512 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:21:30,513 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:21:30,518 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:21:30,760 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:21:31,763 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:21:32,519 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:21:34,362 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:21:34,363 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:21:34,363 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:21:34,368 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:21:34,765 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:21:35,768 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:21:38,122 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:21:38,123 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:21:38,124 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:21:38,124 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:21:38,129 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:21:38,770 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:21:39,772 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:21:43,129 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:21:44,309 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:21:44,309 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:21:44,309 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:21:45,765 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:21:45,766 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:21:45,766 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:21:45,785 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:21:45,786 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:21:47,789 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:21:48,809 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:21:50,191 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:21:50,192 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:21:50,192 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:21:50,197 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:21:50,791 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:21:51,793 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:21:53,586 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:21:53,587 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:21:53,587 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:21:53,592 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:21:53,796 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:21:54,593 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:21:54,788 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 16:21:55,797 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:21:59,309 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:21:59,309 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:21:59,309 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:22:00,543 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:22:02,554 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:22:02,555 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:22:02,555 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:22:02,560 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:22:02,801 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:22:03,804 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:22:05,597 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:22:06,278 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:22:06,279 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:22:06,279 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:22:06,284 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:22:06,807 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:22:07,807 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:22:11,285 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:22:11,656 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:22:11,657 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:22:11,657 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:22:11,662 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:22:11,810 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:22:13,813 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:22:14,309 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:22:14,310 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:22:14,310 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:22:16,044 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:22:16,045 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:22:16,045 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:22:16,050 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:22:16,815 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:22:17,089 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:22:17,817 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:22:21,502 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:22:21,503 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:22:21,503 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:22:21,508 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:22:21,819 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:22:22,508 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:22:23,823 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:22:24,794 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 16:22:27,795 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:22:29,309 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:22:29,310 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:22:29,310 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:22:29,902 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:22:29,902 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:22:29,903 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:22:29,908 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:22:30,828 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:22:31,831 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:22:32,945 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:22:33,424 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:22:33,425 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:22:33,425 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:22:33,430 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:22:33,832 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:22:35,836 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:22:38,431 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:22:38,431 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:22:38,432 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:22:38,432 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:22:38,437 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:22:38,838 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:22:39,840 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:22:41,919 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:22:41,920 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:22:41,920 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:22:41,925 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:22:42,842 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:22:43,843 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:22:43,925 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:22:44,309 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:22:44,310 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:22:44,310 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:22:49,561 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:22:50,276 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:22:50,277 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:22:50,277 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:22:50,295 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:22:50,847 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:22:51,853 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:22:54,794 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 16:22:54,795 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:22:55,550 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:22:55,551 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:22:55,551 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:22:55,556 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:22:55,857 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:22:57,860 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:22:59,309 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:22:59,310 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:22:59,310 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:23:00,513 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:23:01,430 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:23:01,431 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:23:01,431 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:23:01,436 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:23:01,863 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:23:03,866 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:23:06,473 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:23:07,847 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:23:07,848 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:23:07,848 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:23:07,858 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:23:07,868 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:23:09,872 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:23:11,599 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:23:11,600 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:23:11,600 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:23:11,601 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:23:11,605 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:23:11,873 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:23:13,876 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:23:14,309 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:23:14,310 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:23:14,310 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:23:15,924 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:23:15,924 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:23:15,924 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:23:15,929 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:23:16,878 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:23:16,969 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:23:17,880 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:23:20,282 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:23:20,282 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:23:20,283 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:23:20,288 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:23:20,882 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:23:21,885 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:23:22,288 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:23:24,690 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:23:24,691 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:23:24,691 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:23:24,696 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:23:24,796 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 16:23:24,886 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:23:25,889 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:23:27,797 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:23:29,310 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:23:29,310 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:23:29,310 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:23:29,361 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:23:29,525 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:23:29,525 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:23:29,531 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:23:29,891 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:23:31,895 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:23:32,933 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:23:32,934 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:23:32,934 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:23:32,934 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:23:32,939 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:23:33,898 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:23:33,898 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:23:37,391 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:23:37,392 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:23:37,392 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:23:37,397 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:23:37,902 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:23:38,398 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:23:39,905 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:23:41,803 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:23:41,803 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:23:41,804 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:23:41,809 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:23:41,906 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:23:43,809 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:23:43,907 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:23:44,310 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:23:44,310 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:23:44,310 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:23:45,533 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:23:45,533 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:23:45,534 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:23:45,538 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:23:45,908 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:23:47,911 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:23:49,400 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:23:49,401 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:23:49,401 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:23:49,401 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:23:49,406 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:23:49,913 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:23:51,916 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:23:54,134 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:23:54,135 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:23:54,135 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:23:54,140 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:23:54,798 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 16:23:54,799 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:23:54,919 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:23:55,922 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:23:59,310 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:23:59,310 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:23:59,310 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:24:00,197 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:24:00,197 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:24:00,198 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:24:00,198 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:24:00,203 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:24:00,924 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:24:01,927 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:24:05,241 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:24:08,483 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:24:08,484 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:24:08,484 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:24:08,489 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:24:08,932 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:24:09,934 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:24:10,490 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:24:14,310 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:24:14,310 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:24:14,311 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:24:15,610 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:24:17,292 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:24:17,292 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:24:17,292 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:24:17,298 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:24:17,941 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:24:17,941 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:24:21,337 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:24:21,366 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:24:21,367 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:24:21,367 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:24:21,372 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:24:21,945 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:24:21,945 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:24:24,801 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 16:24:25,123 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:24:25,124 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:24:25,124 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:24:25,129 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:24:25,949 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:24:25,949 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:24:27,130 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:24:29,310 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:24:29,310 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:24:29,310 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:24:30,203 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:24:30,203 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:24:30,204 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:24:30,208 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:24:30,953 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:24:31,956 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:24:32,245 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:24:36,746 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:24:36,747 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:24:36,747 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:24:36,753 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:24:36,960 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:24:37,753 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:24:37,963 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:24:40,522 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:24:40,523 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:24:40,523 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:24:40,528 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:24:40,965 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:24:41,967 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:24:43,528 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:24:44,310 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:24:44,310 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:24:44,312 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:24:45,105 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:24:45,106 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:24:45,106 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:24:45,111 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:24:45,971 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:24:45,972 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:24:49,149 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:24:49,903 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:24:49,904 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:24:49,904 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:24:49,909 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:24:49,974 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:24:51,977 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:24:54,807 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 16:24:54,808 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:24:54,977 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:24:54,978 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:24:54,978 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:24:54,983 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:24:54,983 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:24:55,986 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:24:59,310 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:24:59,311 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:24:59,311 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:25:00,544 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:25:03,319 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:25:03,320 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:25:03,320 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:25:03,325 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:25:03,992 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:25:03,993 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:25:06,365 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:25:07,294 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:25:07,295 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:25:07,295 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:25:07,300 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:25:07,997 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:25:07,997 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:25:11,052 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:25:11,053 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:25:11,053 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:25:11,058 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:25:12,002 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:25:12,002 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:25:12,058 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:25:14,310 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:25:14,311 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:25:14,311 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:25:16,511 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:25:16,512 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:25:16,512 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:25:16,517 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:25:17,005 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:25:17,557 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:25:18,007 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:25:20,469 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:25:20,470 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:25:20,470 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:25:20,475 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:25:21,009 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:25:22,009 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:25:23,475 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:25:24,807 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 16:25:28,809 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:25:28,860 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:25:28,860 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:25:28,861 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:25:28,866 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:25:29,013 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:25:29,311 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:25:29,311 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:25:29,312 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:25:30,016 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:25:33,190 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:25:33,191 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:25:33,191 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:25:33,196 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:25:34,019 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:25:34,019 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:25:34,233 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:25:37,131 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:25:37,132 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:25:37,132 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:25:37,137 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:25:38,027 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:25:38,028 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:25:40,138 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:25:44,310 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:25:44,311 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:25:44,312 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:25:44,887 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:25:44,888 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:25:44,888 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:25:44,893 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:25:45,031 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:25:45,933 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:25:46,034 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:25:48,788 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:25:48,789 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:25:48,789 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:25:48,794 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:25:49,036 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:25:50,038 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:25:51,795 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:25:52,862 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:25:52,863 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:25:52,863 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:25:52,868 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:25:53,041 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:25:54,045 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:25:54,813 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 16:25:56,815 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:25:57,377 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:25:57,377 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:25:57,377 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:25:57,382 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:25:58,049 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:25:58,049 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:25:59,311 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:25:59,311 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:25:59,311 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:26:01,780 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:26:01,780 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:26:01,781 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:26:01,786 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:26:01,825 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:26:02,051 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:26:04,054 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:26:06,213 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:26:06,214 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:26:06,214 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:26:06,219 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:26:07,056 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:26:07,219 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:26:08,059 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:26:11,173 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:26:11,174 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:26:11,174 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:26:11,179 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:26:12,063 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:26:12,063 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:26:13,179 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:26:14,311 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:26:14,311 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:26:14,312 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:26:15,153 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:26:15,153 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:26:15,153 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:26:15,158 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:26:16,067 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:26:16,068 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:26:18,197 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:26:21,024 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:26:21,025 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:26:21,025 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:26:21,030 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:26:21,070 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:26:22,073 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:26:24,031 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:26:24,812 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 16:26:24,863 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:26:24,864 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:26:24,864 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:26:24,869 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:26:25,075 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:26:26,077 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:26:28,552 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:26:28,553 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:26:28,553 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:26:28,558 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:26:29,079 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:26:29,311 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:26:29,311 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:26:29,311 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:26:29,582 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:26:30,081 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:26:32,429 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:26:32,429 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:26:32,429 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:26:32,434 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:26:33,084 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:26:34,086 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:26:35,473 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:26:37,112 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:26:37,113 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:26:37,113 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:26:37,118 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:26:38,092 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:26:38,092 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:26:41,119 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:26:41,619 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:26:41,620 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:26:41,620 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:26:41,625 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:26:42,094 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:26:44,097 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:26:44,311 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:26:44,311 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:26:44,311 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:26:45,677 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:26:45,678 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:26:45,678 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:26:45,683 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:26:46,098 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:26:46,721 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:26:48,102 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:26:50,651 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:26:50,652 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:26:50,652 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:26:50,657 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:26:51,103 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:26:52,104 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:26:52,658 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:26:54,816 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 16:26:55,764 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:26:55,768 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:26:55,769 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:26:55,774 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:26:56,108 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:26:57,774 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:26:58,112 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:26:59,311 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:26:59,311 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:26:59,311 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:26:59,754 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:26:59,754 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:26:59,754 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:26:59,760 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:27:00,113 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:27:02,116 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:27:02,797 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:27:04,428 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:27:04,429 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:27:04,429 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:27:04,434 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:27:05,118 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:27:06,120 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:27:08,435 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:27:10,326 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:27:10,327 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:27:10,327 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:27:10,332 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:27:11,123 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:27:12,126 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:27:14,311 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:27:14,312 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:27:14,312 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:27:14,585 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:27:15,615 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:27:15,616 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:27:15,616 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:27:15,621 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:27:16,128 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:27:18,131 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:27:19,661 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:27:20,824 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:27:20,825 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:27:20,825 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:27:20,831 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:27:21,133 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:27:22,135 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:27:24,818 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 16:27:24,820 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:27:25,410 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:27:25,411 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:27:25,411 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:27:25,416 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:27:26,140 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:27:26,140 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:27:29,311 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:27:29,312 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:27:29,312 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:27:30,039 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:27:30,039 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:27:30,039 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:27:30,040 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:27:30,044 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:27:30,142 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:27:32,146 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:27:35,081 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:27:38,345 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:27:38,346 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:27:38,346 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:27:38,351 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:27:39,151 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:27:40,154 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:27:40,352 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:27:42,982 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:27:42,983 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:27:42,983 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:27:42,989 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:27:43,155 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:27:44,158 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:27:44,311 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:27:44,311 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:27:44,313 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:27:45,663 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:27:49,551 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:27:49,552 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:27:49,552 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:27:49,557 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:27:50,164 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:27:50,164 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:27:51,597 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:27:53,376 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:27:53,377 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:27:53,377 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:27:53,382 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:27:54,170 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:27:54,170 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:27:54,820 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 16:27:56,821 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:27:57,897 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:27:57,898 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:27:57,898 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:27:57,903 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:27:58,172 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:27:59,311 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:27:59,312 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:27:59,312 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:28:00,175 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:28:01,942 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:28:01,943 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:28:01,943 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:28:01,943 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:28:01,948 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:28:02,176 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:28:04,179 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:28:06,985 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:28:09,853 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:28:09,854 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:28:09,854 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:28:09,859 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:28:10,183 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:28:12,184 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:28:12,860 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:28:13,996 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:28:13,997 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:28:13,997 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:28:14,003 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:28:14,185 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:28:14,312 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:28:14,312 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:28:14,312 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:28:16,188 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:28:18,475 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:28:18,476 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:28:18,476 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:28:18,476 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:28:18,481 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:28:19,190 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:28:20,193 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:28:23,521 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:28:23,845 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:28:23,846 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:28:23,846 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:28:23,851 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:28:24,195 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:28:24,826 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 16:28:26,198 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:28:28,542 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:28:28,543 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:28:28,543 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:28:28,544 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:28:28,548 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:28:29,200 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:28:29,312 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:28:29,312 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:28:29,313 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:28:30,202 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:28:32,558 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:28:32,558 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:28:32,558 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:28:32,563 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:28:33,204 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:28:33,601 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:28:34,207 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:28:38,589 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:28:38,590 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:28:38,590 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:28:38,595 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:28:39,211 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:28:39,595 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:28:40,213 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:28:42,865 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:28:42,866 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:28:42,866 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:28:42,871 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:28:43,215 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:28:44,218 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:28:44,312 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:28:44,312 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:28:44,312 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:28:45,569 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:28:46,640 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:28:46,641 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:28:46,641 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:28:46,646 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:28:47,219 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:28:48,222 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:28:50,685 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:28:51,177 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:28:51,178 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:28:51,178 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:28:51,183 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:28:51,224 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:28:52,226 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:28:54,826 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 16:28:55,827 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:28:59,312 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:28:59,312 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:28:59,313 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:28:59,841 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:28:59,842 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:28:59,842 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:28:59,847 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:29:00,232 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:29:00,885 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:29:02,235 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:29:03,985 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:29:03,985 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:29:03,986 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:29:03,991 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:29:04,236 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:29:05,991 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:29:06,240 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:29:07,310 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:29:07,311 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:29:07,311 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:29:07,316 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:29:08,243 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:29:08,244 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:29:11,317 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:29:12,177 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:29:12,177 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:29:12,178 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:29:12,183 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:29:12,246 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:29:14,249 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:29:14,312 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:29:14,312 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:29:14,313 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:29:16,085 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:29:16,085 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:29:16,086 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:29:16,091 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:29:16,250 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:29:17,129 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:29:18,253 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:29:20,043 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:29:20,044 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:29:20,044 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:29:20,050 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:29:20,254 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:29:22,258 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:29:23,051 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:29:23,944 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:29:23,945 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:29:23,945 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:29:23,950 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:29:24,259 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:29:24,834 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 16:29:26,262 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:29:28,835 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:29:29,312 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:29:29,312 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:29:29,313 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:29:32,465 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:29:32,466 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:29:32,466 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:29:32,471 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:29:33,266 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:29:34,268 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:29:34,509 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:29:35,892 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:29:35,893 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:29:35,893 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:29:35,898 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:29:36,270 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:29:38,273 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:29:39,899 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:29:40,691 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:29:40,692 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:29:40,692 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:29:40,697 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:29:41,275 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:29:42,278 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:29:44,312 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:29:44,312 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:29:44,314 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:29:44,798 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:29:44,799 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:29:44,799 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:29:44,804 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:29:45,279 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:29:45,841 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:29:46,282 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:29:48,486 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:29:48,487 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:29:48,487 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:29:48,492 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:29:49,284 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:29:50,286 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:29:51,493 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:29:52,734 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:29:52,734 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:29:52,735 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:29:52,740 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:29:53,289 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:29:54,292 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:29:54,831 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 16:29:56,562 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:29:56,563 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:29:56,563 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:29:56,563 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:29:56,568 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:29:57,294 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:29:58,296 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:29:59,312 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:29:59,312 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:29:59,314 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:30:01,394 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:30:01,394 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:30:01,394 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:30:01,399 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:30:02,301 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:30:02,301 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:30:02,437 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:30:04,936 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:30:04,937 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:30:04,937 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:30:04,942 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:30:05,302 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:30:06,305 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:30:07,943 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:30:09,571 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:30:09,572 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:30:09,572 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:30:09,577 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:30:10,307 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:30:10,308 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:30:13,578 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:30:14,312 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:30:14,312 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:30:14,314 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:30:16,622 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:30:16,622 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:30:16,623 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:30:16,627 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:30:17,311 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:30:18,314 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:30:18,665 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:30:20,792 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:30:20,793 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:30:20,793 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:30:20,798 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:30:21,316 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:30:22,318 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:30:23,799 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:30:24,837 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 16:30:28,838 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:30:29,312 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:30:29,312 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:30:29,314 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:30:29,785 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:30:29,785 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:30:29,786 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:30:29,791 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:30:30,323 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:30:32,326 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:30:33,343 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:30:33,344 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:30:33,344 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:30:33,349 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:30:34,327 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:30:34,350 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:30:36,331 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:30:37,040 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:30:37,041 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:30:37,041 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:30:37,046 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:30:37,332 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:30:38,334 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:30:40,046 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:30:40,673 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:30:40,674 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:30:40,674 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:30:40,679 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:30:41,336 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:30:42,339 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:30:44,312 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:30:44,313 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:30:44,314 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:30:45,485 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:30:45,485 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:30:45,485 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:30:45,486 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:30:45,490 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:30:46,341 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:30:48,344 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:30:49,071 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:30:49,072 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:30:49,072 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:30:49,077 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:30:49,345 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:30:50,347 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:30:51,078 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:30:54,284 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:30:54,285 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:30:54,285 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:30:54,290 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:30:54,350 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:30:54,836 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 16:30:56,352 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:30:56,837 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:30:58,553 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:30:58,554 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:30:58,554 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:30:58,559 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:30:59,312 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:30:59,313 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:30:59,314 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:30:59,353 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:31:00,356 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:31:02,584 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:31:06,898 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:31:06,899 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:31:06,899 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:31:06,904 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:31:07,360 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:31:07,941 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:31:08,363 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:31:10,807 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:31:10,808 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:31:10,808 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:31:10,814 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:31:11,364 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:31:12,367 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:31:13,814 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:31:14,312 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:31:14,313 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:31:14,314 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:31:16,738 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:31:16,739 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:31:16,739 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:31:16,744 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:31:17,370 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:31:18,372 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:31:19,781 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:31:20,694 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:31:20,694 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:31:20,695 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:31:20,700 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:31:21,374 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:31:22,376 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:31:24,839 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 16:31:24,840 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:31:25,447 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:31:25,448 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:31:25,448 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:31:25,453 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:31:26,378 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:31:28,382 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:31:29,313 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:31:29,313 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:31:29,313 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:31:29,888 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:31:29,889 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:31:29,889 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:31:29,889 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:31:29,894 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:31:30,383 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:31:32,386 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:31:34,933 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:31:37,021 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:31:37,025 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:31:37,025 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:31:37,030 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:31:37,390 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:31:38,392 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:31:40,031 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:31:40,903 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:31:40,904 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:31:40,904 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:31:40,909 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:31:41,394 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:31:42,395 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:31:44,313 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:31:44,313 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:31:44,314 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:31:44,719 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:31:44,720 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:31:44,720 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:31:44,725 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:31:45,396 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:31:45,765 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:31:46,399 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:31:50,765 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:31:53,143 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:31:53,144 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:31:53,144 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:31:53,149 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:31:53,404 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:31:54,406 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:31:54,841 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 16:31:55,843 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:31:58,788 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:31:58,789 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:31:58,789 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:31:58,794 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:31:59,313 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:31:59,313 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:31:59,314 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:31:59,409 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:32:00,412 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:32:01,525 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:32:03,378 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:32:03,379 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:32:03,379 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:32:03,384 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:32:03,413 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:32:06,417 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:32:07,421 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:32:08,765 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:32:08,766 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:32:08,766 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:32:08,771 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:32:09,419 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:32:10,421 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:32:12,772 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:32:14,313 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:32:14,313 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:32:14,314 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:32:15,423 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:32:15,424 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:32:15,424 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:32:15,429 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:32:16,425 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:32:18,428 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:32:18,465 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:32:20,940 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:32:20,940 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:32:20,941 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:32:20,946 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:32:21,430 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:32:22,432 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:32:23,946 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:32:24,434 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:32:24,435 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:32:24,435 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:32:24,440 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:32:24,843 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 16:32:25,434 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:32:26,436 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:32:29,313 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:32:29,313 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:32:29,315 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:32:29,615 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:32:32,834 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:32:32,835 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:32:32,835 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:32:32,840 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:32:33,440 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:32:34,443 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:32:34,877 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:32:36,762 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:32:36,763 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:32:36,763 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:32:36,769 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:32:37,446 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:32:38,448 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:32:40,769 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:32:41,928 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:32:41,928 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:32:41,929 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:32:41,934 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:32:42,450 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:32:44,313 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:32:44,313 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:32:44,315 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:32:44,453 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:32:46,395 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:32:46,396 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:32:46,396 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:32:46,396 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:32:46,401 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:32:46,454 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:32:48,458 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:32:50,967 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:32:50,967 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:32:50,968 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:32:50,973 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:32:51,459 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:32:51,973 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:32:52,462 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:32:54,850 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 16:32:55,732 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:32:55,733 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:32:55,733 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:32:55,738 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:32:56,465 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:32:57,738 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:32:58,469 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:32:59,313 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:32:59,314 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:32:59,315 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:33:03,522 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:33:04,236 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:33:04,236 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:33:04,236 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:33:04,241 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:33:04,472 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:33:06,475 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:33:09,281 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:33:12,628 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:33:12,629 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:33:12,629 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:33:12,634 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:33:13,479 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:33:14,313 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:33:14,314 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:33:14,314 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:33:14,480 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:33:14,515 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:33:19,515 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:33:20,969 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:33:20,969 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:33:20,970 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:33:20,975 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:33:21,484 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:33:22,486 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:33:24,850 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 16:33:24,851 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:33:25,061 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:33:25,062 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:33:25,062 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:33:25,067 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:33:25,488 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:33:26,489 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:33:28,836 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:33:28,837 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:33:28,837 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:33:28,842 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:33:29,313 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:33:29,314 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:33:29,315 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:33:29,490 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:33:30,493 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:33:30,626 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:33:35,028 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:33:35,029 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:33:35,029 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:33:35,034 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:33:35,497 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:33:36,073 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:33:36,500 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:33:39,122 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:33:39,123 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:33:39,123 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:33:39,128 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:33:39,501 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:33:40,502 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:33:41,128 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:33:44,313 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:33:44,314 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:33:44,314 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:33:44,369 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:33:44,558 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:33:44,558 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:33:44,563 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:33:45,505 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:33:46,507 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:33:46,563 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:33:47,977 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:33:47,978 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:33:47,978 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:33:47,983 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:33:48,509 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:33:50,512 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:33:51,984 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:33:52,249 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:33:52,250 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:33:52,250 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:33:52,255 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:33:52,513 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:33:54,520 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:33:54,856 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 16:33:55,773 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:33:55,774 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:33:55,774 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:33:55,779 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:33:56,521 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:33:57,780 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:33:58,525 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:33:59,313 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:33:59,314 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:33:59,315 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:33:59,944 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:33:59,945 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:33:59,945 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:33:59,950 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:34:00,526 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:34:02,529 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:34:02,989 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:34:04,327 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:34:04,327 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:34:04,327 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:34:04,332 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:34:04,530 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:34:06,533 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:34:08,333 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:34:09,462 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:34:09,463 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:34:09,463 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:34:09,468 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:34:09,535 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:34:10,538 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:34:13,303 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:34:13,304 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:34:13,304 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:34:13,309 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:34:13,540 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:34:14,310 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:34:14,314 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:34:14,314 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:34:14,315 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:34:14,542 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:34:19,016 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:34:19,016 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:34:19,016 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:34:19,022 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:34:19,545 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:34:20,061 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:34:20,548 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:34:23,187 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:34:23,188 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:34:23,188 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:34:23,193 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:34:23,550 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:34:24,550 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:34:24,856 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 16:34:25,857 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:34:27,342 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:34:27,343 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:34:27,343 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:34:27,348 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:34:27,552 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:34:28,555 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:34:29,314 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:34:29,314 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:34:29,315 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:34:31,364 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:34:31,364 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:34:31,364 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:34:31,364 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:34:31,369 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:34:31,557 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:34:32,560 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:34:35,196 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:34:35,197 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:34:35,197 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:34:35,202 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:34:35,562 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:34:36,563 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:34:37,203 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:34:39,469 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:34:39,470 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:34:39,470 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:34:39,475 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:34:39,565 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:34:40,567 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:34:42,475 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:34:44,314 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:34:44,314 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:34:44,315 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:34:47,607 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:34:47,753 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:34:47,754 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:34:47,754 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:34:47,759 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:34:48,572 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:34:50,575 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:34:51,701 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:34:51,701 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:34:51,702 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:34:51,707 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:34:52,576 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:34:52,707 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:34:54,581 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:34:54,860 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 16:34:57,236 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:34:57,237 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:34:57,237 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:34:57,251 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:34:57,582 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:34:58,252 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:34:58,585 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:34:59,314 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:34:59,314 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:34:59,314 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:35:02,758 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:35:02,762 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:35:02,762 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:35:02,767 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:35:03,588 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:35:03,801 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:35:04,588 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:35:08,801 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:35:10,138 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:35:10,139 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:35:10,139 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:35:10,144 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:35:10,592 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:35:12,595 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:35:14,145 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:35:14,314 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:35:14,314 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:35:14,315 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:35:15,203 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:35:15,203 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:35:15,203 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:35:15,208 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:35:15,597 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:35:16,600 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:35:18,892 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:35:18,893 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:35:18,893 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:35:18,897 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:35:19,601 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:35:19,898 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:35:20,604 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:35:22,753 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:35:22,754 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:35:22,754 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:35:22,759 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:35:23,606 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:35:24,608 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:35:24,861 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 16:35:25,862 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:35:29,315 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:35:29,315 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:35:29,316 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:35:31,246 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:35:31,247 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:35:31,247 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:35:31,247 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:35:31,252 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:35:31,612 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:35:32,615 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:35:35,196 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:35:35,196 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:35:35,197 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:35:35,202 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:35:35,618 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:35:36,620 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:35:37,202 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:35:41,373 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:35:41,373 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:35:41,374 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:35:41,379 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:35:41,623 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:35:42,379 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:35:42,626 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:35:44,314 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:35:44,315 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:35:44,315 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:35:45,882 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:35:45,883 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:35:45,883 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:35:45,888 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:35:46,628 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:35:47,925 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:35:48,631 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:35:50,781 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:35:50,782 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:35:50,783 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:35:50,787 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:35:51,633 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:35:52,636 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:35:53,788 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:35:54,864 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 16:35:55,413 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:35:55,414 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:35:55,414 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:35:55,419 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:35:55,639 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:35:56,641 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:35:59,314 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:35:59,315 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:35:59,315 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:35:59,614 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:36:00,387 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:36:00,388 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:36:00,388 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:36:00,393 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:36:00,643 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:36:02,647 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:36:04,526 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:36:04,527 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:36:04,527 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:36:04,532 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:36:04,648 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:36:05,532 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:36:06,651 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:36:09,614 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:36:09,615 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:36:09,616 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:36:09,620 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:36:09,653 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:36:10,621 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:36:10,656 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:36:13,629 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:36:13,630 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:36:13,630 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:36:13,635 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:36:13,658 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:36:14,314 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:36:14,315 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:36:14,315 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:36:14,660 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:36:16,534 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:36:21,535 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:36:22,063 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:36:22,064 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:36:22,064 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:36:22,069 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:36:22,665 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:36:24,668 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:36:24,871 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 16:36:25,932 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:36:25,933 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:36:25,933 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:36:25,938 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:36:26,670 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:36:26,939 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:36:28,673 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:36:29,314 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:36:29,315 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:36:29,315 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:36:29,988 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:36:29,989 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:36:29,989 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:36:29,994 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:36:30,674 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:36:32,033 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:36:32,677 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:36:34,626 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:36:34,626 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:36:34,627 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:36:34,632 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:36:34,678 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:36:36,683 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:36:37,632 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:36:39,908 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:36:39,909 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:36:39,909 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:36:39,914 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:36:40,685 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:36:42,688 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:36:42,915 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:36:43,871 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:36:43,872 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:36:43,873 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:36:43,877 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:36:44,315 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:36:44,315 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:36:44,315 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:36:44,689 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:36:46,692 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:36:47,901 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:36:47,901 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:36:47,901 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:36:47,907 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:36:47,945 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:36:48,694 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:36:50,697 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:36:52,263 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:36:52,263 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:36:52,264 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:36:52,268 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:36:52,698 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:36:53,269 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:36:54,702 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:36:54,871 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 16:36:56,668 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:36:56,668 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:36:56,669 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:36:56,673 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:36:56,703 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:36:58,674 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:36:58,707 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:36:59,315 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:36:59,315 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:36:59,315 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:37:01,231 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:37:01,232 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:37:01,232 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:37:01,237 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:37:01,708 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:37:02,711 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:37:04,273 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:37:06,164 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:37:06,165 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:37:06,165 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:37:06,170 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:37:06,713 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:37:08,717 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:37:10,171 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:37:14,315 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:37:14,315 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:37:14,315 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:37:15,541 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:37:15,895 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:37:15,896 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:37:15,896 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:37:15,900 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:37:16,721 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:37:18,724 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:37:20,261 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:37:20,262 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:37:20,262 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:37:20,267 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:37:20,725 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:37:21,267 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:37:22,728 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:37:24,095 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:37:24,096 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:37:24,096 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:37:24,101 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:37:24,730 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:37:24,876 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 16:37:26,731 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:37:26,877 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:37:28,240 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:37:28,241 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:37:28,241 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:37:28,246 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:37:28,732 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:37:29,315 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:37:29,315 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:37:29,315 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:37:30,735 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:37:32,560 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:37:34,695 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:37:34,696 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:37:34,696 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:37:34,701 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:37:34,738 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:37:36,742 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:37:37,737 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:37:38,623 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:37:38,624 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:37:38,624 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:37:38,629 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:37:38,748 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:37:40,751 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:37:43,576 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:37:43,576 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:37:43,577 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:37:43,577 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:37:43,583 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:37:43,753 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:37:44,315 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:37:44,315 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:37:44,317 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:37:44,755 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:37:48,589 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:37:49,009 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:37:49,010 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:37:49,010 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:37:49,015 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:37:49,758 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:37:50,761 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:37:54,053 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:37:54,096 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:37:54,097 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:37:54,097 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:37:54,102 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:37:54,764 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:37:54,875 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 16:37:56,767 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:37:58,062 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:37:58,063 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:37:58,063 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:37:58,068 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:37:58,768 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:37:59,068 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:37:59,315 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:37:59,315 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:37:59,317 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:38:00,772 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:38:02,074 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:38:02,074 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:38:02,074 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:38:02,079 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:38:02,773 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:38:04,117 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:38:04,774 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:38:07,322 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:38:07,322 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:38:07,323 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:38:07,328 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:38:07,776 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:38:08,778 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:38:09,328 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:38:11,556 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:38:11,556 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:38:11,557 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:38:11,562 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:38:11,780 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:38:12,783 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:38:14,315 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:38:14,316 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:38:14,316 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:38:14,514 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:38:16,580 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:38:16,580 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:38:16,580 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:38:16,585 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:38:16,785 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:38:18,788 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:38:19,625 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:38:23,495 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:38:23,496 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:38:23,496 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:38:23,501 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:38:23,791 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:38:24,793 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:38:24,883 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 16:38:24,885 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:38:27,659 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:38:27,659 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:38:27,660 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:38:27,664 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:38:27,795 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:38:28,796 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:38:29,315 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:38:29,316 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:38:29,316 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:38:30,584 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:38:32,841 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:38:32,842 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:38:32,842 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:38:32,847 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:38:33,799 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:38:34,802 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:38:35,885 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:38:37,651 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:38:37,652 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:38:37,652 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:38:37,657 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:38:37,811 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:38:38,813 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:38:41,658 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:38:43,135 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:38:43,135 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:38:43,136 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:38:43,141 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:38:43,816 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:38:44,315 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:38:44,316 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:38:44,317 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:38:44,819 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:38:47,536 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:38:47,588 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:38:47,589 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:38:47,589 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:38:47,594 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:38:47,821 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:38:48,823 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:38:52,633 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:38:53,289 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:38:53,290 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:38:53,290 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:38:53,295 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:38:53,828 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:38:54,830 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:38:54,880 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 16:38:57,881 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:38:58,170 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:38:58,170 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:38:58,171 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:38:58,175 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:38:58,832 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:38:59,315 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:38:59,316 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:38:59,316 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:39:00,835 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:39:02,334 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:39:02,335 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:39:02,335 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:39:02,340 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:39:02,837 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:39:03,377 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:39:04,840 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:39:06,221 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:39:06,222 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:39:06,222 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:39:06,227 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:39:06,841 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:39:08,844 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:39:09,227 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:39:12,283 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:39:12,284 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:39:12,284 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:39:12,289 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:39:12,846 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:39:14,290 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:39:14,316 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:39:14,316 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:39:14,317 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:39:14,850 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:39:16,144 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:39:16,145 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:39:16,145 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:39:16,150 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:39:16,851 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:39:18,854 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:39:20,189 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:39:21,660 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:39:21,661 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:39:21,661 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:39:21,666 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:39:21,856 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:39:22,858 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:39:24,883 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 16:39:25,876 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:39:25,877 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:39:25,877 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:39:25,877 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:39:25,882 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:39:26,863 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:39:26,863 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:39:29,249 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:39:29,250 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:39:29,250 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:39:29,255 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:39:29,316 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:39:29,316 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:39:29,316 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:39:29,864 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:39:30,867 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:39:31,568 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:39:32,594 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:39:32,595 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:39:32,595 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:39:32,600 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:39:32,868 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:39:34,872 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:39:36,637 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:39:37,468 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:39:37,469 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:39:37,469 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:39:37,474 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:39:37,875 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:39:38,877 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:39:42,474 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:39:44,316 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:39:44,316 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:39:44,316 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:39:45,278 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:39:45,282 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:39:45,282 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:39:45,287 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:39:45,881 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:39:46,884 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:39:48,321 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:39:49,583 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:39:49,584 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:39:49,584 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:39:49,590 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:39:49,886 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:39:50,888 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:39:53,505 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:39:53,506 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:39:53,506 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:39:53,506 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:39:53,511 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:39:53,891 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:39:54,885 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 16:39:54,894 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:39:57,903 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:39:57,904 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:39:57,904 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:39:57,909 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:39:58,898 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:39:58,898 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:39:58,909 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:39:59,316 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:39:59,316 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:39:59,316 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:40:02,060 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:40:02,061 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:40:02,061 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:40:02,066 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:40:02,900 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:40:04,105 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:40:04,903 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:40:06,077 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:40:06,078 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:40:06,078 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:40:06,083 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:40:06,905 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:40:08,908 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:40:10,084 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:40:14,316 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:40:14,316 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:40:14,316 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:40:15,564 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:40:17,037 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:40:17,038 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:40:17,038 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:40:17,043 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:40:17,913 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:40:18,916 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:40:20,860 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:40:20,861 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:40:20,861 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:40:20,861 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:40:20,866 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:40:20,917 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:40:22,920 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:40:24,887 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 16:40:25,889 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:40:26,388 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:40:26,389 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:40:26,389 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:40:26,394 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:40:26,922 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:40:28,925 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:40:29,316 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:40:29,316 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:40:29,317 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:40:31,622 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:40:32,050 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:40:32,050 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:40:32,050 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:40:32,055 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:40:32,928 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:40:34,931 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:40:37,069 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:40:37,070 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:40:37,070 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:40:37,070 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:40:37,075 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:40:37,934 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:40:38,936 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:40:42,076 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:40:42,890 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:40:42,891 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:40:42,891 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:40:42,896 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:40:42,939 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:40:44,316 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:40:44,317 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:40:44,317 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:40:44,942 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:40:47,559 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:40:47,998 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:40:47,999 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:40:47,999 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:40:48,004 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:40:48,944 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:40:50,947 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:40:51,941 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:40:51,942 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:40:51,942 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:40:51,951 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:40:51,951 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:40:52,951 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:40:52,953 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:40:54,893 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 16:40:57,250 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:40:57,250 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:40:57,251 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:40:57,256 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:40:57,957 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:40:58,256 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:40:58,960 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:40:59,316 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:40:59,317 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:40:59,317 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:41:01,451 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:41:01,452 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:41:01,452 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:41:01,457 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:41:01,961 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:41:02,964 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:41:03,497 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:41:06,246 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:41:06,246 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:41:06,247 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:41:06,251 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:41:06,966 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:41:08,970 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:41:09,252 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:41:10,986 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:41:10,987 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:41:10,987 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:41:10,992 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:41:11,971 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:41:12,974 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:41:14,316 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:41:14,317 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:41:14,317 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:41:14,564 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:41:16,241 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:41:16,241 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:41:16,242 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:41:16,246 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:41:16,976 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:41:18,979 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:41:20,285 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:41:24,743 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:41:24,744 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:41:24,744 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:41:24,749 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:41:24,891 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 16:41:24,983 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:41:25,893 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:41:26,986 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:41:28,869 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:41:28,870 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:41:28,870 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:41:28,875 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:41:28,987 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:41:29,317 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:41:29,318 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:41:29,318 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:41:30,990 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:41:31,584 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:41:36,584 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:41:37,804 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:41:37,805 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:41:37,805 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:41:37,810 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:41:37,996 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:41:38,998 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:41:41,849 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:41:42,594 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:41:42,595 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:41:42,595 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:41:42,600 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:41:43,001 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:41:44,317 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:41:44,318 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:41:44,318 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:41:45,004 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:41:47,236 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:41:47,237 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:41:47,237 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:41:47,237 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:41:47,242 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:41:48,005 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:41:49,008 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:41:51,442 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:41:51,442 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:41:51,443 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:41:51,447 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:41:52,009 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:41:52,448 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:41:53,012 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:41:54,897 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 16:41:56,415 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:41:56,416 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:41:56,416 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:41:56,421 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:41:57,015 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:41:58,421 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:41:59,018 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:41:59,317 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:41:59,317 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:41:59,318 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:42:03,541 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:42:04,268 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:42:04,269 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:42:04,269 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:42:04,274 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:42:05,022 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:42:07,025 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:42:08,593 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:42:08,593 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:42:08,594 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:42:08,594 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:42:08,598 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:42:09,027 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:42:11,030 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:42:13,599 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:42:14,317 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:42:14,318 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:42:14,318 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:42:17,021 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:42:17,022 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:42:17,022 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:42:17,027 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:42:17,033 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:42:19,036 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:42:19,065 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:42:21,062 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:42:21,063 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:42:21,063 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:42:21,068 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:42:22,037 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:42:23,040 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:42:24,068 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:42:24,896 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 16:42:24,991 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:42:24,991 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:42:24,992 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:42:24,997 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:42:25,041 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:42:27,044 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:42:29,317 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:42:29,318 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:42:29,318 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:42:29,612 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:42:31,063 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:42:31,063 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:42:31,063 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:42:31,068 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:42:32,047 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:42:33,050 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:42:35,105 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:42:35,288 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:42:35,288 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:42:35,289 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:42:35,293 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:42:36,053 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:42:37,055 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:42:40,294 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:42:40,689 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:42:40,689 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:42:40,689 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:42:40,694 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:42:41,057 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:42:43,060 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:42:44,317 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:42:44,317 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:42:44,318 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:42:44,464 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:42:44,596 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:42:44,596 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:42:44,601 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:42:45,062 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:42:45,602 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:42:47,065 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:42:48,682 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:42:48,682 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:42:48,683 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:42:48,688 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:42:49,066 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:42:50,688 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:42:51,067 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:42:52,450 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:42:52,451 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:42:52,451 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:42:52,456 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:42:53,068 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:42:54,897 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 16:42:55,072 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:42:55,898 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:42:58,834 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:42:58,835 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:42:58,835 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:42:58,840 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:42:59,074 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:42:59,317 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:42:59,318 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:42:59,318 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:43:01,077 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:43:01,642 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:43:02,791 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:43:02,792 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:43:02,792 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:43:02,796 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:43:03,079 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:43:05,082 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:43:06,833 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:43:06,953 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:43:06,953 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:43:06,954 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:43:06,959 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:43:07,083 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:43:09,086 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:43:10,718 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:43:10,719 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:43:10,719 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:43:10,724 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:43:11,087 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:43:12,725 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:43:13,090 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:43:14,317 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:43:14,317 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:43:14,319 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:43:15,302 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:43:15,303 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:43:15,303 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:43:15,308 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:43:16,092 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:43:17,095 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:43:18,345 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:43:18,971 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:43:18,972 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:43:18,972 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:43:18,977 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:43:19,096 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:43:21,099 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:43:23,386 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:43:23,386 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:43:23,387 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:43:23,387 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:43:23,392 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:43:24,100 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:43:24,899 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 16:43:25,103 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:43:27,331 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:43:27,331 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:43:27,332 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:43:27,337 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:43:28,104 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:43:29,105 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:43:29,318 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:43:29,318 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:43:29,318 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:43:29,550 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:43:32,574 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:43:32,575 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:43:32,575 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:43:32,580 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:43:33,108 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:43:34,617 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:43:35,111 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:43:36,145 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:43:36,146 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:43:36,146 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:43:36,151 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:43:37,115 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:43:37,115 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:43:39,765 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:43:39,766 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:43:39,766 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:43:39,766 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:43:39,771 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:43:40,117 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:43:41,119 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:43:43,528 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:43:43,529 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:43:43,529 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:43:43,534 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:43:44,121 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:43:44,317 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:43:44,318 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:43:44,318 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:43:45,122 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:43:45,573 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:43:47,498 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:43:47,498 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:43:47,498 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:43:47,504 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:43:48,123 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:43:49,126 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:43:51,224 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:43:51,224 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:43:51,225 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:43:51,225 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:43:51,229 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:43:52,128 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:43:53,130 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:43:54,902 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 16:43:55,397 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:43:55,397 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:43:55,398 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:43:55,403 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:43:56,133 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:43:56,403 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:43:57,135 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:43:59,317 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:43:59,318 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:43:59,318 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:43:59,598 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:43:59,837 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:43:59,837 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:43:59,842 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:44:00,137 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:44:01,139 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:44:01,842 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:44:04,572 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:44:04,573 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:44:04,573 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:44:04,578 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:44:05,142 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:44:07,143 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:44:07,579 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:44:08,130 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:44:08,131 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:44:08,131 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:44:08,136 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:44:08,146 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:44:09,149 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:44:13,137 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:44:14,318 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:44:14,318 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:44:14,319 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:44:14,610 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:44:14,611 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:44:14,611 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:44:14,616 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:44:15,155 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:44:17,160 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:44:18,392 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:44:18,393 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:44:18,393 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:44:18,393 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:44:18,398 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:44:19,161 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:44:21,166 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:44:22,815 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:44:22,816 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:44:22,816 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:44:22,821 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:44:23,167 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:44:23,821 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:44:24,904 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 16:44:25,172 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:44:27,236 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:44:27,237 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:44:27,237 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:44:27,245 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:44:28,175 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:44:29,176 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:44:29,245 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:44:29,318 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:44:29,318 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:44:29,318 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:44:31,193 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:44:31,194 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:44:31,194 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:44:31,198 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:44:32,181 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:44:33,184 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:44:35,237 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:44:35,654 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:44:35,655 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:44:35,655 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:44:35,660 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:44:36,188 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:44:37,191 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:44:39,188 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:44:39,189 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:44:39,189 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:44:39,194 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:44:39,194 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:44:41,195 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:44:41,197 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:44:44,241 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:44:44,242 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:44:44,242 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:44:44,247 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:44:44,318 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:44:44,318 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:44:44,318 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:44:45,203 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:44:45,203 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:44:46,549 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:44:48,565 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:44:48,566 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:44:48,566 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:44:48,571 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:44:49,207 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:44:51,212 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:44:51,609 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:44:54,906 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 16:44:55,930 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:44:55,931 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:44:55,931 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:44:55,936 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:44:56,218 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:44:56,936 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:44:57,218 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:44:59,318 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:44:59,318 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:44:59,318 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:45:00,159 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:45:00,159 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:45:00,160 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:45:00,165 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:45:00,223 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:45:01,226 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:45:02,201 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:45:04,463 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:45:04,464 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:45:04,464 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:45:04,469 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:45:05,230 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:45:07,234 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:45:07,470 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:45:10,285 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:45:10,286 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:45:10,287 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:45:10,291 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:45:11,240 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:45:13,245 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:45:13,292 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:45:14,318 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:45:14,318 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:45:14,318 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:45:14,689 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:45:14,690 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:45:14,690 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:45:14,695 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:45:15,246 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:45:17,251 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:45:18,733 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:45:18,917 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:45:18,918 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:45:18,918 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:45:18,923 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:45:19,252 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:45:21,257 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:45:22,924 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:45:22,924 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:45:22,925 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:45:22,930 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:45:23,258 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:45:23,930 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:45:24,911 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 16:45:25,262 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:45:28,660 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:45:28,661 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:45:28,661 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:45:28,666 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:45:29,266 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:45:29,318 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:45:29,318 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:45:29,319 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:45:29,582 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:45:31,271 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:45:33,901 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:45:33,902 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:45:33,902 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:45:33,907 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:45:34,274 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:45:34,945 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:45:35,276 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:45:37,770 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:45:37,771 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:45:37,771 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:45:37,776 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:45:38,281 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:45:39,283 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:45:40,777 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:45:43,177 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:45:43,178 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:45:43,178 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:45:43,183 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:45:43,287 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:45:44,318 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:45:44,319 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:45:44,319 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:45:45,292 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:45:46,574 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:45:51,440 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:45:51,441 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:45:51,441 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:45:51,446 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:45:52,301 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:45:52,485 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:45:53,303 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:45:54,910 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 16:45:55,776 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:45:55,777 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:45:55,777 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:45:55,782 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:45:56,306 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:45:57,309 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:45:57,783 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:45:59,318 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:45:59,319 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:45:59,320 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:45:59,533 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:45:59,617 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:45:59,617 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:45:59,622 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:46:00,312 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:46:01,315 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:46:03,623 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:46:03,762 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:46:03,762 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:46:03,762 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:46:03,768 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:46:04,318 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:46:05,320 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:46:08,231 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:46:08,232 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:46:08,232 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:46:08,238 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:46:08,324 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:46:09,238 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:46:09,325 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:46:12,828 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:46:12,829 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:46:12,829 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:46:12,834 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:46:13,331 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:46:14,318 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:46:14,319 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:46:14,320 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:46:14,540 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:46:15,335 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:46:16,777 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:46:16,777 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:46:16,777 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:46:16,782 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:46:17,337 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:46:19,342 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:46:19,821 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:46:21,634 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:46:21,635 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:46:21,635 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:46:21,641 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:46:22,345 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:46:23,347 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:46:24,915 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 16:46:24,916 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:46:27,004 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:46:27,005 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:46:27,005 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:46:27,010 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:46:27,351 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:46:29,318 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:46:29,319 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:46:29,320 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:46:29,356 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:46:30,625 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:46:31,190 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:46:31,191 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:46:31,191 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:46:31,196 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:46:31,357 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:46:33,361 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:46:35,183 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:46:35,184 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:46:35,184 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:46:35,189 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:46:35,363 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:46:36,189 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:46:37,368 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:46:38,399 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:46:38,400 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:46:38,400 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:46:38,405 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:46:39,369 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:46:41,374 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:46:41,405 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:46:42,126 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:46:42,127 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:46:42,127 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:46:42,132 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:46:42,374 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:46:43,377 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:46:44,319 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:46:44,319 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:46:44,319 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:46:46,584 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:46:46,799 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:46:46,800 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:46:46,800 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:46:46,805 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:46:47,381 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:46:49,385 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:46:50,938 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:46:50,939 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:46:50,939 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:46:50,944 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:46:51,387 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:46:51,944 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:46:53,392 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:46:54,601 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:46:54,602 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:46:54,602 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:46:54,607 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:46:54,914 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 16:46:55,394 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:46:57,398 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:46:57,915 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:46:59,319 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:46:59,319 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:46:59,320 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:46:59,564 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:46:59,592 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:46:59,592 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:46:59,597 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:47:00,401 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:47:01,403 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:47:03,598 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:47:05,723 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:47:05,724 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:47:05,724 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:47:05,729 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:47:06,407 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:47:07,410 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:47:08,769 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:47:10,793 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:47:10,794 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:47:10,794 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:47:10,800 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:47:11,416 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:47:13,421 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:47:13,801 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:47:14,319 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:47:14,319 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:47:14,320 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:47:16,066 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:47:16,066 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:47:16,067 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:47:16,071 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:47:16,425 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:47:17,428 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:47:19,109 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:47:21,155 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:47:21,155 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:47:21,156 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:47:21,161 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:47:21,432 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:47:23,437 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:47:24,161 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:47:24,921 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 16:47:26,147 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:47:26,148 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:47:26,148 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:47:26,153 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:47:26,441 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:47:27,443 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:47:29,319 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:47:29,319 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:47:29,320 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:47:29,541 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:47:29,897 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:47:29,897 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:47:29,897 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:47:29,902 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:47:30,447 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:47:31,449 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:47:34,013 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:47:34,013 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:47:34,014 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:47:34,018 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:47:34,452 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:47:35,019 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:47:35,455 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:47:40,019 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:47:42,635 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:47:42,636 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:47:42,636 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:47:42,643 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:47:43,465 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:47:44,319 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:47:44,319 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:47:44,321 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:47:45,468 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:47:45,629 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:47:47,575 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:47:47,576 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:47:47,576 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:47:47,581 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:47:48,473 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:47:49,476 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:47:51,621 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:47:51,638 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:47:51,639 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:47:51,639 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:47:51,644 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:47:52,479 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:47:53,482 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:47:54,918 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 16:47:55,919 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:47:55,920 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:47:55,920 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:47:55,925 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:47:56,485 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:47:56,926 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:47:57,488 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:47:59,319 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:47:59,319 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:47:59,321 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:48:01,829 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:48:01,830 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:48:01,830 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:48:01,835 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:48:02,494 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:48:02,873 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:48:03,496 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:48:06,177 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:48:06,177 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:48:06,178 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:48:06,183 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:48:06,498 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:48:07,501 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:48:08,183 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:48:11,716 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:48:11,717 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:48:11,717 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:48:11,722 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:48:12,509 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:48:13,509 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:48:13,723 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:48:14,319 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:48:14,319 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:48:14,321 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:48:17,172 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:48:17,172 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:48:17,172 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:48:17,177 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:48:17,515 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:48:19,217 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:48:19,520 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:48:21,562 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:48:21,563 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:48:21,563 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:48:21,568 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:48:22,524 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:48:23,527 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:48:24,568 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:48:24,919 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 16:48:26,983 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:48:26,984 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:48:26,984 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:48:26,989 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:48:27,530 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:48:29,319 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:48:29,319 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:48:29,321 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:48:29,541 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:48:30,541 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:48:35,356 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:48:35,357 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:48:35,357 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:48:35,362 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:48:35,545 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:48:36,401 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:48:37,549 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:48:39,167 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:48:39,167 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:48:39,168 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:48:39,173 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:48:39,550 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:48:41,552 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:48:42,173 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:48:43,783 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:48:43,784 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:48:43,784 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:48:43,789 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:48:44,319 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:48:44,319 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:48:44,321 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:48:44,556 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:48:45,559 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:48:47,601 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:48:50,700 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:48:50,701 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:48:50,701 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:48:50,706 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:48:51,566 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:48:52,745 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:48:53,570 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:48:54,925 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 16:48:56,835 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:48:56,835 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:48:56,836 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:48:56,840 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:48:57,574 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:48:57,841 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:48:59,319 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:48:59,320 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:48:59,321 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:48:59,579 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:49:01,928 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:49:01,928 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:49:01,928 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:49:01,933 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:49:02,582 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:49:02,973 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:49:03,583 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:49:07,544 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:49:07,545 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:49:07,545 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:49:07,549 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:49:07,587 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:49:08,550 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:49:09,591 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:49:11,038 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:49:11,038 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:49:11,039 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:49:11,044 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:49:11,592 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:49:13,595 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:49:14,044 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:49:14,319 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:49:14,320 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:49:14,321 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:49:16,351 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:49:16,351 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:49:16,352 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:49:16,356 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:49:16,597 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:49:17,599 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:49:19,393 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:49:20,873 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:49:20,874 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:49:20,874 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:49:20,879 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:49:21,602 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:49:23,605 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:49:24,880 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:49:24,924 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 16:49:26,261 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:49:26,262 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:49:26,262 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:49:26,267 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:49:26,606 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:49:27,609 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:49:29,319 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:49:29,320 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:49:29,321 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:49:30,602 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:49:31,360 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:49:31,360 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:49:31,361 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:49:31,365 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:49:31,611 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:49:33,619 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:49:35,509 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:49:35,510 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:49:35,510 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:49:35,514 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:49:35,620 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:49:36,515 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:49:37,625 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:49:39,334 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:49:39,335 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:49:39,335 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:49:39,339 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:49:39,626 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:49:41,627 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:49:42,340 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:49:44,320 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:49:44,320 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:49:44,321 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:49:45,787 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:49:45,788 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:49:45,788 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:49:45,793 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:49:46,630 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:49:47,633 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:49:47,833 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:49:52,833 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:49:54,044 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:49:54,045 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:49:54,045 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:49:54,050 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:49:54,638 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:49:54,928 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 16:49:55,640 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:49:57,608 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:49:57,609 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:49:57,609 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:49:57,614 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:49:57,641 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:49:58,614 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:49:59,320 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:49:59,320 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:49:59,321 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:49:59,645 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:50:01,938 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:50:01,938 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:50:01,939 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:50:01,943 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:50:02,646 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:50:03,647 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:50:03,981 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:50:06,370 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:50:06,371 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:50:06,371 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:50:06,376 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:50:06,649 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:50:07,651 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:50:09,376 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:50:11,624 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:50:11,624 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:50:11,625 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:50:11,629 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:50:11,654 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:50:13,657 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:50:14,320 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:50:14,320 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:50:14,321 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:50:14,597 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:50:15,585 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:50:15,585 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:50:15,585 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:50:15,590 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:50:15,659 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:50:17,662 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:50:19,629 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:50:20,800 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:50:20,801 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:50:20,801 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:50:20,806 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:50:21,664 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:50:23,667 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:50:24,807 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:50:24,928 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 16:50:25,638 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:50:25,638 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:50:25,639 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:50:25,644 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:50:25,668 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:50:27,671 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:50:29,320 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:50:29,320 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:50:29,321 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:50:29,887 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:50:29,888 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:50:29,888 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:50:29,888 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:50:29,893 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:50:30,673 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:50:31,676 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:50:33,772 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:50:33,773 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:50:33,773 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:50:33,778 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:50:34,678 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:50:35,680 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:50:35,779 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:50:38,322 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:50:38,323 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:50:38,323 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:50:38,328 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:50:38,683 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:50:39,686 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:50:41,328 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:50:43,369 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:50:43,369 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:50:43,369 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:50:43,375 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:50:43,688 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:50:44,320 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:50:44,320 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:50:44,321 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:50:45,692 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:50:46,616 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:50:47,307 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:50:47,308 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:50:47,308 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:50:47,319 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:50:47,693 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:50:49,694 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:50:52,353 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:50:53,002 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:50:53,003 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:50:53,003 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:50:53,008 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:50:53,696 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:50:54,933 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 16:50:55,701 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:50:56,759 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:50:56,760 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:50:56,760 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:50:56,765 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:50:57,702 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:50:57,765 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:50:59,320 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:50:59,320 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:50:59,322 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:50:59,705 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:51:00,796 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:51:00,797 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:51:00,797 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:51:00,801 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:51:01,706 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:51:02,841 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:51:03,710 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:51:05,776 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:51:05,777 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:51:05,777 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:51:05,787 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:51:06,711 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:51:07,714 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:51:08,788 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:51:09,789 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:51:09,790 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:51:09,790 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:51:09,795 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:51:10,717 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:51:11,718 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:51:13,796 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:51:13,914 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:51:13,915 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:51:13,915 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:51:13,920 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:51:14,320 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:51:14,320 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:51:14,322 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:51:14,719 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:51:15,722 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:51:19,688 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:51:20,160 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:51:20,161 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:51:20,161 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:51:20,166 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:51:20,725 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:51:21,728 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:51:23,995 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:51:23,996 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:51:23,996 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:51:24,001 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:51:24,729 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:51:24,932 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 16:51:24,933 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:51:25,732 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:51:27,435 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:51:27,436 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:51:27,436 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:51:27,441 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:51:27,733 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:51:29,320 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:51:29,320 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:51:29,322 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:51:29,736 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:51:30,517 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:51:31,658 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:51:31,659 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:51:31,659 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:51:31,664 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:51:31,738 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:51:33,741 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:51:35,701 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:51:37,570 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:51:37,571 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:51:37,571 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:51:37,576 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:51:37,745 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:51:39,748 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:51:41,328 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:51:41,329 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:51:41,329 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:51:41,329 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:51:41,334 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:51:41,749 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:51:43,752 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:51:44,320 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:51:44,321 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:51:44,322 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:51:45,702 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:51:45,703 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:51:45,703 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:51:45,708 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:51:45,754 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:51:46,745 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:51:47,757 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:51:49,463 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:51:49,463 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:51:49,464 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:51:49,468 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:51:49,758 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:51:51,761 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:51:52,469 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:51:53,688 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:51:53,689 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:51:53,689 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:51:53,694 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:51:53,762 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:51:54,937 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 16:51:55,767 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:51:57,939 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:51:59,320 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:51:59,321 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:51:59,322 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:51:59,765 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:51:59,766 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:51:59,766 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:51:59,769 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:51:59,772 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:52:00,770 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:52:01,772 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:52:03,639 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:52:03,640 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:52:03,640 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:52:03,640 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:52:03,645 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:52:03,774 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:52:05,777 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:52:08,646 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:52:09,429 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:52:09,430 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:52:09,430 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:52:09,435 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:52:09,780 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:52:11,783 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:52:13,774 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:52:13,774 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:52:13,775 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:52:13,775 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:52:13,779 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:52:13,784 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:52:14,320 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:52:14,321 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:52:14,322 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:52:15,788 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:52:17,688 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:52:17,689 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:52:17,689 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:52:17,694 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:52:17,789 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:52:19,733 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:52:19,792 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:52:21,558 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:52:21,558 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:52:21,558 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:52:21,564 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:52:21,794 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:52:23,797 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:52:24,936 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 16:52:24,938 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:52:29,321 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:52:29,321 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:52:29,322 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:52:29,651 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:52:29,652 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:52:29,652 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:52:29,657 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:52:29,800 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:52:30,697 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:52:31,803 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:52:32,918 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:52:32,919 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:52:32,919 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:52:32,924 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:52:33,805 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:52:35,808 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:52:35,925 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:52:39,196 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:52:39,197 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:52:39,197 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:52:39,201 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:52:39,812 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:52:41,202 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:52:41,815 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:52:44,321 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:52:44,321 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:52:44,322 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:52:44,827 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:52:44,828 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:52:44,828 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:52:44,833 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:52:45,817 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:52:46,873 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:52:47,821 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:52:49,686 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:52:49,687 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:52:49,687 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:52:49,692 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:52:49,822 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:52:51,827 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:52:52,693 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:52:54,938 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 16:52:55,923 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:52:55,924 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:52:55,924 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:52:55,929 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:52:56,831 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:52:57,832 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:52:57,930 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:52:59,321 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:52:59,321 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:52:59,322 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:52:59,796 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:52:59,797 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:52:59,797 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:52:59,802 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:52:59,833 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:53:01,837 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:53:03,841 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:53:04,330 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:53:04,331 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:53:04,331 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:53:04,336 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:53:04,838 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:53:05,841 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:53:08,183 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:53:08,184 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:53:08,184 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:53:08,189 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:53:08,843 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:53:09,189 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:53:09,846 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:53:14,190 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:53:14,321 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:53:14,321 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:53:14,322 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:53:16,661 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:53:16,661 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:53:16,661 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:53:16,666 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:53:16,849 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:53:17,852 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:53:19,705 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:53:21,841 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:53:21,842 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:53:21,842 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:53:21,847 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:53:21,854 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:53:23,857 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:53:24,847 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:53:24,943 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 16:53:26,253 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:53:26,254 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:53:26,254 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:53:26,259 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:53:26,859 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:53:27,862 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:53:29,321 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:53:29,321 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:53:29,323 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:53:30,613 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:53:32,283 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:53:32,284 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:53:32,284 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:53:32,289 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:53:32,865 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:53:33,867 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:53:36,329 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:53:37,212 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:53:37,213 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:53:37,213 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:53:37,218 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:53:37,871 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:53:39,874 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:53:41,142 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:53:41,143 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:53:41,143 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:53:41,148 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:53:41,875 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:53:42,149 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:53:43,878 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:53:44,321 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:53:44,321 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:53:44,323 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:53:45,376 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:53:45,377 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:53:45,377 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:53:45,382 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:53:45,879 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:53:47,421 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:53:47,882 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:53:50,009 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:53:50,010 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:53:50,010 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:53:50,015 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:53:50,884 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:53:51,887 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:53:53,016 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:53:54,426 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:53:54,427 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:53:54,427 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:53:54,432 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:53:54,890 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:53:54,942 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 16:53:55,892 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:53:58,944 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:53:59,314 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:53:59,315 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:53:59,316 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:53:59,320 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:53:59,321 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:53:59,321 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:53:59,322 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:53:59,895 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:54:01,898 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:54:04,626 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:54:07,828 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:54:07,829 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:54:07,829 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:54:07,834 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:54:07,901 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:54:09,873 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:54:09,905 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:54:12,116 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:54:12,117 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:54:12,117 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:54:12,122 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:54:12,906 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:54:13,909 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:54:14,321 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:54:14,321 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:54:14,323 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:54:15,586 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:54:17,665 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:54:17,666 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:54:17,666 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:54:17,671 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:54:17,911 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:54:19,914 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:54:20,709 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:54:21,504 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:54:21,505 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:54:21,505 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:54:21,510 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:54:21,916 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:54:23,919 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:54:24,947 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 16:54:25,673 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:54:25,673 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:54:25,673 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:54:25,678 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:54:25,920 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:54:26,679 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:54:27,923 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:54:29,321 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:54:29,322 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:54:29,323 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:54:31,208 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:54:31,209 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:54:31,209 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:54:31,214 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:54:31,926 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:54:32,253 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:54:33,929 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:54:37,253 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:54:37,578 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:54:37,579 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:54:37,579 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:54:37,584 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:54:37,933 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:54:39,936 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:54:42,585 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:54:43,987 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:54:43,988 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:54:43,988 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:54:43,993 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:54:44,321 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:54:44,322 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:54:44,323 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:54:44,939 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:54:45,941 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:54:48,561 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:54:49,378 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:54:49,379 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:54:49,379 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:54:49,384 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:54:49,944 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:54:51,947 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:54:54,083 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:54:54,084 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:54:54,084 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:54:54,084 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:54:54,089 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:54:54,947 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 16:54:54,950 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:54:55,953 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:54:58,077 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:54:58,077 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:54:58,078 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:54:58,082 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:54:58,955 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:54:59,321 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:54:59,322 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:54:59,323 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:54:59,605 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:54:59,955 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:55:02,567 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:55:02,568 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:55:02,568 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:55:02,573 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:55:02,957 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:55:03,960 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:55:04,613 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:55:08,015 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:55:08,016 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:55:08,016 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:55:08,021 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:55:08,964 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:55:09,965 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:55:10,021 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:55:12,485 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:55:12,486 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:55:12,486 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:55:12,491 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:55:12,966 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:55:13,969 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:55:14,322 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:55:14,322 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:55:14,323 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:55:15,580 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:55:16,671 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:55:16,672 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:55:16,672 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:55:16,677 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:55:16,971 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:55:17,973 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:55:20,656 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:55:20,657 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:55:20,657 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:55:20,657 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:55:20,662 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:55:20,975 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:55:21,977 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:55:24,951 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 16:55:25,953 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:55:26,062 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:55:26,063 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:55:26,063 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:55:26,068 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:55:26,980 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:55:27,983 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:55:29,322 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:55:29,322 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:55:29,323 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:55:31,612 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:55:32,067 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:55:32,067 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:55:32,069 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:55:32,074 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:55:32,986 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:55:33,989 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:55:36,297 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:55:36,298 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:55:36,298 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:55:36,303 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:55:36,992 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:55:37,303 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:55:37,995 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:55:40,452 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:55:40,453 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:55:40,453 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:55:40,458 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:55:40,996 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:55:41,999 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:55:42,458 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:55:44,322 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:55:44,322 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:55:44,323 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:55:44,719 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:55:44,719 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:55:44,720 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:55:44,724 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:55:45,001 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:55:46,003 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:55:47,761 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:55:49,079 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:55:49,079 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:55:49,080 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:55:49,084 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:55:50,006 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:55:52,009 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:55:52,859 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:55:52,860 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:55:52,860 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:55:52,860 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:55:52,864 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:55:53,009 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:55:54,012 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:55:54,951 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 16:55:57,016 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:55:57,017 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:55:57,017 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:55:57,022 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:55:58,016 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:55:58,023 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:55:59,322 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:55:59,322 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:55:59,324 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:56:00,020 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:56:01,217 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:56:01,218 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:56:01,218 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:56:01,223 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:56:02,021 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:56:03,261 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:56:04,024 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:56:04,967 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:56:04,968 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:56:04,968 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:56:04,973 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:56:05,025 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:56:06,028 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:56:08,814 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:56:08,815 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:56:08,815 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:56:08,815 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:56:08,820 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:56:09,030 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:56:10,032 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:56:13,789 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:56:13,790 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:56:13,791 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:56:13,795 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:56:14,035 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:56:14,322 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:56:14,322 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:56:14,324 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:56:14,617 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:56:16,038 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:56:19,278 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:56:19,279 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:56:19,279 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:56:19,285 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:56:20,040 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:56:20,321 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:56:22,044 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:56:23,171 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:56:23,172 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:56:23,172 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:56:23,177 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:56:24,045 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:56:24,953 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 16:56:25,954 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:56:26,048 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:56:28,385 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:56:28,386 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:56:28,386 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:56:28,391 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:56:29,050 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:56:29,322 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:56:29,322 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:56:29,324 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:56:30,053 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:56:31,536 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:56:32,372 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:56:32,373 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:56:32,373 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:56:32,378 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:56:33,055 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:56:34,058 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:56:37,417 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:56:41,824 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:56:41,824 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:56:41,825 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:56:41,829 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:56:42,064 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:56:42,830 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:56:44,067 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:56:44,322 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:56:44,322 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:56:44,324 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:56:47,044 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:56:47,045 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:56:47,045 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:56:47,050 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:56:47,069 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:56:48,089 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:56:50,072 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:56:51,438 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:56:51,439 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:56:51,439 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:56:51,444 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:56:52,074 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:56:53,445 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:56:54,077 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:56:54,955 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 16:56:55,753 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:56:55,754 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:56:55,754 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:56:55,759 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:56:56,079 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:56:58,082 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:56:58,760 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:56:59,322 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:56:59,322 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:56:59,324 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:57:01,575 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:57:01,576 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:57:01,576 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:57:01,580 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:57:02,084 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:57:04,086 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:57:04,621 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:57:05,574 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:57:05,575 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:57:05,575 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:57:05,580 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:57:06,087 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:57:08,090 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:57:10,581 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:57:13,950 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:57:13,951 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:57:13,951 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:57:13,957 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:57:14,094 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:57:14,322 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:57:14,323 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:57:14,324 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:57:16,095 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:57:16,536 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:57:17,644 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:57:17,645 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:57:17,645 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:57:17,650 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:57:18,097 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:57:20,101 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:57:21,689 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:57:21,777 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:57:21,778 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:57:21,778 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:57:21,783 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:57:22,102 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:57:24,105 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:57:24,957 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 16:57:25,913 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:57:25,914 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:57:25,914 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:57:25,919 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:57:26,106 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:57:26,919 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:57:28,110 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:57:29,323 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:57:29,323 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:57:29,324 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:57:29,763 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:57:29,763 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:57:29,763 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:57:29,768 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:57:30,111 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:57:32,114 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:57:32,805 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:57:35,127 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:57:35,128 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:57:35,128 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:57:35,133 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:57:36,117 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:57:38,119 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:57:38,134 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:57:40,751 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:57:40,752 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:57:40,752 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:57:40,757 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:57:41,121 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:57:42,124 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:57:43,758 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:57:44,322 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:57:44,323 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:57:44,324 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:57:46,286 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:57:46,287 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:57:46,287 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:57:46,292 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:57:47,127 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:57:48,130 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:57:49,329 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:57:52,964 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:57:52,964 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:57:52,965 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:57:52,969 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:57:53,133 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:57:54,135 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:57:54,962 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 16:57:54,964 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:57:57,657 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:57:57,657 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:57:57,657 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:57:57,663 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:57:58,139 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:57:59,323 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:57:59,323 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:57:59,324 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:58:00,141 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:58:00,642 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:58:02,525 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:58:02,525 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:58:02,526 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:58:02,531 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:58:03,142 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:58:04,145 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:58:06,569 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:58:08,833 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:58:08,833 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:58:08,834 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:58:08,839 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:58:09,148 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:58:10,151 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:58:11,840 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:58:12,809 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:58:12,809 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:58:12,810 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:58:12,815 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:58:13,153 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:58:14,155 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:58:14,323 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:58:14,323 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:58:14,324 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:58:16,509 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:58:16,509 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:58:16,509 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:58:16,514 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:58:17,157 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:58:17,553 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:58:18,160 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:58:20,482 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:58:20,482 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:58:20,483 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:58:20,488 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:58:21,162 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:58:22,164 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:58:23,488 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:58:24,961 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 16:58:24,991 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:58:24,992 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:58:24,992 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:58:24,997 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:58:25,166 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:58:26,169 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:58:28,998 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:58:29,323 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:58:29,323 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:58:29,325 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:58:30,100 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:58:30,101 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:58:30,101 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:58:30,106 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:58:30,171 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:58:32,175 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:58:34,145 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:58:35,292 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:58:35,293 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:58:35,293 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:58:35,298 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:58:36,177 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:58:38,182 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:58:39,299 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:58:39,531 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:58:39,532 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:58:39,532 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:58:39,538 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:58:40,183 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:58:42,186 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:58:43,451 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:58:43,451 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:58:43,452 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:58:43,457 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:58:44,188 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:58:44,323 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:58:44,323 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:58:44,323 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:58:44,627 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:58:46,191 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:58:47,645 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:58:47,645 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:58:47,645 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:58:47,650 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:58:48,192 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:58:49,689 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:58:50,195 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:58:51,845 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:58:51,846 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:58:51,846 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:58:51,851 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:58:52,197 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:58:54,200 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:58:54,852 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:58:54,966 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 16:58:55,974 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:58:55,974 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:58:55,974 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:58:55,979 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:58:56,202 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:58:58,206 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:58:59,323 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:58:59,324 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:58:59,324 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:58:59,851 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:58:59,852 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:58:59,852 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:58:59,852 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:58:59,857 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:59:00,207 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:59:02,210 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:59:04,177 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:59:04,178 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:59:04,178 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:59:04,183 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:59:04,211 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:59:05,184 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:59:06,214 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:59:08,234 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:59:08,235 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:59:08,235 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:59:08,241 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:59:09,217 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:59:10,219 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:59:10,241 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:59:12,550 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:59:12,551 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:59:12,551 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:59:12,557 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:59:13,221 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:59:14,224 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:59:14,323 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:59:14,324 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:59:14,324 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:59:15,614 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:59:17,241 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:59:17,242 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:59:17,242 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:59:17,247 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:59:18,226 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:59:20,229 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:59:21,285 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:59:22,493 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:59:22,494 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:59:22,494 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:59:22,499 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:59:23,231 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:59:24,234 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:59:24,965 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 16:59:26,438 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:59:26,439 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:59:26,439 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:59:26,439 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:59:26,444 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:59:27,235 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:59:28,238 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:59:29,323 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:59:29,324 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:59:29,325 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:59:30,476 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:59:30,477 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:59:30,477 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:59:30,482 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:59:31,240 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:59:31,521 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:59:32,242 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:59:35,075 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:59:35,076 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:59:35,076 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:59:35,081 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:59:35,244 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:59:36,247 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:59:37,082 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:59:42,082 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:59:43,222 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:59:43,223 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:59:43,223 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:59:43,229 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:59:43,252 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:59:44,255 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:59:44,323 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:59:44,323 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:59:44,325 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:59:47,595 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:59:48,084 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:59:48,085 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:59:48,085 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:59:48,090 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:59:48,257 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:59:50,260 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:59:53,129 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:59:53,478 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:59:53,479 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:59:53,479 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:59:53,484 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:59:54,263 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 16:59:54,970 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 16:59:56,267 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 16:59:58,972 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:59:59,323 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:59:59,324 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:59:59,325 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:59:59,454 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:59:59,580 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 16:59:59,580 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:59:59,585 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:00:00,270 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:00:02,273 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:00:03,986 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:00:03,986 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:00:03,986 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:00:03,987 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:00:03,992 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:00:04,274 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:00:06,278 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:00:08,868 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:00:08,869 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:00:08,869 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:00:08,874 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:00:09,280 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:00:09,874 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:00:10,283 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:00:14,323 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:00:14,324 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:00:14,325 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:00:14,947 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:00:14,947 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:00:14,947 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:00:14,947 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:00:14,953 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:00:15,286 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:00:16,288 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:00:18,477 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:00:18,478 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:00:18,478 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:00:18,483 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:00:19,290 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:00:20,291 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:00:20,484 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:00:24,112 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:00:24,113 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:00:24,113 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:00:24,118 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:00:24,293 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:00:24,969 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 17:00:25,971 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:00:26,298 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:00:28,739 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:00:28,740 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:00:28,741 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:00:28,746 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:00:29,300 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:00:29,324 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:00:29,324 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:00:29,325 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:00:30,303 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:00:31,569 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:00:32,942 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:00:32,943 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:00:32,943 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:00:32,948 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:00:33,305 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:00:34,308 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:00:36,985 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:00:37,163 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:00:37,163 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:00:37,164 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:00:37,169 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:00:37,311 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:00:38,314 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:00:42,170 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:00:42,915 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:00:42,915 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:00:42,916 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:00:42,921 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:00:43,317 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:00:44,319 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:00:44,324 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:00:44,324 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:00:44,325 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:00:47,560 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:00:48,369 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:00:48,370 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:00:48,370 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:00:48,375 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:00:49,322 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:00:50,325 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:00:52,630 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:00:52,630 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:00:52,631 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:00:52,631 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:00:52,636 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:00:53,327 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:00:54,329 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:00:54,972 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 17:00:56,956 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:00:56,957 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:00:56,957 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:00:56,962 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:00:57,332 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:00:57,962 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:00:58,333 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:00:59,324 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:00:59,324 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:00:59,324 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:01:02,624 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:01:02,625 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:01:02,625 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:01:02,629 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:01:03,336 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:01:03,672 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:01:04,338 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:01:07,988 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:01:07,989 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:01:07,989 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:01:07,994 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:01:08,341 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:01:08,995 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:01:10,344 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:01:12,957 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:01:12,958 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:01:12,958 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:01:12,964 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:01:13,346 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:01:14,324 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:01:14,324 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:01:14,326 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:01:14,347 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:01:14,577 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:01:16,906 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:01:16,907 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:01:16,907 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:01:16,912 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:01:17,348 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:01:18,351 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:01:19,949 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:01:22,006 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:01:22,006 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:01:22,007 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:01:22,012 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:01:22,353 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:01:24,356 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:01:24,974 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 17:01:24,975 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:01:25,904 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:01:25,904 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:01:25,905 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:01:25,910 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:01:26,357 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:01:28,360 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:01:29,207 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:01:29,207 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:01:29,208 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:01:29,213 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:01:29,324 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:01:29,324 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:01:29,324 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:01:29,361 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:01:30,364 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:01:30,589 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:01:34,072 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:01:34,073 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:01:34,073 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:01:34,078 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:01:34,367 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:01:36,117 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:01:36,370 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:01:40,761 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:01:40,762 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:01:40,762 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:01:40,767 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:01:41,374 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:01:41,768 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:01:42,376 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:01:44,324 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:01:44,324 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:01:44,326 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:01:44,774 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:01:44,775 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:01:44,775 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:01:44,780 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:01:45,378 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:01:46,379 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:01:46,817 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:01:48,305 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:01:48,306 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:01:48,306 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:01:48,311 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:01:48,380 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:01:50,383 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:01:51,756 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:01:51,756 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:01:51,757 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:01:51,762 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:01:52,385 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:01:52,762 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:01:54,389 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:01:54,976 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 17:01:55,850 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:01:55,850 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:01:55,851 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:01:55,856 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:01:56,391 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:01:57,856 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:01:58,394 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:01:59,324 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:01:59,324 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:01:59,326 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:01:59,559 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:01:59,560 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:01:59,560 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:01:59,565 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:02:00,395 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:02:02,398 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:02:03,605 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:02:08,592 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:02:08,593 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:02:08,593 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:02:08,598 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:02:09,403 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:02:09,598 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:02:10,406 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:02:13,433 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:02:13,434 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:02:13,434 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:02:13,439 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:02:14,324 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:02:14,325 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:02:14,325 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:02:14,408 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:02:15,574 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:02:16,411 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:02:16,976 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:02:16,977 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:02:16,977 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:02:16,982 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:02:17,412 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:02:18,414 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:02:21,021 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:02:21,582 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:02:21,586 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:02:21,586 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:02:21,592 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:02:22,417 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:02:24,420 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:02:24,978 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 17:02:25,549 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:02:25,549 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:02:25,550 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:02:25,555 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:02:26,421 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:02:26,555 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:02:28,424 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:02:29,302 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:02:29,303 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:02:29,303 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:02:29,308 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:02:29,325 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:02:29,325 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:02:29,326 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:02:29,425 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:02:30,428 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:02:32,521 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:02:33,582 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:02:33,583 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:02:33,583 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:02:33,588 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:02:34,430 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:02:36,434 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:02:37,625 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:02:38,037 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:02:38,037 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:02:38,038 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:02:38,042 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:02:38,436 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:02:40,439 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:02:42,360 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:02:42,361 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:02:42,361 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:02:42,366 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:02:42,440 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:02:43,367 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:02:44,324 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:02:44,325 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:02:44,326 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:02:44,443 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:02:47,589 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:02:47,590 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:02:47,590 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:02:47,595 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:02:48,446 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:02:48,633 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:02:50,449 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:02:51,727 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:02:51,728 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:02:51,728 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:02:51,733 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:02:52,450 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:02:53,734 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:02:54,453 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:02:54,981 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 17:02:58,184 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:02:58,184 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:02:58,185 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:02:58,191 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:02:58,457 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:02:59,191 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:02:59,325 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:02:59,325 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:02:59,326 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:03:00,460 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:03:02,071 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:03:02,071 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:03:02,071 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:03:02,077 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:03:02,461 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:03:04,462 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:03:05,113 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:03:05,880 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:03:05,881 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:03:05,881 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:03:05,886 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:03:06,464 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:03:08,467 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:03:10,887 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:03:13,866 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:03:13,866 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:03:13,867 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:03:13,871 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:03:14,325 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:03:14,325 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:03:14,325 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:03:14,471 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:03:16,472 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:03:16,538 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:03:18,400 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:03:18,401 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:03:18,401 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:03:18,406 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:03:18,473 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:03:20,476 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:03:22,445 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:03:24,987 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 17:03:26,637 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:03:26,638 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:03:26,638 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:03:26,643 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:03:27,480 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:03:27,643 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:03:28,483 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:03:29,325 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:03:29,325 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:03:29,325 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:03:31,432 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:03:31,432 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:03:31,433 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:03:31,437 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:03:31,485 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:03:32,487 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:03:33,477 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:03:37,501 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:03:37,502 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:03:37,502 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:03:37,507 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:03:38,494 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:03:38,495 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:03:38,507 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:03:40,933 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:03:40,934 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:03:40,934 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:03:40,938 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:03:41,496 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:03:42,499 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:03:43,939 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:03:44,325 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:03:44,325 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:03:44,326 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:03:45,961 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:03:45,961 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:03:45,962 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:03:45,967 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:03:46,501 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:03:48,505 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:03:49,005 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:03:51,017 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:03:51,018 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:03:51,018 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:03:51,023 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:03:51,506 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:03:52,509 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:03:54,023 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:03:54,804 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:03:54,805 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:03:54,805 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:03:54,810 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:03:54,986 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 17:03:55,512 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:03:56,515 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:03:59,325 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:03:59,325 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:03:59,327 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:03:59,544 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:04:01,371 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:04:01,372 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:04:01,372 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:04:01,377 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:04:01,518 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:04:02,521 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:04:05,417 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:04:06,614 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:04:06,614 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:04:06,615 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:04:06,620 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:04:07,524 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:04:08,527 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:04:10,620 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:04:10,731 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:04:10,732 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:04:10,732 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:04:10,738 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:04:11,529 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:04:12,531 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:04:14,325 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:04:14,325 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:04:14,325 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:04:16,218 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:04:16,219 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:04:16,219 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:04:16,219 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:04:16,224 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:04:16,534 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:04:18,538 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:04:21,261 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:04:24,730 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:04:24,731 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:04:24,731 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:04:24,737 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:04:24,991 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 17:04:25,542 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:04:26,543 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:04:26,992 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:04:28,976 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:04:28,976 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:04:28,977 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:04:28,982 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:04:29,325 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:04:29,325 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:04:29,326 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:04:29,544 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:04:30,547 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:04:32,594 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:04:33,752 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:04:33,753 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:04:33,753 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:04:33,758 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:04:34,550 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:04:36,553 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:04:37,797 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:04:37,927 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:04:37,928 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:04:37,928 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:04:37,933 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:04:38,556 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:04:40,560 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:04:42,934 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:04:43,130 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:04:43,130 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:04:43,131 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:04:43,136 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:04:43,561 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:04:44,325 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:04:44,326 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:04:44,326 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:04:44,564 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:04:48,579 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:04:48,671 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:04:48,671 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:04:48,672 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:04:48,677 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:04:49,567 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:04:50,569 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:04:52,634 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:04:52,635 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:04:52,635 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:04:52,640 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:04:53,571 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:04:53,641 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:04:54,574 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:04:54,991 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 17:04:56,403 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:04:56,404 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:04:56,404 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:04:56,409 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:04:56,576 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:04:58,579 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:04:59,325 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:04:59,326 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:04:59,327 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:04:59,555 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:05:00,271 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:05:00,272 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:05:00,272 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:05:00,277 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:05:00,581 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:05:02,584 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:05:05,317 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:05:05,866 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:05:05,867 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:05:05,867 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:05:05,873 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:05:06,586 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:05:08,589 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:05:10,112 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:05:10,113 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:05:10,113 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:05:10,118 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:05:10,591 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:05:11,119 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:05:12,594 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:05:14,101 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:05:14,102 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:05:14,102 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:05:14,107 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:05:14,325 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:05:14,326 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:05:14,327 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:05:14,596 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:05:16,570 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:05:16,599 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:05:18,938 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:05:18,939 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:05:18,939 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:05:18,944 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:05:19,601 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:05:20,603 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:05:21,981 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:05:22,692 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:05:22,693 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:05:22,693 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:05:22,698 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:05:23,605 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:05:24,608 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:05:24,994 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 17:05:26,996 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:05:28,425 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:05:28,426 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:05:28,426 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:05:28,432 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:05:28,611 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:05:29,325 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:05:29,326 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:05:29,326 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:05:30,614 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:05:32,584 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:05:35,062 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:05:35,063 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:05:35,063 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:05:35,068 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:05:35,617 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:05:36,620 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:05:38,105 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:05:39,305 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:05:39,305 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:05:39,305 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:05:39,311 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:05:39,623 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:05:40,624 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:05:43,311 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:05:44,145 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:05:44,145 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:05:44,146 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:05:44,150 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:05:44,326 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:05:44,326 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:05:44,326 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:05:44,626 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:05:46,630 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:05:48,579 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:05:48,948 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:05:48,949 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:05:48,949 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:05:48,954 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:05:49,631 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:05:50,634 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:05:53,993 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:05:54,997 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 17:05:55,658 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:05:55,659 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:05:55,659 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:05:55,664 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:05:56,642 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:05:56,642 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:05:59,326 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:05:59,326 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:05:59,327 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:05:59,620 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:06:00,552 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:06:00,553 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:06:00,553 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:06:00,558 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:06:00,645 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:06:02,648 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:06:05,597 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:06:09,601 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:06:09,601 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:06:09,601 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:06:09,607 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:06:09,653 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:06:10,608 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:06:10,655 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:06:14,326 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:06:14,326 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:06:14,326 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:06:15,481 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:06:15,482 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:06:15,482 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:06:15,488 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:06:15,659 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:06:16,525 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:06:16,662 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:06:21,225 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:06:21,225 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:06:21,226 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:06:21,231 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:06:21,665 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:06:22,231 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:06:22,668 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:06:25,000 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 17:06:25,037 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:06:25,037 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:06:25,037 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:06:25,042 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:06:25,669 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:06:26,670 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:06:28,043 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:06:29,272 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:06:29,273 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:06:29,273 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:06:29,278 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:06:29,326 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:06:29,326 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:06:29,328 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:06:29,672 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:06:30,674 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:06:33,294 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:06:33,295 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:06:33,295 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:06:33,295 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:06:33,300 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:06:33,676 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:06:34,679 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:06:37,968 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:06:37,969 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:06:37,969 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:06:37,974 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:06:38,683 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:06:38,975 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:06:40,686 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:06:41,938 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:06:41,939 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:06:41,939 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:06:41,944 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:06:42,687 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:06:44,326 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:06:44,326 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:06:44,328 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:06:44,574 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:06:44,690 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:06:45,523 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:06:45,524 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:06:45,524 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:06:45,529 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:06:45,691 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:06:46,694 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:06:49,127 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:06:49,128 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:06:49,128 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:06:49,134 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:06:49,696 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:06:50,134 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:06:50,698 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:06:55,006 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 17:06:56,007 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:06:57,169 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:06:57,170 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:06:57,170 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:06:57,176 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:06:57,704 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:06:58,706 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:06:59,326 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:06:59,326 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:06:59,328 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:07:01,132 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:07:01,132 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:07:01,132 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:07:01,133 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:07:01,137 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:07:01,709 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:07:02,711 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:07:05,140 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:07:05,141 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:07:05,141 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:07:05,146 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:07:05,713 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:07:06,147 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:07:06,714 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:07:08,786 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:07:08,787 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:07:08,787 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:07:08,793 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:07:09,716 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:07:10,719 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:07:11,793 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:07:12,651 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:07:12,651 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:07:12,652 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:07:12,657 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:07:12,720 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:07:14,326 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:07:14,326 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:07:14,328 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:07:14,723 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:07:17,590 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:07:18,144 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:07:18,144 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:07:18,145 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:07:18,150 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:07:18,725 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:07:20,729 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:07:21,962 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:07:21,962 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:07:21,963 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:07:21,967 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:07:22,731 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:07:22,968 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:07:24,734 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:07:25,006 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 17:07:28,007 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:07:29,326 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:07:29,326 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:07:29,328 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:07:30,326 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:07:30,326 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:07:30,327 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:07:30,332 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:07:30,737 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:07:32,740 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:07:33,369 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:07:34,165 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:07:34,165 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:07:34,166 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:07:34,171 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:07:34,742 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:07:36,745 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:07:37,747 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:07:37,748 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:07:37,748 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:07:37,753 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:07:38,747 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:07:38,748 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:07:38,753 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:07:41,515 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:07:41,516 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:07:41,516 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:07:41,521 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:07:41,749 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:07:42,752 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:07:44,326 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:07:44,327 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:07:44,328 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:07:44,557 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:07:49,558 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:07:49,567 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:07:49,568 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:07:49,568 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:07:49,573 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:07:49,756 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:07:50,758 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:07:54,613 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:07:55,012 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 17:07:58,106 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:07:58,106 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:07:58,107 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:07:58,112 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:07:58,764 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:07:59,326 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:07:59,327 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:07:59,328 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:08:00,586 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:08:00,765 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:08:01,941 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:08:01,942 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:08:01,942 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:08:01,947 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:08:02,767 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:08:04,770 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:08:05,985 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:08:06,435 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:08:06,436 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:08:06,436 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:08:06,442 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:08:06,771 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:08:08,774 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:08:10,148 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:08:10,149 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:08:10,149 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:08:10,155 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:08:10,776 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:08:11,156 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:08:12,779 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:08:13,989 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:08:13,990 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:08:13,990 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:08:13,995 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:08:14,326 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:08:14,327 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:08:14,328 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:08:14,780 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:08:16,591 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:08:16,781 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:08:20,410 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:08:20,411 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:08:20,411 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:08:20,416 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:08:20,784 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:08:22,453 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:08:22,787 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:08:24,464 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:08:24,465 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:08:24,465 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:08:24,471 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:08:24,788 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:08:25,011 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 17:08:26,791 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:08:28,012 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:08:28,381 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:08:28,382 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:08:28,382 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:08:28,387 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:08:28,792 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:08:29,327 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:08:29,327 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:08:29,328 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:08:30,796 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:08:33,212 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:08:33,213 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:08:33,213 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:08:33,213 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:08:33,218 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:08:33,798 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:08:34,800 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:08:37,848 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:08:37,849 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:08:37,849 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:08:37,854 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:08:38,804 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:08:38,804 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:08:38,854 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:08:41,749 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:08:41,750 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:08:41,750 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:08:41,755 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:08:41,805 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:08:42,808 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:08:44,327 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:08:44,327 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:08:44,329 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:08:44,575 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:08:49,575 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:08:50,089 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:08:50,090 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:08:50,090 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:08:50,095 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:08:50,812 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:08:52,815 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:08:53,894 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:08:53,895 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:08:53,895 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:08:53,900 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:08:54,817 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:08:54,901 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:08:55,016 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 17:08:56,821 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:08:59,203 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:08:59,204 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:08:59,204 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:08:59,209 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:08:59,327 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:08:59,327 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:08:59,329 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:08:59,823 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:09:00,559 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:09:00,826 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:09:03,386 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:09:03,386 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:09:03,386 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:09:03,391 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:09:03,827 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:09:04,830 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:09:06,429 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:09:07,439 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:09:07,440 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:09:07,440 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:09:07,445 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:09:07,832 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:09:08,834 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:09:11,446 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:09:12,198 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:09:12,199 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:09:12,199 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:09:12,204 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:09:12,837 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:09:14,327 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:09:14,327 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:09:14,328 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:09:14,840 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:09:16,568 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:09:17,089 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:09:17,090 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:09:17,090 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:09:17,095 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:09:17,842 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:09:18,844 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:09:22,133 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:09:25,016 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 17:09:25,681 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:09:25,681 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:09:25,681 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:09:25,686 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:09:25,848 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:09:26,851 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:09:27,687 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:09:29,327 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:09:29,327 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:09:29,328 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:09:31,290 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:09:31,290 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:09:31,290 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:09:31,295 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:09:31,854 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:09:32,857 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:09:33,333 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:09:35,140 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:09:35,141 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:09:35,141 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:09:35,146 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:09:35,859 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:09:36,859 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:09:39,147 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:09:39,987 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:09:39,988 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:09:39,988 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:09:39,994 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:09:40,863 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:09:42,866 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:09:43,990 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:09:43,991 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:09:43,991 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:09:43,996 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:09:44,329 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:09:44,329 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:09:44,329 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:09:44,329 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:09:44,867 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:09:46,870 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:09:47,569 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:09:47,569 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:09:47,569 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:09:47,574 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:09:47,871 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:09:48,874 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:09:49,613 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:09:53,306 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:09:53,306 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:09:53,307 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:09:53,311 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:09:53,876 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:09:54,879 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:09:55,019 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 17:09:55,021 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:09:56,618 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:09:56,619 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:09:56,619 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:09:56,624 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:09:56,881 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:09:58,883 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:09:59,327 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:09:59,328 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:09:59,329 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:10:00,621 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:10:02,163 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:10:02,163 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:10:02,163 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:10:02,168 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:10:02,885 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:10:04,888 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:10:06,205 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:10:07,853 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:10:07,853 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:10:07,854 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:10:07,858 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:10:07,890 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:10:08,893 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:10:11,859 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:10:14,002 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:10:14,003 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:10:14,003 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:10:14,011 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:10:14,328 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:10:14,328 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:10:14,329 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:10:14,897 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:10:16,900 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:10:17,619 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:10:18,185 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:10:18,189 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:10:18,189 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:10:18,194 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:10:18,901 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:10:20,904 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:10:21,838 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:10:21,839 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:10:21,839 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:10:21,844 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:10:21,905 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:10:22,844 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:10:22,905 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:10:25,023 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 17:10:25,969 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:10:25,970 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:10:25,970 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:10:25,975 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:10:26,910 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:10:26,910 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:10:27,975 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:10:29,328 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:10:29,328 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:10:29,329 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:10:30,838 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:10:30,838 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:10:30,839 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:10:30,844 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:10:30,912 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:10:32,915 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:10:33,881 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:10:35,625 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:10:35,626 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:10:35,626 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:10:35,631 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:10:35,918 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:10:36,920 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:10:39,632 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:10:40,204 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:10:40,204 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:10:40,204 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:10:40,209 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:10:40,923 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:10:42,926 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:10:44,328 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:10:44,328 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:10:44,330 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:10:45,604 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:10:49,343 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:10:49,343 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:10:49,343 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:10:49,348 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:10:49,930 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:10:50,933 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:10:51,385 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:10:55,026 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 17:10:57,027 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:10:57,674 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:10:57,675 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:10:57,675 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:10:57,680 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:10:57,938 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:10:58,940 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:10:59,328 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:10:59,328 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:10:59,330 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:11:02,609 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:11:03,170 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:11:03,171 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:11:03,171 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:11:03,177 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:11:03,943 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:11:04,946 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:11:07,215 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:11:07,216 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:11:07,216 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:11:07,221 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:11:07,948 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:11:08,222 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:11:08,950 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:11:11,530 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:11:11,530 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:11:11,531 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:11:11,536 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:11:11,953 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:11:12,955 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:11:13,536 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:11:14,328 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:11:14,328 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:11:14,330 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:11:15,062 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:11:15,063 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:11:15,063 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:11:15,068 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:11:15,957 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:11:16,959 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:11:19,105 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:11:19,555 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:11:19,556 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:11:19,556 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:11:19,562 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:11:19,961 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:11:20,963 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:11:24,562 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:11:25,032 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 17:11:27,997 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:11:27,998 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:11:27,998 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:11:28,003 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:11:28,970 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:11:28,970 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:11:29,328 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:11:29,328 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:11:29,330 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:11:29,583 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:11:31,684 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:11:31,685 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:11:31,685 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:11:31,690 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:11:31,972 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:11:32,974 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:11:34,729 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:11:39,729 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:11:40,538 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:11:40,539 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:11:40,539 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:11:40,544 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:11:40,981 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:11:42,984 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:11:44,328 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:11:44,329 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:11:44,329 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:11:44,361 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:11:44,537 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:11:44,537 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:11:44,542 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:11:44,985 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:11:45,542 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:11:46,988 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:11:50,543 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:11:52,768 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:11:52,769 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:11:52,769 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:11:52,774 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:11:52,991 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:11:54,994 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:11:55,032 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 17:11:56,033 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:11:56,587 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:11:56,587 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:11:56,588 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:11:56,593 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:11:56,997 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:11:59,000 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:11:59,328 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:11:59,329 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:11:59,330 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:12:00,377 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:12:00,378 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:12:00,378 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:12:00,384 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:12:01,001 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:12:01,421 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:12:03,004 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:12:04,679 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:12:04,680 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:12:04,681 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:12:04,685 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:12:05,005 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:12:06,686 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:12:07,006 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:12:08,973 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:12:08,973 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:12:08,974 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:12:08,979 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:12:09,007 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:12:11,011 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:12:11,979 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:12:12,989 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:12:12,990 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:12:12,990 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:12:12,995 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:12:13,012 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:12:14,328 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:12:14,329 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:12:14,330 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:12:15,015 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:12:17,617 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:12:21,212 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:12:21,213 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:12:21,213 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:12:21,218 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:12:22,019 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:12:23,020 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:12:23,257 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:12:25,039 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 17:12:26,878 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:12:26,879 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:12:26,879 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:12:26,884 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:12:27,022 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:12:28,885 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:12:29,025 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:12:29,328 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:12:29,329 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:12:29,330 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:12:30,515 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:12:30,516 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:12:30,516 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:12:30,521 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:12:31,026 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:12:33,029 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:12:34,561 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:12:35,012 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:12:35,013 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:12:35,013 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:12:35,018 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:12:35,031 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:12:37,034 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:12:38,760 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:12:38,760 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:12:38,761 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:12:38,765 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:12:39,036 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:12:39,766 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:12:41,039 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:12:42,502 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:12:42,503 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:12:42,504 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:12:42,509 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:12:43,040 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:12:44,329 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:12:44,329 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:12:44,330 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:12:45,041 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:12:45,535 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:12:46,435 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:12:46,435 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:12:46,436 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:12:46,441 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:12:47,042 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:12:49,046 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:12:50,020 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:12:50,021 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:12:50,021 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:12:50,026 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:12:50,046 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:12:51,027 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:12:51,049 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:12:55,039 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 17:12:56,040 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:12:56,945 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:12:56,946 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:12:56,946 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:12:56,951 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:12:57,054 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:12:59,057 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:12:59,329 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:12:59,329 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:12:59,330 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:13:01,355 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:13:01,356 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:13:01,356 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:13:01,357 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:13:01,361 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:13:02,058 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:13:03,061 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:13:05,488 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:13:05,489 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:13:05,489 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:13:05,494 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:13:06,063 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:13:06,495 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:13:07,063 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:13:09,244 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:13:09,245 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:13:09,245 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:13:09,250 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:13:10,065 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:13:11,068 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:13:12,250 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:13:12,974 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:13:12,975 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:13:12,975 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:13:12,981 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:13:13,069 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:13:14,329 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:13:14,329 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:13:14,330 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:13:15,072 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:13:16,890 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:13:16,890 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:13:16,891 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:13:16,896 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:13:17,073 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:13:17,933 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:13:19,076 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:13:21,689 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:13:21,690 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:13:21,691 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:13:21,695 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:13:22,078 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:13:23,081 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:13:23,696 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:13:25,041 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 17:13:26,480 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:13:26,481 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:13:26,481 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:13:26,486 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:13:27,083 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:13:29,084 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:13:29,329 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:13:29,329 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:13:29,330 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:13:29,558 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:13:31,057 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:13:31,058 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:13:31,058 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:13:31,063 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:13:31,085 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:13:33,088 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:13:35,101 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:13:35,902 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:13:35,903 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:13:35,903 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:13:35,908 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:13:36,090 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:13:37,093 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:13:40,027 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:13:40,028 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:13:40,028 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:13:40,033 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:13:40,096 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:13:41,033 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:13:41,098 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:13:43,942 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:13:43,943 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:13:43,943 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:13:43,948 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:13:44,100 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:13:44,329 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:13:44,329 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:13:44,331 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:13:45,102 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:13:46,579 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:13:51,580 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:13:52,415 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:13:52,416 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:13:52,416 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:13:52,421 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:13:53,107 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:13:55,044 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 17:13:55,110 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:13:57,046 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:13:59,329 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:13:59,329 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:13:59,331 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:13:59,916 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:13:59,917 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:13:59,917 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:13:59,922 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:14:00,114 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:14:01,117 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:14:02,961 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:14:04,293 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:14:04,294 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:14:04,294 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:14:04,299 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:14:05,120 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:14:07,123 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:14:08,070 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:14:08,071 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:14:08,071 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:14:08,071 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:14:08,076 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:14:08,123 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:14:09,126 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:14:13,077 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:14:13,316 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:14:13,316 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:14:13,317 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:14:13,321 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:14:14,129 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:14:14,329 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:14:14,329 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:14:14,331 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:14:15,131 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:14:17,355 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:14:17,356 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:14:17,356 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:14:17,361 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:14:18,133 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:14:18,397 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:14:19,136 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:14:21,518 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:14:21,518 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:14:21,519 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:14:21,524 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:14:22,137 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:14:23,145 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:14:23,525 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:14:25,048 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 17:14:25,217 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:14:25,218 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:14:25,218 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:14:25,224 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:14:26,147 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:14:27,150 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:14:29,036 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:14:29,037 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:14:29,037 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:14:29,037 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:14:29,042 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:14:29,151 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:14:29,329 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:14:29,329 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:14:29,331 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:14:31,154 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:14:34,402 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:14:34,403 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:14:34,403 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:14:34,403 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:14:34,408 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:14:35,157 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:14:37,160 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:14:38,222 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:14:38,222 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:14:38,223 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:14:38,228 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:14:39,164 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:14:39,164 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:14:40,228 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:14:42,308 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:14:42,309 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:14:42,309 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:14:42,314 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:14:43,166 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:14:44,329 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:14:44,330 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:14:44,331 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:14:45,170 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:14:45,577 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:14:47,667 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:14:47,667 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:14:47,668 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:14:47,673 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:14:48,171 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:14:49,174 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:14:50,712 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:14:51,437 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:14:51,437 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:14:51,438 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:14:51,442 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:14:52,176 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:14:53,176 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:14:55,051 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 17:14:56,052 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:14:57,719 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:14:57,720 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:14:57,720 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:14:57,725 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:14:58,181 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:14:59,183 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:14:59,329 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:14:59,330 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:14:59,331 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:15:01,549 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:15:03,630 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:15:03,631 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:15:03,631 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:15:03,636 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:15:04,186 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:15:05,189 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:15:06,673 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:15:11,673 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:15:12,126 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:15:12,127 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:15:12,127 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:15:12,132 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:15:12,193 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:15:13,196 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:15:14,329 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:15:14,330 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:15:14,331 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:15:17,626 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:15:20,568 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:15:20,569 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:15:20,569 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:15:20,574 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:15:21,200 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:15:23,201 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:15:23,613 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:15:25,054 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 17:15:26,071 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:15:26,072 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:15:26,072 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:15:26,077 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:15:26,203 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:15:27,206 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:15:29,078 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:15:29,329 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:15:29,330 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:15:29,331 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:15:30,213 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:15:30,214 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:15:30,214 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:15:30,219 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:15:31,210 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:15:31,210 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:15:34,257 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:15:35,627 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:15:35,628 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:15:35,628 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:15:35,633 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:15:36,213 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:15:37,216 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:15:39,634 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:15:40,648 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:15:40,648 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:15:40,648 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:15:40,653 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:15:41,220 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:15:43,223 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:15:44,330 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:15:44,330 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:15:44,332 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:15:45,627 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:15:45,680 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:15:45,680 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:15:45,680 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:15:45,685 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:15:46,224 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:15:47,227 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:15:50,110 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:15:50,111 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:15:50,111 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:15:50,117 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:15:50,229 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:15:51,117 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:15:51,229 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:15:55,060 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 17:15:57,062 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:15:57,239 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:15:57,240 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:15:57,240 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:15:57,245 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:15:58,235 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:15:59,237 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:15:59,330 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:15:59,330 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:15:59,332 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:16:01,328 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:16:01,329 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:16:01,329 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:16:01,334 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:16:02,239 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:16:02,373 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:16:03,241 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:16:06,569 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:16:06,570 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:16:06,570 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:16:06,575 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:16:07,244 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:16:07,576 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:16:09,247 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:16:09,948 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:16:09,949 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:16:09,949 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:16:09,954 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:16:10,248 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:16:11,250 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:16:12,955 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:16:13,960 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:16:13,961 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:16:13,961 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:16:13,966 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:16:14,252 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:16:14,330 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:16:14,330 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:16:14,332 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:16:15,255 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:16:18,535 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:16:20,612 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:16:20,613 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:16:20,613 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:16:20,618 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:16:21,258 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:16:23,261 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:16:23,657 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:16:24,625 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:16:24,626 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:16:24,626 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:16:24,631 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:16:25,059 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 17:16:25,262 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:16:27,266 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:16:29,060 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:16:29,330 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:16:29,330 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:16:29,332 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:16:34,369 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:16:34,370 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:16:34,370 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:16:34,370 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:16:34,375 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:16:35,271 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:16:37,274 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:16:38,408 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:16:38,409 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:16:38,409 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:16:38,414 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:16:39,276 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:16:39,414 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:16:41,279 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:16:43,293 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:16:43,294 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:16:43,294 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:16:43,299 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:16:44,281 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:16:44,330 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:16:44,330 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:16:44,331 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:16:44,592 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:16:45,284 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:16:47,535 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:16:47,536 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:16:47,536 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:16:47,541 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:16:48,286 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:16:49,289 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:16:50,581 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:16:51,925 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:16:51,925 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:16:51,926 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:16:51,931 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:16:52,290 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:16:53,295 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:16:55,065 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 17:16:56,067 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:16:58,837 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:16:58,838 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:16:58,838 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:16:58,843 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:16:59,300 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:16:59,330 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:16:59,330 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:16:59,332 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:17:01,303 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:17:01,594 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:17:04,378 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:17:04,379 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:17:04,379 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:17:04,384 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:17:05,307 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:17:05,307 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:17:07,421 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:17:10,519 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:17:10,520 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:17:10,520 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:17:10,525 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:17:11,311 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:17:12,525 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:17:13,314 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:17:14,330 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:17:14,331 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:17:14,332 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:17:14,848 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:17:14,849 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:17:14,849 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:17:14,854 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:17:15,315 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:17:17,318 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:17:17,893 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:17:18,709 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:17:18,710 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:17:18,710 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:17:18,715 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:17:19,320 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:17:21,323 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:17:22,626 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:17:22,627 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:17:22,627 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:17:22,632 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:17:23,324 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:17:23,632 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:17:25,065 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 17:17:25,327 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:17:29,067 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:17:29,330 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:17:29,331 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:17:29,332 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:17:31,810 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:17:31,811 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:17:31,811 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:17:31,816 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:17:32,331 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:17:33,333 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:17:34,853 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:17:35,571 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:17:35,571 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:17:35,572 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:17:35,577 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:17:36,335 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:17:37,338 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:17:39,743 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:17:39,743 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:17:39,744 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:17:39,749 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:17:40,341 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:17:40,749 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:17:41,343 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:17:43,697 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:17:43,698 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:17:43,698 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:17:43,703 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:17:44,330 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:17:44,331 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:17:44,332 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:17:44,345 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:17:45,347 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:17:46,634 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:17:49,185 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:17:49,186 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:17:49,186 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:17:49,191 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:17:49,350 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:17:51,353 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:17:52,229 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:17:52,876 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:17:52,877 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:17:52,877 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:17:52,882 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:17:53,354 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:17:55,069 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 17:17:55,357 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:17:57,200 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:17:57,201 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:17:57,201 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:17:57,206 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:17:57,360 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:17:58,206 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:17:59,330 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:17:59,331 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:17:59,332 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:17:59,363 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:18:01,281 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:18:01,281 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:18:01,281 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:18:01,286 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:18:01,364 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:18:03,325 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:18:03,367 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:18:06,979 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:18:06,983 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:18:06,983 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:18:06,988 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:18:07,370 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:18:08,988 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:18:09,372 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:18:12,455 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:18:12,456 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:18:12,456 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:18:12,461 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:18:13,377 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:18:13,377 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:18:14,331 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:18:14,331 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:18:14,333 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:18:14,619 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:18:16,906 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:18:16,906 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:18:16,907 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:18:16,911 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:18:17,379 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:18:19,380 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:18:19,949 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:18:22,039 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:18:22,039 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:18:22,040 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:18:22,044 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:18:22,382 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:18:23,385 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:18:25,045 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:18:25,072 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 17:18:26,036 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:18:26,036 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:18:26,037 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:18:26,041 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:18:26,386 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:18:27,389 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:18:29,331 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:18:29,331 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:18:29,331 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:18:29,869 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:18:29,869 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:18:29,870 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:18:29,875 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:18:30,391 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:18:30,913 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:18:31,393 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:18:34,562 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:18:34,563 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:18:34,563 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:18:34,568 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:18:35,396 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:18:36,569 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:18:37,399 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:18:38,389 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:18:38,390 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:18:38,390 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:18:38,396 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:18:38,399 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:18:39,402 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:18:42,396 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:18:44,034 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:18:44,035 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:18:44,035 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:18:44,040 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:18:44,331 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:18:44,331 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:18:44,332 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:18:44,405 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:18:45,407 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:18:47,595 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:18:49,154 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:18:49,155 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:18:49,155 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:18:49,160 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:18:49,410 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:18:51,413 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:18:53,175 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:18:53,177 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:18:53,177 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:18:53,178 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:18:53,183 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:18:53,414 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:18:55,074 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 17:18:55,418 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:18:57,011 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:18:57,012 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:18:57,012 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:18:57,017 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:18:57,421 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:18:59,018 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:18:59,331 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:18:59,331 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:18:59,333 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:18:59,424 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:19:00,940 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:19:00,941 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:19:00,941 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:19:00,946 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:19:01,425 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:19:03,428 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:19:04,757 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:19:04,758 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:19:04,758 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:19:04,758 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:19:04,763 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:19:05,429 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:19:07,432 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:19:08,514 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:19:08,515 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:19:08,515 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:19:08,520 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:19:09,435 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:19:09,435 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:19:10,520 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:19:14,331 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:19:14,331 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:19:14,333 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:19:15,029 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:19:15,029 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:19:15,030 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:19:15,034 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:19:15,439 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:19:16,073 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:19:17,442 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:19:19,903 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:19:19,904 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:19:19,904 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:19:19,908 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:19:20,444 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:19:21,444 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:19:21,909 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:19:25,081 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 17:19:27,083 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:19:28,241 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:19:28,242 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:19:28,242 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:19:28,248 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:19:28,448 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:19:29,331 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:19:29,331 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:19:29,333 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:19:29,451 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:19:32,246 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:19:32,246 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:19:32,246 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:19:32,247 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:19:32,251 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:19:32,453 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:19:33,455 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:19:36,207 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:19:36,207 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:19:36,208 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:19:36,212 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:19:36,457 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:19:37,460 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:19:38,213 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:19:39,795 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:19:39,796 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:19:39,796 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:19:39,801 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:19:40,463 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:19:41,465 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:19:43,802 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:19:44,331 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:19:44,331 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:19:44,333 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:19:45,466 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:19:45,467 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:19:45,467 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:19:45,472 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:19:45,472 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:19:47,475 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:19:49,509 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:19:50,390 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:19:50,391 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:19:50,391 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:19:50,396 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:19:50,477 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:19:51,479 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:19:54,821 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:19:54,822 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:19:54,822 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:19:54,822 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:19:54,827 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:19:55,080 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 17:19:55,482 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:19:57,486 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:19:59,331 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:19:59,332 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:19:59,332 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:20:00,238 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:20:00,239 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:20:00,239 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:20:00,239 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:20:00,244 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:20:00,488 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:20:01,490 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:20:04,619 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:20:04,620 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:20:04,620 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:20:04,626 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:20:05,493 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:20:05,493 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:20:05,626 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:20:08,564 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:20:08,564 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:20:08,564 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:20:08,569 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:20:09,497 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:20:09,497 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:20:11,570 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:20:14,278 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:20:14,278 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:20:14,278 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:20:14,283 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:20:14,331 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:20:14,332 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:20:14,332 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:20:14,500 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:20:15,503 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:20:16,573 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:20:18,298 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:20:18,299 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:20:18,299 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:20:18,304 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:20:18,505 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:20:19,507 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:20:21,853 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:20:21,854 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:20:21,854 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:20:21,854 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:20:21,859 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:20:22,509 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:20:23,511 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:20:25,086 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 17:20:26,242 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:20:26,243 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:20:26,243 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:20:26,248 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:20:26,513 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:20:27,249 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:20:27,515 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:20:29,331 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:20:29,332 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:20:29,333 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:20:32,042 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:20:32,043 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:20:32,043 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:20:32,048 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:20:32,518 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:20:33,085 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:20:33,521 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:20:37,512 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:20:37,513 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:20:37,513 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:20:37,518 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:20:37,523 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:20:38,519 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:20:39,528 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:20:41,521 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:20:41,521 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:20:41,521 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:20:41,526 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:20:41,529 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:20:43,527 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:20:43,532 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:20:44,331 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:20:44,332 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:20:44,333 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:20:45,458 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:20:45,459 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:20:45,459 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:20:45,464 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:20:45,533 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:20:47,536 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:20:49,023 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:20:49,023 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:20:49,024 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:20:49,025 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:20:49,029 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:20:49,537 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:20:51,541 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:20:54,030 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:20:55,086 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 17:20:57,378 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:20:57,379 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:20:57,379 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:20:57,384 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:20:57,545 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:20:59,332 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:20:59,332 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:20:59,333 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:20:59,549 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:20:59,765 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:21:01,263 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:21:01,264 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:21:01,264 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:21:01,269 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:21:01,550 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:21:03,553 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:21:05,305 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:21:05,461 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:21:05,462 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:21:05,462 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:21:05,467 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:21:05,554 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:21:07,557 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:21:09,068 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:21:09,069 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:21:09,069 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:21:09,074 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:21:09,559 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:21:11,075 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:21:11,562 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:21:12,991 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:21:12,991 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:21:12,992 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:21:12,997 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:21:13,563 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:21:14,332 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:21:14,332 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:21:14,332 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:21:15,566 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:21:16,598 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:21:21,410 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:21:21,411 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:21:21,411 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:21:21,417 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:21:21,570 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:21:22,453 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:21:23,573 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:21:25,092 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 17:21:25,766 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:21:25,766 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:21:25,766 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:21:25,771 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:21:26,574 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:21:27,577 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:21:27,772 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:21:29,332 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:21:29,332 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:21:29,333 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:21:29,568 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:21:29,622 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:21:29,623 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:21:29,628 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:21:30,579 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:21:31,581 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:21:33,357 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:21:33,358 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:21:33,358 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:21:33,358 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:21:33,364 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:21:33,582 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:21:35,586 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:21:37,796 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:21:37,797 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:21:37,797 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:21:37,802 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:21:38,588 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:21:38,802 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:21:39,591 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:21:43,242 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:21:43,243 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:21:43,243 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:21:43,248 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:21:43,593 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:21:44,248 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:21:44,332 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:21:44,332 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:21:44,334 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:21:45,596 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:21:47,200 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:21:47,201 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:21:47,201 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:21:47,208 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:21:47,597 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:21:49,599 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:21:50,245 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:21:51,003 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:21:51,003 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:21:51,004 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:21:51,008 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:21:51,600 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:21:53,603 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:21:55,092 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 17:21:55,824 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:21:55,825 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:21:55,825 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:21:55,825 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:21:55,830 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:21:56,606 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:21:57,608 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:21:59,332 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:21:59,332 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:21:59,332 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:22:01,643 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:22:04,278 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:22:04,279 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:22:04,279 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:22:04,284 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:22:04,612 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:22:05,615 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:22:07,321 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:22:08,874 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:22:08,875 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:22:08,875 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:22:08,880 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:22:09,617 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:22:11,621 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:22:12,711 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:22:12,712 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:22:12,712 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:22:12,713 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:22:12,717 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:22:13,624 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:22:13,624 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:22:14,332 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:22:14,332 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:22:14,334 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:22:17,624 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:22:17,624 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:22:17,625 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:22:17,629 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:22:17,629 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:22:18,669 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:22:19,633 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:22:23,669 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:22:25,096 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 17:22:26,167 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:22:26,168 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:22:26,168 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:22:26,173 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:22:26,636 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:22:27,639 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:22:29,174 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:22:29,332 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:22:29,332 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:22:29,334 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:22:31,395 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:22:31,396 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:22:31,396 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:22:31,401 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:22:31,641 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:22:33,644 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:22:34,444 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:22:37,053 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:22:37,053 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:22:37,053 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:22:37,058 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:22:37,647 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:22:39,651 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:22:40,059 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:22:40,940 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:22:40,941 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:22:40,941 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:22:40,946 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:22:41,652 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:22:43,656 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:22:44,332 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:22:44,333 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:22:44,334 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:22:44,466 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:22:44,591 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:22:44,591 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:22:44,596 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:22:44,656 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:22:45,596 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:22:45,659 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:22:48,583 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:22:48,584 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:22:48,584 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:22:48,589 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:22:48,661 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:22:49,663 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:22:51,590 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:22:52,570 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:22:52,571 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:22:52,571 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:22:52,576 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:22:52,665 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:22:53,667 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:22:55,099 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 17:22:57,100 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:22:58,049 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:22:58,050 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:22:58,050 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:22:58,055 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:22:58,672 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:22:59,332 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:22:59,333 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:22:59,334 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:22:59,674 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:23:02,631 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:23:03,741 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:23:03,741 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:23:03,742 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:23:03,747 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:23:04,677 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:23:05,679 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:23:07,785 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:23:12,073 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:23:12,074 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:23:12,074 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:23:12,079 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:23:12,684 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:23:13,079 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:23:13,686 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:23:14,333 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:23:14,333 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:23:14,334 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:23:15,749 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:23:15,749 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:23:15,749 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:23:15,754 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:23:16,692 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:23:17,692 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:23:18,793 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:23:21,615 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:23:21,616 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:23:21,616 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:23:21,621 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:23:21,695 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:23:23,698 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:23:24,622 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:23:25,102 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 17:23:25,463 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:23:25,464 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:23:25,464 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:23:25,469 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:23:25,699 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:23:27,702 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:23:29,332 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:23:29,333 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:23:29,334 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:23:30,602 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:23:33,927 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:23:33,927 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:23:33,927 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:23:33,932 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:23:34,707 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:23:35,709 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:23:35,969 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:23:40,174 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:23:40,175 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:23:40,175 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:23:40,180 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:23:40,713 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:23:41,180 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:23:41,716 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:23:43,446 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:23:43,447 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:23:43,447 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:23:43,452 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:23:43,717 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:23:44,333 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:23:44,333 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:23:44,334 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:23:45,718 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:23:46,600 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:23:48,862 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:23:48,862 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:23:48,863 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:23:48,868 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:23:49,721 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:23:51,724 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:23:51,905 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:23:55,108 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 17:23:57,109 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:23:57,373 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:23:57,373 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:23:57,374 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:23:57,378 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:23:57,729 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:23:59,333 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:23:59,333 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:23:59,334 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:23:59,732 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:24:01,143 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:24:04,045 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:24:04,045 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:24:04,045 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:24:04,051 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:24:04,735 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:24:05,145 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:24:05,146 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:24:05,146 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:24:05,151 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:24:05,737 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:24:05,738 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:24:07,741 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:24:08,586 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:24:08,587 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:24:08,587 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:24:08,592 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:24:08,741 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:24:09,593 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:24:09,742 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:24:13,077 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:24:13,078 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:24:13,078 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:24:13,083 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:24:13,745 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:24:14,333 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:24:14,333 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:24:14,334 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:24:15,550 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:24:15,748 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:24:18,886 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:24:18,887 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:24:18,887 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:24:18,892 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:24:19,750 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:24:20,929 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:24:21,753 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:24:22,479 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:24:22,480 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:24:22,480 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:24:22,486 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:24:22,754 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:24:23,756 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:24:25,108 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 17:24:25,282 WARNING FileStreamThread:1827008 [file_stream.py:request_with_retry():668] requests_with_retry encountered retryable exception: 502 Server Error: Bad Gateway for url: https://api.wandb.ai/files/lilei_stones/hf-vlms/zo3nkbji/file_stream. func: functools.partial(>, timeout=180.0), args: ('https://api.wandb.ai/files/lilei_stones/hf-vlms/zo3nkbji/file_stream',), kwargs: {'json': {'files': {'wandb-summary.json': {'offset': 0, 'content': ['{"Finetune/Step": 2113, "Finetune/Loss": 0.8899515867233276, "Finetune/Loss (Raw)": 0.9392950534820557, "Finetune/Learning Rate": 1.3438732176917437e-05, "Finetune/Step Time": 4.916087483987212, "_timestamp": 1710264195.7488248, "_runtime": 10462.06027674675, "_step": 2113}']}, 'wandb-history.jsonl': {'offset': 2108, 'content': ['{"Finetune/Step": 2109, "Finetune/Loss": 0.8953642249107361, "Finetune/Loss (Raw)": 0.9452600479125977, "Finetune/Learning Rate": 1.3462120295134476e-05, "Finetune/Step Time": 4.860195890069008, "_timestamp": 1710264172.5707068, "_runtime": 10438.882158756256, "_step": 2109}', '{"Finetune/Step": 2110, "Finetune/Loss": 0.8953254222869873, "Finetune/Loss (Raw)": 0.962855875492096, "Finetune/Learning Rate": 1.3456275275280661e-05, "Finetune/Step Time": 4.868157049641013, "_timestamp": 1710264178.0491347, "_runtime": 10444.360586643219, "_step": 2110}', '{"Finetune/Step": 2111, "Finetune/Loss": 0.8954590559005737, "Finetune/Loss (Raw)": 0.9165306687355042, "Finetune/Learning Rate": 1.3450428914113892e-05, "Finetune/Step Time": 4.882664466276765, "_timestamp": 1710264183.7410262, "_runtime": 10450.052478075027, "_step": 2111}', '{"Finetune/Step": 2112, "Finetune/Loss": 0.890485405921936, "Finetune/Loss (Raw)": 0.4575220048427582, "Finetune/Learning Rate": 1.3444581213903026e-05, "Finetune/Step Time": 4.918228147551417, "_timestamp": 1710264192.0731206, "_runtime": 10458.384572505951, "_step": 2112}', '{"Finetune/Step": 2113, "Finetune/Loss": 0.8899515867233276, "Finetune/Loss (Raw)": 0.9392950534820557, "Finetune/Learning Rate": 1.3438732176917437e-05, "Finetune/Step Time": 4.916087483987212, "_timestamp": 1710264195.7488248, "_runtime": 10462.06027674675, "_step": 2113}']}, 'wandb-events.jsonl': {'offset': 347, 'content': ['{"system.proc.memory.availableMB": 1652372.86, "system.memory": 19.9, "system.proc.memory.rssMB": 8296.52, "system.proc.memory.percent": 0.4, "system.network.sent": 6265215864.67, "system.network.recv": 29959944514.6, "system.cpu": 0.39, "system.cpu.0.cpu_percent": 0.07, "system.cpu.1.cpu_percent": 0.0, "system.cpu.2.cpu_percent": 0.03, "system.cpu.3.cpu_percent": 0.17, "system.cpu.4.cpu_percent": 0.1, "system.cpu.5.cpu_percent": 1.2, "system.cpu.6.cpu_percent": 67.55, "system.cpu.7.cpu_percent": 0.0, "system.cpu.8.cpu_percent": 0.03, "system.cpu.9.cpu_percent": 32.02, "system.cpu.10.cpu_percent": 0.07, "system.cpu.11.cpu_percent": 0.63, "system.cpu.12.cpu_percent": 1.2, "system.cpu.13.cpu_percent": 0.2, "system.cpu.14.cpu_percent": 0.03, "system.cpu.15.cpu_percent": 2.66, "system.cpu.16.cpu_percent": 0.63, "system.cpu.17.cpu_percent": 1.07, "system.cpu.18.cpu_percent": 0.27, "system.cpu.19.cpu_percent": 0.07, "system.cpu.20.cpu_percent": 0.07, "system.cpu.21.cpu_percent": 0.53, "system.cpu.22.cpu_percent": 0.6, "system.cpu.23.cpu_percent": 0.07, "system.cpu.24.cpu_percent": 31.73, "system.cpu.25.cpu_percent": 1.23, "system.cpu.26.cpu_percent": 0.07, "system.cpu.27.cpu_percent": 0.1, "system.cpu.28.cpu_percent": 0.07, "system.cpu.29.cpu_percent": 0.2, "system.cpu.30.cpu_percent": 0.07, "system.cpu.31.cpu_percent": 68.27, "system.cpu.32.cpu_percent": 0.0, "system.cpu.33.cpu_percent": 0.0, "system.cpu.34.cpu_percent": 32.35, "system.cpu.35.cpu_percent": 0.0, "system.cpu.36.cpu_percent": 0.03, "system.cpu.37.cpu_percent": 1.2, "system.cpu.38.cpu_percent": 0.2, "system.cpu.39.cpu_percent": 1.53, "system.cpu.40.cpu_percent": 0.13, "system.cpu.41.cpu_percent": 32.23, "system.cpu.42.cpu_percent": 0.0, "system.cpu.43.cpu_percent": 0.0, "system.cpu.44.cpu_percent": 0.03, "system.cpu.45.cpu_percent": 0.07, "system.cpu.46.cpu_percent": 0.5, "system.cpu.47.cpu_percent": 69.47, "system.cpu.48.cpu_percent": 12.12, "system.cpu.49.cpu_percent": 0.1, "system.cpu.50.cpu_percent": 0.0, "system.cpu.51.cpu_percent": 0.57, "system.cpu.52.cpu_percent": 0.83, "system.cpu.53.cpu_percent": 0.1, "system.cpu.54.cpu_percent": 0.07, "system.cpu.55.cpu_percent": 0.2, "system.cpu.56.cpu_percent": 20.01, "system.cpu.57.cpu_percent": 0.0, "system.cpu.58.cpu_percent": 0.0, "system.cpu.59.cpu_percent": 0.63, "system.cpu.60.cpu_percent": 0.6, "system.cpu.61.cpu_percent": 0.0, "system.cpu.62.cpu_percent": 0.13, "system.cpu.63.cpu_percent": 0.17, "system.cpu.64.cpu_percent": 0.0, "system.cpu.65.cpu_percent": 0.07, "system.cpu.66.cpu_percent": 0.0, "system.cpu.67.cpu_percent": 0.0, "system.cpu.68.cpu_percent": 0.0, "system.cpu.69.cpu_percent": 0.0, "system.cpu.70.cpu_percent": 0.17, "system.cpu.71.cpu_percent": 0.0, "system.cpu.72.cpu_percent": 0.0, "system.cpu.73.cpu_percent": 0.0, "system.cpu.74.cpu_percent": 0.0, "system.cpu.75.cpu_percent": 0.0, "system.cpu.76.cpu_percent": 0.0, "system.cpu.77.cpu_percent": 0.0, "system.cpu.78.cpu_percent": 0.0, "system.cpu.79.cpu_percent": 0.0, "system.cpu.80.cpu_percent": 0.03, "system.cpu.81.cpu_percent": 0.0, "system.cpu.82.cpu_percent": 0.03, "system.cpu.83.cpu_percent": 0.03, "system.cpu.84.cpu_percent": 0.0, "system.cpu.85.cpu_percent": 0.9, "system.cpu.86.cpu_percent": 0.03, "system.cpu.87.cpu_percent": 0.0, "system.cpu.88.cpu_percent": 0.0, "system.cpu.89.cpu_percent": 0.03, "system.cpu.90.cpu_percent": 0.0, "system.cpu.91.cpu_percent": 0.0, "system.cpu.92.cpu_percent": 0.07, "system.cpu.93.cpu_percent": 0.03, "system.cpu.94.cpu_percent": 0.0, "system.cpu.95.cpu_percent": 0.07, "system.cpu.96.cpu_percent": 0.0, "system.cpu.97.cpu_percent": 4.5, "system.cpu.98.cpu_percent": 0.0, "system.cpu.99.cpu_percent": 0.0, "system.cpu.100.cpu_percent": 0.03, "system.cpu.101.cpu_percent": 0.0, "system.cpu.102.cpu_percent": 0.0, "system.cpu.103.cpu_percent": 0.0, "system.cpu.104.cpu_percent": 3.23, "system.cpu.105.cpu_percent": 1.66, "system.cpu.106.cpu_percent": 1.46, "system.cpu.107.cpu_percent": 0.0, "system.cpu.108.cpu_percent": 1.6, "system.cpu.109.cpu_percent": 0.0, "system.cpu.110.cpu_percent": 0.0, "system.cpu.111.cpu_percent": 0.0, "system.cpu.112.cpu_percent": 0.0, "system.cpu.113.cpu_percent": 0.0, "system.cpu.114.cpu_percent": 0.0, "system.cpu.115.cpu_percent": 0.0, "system.cpu.116.cpu_percent": 0.0, "system.cpu.117.cpu_percent": 0.0, "system.cpu.118.cpu_percent": 0.0, "system.cpu.119.cpu_percent": 0.03, "system.cpu.120.cpu_percent": 0.0, "system.cpu.121.cpu_percent": 0.03, "system.cpu.122.cpu_percent": 0.0, "system.cpu.123.cpu_percent": 0.03, "system.cpu.124.cpu_percent": 0.0, "system.cpu.125.cpu_percent": 0.6, "system.cpu.126.cpu_percent": 0.0, "system.cpu.127.cpu_percent": 0.03, "system.cpu.128.cpu_percent": 0.03, "system.cpu.129.cpu_percent": 0.03, "system.cpu.130.cpu_percent": 0.2, "system.cpu.131.cpu_percent": 32.37, "system.cpu.132.cpu_percent": 0.0, "system.cpu.133.cpu_percent": 1.33, "system.cpu.134.cpu_percent": 0.0, "system.cpu.135.cpu_percent": 0.0, "system.cpu.136.cpu_percent": 0.03, "system.cpu.137.cpu_percent": 0.13, "system.cpu.138.cpu_percent": 67.95, "system.cpu.139.cpu_percent": 0.07, "system.cpu.140.cpu_percent": 0.0, "system.cpu.141.cpu_percent": 0.03, "system.cpu.142.cpu_percent": 0.67, "system.cpu.143.cpu_percent": 7.65, "system.cpu.144.cpu_percent": 0.0, "system.cpu.145.cpu_percent": 0.0, "system.cpu.146.cpu_percent": 0.0, "system.cpu.147.cpu_percent": 0.03, "system.cpu.148.cpu_percent": 0.07, "system.cpu.149.cpu_percent": 0.07, "system.cpu.150.cpu_percent": 0.0, "system.cpu.151.cpu_percent": 0.0, "system.cpu.152.cpu_percent": 0.1, "system.cpu.153.cpu_percent": 0.03, "system.cpu.154.cpu_percent": 0.07, "system.cpu.155.cpu_percent": 0.0, "system.cpu.156.cpu_percent": 0.0, "system.cpu.157.cpu_percent": 0.1, "system.cpu.158.cpu_percent": 0.0, "system.cpu.159.cpu_percent": 0.0, "system.cpu.160.cpu_percent": 32.39, "system.cpu.161.cpu_percent": 67.7, "system.cpu.162.cpu_percent": 0.0, "system.cpu.163.cpu_percent": 0.2, "system.cpu.164.cpu_percent": 0.7, "system.cpu.165.cpu_percent": 0.77, "system.cpu.166.cpu_percent": 0.03, "system.cpu.167.cpu_percent": 67.59, "system.cpu.168.cpu_percent": 0.6, "system.cpu.169.cpu_percent": 0.03, "system.cpu.170.cpu_percent": 31.04, "system.cpu.171.cpu_percent": 0.17, "system.cpu.172.cpu_percent": 67.91, "system.cpu.173.cpu_percent": 1.23, "system.cpu.174.cpu_percent": 0.17, "system.cpu.175.cpu_percent": 0.1, "system.cpu.176.cpu_percent": 0.03, "system.cpu.177.cpu_percent": 0.03, "system.cpu.178.cpu_percent": 0.0, "system.cpu.179.cpu_percent": 0.1, "system.cpu.180.cpu_percent": 0.03, "system.cpu.181.cpu_percent": 0.13, "system.cpu.182.cpu_percent": 1.27, "system.cpu.183.cpu_percent": 0.07, "system.cpu.184.cpu_percent": 0.03, "system.cpu.185.cpu_percent": 0.1, "system.cpu.186.cpu_percent": 0.17, "system.cpu.187.cpu_percent": 67.97, "system.cpu.188.cpu_percent": 0.13, "system.cpu.189.cpu_percent": 0.1, "system.cpu.190.cpu_percent": 0.13, "system.cpu.191.cpu_percent": 0.0, "system.cpu.192.cpu_percent": 0.0, "system.cpu.193.cpu_percent": 0.03, "system.cpu.194.cpu_percent": 0.0, "system.cpu.195.cpu_percent": 0.03, "system.cpu.196.cpu_percent": 0.07, "system.cpu.197.cpu_percent": 0.1, "system.cpu.198.cpu_percent": 0.0, "system.cpu.199.cpu_percent": 0.03, "system.cpu.200.cpu_percent": 0.0, "system.cpu.201.cpu_percent": 0.0, "system.cpu.202.cpu_percent": 0.0, "system.cpu.203.cpu_percent": 0.03, "system.cpu.204.cpu_percent": 0.0, "system.cpu.205.cpu_percent": 0.0, "system.cpu.206.cpu_percent": 0.0, "system.cpu.207.cpu_percent": 0.0, "system.cpu.208.cpu_percent": 0.0, "system.cpu.209.cpu_percent": 0.0, "system.cpu.210.cpu_percent": 0.0, "system.cpu.211.cpu_percent": 0.0, "system.cpu.212.cpu_percent": 0.03, "system.cpu.213.cpu_percent": 0.07, "system.cpu.214.cpu_percent": 0.03, "system.cpu.215.cpu_percent": 0.0, "system.cpu.216.cpu_percent": 0.1, "system.cpu.217.cpu_percent": 0.0, "system.cpu.218.cpu_percent": 0.0, "system.cpu.219.cpu_percent": 0.0, "system.cpu.220.cpu_percent": 0.0, "system.cpu.221.cpu_percent": 0.03, "system.cpu.222.cpu_percent": 0.03, "system.cpu.223.cpu_percent": 0.0, "system.cpu.224.cpu_percent": 0.0, "system.cpu.225.cpu_percent": 0.0, "system.cpu.226.cpu_percent": 0.0, "system.cpu.227.cpu_percent": 0.0, "system.cpu.228.cpu_percent": 0.0, "system.cpu.229.cpu_percent": 0.0, "system.cpu.230.cpu_percent": 4.66, "system.cpu.231.cpu_percent": 0.0, "system.cpu.232.cpu_percent": 0.0, "system.cpu.233.cpu_percent": 0.0, "system.cpu.234.cpu_percent": 0.0, "system.cpu.235.cpu_percent": 0.0, "system.cpu.236.cpu_percent": 0.5, "system.cpu.237.cpu_percent": 0.0, "system.cpu.238.cpu_percent": 0.0, "system.cpu.239.cpu_percent": 1.46, "system.cpu.240.cpu_percent": 0.0, "system.cpu.241.cpu_percent": 0.03, "system.cpu.242.cpu_percent": 0.07, "system.cpu.243.cpu_percent": 0.03, "system.cpu.244.cpu_percent": 0.0, "system.cpu.245.cpu_percent": 0.0, "system.cpu.246.cpu_percent": 0.0, "system.cpu.247.cpu_percent": 0.03, "system.cpu.248.cpu_percent": 0.0, "system.cpu.249.cpu_percent": 0.03, "system.cpu.250.cpu_percent": 0.0, "system.cpu.251.cpu_percent": 0.0, "system.cpu.252.cpu_percent": 0.03, "system.cpu.253.cpu_percent": 0.0, "system.cpu.254.cpu_percent": 0.03, "system.proc.cpu.threads": 45, "system.disk.\\\\.usagePercent": 22.9, "system.disk.\\\\.usageGB": 55.32, "system.disk.in": 64.2, "system.disk.out": 567.52, "system.gpu.0.memoryAllocated": 99.19, "system.gpu.process.0.memoryAllocated": 99.19, "system.gpu.1.memoryAllocated": 81.67, "system.gpu.2.memoryAllocated": 80.95, "system.gpu.3.memoryAllocated": 78.9, "system.gpu.4.memoryAllocated": 79.57, "system.gpu.5.memoryAllocated": 87.99, "system.gpu.6.memoryAllocated": 82.35, "system.gpu.7.memoryAllocated": 91.58, "system.gpu.0.memoryAllocatedBytes": 85206630400.0, "system.gpu.process.0.memoryAllocatedBytes": 85206630400.0, "system.gpu.1.memoryAllocatedBytes": 70155370496.0, "system.gpu.2.memoryAllocatedBytes": 69536710656.0, "system.gpu.3.memoryAllocatedBytes": 67777200128.0, "system.gpu.4.memoryAllocatedBytes": 68353916928.0, "system.gpu.5.memoryAllocatedBytes": 75586994176.0, "system.gpu.6.memoryAllocatedBytes": 70738378752.0, "system.gpu.7.memoryAllocatedBytes": 78669807616.0, "system.gpu.0.memory": 38.73, "system.gpu.process.0.memory": 38.73, "system.gpu.1.memory": 34.93, "system.gpu.2.memory": 32.67, "system.gpu.3.memory": 30.73, "system.gpu.4.memory": 30.73, "system.gpu.5.memory": 32.47, "system.gpu.6.memory": 29.8, "system.gpu.7.memory": 31.07, "system.gpu.0.gpu": 99.93, "system.gpu.process.0.gpu": 99.93, "system.gpu.1.gpu": 99.93, "system.gpu.2.gpu": 99.93, "system.gpu.3.gpu": 99.93, "system.gpu.4.gpu": 99.93, "system.gpu.5.gpu": 99.93, "system.gpu.6.gpu": 99.93, "system.gpu.7.gpu": 99.93, "system.gpu.0.temp": 73.6, "system.gpu.process.0.temp": 73.6, "system.gpu.1.temp": 63.8, "system.gpu.2.temp": 66.47, "system.gpu.3.temp": 73.93, "system.gpu.4.temp": 69.73, "system.gpu.5.temp": 64.4, "system.gpu.6.temp": 65.8, "system.gpu.7.temp": 74.0, "system.gpu.0.powerWatts": 369.2, "system.gpu.process.0.powerWatts": 369.2, "system.gpu.1.powerWatts": 368.79, "system.gpu.2.powerWatts": 331.63, "system.gpu.3.powerWatts": 316.25, "system.gpu.4.powerWatts": 334.68, "system.gpu.5.powerWatts": 337.44, "system.gpu.6.powerWatts": 338.5, "system.gpu.7.powerWatts": 322.51, "system.gpu.0.powerPercent": 91.41, "system.gpu.process.0.powerPercent": 91.41, "system.gpu.1.powerPercent": 89.03, "system.gpu.2.powerPercent": 85.94, "system.gpu.3.powerPercent": 73.85, "system.gpu.4.powerPercent": 82.96, "system.gpu.5.powerPercent": 87.93, "system.gpu.6.powerPercent": 84.56, "system.gpu.7.powerPercent": 83.14, "_wandb": true, "_timestamp": 1710264205.099411, "_runtime": 10471.410863}']}, 'output.log': {'offset': 1, 'content': ['ERROR 2024-03-12T17:23:22.980327 =>> [Global Step] 002114 =>> LR :: 0.000013 -- Loss :: 0.8894: 41%|█████████████████▍ | 2114/5198 [2:54:19<4:41:06, 5.47s/it]\n']}}, 'dropped': 0}} +2024-03-12 17:24:26,109 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:24:26,667 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:24:26,668 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:24:26,668 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:24:26,673 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:24:26,758 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:24:27,761 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:24:29,333 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:24:29,333 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:24:29,334 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:24:31,542 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:24:35,609 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:24:35,610 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:24:35,610 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:24:35,614 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:24:35,766 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:24:36,653 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:24:37,769 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:24:41,653 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:24:43,948 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:24:43,948 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:24:43,949 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:24:43,953 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:24:44,333 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:24:44,333 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:24:44,335 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:24:44,774 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:24:45,776 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:24:47,472 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:24:47,472 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:24:47,473 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:24:47,473 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:24:47,477 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:24:47,778 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:24:49,781 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:24:51,413 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:24:51,414 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:24:51,414 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:24:51,419 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:24:51,782 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:24:53,419 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:24:53,785 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:24:55,114 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 17:24:55,990 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:24:55,991 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:24:55,991 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:24:55,996 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:24:56,788 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:24:57,791 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:24:58,997 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:24:59,333 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:24:59,333 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:24:59,335 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:24:59,795 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:24:59,795 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:24:59,796 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:24:59,800 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:25:00,792 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:25:01,795 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:25:04,837 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:25:04,894 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:25:04,894 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:25:04,894 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:25:04,900 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:25:05,799 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:25:05,800 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:25:08,168 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:25:08,169 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:25:08,169 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:25:08,174 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:25:08,801 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:25:09,805 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:25:10,174 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:25:11,779 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:25:11,780 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:25:11,780 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:25:11,785 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:25:11,807 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:25:13,808 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:25:14,333 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:25:14,333 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:25:14,335 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:25:15,570 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:25:20,304 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:25:20,305 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:25:20,305 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:25:20,310 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:25:20,812 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:25:21,349 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:25:21,814 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:25:24,084 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:25:24,085 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:25:24,085 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:25:24,089 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:25:24,816 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:25:25,113 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 17:25:25,817 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:25:27,114 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:25:29,333 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:25:29,333 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:25:29,335 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:25:30,594 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:25:30,594 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:25:30,594 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:25:30,599 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:25:30,819 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:25:31,822 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:25:32,637 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:25:35,493 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:25:35,494 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:25:35,495 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:25:35,500 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:25:35,825 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:25:37,828 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:25:38,501 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:25:40,020 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:25:40,020 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:25:40,020 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:25:40,025 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:25:40,831 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:25:41,833 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:25:44,026 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:25:44,333 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:25:44,334 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:25:44,335 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:25:44,491 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:25:44,627 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:25:44,628 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:25:44,632 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:25:44,835 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:25:45,838 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:25:49,633 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:25:51,070 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:25:51,071 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:25:51,071 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:25:51,076 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:25:51,841 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:25:53,846 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:25:55,077 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:25:55,116 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 17:25:57,168 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:25:57,172 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:25:57,172 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:25:57,177 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:25:57,850 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:25:59,333 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:25:59,334 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:25:59,335 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:25:59,853 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:26:00,561 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:26:02,794 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:26:02,794 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:26:02,794 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:26:02,799 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:26:02,854 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:26:03,857 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:26:05,837 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:26:08,862 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:26:08,863 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:26:08,863 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:26:08,868 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:26:09,862 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:26:09,862 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:26:10,869 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:26:13,739 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:26:13,740 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:26:13,740 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:26:13,745 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:26:13,865 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:26:14,334 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:26:14,334 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:26:14,335 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:26:15,868 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:26:16,598 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:26:17,566 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:26:17,567 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:26:17,567 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:26:17,573 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:26:17,869 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:26:19,872 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:26:21,609 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:26:21,964 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:26:21,965 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:26:21,965 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:26:21,971 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:26:22,874 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:26:23,876 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:26:25,119 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 17:26:26,118 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:26:26,118 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:26:26,118 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:26:26,124 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:26:26,878 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:26:27,124 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:26:27,881 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:26:29,333 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:26:29,334 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:26:29,335 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:26:31,203 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:26:31,204 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:26:31,204 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:26:31,209 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:26:31,883 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:26:32,249 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:26:33,886 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:26:36,018 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:26:36,019 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:26:36,020 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:26:36,025 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:26:36,888 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:26:37,891 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:26:38,026 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:26:39,983 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:26:39,984 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:26:39,984 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:26:39,989 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:26:40,893 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:26:41,894 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:26:43,990 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:26:44,334 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:26:44,334 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:26:44,335 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:26:44,617 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:26:44,618 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:26:44,618 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:26:44,623 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:26:44,896 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:26:45,898 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:26:49,495 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:26:49,496 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:26:49,496 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:26:49,497 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:26:49,501 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:26:49,900 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:26:51,904 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:26:54,502 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:26:54,918 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:26:54,919 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:26:54,919 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:26:54,924 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:26:55,121 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 17:26:55,908 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:26:55,908 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:26:59,334 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:26:59,334 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:26:59,336 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:26:59,564 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:27:00,823 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:27:00,824 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:27:00,824 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:27:00,828 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:27:00,912 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:27:01,914 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:27:04,760 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:27:04,761 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:27:04,761 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:27:04,761 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:27:04,766 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:27:04,916 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:27:05,919 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:27:09,766 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:27:10,846 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:27:10,847 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:27:10,847 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:27:10,852 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:27:10,921 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:27:11,924 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:27:14,334 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:27:14,334 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:27:14,336 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:27:14,483 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:27:14,543 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:27:14,543 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:27:14,547 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:27:14,926 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:27:15,548 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:27:15,928 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:27:18,092 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:27:18,092 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:27:18,092 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:27:18,097 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:27:18,930 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:27:19,932 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:27:21,137 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:27:23,681 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:27:23,682 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:27:23,682 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:27:23,687 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:27:23,935 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:27:25,123 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 17:27:25,938 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:27:27,125 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:27:29,334 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:27:29,334 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:27:29,336 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:27:29,366 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:27:29,613 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:27:29,613 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:27:29,618 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:27:29,941 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:27:31,944 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:27:32,619 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:27:33,181 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:27:33,182 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:27:33,182 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:27:33,187 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:27:33,945 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:27:35,949 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:27:38,188 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:27:40,583 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:27:40,584 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:27:40,584 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:27:40,589 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:27:40,952 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:27:41,957 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:27:43,590 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:27:44,334 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:27:44,334 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:27:44,336 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:27:44,932 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:27:44,933 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:27:44,933 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:27:44,938 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:27:44,958 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:27:45,963 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:27:48,713 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:27:48,714 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:27:48,714 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:27:48,714 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:27:48,719 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:27:48,964 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:27:49,969 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:27:53,148 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:27:53,148 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:27:53,149 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:27:53,154 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:27:53,972 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:27:54,154 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:27:55,127 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 17:27:55,977 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:27:56,838 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:27:56,839 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:27:56,839 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:27:56,845 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:27:56,979 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:27:57,982 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:27:59,334 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:27:59,334 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:27:59,336 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:27:59,636 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:28:00,758 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:28:00,758 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:28:00,758 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:28:00,763 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:28:00,983 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:28:01,986 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:28:04,801 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:28:06,181 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:28:06,181 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:28:06,182 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:28:06,186 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:28:06,991 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:28:07,995 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:28:10,122 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:28:10,123 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:28:10,123 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:28:10,123 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:28:10,128 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:28:10,998 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:28:12,002 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:28:14,334 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:28:14,335 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:28:14,336 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:28:15,546 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:28:16,156 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:28:16,156 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:28:16,156 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:28:16,161 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:28:17,006 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:28:18,010 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:28:21,201 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:28:21,466 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:28:21,467 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:28:21,467 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:28:21,472 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:28:22,014 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:28:24,019 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:28:25,133 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 17:28:27,134 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:28:27,814 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:28:27,815 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:28:27,815 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:28:27,820 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:28:28,023 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:28:29,334 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:28:29,335 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:28:29,336 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:28:30,028 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:28:31,864 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:28:31,865 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:28:31,865 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:28:31,870 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:28:32,029 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:28:32,909 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:28:34,034 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:28:36,219 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:28:36,220 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:28:36,220 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:28:36,225 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:28:37,036 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:28:38,040 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:28:38,225 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:28:40,071 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:28:40,072 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:28:40,072 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:28:40,077 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:28:41,043 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:28:42,047 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:28:44,077 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:28:44,334 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:28:44,335 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:28:44,336 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:28:45,768 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:28:45,769 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:28:45,769 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:28:45,774 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:28:46,051 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:28:48,056 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:28:49,813 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:28:50,078 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:28:50,079 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:28:50,079 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:28:50,084 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:28:51,057 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:28:52,062 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:28:53,885 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:28:53,886 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:28:53,886 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:28:53,891 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:28:54,063 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:28:54,892 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:28:55,133 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 17:28:56,068 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:28:58,212 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:28:58,213 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:28:58,213 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:28:58,218 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:28:59,071 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:28:59,334 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:28:59,335 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:28:59,336 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:29:00,074 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:29:00,573 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:29:04,909 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:29:04,910 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:29:04,910 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:29:04,915 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:29:05,078 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:29:05,953 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:29:06,081 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:29:10,732 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:29:10,732 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:29:10,733 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:29:10,738 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:29:11,086 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:29:11,739 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:29:12,089 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:29:14,335 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:29:14,335 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:29:14,337 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:29:14,597 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:29:14,597 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:29:14,597 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:29:14,602 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:29:15,091 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:29:16,095 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:29:17,641 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:29:17,813 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:29:17,814 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:29:17,814 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:29:17,819 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:29:18,096 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:29:20,100 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:29:22,820 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:29:25,139 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 17:29:26,114 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:29:26,115 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:29:26,115 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:29:26,120 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:29:27,104 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:29:28,104 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:29:28,121 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:29:29,335 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:29:29,335 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:29:29,337 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:29:33,619 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:29:34,298 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:29:34,298 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:29:34,298 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:29:34,303 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:29:35,109 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:29:36,111 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:29:39,193 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:29:39,194 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:29:39,194 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:29:39,194 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:29:39,199 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:29:40,116 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:29:40,117 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:29:44,200 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:29:44,283 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:29:44,284 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:29:44,284 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:29:44,289 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:29:44,335 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:29:44,335 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:29:44,337 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:29:45,119 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:29:46,122 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:29:49,581 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:29:49,985 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:29:49,986 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:29:49,986 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:29:49,991 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:29:50,124 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:29:52,127 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:29:54,151 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:29:54,152 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:29:54,152 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:29:54,158 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:29:55,129 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:29:55,138 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 17:29:55,139 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:29:56,132 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:29:58,462 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:29:58,463 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:29:58,463 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:29:58,468 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:29:59,135 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:29:59,335 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:29:59,335 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:29:59,336 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:30:00,137 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:30:00,787 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:30:02,662 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:30:02,663 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:30:02,663 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:30:02,668 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:30:03,140 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:30:04,140 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:30:06,468 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:30:06,469 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:30:06,469 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:30:06,470 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:30:06,474 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:30:07,142 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:30:08,144 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:30:11,475 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:30:11,556 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:30:11,556 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:30:11,557 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:30:11,561 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:30:12,147 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:30:14,150 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:30:14,335 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:30:14,335 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:30:14,337 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:30:15,483 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:30:15,484 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:30:15,484 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:30:15,489 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:30:16,151 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:30:16,529 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:30:18,155 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:30:20,297 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:30:20,298 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:30:20,298 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:30:20,303 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:30:21,156 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:30:22,159 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:30:22,304 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:30:24,062 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:30:24,063 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:30:24,063 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:30:24,068 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:30:24,160 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:30:25,143 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 17:30:26,163 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:30:28,145 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:30:29,335 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:30:29,335 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:30:29,337 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:30:29,425 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:30:29,628 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:30:29,629 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:30:29,634 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:30:30,165 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:30:32,168 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:30:33,634 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:30:37,816 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:30:37,817 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:30:37,817 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:30:37,822 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:30:38,172 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:30:38,822 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:30:40,176 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:30:41,116 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:30:41,117 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:30:41,117 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:30:41,122 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:30:41,177 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:30:42,179 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:30:44,123 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:30:44,335 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:30:44,335 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:30:44,337 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:30:45,131 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:30:45,131 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:30:45,131 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:30:45,136 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:30:45,181 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:30:46,184 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:30:49,173 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:30:50,500 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:30:50,501 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:30:50,501 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:30:50,506 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:30:51,187 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:30:52,189 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:30:54,506 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:30:55,143 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 17:30:59,335 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:30:59,336 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:30:59,337 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:30:59,499 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:30:59,591 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:30:59,591 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:30:59,591 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:30:59,597 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:31:00,195 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:31:02,198 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:31:04,598 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:31:09,598 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:31:10,794 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:31:10,795 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:31:10,795 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:31:10,800 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:31:11,203 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:31:12,206 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:31:14,335 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:31:14,336 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:31:14,337 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:31:15,285 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:31:15,285 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:31:15,285 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:31:15,285 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:31:15,290 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:31:16,208 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:31:16,208 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:31:20,138 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:31:20,139 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:31:20,139 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:31:20,144 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:31:20,210 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:31:21,145 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:31:22,214 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:31:24,275 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:31:24,276 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:31:24,276 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:31:24,282 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:31:25,145 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 17:31:25,215 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:31:26,146 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:31:26,218 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:31:29,335 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:31:29,336 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:31:29,337 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:31:31,547 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:31:33,246 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:31:33,247 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:31:33,247 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:31:33,252 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:31:34,224 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:31:34,224 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:31:37,289 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:31:40,915 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:31:40,915 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:31:40,916 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:31:40,920 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:31:41,230 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:31:42,232 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:31:42,921 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:31:44,335 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:31:44,336 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:31:44,337 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:31:47,419 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:31:47,420 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:31:47,420 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:31:47,425 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:31:48,235 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:31:48,461 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:31:50,238 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:31:51,909 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:31:51,910 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:31:51,910 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:31:51,915 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:31:52,240 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:31:53,915 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:31:54,243 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:31:55,152 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 17:31:56,839 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:31:56,840 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:31:56,840 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:31:56,844 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:31:57,245 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:31:58,248 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:31:59,336 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:31:59,336 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:31:59,337 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:31:59,338 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:32:00,331 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:32:00,332 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:32:00,332 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:32:00,337 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:32:01,250 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:32:02,252 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:32:04,269 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:32:04,270 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:32:04,270 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:32:04,275 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:32:05,254 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:32:05,276 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:32:06,256 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:32:08,256 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:32:08,256 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:32:08,257 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:32:08,262 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:32:09,258 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:32:10,261 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:32:11,262 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:32:13,502 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:32:13,502 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:32:13,503 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:32:13,507 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:32:14,264 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:32:14,336 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:32:14,336 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:32:14,337 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:32:16,267 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:32:16,662 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:32:21,662 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:32:21,697 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:32:21,697 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:32:21,697 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:32:21,702 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:32:22,270 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:32:24,273 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:32:25,152 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 17:32:25,553 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:32:25,554 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:32:25,554 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:32:25,560 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:32:26,274 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:32:27,560 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:32:28,277 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:32:29,336 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:32:29,336 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:32:29,338 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:32:31,477 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:32:31,478 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:32:31,478 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:32:31,483 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:32:32,280 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:32:33,521 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:32:34,283 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:32:36,328 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:32:36,329 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:32:36,329 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:32:36,334 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:32:37,285 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:32:38,288 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:32:39,335 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:32:41,051 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:32:41,052 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:32:41,052 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:32:41,057 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:32:41,291 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:32:42,293 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:32:44,336 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:32:44,336 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:32:44,338 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:32:44,547 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:32:45,082 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:32:45,083 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:32:45,083 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:32:45,088 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:32:45,295 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:32:46,297 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:32:50,125 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:32:50,290 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:32:50,290 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:32:50,290 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:32:50,295 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:32:50,300 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:32:52,303 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:32:55,158 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 17:32:55,160 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:32:56,528 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:32:56,529 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:32:56,529 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:32:56,534 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:32:57,307 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:32:58,309 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:32:59,336 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:32:59,336 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:32:59,338 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:33:00,603 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:33:01,017 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:33:01,018 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:33:01,018 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:33:01,023 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:33:01,311 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:33:02,313 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:33:06,035 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:33:06,036 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:33:06,036 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:33:06,036 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:33:06,040 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:33:06,316 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:33:08,319 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:33:09,801 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:33:09,802 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:33:09,802 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:33:09,807 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:33:10,320 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:33:11,807 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:33:12,324 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:33:14,336 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:33:14,336 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:33:14,338 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:33:14,793 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:33:14,793 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:33:14,793 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:33:14,798 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:33:15,325 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:33:16,328 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:33:16,837 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:33:18,718 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:33:18,718 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:33:18,719 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:33:18,723 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:33:19,329 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:33:20,332 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:33:22,724 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:33:22,831 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:33:22,832 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:33:22,832 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:33:22,837 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:33:23,334 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:33:24,337 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:33:25,158 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 17:33:28,160 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:33:28,211 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:33:28,211 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:33:28,212 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:33:28,216 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:33:28,339 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:33:29,336 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:33:29,337 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:33:29,338 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:33:30,342 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:33:32,532 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:33:32,533 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:33:32,533 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:33:32,539 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:33:33,344 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:33:33,577 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:33:34,346 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:33:37,592 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:33:37,593 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:33:37,593 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:33:37,598 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:33:38,349 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:33:38,599 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:33:40,353 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:33:41,813 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:33:41,813 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:33:41,814 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:33:41,818 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:33:42,354 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:33:43,819 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:33:44,336 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:33:44,337 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:33:44,338 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:33:44,356 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:33:45,984 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:33:45,984 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:33:45,984 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:33:45,989 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:33:46,357 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:33:48,360 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:33:49,029 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:33:51,965 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:33:51,966 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:33:51,966 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:33:51,971 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:33:52,362 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:33:54,363 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:33:54,971 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:33:55,164 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 17:33:55,886 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:33:55,887 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:33:55,887 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:33:55,892 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:33:56,364 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:33:58,368 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:33:59,336 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:33:59,337 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:33:59,338 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:33:59,901 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:33:59,901 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:33:59,901 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:33:59,907 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:34:00,369 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:34:00,945 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:34:02,373 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:34:04,670 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:34:04,671 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:34:04,671 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:34:04,676 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:34:05,374 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:34:06,377 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:34:06,676 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:34:08,324 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:34:08,325 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:34:08,325 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:34:08,330 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:34:08,378 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:34:10,381 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:34:12,330 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:34:12,515 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:34:12,516 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:34:12,516 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:34:12,521 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:34:13,383 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:34:14,336 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:34:14,337 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:34:14,338 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:34:14,385 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:34:16,174 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:34:16,175 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:34:16,175 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:34:16,180 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:34:16,387 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:34:18,217 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:34:18,390 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:34:19,882 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:34:19,883 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:34:19,883 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:34:19,888 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:34:20,391 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:34:22,394 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:34:23,503 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:34:23,504 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:34:23,504 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:34:23,505 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:34:23,509 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:34:24,397 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:34:24,397 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:34:25,164 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 17:34:28,772 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:34:28,773 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:34:28,773 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:34:28,773 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:34:28,779 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:34:29,337 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:34:29,337 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:34:29,338 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:34:29,400 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:34:30,402 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:34:32,228 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:34:32,229 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:34:32,229 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:34:32,234 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:34:32,403 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:34:34,273 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:34:34,404 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:34:36,011 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:34:36,011 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:34:36,012 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:34:36,016 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:34:36,406 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:34:38,409 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:34:40,017 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:34:42,735 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:34:42,736 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:34:42,736 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:34:42,741 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:34:43,413 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:34:44,337 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:34:44,337 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:34:44,339 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:34:44,415 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:34:45,620 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:34:46,348 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:34:46,349 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:34:46,349 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:34:46,354 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:34:46,416 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:34:48,419 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:34:51,393 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:34:54,645 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:34:54,646 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:34:54,646 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:34:54,651 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:34:55,166 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 17:34:55,423 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:34:56,426 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:34:57,167 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:34:58,094 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:34:58,095 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:34:58,095 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:34:58,100 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:34:58,428 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:34:59,337 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:34:59,337 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:34:59,339 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:35:00,431 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:35:02,475 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:35:02,475 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:35:02,476 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:35:02,476 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:35:02,480 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:35:03,433 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:35:04,435 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:35:06,443 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:35:06,443 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:35:06,444 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:35:06,448 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:35:07,437 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:35:08,439 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:35:08,449 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:35:10,647 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:35:10,647 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:35:10,648 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:35:10,652 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:35:11,442 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:35:12,444 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:35:13,653 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:35:14,337 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:35:14,337 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:35:14,339 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:35:14,739 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:35:14,740 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:35:14,740 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:35:14,745 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:35:15,446 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:35:16,448 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:35:18,370 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:35:18,370 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:35:18,371 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:35:18,375 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:35:18,450 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:35:19,376 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:35:20,453 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:35:21,994 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:35:21,995 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:35:21,995 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:35:22,000 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:35:22,454 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:35:24,455 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:35:25,000 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:35:25,169 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 17:35:27,316 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:35:27,317 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:35:27,317 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:35:27,322 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:35:27,456 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:35:28,459 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:35:29,337 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:35:29,337 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:35:29,339 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:35:30,602 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:35:31,316 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:35:31,317 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:35:31,317 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:35:31,322 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:35:31,461 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:35:32,463 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:35:35,701 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:35:35,701 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:35:35,702 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:35:35,702 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:35:35,707 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:35:36,466 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:35:38,469 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:35:39,524 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:35:39,524 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:35:39,525 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:35:39,529 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:35:40,473 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:35:40,473 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:35:41,530 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:35:43,566 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:35:43,566 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:35:43,567 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:35:43,571 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:35:44,337 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:35:44,337 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:35:44,339 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:35:44,477 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:35:44,478 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:35:46,615 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:35:47,829 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:35:47,830 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:35:47,830 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:35:47,835 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:35:48,480 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:35:50,483 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:35:51,873 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:35:53,162 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:35:53,163 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:35:53,163 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:35:53,168 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:35:53,484 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:35:54,487 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:35:55,173 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 17:35:57,144 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:35:57,144 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:35:57,145 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:35:57,145 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:35:57,150 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:35:57,490 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:35:58,491 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:35:59,337 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:35:59,337 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:35:59,339 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:36:00,936 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:36:00,937 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:36:00,937 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:36:00,942 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:36:01,492 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:36:02,495 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:36:02,981 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:36:04,521 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:36:04,521 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:36:04,522 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:36:04,527 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:36:05,497 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:36:06,499 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:36:08,528 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:36:10,158 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:36:10,159 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:36:10,159 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:36:10,165 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:36:10,501 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:36:12,505 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:36:13,558 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:36:13,559 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:36:13,559 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:36:13,559 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:36:13,564 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:36:14,337 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:36:14,338 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:36:14,339 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:36:14,508 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:36:14,508 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:36:18,204 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:36:18,204 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:36:18,204 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:36:18,209 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:36:18,510 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:36:19,249 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:36:20,513 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:36:22,611 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:36:22,612 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:36:22,612 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:36:22,617 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:36:23,515 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:36:24,518 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:36:24,617 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:36:25,176 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 17:36:29,337 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:36:29,338 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:36:29,339 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:36:29,636 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:36:30,123 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:36:30,123 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:36:30,124 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:36:30,128 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:36:30,521 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:36:32,524 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:36:33,922 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:36:33,923 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:36:33,923 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:36:33,928 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:36:34,526 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:36:34,928 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:36:36,529 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:36:38,809 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:36:38,810 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:36:38,810 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:36:38,815 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:36:39,532 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:36:40,532 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:36:40,815 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:36:43,866 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:36:43,866 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:36:43,866 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:36:43,871 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:36:44,337 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:36:44,338 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:36:44,339 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:36:44,535 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:36:46,538 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:36:46,622 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:36:47,684 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:36:47,685 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:36:47,685 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:36:47,690 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:36:48,541 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:36:48,541 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:36:51,411 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:36:51,412 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:36:51,412 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:36:51,417 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:36:51,543 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:36:52,417 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:36:52,545 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:36:55,179 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 17:36:55,575 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:36:55,575 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:36:55,576 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:36:55,581 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:36:56,550 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:36:56,551 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:36:57,581 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:36:59,337 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:36:59,338 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:36:59,339 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:36:59,728 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:36:59,729 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:36:59,729 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:36:59,734 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:37:00,553 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:37:02,556 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:37:02,774 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:37:03,541 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:37:03,542 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:37:03,542 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:37:03,547 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:37:03,557 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:37:04,559 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:37:06,826 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:37:06,827 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:37:06,827 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:37:06,832 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:37:07,561 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:37:07,832 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:37:08,564 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:37:12,832 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:37:14,337 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:37:14,338 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:37:14,340 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:37:14,656 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:37:14,657 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:37:14,657 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:37:14,662 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:37:15,568 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:37:16,570 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:37:18,701 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:37:19,887 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:37:19,888 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:37:19,888 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:37:19,893 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:37:20,573 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:37:22,576 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:37:23,893 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:37:25,082 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:37:25,083 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:37:25,083 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:37:25,088 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:37:25,181 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 17:37:25,577 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:37:26,580 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:37:29,182 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:37:29,279 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:37:29,279 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:37:29,280 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:37:29,284 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:37:29,338 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:37:29,338 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:37:29,339 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:37:29,582 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:37:30,582 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:37:34,169 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:37:34,170 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:37:34,170 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:37:34,175 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:37:34,213 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:37:34,584 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:37:36,588 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:37:39,213 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:37:39,460 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:37:39,461 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:37:39,461 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:37:39,466 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:37:39,591 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:37:40,594 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:37:44,159 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:37:44,160 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:37:44,160 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:37:44,164 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:37:44,338 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:37:44,338 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:37:44,340 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:37:44,572 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:37:44,596 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:37:46,599 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:37:49,573 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:37:50,744 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:37:50,745 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:37:50,745 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:37:50,749 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:37:51,602 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:37:52,604 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:37:54,789 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:37:55,184 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 17:37:55,650 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:37:55,651 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:37:55,651 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:37:55,655 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:37:56,608 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:37:56,608 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:37:59,338 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:37:59,338 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:37:59,340 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:38:00,582 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:38:01,065 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:38:01,066 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:38:01,066 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:38:01,071 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:38:01,612 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:38:02,615 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:38:05,121 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:38:05,122 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:38:05,122 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:38:05,127 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:38:05,616 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:38:06,128 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:38:06,619 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:38:11,128 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:38:11,258 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:38:11,258 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:38:11,259 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:38:11,263 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:38:11,622 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:38:12,625 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:38:14,338 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:38:14,338 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:38:14,340 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:38:15,097 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:38:15,097 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:38:15,097 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:38:15,102 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:38:15,626 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:38:16,141 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:38:16,627 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:38:19,094 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:38:19,095 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:38:19,095 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:38:19,100 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:38:19,629 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:38:20,631 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:38:22,101 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:38:23,217 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:38:23,218 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:38:23,218 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:38:23,223 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:38:23,633 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:38:24,636 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:38:25,191 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 17:38:27,192 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:38:28,317 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:38:28,318 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:38:28,318 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:38:28,323 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:38:28,638 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:38:29,338 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:38:29,338 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:38:29,340 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:38:30,641 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:38:32,590 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:38:33,698 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:38:33,699 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:38:33,699 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:38:33,703 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:38:34,645 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:38:34,645 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:38:37,741 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:38:37,952 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:38:37,952 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:38:37,953 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:38:37,957 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:38:38,647 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:38:40,652 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:38:42,376 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:38:42,377 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:38:42,377 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:38:42,382 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:38:42,653 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:38:43,382 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:38:44,338 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:38:44,338 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:38:44,340 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:38:44,656 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:38:46,395 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:38:46,396 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:38:46,396 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:38:46,400 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:38:46,657 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:38:48,437 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:38:48,658 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:38:51,520 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:38:51,521 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:38:51,521 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:38:51,526 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:38:51,659 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:38:52,662 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:38:53,526 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:38:55,189 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 17:38:55,273 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:38:55,274 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:38:55,274 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:38:55,279 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:38:55,664 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:38:56,666 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:38:59,280 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:38:59,338 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:38:59,338 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:38:59,340 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:38:59,476 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:38:59,615 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:38:59,615 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:38:59,620 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:38:59,669 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:39:00,670 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:39:03,095 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:39:03,095 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:39:03,096 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:39:03,100 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:39:03,672 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:39:04,674 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:39:05,101 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:39:07,749 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:39:07,750 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:39:07,750 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:39:07,755 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:39:08,678 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:39:08,679 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:39:10,756 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:39:11,572 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:39:11,573 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:39:11,573 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:39:11,577 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:39:11,680 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:39:12,683 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:39:14,338 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:39:14,339 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:39:14,340 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:39:15,146 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:39:15,146 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:39:15,146 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:39:15,151 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:39:15,684 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:39:16,189 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:39:16,687 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:39:19,323 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:39:19,323 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:39:19,324 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:39:19,329 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:39:19,689 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:39:20,691 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:39:21,329 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:39:25,195 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 17:39:26,261 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:39:26,261 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:39:26,262 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:39:26,267 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:39:26,695 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:39:27,268 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:39:28,698 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:39:29,338 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:39:29,339 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:39:29,340 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:39:30,845 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:39:30,845 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:39:30,845 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:39:30,850 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:39:31,700 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:39:32,702 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:39:32,889 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:39:35,208 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:39:35,209 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:39:35,209 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:39:35,214 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:39:35,704 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:39:36,705 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:39:38,214 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:39:39,079 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:39:39,080 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:39:39,080 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:39:39,085 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:39:39,707 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:39:40,710 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:39:42,755 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:39:42,756 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:39:42,756 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:39:42,760 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:39:43,713 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:39:43,761 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:39:44,338 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:39:44,339 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:39:44,340 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:39:44,715 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:39:47,204 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:39:47,205 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:39:47,205 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:39:47,210 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:39:47,717 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:39:48,719 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:39:49,249 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:39:50,734 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:39:50,735 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:39:50,735 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:39:50,740 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:39:51,721 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:39:52,723 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:39:54,741 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:39:55,194 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 17:39:55,270 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:39:55,271 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:39:55,271 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:39:55,275 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:39:55,725 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:39:56,728 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:39:59,235 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:39:59,236 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:39:59,236 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:39:59,241 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:39:59,338 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:39:59,339 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:39:59,340 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:39:59,731 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:40:00,600 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:40:00,733 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:40:04,421 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:40:04,421 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:40:04,422 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:40:04,426 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:40:04,736 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:40:06,465 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:40:06,739 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:40:08,741 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:40:08,742 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:40:08,742 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:40:08,747 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:40:09,740 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:40:10,743 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:40:11,747 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:40:14,339 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:40:14,339 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:40:14,340 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:40:14,404 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:40:14,617 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:40:14,617 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:40:14,622 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:40:14,745 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:40:16,749 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:40:17,623 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:40:18,222 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:40:18,223 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:40:18,223 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:40:18,228 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:40:18,750 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:40:20,753 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:40:22,978 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:40:22,979 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:40:22,979 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:40:22,979 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:40:22,984 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:40:23,754 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:40:24,757 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:40:25,198 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 17:40:28,199 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:40:29,339 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:40:29,339 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:40:29,340 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:40:32,135 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:40:32,138 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:40:32,139 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:40:32,144 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:40:32,762 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:40:34,177 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:40:34,765 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:40:37,940 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:40:37,941 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:40:37,941 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:40:37,946 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:40:38,769 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:40:38,769 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:40:39,946 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:40:42,521 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:40:42,521 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:40:42,521 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:40:42,527 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:40:42,773 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:40:44,339 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:40:44,339 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:40:44,341 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:40:44,776 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:40:45,635 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:40:46,686 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:40:46,686 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:40:46,686 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:40:46,693 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:40:46,777 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:40:48,780 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:40:50,729 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:40:51,641 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:40:51,642 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:40:51,642 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:40:51,647 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:40:51,782 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:40:52,784 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:40:55,201 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 17:40:56,202 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:40:56,723 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:40:56,723 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:40:56,724 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:40:56,728 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:40:56,787 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:40:58,791 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:40:59,339 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:40:59,339 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:40:59,341 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:41:01,570 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:41:01,662 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:41:01,663 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:41:01,663 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:41:01,668 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:41:01,793 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:41:02,795 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:41:06,705 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:41:07,775 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:41:07,776 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:41:07,776 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:41:07,781 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:41:07,798 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:41:08,801 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:41:11,782 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:41:13,843 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:41:13,843 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:41:13,844 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:41:13,849 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:41:14,339 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:41:14,339 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:41:14,341 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:41:14,807 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:41:14,807 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:41:17,554 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:41:17,891 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:41:17,892 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:41:17,892 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:41:17,897 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:41:18,811 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:41:18,811 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:41:22,937 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:41:23,320 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:41:23,320 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:41:23,321 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:41:23,325 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:41:23,814 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:41:24,816 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:41:25,204 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 17:41:28,189 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:41:28,190 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:41:28,190 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:41:28,190 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:41:28,195 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:41:28,819 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:41:29,339 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:41:29,339 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:41:29,341 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:41:30,822 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:41:32,134 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:41:32,134 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:41:32,135 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:41:32,139 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:41:32,823 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:41:34,177 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:41:34,826 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:41:38,606 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:41:38,607 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:41:38,607 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:41:38,612 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:41:38,829 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:41:39,613 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:41:40,833 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:41:43,317 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:41:43,318 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:41:43,318 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:41:43,323 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:41:43,835 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:41:44,339 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:41:44,339 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:41:44,341 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:41:44,835 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:41:45,581 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:41:50,582 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:41:51,294 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:41:51,294 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:41:51,294 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:41:51,299 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:41:51,839 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:41:52,842 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:41:55,210 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 17:41:56,212 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:41:59,339 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:41:59,340 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:41:59,341 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:41:59,657 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:41:59,658 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:41:59,658 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:41:59,663 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:41:59,847 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:42:01,701 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:42:02,850 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:42:05,938 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:42:05,938 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:42:05,939 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:42:05,943 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:42:06,852 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:42:06,944 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:42:08,856 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:42:11,944 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:42:13,578 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:42:13,579 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:42:13,579 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:42:13,584 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:42:13,859 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:42:14,339 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:42:14,340 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:42:14,341 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:42:14,862 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:42:16,880 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:42:16,881 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:42:16,881 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:42:16,886 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:42:17,863 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:42:17,925 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:42:18,866 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:42:20,872 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:42:20,873 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:42:20,873 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:42:20,878 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:42:21,867 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:42:22,870 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:42:23,879 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:42:25,046 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:42:25,047 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:42:25,047 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:42:25,052 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:42:25,209 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 17:42:25,871 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:42:26,874 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:42:29,097 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:42:29,098 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:42:29,099 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:42:29,099 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:42:29,104 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:42:29,340 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:42:29,340 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:42:29,342 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:42:29,876 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:42:30,878 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:42:33,913 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:42:33,913 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:42:33,914 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:42:33,919 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:42:34,880 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:42:34,957 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:42:36,884 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:42:37,860 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:42:37,860 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:42:37,861 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:42:37,866 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:42:37,884 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:42:40,866 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:42:40,889 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:42:42,151 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:42:42,152 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:42:42,152 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:42:42,157 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:42:42,890 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:42:44,339 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:42:44,340 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:42:44,341 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:42:44,893 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:42:46,593 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:42:47,251 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:42:47,252 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:42:47,252 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:42:47,257 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:42:47,895 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:42:48,898 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:42:52,293 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:42:52,429 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:42:52,430 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:42:52,430 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:42:52,435 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:42:52,900 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:42:54,903 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:42:55,215 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 17:42:56,192 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:42:56,193 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:42:56,193 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:42:56,198 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:42:56,904 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:42:58,199 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:42:58,908 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:42:59,340 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:42:59,340 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:42:59,341 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:43:00,338 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:43:00,338 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:43:00,338 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:43:00,343 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:43:00,909 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:43:02,912 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:43:03,381 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:43:04,851 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:43:04,852 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:43:04,853 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:43:04,858 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:43:04,914 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:43:06,917 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:43:08,858 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:43:09,068 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:43:09,069 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:43:09,069 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:43:09,075 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:43:09,918 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:43:10,921 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:43:12,920 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:43:12,921 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:43:12,921 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:43:12,926 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:43:13,923 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:43:13,927 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:43:14,340 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:43:14,340 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:43:14,342 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:43:14,926 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:43:19,564 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:43:22,020 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:43:22,020 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:43:22,021 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:43:22,026 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:43:22,930 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:43:24,931 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:43:25,065 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:43:25,216 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 17:43:26,707 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:43:26,708 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:43:26,708 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:43:26,713 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:43:26,933 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:43:28,936 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:43:29,340 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:43:29,340 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:43:29,342 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:43:30,563 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:43:31,311 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:43:31,312 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:43:31,312 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:43:31,317 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:43:31,937 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:43:32,940 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:43:34,722 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:43:34,723 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:43:34,723 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:43:34,728 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:43:34,941 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:43:35,728 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:43:36,945 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:43:40,535 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:43:40,536 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:43:40,537 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:43:40,542 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:43:40,948 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:43:41,542 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:43:42,951 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:43:44,340 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:43:44,340 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:43:44,342 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:43:44,388 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:43:44,555 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:43:44,555 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:43:44,560 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:43:44,952 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:43:46,561 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:43:46,955 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:43:48,004 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:43:48,005 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:43:48,005 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:43:48,010 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:43:48,956 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:43:50,959 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:43:52,010 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:43:53,539 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:43:53,540 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:43:53,540 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:43:53,545 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:43:53,961 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:43:54,983 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:43:55,222 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 17:43:57,223 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:43:58,130 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:43:58,131 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:43:58,131 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:43:58,137 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:43:58,986 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:43:59,340 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:43:59,340 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:43:59,342 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:44:00,990 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:44:02,317 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:44:02,318 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:44:02,318 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:44:02,318 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:44:02,323 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:44:02,991 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:44:04,994 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:44:06,636 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:44:06,637 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:44:06,637 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:44:06,642 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:44:06,995 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:44:07,642 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:44:08,998 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:44:10,566 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:44:10,567 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:44:10,567 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:44:10,572 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:44:10,999 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:44:13,001 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:44:13,573 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:44:14,340 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:44:14,340 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:44:14,342 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:44:14,691 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:44:14,692 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:44:14,692 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:44:14,698 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:44:15,003 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:44:17,006 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:44:18,603 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:44:18,604 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:44:18,605 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:44:18,605 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:44:18,610 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:44:19,007 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:44:21,010 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:44:23,610 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:44:25,117 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:44:25,118 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:44:25,118 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:44:25,123 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:44:25,221 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 17:44:26,013 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:44:27,016 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:44:28,914 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:44:28,915 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:44:28,915 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:44:28,915 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:44:28,920 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:44:29,017 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:44:29,340 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:44:29,341 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:44:29,342 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:44:31,020 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:44:34,552 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:44:34,593 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:44:34,593 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:44:34,594 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:44:34,599 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:44:35,022 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:44:37,026 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:44:38,747 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:44:38,748 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:44:38,748 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:44:38,753 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:44:39,027 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:44:39,754 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:44:41,031 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:44:42,532 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:44:42,533 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:44:42,534 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:44:42,539 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:44:43,032 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:44:44,340 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:44:44,341 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:44:44,342 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:44:45,035 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:44:45,605 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:44:46,474 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:44:46,475 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:44:46,475 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:44:46,481 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:44:47,036 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:44:49,039 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:44:51,517 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:44:52,281 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:44:52,282 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:44:52,282 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:44:52,287 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:44:53,042 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:44:55,045 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:44:55,224 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 17:44:55,977 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:44:55,978 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:44:55,979 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:44:55,983 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:44:56,045 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:44:56,984 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:44:59,051 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:44:59,340 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:44:59,341 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:44:59,342 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:44:59,902 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:44:59,906 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:44:59,906 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:44:59,911 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:45:00,052 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:45:02,945 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:45:03,056 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:45:05,716 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:45:05,716 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:45:05,717 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:45:05,722 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:45:06,057 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:45:07,060 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:45:08,722 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:45:11,374 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:45:11,375 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:45:11,375 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:45:11,380 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:45:12,063 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:45:13,066 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:45:14,340 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:45:14,341 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:45:14,342 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:45:14,643 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:45:15,118 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:45:15,118 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:45:15,119 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:45:15,124 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:45:16,068 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:45:17,068 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:45:19,867 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:45:19,868 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:45:19,868 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:45:19,869 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:45:19,874 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:45:20,070 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:45:23,074 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:45:24,385 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:45:24,386 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:45:24,386 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:45:24,391 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:45:25,075 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:45:25,226 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 17:45:25,228 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:45:27,078 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:45:29,340 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:45:29,341 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:45:29,342 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:45:29,740 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:45:29,741 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:45:29,741 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:45:29,746 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:45:30,080 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:45:30,785 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:45:31,082 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:45:35,785 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:45:36,291 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:45:36,291 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:45:36,292 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:45:36,297 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:45:37,086 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:45:39,089 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:45:40,137 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:45:40,138 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:45:40,138 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:45:40,143 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:45:41,092 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:45:41,144 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:45:43,096 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:45:44,341 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:45:44,341 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:45:44,343 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:45:45,007 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:45:45,008 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:45:45,008 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:45:45,013 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:45:45,097 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:45:47,049 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:45:47,100 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:45:48,760 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:45:48,761 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:45:48,761 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:45:48,766 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:45:49,101 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:45:51,104 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:45:52,767 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:45:52,830 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:45:52,831 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:45:52,832 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:45:52,836 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:45:53,106 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:45:55,108 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:45:55,229 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 17:45:58,231 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:45:58,737 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:45:58,737 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:45:58,738 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:45:58,743 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:45:59,112 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:45:59,341 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:45:59,341 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:45:59,343 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:46:01,115 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:46:03,650 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:46:08,109 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:46:08,110 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:46:08,110 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:46:08,115 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:46:08,119 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:46:09,153 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:46:11,123 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:46:12,693 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:46:12,694 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:46:12,694 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:46:12,700 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:46:13,124 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:46:14,341 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:46:14,341 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:46:14,343 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:46:14,592 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:46:15,126 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:46:16,822 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:46:16,823 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:46:16,823 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:46:16,828 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:46:17,127 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:46:19,130 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:46:19,865 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:46:20,102 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:46:20,106 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:46:20,106 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:46:20,111 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:46:20,130 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:46:23,134 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:46:24,674 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:46:24,675 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:46:24,675 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:46:24,680 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:46:25,135 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:46:25,232 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 17:46:25,233 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:46:27,139 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:46:29,341 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:46:29,341 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:46:29,343 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:46:29,590 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:46:29,591 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:46:29,591 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:46:29,597 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:46:30,140 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:46:30,633 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:46:31,143 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:46:33,140 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:46:33,141 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:46:33,141 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:46:33,146 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:46:33,146 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:46:35,149 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:46:36,147 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:46:37,381 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:46:37,382 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:46:37,382 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:46:37,387 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:46:38,151 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:46:39,154 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:46:41,388 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:46:42,813 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:46:42,814 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:46:42,814 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:46:42,819 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:46:43,157 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:46:44,341 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:46:44,341 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:46:44,343 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:46:45,161 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:46:46,620 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:46:47,701 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:46:47,702 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:46:47,702 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:46:47,707 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:46:48,162 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:46:49,165 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:46:51,654 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:46:51,655 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:46:51,655 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:46:51,655 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:46:51,660 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:46:52,166 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:46:53,167 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:46:55,236 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 17:46:56,119 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:46:56,120 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:46:56,120 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:46:56,125 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:46:56,169 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:46:57,126 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:46:59,172 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:46:59,341 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:46:59,341 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:46:59,343 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:47:00,155 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:47:00,155 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:47:00,156 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:47:00,160 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:47:00,172 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:47:02,197 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:47:03,176 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:47:06,639 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:47:06,640 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:47:06,641 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:47:06,645 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:47:07,178 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:47:07,646 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:47:09,181 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:47:11,884 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:47:11,885 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:47:11,886 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:47:11,891 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:47:12,184 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:47:12,892 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:47:13,187 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:47:14,341 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:47:14,341 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:47:14,343 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:47:16,039 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:47:16,039 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:47:16,040 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:47:16,045 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:47:16,188 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:47:18,081 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:47:19,192 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:47:21,045 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:47:21,046 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:47:21,046 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:47:21,052 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:47:21,193 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:47:23,196 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:47:24,053 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:47:25,240 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:47:25,241 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 17:47:25,243 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:47:25,243 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:47:25,248 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:47:26,198 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:47:27,201 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:47:29,249 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:47:29,341 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:47:29,341 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:47:29,343 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:47:29,653 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:47:29,653 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:47:29,654 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:47:29,658 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:47:30,203 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:47:31,205 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:47:34,318 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:47:34,319 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:47:34,319 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:47:34,320 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:47:34,324 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:47:35,207 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:47:37,211 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:47:37,693 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:47:37,695 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:47:37,696 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:47:37,701 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:47:38,211 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:47:39,214 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:47:39,701 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:47:41,880 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:47:41,881 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:47:41,881 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:47:41,887 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:47:42,216 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:47:43,219 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:47:44,341 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:47:44,342 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:47:44,343 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:47:45,645 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:47:45,705 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:47:45,706 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:47:45,708 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:47:45,713 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:47:46,221 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:47:47,221 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:47:50,750 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:47:50,770 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:47:50,771 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:47:50,771 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:47:50,776 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:47:51,224 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:47:53,226 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:47:55,241 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 17:47:56,242 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:47:59,132 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:47:59,133 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:47:59,133 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:47:59,138 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:47:59,231 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:47:59,341 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:47:59,342 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:47:59,344 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:48:01,234 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:48:01,591 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:48:06,592 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:48:06,632 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:48:06,632 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:48:06,633 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:48:06,638 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:48:07,237 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:48:09,240 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:48:10,151 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:48:10,152 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:48:10,152 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:48:10,157 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:48:10,241 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:48:12,158 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:48:13,245 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:48:14,341 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:48:14,342 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:48:14,343 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:48:16,805 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:48:16,805 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:48:16,806 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:48:16,811 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:48:17,247 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:48:17,849 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:48:19,251 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:48:22,168 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:48:22,169 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:48:22,169 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:48:22,175 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:48:22,253 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:48:23,175 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:48:25,247 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 17:48:25,256 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:48:28,156 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:48:28,156 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:48:28,157 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:48:28,162 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:48:28,258 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:48:29,162 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:48:29,341 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:48:29,342 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:48:29,343 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:48:31,261 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:48:32,636 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:48:32,637 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:48:32,637 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:48:32,642 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:48:33,262 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:48:34,681 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:48:35,265 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:48:37,947 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:48:37,948 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:48:37,948 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:48:37,953 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:48:38,267 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:48:39,270 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:48:39,954 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:48:44,342 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:48:44,342 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:48:44,344 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:48:45,594 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:48:47,018 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:48:47,018 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:48:47,019 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:48:47,023 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:48:47,275 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:48:49,278 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:48:50,970 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:48:50,971 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:48:50,972 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:48:50,972 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:48:50,977 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:48:51,279 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:48:53,282 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:48:54,927 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:48:54,928 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:48:54,928 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:48:54,934 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:48:55,246 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 17:48:55,284 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:48:56,248 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:48:57,287 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:48:59,342 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:48:59,342 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:48:59,344 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:49:00,761 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:49:00,761 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:49:00,762 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:49:00,766 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:49:01,290 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:49:01,805 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:49:03,293 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:49:06,805 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:49:08,930 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:49:08,931 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:49:08,931 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:49:08,936 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:49:09,296 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:49:11,299 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:49:11,936 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:49:12,757 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:49:12,758 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:49:12,758 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:49:12,763 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:49:13,301 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:49:14,342 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:49:14,342 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:49:14,344 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:49:15,304 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:49:17,638 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:49:18,212 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:49:18,213 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:49:18,213 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:49:18,218 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:49:18,305 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:49:21,309 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:49:21,855 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:49:21,856 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:49:21,856 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:49:21,861 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:49:22,310 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:49:22,861 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:49:23,310 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:49:25,249 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 17:49:25,921 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:49:25,922 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:49:25,922 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:49:25,927 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:49:26,312 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:49:27,315 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:49:27,927 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:49:29,342 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:49:29,342 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:49:29,344 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:49:30,761 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:49:30,762 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:49:30,762 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:49:30,767 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:49:31,317 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:49:33,320 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:49:33,808 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:49:34,508 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:49:34,509 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:49:34,509 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:49:34,514 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:49:35,321 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:49:37,325 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:49:39,045 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:49:39,046 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:49:39,046 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:49:39,047 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:49:39,051 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:49:39,326 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:49:41,331 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:49:43,074 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:49:43,075 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:49:43,075 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:49:43,080 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:49:43,332 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:49:44,081 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:49:44,342 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:49:44,342 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:49:44,344 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:49:45,335 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:49:47,301 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:49:47,302 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:49:47,302 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:49:47,307 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:49:47,336 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:49:49,339 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:49:49,345 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:49:51,677 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:49:51,678 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:49:51,678 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:49:51,683 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:49:52,341 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:49:53,343 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:49:54,684 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:49:55,252 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 17:49:55,687 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:49:55,688 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:49:55,688 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:49:55,693 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:49:56,345 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:49:57,347 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:49:59,342 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:49:59,343 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:49:59,344 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:49:59,466 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:49:59,604 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:49:59,604 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:49:59,609 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:50:00,350 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:50:00,610 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:50:01,353 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:50:04,188 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:50:04,188 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:50:04,189 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:50:04,193 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:50:04,355 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:50:06,194 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:50:07,358 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:50:08,722 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:50:08,722 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:50:08,723 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:50:08,727 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:50:09,360 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:50:11,363 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:50:11,728 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:50:14,342 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:50:14,343 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:50:14,344 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:50:17,127 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:50:17,128 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:50:17,128 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:50:17,128 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:50:17,133 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:50:17,367 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:50:19,370 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:50:20,886 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:50:20,887 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:50:20,887 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:50:20,892 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:50:21,371 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:50:22,893 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:50:23,374 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:50:25,255 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 17:50:26,158 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:50:26,159 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:50:26,159 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:50:26,164 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:50:26,376 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:50:27,379 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:50:28,165 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:50:29,342 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:50:29,343 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:50:29,344 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:50:30,182 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:50:30,183 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:50:30,183 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:50:30,188 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:50:30,380 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:50:33,225 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:50:33,382 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:50:34,351 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:50:34,351 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:50:34,352 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:50:34,357 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:50:34,383 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:50:37,387 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:50:38,357 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:50:42,745 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:50:42,746 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:50:42,746 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:50:42,751 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:50:43,391 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:50:43,751 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:50:44,342 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:50:44,343 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:50:44,344 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:50:45,395 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:50:49,561 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:50:51,884 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:50:51,885 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:50:51,885 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:50:51,890 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:50:52,398 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:50:53,401 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:50:54,932 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:50:55,258 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 17:50:55,909 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:50:55,910 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:50:55,910 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:50:55,915 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:50:56,403 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:50:57,405 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:50:59,342 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:50:59,343 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:50:59,344 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:51:00,620 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:51:02,911 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:51:02,911 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:51:02,912 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:51:02,916 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:51:03,410 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:51:05,413 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:51:05,953 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:51:08,303 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:51:08,304 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:51:08,304 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:51:08,309 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:51:08,414 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:51:11,310 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:51:11,416 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:51:12,444 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:51:12,445 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:51:12,445 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:51:12,451 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:51:13,418 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:51:14,343 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:51:14,343 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:51:14,345 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:51:15,421 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:51:15,731 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:51:15,732 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:51:15,732 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:51:15,737 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:51:16,421 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:51:16,777 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:51:17,424 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:51:19,489 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:51:19,489 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:51:19,490 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:51:19,495 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:51:20,426 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:51:21,428 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:51:22,495 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:51:25,169 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:51:25,170 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:51:25,170 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:51:25,175 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:51:25,260 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 17:51:25,430 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:51:27,434 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:51:28,262 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:51:29,343 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:51:29,343 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:51:29,344 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:51:29,458 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:51:29,618 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:51:29,619 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:51:29,624 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:51:30,435 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:51:31,438 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:51:33,527 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:51:33,528 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:51:33,529 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:51:33,529 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:51:33,533 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:51:34,440 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:51:35,442 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:51:38,534 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:51:39,247 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:51:39,248 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:51:39,248 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:51:39,253 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:51:39,445 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:51:41,449 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:51:42,964 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:51:42,965 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:51:42,966 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:51:42,970 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:51:43,450 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:51:43,971 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:51:44,779 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:51:44,779 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:51:44,779 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:51:45,453 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:51:47,768 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:51:47,769 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:51:47,769 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:51:47,774 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:51:48,455 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:51:49,458 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:51:49,813 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:51:51,158 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:51:51,158 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:51:51,159 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:51:51,163 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:51:51,459 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:51:53,462 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:51:55,164 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:51:55,267 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 17:51:55,617 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:51:55,618 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:51:55,618 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:51:55,624 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:51:56,464 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:51:57,466 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:51:59,779 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:51:59,779 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:51:59,779 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:52:00,452 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:52:00,453 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:52:00,453 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:52:00,453 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:52:00,458 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:52:00,469 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:52:03,473 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:52:04,253 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:52:04,254 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:52:04,254 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:52:04,259 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:52:04,474 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:52:05,476 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:52:06,260 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:52:09,476 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:52:09,477 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:52:09,478 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:52:09,482 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:52:10,479 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:52:11,482 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:52:11,483 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:52:13,774 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:52:13,774 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:52:13,775 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:52:13,780 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:52:14,484 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:52:14,778 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:52:14,779 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:52:14,779 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:52:15,487 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:52:16,991 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:52:18,513 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:52:18,514 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:52:18,514 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:52:18,519 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:52:19,489 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:52:21,493 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:52:22,087 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:52:22,088 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:52:22,088 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:52:22,088 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:52:22,093 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:52:22,493 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:52:23,496 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:52:25,267 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 17:52:25,669 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:52:25,670 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:52:25,670 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:52:25,675 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:52:26,498 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:52:27,500 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:52:27,676 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:52:29,779 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:52:29,779 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:52:29,779 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:52:30,448 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:52:30,448 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:52:30,449 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:52:30,453 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:52:30,502 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:52:33,493 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:52:33,506 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:52:35,608 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:52:35,609 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:52:35,609 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:52:35,615 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:52:36,508 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:52:37,511 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:52:38,615 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:52:39,737 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:52:39,738 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:52:39,739 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:52:39,743 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:52:40,514 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:52:41,517 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:52:43,516 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:52:43,517 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:52:43,517 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:52:43,522 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:52:44,519 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:52:44,522 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:52:44,779 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:52:44,779 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:52:44,780 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:52:45,522 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:52:48,017 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:52:48,018 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:52:48,019 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:52:48,023 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:52:48,524 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:52:49,526 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:52:50,024 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:52:51,829 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:52:51,830 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:52:51,830 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:52:51,835 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:52:52,528 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:52:53,529 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:52:55,273 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 17:52:55,275 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:52:56,148 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:52:56,149 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:52:56,149 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:52:56,154 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:52:56,530 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:52:57,533 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:52:59,779 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:52:59,779 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:52:59,779 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:53:01,078 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:53:01,100 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:53:01,100 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:53:01,100 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:53:01,105 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:53:01,536 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:53:03,539 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:53:06,145 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:53:10,219 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:53:10,220 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:53:10,220 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:53:10,225 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:53:10,544 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:53:11,226 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:53:11,544 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:53:13,582 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:53:13,583 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:53:13,583 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:53:13,588 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:53:14,547 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:53:14,779 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:53:14,779 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:53:14,779 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:53:15,549 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:53:17,060 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:53:18,391 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:53:18,392 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:53:18,392 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:53:18,397 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:53:18,551 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:53:21,554 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:53:22,437 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:53:23,486 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:53:23,486 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:53:23,487 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:53:23,491 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:53:23,556 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:53:25,273 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 17:53:25,559 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:53:27,885 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:53:27,886 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:53:27,886 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:53:27,887 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:53:27,892 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:53:28,560 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:53:29,563 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:53:29,779 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:53:29,779 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:53:29,779 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:53:33,007 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:53:33,064 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:53:33,064 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:53:33,064 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:53:33,069 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:53:33,565 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:53:35,568 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:53:38,109 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:53:41,187 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:53:41,188 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:53:41,188 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:53:41,193 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:53:41,573 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:53:43,193 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:53:43,574 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:53:44,779 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:53:44,779 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:53:44,779 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:53:45,716 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:53:45,717 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:53:45,717 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:53:45,722 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:53:46,576 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:53:47,579 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:53:48,761 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:53:53,190 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:53:53,190 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:53:53,191 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:53:53,196 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:53:53,582 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:53:54,196 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:53:55,278 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 17:53:55,585 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:53:59,280 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:53:59,779 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:53:59,779 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:53:59,779 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:54:01,644 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:54:01,644 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:54:01,645 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:54:01,650 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:54:02,590 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:54:03,593 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:54:04,689 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:54:06,517 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:54:06,518 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:54:06,518 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:54:06,523 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:54:06,595 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:54:09,598 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:54:10,524 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:54:11,313 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:54:11,314 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:54:11,314 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:54:11,319 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:54:11,600 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:54:13,604 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:54:14,779 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:54:14,779 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:54:14,780 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:54:15,876 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:54:15,877 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:54:15,877 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:54:15,877 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:54:15,882 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:54:16,606 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:54:17,608 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:54:20,710 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:54:20,711 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:54:20,711 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:54:20,716 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:54:21,611 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:54:21,716 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:54:23,614 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:54:24,406 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:54:24,407 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:54:24,407 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:54:24,412 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:54:24,614 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:54:25,277 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 17:54:25,617 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:54:27,278 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:54:29,779 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:54:29,779 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:54:29,780 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:54:30,666 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:54:30,667 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:54:30,667 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:54:30,672 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:54:31,620 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:54:32,709 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:54:33,623 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:54:34,849 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:54:34,850 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:54:34,850 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:54:34,855 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:54:35,625 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:54:37,628 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:54:37,855 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:54:38,967 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:54:38,968 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:54:38,968 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:54:38,973 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:54:39,629 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:54:41,633 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:54:42,974 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:54:44,591 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:54:44,592 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:54:44,592 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:54:44,597 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:54:44,635 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:54:44,779 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:54:44,779 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:54:44,780 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:54:47,639 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:54:48,074 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:54:48,577 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:54:48,578 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:54:48,578 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:54:48,583 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:54:48,639 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:54:51,643 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:54:52,680 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:54:52,681 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:54:52,681 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:54:52,686 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:54:53,645 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:54:53,687 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:54:55,280 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 17:54:55,648 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:54:56,500 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:54:56,501 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:54:56,501 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:54:56,506 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:54:56,648 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:54:59,507 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:54:59,653 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:54:59,779 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:54:59,780 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:54:59,780 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:55:00,792 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:55:00,793 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:55:00,793 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:55:00,798 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:55:01,655 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:55:03,658 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:55:04,837 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:55:05,543 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:55:05,544 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:55:05,544 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:55:05,549 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:55:05,659 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:55:07,662 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:55:09,164 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:55:09,165 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:55:09,165 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:55:09,170 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:55:09,663 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:55:10,171 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:55:11,666 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:55:13,604 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:55:13,605 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:55:13,605 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:55:13,610 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:55:13,668 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:55:14,779 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:55:14,780 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:55:14,780 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:55:15,671 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:55:16,058 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:55:17,829 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:55:17,830 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:55:17,830 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:55:17,835 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:55:18,673 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:55:19,675 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:55:21,873 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:55:22,463 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:55:22,464 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:55:22,465 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:55:22,470 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:55:22,677 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:55:23,679 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:55:25,283 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 17:55:26,725 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:55:26,726 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:55:26,726 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:55:26,733 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:55:27,682 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:55:27,734 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:55:29,685 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:55:29,779 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:55:29,780 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:55:29,780 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:55:30,450 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:55:30,451 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:55:30,451 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:55:30,456 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:55:30,686 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:55:31,688 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:55:33,493 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:55:35,669 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:55:35,670 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:55:35,670 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:55:35,675 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:55:35,691 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:55:37,694 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:55:38,676 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:55:40,661 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:55:40,662 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:55:40,662 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:55:40,668 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:55:40,696 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:55:43,699 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:55:44,669 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:55:44,779 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:55:44,780 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:55:44,780 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:55:45,263 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:55:45,264 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:55:45,264 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:55:45,269 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:55:45,701 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:55:47,704 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:55:50,309 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:55:50,527 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:55:50,528 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:55:50,528 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:55:50,533 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:55:50,705 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:55:53,709 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:55:55,287 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 17:55:56,288 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:55:56,621 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:55:56,622 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:55:56,622 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:55:56,627 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:55:56,711 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:55:59,716 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:55:59,780 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:55:59,780 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:55:59,780 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:56:00,327 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:56:00,328 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:56:00,328 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:56:00,333 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:56:00,717 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:56:01,373 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:56:01,717 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:56:04,634 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:56:04,634 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:56:04,635 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:56:04,640 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:56:04,719 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:56:06,640 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:56:07,723 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:56:10,250 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:56:10,250 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:56:10,251 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:56:10,255 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:56:10,724 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:56:11,727 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:56:12,256 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:56:14,583 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:56:14,584 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:56:14,584 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:56:14,589 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:56:14,729 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:56:14,780 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:56:14,780 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:56:14,780 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:56:17,731 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:56:18,029 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:56:18,562 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:56:18,562 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:56:18,563 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:56:18,567 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:56:18,732 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:56:21,735 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:56:23,605 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:56:25,293 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 17:56:26,932 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:56:26,932 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:56:26,933 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:56:26,938 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:56:27,739 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:56:28,938 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:56:29,742 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:56:29,780 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:56:29,780 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:56:29,780 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:56:33,088 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:56:33,089 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:56:33,089 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:56:33,094 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:56:33,744 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:56:34,133 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:56:35,747 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:56:37,845 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:56:37,845 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:56:37,846 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:56:37,851 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:56:38,749 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:56:39,752 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:56:39,852 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:56:41,223 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:56:41,224 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:56:41,224 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:56:41,229 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:56:41,754 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:56:43,757 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:56:44,780 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:56:44,780 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:56:44,781 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:56:45,080 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:56:46,199 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:56:46,200 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:56:46,200 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:56:46,205 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:56:46,759 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:56:47,761 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:56:50,245 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:56:51,113 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:56:51,113 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:56:51,114 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:56:51,119 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:56:51,764 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:56:53,767 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:56:55,293 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 17:56:55,295 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:56:56,587 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:56:56,588 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:56:56,588 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:56:56,593 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:56:56,769 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:56:57,771 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:56:59,780 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:56:59,780 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:56:59,781 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:57:01,025 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:57:01,026 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:57:01,026 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:57:01,026 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:57:01,031 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:57:01,775 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:57:03,778 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:57:06,069 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:57:09,534 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:57:09,535 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:57:09,535 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:57:09,540 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:57:09,781 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:57:11,541 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:57:11,783 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:57:13,899 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:57:13,900 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:57:13,901 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:57:13,906 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:57:14,780 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:57:14,780 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:57:14,781 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:57:14,785 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:57:15,787 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:57:17,015 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:57:18,649 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:57:18,650 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:57:18,650 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:57:18,655 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:57:18,789 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:57:21,793 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:57:22,693 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:57:24,787 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:57:24,787 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:57:24,788 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:57:24,792 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:57:24,795 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:57:25,298 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 17:57:27,798 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:57:28,300 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:57:29,780 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:57:29,780 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:57:29,781 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:57:30,266 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:57:30,266 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:57:30,267 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:57:30,271 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:57:30,800 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:57:31,803 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:57:33,309 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:57:38,309 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:57:38,704 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:57:38,705 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:57:38,705 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:57:38,710 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:57:38,807 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:57:41,812 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:57:43,262 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:57:43,263 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:57:43,263 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:57:43,268 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:57:43,813 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:57:44,268 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:57:44,780 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:57:44,781 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:57:44,781 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:57:45,816 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:57:49,495 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:57:49,495 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:57:49,496 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:57:49,496 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:57:49,501 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:57:49,818 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:57:51,821 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:57:54,434 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:57:54,435 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:57:54,435 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:57:54,440 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:57:54,823 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:57:55,297 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 17:57:55,298 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:57:55,825 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:57:59,780 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:57:59,781 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:57:59,781 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:58:00,578 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:58:00,578 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:58:00,579 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:58:00,579 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:58:00,583 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:58:00,829 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:58:01,832 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:58:04,580 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:58:04,581 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:58:04,581 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:58:04,586 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:58:04,834 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:58:05,586 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:58:05,834 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:58:08,946 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:58:08,947 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:58:08,947 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:58:08,952 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:58:09,837 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:58:10,953 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:58:11,840 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:58:14,780 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:58:14,781 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:58:14,781 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:58:15,023 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:58:15,023 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:58:15,023 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:58:15,028 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:58:15,843 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:58:16,065 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:58:17,846 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:58:19,665 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:58:19,666 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:58:19,666 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:58:19,671 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:58:19,847 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:58:21,672 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:58:21,848 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:58:23,154 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:58:23,154 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:58:23,155 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:58:23,160 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:58:23,850 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:58:25,304 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 17:58:25,853 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:58:27,306 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:58:28,283 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:58:28,284 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:58:28,284 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:58:28,289 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:58:28,855 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:58:29,781 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:58:29,781 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:58:29,781 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:58:29,857 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:58:32,508 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:58:32,508 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:58:32,509 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:58:32,509 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:58:32,513 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:58:32,859 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:58:33,862 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:58:37,553 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:58:40,866 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:58:40,867 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:58:40,867 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:58:40,872 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:58:41,868 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:58:42,873 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:58:43,871 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:58:44,695 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:58:44,696 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:58:44,696 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:58:44,701 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:58:44,781 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:58:44,781 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:58:44,781 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:58:44,872 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:58:45,875 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:58:48,006 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:58:48,589 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:58:48,590 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:58:48,590 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:58:48,595 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:58:48,877 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:58:49,879 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:58:53,633 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:58:55,304 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 17:58:57,114 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:58:57,115 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:58:57,115 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:58:57,120 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:58:57,884 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:58:59,121 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:58:59,781 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:58:59,781 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:58:59,781 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:58:59,888 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:59:01,198 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:59:01,198 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:59:01,198 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:59:01,203 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:59:01,890 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:59:03,893 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:59:04,241 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:59:06,590 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:59:06,591 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:59:06,591 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:59:06,595 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:59:06,894 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:59:07,897 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:59:09,596 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:59:10,518 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:59:10,519 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:59:10,519 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:59:10,524 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:59:10,899 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:59:11,899 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:59:14,781 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:59:14,781 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:59:14,781 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:59:15,005 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:59:15,993 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:59:15,994 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:59:15,994 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:59:15,999 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:59:16,903 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:59:17,905 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:59:20,037 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:59:23,652 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:59:23,652 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:59:23,653 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:59:23,658 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:59:23,909 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:59:25,306 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 17:59:25,307 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:59:25,912 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:59:29,781 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:59:29,781 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:59:29,782 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:59:31,005 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:59:32,751 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:59:32,751 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:59:32,752 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:59:32,757 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:59:32,916 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:59:35,920 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:59:36,675 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:59:36,675 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:59:36,676 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:59:36,676 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:59:36,681 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:59:36,921 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:59:37,923 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:59:40,326 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:59:40,327 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:59:40,327 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:59:40,332 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:59:40,926 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:59:41,929 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:59:42,332 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:59:43,964 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:59:43,964 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:59:43,965 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:59:43,969 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:59:44,781 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:59:44,781 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:59:44,782 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:59:44,931 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:59:45,932 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:59:48,052 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:59:48,403 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:59:48,407 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:59:48,407 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:59:48,412 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:59:48,933 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:59:49,936 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:59:53,449 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:59:54,994 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:59:54,995 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:59:54,995 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:59:55,000 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:59:55,308 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 17:59:55,940 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 17:59:57,943 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 17:59:59,079 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:59:59,080 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 17:59:59,080 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:59:59,080 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:59:59,085 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:59:59,781 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:59:59,781 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:59:59,782 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:59:59,946 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:00:01,949 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:00:03,059 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:00:03,060 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:00:03,060 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:00:03,065 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:00:03,950 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:00:04,105 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:00:05,953 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:00:07,650 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:00:07,651 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:00:07,651 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:00:07,657 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:00:07,955 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:00:09,657 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:00:09,958 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:00:12,459 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:00:12,460 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:00:12,460 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:00:12,465 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:00:12,960 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:00:13,963 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:00:14,781 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:00:14,781 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:00:14,782 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:00:15,077 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:00:16,654 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:00:16,654 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:00:16,655 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:00:16,660 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:00:16,965 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:00:17,965 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:00:20,697 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:00:21,189 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:00:21,190 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:00:21,190 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:00:21,195 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:00:21,968 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:00:23,971 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:00:25,311 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 18:00:26,046 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:00:26,047 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:00:26,047 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:00:26,047 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:00:26,052 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:00:26,973 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:00:27,975 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:00:29,781 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:00:29,782 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:00:29,782 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:00:30,895 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:00:30,895 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:00:30,896 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:00:30,901 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:00:30,977 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:00:31,937 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:00:33,981 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:00:34,537 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:00:34,538 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:00:34,538 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:00:34,543 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:00:34,981 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:00:35,984 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:00:37,544 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:00:38,914 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:00:38,915 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:00:38,915 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:00:38,920 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:00:38,986 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:00:41,991 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:00:42,921 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:00:43,283 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:00:43,284 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:00:43,284 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:00:43,289 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:00:43,992 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:00:44,781 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:00:44,782 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:00:44,782 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:00:45,995 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:00:47,466 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:00:47,467 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:00:47,467 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:00:47,472 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:00:47,997 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:00:48,509 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:00:50,000 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:00:52,389 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:00:52,390 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:00:52,390 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:00:52,395 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:00:53,001 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:00:54,004 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:00:54,395 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:00:55,317 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 18:00:58,936 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:00:58,937 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:00:58,937 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:00:58,942 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:00:59,008 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:00:59,781 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:00:59,782 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:00:59,782 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:01:00,038 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:01:02,012 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:01:03,094 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:01:03,095 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:01:03,095 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:01:03,100 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:01:04,013 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:01:05,137 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:01:06,016 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:01:07,399 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:01:07,399 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:01:07,400 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:01:07,404 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:01:08,017 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:01:10,020 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:01:10,405 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:01:12,170 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:01:12,171 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:01:12,171 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:01:12,176 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:01:13,023 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:01:14,025 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:01:14,781 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:01:14,782 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:01:14,782 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:01:16,061 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:01:16,063 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:01:16,063 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:01:16,064 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:01:16,069 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:01:17,027 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:01:18,030 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:01:20,085 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:01:20,086 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:01:20,086 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:01:20,092 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:01:21,031 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:01:21,092 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:01:22,034 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:01:25,316 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 18:01:25,374 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:01:25,375 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:01:25,375 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:01:25,380 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:01:26,036 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:01:26,381 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:01:28,039 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:01:29,358 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:01:29,358 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:01:29,359 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:01:29,363 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:01:29,782 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:01:29,782 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:01:29,782 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:01:30,041 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:01:32,044 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:01:32,068 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:01:33,353 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:01:33,353 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:01:33,354 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:01:33,359 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:01:34,045 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:01:36,048 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:01:37,397 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:01:38,574 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:01:38,575 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:01:38,575 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:01:38,580 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:01:39,050 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:01:40,053 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:01:41,923 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:01:41,924 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:01:41,924 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:01:41,929 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:01:42,055 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:01:42,930 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:01:44,058 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:01:44,782 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:01:44,782 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:01:44,782 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:01:46,028 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:01:46,029 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:01:46,029 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:01:46,034 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:01:46,059 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:01:48,061 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:01:48,073 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:01:49,919 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:01:49,920 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:01:49,920 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:01:49,925 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:01:50,062 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:01:52,065 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:01:53,926 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:01:54,301 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:01:54,302 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:01:54,302 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:01:54,307 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:01:55,066 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:01:55,321 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 18:01:56,069 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:01:59,323 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:01:59,397 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:01:59,398 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:01:59,398 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:01:59,403 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:01:59,782 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:01:59,782 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:01:59,782 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:02:00,073 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:02:02,076 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:02:05,000 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:02:05,691 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:02:05,691 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:02:05,692 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:02:05,697 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:02:06,078 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:02:08,081 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:02:09,828 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:02:09,829 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:02:09,829 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:02:09,834 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:02:10,083 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:02:10,835 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:02:12,086 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:02:14,782 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:02:14,782 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:02:14,783 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:02:16,075 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:02:19,763 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:02:19,764 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:02:19,764 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:02:19,769 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:02:20,091 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:02:21,809 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:02:22,094 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:02:24,386 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:02:24,387 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:02:24,387 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:02:24,392 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:02:25,096 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:02:25,320 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 18:02:26,098 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:02:27,322 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:02:29,782 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:02:29,782 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:02:29,783 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:02:31,347 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:02:31,348 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:02:31,348 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:02:31,353 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:02:32,102 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:02:32,393 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:02:34,105 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:02:36,892 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:02:36,893 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:02:36,893 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:02:36,898 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:02:37,107 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:02:37,898 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:02:38,109 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:02:41,497 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:02:41,497 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:02:41,498 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:02:41,503 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:02:42,113 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:02:43,503 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:02:44,116 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:02:44,782 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:02:44,782 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:02:44,783 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:02:46,417 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:02:46,417 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:02:46,418 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:02:46,423 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:02:47,118 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:02:48,121 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:02:49,461 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:02:51,137 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:02:51,137 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:02:51,138 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:02:51,143 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:02:52,123 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:02:54,126 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:02:55,144 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:02:55,327 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 18:02:57,229 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:02:57,230 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:02:57,230 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:02:57,236 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:02:58,129 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:02:59,782 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:02:59,782 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:02:59,783 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:03:00,133 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:03:01,003 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:03:01,443 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:03:01,444 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:03:01,444 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:03:01,449 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:03:02,134 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:03:04,138 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:03:05,649 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:03:05,649 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:03:05,650 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:03:05,655 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:03:06,139 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:03:06,655 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:03:08,142 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:03:10,529 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:03:10,530 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:03:10,530 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:03:10,536 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:03:11,144 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:03:12,147 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:03:12,536 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:03:14,771 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:03:14,772 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:03:14,772 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:03:14,777 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:03:14,782 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:03:14,782 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:03:14,783 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:03:15,149 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:03:16,151 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:03:18,018 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:03:19,439 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:03:19,439 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:03:19,440 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:03:19,445 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:03:20,154 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:03:22,157 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:03:23,481 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:03:24,050 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:03:24,050 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:03:24,051 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:03:24,056 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:03:24,158 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:03:25,327 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 18:03:26,162 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:03:28,277 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:03:28,278 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:03:28,278 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:03:28,283 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:03:29,163 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:03:29,283 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:03:29,782 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:03:29,783 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:03:29,783 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:03:30,166 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:03:35,000 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:03:35,899 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:03:35,899 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:03:35,900 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:03:35,905 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:03:36,169 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:03:38,173 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:03:40,941 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:03:42,847 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:03:42,848 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:03:42,849 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:03:42,854 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:03:43,177 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:03:44,179 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:03:44,782 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:03:44,783 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:03:44,783 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:03:46,001 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:03:46,889 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:03:46,890 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:03:46,890 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:03:46,896 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:03:47,181 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:03:48,184 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:03:51,378 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:03:51,378 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:03:51,379 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:03:51,379 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:03:51,384 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:03:52,186 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:03:54,189 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:03:55,330 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 18:03:57,332 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:03:57,520 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:03:57,520 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:03:57,521 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:03:57,526 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:03:58,191 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:03:59,782 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:03:59,783 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:03:59,783 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:04:00,195 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:04:02,485 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:04:02,485 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:04:02,486 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:04:02,486 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:04:02,491 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:04:03,197 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:04:04,200 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:04:06,078 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:04:06,079 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:04:06,079 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:04:06,084 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:04:06,201 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:04:08,084 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:04:08,202 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:04:10,264 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:04:10,265 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:04:10,265 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:04:10,270 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:04:11,204 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:04:12,206 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:04:13,271 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:04:14,473 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:04:14,473 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:04:14,474 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:04:14,479 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:04:14,783 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:04:14,783 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:04:14,783 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:04:15,208 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:04:16,211 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:04:19,069 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:04:19,406 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:04:19,407 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:04:19,407 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:04:19,412 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:04:20,213 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:04:22,216 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:04:23,664 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:04:23,665 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:04:23,665 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:04:23,670 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:04:24,217 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:04:24,671 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:04:25,332 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 18:04:26,220 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:04:29,783 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:04:29,783 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:04:29,783 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:04:30,042 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:04:32,614 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:04:32,614 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:04:32,615 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:04:32,620 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:04:33,224 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:04:34,227 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:04:35,657 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:04:37,357 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:04:37,358 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:04:37,358 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:04:37,363 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:04:38,230 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:04:40,233 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:04:41,364 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:04:41,503 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:04:41,504 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:04:41,504 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:04:41,509 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:04:42,235 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:04:44,238 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:04:44,783 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:04:44,783 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:04:44,783 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:04:45,051 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:04:45,101 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:04:45,102 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:04:45,106 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:04:45,239 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:04:46,241 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:04:47,107 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:04:50,038 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:04:50,039 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:04:50,039 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:04:50,044 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:04:50,243 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:04:52,247 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:04:53,081 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:04:53,471 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:04:53,472 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:04:53,472 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:04:53,478 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:04:54,248 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:04:55,335 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 18:04:56,251 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:04:58,336 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:04:59,783 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:04:59,784 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:04:59,784 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:04:59,784 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:04:59,784 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:04:59,789 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:04:59,789 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:05:00,255 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:05:02,258 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:05:04,056 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:05:05,295 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:05:05,296 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:05:05,296 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:05:05,301 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:05:06,261 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:05:08,264 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:05:09,341 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:05:11,183 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:05:11,183 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:05:11,184 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:05:11,189 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:05:11,265 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:05:14,270 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:05:14,783 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:05:14,783 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:05:14,784 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:05:15,066 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:05:20,067 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:05:20,521 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:05:20,522 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:05:20,522 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:05:20,528 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:05:21,273 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:05:22,276 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:05:23,977 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:05:23,978 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:05:23,978 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:05:23,983 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:05:24,277 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:05:25,342 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 18:05:25,344 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:05:26,281 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:05:27,846 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:05:27,847 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:05:27,847 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:05:27,852 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:05:28,282 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:05:29,783 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:05:29,783 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:05:29,784 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:05:30,285 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:05:31,043 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:05:33,879 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:05:33,879 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:05:33,880 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:05:33,885 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:05:34,287 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:05:36,288 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:05:36,921 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:05:37,727 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:05:37,728 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:05:37,729 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:05:37,734 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:05:38,290 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:05:40,293 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:05:41,579 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:05:41,580 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:05:41,580 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:05:41,585 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:05:42,295 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:05:42,586 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:05:44,299 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:05:44,783 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:05:44,783 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:05:44,784 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:05:45,729 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:05:45,729 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:05:45,730 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:05:45,734 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:05:46,300 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:05:47,773 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:05:48,303 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:05:49,676 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:05:49,677 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:05:49,677 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:05:49,682 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:05:50,304 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:05:52,307 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:05:53,683 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:05:54,284 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:05:54,285 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:05:54,285 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:05:54,290 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:05:54,308 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:05:55,341 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 18:05:56,312 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:05:57,659 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:05:57,659 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:05:57,660 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:05:57,665 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:05:58,313 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:05:59,666 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:05:59,784 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:05:59,784 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:05:59,785 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:06:00,318 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:06:02,270 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:06:02,271 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:06:02,271 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:06:02,276 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:06:02,319 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:06:04,322 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:06:05,313 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:06:07,084 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:06:07,085 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:06:07,085 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:06:07,090 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:06:07,324 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:06:08,326 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:06:11,091 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:06:13,618 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:06:13,619 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:06:13,619 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:06:13,624 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:06:14,331 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:06:14,783 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:06:14,784 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:06:14,784 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:06:16,332 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:06:17,029 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:06:17,467 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:06:17,467 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:06:17,468 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:06:17,473 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:06:18,333 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:06:20,336 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:06:22,474 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:06:25,347 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 18:06:25,749 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:06:25,750 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:06:25,750 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:06:25,755 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:06:26,340 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:06:27,756 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:06:28,343 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:06:29,335 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:06:29,336 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:06:29,336 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:06:29,342 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:06:29,343 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:06:29,783 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:06:29,784 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:06:29,784 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:06:32,347 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:06:33,096 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:06:33,309 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:06:33,310 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:06:33,310 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:06:33,315 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:06:33,348 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:06:36,352 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:06:38,316 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:06:39,033 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:06:39,034 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:06:39,034 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:06:39,040 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:06:39,356 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:06:40,358 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:06:44,041 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:06:44,783 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:06:44,784 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:06:44,784 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:06:47,019 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:06:47,020 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:06:47,020 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:06:47,025 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:06:47,368 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:06:48,371 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:06:50,025 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:06:51,755 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:06:51,755 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:06:51,756 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:06:51,761 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:06:52,374 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:06:54,379 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:06:55,347 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 18:06:55,349 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:06:55,596 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:06:55,596 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:06:55,597 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:06:55,602 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:06:56,380 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:06:58,384 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:06:59,784 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:06:59,784 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:06:59,784 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:07:00,660 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:07:00,661 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:07:00,661 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:07:00,661 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:07:00,666 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:07:01,387 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:07:02,390 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:07:05,667 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:07:06,108 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:07:06,109 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:07:06,109 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:07:06,115 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:07:06,393 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:07:08,399 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:07:11,116 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:07:11,608 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:07:11,608 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:07:11,609 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:07:11,614 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:07:12,402 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:07:14,407 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:07:14,784 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:07:14,784 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:07:14,784 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:07:15,616 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:07:15,616 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:07:15,617 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:07:15,621 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:07:16,409 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:07:16,622 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:07:18,414 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:07:19,800 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:07:19,800 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:07:19,801 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:07:19,806 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:07:20,415 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:07:21,806 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:07:22,419 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:07:23,982 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:07:23,982 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:07:23,983 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:07:23,987 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:07:24,421 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:07:25,353 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 18:07:26,425 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:07:27,354 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:07:29,784 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:07:29,784 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:07:29,784 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:07:30,433 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:07:30,434 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:07:30,434 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:07:30,439 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:07:31,431 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:07:32,434 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:07:32,439 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:07:37,440 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:07:38,919 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:07:38,920 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:07:38,920 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:07:38,925 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:07:39,441 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:07:40,444 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:07:42,926 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:07:44,214 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:07:44,215 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:07:44,216 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:07:44,221 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:07:44,449 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:07:44,784 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:07:44,784 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:07:44,784 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:07:46,454 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:07:48,032 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:07:48,223 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:07:48,224 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:07:48,224 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:07:48,229 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:07:48,455 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:07:50,460 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:07:53,230 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:07:54,099 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:07:54,100 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:07:54,100 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:07:54,105 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:07:54,463 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:07:55,351 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 18:07:56,468 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:07:58,353 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:07:59,784 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:07:59,784 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:07:59,785 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:08:00,822 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:08:00,822 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:08:00,823 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:08:00,827 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:08:01,474 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:08:02,476 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:08:03,828 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:08:04,476 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:08:04,477 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:08:04,478 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:08:04,482 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:08:05,479 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:08:06,482 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:08:08,573 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:08:08,574 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:08:08,574 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:08:08,579 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:08:09,485 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:08:09,579 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:08:10,488 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:08:13,263 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:08:13,263 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:08:13,264 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:08:13,268 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:08:13,491 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:08:14,494 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:08:14,784 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:08:14,785 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:08:14,785 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:08:15,058 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:08:16,697 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:08:16,698 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:08:16,698 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:08:16,703 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:08:17,497 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:08:18,498 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:08:20,704 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:08:22,079 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:08:22,079 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:08:22,080 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:08:22,085 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:08:22,503 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:08:24,508 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:08:25,355 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 18:08:26,356 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:08:29,784 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:08:29,785 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:08:29,785 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:08:31,103 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:08:31,104 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:08:31,104 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:08:31,110 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:08:31,517 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:08:32,110 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:08:32,520 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:08:35,279 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:08:35,280 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:08:35,280 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:08:35,285 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:08:35,522 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:08:36,525 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:08:37,286 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:08:40,977 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:08:40,978 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:08:40,978 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:08:40,983 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:08:41,531 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:08:42,534 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:08:42,983 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:08:44,314 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:08:44,315 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:08:44,315 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:08:44,320 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:08:44,535 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:08:44,784 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:08:44,785 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:08:44,785 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:08:46,538 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:08:48,081 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:08:48,247 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:08:48,252 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:08:48,253 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:08:48,258 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:08:48,542 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:08:50,547 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:08:52,848 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:08:52,849 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:08:52,849 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:08:52,854 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:08:53,550 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:08:53,854 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:08:54,552 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:08:55,358 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 18:08:56,164 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:08:56,165 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:08:56,165 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:08:56,170 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:08:56,553 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:08:58,556 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:08:59,170 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:08:59,784 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:08:59,785 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:08:59,785 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:09:00,519 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:09:00,520 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:09:00,520 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:09:00,525 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:09:00,560 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:09:02,564 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:09:04,526 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:09:06,347 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:09:06,348 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:09:06,348 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:09:06,353 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:09:06,567 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:09:08,573 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:09:10,354 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:09:10,575 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:09:10,576 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:09:10,576 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:09:10,581 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:09:11,576 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:09:12,578 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:09:14,784 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:09:14,785 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:09:14,785 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:09:15,842 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:09:15,842 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:09:15,843 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:09:15,843 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:09:15,848 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:09:16,583 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:09:18,587 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:09:20,849 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:09:20,975 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:09:20,976 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:09:20,976 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:09:20,981 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:09:21,591 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:09:22,594 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:09:25,223 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:09:25,224 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:09:25,224 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:09:25,229 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:09:25,360 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 18:09:25,597 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:09:26,361 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:09:26,597 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:09:29,057 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:09:29,057 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:09:29,058 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:09:29,065 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:09:29,602 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:09:29,785 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:09:29,785 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:09:29,785 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:09:30,605 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:09:32,056 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:09:32,936 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:09:32,937 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:09:32,937 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:09:32,942 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:09:33,608 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:09:34,611 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:09:37,454 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:09:37,455 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:09:37,456 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:09:37,456 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:09:37,460 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:09:37,614 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:09:38,617 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:09:41,620 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:09:41,621 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:09:41,621 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:09:41,626 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:09:42,622 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:09:42,626 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:09:44,626 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:09:44,785 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:09:44,785 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:09:44,785 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:09:48,010 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:09:50,002 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:09:50,003 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:09:50,003 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:09:50,008 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:09:50,633 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:09:52,638 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:09:54,008 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:09:54,572 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:09:54,573 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:09:54,573 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:09:54,578 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:09:54,639 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:09:55,365 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 18:09:56,644 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:09:58,198 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:09:58,199 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:09:58,199 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:09:58,204 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:09:58,645 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:09:59,204 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:09:59,785 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:09:59,785 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:09:59,786 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:10:00,648 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:10:02,168 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:10:02,169 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:10:02,169 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:10:02,174 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:10:02,650 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:10:04,655 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:10:05,175 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:10:06,323 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:10:06,324 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:10:06,324 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:10:06,329 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:10:06,656 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:10:08,661 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:10:10,305 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:10:10,305 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:10:10,306 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:10:10,306 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:10:10,310 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:10:10,662 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:10:12,666 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:10:14,165 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:10:14,166 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:10:14,166 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:10:14,171 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:10:14,668 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:10:14,785 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:10:14,785 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:10:14,786 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:10:16,270 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:10:16,673 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:10:19,732 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:10:19,733 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:10:19,733 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:10:19,738 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:10:20,677 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:10:21,739 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:10:22,682 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:10:23,494 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:10:23,495 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:10:23,495 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:10:23,500 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:10:23,682 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:10:24,685 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:10:25,364 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 18:10:27,366 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:10:29,785 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:10:29,786 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:10:29,786 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:10:32,719 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:10:32,720 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:10:32,721 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:10:32,721 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:10:32,726 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:10:33,696 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:10:34,699 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:10:36,357 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:10:36,357 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:10:36,358 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:10:36,363 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:10:36,700 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:10:38,363 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:10:38,703 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:10:40,365 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:10:40,365 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:10:40,365 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:10:40,370 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:10:40,705 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:10:42,710 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:10:43,371 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:10:44,224 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:10:44,225 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:10:44,225 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:10:44,230 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:10:44,712 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:10:44,785 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:10:44,785 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:10:44,786 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:10:46,717 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:10:49,050 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:10:49,562 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:10:49,563 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:10:49,564 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:10:49,568 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:10:49,720 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:10:52,726 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:10:54,359 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:10:54,360 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:10:54,360 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:10:54,360 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:10:54,365 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:10:54,727 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:10:55,369 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 18:10:56,732 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:10:59,371 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:10:59,785 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:10:59,786 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:10:59,786 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:11:00,655 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:11:00,655 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:11:00,656 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:11:00,661 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:11:00,736 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:11:02,741 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:11:04,662 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:11:04,698 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:11:04,699 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:11:04,699 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:11:04,704 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:11:04,742 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:11:06,746 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:11:08,459 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:11:08,460 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:11:08,460 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:11:08,465 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:11:08,747 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:11:10,465 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:11:10,752 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:11:14,726 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:11:14,727 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:11:14,727 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:11:14,732 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:11:14,756 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:11:14,785 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:11:14,786 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:11:14,786 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:11:16,025 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:11:16,761 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:11:18,852 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:11:18,853 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:11:18,853 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:11:18,859 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:11:19,764 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:11:20,765 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:11:21,859 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:11:25,369 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 18:11:26,621 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:11:26,623 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:11:26,623 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:11:26,628 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:11:26,773 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:11:27,629 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:11:28,778 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:11:29,785 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:11:29,786 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:11:29,786 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:11:31,810 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:11:31,811 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:11:31,811 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:11:31,816 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:11:32,782 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:11:32,816 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:11:34,787 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:11:37,817 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:11:38,140 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:11:38,140 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:11:38,141 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:11:38,146 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:11:38,791 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:11:40,796 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:11:43,147 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:11:44,013 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:11:44,014 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:11:44,014 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:11:44,019 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:11:44,785 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:11:44,786 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:11:44,786 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:11:44,800 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:11:46,805 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:11:48,764 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:11:48,765 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:11:48,765 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:11:48,766 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:11:48,771 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:11:48,806 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:11:50,811 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:11:53,623 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:11:53,624 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:11:53,624 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:11:53,629 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:11:53,814 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:11:54,630 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:11:55,376 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 18:11:56,820 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:11:57,177 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:11:57,177 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:11:57,178 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:11:57,182 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:11:57,820 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:11:58,823 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:11:59,785 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:11:59,786 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:11:59,786 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:12:00,062 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:12:02,166 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:12:02,167 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:12:02,168 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:12:02,172 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:12:02,827 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:12:04,831 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:12:05,173 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:12:07,524 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:12:07,525 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:12:07,525 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:12:07,530 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:12:07,835 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:12:08,837 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:12:10,531 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:12:14,786 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:12:14,786 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:12:14,786 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:12:15,998 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:12:16,137 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:12:16,138 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:12:16,138 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:12:16,143 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:12:16,847 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:12:18,852 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:12:20,148 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:12:20,148 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:12:20,149 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:12:20,154 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:12:20,853 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:12:21,154 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:12:22,858 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:12:25,377 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 18:12:25,855 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:12:25,856 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:12:25,856 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:12:25,863 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:12:25,863 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:12:26,864 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:12:28,867 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:12:29,786 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:12:29,786 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:12:29,786 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:12:30,804 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:12:30,805 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:12:30,806 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:12:30,810 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:12:30,868 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:12:32,811 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:12:32,873 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:12:34,723 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:12:34,724 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:12:34,724 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:12:34,729 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:12:34,874 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:12:36,878 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:12:38,110 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:12:38,111 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:12:38,112 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:12:38,112 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:12:38,117 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:12:38,880 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:12:40,885 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:12:42,293 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:12:42,294 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:12:42,294 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:12:42,299 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:12:42,888 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:12:43,300 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:12:44,786 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:12:44,786 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:12:44,786 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:12:44,892 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:12:46,168 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:12:46,169 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:12:46,169 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:12:46,175 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:12:46,893 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:12:48,898 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:12:49,176 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:12:50,222 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:12:50,222 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:12:50,223 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:12:50,229 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:12:50,899 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:12:52,903 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:12:54,230 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:12:54,470 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:12:54,471 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:12:54,471 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:12:54,477 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:12:54,904 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:12:55,379 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 18:12:56,909 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:12:58,450 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:12:58,451 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:12:58,452 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:12:58,458 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:12:58,910 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:12:59,458 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:12:59,786 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:12:59,786 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:12:59,787 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:13:00,917 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:13:04,308 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:13:04,309 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:13:04,309 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:13:04,315 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:13:04,920 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:13:05,316 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:13:06,923 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:13:08,667 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:13:08,671 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:13:08,672 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:13:08,677 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:13:08,925 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:13:10,677 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:13:10,928 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:13:12,513 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:13:12,514 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:13:12,514 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:13:12,520 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:13:12,930 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:13:14,786 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:13:14,786 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:13:14,787 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:13:14,936 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:13:16,017 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:13:17,166 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:13:17,167 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:13:17,168 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:13:17,173 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:13:17,938 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:13:18,941 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:13:20,968 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:13:20,969 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:13:20,969 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:13:20,974 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:13:21,943 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:13:21,975 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:13:22,946 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:13:25,382 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 18:13:26,789 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:13:26,790 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:13:26,790 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:13:26,796 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:13:26,950 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:13:27,797 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:13:28,956 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:13:29,786 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:13:29,786 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:13:29,787 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:13:31,368 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:13:31,369 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:13:31,369 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:13:31,375 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:13:31,958 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:13:32,961 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:13:33,375 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:13:35,374 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:13:35,375 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:13:35,375 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:13:35,381 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:13:35,964 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:13:36,967 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:13:38,381 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:13:43,382 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:13:43,572 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:13:43,572 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:13:43,573 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:13:43,579 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:13:43,975 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:13:44,786 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:13:44,787 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:13:44,787 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:13:44,978 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:13:48,056 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:13:48,057 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:13:48,057 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:13:48,063 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:13:48,983 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:13:49,064 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:13:50,988 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:13:52,383 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:13:52,384 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:13:52,384 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:13:52,389 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:13:52,989 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:13:54,390 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:13:54,992 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:13:55,385 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 18:13:56,346 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:13:56,347 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:13:56,347 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:13:56,352 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:13:56,994 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:13:58,998 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:13:59,786 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:13:59,787 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:13:59,787 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:14:00,026 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:14:01,992 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:14:01,993 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:14:01,993 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:14:01,998 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:14:02,002 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:14:05,007 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:14:05,999 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:14:06,563 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:14:06,564 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:14:06,564 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:14:06,569 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:14:07,009 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:14:09,014 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:14:10,565 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:14:10,565 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:14:10,566 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:14:10,571 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:14:11,015 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:14:11,571 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:14:13,019 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:14:14,786 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:14:14,787 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:14:14,787 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:14:17,053 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:14:18,870 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:14:18,870 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:14:18,871 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:14:18,877 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:14:19,026 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:14:21,033 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:14:22,878 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:14:23,953 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:14:23,953 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:14:23,954 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:14:23,963 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:14:24,035 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:14:25,393 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 18:14:27,048 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:14:28,166 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:14:28,167 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:14:28,167 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:14:28,167 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:14:28,174 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:14:29,049 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:14:29,786 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:14:29,787 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:14:29,787 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:14:31,065 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:14:32,733 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:14:32,734 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:14:32,735 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:14:32,745 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:14:33,066 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:14:33,745 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:14:35,071 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:14:37,510 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:14:37,511 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:14:37,511 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:14:37,521 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:14:38,074 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:14:39,075 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:14:39,522 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:14:41,714 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:14:41,715 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:14:41,715 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:14:41,725 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:14:42,079 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:14:43,085 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:14:44,726 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:14:44,786 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:14:44,787 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:14:44,787 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:14:46,696 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:14:46,696 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:14:46,697 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:14:46,707 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:14:47,088 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:14:49,094 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:14:50,487 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:14:50,488 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:14:50,488 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:14:50,488 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:14:50,496 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:14:51,096 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:14:52,102 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:14:54,251 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:14:54,252 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:14:54,252 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:14:54,262 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:14:55,105 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:14:55,391 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 18:14:56,115 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:14:56,392 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:14:58,216 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:14:58,216 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:14:58,217 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:14:58,222 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:14:59,118 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:14:59,787 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:14:59,787 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:14:59,787 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:15:00,123 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:15:02,022 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:15:03,763 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:15:03,764 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:15:03,765 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:15:03,774 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:15:04,131 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:15:04,132 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:15:07,774 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:15:10,030 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:15:10,031 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:15:10,031 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:15:10,038 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:15:10,136 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:15:12,143 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:15:13,039 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:15:13,859 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:15:13,860 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:15:13,861 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:15:13,869 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:15:14,145 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:15:14,787 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:15:14,787 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:15:14,787 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:15:16,151 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:15:17,638 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:15:17,639 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:15:17,639 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:15:17,646 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:15:18,157 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:15:18,157 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:15:18,646 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:15:21,593 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:15:21,594 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:15:21,594 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:15:21,600 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:15:22,164 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:15:22,165 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:15:24,601 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:15:25,399 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 18:15:26,558 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:15:26,559 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:15:26,559 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:15:26,570 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:15:27,170 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:15:28,177 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:15:29,787 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:15:29,787 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:15:29,788 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:15:30,004 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:15:32,005 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:15:32,006 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:15:32,006 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:15:32,019 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:15:32,181 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:15:34,188 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:15:35,019 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:15:36,068 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:15:36,069 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:15:36,069 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:15:36,077 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:15:36,190 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:15:38,197 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:15:39,807 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:15:39,808 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:15:39,808 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:15:39,817 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:15:40,206 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:15:40,206 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:15:40,818 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:15:44,787 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:15:44,787 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:15:44,788 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:15:46,047 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:15:48,229 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:15:48,230 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:15:48,230 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:15:48,238 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:15:49,215 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:15:50,224 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:15:51,239 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:15:54,934 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:15:54,934 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:15:54,935 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:15:54,942 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:15:55,229 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:15:55,396 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 18:15:56,237 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:15:56,397 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:15:59,787 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:15:59,787 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:15:59,788 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:16:02,052 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:16:03,602 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:16:03,602 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:16:03,603 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:16:03,612 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:16:04,252 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:16:04,252 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:16:07,465 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:16:07,466 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:16:07,466 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:16:07,466 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:16:07,472 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:16:08,262 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:16:08,262 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:16:11,462 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:16:11,463 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:16:11,463 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:16:11,474 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:16:12,272 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:16:12,273 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:16:12,474 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:16:14,787 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:16:14,787 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:16:14,788 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:16:15,522 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:16:15,523 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:16:15,523 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:16:15,532 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:16:16,285 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:16:16,285 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:16:17,532 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:16:22,533 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:16:24,171 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:16:24,172 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:16:24,172 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:16:24,179 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:16:24,293 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:16:25,403 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 18:16:26,300 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:16:28,404 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:16:29,787 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:16:29,787 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:16:29,788 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:16:29,837 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:16:30,022 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:16:30,022 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:16:30,032 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:16:30,305 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:16:32,311 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:16:33,676 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:16:33,677 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:16:33,677 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:16:33,677 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:16:33,684 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:16:34,319 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:16:34,319 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:16:38,420 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:16:38,421 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:16:38,421 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:16:38,431 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:16:39,326 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:16:39,431 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:16:40,329 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:16:44,432 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:16:44,787 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:16:44,788 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:16:44,788 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:16:46,952 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:16:46,953 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:16:46,953 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:16:46,960 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:16:47,339 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:16:48,345 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:16:49,961 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:16:51,407 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:16:51,407 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:16:51,408 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:16:51,417 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:16:52,353 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:16:52,353 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:16:55,401 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 18:16:55,403 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:16:57,108 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:16:57,108 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:16:57,109 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:16:57,117 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:16:57,357 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:16:58,366 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:16:59,787 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:16:59,788 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:16:59,788 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:17:01,029 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:17:01,061 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:17:01,062 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:17:01,062 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:17:01,073 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:17:01,372 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:17:02,376 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:17:05,774 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:17:05,774 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:17:05,775 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:17:05,781 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:17:06,381 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:17:06,381 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:17:06,782 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:17:11,782 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:17:11,802 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:17:11,803 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:17:11,803 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:17:11,810 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:17:12,390 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:17:12,390 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:17:14,787 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:17:14,788 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:17:14,788 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:17:16,719 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:17:16,720 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:17:16,720 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:17:16,727 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:17:17,395 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:17:17,727 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:17:18,402 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:17:20,619 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:17:20,619 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:17:20,620 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:17:20,625 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:17:21,405 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:17:22,408 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:17:23,626 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:17:24,420 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:17:24,421 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:17:24,421 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:17:24,429 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:17:25,404 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 18:17:25,411 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:17:26,417 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:17:29,405 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:17:29,787 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:17:29,788 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:17:29,788 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:17:32,781 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:17:32,782 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:17:32,782 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:17:32,790 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:17:33,424 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:17:34,430 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:17:34,791 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:17:36,547 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:17:36,547 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:17:36,548 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:17:36,555 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:17:37,432 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:17:38,437 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:17:40,556 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:17:40,996 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:17:40,997 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:17:40,997 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:17:41,005 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:17:41,439 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:17:42,450 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:17:44,781 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:17:44,782 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:17:44,782 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:17:44,788 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:17:44,788 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:17:44,792 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:17:44,792 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:17:45,453 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:17:46,011 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:17:46,461 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:17:48,838 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:17:48,839 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:17:48,839 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:17:48,846 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:17:49,464 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:17:50,468 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:17:51,846 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:17:53,745 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:17:53,746 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:17:53,746 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:17:53,756 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:17:54,475 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:17:54,475 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:17:55,407 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 18:17:57,408 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:17:57,774 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:17:57,774 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:17:57,775 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:17:57,783 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:17:58,483 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:17:58,483 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:17:59,788 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:17:59,788 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:17:59,789 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:18:02,292 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:18:02,293 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:18:02,293 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:18:02,299 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:18:02,488 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:18:03,337 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:18:04,497 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:18:06,417 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:18:06,418 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:18:06,418 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:18:06,424 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:18:06,498 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:18:08,424 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:18:08,500 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:18:12,278 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:18:12,279 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:18:12,280 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:18:12,286 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:18:12,504 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:18:14,287 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:18:14,509 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:18:14,788 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:18:14,788 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:18:14,788 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:18:17,906 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:18:17,907 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:18:17,907 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:18:17,913 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:18:18,516 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:18:18,516 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:18:19,914 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:18:22,339 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:18:22,340 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:18:22,340 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:18:22,346 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:18:22,519 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:18:24,524 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:18:25,347 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:18:25,410 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 18:18:26,145 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:18:26,146 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:18:26,146 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:18:26,152 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:18:26,525 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:18:28,530 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:18:29,786 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:18:29,787 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:18:29,787 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:18:29,788 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:18:29,788 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:18:29,793 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:18:29,794 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:18:30,531 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:18:30,531 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:18:31,057 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:18:35,216 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:18:35,217 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:18:35,217 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:18:35,226 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:18:35,535 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:18:36,226 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:18:36,539 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:18:41,226 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:18:41,489 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:18:41,490 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:18:41,490 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:18:41,497 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:18:41,543 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:18:42,555 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:18:44,788 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:18:44,788 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:18:44,789 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:18:45,478 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:18:45,479 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:18:45,479 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:18:45,485 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:18:45,558 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:18:46,485 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:18:46,561 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:18:51,486 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:18:52,098 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:18:52,099 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:18:52,099 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:18:52,105 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:18:52,566 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:18:54,569 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:18:55,417 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 18:18:57,419 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:18:57,902 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:18:57,902 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:18:57,903 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:18:57,910 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:18:58,575 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:18:58,575 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:18:59,788 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:18:59,788 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:18:59,789 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:19:01,763 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:19:01,763 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:19:01,763 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:19:01,770 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:19:02,579 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:19:02,580 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:19:02,805 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:19:07,805 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:19:07,841 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:19:07,842 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:19:07,842 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:19:07,847 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:19:08,586 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:19:08,587 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:19:12,547 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:19:12,548 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:19:12,548 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:19:12,553 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:19:12,590 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:19:13,553 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:19:14,593 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:19:14,788 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:19:14,789 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:19:14,789 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:19:18,627 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:19:18,628 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:19:18,630 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:19:18,630 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:19:18,635 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:19:19,597 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:19:20,600 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:19:23,636 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:19:24,925 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:19:24,926 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:19:24,926 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:19:24,931 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:19:25,415 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 18:19:25,603 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:19:26,606 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:19:29,417 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:19:29,788 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:19:29,789 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:19:29,789 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:19:33,412 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:19:33,413 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:19:33,413 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:19:33,420 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:19:33,610 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:19:34,420 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:19:34,613 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:19:37,352 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:19:37,353 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:19:37,353 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:19:37,358 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:19:37,615 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:19:38,616 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:19:40,359 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:19:43,107 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:19:43,108 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:19:43,108 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:19:43,113 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:19:43,621 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:19:44,623 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:19:44,788 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:19:44,789 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:19:44,789 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:19:46,040 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:19:49,257 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:19:49,257 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:19:49,258 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:19:49,263 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:19:49,626 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:19:50,629 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:19:51,263 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:19:54,361 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:19:54,362 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:19:54,362 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:19:54,367 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:19:54,632 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:19:55,420 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 18:19:56,422 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:19:56,636 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:19:58,680 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:19:58,681 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:19:58,681 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:19:58,686 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:19:59,638 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:19:59,788 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:19:59,789 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:19:59,789 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:20:00,640 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:20:02,049 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:20:02,270 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:20:02,271 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:20:02,271 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:20:02,276 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:20:02,641 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:20:04,644 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:20:05,966 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:20:05,967 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:20:05,967 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:20:05,972 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:20:06,647 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:20:06,648 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:20:07,973 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:20:10,703 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:20:10,704 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:20:10,704 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:20:10,709 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:20:11,651 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:20:12,653 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:20:13,710 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:20:14,789 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:20:14,789 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:20:14,789 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:20:14,910 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:20:15,073 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:20:15,073 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:20:15,078 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:20:15,656 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:20:16,658 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:20:19,079 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:20:20,468 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:20:20,469 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:20:20,469 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:20:20,474 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:20:20,661 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:20:22,664 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:20:24,475 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:20:25,420 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 18:20:28,889 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:20:28,890 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:20:28,890 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:20:28,895 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:20:29,668 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:20:29,789 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:20:29,789 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:20:29,789 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:20:30,036 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:20:30,671 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:20:33,208 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:20:33,208 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:20:33,209 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:20:33,213 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:20:33,673 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:20:34,676 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:20:35,214 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:20:37,337 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:20:37,337 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:20:37,338 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:20:37,343 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:20:37,678 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:20:38,681 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:20:40,343 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:20:41,228 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:20:41,229 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:20:41,229 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:20:41,234 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:20:41,682 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:20:42,686 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:20:44,789 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:20:44,789 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:20:44,789 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:20:45,288 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:20:45,289 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:20:45,289 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:20:45,294 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:20:45,688 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:20:46,333 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:20:46,690 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:20:51,333 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:20:53,662 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:20:53,663 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:20:53,663 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:20:53,668 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:20:53,695 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:20:54,697 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:20:55,427 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 18:20:56,428 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:20:57,846 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:20:57,847 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:20:57,847 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:20:57,852 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:20:58,702 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:20:58,702 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:20:59,789 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:20:59,789 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:20:59,790 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:21:01,740 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:21:01,741 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:21:01,741 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:21:01,742 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:21:01,749 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:21:02,713 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:21:02,713 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:21:06,433 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:21:06,434 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:21:06,434 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:21:06,439 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:21:06,715 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:21:07,440 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:21:08,718 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:21:11,085 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:21:11,086 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:21:11,086 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:21:11,091 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:21:11,720 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:21:12,720 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:21:13,092 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:21:14,789 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:21:14,789 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:21:14,790 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:21:15,051 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:21:15,052 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:21:15,052 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:21:15,058 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:21:15,722 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:21:16,725 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:21:18,097 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:21:19,043 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:21:19,044 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:21:19,044 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:21:19,049 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:21:19,727 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:21:20,729 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:21:24,050 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:21:25,427 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 18:21:25,952 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:21:25,953 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:21:25,953 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:21:25,958 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:21:26,734 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:21:26,735 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:21:29,789 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:21:29,789 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:21:29,790 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:21:30,018 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:21:30,062 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:21:30,063 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:21:30,063 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:21:30,068 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:21:30,739 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:21:30,739 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:21:35,105 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:21:38,606 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:21:38,607 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:21:38,607 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:21:38,612 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:21:38,744 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:21:40,613 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:21:40,747 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:21:43,087 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:21:43,088 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:21:43,088 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:21:43,093 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:21:43,750 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:21:44,752 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:21:44,789 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:21:44,789 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:21:44,790 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:21:46,004 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:21:47,040 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:21:47,040 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:21:47,041 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:21:47,045 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:21:47,754 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:21:48,757 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:21:51,085 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:21:51,770 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:21:51,770 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:21:51,771 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:21:51,776 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:21:52,761 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:21:52,761 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:21:55,430 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 18:21:56,431 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:21:56,492 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:21:56,492 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:21:56,493 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:21:56,497 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:21:56,763 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:21:58,766 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:21:59,789 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:21:59,790 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:21:59,790 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:22:00,612 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:22:00,613 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:22:00,613 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:22:00,618 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:22:00,769 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:22:01,657 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:22:02,772 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:22:06,088 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:22:06,089 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:22:06,089 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:22:06,093 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:22:06,776 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:22:06,776 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:22:07,094 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:22:10,257 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:22:10,258 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:22:10,258 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:22:10,263 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:22:10,780 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:22:10,780 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:22:12,264 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:22:14,459 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:22:14,460 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:22:14,460 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:22:14,465 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:22:14,783 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:22:14,789 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:22:14,790 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:22:14,790 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:22:16,786 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:22:18,013 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:22:19,667 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:22:19,667 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:22:19,668 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:22:19,673 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:22:19,788 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:22:20,790 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:22:23,673 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:22:25,389 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:22:25,390 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:22:25,390 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:22:25,394 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:22:25,432 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 18:22:25,793 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:22:26,796 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:22:29,434 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:22:29,789 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:22:29,790 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:22:29,790 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:22:33,834 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:22:33,835 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:22:33,836 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:22:33,840 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:22:34,802 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:22:34,802 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:22:34,841 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:22:37,659 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:22:37,660 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:22:37,661 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:22:37,665 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:22:37,804 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:22:38,807 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:22:40,666 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:22:41,752 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:22:41,753 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:22:41,753 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:22:41,758 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:22:41,808 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:22:42,813 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:22:44,790 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:22:44,790 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:22:44,790 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:22:45,443 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:22:45,444 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:22:45,444 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:22:45,449 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:22:45,814 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:22:46,489 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:22:46,817 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:22:49,017 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:22:49,018 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:22:49,018 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:22:49,023 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:22:49,818 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:22:50,821 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:22:52,024 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:22:53,683 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:22:53,684 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:22:53,685 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:22:53,690 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:22:53,823 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:22:54,904 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:22:55,435 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 18:22:57,311 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:22:57,312 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:22:57,313 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:22:57,313 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:22:57,317 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:22:57,906 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:22:58,909 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:22:59,790 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:22:59,790 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:22:59,790 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:23:01,528 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:23:01,528 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:23:01,529 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:23:01,534 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:23:01,912 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:23:02,573 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:23:02,915 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:23:05,694 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:23:05,695 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:23:05,695 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:23:05,700 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:23:05,917 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:23:06,919 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:23:07,701 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:23:09,748 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:23:09,749 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:23:09,749 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:23:09,754 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:23:09,921 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:23:10,923 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:23:12,754 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:23:13,994 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:23:13,995 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:23:13,995 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:23:14,000 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:23:14,790 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:23:14,790 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:23:14,790 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:23:14,926 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:23:14,926 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:23:18,031 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:23:19,349 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:23:19,349 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:23:19,350 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:23:19,355 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:23:19,929 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:23:20,931 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:23:23,190 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:23:23,190 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:23:23,191 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:23:23,191 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:23:23,195 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:23:23,933 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:23:24,936 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:23:25,439 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 18:23:28,440 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:23:29,000 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:23:29,001 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:23:29,001 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:23:29,006 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:23:29,790 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:23:29,790 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:23:29,791 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:23:29,939 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:23:30,941 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:23:33,844 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:23:33,844 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:23:33,845 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:23:33,845 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:23:33,849 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:23:33,943 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:23:34,945 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:23:38,889 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:23:39,217 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:23:39,218 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:23:39,218 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:23:39,223 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:23:39,949 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:23:40,951 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:23:42,955 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:23:42,956 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:23:42,956 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:23:42,961 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:23:43,954 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:23:43,961 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:23:44,790 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:23:44,790 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:23:44,790 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:23:44,957 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:23:46,365 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:23:46,365 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:23:46,366 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:23:46,371 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:23:46,960 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:23:46,960 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:23:49,372 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:23:51,047 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:23:51,048 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:23:51,048 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:23:51,054 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:23:51,963 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:23:52,965 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:23:54,964 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:23:54,965 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:23:54,965 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:23:54,965 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:23:54,970 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:23:55,441 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 18:23:55,967 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:23:56,969 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:23:59,790 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:23:59,790 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:23:59,791 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:23:59,902 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:24:00,089 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:24:00,089 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:24:00,089 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:24:00,095 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:24:00,975 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:24:00,975 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:24:04,990 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:24:04,991 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:24:04,991 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:24:04,997 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:24:05,977 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:24:05,997 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:24:06,980 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:24:10,771 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:24:10,772 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:24:10,772 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:24:10,777 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:24:10,982 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:24:11,777 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:24:12,985 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:24:14,790 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:24:14,790 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:24:14,791 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:24:14,809 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:24:15,098 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:24:15,098 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:24:15,103 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:24:15,988 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:24:16,990 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:24:17,104 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:24:18,378 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:24:18,379 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:24:18,380 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:24:18,385 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:24:18,993 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:24:18,993 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:24:22,385 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:24:22,924 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:24:22,925 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:24:22,925 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:24:22,930 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:24:22,996 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:24:24,999 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:24:25,447 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 18:24:27,448 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:24:28,175 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:24:28,176 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:24:28,176 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:24:28,181 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:24:29,003 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:24:29,003 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:24:29,790 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:24:29,790 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:24:29,791 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:24:32,510 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:24:32,510 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:24:32,510 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:24:32,511 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:24:32,516 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:24:33,005 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:24:35,009 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:24:37,553 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:24:39,109 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:24:39,109 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:24:39,110 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:24:39,115 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:24:40,012 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:24:41,017 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:24:42,776 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:24:42,777 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:24:42,777 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:24:42,777 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:24:42,782 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:24:43,020 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:24:45,023 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:24:45,058 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:24:45,058 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:24:45,058 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:24:47,795 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:24:47,795 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:24:47,796 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:24:47,796 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:24:47,801 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:24:48,025 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:24:49,027 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:24:51,835 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:24:51,836 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:24:51,836 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:24:51,841 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:24:52,029 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:24:52,841 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:24:53,032 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:24:55,448 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 18:24:58,449 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:25:00,057 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:25:00,058 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:25:00,058 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:25:00,084 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:25:00,329 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:25:00,330 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:25:00,334 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:25:01,040 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:25:01,040 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:25:04,335 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:25:04,982 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:25:04,983 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:25:04,983 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:25:04,988 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:25:05,042 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:25:07,046 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:25:09,110 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:25:09,110 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:25:09,111 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:25:09,116 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:25:10,047 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:25:10,116 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:25:11,050 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:25:15,057 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:25:15,058 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:25:15,058 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:25:15,355 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:25:17,924 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:25:17,925 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:25:17,925 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:25:17,930 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:25:18,054 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:25:19,057 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:25:20,969 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:25:25,453 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 18:25:26,454 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:25:26,521 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:25:26,522 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:25:26,522 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:25:26,527 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:25:27,061 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:25:29,064 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:25:30,058 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:25:30,058 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:25:30,058 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:25:30,207 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:25:30,354 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:25:30,354 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:25:30,359 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:25:31,067 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:25:31,068 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:25:32,360 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:25:34,001 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:25:34,002 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:25:34,002 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:25:34,007 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:25:34,069 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:25:35,071 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:25:38,008 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:25:38,154 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:25:38,155 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:25:38,155 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:25:38,160 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:25:39,076 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:25:39,076 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:25:43,161 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:25:45,058 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:25:45,058 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:25:45,058 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:25:46,526 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:25:46,526 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:25:46,527 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:25:46,532 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:25:47,081 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:25:48,569 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:25:49,085 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:25:50,136 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:25:50,137 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:25:50,137 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:25:50,142 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:25:51,088 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:25:51,088 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:25:54,142 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:25:55,140 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:25:55,141 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:25:55,141 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:25:55,146 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:25:55,452 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 18:25:56,091 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:25:57,093 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:25:59,216 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:25:59,216 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:25:59,217 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:25:59,217 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:25:59,222 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:26:00,058 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:26:00,058 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:26:00,058 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:26:00,095 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:26:01,099 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:26:03,927 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:26:03,927 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:26:03,928 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:26:03,932 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:26:04,100 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:26:04,969 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:26:05,103 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:26:08,980 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:26:08,981 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:26:08,981 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:26:08,986 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:26:09,105 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:26:09,986 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:26:11,109 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:26:12,566 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:26:12,566 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:26:12,567 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:26:12,571 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:26:13,112 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:26:13,112 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:26:15,058 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:26:15,058 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:26:15,059 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:26:15,060 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:26:17,346 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:26:17,346 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:26:17,347 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:26:17,352 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:26:18,115 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:26:19,117 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:26:20,389 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:26:21,833 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:26:21,834 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:26:21,834 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:26:21,839 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:26:22,120 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:26:23,122 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:26:25,456 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 18:26:25,457 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:26:26,103 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:26:26,104 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:26:26,104 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:26:26,109 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:26:26,124 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:26:27,126 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:26:30,058 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:26:30,058 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:26:30,059 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:26:31,350 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:26:32,868 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:26:32,869 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:26:32,869 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:26:32,874 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:26:33,130 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:26:35,133 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:26:36,916 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:26:37,585 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:26:37,586 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:26:37,586 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:26:37,591 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:26:38,135 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:26:39,138 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:26:42,592 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:26:44,455 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:26:44,456 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:26:44,456 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:26:44,461 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:26:45,058 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:26:45,058 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:26:45,059 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:26:45,144 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:26:45,144 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:26:48,298 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:26:49,057 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:26:49,058 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:26:49,058 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:26:49,063 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:26:49,147 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:26:51,150 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:26:54,101 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:26:54,107 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:26:54,108 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:26:54,108 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:26:54,113 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:26:54,152 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:26:55,154 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:26:55,458 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 18:26:59,459 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:27:00,058 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:27:00,058 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:27:00,059 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:27:02,614 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:27:02,615 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:27:02,615 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:27:02,620 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:27:03,160 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:27:04,657 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:27:05,163 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:27:07,502 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:27:07,503 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:27:07,503 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:27:07,508 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:27:08,165 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:27:09,167 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:27:10,509 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:27:11,240 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:27:11,241 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:27:11,241 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:27:11,246 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:27:12,169 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:27:13,172 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:27:15,058 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:27:15,058 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:27:15,059 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:27:16,256 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:27:17,095 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:27:17,096 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:27:17,096 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:27:17,101 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:27:17,174 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:27:19,177 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:27:20,873 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:27:20,874 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:27:20,874 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:27:20,879 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:27:21,178 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:27:21,880 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:27:23,182 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:27:25,238 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:27:25,239 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:27:25,239 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:27:25,244 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:27:25,460 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 18:27:26,183 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:27:27,184 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:27:27,461 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:27:29,191 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:27:29,192 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:27:29,192 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:27:29,198 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:27:30,058 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:27:30,059 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:27:30,059 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:27:30,186 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:27:31,188 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:27:33,358 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:27:33,443 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:27:33,443 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:27:33,444 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:27:33,448 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:27:34,190 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:27:35,192 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:27:37,279 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:27:37,280 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:27:37,280 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:27:37,285 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:27:38,194 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:27:39,197 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:27:39,285 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:27:41,224 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:27:41,225 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:27:41,225 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:27:41,231 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:27:42,199 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:27:43,202 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:27:45,058 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:27:45,059 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:27:45,059 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:27:45,059 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:27:46,073 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:27:46,074 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:27:46,074 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:27:46,079 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:27:46,204 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:27:47,206 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:27:50,117 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:27:50,465 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:27:50,466 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:27:50,466 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:27:50,471 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:27:51,211 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:27:51,211 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:27:55,462 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 18:27:55,463 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:27:55,484 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:27:55,484 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:27:55,484 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:27:55,489 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:27:56,214 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:27:57,216 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:28:00,058 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:28:00,059 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:28:00,059 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:28:01,352 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:28:02,234 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:28:02,235 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:28:02,235 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:28:02,240 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:28:03,223 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:28:03,223 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:28:06,367 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:28:06,368 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:28:06,368 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:28:06,368 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:28:06,373 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:28:07,228 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:28:07,228 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:28:10,560 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:28:10,561 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:28:10,561 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:28:10,566 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:28:11,234 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:28:11,234 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:28:11,566 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:28:14,584 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:28:14,585 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:28:14,585 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:28:14,590 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:28:15,058 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:28:15,059 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:28:15,059 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:28:15,240 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:28:15,240 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:28:17,323 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:28:18,857 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:28:18,858 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:28:18,858 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:28:18,863 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:28:19,244 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:28:21,249 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:28:22,717 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:28:22,718 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:28:22,718 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:28:22,718 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:28:22,723 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:28:23,252 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:28:23,252 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:28:25,465 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 18:28:27,141 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:28:27,142 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:28:27,143 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:28:27,147 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:28:27,256 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:28:28,148 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:28:29,265 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:28:30,059 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:28:30,059 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:28:30,059 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:28:30,878 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:28:30,879 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:28:30,879 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:28:30,884 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:28:31,266 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:28:33,270 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:28:33,921 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:28:35,210 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:28:35,210 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:28:35,211 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:28:35,216 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:28:35,271 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:28:37,274 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:28:39,196 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:28:39,197 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:28:39,197 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:28:39,197 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:28:39,202 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:28:39,275 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:28:41,279 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:28:43,838 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:28:43,839 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:28:43,839 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:28:43,846 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:28:44,282 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:28:44,846 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:28:45,059 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:28:45,059 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:28:45,060 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:28:45,288 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:28:47,310 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:28:47,311 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:28:47,311 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:28:47,316 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:28:48,290 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:28:49,293 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:28:50,353 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:28:54,479 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:28:54,480 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:28:54,481 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:28:54,486 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:28:55,300 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:28:55,300 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:28:55,470 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 18:28:55,471 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:28:59,686 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:28:59,687 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:28:59,687 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:28:59,693 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:29:00,059 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:29:00,059 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:29:00,059 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:29:00,304 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:29:01,309 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:29:01,309 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:29:03,446 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:29:03,447 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:29:03,447 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:29:03,453 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:29:04,311 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:29:05,314 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:29:06,489 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:29:07,644 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:29:07,644 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:29:07,645 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:29:07,650 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:29:08,316 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:29:09,319 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:29:11,651 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:29:12,295 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:29:12,296 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:29:12,296 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:29:12,301 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:29:12,321 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:29:13,324 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:29:15,059 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:29:15,059 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:29:15,060 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:29:16,118 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:29:16,119 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:29:16,119 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:29:16,124 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:29:16,327 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:29:17,161 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:29:17,331 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:29:22,161 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:29:24,701 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:29:24,702 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:29:24,702 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:29:24,708 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:29:25,342 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:29:25,342 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:29:25,469 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 18:29:27,471 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:29:30,059 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:29:30,059 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:29:30,060 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:29:30,811 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:29:30,812 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:29:30,812 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:29:30,817 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:29:31,349 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:29:32,853 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:29:33,354 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:29:35,964 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:29:35,965 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:29:35,965 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:29:35,971 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:29:36,356 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:29:37,359 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:29:37,971 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:29:42,972 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:29:43,511 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:29:43,512 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:29:43,512 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:29:43,518 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:29:44,366 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:29:45,059 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:29:45,059 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:29:45,060 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:29:45,371 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:29:47,446 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:29:47,447 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:29:47,447 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:29:47,453 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:29:48,373 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:29:48,489 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:29:49,377 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:29:51,706 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:29:51,706 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:29:51,707 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:29:51,712 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:29:52,379 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:29:53,383 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:29:53,713 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:29:55,476 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 18:29:59,477 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:30:00,059 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:30:00,059 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:30:00,060 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:30:00,452 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:30:00,453 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:30:00,453 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:30:00,458 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:30:01,396 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:30:01,396 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:30:04,497 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:30:05,707 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:30:05,708 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:30:05,708 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:30:05,714 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:30:06,399 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:30:07,403 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:30:09,714 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:30:11,986 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:30:11,987 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:30:11,987 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:30:11,993 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:30:12,407 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:30:13,411 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:30:14,993 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:30:15,059 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:30:15,060 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:30:15,060 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:30:15,471 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:30:15,574 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:30:15,575 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:30:15,581 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:30:16,414 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:30:17,415 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:30:19,680 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:30:19,681 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:30:19,681 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:30:19,687 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:30:20,420 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:30:20,687 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:30:21,423 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:30:24,407 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:30:24,407 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:30:24,408 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:30:24,413 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:30:24,425 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:30:25,430 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:30:25,475 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 18:30:26,476 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:30:29,036 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:30:29,036 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:30:29,037 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:30:29,042 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:30:29,433 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:30:30,059 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:30:30,060 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:30:30,060 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:30:31,438 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:30:32,326 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:30:34,121 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:30:34,122 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:30:34,122 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:30:34,128 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:30:34,440 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:30:35,441 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:30:38,072 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:30:38,073 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:30:38,073 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:30:38,073 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:30:38,086 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:30:38,445 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:30:39,449 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:30:43,003 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:30:43,004 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:30:43,004 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:30:43,010 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:30:43,453 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:30:44,011 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:30:45,059 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:30:45,060 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:30:45,060 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:30:45,458 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:30:49,265 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:30:49,265 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:30:49,266 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:30:49,266 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:30:49,271 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:30:49,462 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:30:51,467 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:30:53,371 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:30:53,372 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:30:53,373 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:30:53,378 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:30:53,468 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:30:54,379 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:30:55,472 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:30:55,478 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 18:30:57,829 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:30:57,830 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:30:57,830 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:30:57,836 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:30:58,474 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:30:59,478 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:30:59,836 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:31:00,059 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:31:00,060 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:31:00,060 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:31:01,781 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:31:01,782 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:31:01,782 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:31:01,788 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:31:02,482 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:31:03,485 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:31:05,825 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:31:05,944 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:31:05,945 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:31:05,945 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:31:05,951 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:31:06,487 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:31:07,490 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:31:10,952 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:31:12,273 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:31:12,274 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:31:12,274 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:31:12,281 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:31:12,495 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:31:13,500 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:31:15,060 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:31:15,060 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:31:15,060 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:31:16,356 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:31:16,692 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:31:16,692 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:31:16,693 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:31:16,699 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:31:17,504 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:31:17,504 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:31:21,531 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:31:21,532 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:31:21,532 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:31:21,533 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:31:21,539 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:31:22,510 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:31:23,515 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:31:25,481 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 18:31:26,322 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:31:26,323 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:31:26,323 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:31:26,329 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:31:26,517 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:31:27,330 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:31:27,520 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:31:30,060 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:31:30,060 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:31:30,061 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:31:32,130 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:31:32,131 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:31:32,131 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:31:32,137 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:31:32,525 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:31:33,173 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:31:33,529 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:31:35,662 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:31:35,663 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:31:35,663 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:31:35,668 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:31:36,531 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:31:37,535 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:31:38,669 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:31:41,819 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:31:41,820 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:31:41,821 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:31:41,826 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:31:42,539 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:31:43,544 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:31:43,827 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:31:45,060 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:31:45,060 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:31:45,060 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:31:47,115 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:31:47,115 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:31:47,116 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:31:47,121 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:31:47,548 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:31:49,157 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:31:49,553 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:31:51,458 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:31:51,459 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:31:51,459 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:31:51,465 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:31:51,555 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:31:53,559 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:31:54,466 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:31:55,446 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:31:55,447 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:31:55,447 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:31:55,453 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:31:55,483 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 18:31:55,561 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:31:57,565 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:31:59,485 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:32:00,060 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:32:00,060 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:32:00,061 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:32:04,057 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:32:04,058 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:32:04,058 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:32:04,064 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:32:04,572 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:32:05,101 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:32:05,576 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:32:09,049 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:32:09,050 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:32:09,050 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:32:09,056 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:32:09,580 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:32:11,056 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:32:11,585 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:32:12,754 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:32:12,755 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:32:12,755 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:32:12,761 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:32:13,589 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:32:13,589 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:32:15,060 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:32:15,060 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:32:15,061 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:32:16,329 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:32:18,148 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:32:18,152 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:32:18,152 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:32:18,158 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:32:18,593 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:32:19,598 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:32:22,193 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:32:23,726 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:32:23,727 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:32:23,727 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:32:23,733 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:32:24,602 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:32:25,487 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 18:32:25,606 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:32:27,307 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:32:27,307 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:32:27,308 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:32:27,308 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:32:27,313 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:32:27,608 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:32:29,613 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:32:30,060 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:32:30,060 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:32:30,061 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:32:32,062 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:32:32,062 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:32:32,063 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:32:32,068 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:32:32,615 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:32:33,105 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:32:33,619 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:32:36,661 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:32:36,662 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:32:36,662 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:32:36,667 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:32:37,624 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:32:37,624 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:32:38,668 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:32:43,669 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:32:44,965 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:32:44,966 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:32:44,966 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:32:44,972 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:32:45,060 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:32:45,061 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:32:45,061 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:32:45,634 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:32:45,634 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:32:49,329 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:32:49,873 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:32:49,874 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:32:49,874 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:32:49,879 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:32:50,640 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:32:51,645 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:32:53,647 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:32:53,647 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:32:53,648 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:32:53,653 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:32:54,647 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:32:54,654 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:32:55,490 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 18:32:55,650 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:32:58,512 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:32:58,512 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:32:58,513 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:32:58,519 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:32:58,652 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:32:59,656 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:33:00,060 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:33:00,061 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:33:00,061 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:33:00,061 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:33:03,738 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:33:03,739 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:33:03,739 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:33:03,744 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:33:04,661 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:33:05,665 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:33:05,781 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:33:08,072 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:33:08,073 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:33:08,073 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:33:08,079 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:33:08,667 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:33:09,671 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:33:11,079 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:33:11,260 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:33:11,261 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:33:11,261 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:33:11,267 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:33:11,672 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:33:13,676 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:33:15,060 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:33:15,061 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:33:15,061 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:33:15,850 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:33:15,851 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:33:15,851 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:33:15,857 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:33:16,679 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:33:16,893 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:33:17,683 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:33:21,893 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:33:22,063 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:33:22,063 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:33:22,064 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:33:22,069 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:33:22,687 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:33:23,692 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:33:25,496 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 18:33:26,806 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:33:26,807 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:33:26,807 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:33:26,813 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:33:27,697 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:33:27,697 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:33:27,813 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:33:30,060 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:33:30,061 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:33:30,061 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:33:30,609 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:33:30,610 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:33:30,610 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:33:30,616 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:33:30,699 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:33:31,703 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:33:33,653 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:33:36,260 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:33:36,261 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:33:36,262 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:33:36,267 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:33:36,707 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:33:37,712 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:33:39,268 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:33:41,268 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:33:41,269 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:33:41,269 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:33:41,274 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:33:41,715 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:33:43,722 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:33:44,275 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:33:44,999 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:33:45,000 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:33:45,000 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:33:45,006 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:33:45,061 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:33:45,061 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:33:45,062 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:33:45,725 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:33:45,725 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:33:49,329 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:33:50,744 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:33:50,745 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:33:50,745 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:33:50,751 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:33:51,733 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:33:51,733 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:33:54,746 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:33:54,746 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:33:54,747 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:33:54,747 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:33:54,752 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:33:55,496 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 18:33:55,739 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:33:55,739 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:34:00,061 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:34:00,061 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:34:00,062 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:34:00,062 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:34:03,566 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:34:03,566 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:34:03,567 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:34:03,572 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:34:03,748 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:34:05,609 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:34:05,753 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:34:07,612 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:34:07,612 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:34:07,613 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:34:07,618 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:34:07,754 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:34:09,759 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:34:10,619 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:34:12,666 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:34:12,666 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:34:12,667 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:34:12,672 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:34:12,761 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:34:13,765 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:34:15,061 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:34:15,061 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:34:15,062 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:34:16,305 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:34:16,490 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:34:16,491 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:34:16,491 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:34:16,497 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:34:16,768 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:34:17,772 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:34:21,286 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:34:21,287 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:34:21,287 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:34:21,294 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:34:21,775 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:34:22,295 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:34:23,780 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:34:25,502 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 18:34:26,323 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:34:26,324 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:34:26,324 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:34:26,330 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:34:26,782 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:34:27,330 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:34:27,787 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:34:30,061 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:34:30,061 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:34:30,061 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:34:33,275 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:34:33,669 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:34:33,669 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:34:33,670 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:34:33,676 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:34:33,792 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:34:35,797 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:34:37,755 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:34:37,756 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:34:37,756 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:34:37,762 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:34:37,799 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:34:38,762 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:34:39,804 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:34:41,682 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:34:41,683 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:34:41,683 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:34:41,688 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:34:41,805 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:34:43,811 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:34:44,689 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:34:45,061 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:34:45,061 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:34:45,061 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:34:47,397 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:34:47,398 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:34:47,398 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:34:47,404 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:34:47,814 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:34:49,817 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:34:50,441 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:34:51,350 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:34:51,351 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:34:51,352 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:34:51,357 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:34:51,820 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:34:53,825 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:34:55,502 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 18:34:55,503 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:34:59,767 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:34:59,768 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:34:59,768 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:34:59,774 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:34:59,831 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:35:00,061 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:35:00,061 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:35:00,062 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:35:01,286 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:35:01,838 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:35:05,899 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:35:05,899 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:35:05,899 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:35:05,905 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:35:06,841 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:35:06,941 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:35:07,845 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:35:10,910 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:35:10,911 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:35:10,911 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:35:10,917 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:35:11,851 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:35:11,851 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:35:12,917 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:35:15,061 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:35:15,062 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:35:15,062 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:35:15,617 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:35:15,617 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:35:15,617 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:35:15,623 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:35:15,855 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:35:17,860 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:35:18,661 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:35:19,947 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:35:19,948 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:35:19,948 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:35:19,954 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:35:20,862 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:35:21,866 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:35:23,955 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:35:24,187 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:35:24,188 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:35:24,188 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:35:24,194 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:35:24,868 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:35:25,508 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 18:35:25,872 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:35:29,240 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:35:29,241 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:35:29,241 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:35:29,241 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:35:29,246 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:35:29,876 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:35:30,062 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:35:30,062 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:35:30,063 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:35:31,884 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:35:32,792 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:35:32,792 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:35:32,792 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:35:32,798 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:35:32,885 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:35:33,887 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:35:34,837 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:35:36,768 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:35:36,769 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:35:36,769 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:35:36,776 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:35:36,889 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:35:37,892 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:35:40,777 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:35:42,169 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:35:42,169 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:35:42,170 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:35:42,175 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:35:42,896 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:35:43,900 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:35:45,061 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:35:45,062 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:35:45,062 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:35:46,338 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:35:48,178 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:35:48,179 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:35:48,179 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:35:48,185 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:35:48,904 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:35:49,908 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:35:52,221 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:35:52,556 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:35:52,557 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:35:52,557 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:35:52,563 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:35:52,910 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:35:53,913 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:35:55,508 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 18:35:57,509 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:35:57,794 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:35:57,795 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:35:57,795 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:35:57,801 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:35:57,917 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:35:59,922 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:36:00,061 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:36:00,062 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:36:00,062 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:36:02,332 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:36:02,332 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:36:02,332 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:36:02,338 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:36:02,925 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:36:03,377 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:36:03,928 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:36:06,744 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:36:06,745 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:36:06,745 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:36:06,751 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:36:06,930 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:36:07,934 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:36:08,752 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:36:11,272 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:36:11,273 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:36:11,273 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:36:11,279 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:36:11,937 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:36:13,940 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:36:14,280 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:36:15,061 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:36:15,062 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:36:15,062 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:36:15,453 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:36:15,454 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:36:15,454 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:36:15,460 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:36:15,943 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:36:17,948 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:36:19,461 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:36:19,851 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:36:19,851 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:36:19,852 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:36:19,857 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:36:19,950 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:36:21,955 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:36:23,842 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:36:23,843 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:36:23,843 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:36:23,848 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:36:23,956 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:36:24,849 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:36:25,510 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 18:36:25,961 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:36:30,062 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:36:30,062 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:36:30,062 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:36:30,063 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:36:30,874 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:36:30,875 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:36:30,875 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:36:30,880 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:36:30,966 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:36:31,970 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:36:34,600 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:36:34,601 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:36:34,601 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:36:34,607 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:36:34,972 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:36:35,607 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:36:35,976 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:36:38,550 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:36:38,551 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:36:38,551 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:36:38,556 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:36:38,978 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:36:39,982 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:36:41,557 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:36:42,361 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:36:42,361 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:36:42,362 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:36:42,368 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:36:42,984 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:36:43,988 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:36:45,062 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:36:45,062 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:36:45,063 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:36:46,185 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:36:46,186 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:36:46,186 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:36:46,192 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:36:46,990 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:36:47,229 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:36:47,993 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:36:50,143 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:36:50,144 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:36:50,144 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:36:50,151 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:36:50,996 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:36:51,999 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:36:53,151 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:36:53,940 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:36:53,941 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:36:53,941 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:36:53,947 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:36:54,000 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:36:55,513 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 18:36:56,005 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:36:57,181 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:36:57,182 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:36:57,182 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:36:57,188 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:36:58,009 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:36:58,009 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:36:58,188 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:37:00,062 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:37:00,062 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:37:00,062 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:37:00,742 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:37:00,742 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:37:00,743 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:37:00,748 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:37:01,011 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:37:02,015 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:37:03,785 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:37:06,156 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:37:06,157 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:37:06,157 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:37:06,163 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:37:07,019 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:37:08,023 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:37:09,163 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:37:11,901 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:37:11,902 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:37:11,902 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:37:11,908 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:37:12,026 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:37:14,031 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:37:14,908 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:37:15,062 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:37:15,062 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:37:15,062 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:37:16,697 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:37:16,697 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:37:16,698 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:37:16,703 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:37:17,034 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:37:18,038 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:37:20,704 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:37:21,316 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:37:21,317 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:37:21,317 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:37:21,323 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:37:22,044 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:37:22,044 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:37:25,516 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 18:37:25,738 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:37:25,739 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:37:25,739 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:37:25,739 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:37:25,745 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:37:26,047 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:37:28,052 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:37:29,679 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:37:29,680 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:37:29,680 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:37:29,686 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:37:30,054 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:37:30,062 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:37:30,063 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:37:30,063 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:37:31,357 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:37:32,058 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:37:33,184 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:37:33,185 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:37:33,185 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:37:33,191 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:37:34,062 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:37:34,062 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:37:37,229 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:37:38,169 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:37:38,169 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:37:38,170 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:37:38,175 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:37:39,066 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:37:40,071 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:37:42,551 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:37:42,552 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:37:42,552 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:37:42,553 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:37:42,559 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:37:43,073 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:37:44,077 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:37:45,062 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:37:45,063 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:37:45,063 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:37:46,943 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:37:46,944 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:37:46,944 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:37:46,950 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:37:47,079 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:37:47,951 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:37:48,080 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:37:51,743 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:37:51,744 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:37:51,744 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:37:51,750 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:37:52,086 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:37:53,750 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:37:54,091 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:37:55,518 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 18:37:55,570 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:37:55,570 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:37:55,571 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:37:55,577 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:37:56,092 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:37:58,097 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:37:59,577 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:38:00,033 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:38:00,034 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:38:00,035 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:38:00,040 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:38:00,063 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:38:00,063 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:38:00,064 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:38:00,098 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:38:02,104 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:38:04,807 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:38:04,808 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:38:04,808 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:38:04,808 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:38:04,814 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:38:05,107 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:38:06,109 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:38:09,591 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:38:09,592 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:38:09,592 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:38:09,598 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:38:10,112 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:38:10,598 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:38:12,117 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:38:13,370 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:38:13,370 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:38:13,371 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:38:13,377 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:38:14,121 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:38:14,121 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:38:15,062 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:38:15,063 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:38:15,063 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:38:16,288 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:38:19,757 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:38:19,758 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:38:19,758 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:38:19,763 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:38:20,127 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:38:21,801 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:38:22,132 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:38:23,508 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:38:23,509 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:38:23,509 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:38:23,515 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:38:24,134 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:38:25,521 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 18:38:26,138 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:38:27,523 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:38:28,317 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:38:28,317 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:38:28,318 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:38:28,323 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:38:29,140 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:38:30,062 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:38:30,063 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:38:30,063 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:38:30,144 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:38:31,951 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:38:31,952 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:38:31,952 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:38:31,958 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:38:32,145 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:38:32,997 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:38:34,150 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:38:35,651 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:38:35,652 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:38:35,652 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:38:35,658 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:38:36,151 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:38:38,153 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:38:38,658 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:38:39,823 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:38:39,824 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:38:39,824 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:38:39,830 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:38:40,156 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:38:42,161 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:38:43,831 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:38:45,062 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:38:45,063 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:38:45,063 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:38:45,864 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:38:45,865 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:38:45,865 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:38:45,871 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:38:46,166 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:38:48,171 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:38:48,909 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:38:49,995 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:38:49,996 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:38:49,996 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:38:50,002 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:38:50,172 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:38:52,177 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:38:54,003 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:38:54,053 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:38:54,053 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:38:54,054 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:38:54,059 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:38:54,178 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:38:55,528 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 18:38:56,184 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:38:58,123 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:38:58,123 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:38:58,124 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:38:58,129 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:38:58,185 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:38:59,130 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:39:00,063 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:39:00,063 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:39:00,063 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:39:00,190 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:39:01,805 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:39:01,806 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:39:01,806 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:39:01,812 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:39:02,192 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:39:04,196 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:39:04,812 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:39:05,983 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:39:05,984 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:39:05,984 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:39:05,990 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:39:06,197 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:39:08,201 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:39:09,990 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:39:09,991 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:39:09,991 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:39:09,991 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:39:09,997 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:39:10,202 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:39:12,207 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:39:14,998 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:39:15,063 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:39:15,063 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:39:15,063 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:39:18,384 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:39:18,385 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:39:18,385 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:39:18,390 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:39:19,215 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:39:20,217 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:39:20,429 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:39:24,544 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:39:24,544 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:39:24,545 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:39:24,551 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:39:25,223 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:39:25,527 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 18:39:25,528 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:39:26,227 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:39:29,571 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:39:29,572 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:39:29,572 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:39:29,578 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:39:30,063 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:39:30,063 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:39:30,064 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:39:30,231 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:39:31,366 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:39:32,236 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:39:33,422 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:39:33,422 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:39:33,423 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:39:33,428 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:39:34,239 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:39:34,240 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:39:36,465 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:39:38,111 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:39:38,112 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:39:38,112 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:39:38,118 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:39:38,242 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:39:40,248 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:39:42,052 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:39:42,053 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:39:42,053 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:39:42,053 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:39:42,059 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:39:42,250 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:39:44,255 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:39:45,063 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:39:45,064 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:39:45,064 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:39:46,040 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:39:46,040 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:39:46,040 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:39:46,046 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:39:46,257 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:39:47,085 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:39:48,260 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:39:50,005 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:39:50,006 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:39:50,006 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:39:50,012 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:39:50,262 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:39:52,266 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:39:53,013 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:39:54,890 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:39:54,891 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:39:54,891 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:39:54,897 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:39:55,268 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:39:55,532 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 18:39:56,271 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:39:58,533 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:39:59,247 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:39:59,248 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:39:59,248 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:39:59,254 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:39:59,273 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:40:00,063 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:40:00,064 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:40:00,064 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:40:00,277 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:40:03,858 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:40:03,859 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:40:03,859 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:40:03,859 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:40:03,864 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:40:04,281 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:40:06,286 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:40:07,935 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:40:07,936 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:40:07,937 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:40:07,942 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:40:08,287 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:40:08,943 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:40:10,291 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:40:12,095 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:40:12,095 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:40:12,096 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:40:12,101 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:40:12,293 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:40:14,102 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:40:14,295 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:40:15,063 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:40:15,064 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:40:15,064 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:40:15,635 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:40:15,636 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:40:15,636 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:40:15,642 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:40:16,298 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:40:18,303 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:40:19,681 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:40:20,718 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:40:20,719 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:40:20,719 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:40:20,725 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:40:21,305 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:40:22,310 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:40:24,546 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:40:24,546 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:40:24,547 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:40:24,552 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:40:25,312 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:40:25,531 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 18:40:25,532 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:40:26,316 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:40:28,738 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:40:28,739 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:40:28,739 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:40:28,745 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:40:29,318 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:40:30,063 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:40:30,064 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:40:30,064 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:40:30,322 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:40:31,357 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:40:34,064 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:40:34,065 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:40:34,065 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:40:34,071 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:40:34,325 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:40:36,330 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:40:37,109 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:40:38,763 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:40:38,764 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:40:38,764 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:40:38,769 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:40:39,332 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:40:40,336 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:40:42,770 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:40:45,063 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:40:45,064 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:40:45,064 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:40:46,936 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:40:46,936 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:40:46,937 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:40:46,942 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:40:47,343 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:40:47,981 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:40:48,346 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:40:50,704 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:40:50,704 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:40:50,705 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:40:50,710 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:40:51,349 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:40:52,353 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:40:53,711 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:40:54,716 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:40:54,716 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:40:54,717 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:40:54,722 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:40:55,355 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:40:55,536 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 18:40:56,359 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:40:59,537 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:41:00,063 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:41:00,064 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:41:00,064 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:41:00,890 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:41:00,891 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:41:00,891 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:41:00,896 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:41:01,363 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:41:02,369 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:41:04,933 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:41:09,038 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:41:09,039 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:41:09,039 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:41:09,045 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:41:09,374 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:41:10,045 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:41:10,376 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:41:13,059 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:41:13,060 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:41:13,060 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:41:13,066 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:41:13,380 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:41:14,384 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:41:15,063 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:41:15,064 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:41:15,064 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:41:15,065 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:41:18,710 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:41:18,711 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:41:18,711 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:41:18,717 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:41:19,389 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:41:20,393 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:41:20,717 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:41:23,920 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:41:23,921 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:41:23,921 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:41:23,927 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:41:24,397 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:41:25,536 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 18:41:26,402 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:41:26,537 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:41:28,080 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:41:28,081 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:41:28,081 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:41:28,086 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:41:28,403 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:41:30,064 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:41:30,064 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:41:30,065 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:41:30,408 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:41:31,822 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:41:31,823 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:41:31,823 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:41:31,823 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:41:31,829 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:41:32,409 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:41:34,414 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:41:36,865 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:41:37,607 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:41:37,608 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:41:37,608 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:41:37,614 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:41:38,419 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:41:38,420 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:41:42,147 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:41:42,148 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:41:42,148 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:41:42,149 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:41:42,154 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:41:42,423 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:41:44,428 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:41:45,064 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:41:45,064 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:41:45,065 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:41:45,918 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:41:45,919 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:41:45,919 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:41:45,924 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:41:46,430 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:41:47,961 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:41:48,434 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:41:52,961 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:41:54,538 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:41:54,539 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:41:54,539 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:41:54,545 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:41:55,441 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:41:55,539 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 18:41:56,446 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:41:58,094 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:41:58,095 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:41:58,095 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:41:58,095 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:41:58,101 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:41:58,447 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:42:00,064 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:42:00,064 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:42:00,065 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:42:00,451 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:42:02,328 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:42:02,332 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:42:02,332 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:42:02,337 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:42:02,454 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:42:03,373 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:42:04,458 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:42:05,912 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:42:05,913 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:42:05,914 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:42:05,919 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:42:06,460 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:42:08,464 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:42:08,920 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:42:12,841 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:42:12,842 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:42:12,842 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:42:12,848 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:42:13,468 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:42:14,472 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:42:14,848 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:42:15,064 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:42:15,064 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:42:15,065 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:42:17,777 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:42:17,778 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:42:17,778 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:42:17,784 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:42:18,476 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:42:20,481 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:42:20,821 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:42:21,938 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:42:21,939 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:42:21,939 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:42:21,944 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:42:22,483 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:42:24,487 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:42:25,542 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 18:42:25,892 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:42:25,893 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:42:25,894 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:42:25,894 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:42:25,899 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:42:26,488 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:42:28,492 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:42:30,064 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:42:30,064 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:42:30,065 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:42:31,286 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:42:32,841 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:42:32,841 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:42:32,842 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:42:32,847 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:42:33,497 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:42:34,501 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:42:36,885 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:42:38,066 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:42:38,067 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:42:38,067 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:42:38,073 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:42:38,505 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:42:40,510 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:42:41,612 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:42:41,613 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:42:41,614 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:42:41,619 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:42:42,512 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:42:42,512 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:42:42,620 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:42:45,064 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:42:45,065 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:42:45,065 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:42:46,807 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:42:46,807 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:42:46,807 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:42:46,813 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:42:47,518 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:42:47,849 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:42:48,522 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:42:51,214 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:42:51,215 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:42:51,215 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:42:51,221 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:42:51,524 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:42:52,528 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:42:53,221 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:42:55,545 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 18:42:55,906 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:42:55,907 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:42:55,907 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:42:55,912 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:42:56,531 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:42:58,534 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:42:58,913 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:43:00,064 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:43:00,065 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:43:00,065 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:43:01,001 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:43:01,001 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:43:01,001 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:43:01,007 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:43:01,538 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:43:02,544 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:43:04,045 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:43:04,881 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:43:04,881 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:43:04,882 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:43:04,888 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:43:05,546 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:43:06,549 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:43:09,888 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:43:10,601 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:43:10,602 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:43:10,602 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:43:10,608 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:43:11,555 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:43:12,558 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:43:15,064 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:43:15,065 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:43:15,065 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:43:15,066 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:43:19,174 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:43:19,174 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:43:19,174 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:43:19,180 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:43:19,566 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:43:20,220 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:43:20,568 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:43:23,349 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:43:23,350 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:43:23,350 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:43:23,356 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:43:23,572 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:43:24,576 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:43:25,356 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:43:25,549 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 18:43:27,233 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:43:27,234 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:43:27,234 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:43:27,240 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:43:27,578 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:43:28,582 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:43:30,064 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:43:30,065 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:43:30,065 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:43:31,038 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:43:31,038 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:43:31,038 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:43:31,039 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:43:31,044 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:43:31,584 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:43:32,588 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:43:35,064 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:43:35,064 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:43:35,065 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:43:35,070 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:43:35,590 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:43:36,071 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:43:36,591 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:43:41,071 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:43:43,642 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:43:43,643 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:43:43,643 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:43:43,648 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:43:44,605 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:43:44,605 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:43:45,064 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:43:45,065 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:43:45,065 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:43:46,263 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:43:47,817 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:43:47,818 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:43:47,818 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:43:47,824 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:43:48,609 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:43:50,615 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:43:51,861 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:43:53,246 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:43:53,247 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:43:53,247 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:43:53,252 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:43:53,617 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:43:54,620 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:43:55,552 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 18:43:57,242 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:43:57,243 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:43:57,243 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:43:57,243 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:43:57,255 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:43:57,622 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:43:58,626 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:44:00,064 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:44:00,065 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:44:00,065 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:44:02,262 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:44:03,293 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:44:03,293 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:44:03,293 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:44:03,299 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:44:03,631 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:44:04,634 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:44:07,337 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:44:08,306 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:44:08,307 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:44:08,307 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:44:08,312 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:44:08,638 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:44:10,643 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:44:13,313 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:44:13,902 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:44:13,903 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:44:13,903 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:44:13,909 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:44:14,646 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:44:15,065 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:44:15,065 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:44:15,065 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:44:16,652 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:44:17,327 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:44:17,328 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:44:17,328 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:44:17,334 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:44:17,653 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:44:18,373 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:44:18,653 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:44:21,153 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:44:21,154 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:44:21,155 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:44:21,160 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:44:21,657 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:44:22,661 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:44:24,161 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:44:24,951 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:44:24,952 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:44:24,952 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:44:24,958 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:44:25,557 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 18:44:25,664 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:44:26,667 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:44:28,841 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:44:28,842 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:44:28,842 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:44:28,848 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:44:29,669 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:44:29,848 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:44:30,065 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:44:30,065 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:44:30,066 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:44:30,673 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:44:32,926 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:44:32,927 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:44:32,927 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:44:32,933 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:44:33,675 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:44:34,679 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:44:34,969 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:44:37,088 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:44:37,089 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:44:37,089 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:44:37,095 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:44:37,681 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:44:38,684 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:44:40,096 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:44:40,712 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:44:40,713 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:44:40,713 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:44:40,719 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:44:41,687 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:44:42,690 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:44:45,065 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:44:45,065 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:44:45,066 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:44:45,345 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:44:45,819 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:44:45,820 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:44:45,820 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:44:45,826 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:44:46,694 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:44:48,699 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:44:49,578 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:44:49,579 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:44:49,579 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:44:49,585 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:44:49,700 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:44:50,585 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:44:50,701 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:44:55,254 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:44:55,254 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:44:55,255 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:44:55,261 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:44:55,556 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 18:44:55,706 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:44:56,558 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:44:56,710 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:44:59,858 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:44:59,859 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:44:59,859 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:44:59,865 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:45:00,066 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:45:00,066 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:45:00,066 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:45:00,714 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:45:02,287 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:45:02,720 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:45:03,723 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:45:03,724 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:45:03,724 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:45:03,730 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:45:04,724 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:45:04,724 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:45:07,769 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:45:12,160 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:45:12,161 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:45:12,161 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:45:12,167 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:45:12,731 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:45:13,167 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:45:14,736 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:45:15,066 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:45:15,066 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:45:15,066 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:45:16,347 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:45:16,347 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:45:16,348 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:45:16,353 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:45:16,738 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:45:18,389 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:45:18,740 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:45:23,389 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:45:24,742 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:45:24,742 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:45:24,743 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:45:24,750 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:45:24,751 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:45:25,561 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 18:45:26,754 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:45:28,563 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:45:29,019 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:45:29,019 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:45:29,020 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:45:29,025 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:45:29,756 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:45:30,066 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:45:30,066 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:45:30,067 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:45:30,757 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:45:33,243 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:45:33,244 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:45:33,244 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:45:33,250 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:45:33,761 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:45:34,289 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:45:34,765 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:45:38,253 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:45:38,254 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:45:38,254 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:45:38,260 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:45:38,768 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:45:40,261 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:45:40,774 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:45:41,781 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:45:41,782 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:45:41,782 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:45:41,787 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:45:42,777 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:45:42,778 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:45:45,066 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:45:45,066 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:45:45,067 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:45:45,370 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:45:45,786 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:45:45,787 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:45:45,787 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:45:45,793 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:45:46,784 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:45:46,784 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:45:50,317 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:45:50,318 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:45:50,318 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:45:50,324 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:45:50,786 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:45:51,324 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:45:52,791 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:45:55,413 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:45:55,413 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:45:55,414 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:45:55,419 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:45:55,560 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 18:45:55,794 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:45:56,562 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:45:56,797 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:45:59,394 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:45:59,394 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:45:59,395 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:45:59,401 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:45:59,799 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:46:00,066 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:46:00,067 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:46:00,067 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:46:00,801 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:46:02,358 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:46:03,419 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:46:03,420 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:46:03,420 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:46:03,427 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:46:03,806 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:46:04,809 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:46:07,465 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:46:12,465 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:46:13,011 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:46:13,012 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:46:13,012 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:46:13,018 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:46:13,818 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:46:14,823 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:46:15,066 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:46:15,067 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:46:15,067 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:46:17,084 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:46:17,085 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:46:17,085 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:46:17,091 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:46:17,825 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:46:18,091 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:46:18,827 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:46:23,092 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:46:23,184 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:46:23,185 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:46:23,185 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:46:23,190 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:46:23,833 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:46:24,838 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:46:25,566 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 18:46:27,978 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:46:27,979 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:46:27,979 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:46:27,984 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:46:28,841 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:46:28,985 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:46:30,066 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:46:30,066 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:46:30,067 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:46:30,846 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:46:34,364 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:46:36,438 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:46:36,439 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:46:36,439 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:46:36,445 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:46:36,853 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:46:38,858 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:46:39,446 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:46:40,443 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:46:40,444 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:46:40,444 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:46:40,450 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:46:40,860 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:46:42,864 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:46:44,451 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:46:45,066 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:46:45,066 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:46:45,067 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:46:46,000 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:46:46,001 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:46:46,001 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:46:46,007 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:46:46,868 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:46:48,873 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:46:50,008 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:46:50,331 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:46:50,332 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:46:50,332 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:46:50,338 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:46:50,874 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:46:52,878 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:46:54,971 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:46:54,972 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:46:54,972 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:46:54,978 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:46:55,566 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 18:46:55,567 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:46:55,881 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:46:56,884 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:47:00,066 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:47:00,067 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:47:00,067 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:47:00,882 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:47:00,882 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:47:00,883 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:47:00,883 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:47:00,890 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:47:00,890 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:47:02,895 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:47:05,861 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:47:05,862 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:47:05,863 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:47:05,868 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:47:05,897 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:47:06,869 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:47:06,900 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:47:09,844 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:47:09,845 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:47:09,845 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:47:09,851 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:47:09,902 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:47:10,905 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:47:12,851 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:47:14,602 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:47:14,603 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:47:14,603 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:47:14,609 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:47:14,908 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:47:15,066 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:47:15,067 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:47:15,067 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:47:16,914 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:47:18,316 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:47:18,331 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:47:18,332 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:47:18,332 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:47:18,338 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:47:18,915 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:47:20,919 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:47:23,377 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:47:23,644 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:47:23,645 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:47:23,645 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:47:23,651 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:47:23,921 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:47:24,925 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:47:25,569 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 18:47:28,570 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:47:29,326 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:47:29,327 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:47:29,327 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:47:29,333 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:47:29,929 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:47:30,066 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:47:30,067 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:47:30,067 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:47:30,933 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:47:33,080 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:47:33,081 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:47:33,081 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:47:33,087 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:47:33,936 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:47:34,087 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:47:34,940 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:47:37,030 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:47:37,031 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:47:37,031 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:47:37,037 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:47:37,942 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:47:38,945 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:47:40,038 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:47:41,525 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:47:41,529 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:47:41,529 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:47:41,535 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:47:41,948 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:47:42,951 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:47:45,067 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:47:45,067 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:47:45,067 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:47:45,331 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:47:47,042 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:47:47,042 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:47:47,042 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:47:47,048 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:47:47,957 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:47:48,960 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:47:51,085 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:47:53,149 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:47:53,150 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:47:53,150 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:47:53,156 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:47:53,965 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:47:54,969 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:47:55,572 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 18:47:56,573 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:48:00,067 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:48:00,067 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:48:00,067 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:48:01,637 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:48:01,638 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:48:01,638 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:48:01,638 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:48:01,646 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:48:01,976 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:48:02,982 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:48:05,611 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:48:05,612 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:48:05,612 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:48:05,618 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:48:05,984 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:48:06,986 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:48:07,619 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:48:10,880 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:48:10,881 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:48:10,881 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:48:10,887 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:48:10,990 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:48:12,888 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:48:12,993 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:48:15,067 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:48:15,067 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:48:15,067 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:48:15,075 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:48:15,332 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:48:15,332 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:48:15,338 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:48:15,996 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:48:17,000 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:48:18,339 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:48:18,913 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:48:18,914 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:48:18,914 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:48:18,919 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:48:19,002 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:48:21,007 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:48:22,353 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:48:22,353 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:48:22,354 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:48:22,359 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:48:23,008 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:48:23,360 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:48:25,012 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:48:25,575 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 18:48:26,319 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:48:26,320 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:48:26,320 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:48:26,326 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:48:27,014 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:48:29,018 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:48:29,327 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:48:30,067 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:48:30,067 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:48:30,067 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:48:31,109 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:48:31,110 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:48:31,110 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:48:31,116 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:48:32,020 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:48:33,024 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:48:35,153 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:48:35,719 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:48:35,720 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:48:35,720 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:48:35,725 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:48:36,026 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:48:37,029 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:48:40,726 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:48:44,163 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:48:44,163 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:48:44,164 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:48:44,169 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:48:45,039 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:48:45,067 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:48:45,067 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:48:45,068 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:48:46,309 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:48:47,044 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:48:48,367 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:48:48,368 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:48:48,368 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:48:48,374 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:48:49,045 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:48:51,051 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:48:51,413 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:48:51,892 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:48:51,892 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:48:51,893 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:48:51,898 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:48:52,051 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:48:53,054 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:48:55,582 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 18:48:56,583 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:48:58,008 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:48:58,008 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:48:58,009 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:48:58,014 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:48:58,057 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:48:59,061 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:49:00,067 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:49:00,067 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:49:00,068 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:49:02,315 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:49:06,239 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:49:06,239 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:49:06,239 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:49:06,245 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:49:07,070 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:49:08,281 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:49:09,076 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:49:11,766 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:49:11,766 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:49:11,767 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:49:11,773 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:49:12,078 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:49:13,081 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:49:13,773 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:49:15,067 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:49:15,068 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:49:15,068 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:49:15,386 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:49:15,386 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:49:15,386 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:49:15,392 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:49:16,083 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:49:17,087 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:49:19,429 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:49:19,698 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:49:19,698 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:49:19,699 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:49:19,704 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:49:20,090 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:49:21,093 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:49:23,899 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:49:23,900 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:49:23,900 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:49:23,906 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:49:24,095 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:49:24,906 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:49:25,097 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:49:25,582 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 18:49:27,662 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:49:27,663 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:49:27,663 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:49:27,669 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:49:28,101 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:49:29,105 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:49:30,067 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:49:30,068 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:49:30,068 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:49:30,299 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:49:32,079 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:49:32,080 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:49:32,080 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:49:32,086 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:49:32,107 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:49:33,110 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:49:36,121 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:49:40,336 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:49:40,337 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:49:40,337 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:49:40,343 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:49:41,119 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:49:41,343 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:49:43,124 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:49:44,544 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:49:44,545 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:49:44,545 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:49:44,551 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:49:45,067 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:49:45,068 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:49:45,068 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:49:45,127 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:49:46,379 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:49:47,131 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:49:48,675 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:49:48,675 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:49:48,676 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:49:48,681 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:49:49,133 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:49:51,137 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:49:51,717 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:49:52,518 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:49:52,519 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:49:52,519 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:49:52,525 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:49:53,138 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:49:55,142 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:49:55,588 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 18:49:57,589 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:49:58,966 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:49:58,967 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:49:58,967 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:49:58,973 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:49:59,146 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:50:00,067 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:50:00,068 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:50:00,068 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:50:01,151 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:50:03,289 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:50:03,624 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:50:03,625 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:50:03,625 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:50:03,631 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:50:04,155 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:50:05,157 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:50:07,660 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:50:07,660 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:50:07,661 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:50:07,666 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:50:08,159 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:50:08,667 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:50:09,163 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:50:11,860 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:50:11,861 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:50:11,861 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:50:11,866 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:50:12,165 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:50:13,168 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:50:13,867 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:50:15,067 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:50:15,068 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:50:15,068 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:50:16,246 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:50:16,247 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:50:16,247 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:50:16,253 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:50:17,172 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:50:19,177 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:50:19,289 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:50:21,033 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:50:21,034 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:50:21,034 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:50:21,040 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:50:21,179 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:50:23,181 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:50:25,041 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:50:25,588 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 18:50:26,565 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:50:26,566 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:50:26,566 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:50:26,572 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:50:27,187 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:50:29,192 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:50:30,068 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:50:30,068 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:50:30,068 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:50:30,272 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:50:30,516 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:50:30,516 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:50:30,517 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:50:30,522 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:50:31,193 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:50:33,197 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:50:34,533 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:50:34,534 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:50:34,534 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:50:34,539 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:50:35,198 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:50:35,540 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:50:37,203 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:50:39,070 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:50:39,071 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:50:39,071 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:50:39,076 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:50:39,204 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:50:41,077 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:50:41,210 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:50:43,085 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:50:43,086 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:50:43,086 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:50:43,092 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:50:43,211 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:50:45,068 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:50:45,068 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:50:45,068 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:50:45,216 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:50:46,296 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:50:48,004 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:50:48,005 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:50:48,005 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:50:48,010 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:50:48,218 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:50:49,222 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:50:52,049 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:50:52,345 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:50:52,345 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:50:52,346 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:50:52,351 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:50:53,224 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:50:55,230 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:50:55,594 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 18:50:56,719 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:50:56,720 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:50:56,720 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:50:56,725 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:50:57,231 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:50:57,726 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:50:59,235 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:51:00,068 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:51:00,068 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:51:00,068 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:51:02,095 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:51:02,096 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:51:02,096 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:51:02,102 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:51:02,238 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:51:03,141 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:51:03,242 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:51:07,772 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:51:07,772 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:51:07,773 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:51:07,778 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:51:08,246 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:51:08,779 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:51:09,250 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:51:11,368 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:51:11,369 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:51:11,369 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:51:11,375 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:51:12,252 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:51:13,253 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:51:14,375 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:51:15,068 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:51:15,068 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:51:15,069 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:51:15,765 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:51:15,766 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:51:15,766 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:51:15,772 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:51:16,257 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:51:17,262 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:51:19,809 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:51:22,773 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:51:22,774 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:51:22,775 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:51:22,780 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:51:23,268 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:51:25,271 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:51:25,594 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 18:51:25,595 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:51:27,543 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:51:27,544 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:51:27,544 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:51:27,550 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:51:28,275 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:51:29,279 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:51:30,068 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:51:30,068 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:51:30,069 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:51:31,306 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:51:31,698 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:51:31,699 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:51:31,699 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:51:31,705 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:51:32,281 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:51:33,285 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:51:36,334 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:51:36,335 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:51:36,335 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:51:36,335 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:51:36,341 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:51:37,288 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:51:39,293 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:51:41,342 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:51:41,813 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:51:41,814 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:51:41,814 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:51:41,820 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:51:42,296 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:51:45,068 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:51:45,068 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:51:45,069 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:51:45,302 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:51:46,159 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:51:46,159 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:51:46,159 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:51:46,166 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:51:46,303 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:51:47,201 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:51:49,309 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:51:50,205 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:51:50,205 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:51:50,206 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:51:50,211 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:51:50,309 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:51:52,212 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:51:53,315 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:51:55,432 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:51:55,433 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:51:55,433 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:51:55,439 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:51:55,596 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 18:51:56,317 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:51:57,320 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:51:57,597 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:51:59,173 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:51:59,173 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:51:59,174 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:51:59,179 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:51:59,322 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:52:00,068 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:52:00,068 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:52:00,069 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:52:01,325 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:52:03,327 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:52:03,641 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:52:03,642 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:52:03,642 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:52:03,648 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:52:04,330 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:52:05,332 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:52:08,336 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:52:08,337 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:52:08,337 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:52:08,337 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:52:08,343 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:52:09,336 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:52:11,341 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:52:11,719 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:52:11,719 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:52:11,720 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:52:11,725 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:52:12,344 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:52:12,344 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:52:13,726 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:52:15,068 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:52:15,068 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:52:15,069 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:52:15,736 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:52:15,736 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:52:15,736 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:52:15,742 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:52:16,349 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:52:16,349 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:52:18,781 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:52:20,113 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:52:20,114 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:52:20,114 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:52:20,120 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:52:20,352 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:52:22,357 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:52:23,882 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:52:23,883 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:52:23,883 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:52:23,883 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:52:23,889 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:52:24,358 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:52:25,599 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 18:52:26,363 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:52:27,655 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:52:27,656 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:52:27,656 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:52:27,663 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:52:28,366 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:52:28,366 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:52:29,663 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:52:30,068 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:52:30,069 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:52:30,069 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:52:34,524 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:52:34,525 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:52:34,526 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:52:34,532 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:52:35,374 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:52:35,532 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:52:36,377 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:52:38,544 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:52:38,545 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:52:38,545 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:52:38,551 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:52:39,380 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:52:40,383 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:52:40,552 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:52:43,114 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:52:43,114 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:52:43,115 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:52:43,121 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:52:43,387 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:52:44,389 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:52:45,068 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:52:45,069 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:52:45,069 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:52:46,339 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:52:47,638 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:52:47,638 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:52:47,638 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:52:47,644 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:52:48,396 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:52:48,396 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:52:51,681 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:52:52,459 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:52:52,460 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:52:52,460 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:52:52,466 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:52:53,402 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:52:54,404 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:52:55,602 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 18:52:57,604 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:53:00,068 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:53:00,069 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:53:00,069 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:53:01,124 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:53:01,124 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:53:01,125 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:53:01,130 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:53:01,413 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:53:02,416 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:53:03,169 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:53:04,561 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:53:04,562 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:53:04,562 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:53:04,568 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:53:05,420 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:53:06,424 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:53:08,568 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:53:11,257 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:53:11,258 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:53:11,259 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:53:11,264 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:53:11,428 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:53:12,430 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:53:14,265 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:53:15,069 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:53:15,069 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:53:15,069 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:53:16,808 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:53:16,812 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:53:16,812 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:53:16,818 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:53:17,436 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:53:18,439 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:53:19,853 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:53:24,209 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:53:24,210 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:53:24,210 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:53:24,216 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:53:24,446 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:53:25,216 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:53:25,605 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 18:53:26,451 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:53:27,510 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:53:27,511 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:53:27,511 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:53:27,517 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:53:28,454 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:53:28,455 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:53:30,069 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:53:30,069 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:53:30,069 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:53:30,296 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:53:31,178 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:53:31,178 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:53:31,178 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:53:31,184 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:53:31,457 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:53:32,460 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:53:35,655 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:53:35,655 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:53:35,656 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:53:35,656 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:53:35,661 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:53:36,465 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:53:36,465 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:53:39,494 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:53:39,495 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:53:39,495 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:53:39,500 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:53:40,471 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:53:40,472 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:53:41,501 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:53:44,235 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:53:44,235 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:53:44,236 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:53:44,241 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:53:44,475 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:53:45,069 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:53:45,070 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:53:45,070 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:53:46,480 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:53:47,302 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:53:52,303 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:53:52,405 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:53:52,406 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:53:52,406 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:53:52,412 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:53:52,487 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:53:54,492 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:53:55,609 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 18:53:56,919 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:53:56,919 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:53:56,920 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:53:56,925 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:53:57,495 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:53:57,925 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:53:58,498 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:54:00,069 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:54:00,069 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:54:00,070 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:54:01,151 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:54:01,151 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:54:01,151 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:54:01,157 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:54:01,501 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:54:02,503 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:54:03,193 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:54:05,558 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:54:05,559 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:54:05,559 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:54:05,565 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:54:06,510 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:54:06,510 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:54:08,565 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:54:10,186 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:54:10,187 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:54:10,187 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:54:10,192 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:54:10,513 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:54:12,518 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:54:14,193 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:54:15,069 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:54:15,069 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:54:15,070 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:54:15,676 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:54:15,676 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:54:15,677 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:54:15,682 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:54:16,524 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:54:16,524 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:54:19,371 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:54:19,372 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:54:19,372 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:54:19,372 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:54:19,377 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:54:19,527 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:54:20,530 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:54:23,373 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:54:23,373 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:54:23,374 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:54:23,379 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:54:23,533 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:54:24,379 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:54:24,534 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:54:25,615 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 18:54:28,621 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:54:28,622 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:54:28,622 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:54:28,628 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:54:29,542 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:54:29,628 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:54:30,069 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:54:30,070 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:54:30,070 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:54:30,545 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:54:35,355 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:54:35,970 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:54:35,971 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:54:35,971 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:54:35,976 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:54:36,553 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:54:36,553 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:54:39,660 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:54:39,661 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:54:39,661 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:54:39,666 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:54:40,559 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:54:40,559 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:54:40,667 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:54:43,923 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:54:43,924 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:54:43,924 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:54:43,930 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:54:44,564 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:54:44,565 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:54:45,069 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:54:45,069 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:54:45,070 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:54:46,405 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:54:50,470 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:54:50,470 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:54:50,471 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:54:50,476 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:54:50,572 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:54:51,513 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:54:52,577 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:54:55,059 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:54:55,059 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:54:55,060 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:54:55,066 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:54:55,580 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:54:55,614 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 18:54:56,583 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:54:56,615 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:54:59,873 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:54:59,874 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:54:59,874 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:54:59,880 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:55:00,069 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:55:00,070 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:55:00,070 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:55:00,588 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:55:00,589 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:55:02,312 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:55:03,832 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:55:03,832 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:55:03,832 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:55:03,838 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:55:04,596 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:55:04,596 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:55:07,877 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:55:12,238 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:55:12,239 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:55:12,239 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:55:12,245 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:55:12,603 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:55:13,245 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:55:14,608 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:55:15,069 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:55:15,070 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:55:15,070 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:55:16,793 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:55:16,794 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:55:16,794 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:55:16,800 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:55:17,611 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:55:18,614 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:55:18,837 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:55:20,622 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:55:20,623 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:55:20,623 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:55:20,629 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:55:21,617 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:55:22,620 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:55:24,630 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:55:24,884 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:55:24,885 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:55:24,885 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:55:24,891 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:55:25,619 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 18:55:25,624 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:55:26,627 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:55:29,045 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:55:29,046 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:55:29,046 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:55:29,052 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:55:29,630 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:55:30,053 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:55:30,069 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:55:30,070 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:55:30,070 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:55:30,633 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:55:32,752 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:55:32,752 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:55:32,753 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:55:32,758 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:55:33,636 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:55:34,638 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:55:35,797 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:55:36,557 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:55:36,557 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:55:36,558 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:55:36,563 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:55:36,640 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:55:38,644 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:55:41,423 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:55:41,424 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:55:41,425 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:55:41,425 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:55:41,430 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:55:41,647 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:55:42,650 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:55:45,070 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:55:45,070 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:55:45,070 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:55:45,592 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:55:45,593 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:55:45,593 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:55:45,599 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:55:45,653 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:55:46,637 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:55:46,656 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:55:49,747 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:55:49,748 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:55:49,748 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:55:49,753 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:55:50,663 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:55:50,663 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:55:51,754 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:55:54,529 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:55:54,530 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:55:54,530 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:55:54,536 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:55:54,666 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:55:55,619 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 18:55:56,671 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:55:57,621 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:55:58,074 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:55:58,075 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:55:58,075 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:55:58,080 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:55:58,676 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:55:58,676 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:56:00,070 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:56:00,070 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:56:00,070 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:56:03,339 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:56:04,000 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:56:04,000 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:56:04,001 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:56:04,006 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:56:04,684 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:56:04,684 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:56:09,045 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:56:12,224 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:56:12,225 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:56:12,225 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:56:12,231 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:56:12,691 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:56:14,231 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:56:14,697 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:56:15,070 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:56:15,070 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:56:15,070 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:56:16,190 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:56:16,191 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:56:16,191 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:56:16,196 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:56:16,698 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:56:18,703 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:56:19,233 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:56:21,539 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:56:21,540 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:56:21,540 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:56:21,546 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:56:21,706 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:56:22,709 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:56:24,546 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:56:25,500 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:56:25,501 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:56:25,501 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:56:25,507 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:56:25,623 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 18:56:25,713 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:56:26,714 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:56:29,625 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:56:30,070 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:56:30,070 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:56:30,070 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:56:30,961 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:56:30,962 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:56:30,962 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:56:30,968 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:56:31,722 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:56:32,725 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:56:35,005 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:56:35,591 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:56:35,591 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:56:35,592 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:56:35,597 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:56:35,728 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:56:36,731 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:56:40,598 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:56:41,002 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:56:41,002 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:56:41,003 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:56:41,008 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:56:41,737 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:56:42,740 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:56:45,070 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:56:45,070 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:56:45,071 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:56:45,711 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:56:45,712 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:56:45,712 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:56:45,712 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:56:45,718 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:56:45,744 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:56:46,747 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:56:50,753 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:56:50,797 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:56:50,797 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:56:50,798 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:56:50,803 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:56:51,753 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:56:52,756 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:56:55,624 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 18:56:56,626 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:56:57,639 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:56:57,639 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:56:57,639 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:56:57,645 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:56:57,762 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:56:58,765 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:57:00,070 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:57:00,070 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:57:00,071 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:57:01,725 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:57:01,726 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:57:01,726 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:57:01,726 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:57:01,732 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:57:01,769 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:57:02,770 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:57:05,595 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:57:05,596 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:57:05,596 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:57:05,602 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:57:05,775 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:57:06,778 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:57:07,603 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:57:09,617 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:57:09,618 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:57:09,618 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:57:09,624 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:57:09,782 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:57:10,784 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:57:12,625 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:57:14,369 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:57:14,370 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:57:14,370 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:57:14,376 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:57:14,788 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:57:15,070 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:57:15,070 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:57:15,071 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:57:16,793 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:57:18,132 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:57:18,132 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:57:18,132 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:57:18,133 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:57:18,138 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:57:18,798 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:57:18,798 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:57:22,335 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:57:22,336 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:57:22,336 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:57:22,342 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:57:22,801 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:57:23,342 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:57:24,805 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:57:25,627 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 18:57:28,129 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:57:28,130 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:57:28,130 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:57:28,136 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:57:28,811 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:57:28,811 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:57:29,136 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:57:30,070 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:57:30,071 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:57:30,071 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:57:32,009 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:57:32,010 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:57:32,010 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:57:32,015 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:57:32,817 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:57:32,817 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:57:35,056 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:57:36,202 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:57:36,203 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:57:36,203 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:57:36,209 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:57:36,823 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:57:36,823 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:57:40,080 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:57:40,080 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:57:40,081 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:57:40,081 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:57:40,086 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:57:40,830 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:57:40,830 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:57:43,769 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:57:43,770 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:57:43,770 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:57:43,776 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:57:43,832 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:57:44,835 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:57:45,071 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:57:45,071 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:57:45,071 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:57:45,301 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:57:50,302 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:57:51,294 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:57:51,295 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:57:51,295 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:57:51,300 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:57:51,843 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:57:52,846 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:57:55,337 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:57:55,630 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 18:57:56,656 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:57:56,657 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:57:56,657 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:57:56,663 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:57:56,849 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:57:58,854 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:58:00,070 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:58:00,071 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:58:00,071 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:58:00,364 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:58:05,005 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:58:05,006 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:58:05,006 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:58:05,012 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:58:05,863 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:58:06,049 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:58:06,866 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:58:09,707 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:58:09,708 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:58:09,708 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:58:09,714 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:58:09,869 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:58:10,872 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:58:11,714 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:58:13,787 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:58:13,788 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:58:13,788 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:58:13,793 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:58:13,875 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:58:14,878 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:58:15,071 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:58:15,071 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:58:15,071 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:58:17,274 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:58:17,547 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:58:17,548 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:58:17,548 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:58:17,553 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:58:17,881 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:58:18,885 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:58:21,952 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:58:21,953 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:58:21,953 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:58:21,959 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:58:22,890 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:58:22,890 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:58:22,960 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:58:25,634 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 18:58:28,110 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:58:28,111 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:58:28,111 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:58:28,111 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:58:28,116 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:58:28,899 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:58:28,899 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:58:30,071 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:58:30,071 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:58:30,071 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:58:32,216 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:58:32,216 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:58:32,216 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:58:32,222 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:58:32,905 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:58:32,905 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:58:33,261 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:58:36,264 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:58:36,265 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:58:36,265 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:58:36,271 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:58:36,911 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:58:36,911 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:58:38,271 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:58:39,865 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:58:39,866 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:58:39,866 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:58:39,872 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:58:39,915 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:58:40,918 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:58:43,872 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:58:45,071 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:58:45,071 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:58:45,071 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:58:48,374 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:58:48,375 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:58:48,375 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:58:48,381 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:58:48,927 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:58:49,420 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:58:50,932 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:58:52,864 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:58:52,865 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:58:52,865 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:58:52,871 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:58:52,934 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:58:54,871 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:58:54,938 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:58:55,637 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 18:58:56,625 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:58:56,626 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:58:56,626 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:58:56,632 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:58:56,940 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:58:58,944 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:59:00,071 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:59:00,071 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:59:00,072 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:59:00,356 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:59:01,340 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:59:01,340 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:59:01,340 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:59:01,346 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:59:01,947 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:59:02,950 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:59:05,385 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:59:07,150 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:59:07,150 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:59:07,151 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:59:07,156 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:59:07,955 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:59:08,958 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:59:11,156 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:59:11,366 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:59:11,367 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:59:11,367 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:59:11,372 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:59:11,961 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:59:12,964 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:59:15,071 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:59:15,071 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:59:15,072 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:59:15,109 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:59:15,365 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:59:15,365 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:59:15,370 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:59:15,967 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:59:16,371 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:59:16,969 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:59:18,912 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:59:18,913 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:59:18,913 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:59:18,918 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:59:18,972 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:59:20,976 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:59:21,919 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:59:24,643 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:59:24,644 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:59:24,645 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:59:24,649 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:59:24,979 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:59:25,640 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 18:59:26,985 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:59:27,642 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:59:28,289 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:59:28,290 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:59:28,290 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:59:28,295 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:59:28,988 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:59:28,988 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:59:30,071 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:59:30,071 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:59:30,072 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:59:32,093 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:59:32,093 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:59:32,094 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:59:32,098 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:59:32,993 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:59:32,993 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:59:33,137 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:59:38,137 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:59:39,686 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:59:39,687 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:59:39,687 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:59:39,694 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:59:40,002 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:59:41,004 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:59:43,695 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:59:43,851 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:59:43,851 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:59:43,852 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:59:43,857 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:59:44,007 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:59:45,010 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:59:45,071 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:59:45,072 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:59:45,072 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:59:48,428 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:59:48,429 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:59:48,429 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:59:48,434 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:59:49,015 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:59:49,473 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:59:51,020 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:59:54,473 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:59:54,773 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:59:54,773 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:59:54,774 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:59:54,778 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:59:55,023 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:59:55,648 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 18:59:57,028 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 18:59:58,719 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:59:58,720 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 18:59:58,721 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:59:58,725 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:59:59,030 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 18:59:59,726 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:00:00,071 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:00:00,071 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:00:00,072 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:00:01,034 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:00:05,302 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:00:07,202 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:00:07,202 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:00:07,202 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:00:07,208 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:00:08,043 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:00:09,045 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:00:11,245 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:00:11,341 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:00:11,342 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:00:11,342 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:00:11,347 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:00:12,048 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:00:13,051 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:00:15,071 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:00:15,072 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:00:15,072 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:00:16,318 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:00:16,366 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:00:16,367 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:00:16,367 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:00:16,372 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:00:17,054 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:00:19,060 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:00:20,683 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:00:20,684 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:00:20,684 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:00:20,689 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:00:21,061 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:00:21,690 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:00:23,066 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:00:24,529 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:00:24,529 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:00:24,530 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:00:24,534 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:00:25,067 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:00:25,644 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 19:00:27,072 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:00:27,646 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:00:30,000 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:00:30,001 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:00:30,001 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:00:30,006 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:00:30,072 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:00:30,072 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:00:30,072 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:00:30,076 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:00:31,079 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:00:33,311 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:00:33,927 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:00:33,927 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:00:33,927 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:00:33,932 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:00:34,082 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:00:35,085 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:00:37,637 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:00:37,637 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:00:37,638 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:00:37,643 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:00:38,088 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:00:38,643 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:00:39,091 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:00:41,840 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:00:41,841 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:00:41,841 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:00:41,846 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:00:42,094 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:00:43,095 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:00:43,846 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:00:45,072 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:00:45,072 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:00:45,072 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:00:46,524 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:00:46,525 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:00:46,525 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:00:46,530 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:00:47,100 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:00:49,103 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:00:49,569 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:00:51,921 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:00:51,921 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:00:51,922 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:00:51,926 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:00:52,108 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:00:53,111 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:00:54,927 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:00:55,651 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 19:00:56,079 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:00:56,080 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:00:56,080 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:00:56,085 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:00:56,115 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:00:57,117 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:01:00,072 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:01:00,072 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:01:00,072 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:01:00,375 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:01:01,157 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:01:01,157 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:01:01,157 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:01:01,163 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:01:02,123 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:01:03,126 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:01:05,306 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:01:05,307 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:01:05,307 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:01:05,313 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:01:06,129 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:01:06,313 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:01:07,131 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:01:11,314 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:01:13,408 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:01:13,409 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:01:13,409 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:01:13,414 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:01:14,140 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:01:15,072 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:01:15,072 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:01:15,073 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:01:15,143 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:01:16,346 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:01:17,318 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:01:17,319 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:01:17,319 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:01:17,324 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:01:18,146 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:01:19,149 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:01:21,277 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:01:21,277 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:01:21,278 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:01:21,283 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:01:22,152 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:01:22,283 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:01:23,155 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:01:25,651 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 19:01:25,969 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:01:25,970 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:01:25,970 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:01:25,978 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:01:26,158 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:01:27,161 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:01:27,979 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:01:29,670 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:01:29,670 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:01:29,670 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:01:29,676 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:01:30,072 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:01:30,072 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:01:30,072 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:01:30,164 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:01:31,166 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:01:33,360 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:01:37,917 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:01:37,918 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:01:37,918 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:01:37,923 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:01:38,175 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:01:38,961 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:01:39,178 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:01:42,491 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:01:42,491 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:01:42,491 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:01:42,497 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:01:43,182 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:01:44,497 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:01:45,072 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:01:45,072 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:01:45,073 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:01:45,186 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:01:46,900 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:01:46,901 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:01:46,901 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:01:46,907 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:01:47,189 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:01:49,193 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:01:49,945 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:01:51,058 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:01:51,059 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:01:51,059 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:01:51,064 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:01:51,194 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:01:53,199 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:01:55,065 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:01:55,497 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:01:55,498 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:01:55,498 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:01:55,503 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:01:55,656 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 19:01:56,202 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:01:57,204 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:02:00,072 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:02:00,072 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:02:00,073 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:02:00,361 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:02:01,206 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:02:01,207 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:02:01,207 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:02:01,212 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:02:02,210 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:02:03,213 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:02:06,249 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:02:07,214 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:02:07,215 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:02:07,215 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:02:07,220 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:02:08,218 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:02:09,221 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:02:11,296 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:02:11,297 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:02:11,297 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:02:11,297 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:02:11,303 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:02:12,225 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:02:13,227 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:02:15,072 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:02:15,072 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:02:15,073 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:02:15,433 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:02:15,433 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:02:15,434 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:02:15,438 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:02:16,230 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:02:16,477 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:02:17,232 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:02:21,421 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:02:21,422 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:02:21,422 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:02:21,427 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:02:22,239 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:02:22,427 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:02:23,242 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:02:25,650 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:02:25,651 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:02:25,651 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:02:25,657 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:02:25,657 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 19:02:26,245 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:02:27,247 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:02:27,658 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:02:30,072 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:02:30,072 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:02:30,073 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:02:30,744 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:02:30,745 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:02:30,745 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:02:30,751 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:02:31,251 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:02:32,789 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:02:33,256 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:02:35,854 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:02:35,855 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:02:35,855 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:02:35,860 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:02:36,259 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:02:37,262 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:02:37,861 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:02:41,652 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:02:41,653 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:02:41,654 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:02:41,659 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:02:42,267 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:02:43,270 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:02:43,660 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:02:45,072 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:02:45,073 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:02:45,073 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:02:45,944 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:02:45,945 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:02:45,945 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:02:45,950 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:02:46,273 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:02:47,279 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:02:48,989 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:02:49,731 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:02:49,731 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:02:49,732 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:02:49,736 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:02:50,281 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:02:51,283 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:02:53,551 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:02:53,552 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:02:53,552 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:02:53,557 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:02:54,285 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:02:54,558 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:02:55,288 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:02:55,658 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 19:02:57,618 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:02:57,619 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:02:57,619 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:02:57,625 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:02:58,291 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:02:59,294 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:02:59,626 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:03:00,072 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:03:00,073 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:03:00,073 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:03:01,491 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:03:01,492 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:03:01,492 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:03:01,497 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:03:02,297 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:03:03,299 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:03:05,540 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:03:06,485 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:03:06,486 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:03:06,486 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:03:06,491 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:03:07,305 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:03:07,306 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:03:10,776 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:03:10,776 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:03:10,777 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:03:10,777 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:03:10,782 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:03:11,309 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:03:13,314 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:03:14,593 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:03:14,594 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:03:14,594 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:03:14,599 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:03:15,073 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:03:15,073 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:03:15,073 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:03:15,315 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:03:16,363 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:03:17,319 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:03:19,736 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:03:19,736 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:03:19,736 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:03:19,742 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:03:20,323 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:03:21,326 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:03:21,781 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:03:23,990 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:03:23,990 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:03:23,991 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:03:23,996 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:03:24,329 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:03:25,332 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:03:25,661 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 19:03:27,663 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:03:28,240 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:03:28,241 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:03:28,241 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:03:28,246 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:03:28,335 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:03:29,338 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:03:30,073 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:03:30,073 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:03:30,073 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:03:32,199 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:03:32,200 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:03:32,200 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:03:32,206 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:03:32,341 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:03:33,245 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:03:33,343 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:03:37,032 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:03:37,033 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:03:37,033 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:03:37,038 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:03:37,347 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:03:39,039 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:03:39,352 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:03:44,039 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:03:45,073 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:03:45,073 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:03:45,073 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:03:45,400 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:03:45,400 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:03:45,401 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:03:45,406 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:03:46,361 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:03:47,364 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:03:48,951 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:03:48,952 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:03:48,952 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:03:48,957 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:03:49,365 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:03:49,957 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:03:51,370 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:03:52,795 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:03:52,796 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:03:52,796 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:03:52,801 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:03:53,371 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:03:55,375 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:03:55,665 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 19:03:55,667 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:03:57,246 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:03:57,247 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:03:57,247 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:03:57,252 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:03:57,377 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:03:59,381 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:04:00,073 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:04:00,073 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:04:00,074 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:04:01,340 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:04:01,472 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:04:01,473 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:04:01,473 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:04:01,478 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:04:02,384 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:04:03,387 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:04:06,075 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:04:06,076 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:04:06,076 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:04:06,081 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:04:06,390 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:04:07,082 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:04:07,393 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:04:12,082 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:04:12,085 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:04:12,086 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:04:12,086 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:04:12,091 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:04:12,399 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:04:13,402 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:04:15,073 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:04:15,073 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:04:15,073 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:04:16,867 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:04:16,871 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:04:16,871 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:04:16,877 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:04:17,405 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:04:17,913 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:04:19,410 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:04:22,913 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:04:23,985 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:04:23,986 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:04:23,986 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:04:23,991 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:04:24,416 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:04:25,419 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:04:25,669 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 19:04:27,754 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:04:27,755 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:04:27,755 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:04:27,762 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:04:28,422 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:04:28,763 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:04:29,424 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:04:30,073 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:04:30,073 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:04:30,074 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:04:32,272 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:04:32,273 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:04:32,273 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:04:32,282 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:04:32,427 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:04:33,430 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:04:34,317 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:04:36,159 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:04:36,159 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:04:36,160 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:04:36,165 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:04:36,433 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:04:37,436 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:04:40,166 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:04:40,303 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:04:40,304 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:04:40,304 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:04:40,309 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:04:40,439 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:04:41,441 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:04:44,828 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:04:44,829 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:04:44,829 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:04:44,835 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:04:45,073 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:04:45,073 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:04:45,074 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:04:45,333 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:04:45,445 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:04:47,451 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:04:48,672 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:04:48,673 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:04:48,673 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:04:48,678 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:04:49,452 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:04:50,717 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:04:51,456 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:04:52,473 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:04:52,474 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:04:52,474 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:04:52,479 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:04:53,461 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:04:53,461 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:04:55,672 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 19:04:56,673 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:04:58,403 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:04:58,404 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:04:58,404 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:04:58,409 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:04:58,467 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:04:59,480 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:05:00,073 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:05:00,073 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:05:00,074 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:05:02,368 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:05:02,804 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:05:02,805 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:05:02,805 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:05:02,810 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:05:03,483 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:05:05,487 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:05:07,283 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:05:07,283 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:05:07,284 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:05:07,288 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:05:07,489 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:05:08,289 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:05:09,492 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:05:11,153 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:05:11,154 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:05:11,154 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:05:11,159 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:05:11,494 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:05:13,495 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:05:14,160 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:05:15,073 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:05:15,073 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:05:15,074 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:05:16,888 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:05:16,889 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:05:16,889 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:05:16,894 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:05:17,497 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:05:19,502 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:05:19,933 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:05:22,213 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:05:22,214 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:05:22,214 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:05:22,222 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:05:22,503 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:05:23,506 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:05:25,222 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:05:25,678 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 19:05:26,339 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:05:26,340 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:05:26,340 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:05:26,346 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:05:26,508 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:05:27,510 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:05:30,073 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:05:30,074 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:05:30,074 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:05:30,387 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:05:34,646 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:05:34,647 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:05:34,647 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:05:34,652 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:05:35,519 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:05:35,520 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:05:35,689 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:05:38,695 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:05:38,696 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:05:38,696 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:05:38,701 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:05:39,528 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:05:39,528 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:05:40,702 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:05:42,537 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:05:42,538 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:05:42,538 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:05:42,544 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:05:43,534 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:05:43,534 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:05:45,073 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:05:45,074 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:05:45,074 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:05:46,298 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:05:49,097 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:05:49,097 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:05:49,098 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:05:49,103 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:05:49,541 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:05:51,544 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:05:52,141 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:05:54,357 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:05:54,358 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:05:54,358 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:05:54,363 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:05:54,550 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:05:55,552 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:05:55,677 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 19:05:57,678 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:05:59,891 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:05:59,892 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:05:59,893 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:05:59,897 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:06:00,073 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:06:00,074 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:06:00,074 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:06:00,559 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:06:01,566 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:06:03,317 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:06:04,467 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:06:04,468 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:06:04,468 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:06:04,473 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:06:04,569 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:06:05,572 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:06:08,332 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:06:08,333 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:06:08,333 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:06:08,334 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:06:08,339 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:06:08,574 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:06:09,577 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:06:12,148 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:06:12,148 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:06:12,149 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:06:12,154 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:06:12,578 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:06:13,581 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:06:14,154 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:06:15,074 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:06:15,074 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:06:15,074 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:06:17,120 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:06:17,120 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:06:17,121 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:06:17,126 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:06:17,584 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:06:19,165 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:06:19,586 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:06:23,232 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:06:23,232 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:06:23,233 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:06:23,238 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:06:23,591 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:06:24,239 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:06:25,597 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:06:25,683 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 19:06:27,555 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:06:27,556 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:06:27,556 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:06:27,561 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:06:27,598 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:06:29,562 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:06:29,603 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:06:30,074 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:06:30,074 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:06:30,074 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:06:31,149 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:06:31,150 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:06:31,150 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:06:31,156 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:06:31,604 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:06:33,608 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:06:35,164 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:06:35,164 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:06:35,165 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:06:35,165 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:06:35,170 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:06:35,610 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:06:37,614 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:06:39,323 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:06:39,324 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:06:39,324 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:06:39,329 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:06:39,615 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:06:40,329 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:06:41,620 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:06:43,150 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:06:43,151 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:06:43,151 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:06:43,157 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:06:43,622 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:06:45,074 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:06:45,074 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:06:45,074 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:06:45,345 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:06:45,626 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:06:47,191 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:06:47,192 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:06:47,192 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:06:47,197 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:06:47,628 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:06:49,632 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:06:51,189 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:06:51,190 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:06:51,191 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:06:51,191 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:06:51,195 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:06:51,634 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:06:53,638 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:06:54,771 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:06:54,772 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:06:54,772 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:06:54,777 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:06:55,641 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:06:55,641 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:06:55,682 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 19:06:56,683 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:06:59,160 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:06:59,161 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:06:59,162 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:06:59,166 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:06:59,645 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:07:00,074 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:07:00,074 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:07:00,075 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:07:01,650 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:07:02,318 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:07:03,096 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:07:03,096 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:07:03,097 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:07:03,101 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:07:03,651 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:07:05,655 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:07:07,814 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:07:07,815 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:07:07,815 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:07:07,815 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:07:07,820 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:07:08,659 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:07:09,661 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:07:12,235 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:07:12,236 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:07:12,236 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:07:12,242 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:07:12,664 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:07:13,242 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:07:13,667 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:07:15,074 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:07:15,074 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:07:15,075 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:07:15,824 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:07:15,825 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:07:15,825 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:07:15,830 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:07:16,670 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:07:17,671 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:07:18,869 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:07:19,893 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:07:19,894 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:07:19,894 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:07:19,899 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:07:20,676 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:07:21,679 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:07:23,256 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:07:23,257 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:07:23,257 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:07:23,262 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:07:23,681 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:07:24,262 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:07:25,686 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:07:25,687 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 19:07:29,689 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:07:30,074 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:07:30,074 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:07:30,075 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:07:30,738 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:07:30,739 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:07:30,739 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:07:30,744 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:07:31,694 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:07:31,694 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:07:34,367 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:07:34,368 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:07:34,369 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:07:34,373 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:07:34,697 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:07:35,374 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:07:35,700 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:07:38,220 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:07:38,221 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:07:38,221 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:07:38,226 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:07:38,703 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:07:39,706 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:07:41,227 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:07:42,112 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:07:42,113 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:07:42,113 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:07:42,118 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:07:42,709 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:07:43,711 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:07:45,074 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:07:45,074 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:07:45,075 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:07:45,911 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:07:45,912 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:07:45,912 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:07:45,917 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:07:46,715 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:07:46,953 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:07:47,718 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:07:50,761 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:07:50,762 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:07:50,762 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:07:50,767 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:07:51,723 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:07:51,723 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:07:52,768 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:07:54,735 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:07:54,735 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:07:54,736 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:07:54,740 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:07:55,686 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 19:07:55,730 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:07:55,731 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:07:58,687 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:07:59,112 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:07:59,113 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:07:59,113 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:07:59,118 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:07:59,734 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:08:00,074 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:08:00,074 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:08:00,075 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:08:01,738 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:08:02,869 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:08:02,870 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:08:02,870 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:08:02,875 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:08:03,742 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:08:03,742 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:08:03,913 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:08:06,845 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:08:06,846 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:08:06,846 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:08:06,851 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:08:07,747 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:08:07,748 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:08:09,852 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:08:10,609 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:08:10,610 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:08:10,610 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:08:10,615 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:08:10,752 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:08:11,754 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:08:15,074 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:08:15,075 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:08:15,075 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:08:15,337 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:08:20,337 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:08:24,441 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:08:24,441 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:08:24,441 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:08:24,446 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:08:24,771 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:08:25,485 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:08:25,692 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 19:08:25,772 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:08:28,441 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:08:28,442 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:08:28,442 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:08:28,447 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:08:28,777 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:08:29,780 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:08:30,074 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:08:30,075 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:08:30,075 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:08:31,353 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:08:32,380 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:08:32,381 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:08:32,381 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:08:32,386 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:08:32,783 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:08:33,785 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:08:36,425 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:08:36,539 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:08:36,540 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:08:36,540 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:08:36,545 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:08:36,788 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:08:37,791 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:08:41,546 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:08:41,776 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:08:41,777 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:08:41,777 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:08:41,782 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:08:41,795 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:08:43,800 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:08:45,074 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:08:45,075 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:08:45,075 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:08:45,848 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:08:45,849 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:08:45,849 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:08:45,854 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:08:46,803 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:08:46,893 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:08:47,806 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:08:49,708 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:08:49,708 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:08:49,709 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:08:49,714 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:08:49,808 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:08:51,813 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:08:52,715 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:08:53,731 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:08:53,732 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:08:53,732 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:08:53,737 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:08:53,814 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:08:55,692 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 19:08:55,818 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:08:58,694 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:08:59,150 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:08:59,150 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:08:59,151 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:08:59,156 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:08:59,822 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:09:00,075 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:09:00,075 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:09:00,075 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:09:01,827 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:09:03,547 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:09:03,548 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:09:03,548 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:09:03,553 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:09:03,828 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:09:04,593 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:09:05,834 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:09:08,335 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:09:08,336 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:09:08,336 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:09:08,341 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:09:08,837 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:09:09,838 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:09:10,341 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:09:15,075 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:09:15,075 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:09:15,075 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:09:15,364 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:09:16,745 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:09:16,746 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:09:16,746 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:09:16,751 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:09:16,847 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:09:17,850 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:09:20,789 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:09:21,336 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:09:21,337 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:09:21,337 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:09:21,342 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:09:21,855 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:09:23,860 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:09:25,695 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 19:09:26,607 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:09:26,608 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:09:26,609 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:09:26,609 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:09:26,613 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:09:26,863 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:09:27,866 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:09:30,075 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:09:30,075 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:09:30,076 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:09:30,598 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:09:30,599 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:09:30,599 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:09:30,604 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:09:30,869 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:09:31,641 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:09:31,869 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:09:36,641 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:09:37,549 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:09:37,550 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:09:37,550 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:09:37,555 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:09:37,877 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:09:39,882 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:09:42,384 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:09:42,385 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:09:42,385 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:09:42,385 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:09:42,390 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:09:42,885 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:09:43,888 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:09:45,075 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:09:45,075 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:09:45,076 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:09:47,389 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:09:49,097 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:09:49,097 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:09:49,097 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:09:49,102 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:09:49,893 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:09:51,897 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:09:53,064 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:09:53,065 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:09:53,065 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:09:53,065 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:09:53,070 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:09:53,898 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:09:55,699 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 19:09:55,901 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:09:58,700 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:10:00,075 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:10:00,075 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:10:00,076 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:10:01,434 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:10:01,435 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:10:01,435 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:10:01,440 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:10:01,905 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:10:03,908 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:10:04,477 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:10:06,132 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:10:06,132 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:10:06,133 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:10:06,137 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:10:06,912 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:10:07,914 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:10:10,138 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:10:10,278 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:10:10,278 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:10:10,279 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:10:10,284 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:10:10,916 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:10:11,919 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:10:14,812 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:10:14,813 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:10:14,813 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:10:14,818 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:10:14,921 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:10:15,075 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:10:15,076 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:10:15,076 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:10:15,353 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:10:15,923 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:10:19,902 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:10:19,903 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:10:19,903 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:10:19,908 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:10:19,926 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:10:20,945 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:10:21,929 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:10:24,507 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:10:24,508 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:10:24,508 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:10:24,513 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:10:24,931 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:10:25,702 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 19:10:25,934 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:10:26,703 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:10:28,092 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:10:28,093 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:10:28,093 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:10:28,098 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:10:28,935 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:10:29,939 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:10:30,075 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:10:30,076 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:10:30,076 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:10:32,354 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:10:32,591 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:10:32,592 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:10:32,592 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:10:32,597 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:10:32,941 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:10:33,943 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:10:37,598 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:10:38,255 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:10:38,256 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:10:38,256 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:10:38,261 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:10:38,947 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:10:39,949 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:10:43,262 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:10:44,414 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:10:44,415 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:10:44,415 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:10:44,420 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:10:44,952 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:10:45,075 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:10:45,076 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:10:45,076 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:10:45,955 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:10:48,299 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:10:51,349 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:10:51,349 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:10:51,350 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:10:51,354 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:10:51,960 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:10:53,355 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:10:53,963 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:10:55,708 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 19:10:57,271 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:10:57,272 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:10:57,272 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:10:57,277 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:10:57,966 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:10:59,278 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:10:59,969 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:11:00,075 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:11:00,076 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:11:00,076 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:11:01,202 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:11:01,203 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:11:01,203 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:11:01,208 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:11:01,970 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:11:03,973 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:11:05,077 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:11:05,078 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:11:05,078 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:11:05,078 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:11:05,083 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:11:05,978 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:11:05,978 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:11:09,028 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:11:09,029 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:11:09,029 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:11:09,034 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:11:09,982 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:11:09,983 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:11:11,035 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:11:14,732 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:11:14,733 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:11:14,733 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:11:14,738 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:11:14,985 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:11:15,075 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:11:15,076 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:11:15,076 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:11:15,988 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:11:16,354 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:11:19,110 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:11:19,111 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:11:19,111 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:11:19,116 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:11:19,993 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:11:19,993 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:11:22,117 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:11:25,580 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:11:25,581 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:11:25,581 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:11:25,586 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:11:25,707 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 19:11:25,997 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:11:27,708 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:11:28,000 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:11:30,075 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:11:30,076 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:11:30,076 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:11:30,498 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:11:30,499 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:11:30,499 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:11:30,503 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:11:31,002 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:11:32,004 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:11:33,504 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:11:35,710 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:11:35,710 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:11:35,711 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:11:35,716 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:11:36,007 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:11:38,010 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:11:38,716 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:11:39,517 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:11:39,518 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:11:39,518 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:11:39,523 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:11:40,011 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:11:42,014 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:11:44,524 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:11:45,076 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:11:45,076 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:11:45,076 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:11:45,561 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:11:45,561 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:11:45,561 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:11:45,566 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:11:46,018 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:11:48,023 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:11:49,567 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:11:50,513 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:11:50,513 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:11:50,513 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:11:50,518 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:11:51,024 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:11:52,027 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:11:54,400 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:11:54,401 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:11:54,401 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:11:54,406 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:11:55,029 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:11:55,406 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:11:55,712 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 19:11:56,030 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:11:58,793 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:11:58,794 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:11:58,794 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:11:58,799 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:11:59,032 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:12:00,034 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:12:00,076 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:12:00,076 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:12:00,076 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:12:01,284 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:12:02,404 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:12:02,404 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:12:02,405 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:12:02,409 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:12:03,036 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:12:04,039 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:12:06,449 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:12:09,329 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:12:09,330 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:12:09,330 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:12:09,335 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:12:10,044 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:12:12,047 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:12:12,335 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:12:12,883 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:12:12,884 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:12:12,884 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:12:12,889 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:12:13,048 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:12:14,050 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:12:15,076 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:12:15,076 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:12:15,076 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:12:16,879 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:12:16,880 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:12:16,880 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:12:16,885 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:12:17,052 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:12:17,885 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:12:18,055 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:12:21,385 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:12:21,385 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:12:21,386 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:12:21,390 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:12:22,058 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:12:23,391 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:12:24,061 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:12:25,713 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 19:12:26,572 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:12:26,572 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:12:26,573 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:12:26,577 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:12:27,063 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:12:28,065 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:12:28,578 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:12:30,076 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:12:30,076 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:12:30,077 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:12:34,379 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:12:35,283 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:12:35,284 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:12:35,284 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:12:35,289 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:12:36,071 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:12:38,074 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:12:39,009 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:12:39,010 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:12:39,010 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:12:39,015 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:12:39,075 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:12:40,016 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:12:40,077 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:12:42,866 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:12:42,867 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:12:42,867 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:12:42,872 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:12:43,080 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:12:44,082 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:12:45,076 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:12:45,076 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:12:45,077 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:12:45,077 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:12:49,028 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:12:49,029 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:12:49,029 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:12:49,035 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:12:49,087 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:12:50,089 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:12:51,073 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:12:54,233 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:12:54,234 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:12:54,234 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:12:54,240 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:12:55,092 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:12:55,719 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 19:12:56,095 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:12:56,720 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:12:58,054 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:12:58,054 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:12:58,055 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:12:58,059 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:12:58,096 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:13:00,076 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:13:00,076 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:13:00,077 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:13:00,099 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:13:02,305 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:13:06,273 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:13:06,274 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:13:06,274 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:13:06,281 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:13:07,105 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:13:07,317 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:13:08,108 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:13:09,627 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:13:09,628 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:13:09,628 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:13:09,633 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:13:10,109 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:13:12,112 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:13:12,633 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:13:14,033 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:13:14,034 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:13:14,034 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:13:14,039 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:13:14,114 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:13:15,076 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:13:15,077 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:13:15,077 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:13:16,118 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:13:17,970 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:13:17,971 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:13:17,971 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:13:17,971 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:13:17,976 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:13:18,119 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:13:20,123 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:13:22,353 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:13:22,353 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:13:22,354 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:13:22,360 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:13:23,125 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:13:23,360 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:13:24,127 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:13:25,720 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 19:13:27,092 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:13:27,093 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:13:27,093 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:13:27,098 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:13:27,129 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:13:28,132 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:13:29,099 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:13:30,076 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:13:30,077 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:13:30,077 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:13:30,745 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:13:30,746 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:13:30,746 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:13:30,751 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:13:31,134 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:13:32,137 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:13:34,789 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:13:36,248 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:13:36,249 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:13:36,249 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:13:36,254 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:13:37,140 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:13:38,143 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:13:40,254 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:13:41,613 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:13:41,614 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:13:41,614 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:13:41,619 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:13:42,146 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:13:44,150 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:13:45,076 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:13:45,077 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:13:45,077 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:13:45,372 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:13:46,155 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:13:46,156 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:13:46,156 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:13:46,161 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:13:47,153 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:13:48,155 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:13:51,201 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:13:51,552 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:13:51,553 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:13:51,553 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:13:51,558 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:13:52,158 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:13:54,161 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:13:55,723 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 19:13:56,483 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:13:56,484 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:13:56,484 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:13:56,484 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:13:56,489 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:13:57,163 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:13:58,166 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:14:00,076 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:14:00,077 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:14:00,077 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:14:02,333 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:14:03,462 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:14:03,463 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:14:03,463 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:14:03,468 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:14:04,170 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:14:06,174 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:14:07,508 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:14:07,684 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:14:07,685 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:14:07,685 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:14:07,690 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:14:08,175 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:14:10,179 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:14:12,439 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:14:12,440 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:14:12,440 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:14:12,445 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:14:13,181 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:14:13,445 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:14:14,184 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:14:15,077 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:14:15,077 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:14:15,077 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:14:17,476 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:14:17,477 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:14:17,477 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:14:17,482 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:14:18,186 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:14:18,521 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:14:20,190 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:14:23,521 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:14:24,013 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:14:24,014 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:14:24,014 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:14:24,019 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:14:24,192 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:14:25,726 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 19:14:26,195 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:14:27,613 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:14:27,614 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:14:27,615 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:14:27,619 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:14:28,197 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:14:28,620 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:14:30,077 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:14:30,077 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:14:30,077 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:14:30,200 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:14:31,309 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:14:31,310 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:14:31,310 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:14:31,315 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:14:32,203 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:14:32,203 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:14:34,353 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:14:34,939 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:14:34,940 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:14:34,940 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:14:34,945 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:14:35,205 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:14:36,207 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:14:39,209 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:14:39,210 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:14:39,210 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:14:39,215 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:14:40,211 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:14:40,211 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:14:40,215 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:14:42,959 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:14:42,960 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:14:42,960 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:14:42,966 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:14:43,213 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:14:44,216 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:14:45,077 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:14:45,077 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:14:45,077 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:14:45,391 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:14:50,392 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:14:52,902 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:14:52,903 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:14:52,903 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:14:52,908 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:14:53,223 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:14:54,225 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:14:55,730 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 19:14:55,732 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:14:58,999 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:14:59,000 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:14:59,000 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:14:59,005 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:14:59,228 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:15:00,077 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:15:00,077 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:15:00,077 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:15:00,231 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:15:01,342 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:15:02,766 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:15:02,767 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:15:02,767 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:15:02,773 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:15:03,233 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:15:04,234 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:15:06,809 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:15:07,141 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:15:07,142 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:15:07,142 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:15:07,147 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:15:07,237 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:15:08,240 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:15:10,867 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:15:10,868 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:15:10,868 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:15:10,874 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:15:11,242 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:15:11,874 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:15:12,245 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:15:14,624 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:15:14,624 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:15:14,625 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:15:14,630 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:15:15,077 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:15:15,077 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:15:15,078 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:15:15,247 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:15:16,247 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:15:17,329 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:15:18,965 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:15:18,966 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:15:18,966 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:15:18,971 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:15:19,250 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:15:20,252 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:15:23,005 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:15:23,006 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:15:23,006 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:15:23,007 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:15:23,012 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:15:23,254 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:15:24,257 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:15:25,734 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 19:15:26,873 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:15:26,873 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:15:26,874 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:15:26,879 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:15:27,259 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:15:28,260 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:15:28,879 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:15:30,077 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:15:30,077 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:15:30,078 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:15:32,036 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:15:32,037 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:15:32,037 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:15:32,042 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:15:32,262 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:15:34,081 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:15:34,265 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:15:36,734 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:15:36,734 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:15:36,735 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:15:36,740 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:15:37,267 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:15:38,269 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:15:39,741 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:15:40,482 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:15:40,483 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:15:40,483 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:15:40,487 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:15:41,271 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:15:42,274 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:15:44,509 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:15:44,509 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:15:44,510 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:15:44,514 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:15:45,077 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:15:45,077 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:15:45,078 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:15:45,276 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:15:45,356 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:15:46,279 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:15:49,306 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:15:49,307 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:15:49,307 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:15:49,312 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:15:50,283 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:15:50,283 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:15:51,349 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:15:54,552 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:15:54,553 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:15:54,553 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:15:54,558 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:15:55,286 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:15:55,737 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 19:15:56,289 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:15:56,738 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:15:58,929 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:15:58,930 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:15:58,930 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:15:58,935 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:15:59,290 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:16:00,077 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:16:00,077 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:16:00,078 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:16:00,293 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:16:02,324 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:16:03,678 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:16:03,678 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:16:03,679 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:16:03,683 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:16:04,295 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:16:06,300 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:16:07,656 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:16:07,657 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:16:07,657 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:16:07,658 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:16:07,662 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:16:08,301 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:16:10,305 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:16:11,422 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:16:11,423 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:16:11,423 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:16:11,428 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:16:12,308 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:16:12,308 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:16:13,429 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:16:15,077 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:16:15,078 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:16:15,078 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:16:15,414 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:16:15,414 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:16:15,414 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:16:15,419 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:16:16,313 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:16:16,313 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:16:18,458 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:16:19,254 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:16:19,255 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:16:19,256 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:16:19,260 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:16:19,314 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:16:20,317 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:16:23,228 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:16:23,232 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:16:23,233 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:16:23,237 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:16:23,319 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:16:24,238 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:16:24,322 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:16:25,740 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 19:16:27,684 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:16:27,684 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:16:27,685 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:16:27,690 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:16:28,324 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:16:29,690 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:16:30,077 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:16:30,078 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:16:30,078 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:16:30,328 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:16:32,676 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:16:32,677 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:16:32,677 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:16:32,682 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:16:33,330 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:16:34,332 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:16:34,721 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:16:38,841 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:16:38,842 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:16:38,842 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:16:38,847 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:16:39,335 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:16:39,848 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:16:40,338 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:16:42,543 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:16:42,544 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:16:42,544 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:16:42,549 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:16:43,340 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:16:44,341 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:16:45,077 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:16:45,078 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:16:45,078 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:16:45,370 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:16:48,892 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:16:48,893 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:16:48,893 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:16:48,898 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:16:49,345 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:16:50,348 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:16:50,937 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:16:52,925 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:16:52,926 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:16:52,926 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:16:52,931 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:16:53,349 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:16:54,352 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:16:55,742 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 19:16:56,744 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:16:57,742 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:16:57,743 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:16:57,743 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:16:57,748 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:16:58,355 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:17:00,078 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:17:00,078 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:17:00,078 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:17:00,358 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:17:01,565 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:17:01,565 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:17:01,566 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:17:01,570 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:17:02,359 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:17:02,609 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:17:04,362 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:17:05,554 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:17:05,555 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:17:05,555 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:17:05,560 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:17:06,365 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:17:08,368 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:17:08,561 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:17:09,738 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:17:09,738 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:17:09,739 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:17:09,743 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:17:10,369 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:17:12,372 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:17:13,744 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:17:14,279 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:17:14,279 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:17:14,280 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:17:14,284 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:17:14,374 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:17:15,078 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:17:15,078 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:17:15,078 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:17:16,377 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:17:18,284 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:17:18,285 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:17:18,285 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:17:18,290 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:17:18,378 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:17:19,329 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:17:20,382 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:17:22,613 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:17:22,614 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:17:22,614 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:17:22,619 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:17:23,384 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:17:24,386 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:17:24,620 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:17:25,749 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 19:17:29,750 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:17:30,078 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:17:30,078 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:17:30,078 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:17:30,331 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:17:30,358 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:17:30,359 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:17:30,363 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:17:30,390 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:17:32,393 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:17:35,364 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:17:36,958 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:17:36,959 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:17:36,959 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:17:36,964 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:17:37,396 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:17:38,399 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:17:41,001 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:17:41,213 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:17:41,214 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:17:41,214 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:17:41,219 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:17:41,401 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:17:42,403 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:17:45,078 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:17:45,078 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:17:45,079 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:17:45,335 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:17:45,338 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:17:45,338 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:17:45,343 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:17:45,406 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:17:46,343 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:17:46,406 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:17:51,344 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:17:51,620 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:17:51,620 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:17:51,620 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:17:51,625 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:17:52,411 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:17:54,415 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:17:55,749 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 19:17:56,750 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:17:58,510 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:17:58,511 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:17:58,511 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:17:58,516 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:17:59,418 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:18:00,078 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:18:00,078 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:18:00,079 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:18:00,420 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:18:02,314 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:18:03,581 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:18:03,582 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:18:03,582 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:18:03,587 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:18:04,423 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:18:06,428 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:18:07,625 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:18:07,906 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:18:07,906 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:18:07,907 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:18:07,912 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:18:08,429 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:18:10,432 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:18:12,754 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:18:12,754 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:18:12,755 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:18:12,755 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:18:12,760 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:18:13,434 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:18:14,437 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:18:15,078 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:18:15,078 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:18:15,079 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:18:16,044 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:18:16,044 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:18:16,044 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:18:16,049 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:18:16,438 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:18:18,089 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:18:18,441 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:18:20,005 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:18:20,006 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:18:20,006 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:18:20,011 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:18:20,443 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:18:22,446 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:18:24,011 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:18:25,659 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:18:25,660 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:18:25,660 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:18:25,666 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:18:25,754 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 19:18:26,449 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:18:28,452 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:18:29,742 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:18:29,743 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:18:29,744 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:18:29,744 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:18:29,748 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:18:30,078 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:18:30,079 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:18:30,079 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:18:30,453 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:18:32,459 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:18:33,617 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:18:33,618 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:18:33,618 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:18:33,623 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:18:34,460 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:18:35,662 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:18:36,464 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:18:39,013 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:18:39,014 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:18:39,014 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:18:39,019 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:18:39,467 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:18:40,469 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:18:41,019 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:18:45,078 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:18:45,079 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:18:45,079 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:18:46,306 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:18:47,552 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:18:47,553 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:18:47,553 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:18:47,558 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:18:48,477 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:18:48,478 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:18:51,559 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:18:54,030 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:18:54,031 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:18:54,031 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:18:54,036 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:18:54,481 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:18:55,753 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 19:18:56,484 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:18:56,754 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:19:00,078 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:19:00,079 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:19:00,079 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:19:02,323 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:19:02,520 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:19:02,521 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:19:02,521 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:19:02,526 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:19:03,488 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:19:04,491 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:19:06,273 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:19:06,274 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:19:06,275 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:19:06,279 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:19:06,493 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:19:08,280 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:19:08,496 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:19:11,148 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:19:11,149 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:19:11,149 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:19:11,154 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:19:11,499 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:19:12,501 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:19:14,155 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:19:15,078 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:19:15,079 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:19:15,079 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:19:15,517 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:19:15,518 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:19:15,518 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:19:15,523 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:19:16,505 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:19:16,506 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:19:19,561 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:19:19,874 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:19:19,875 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:19:19,875 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:19:19,880 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:19:20,508 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:19:22,512 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:19:24,405 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:19:24,406 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:19:24,406 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:19:24,411 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:19:24,513 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:19:25,412 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:19:25,756 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 19:19:26,516 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:19:27,858 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:19:27,858 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:19:27,859 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:19:27,864 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:19:28,517 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:19:30,078 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:19:30,079 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:19:30,079 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:19:30,521 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:19:31,316 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:19:33,575 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:19:33,576 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:19:33,576 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:19:33,580 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:19:34,525 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:19:34,525 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:19:36,581 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:19:39,941 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:19:39,942 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:19:39,942 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:19:39,947 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:19:40,529 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:19:41,948 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:19:42,532 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:19:43,748 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:19:43,749 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:19:43,749 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:19:43,754 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:19:44,534 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:19:45,079 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:19:45,079 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:19:45,079 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:19:46,537 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:19:47,380 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:19:52,180 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:19:52,180 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:19:52,181 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:19:52,185 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:19:52,542 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:19:53,225 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:19:54,545 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:19:55,759 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 19:19:57,674 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:19:57,675 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:19:57,675 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:19:57,680 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:19:58,547 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:19:58,547 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:19:58,681 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:20:00,079 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:20:00,079 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:20:00,079 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:20:01,278 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:20:01,279 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:20:01,279 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:20:01,284 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:20:01,549 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:20:02,552 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:20:04,321 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:20:06,696 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:20:06,696 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:20:06,697 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:20:06,702 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:20:07,556 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:20:08,559 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:20:09,703 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:20:10,659 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:20:10,660 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:20:10,660 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:20:10,665 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:20:11,561 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:20:12,563 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:20:15,079 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:20:15,079 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:20:15,080 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:20:15,080 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:20:19,245 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:20:19,246 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:20:19,246 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:20:19,251 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:20:19,568 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:20:20,289 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:20:20,569 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:20:25,289 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:20:25,547 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:20:25,548 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:20:25,548 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:20:25,553 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:20:25,572 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:20:25,762 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 19:20:26,574 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:20:29,014 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:20:29,015 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:20:29,015 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:20:29,020 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:20:29,576 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:20:30,079 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:20:30,079 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:20:30,080 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:20:30,364 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:20:30,579 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:20:35,365 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:20:35,400 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:20:35,401 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:20:35,401 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:20:35,406 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:20:35,582 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:20:36,585 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:20:39,949 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:20:39,950 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:20:39,950 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:20:39,955 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:20:40,587 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:20:40,956 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:20:42,590 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:20:44,226 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:20:44,227 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:20:44,227 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:20:44,232 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:20:44,592 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:20:45,079 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:20:45,079 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:20:45,080 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:20:46,311 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:20:46,595 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:20:51,312 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:20:52,784 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:20:52,785 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:20:52,785 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:20:52,791 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:20:53,601 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:20:54,603 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:20:55,765 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 19:20:56,766 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:20:57,172 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:20:57,173 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:20:57,173 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:20:57,179 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:20:57,605 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:20:58,608 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:21:00,079 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:21:00,079 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:21:00,080 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:21:00,933 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:21:00,934 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:21:00,934 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:21:00,939 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:21:01,610 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:21:01,977 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:21:02,612 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:21:05,240 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:21:05,241 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:21:05,241 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:21:05,246 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:21:05,615 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:21:06,618 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:21:07,247 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:21:08,834 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:21:08,835 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:21:08,835 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:21:08,840 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:21:09,620 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:21:10,622 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:21:12,841 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:21:12,879 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:21:12,880 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:21:12,880 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:21:12,885 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:21:13,624 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:21:14,627 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:21:15,079 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:21:15,080 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:21:15,080 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:21:16,632 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:21:16,632 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:21:16,633 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:21:16,637 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:21:17,629 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:21:18,631 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:21:18,677 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:21:20,793 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:21:20,794 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:21:20,794 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:21:20,799 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:21:21,634 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:21:22,636 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:21:23,800 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:21:25,046 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:21:25,047 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:21:25,047 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:21:25,052 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:21:25,638 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:21:25,767 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 19:21:26,641 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:21:29,351 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:21:29,352 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:21:29,352 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:21:29,352 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:21:29,357 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:21:29,642 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:21:30,079 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:21:30,080 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:21:30,080 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:21:30,645 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:21:33,165 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:21:33,166 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:21:33,166 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:21:33,171 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:21:33,647 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:21:34,648 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:21:35,209 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:21:39,449 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:21:39,450 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:21:39,450 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:21:39,455 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:21:39,651 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:21:40,455 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:21:40,653 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:21:44,485 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:21:44,486 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:21:44,486 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:21:44,491 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:21:44,656 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:21:45,079 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:21:45,080 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:21:45,080 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:21:46,382 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:21:46,659 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:21:49,409 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:21:49,410 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:21:49,410 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:21:49,415 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:21:49,662 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:21:50,665 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:21:51,453 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:21:53,522 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:21:53,523 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:21:53,523 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:21:53,528 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:21:53,667 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:21:54,669 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:21:55,773 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 19:21:56,774 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:21:57,810 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:21:57,810 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:21:57,811 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:21:57,815 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:21:58,674 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:21:58,674 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:22:00,079 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:22:00,080 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:22:00,080 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:22:01,812 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:22:01,813 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:22:01,813 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:22:01,814 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:22:01,818 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:22:02,678 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:22:02,679 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:22:05,509 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:22:05,510 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:22:05,510 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:22:05,517 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:22:05,682 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:22:06,684 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:22:07,518 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:22:10,192 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:22:10,193 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:22:10,193 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:22:10,198 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:22:10,687 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:22:12,688 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:22:13,199 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:22:14,305 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:22:14,305 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:22:14,306 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:22:14,311 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:22:14,689 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:22:15,079 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:22:15,080 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:22:15,080 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:22:16,692 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:22:18,357 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:22:18,358 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:22:18,358 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:22:18,358 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:22:18,363 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:22:18,694 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:22:20,697 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:22:23,401 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:22:24,287 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:22:24,288 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:22:24,288 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:22:24,293 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:22:24,700 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:22:25,772 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 19:22:26,703 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:22:28,773 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:22:29,276 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:22:29,277 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:22:29,277 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:22:29,281 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:22:29,705 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:22:30,080 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:22:30,080 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:22:30,080 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:22:30,707 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:22:32,911 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:22:32,912 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:22:32,912 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:22:32,917 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:22:33,709 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:22:33,957 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:22:34,712 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:22:37,898 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:22:37,898 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:22:37,899 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:22:37,904 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:22:38,717 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:22:38,717 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:22:39,905 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:22:43,066 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:22:43,066 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:22:43,067 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:22:43,071 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:22:43,720 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:22:44,723 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:22:45,072 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:22:45,080 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:22:45,080 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:22:45,080 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:22:48,398 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:22:48,399 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:22:48,399 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:22:48,404 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:22:48,726 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:22:50,404 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:22:50,729 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:22:52,484 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:22:52,484 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:22:52,485 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:22:52,489 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:22:52,731 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:22:54,734 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:22:55,490 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:22:55,778 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 19:22:56,570 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:22:56,570 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:22:56,571 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:22:56,576 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:22:56,735 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:22:58,739 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:23:00,080 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:23:00,080 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:23:00,081 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:23:00,500 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:23:00,501 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:23:00,501 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:23:00,501 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:23:00,506 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:23:00,740 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:23:02,743 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:23:05,545 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:23:06,352 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:23:06,353 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:23:06,353 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:23:06,357 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:23:06,747 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:23:08,751 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:23:10,032 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:23:10,032 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:23:10,033 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:23:10,037 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:23:10,752 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:23:11,038 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:23:12,756 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:23:13,980 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:23:13,980 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:23:13,981 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:23:13,985 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:23:14,757 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:23:15,080 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:23:15,080 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:23:15,081 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:23:16,385 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:23:16,761 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:23:19,511 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:23:19,511 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:23:19,512 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:23:19,516 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:23:19,763 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:23:20,766 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:23:21,553 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:23:23,310 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:23:23,311 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:23:23,311 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:23:23,316 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:23:23,767 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:23:24,770 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:23:25,778 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 19:23:26,780 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:23:28,199 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:23:28,200 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:23:28,200 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:23:28,205 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:23:28,772 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:23:30,080 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:23:30,080 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:23:30,081 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:23:30,776 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:23:32,344 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:23:32,962 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:23:32,963 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:23:32,963 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:23:32,969 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:23:33,778 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:23:34,780 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:23:36,803 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:23:36,804 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:23:36,804 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:23:36,809 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:23:37,782 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:23:37,809 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:23:38,791 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:23:41,775 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:23:41,776 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:23:41,776 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:23:41,781 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:23:41,793 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:23:42,796 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:23:43,782 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:23:45,080 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:23:45,080 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:23:45,081 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:23:47,702 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:23:47,703 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:23:47,703 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:23:47,708 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:23:47,801 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:23:48,803 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:23:49,745 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:23:51,352 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:23:51,353 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:23:51,353 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:23:51,358 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:23:51,805 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:23:52,808 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:23:55,358 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:23:55,784 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 19:23:59,792 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:23:59,792 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:23:59,793 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:23:59,798 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:23:59,812 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:24:00,080 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:24:00,080 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:24:00,081 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:24:00,813 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:24:01,331 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:24:06,331 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:24:06,726 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:24:06,727 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:24:06,727 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:24:06,732 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:24:06,818 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:24:08,822 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:24:10,360 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:24:10,361 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:24:10,361 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:24:10,366 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:24:10,823 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:24:11,366 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:24:12,826 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:24:15,080 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:24:15,081 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:24:15,081 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:24:16,379 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:24:18,781 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:24:18,782 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:24:18,782 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:24:18,787 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:24:18,830 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:24:20,833 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:24:21,825 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:24:23,548 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:24:23,548 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:24:23,549 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:24:23,554 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:24:23,835 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:24:24,839 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:24:25,785 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 19:24:27,133 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:24:27,134 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:24:27,134 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:24:27,134 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:24:27,139 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:24:27,840 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:24:28,843 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:24:30,080 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:24:30,081 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:24:30,081 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:24:31,703 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:24:31,703 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:24:31,704 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:24:31,708 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:24:31,845 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:24:32,745 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:24:32,848 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:24:36,310 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:24:36,311 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:24:36,311 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:24:36,316 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:24:36,850 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:24:38,316 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:24:38,853 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:24:41,126 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:24:41,126 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:24:41,126 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:24:41,131 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:24:41,855 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:24:42,858 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:24:44,132 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:24:45,080 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:24:45,081 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:24:45,081 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:24:46,940 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:24:46,940 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:24:46,941 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:24:46,946 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:24:47,863 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:24:48,866 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:24:49,946 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:24:51,093 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:24:51,094 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:24:51,094 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:24:51,099 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:24:51,868 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:24:52,870 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:24:55,099 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:24:55,788 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 19:24:58,266 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:24:58,267 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:24:58,267 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:24:58,272 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:24:58,875 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:25:00,081 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:25:00,081 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:25:00,081 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:25:00,336 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:25:00,878 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:25:01,894 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:25:01,895 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:25:01,895 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:25:01,900 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:25:02,881 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:25:02,881 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:25:05,546 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:25:05,546 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:25:05,547 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:25:05,547 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:25:05,551 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:25:05,884 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:25:06,887 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:25:10,552 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:25:13,996 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:25:13,997 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:25:13,997 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:25:14,003 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:25:14,894 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:25:14,894 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:25:15,081 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:25:15,081 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:25:15,081 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:25:16,294 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:25:19,723 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:25:19,724 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:25:19,724 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:25:19,729 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:25:19,897 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:25:20,900 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:25:21,730 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:25:23,553 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:25:23,554 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:25:23,554 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:25:23,559 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:25:23,901 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:25:24,902 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:25:25,791 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 19:25:26,793 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:25:27,731 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:25:27,732 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:25:27,732 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:25:27,737 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:25:27,904 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:25:28,907 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:25:30,081 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:25:30,081 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:25:30,081 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:25:32,135 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:25:32,135 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:25:32,136 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:25:32,136 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:25:32,141 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:25:32,909 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:25:34,913 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:25:36,317 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:25:36,318 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:25:36,318 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:25:36,323 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:25:36,914 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:25:37,324 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:25:38,918 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:25:41,684 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:25:41,684 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:25:41,685 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:25:41,689 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:25:41,919 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:25:42,690 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:25:42,922 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:25:45,081 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:25:45,081 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:25:45,081 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:25:46,778 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:25:46,779 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:25:46,779 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:25:46,784 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:25:46,925 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:25:47,784 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:25:48,930 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:25:50,415 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:25:50,415 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:25:50,415 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:25:50,421 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:25:50,931 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:25:52,934 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:25:53,421 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:25:54,245 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:25:54,245 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:25:54,246 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:25:54,251 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:25:54,935 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:25:55,794 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 19:25:56,939 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:25:58,796 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:25:58,879 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:25:58,879 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:25:58,880 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:25:58,885 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:25:58,940 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:26:00,081 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:26:00,081 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:26:00,082 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:26:00,943 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:26:02,417 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:26:04,784 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:26:04,784 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:26:04,784 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:26:04,790 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:26:04,946 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:26:06,616 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:26:06,617 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:26:06,617 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:26:06,622 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:26:06,950 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:26:06,950 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:26:08,953 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:26:10,558 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:26:10,559 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:26:10,559 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:26:10,559 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:26:10,564 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:26:10,955 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:26:12,958 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:26:14,640 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:26:14,640 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:26:14,640 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:26:14,645 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:26:14,959 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:26:15,081 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:26:15,081 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:26:15,082 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:26:16,291 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:26:16,963 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:26:18,476 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:26:18,476 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:26:18,477 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:26:18,481 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:26:18,964 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:26:20,967 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:26:21,482 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:26:25,079 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:26:25,080 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:26:25,080 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:26:25,085 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:26:25,796 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 19:26:25,971 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:26:26,798 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:26:26,971 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:26:29,864 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:26:29,865 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:26:29,865 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:26:29,871 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:26:29,973 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:26:30,081 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:26:30,081 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:26:30,082 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:26:30,976 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:26:34,491 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:26:34,832 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:26:34,832 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:26:34,832 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:26:34,837 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:26:34,978 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:26:36,982 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:26:38,500 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:26:38,501 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:26:38,501 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:26:38,506 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:26:38,983 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:26:40,506 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:26:40,986 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:26:44,182 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:26:44,183 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:26:44,183 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:26:44,188 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:26:44,991 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:26:44,991 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:26:45,081 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:26:45,081 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:26:45,082 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:26:46,317 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:26:48,547 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:26:48,548 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:26:48,548 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:26:48,554 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:26:48,995 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:26:50,998 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:26:51,554 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:26:52,294 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:26:52,294 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:26:52,294 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:26:52,299 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:26:52,999 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:26:55,003 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:26:55,799 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 19:26:56,325 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:26:56,326 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:26:56,326 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:26:56,331 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:26:57,004 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:26:57,332 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:26:59,008 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:27:00,081 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:27:00,082 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:27:00,082 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:27:00,465 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:27:00,466 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:27:00,466 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:27:00,470 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:27:01,009 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:27:02,509 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:27:03,012 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:27:04,041 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:27:04,042 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:27:04,042 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:27:04,047 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:27:05,015 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:27:05,016 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:27:07,962 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:27:07,963 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:27:07,963 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:27:07,963 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:27:07,968 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:27:08,019 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:27:09,021 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:27:11,694 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:27:11,695 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:27:11,695 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:27:11,700 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:27:12,023 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:27:13,026 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:27:13,701 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:27:15,081 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:27:15,082 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:27:15,082 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:27:15,087 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:27:15,381 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:27:15,381 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:27:15,386 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:27:16,028 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:27:17,030 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:27:19,387 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:27:23,531 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:27:23,532 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:27:23,532 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:27:23,538 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:27:24,036 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:27:24,539 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:27:25,036 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:27:25,805 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 19:27:27,847 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:27:27,848 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:27:27,848 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:27:27,854 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:27:28,038 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:27:29,041 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:27:29,854 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:27:30,081 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:27:30,082 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:27:30,082 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:27:32,479 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:27:32,479 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:27:32,480 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:27:32,484 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:27:33,044 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:27:35,047 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:27:35,485 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:27:36,124 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:27:36,124 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:27:36,125 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:27:36,129 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:27:37,051 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:27:37,051 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:27:39,870 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:27:39,870 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:27:39,870 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:27:39,875 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:27:40,053 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:27:40,876 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:27:41,056 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:27:44,064 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:27:44,064 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:27:44,065 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:27:44,070 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:27:45,060 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:27:45,061 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:27:45,081 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:27:45,082 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:27:45,082 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:27:46,387 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:27:50,415 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:27:50,416 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:27:50,416 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:27:50,420 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:27:51,065 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:27:51,421 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:27:53,068 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:27:54,234 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:27:54,235 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:27:54,235 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:27:54,240 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:27:55,072 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:27:55,072 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:27:55,804 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 19:27:56,805 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:27:59,415 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:27:59,416 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:27:59,416 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:27:59,421 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:28:00,075 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:28:00,082 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:28:00,082 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:28:00,082 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:28:01,078 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:28:02,366 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:28:03,367 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:28:03,368 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:28:03,368 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:28:03,373 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:28:04,080 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:28:05,080 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:28:07,062 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:28:07,063 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:28:07,063 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:28:07,068 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:28:07,083 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:28:08,068 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:28:09,086 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:28:12,581 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:28:12,582 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:28:12,582 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:28:12,587 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:28:13,089 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:28:13,587 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:28:15,082 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:28:15,082 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:28:15,082 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:28:15,092 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:28:16,721 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:28:16,722 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:28:16,722 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:28:16,727 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:28:17,094 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:28:18,765 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:28:19,097 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:28:20,716 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:28:20,717 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:28:20,717 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:28:20,722 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:28:21,099 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:28:23,102 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:28:24,722 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:28:25,810 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 19:28:26,781 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:28:26,781 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:28:26,781 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:28:26,786 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:28:27,105 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:28:29,108 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:28:29,787 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:28:30,082 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:28:30,082 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:28:30,082 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:28:31,290 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:28:31,291 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:28:31,291 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:28:31,296 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:28:32,110 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:28:33,113 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:28:35,297 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:28:35,764 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:28:35,764 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:28:35,765 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:28:35,770 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:28:36,115 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:28:37,118 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:28:40,021 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:28:40,021 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:28:40,021 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:28:40,026 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:28:40,120 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:28:41,027 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:28:41,120 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:28:45,082 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:28:45,082 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:28:45,083 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:28:46,010 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:28:46,010 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:28:46,010 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:28:46,015 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:28:46,053 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:28:46,124 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:28:47,126 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:28:50,397 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:28:50,398 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:28:50,398 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:28:50,403 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:28:51,130 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:28:51,404 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:28:53,136 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:28:54,123 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:28:54,124 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:28:54,124 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:28:54,129 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:28:54,137 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:28:55,139 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:28:55,811 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 19:28:56,812 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:28:58,026 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:28:58,027 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:28:58,027 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:28:58,032 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:28:58,141 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:28:59,144 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:29:00,082 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:29:00,082 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:29:00,083 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:29:01,795 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:29:01,796 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:29:01,796 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:29:01,801 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:29:01,841 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:29:02,146 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:29:03,149 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:29:05,512 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:29:05,512 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:29:05,513 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:29:05,517 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:29:06,152 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:29:07,155 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:29:07,518 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:29:11,622 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:29:11,622 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:29:11,623 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:29:11,628 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:29:12,158 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:29:12,628 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:29:13,158 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:29:15,082 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:29:15,083 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:29:15,083 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:29:16,098 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:29:16,098 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:29:16,098 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:29:16,103 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:29:16,160 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:29:17,163 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:29:18,141 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:29:20,510 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:29:20,510 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:29:20,510 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:29:20,515 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:29:21,166 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:29:23,169 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:29:23,516 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:29:25,816 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 19:29:28,818 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:29:29,029 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:29:29,030 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:29:29,030 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:29:29,035 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:29:29,173 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:29:30,082 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:29:30,082 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:29:30,083 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:29:31,174 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:29:32,328 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:29:32,329 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:29:32,329 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:29:32,334 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:29:33,177 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:29:33,178 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:29:34,373 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:29:36,372 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:29:36,373 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:29:36,373 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:29:36,378 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:29:37,182 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:29:37,182 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:29:39,378 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:29:40,140 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:29:40,141 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:29:40,141 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:29:40,146 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:29:40,184 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:29:41,187 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:29:44,160 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:29:44,160 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:29:44,161 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:29:44,166 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:29:44,188 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:29:45,082 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:29:45,083 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:29:45,083 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:29:45,083 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:29:45,191 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:29:48,993 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:29:48,993 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:29:48,993 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:29:48,998 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:29:49,195 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:29:51,037 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:29:51,198 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:29:54,081 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:29:54,082 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:29:54,082 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:29:54,087 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:29:54,200 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:29:55,203 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:29:55,817 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 19:29:56,819 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:29:57,781 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:29:57,782 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:29:57,782 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:29:57,787 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:29:58,205 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:29:59,208 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:30:00,082 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:30:00,083 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:30:00,083 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:30:02,324 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:30:06,473 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:30:06,474 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:30:06,474 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:30:06,479 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:30:07,214 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:30:07,517 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:30:09,217 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:30:12,292 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:30:12,293 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:30:12,293 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:30:12,298 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:30:13,224 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:30:13,224 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:30:13,298 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:30:15,082 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:30:15,083 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:30:15,083 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:30:16,222 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:30:16,222 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:30:16,223 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:30:16,227 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:30:16,228 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:30:17,230 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:30:19,265 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:30:20,385 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:30:20,385 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:30:20,386 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:30:20,390 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:30:21,235 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:30:21,235 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:30:24,391 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:30:25,743 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:30:25,744 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:30:25,744 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:30:25,749 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:30:25,822 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 19:30:26,238 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:30:27,241 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:30:29,823 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:30:29,946 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:30:29,946 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:30:29,947 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:30:29,952 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:30:30,083 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:30:30,083 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:30:30,084 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:30:30,243 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:30:31,245 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:30:33,217 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:30:33,218 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:30:33,218 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:30:33,223 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:30:33,247 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:30:35,248 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:30:35,261 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:30:37,393 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:30:37,394 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:30:37,394 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:30:37,399 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:30:38,250 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:30:39,252 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:30:40,400 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:30:41,613 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:30:41,614 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:30:41,614 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:30:41,620 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:30:42,254 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:30:43,257 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:30:45,083 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:30:45,083 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:30:45,083 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:30:45,707 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:30:48,093 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:30:48,094 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:30:48,094 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:30:48,099 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:30:48,261 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:30:49,264 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:30:51,137 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:30:51,985 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:30:51,986 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:30:51,986 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:30:51,991 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:30:52,266 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:30:53,269 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:30:55,724 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:30:55,725 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:30:55,725 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:30:55,730 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:30:55,821 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 19:30:56,271 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:30:56,822 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:30:57,274 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:30:59,741 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:30:59,741 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:30:59,742 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:30:59,747 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:31:00,083 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:31:00,083 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:31:00,084 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:31:00,276 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:31:01,276 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:31:02,377 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:31:04,498 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:31:04,499 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:31:04,499 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:31:04,504 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:31:05,279 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:31:07,283 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:31:07,541 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:31:08,525 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:31:08,526 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:31:08,526 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:31:08,531 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:31:09,285 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:31:11,288 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:31:12,694 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:31:12,695 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:31:12,695 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:31:12,695 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:31:12,700 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:31:13,290 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:31:15,083 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:31:15,083 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:31:15,084 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:31:15,293 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:31:18,383 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:31:21,222 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:31:21,223 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:31:21,223 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:31:21,228 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:31:21,297 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:31:23,301 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:31:24,265 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:31:24,650 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:31:24,651 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:31:24,651 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:31:24,657 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:31:25,302 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:31:25,827 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 19:31:27,306 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:31:28,879 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:31:28,880 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:31:28,880 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:31:28,885 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:31:29,307 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:31:29,886 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:31:30,083 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:31:30,083 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:31:30,084 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:31:31,310 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:31:35,349 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:31:35,753 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:31:35,754 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:31:35,754 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:31:35,759 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:31:36,313 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:31:37,316 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:31:39,291 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:31:39,292 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:31:39,292 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:31:39,297 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:31:39,317 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:31:41,298 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:31:41,321 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:31:43,440 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:31:43,440 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:31:43,440 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:31:43,445 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:31:44,323 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:31:45,083 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:31:45,084 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:31:45,084 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:31:45,326 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:31:46,340 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:31:50,319 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:31:50,320 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:31:50,320 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:31:50,325 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:31:50,330 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:31:51,333 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:31:52,326 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:31:55,290 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:31:55,291 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:31:55,291 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:31:55,296 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:31:55,336 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:31:55,826 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 19:31:57,339 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:31:57,828 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:31:59,255 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:31:59,256 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:31:59,256 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:31:59,261 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:31:59,340 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:32:00,083 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:32:00,084 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:32:00,084 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:32:01,344 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:32:03,324 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:32:03,325 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:32:03,325 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:32:03,325 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:32:03,330 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:32:03,345 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:32:05,348 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:32:08,011 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:32:08,011 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:32:08,012 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:32:08,017 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:32:08,352 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:32:09,018 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:32:09,354 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:32:11,317 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:32:11,317 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:32:11,318 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:32:11,322 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:32:11,356 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:32:13,359 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:32:14,323 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:32:15,079 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:32:15,079 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:32:15,080 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:32:15,084 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:32:15,084 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:32:15,084 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:32:15,085 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:32:15,360 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:32:17,364 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:32:19,377 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:32:19,868 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:32:19,869 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:32:19,869 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:32:19,874 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:32:20,366 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:32:21,369 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:32:23,500 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:32:23,500 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:32:23,501 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:32:23,505 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:32:24,371 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:32:24,506 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:32:25,371 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:32:25,829 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 19:32:28,154 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:32:28,155 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:32:28,155 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:32:28,160 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:32:28,373 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:32:29,376 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:32:30,083 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:32:30,084 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:32:30,084 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:32:30,084 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:32:35,074 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:32:35,074 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:32:35,075 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:32:35,079 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:32:35,117 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:32:35,380 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:32:37,383 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:32:40,117 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:32:41,494 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:32:41,495 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:32:41,495 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:32:41,500 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:32:42,386 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:32:43,389 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:32:45,083 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:32:45,084 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:32:45,084 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:32:45,378 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:32:45,407 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:32:45,407 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:32:45,407 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:32:45,412 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:32:46,391 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:32:47,394 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:32:49,114 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:32:49,115 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:32:49,115 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:32:49,120 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:32:49,397 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:32:51,120 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:32:51,400 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:32:54,710 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:32:54,711 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:32:54,711 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:32:54,716 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:32:55,402 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:32:55,832 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 19:32:56,834 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:32:57,406 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:32:58,667 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:32:58,668 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:32:58,668 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:32:58,674 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:32:59,407 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:33:00,083 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:33:00,084 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:33:00,084 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:33:01,410 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:33:02,360 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:33:06,967 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:33:06,968 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:33:06,968 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:33:06,973 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:33:07,415 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:33:07,973 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:33:09,419 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:33:12,054 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:33:12,055 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:33:12,055 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:33:12,060 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:33:12,421 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:33:13,061 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:33:13,423 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:33:15,084 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:33:15,084 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:33:15,084 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:33:16,868 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:33:16,868 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:33:16,869 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:33:16,874 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:33:17,426 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:33:18,874 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:33:19,429 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:33:23,757 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:33:23,757 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:33:23,758 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:33:23,763 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:33:24,432 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:33:24,763 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:33:25,435 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:33:25,835 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 19:33:29,836 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:33:30,084 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:33:30,084 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:33:30,084 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:33:30,498 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:33:30,498 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:33:30,498 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:33:30,504 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:33:31,441 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:33:31,441 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:33:34,900 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:33:34,901 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:33:34,901 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:33:34,902 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:33:34,906 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:33:35,444 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:33:37,448 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:33:39,907 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:33:40,460 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:33:40,461 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:33:40,461 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:33:40,465 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:33:41,452 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:33:41,453 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:33:45,084 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:33:45,084 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:33:45,085 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:33:45,085 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:33:45,674 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:33:45,674 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:33:45,674 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:33:45,679 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:33:46,456 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:33:47,459 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:33:49,609 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:33:49,609 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:33:49,610 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:33:49,615 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:33:50,462 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:33:50,615 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:33:51,464 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:33:54,204 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:33:54,204 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:33:54,205 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:33:54,210 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:33:54,466 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:33:55,469 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:33:55,841 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 19:33:55,843 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:33:59,006 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:33:59,007 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:33:59,007 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:33:59,012 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:33:59,472 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:34:00,084 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:34:00,084 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:34:00,085 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:34:01,346 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:34:01,475 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:34:03,465 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:34:03,466 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:34:03,466 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:34:03,471 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:34:03,476 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:34:05,479 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:34:06,509 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:34:08,542 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:34:08,543 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:34:08,543 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:34:08,548 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:34:09,486 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:34:09,486 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:34:11,548 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:34:15,084 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:34:15,084 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:34:15,085 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:34:17,286 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:34:17,592 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:34:17,593 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:34:17,593 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:34:17,598 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:34:18,491 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:34:19,494 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:34:21,817 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:34:21,818 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:34:21,818 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:34:21,823 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:34:22,496 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:34:22,823 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:34:23,499 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:34:25,842 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 19:34:26,702 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:34:26,702 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:34:26,703 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:34:26,708 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:34:27,503 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:34:27,503 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:34:28,709 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:34:30,084 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:34:30,084 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:34:30,085 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:34:31,938 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:34:31,939 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:34:31,939 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:34:31,944 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:34:32,506 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:34:33,509 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:34:33,945 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:34:38,945 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:34:39,229 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:34:39,230 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:34:39,230 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:34:39,235 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:34:39,513 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:34:41,517 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:34:43,778 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:34:43,779 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:34:43,779 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:34:43,784 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:34:44,519 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:34:44,785 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:34:45,086 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:34:45,086 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:34:45,086 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:34:45,521 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:34:48,183 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:34:48,184 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:34:48,184 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:34:48,189 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:34:48,525 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:34:49,527 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:34:50,225 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:34:53,903 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:34:53,903 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:34:53,904 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:34:53,909 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:34:54,530 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:34:55,531 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:34:55,847 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 19:34:55,849 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:34:58,143 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:34:58,144 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:34:58,144 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:34:58,149 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:34:58,533 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:34:59,536 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:35:00,084 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:35:00,085 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:35:00,085 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:35:01,353 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:35:02,410 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:35:02,410 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:35:02,411 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:35:02,415 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:35:02,538 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:35:03,540 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:35:05,777 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:35:05,778 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:35:05,778 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:35:05,783 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:35:06,543 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:35:06,783 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:35:07,546 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:35:11,784 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:35:11,797 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:35:11,798 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:35:11,798 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:35:11,803 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:35:12,549 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:35:13,552 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:35:15,084 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:35:15,085 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:35:15,085 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:35:17,281 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:35:17,327 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:35:17,328 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:35:17,328 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:35:17,333 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:35:17,554 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:35:19,558 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:35:21,212 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:35:21,213 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:35:21,213 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:35:21,218 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:35:21,559 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:35:23,218 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:35:23,562 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:35:24,818 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:35:24,819 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:35:24,819 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:35:24,824 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:35:25,564 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:35:25,848 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 19:35:27,567 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:35:28,644 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:35:28,645 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:35:28,645 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:35:28,645 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:35:28,650 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:35:29,570 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:35:29,571 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:35:30,084 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:35:30,085 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:35:30,085 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:35:32,292 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:35:32,293 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:35:32,293 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:35:32,298 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:35:32,572 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:35:33,575 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:35:34,337 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:35:39,337 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:35:41,482 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:35:41,483 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:35:41,483 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:35:41,488 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:35:41,579 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:35:43,583 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:35:44,489 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:35:45,085 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:35:45,085 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:35:45,085 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:35:49,796 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:35:49,796 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:35:49,797 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:35:49,797 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:35:49,801 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:35:50,588 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:35:51,591 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:35:54,841 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:35:55,854 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 19:35:58,123 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:35:58,124 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:35:58,124 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:35:58,129 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:35:58,595 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:35:59,600 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:36:00,085 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:36:00,085 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:36:00,085 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:36:00,086 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:36:02,811 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:36:02,811 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:36:02,811 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:36:02,816 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:36:03,602 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:36:05,606 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:36:05,853 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:36:09,079 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:36:09,080 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:36:09,080 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:36:09,087 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:36:09,610 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:36:11,087 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:36:11,613 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:36:14,321 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:36:14,321 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:36:14,322 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:36:14,327 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:36:14,615 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:36:15,085 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:36:15,085 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:36:15,085 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:36:15,618 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:36:16,343 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:36:19,165 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:36:19,166 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:36:19,166 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:36:19,171 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:36:19,620 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:36:21,625 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:36:22,172 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:36:23,314 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:36:23,315 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:36:23,315 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:36:23,320 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:36:23,626 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:36:25,630 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:36:25,854 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 19:36:27,856 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:36:28,369 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:36:28,370 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:36:28,370 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:36:28,376 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:36:28,632 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:36:29,634 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:36:30,085 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:36:30,085 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:36:30,086 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:36:32,740 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:36:32,741 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:36:32,741 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:36:32,746 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:36:33,639 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:36:33,639 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:36:33,785 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:36:38,648 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:36:38,649 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:36:38,650 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:36:38,655 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:36:39,645 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:36:39,645 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:36:39,655 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:36:42,902 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:36:42,903 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:36:42,903 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:36:42,908 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:36:43,647 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:36:44,909 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:36:45,085 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:36:45,085 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:36:45,086 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:36:45,651 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:36:47,890 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:36:47,891 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:36:47,891 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:36:47,896 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:36:48,654 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:36:49,665 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:36:50,897 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:36:52,861 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:36:52,862 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:36:52,862 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:36:52,867 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:36:53,670 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:36:53,670 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:36:55,857 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 19:36:56,859 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:36:57,875 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:36:57,876 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:36:57,877 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:36:57,881 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:36:58,673 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:36:59,675 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:37:00,085 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:37:00,086 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:37:00,086 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:37:02,288 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:37:03,692 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:37:03,692 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:37:03,693 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:37:03,698 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:37:04,678 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:37:05,681 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:37:07,449 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:37:07,450 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:37:07,451 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:37:07,451 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:37:07,456 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:37:07,683 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:37:09,687 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:37:12,457 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:37:12,516 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:37:12,517 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:37:12,517 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:37:12,522 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:37:12,689 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:37:13,691 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:37:15,085 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:37:15,085 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:37:15,086 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:37:17,723 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:37:17,724 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:37:17,724 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:37:17,724 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:37:17,729 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:37:18,695 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:37:19,697 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:37:22,530 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:37:22,531 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:37:22,532 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:37:22,536 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:37:22,700 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:37:23,537 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:37:23,702 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:37:25,860 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 19:37:26,721 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:37:26,722 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:37:26,722 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:37:26,727 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:37:27,707 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:37:27,707 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:37:28,728 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:37:30,085 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:37:30,085 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:37:30,086 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:37:30,350 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:37:30,386 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:37:30,387 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:37:30,392 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:37:30,709 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:37:31,710 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:37:34,393 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:37:35,405 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:37:35,406 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:37:35,406 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:37:35,411 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:37:35,712 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:37:37,716 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:37:39,449 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:37:39,601 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:37:39,601 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:37:39,602 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:37:39,606 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:37:39,717 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:37:41,720 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:37:43,420 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:37:43,421 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:37:43,422 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:37:43,426 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:37:43,722 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:37:45,085 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:37:45,086 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:37:45,086 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:37:45,358 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:37:45,725 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:37:48,450 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:37:48,450 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:37:48,450 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:37:48,455 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:37:48,728 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:37:49,731 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:37:50,493 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:37:55,493 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:37:55,863 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 19:37:56,975 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:37:56,975 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:37:56,976 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:37:56,980 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:37:57,736 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:37:59,739 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:38:00,085 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:38:00,086 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:38:00,086 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:38:00,501 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:38:00,502 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:38:00,502 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:38:00,502 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:38:00,507 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:38:00,740 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:38:01,743 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:38:04,382 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:38:04,382 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:38:04,383 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:38:04,388 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:38:04,745 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:38:05,747 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:38:06,388 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:38:08,843 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:38:08,844 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:38:08,844 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:38:08,850 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:38:09,753 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:38:09,753 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:38:11,850 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:38:15,085 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:38:15,086 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:38:15,086 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:38:17,272 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:38:17,273 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:38:17,273 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:38:17,273 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:38:17,278 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:38:17,758 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:38:19,761 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:38:22,317 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:38:23,267 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:38:23,268 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:38:23,269 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:38:23,274 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:38:23,764 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:38:25,767 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:38:25,869 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 19:38:27,734 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:38:27,735 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:38:27,736 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:38:27,736 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:38:27,741 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:38:27,768 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:38:29,772 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:38:30,086 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:38:30,086 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:38:30,086 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:38:32,578 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:38:32,578 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:38:32,579 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:38:32,583 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:38:32,774 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:38:33,584 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:38:33,776 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:38:38,584 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:38:41,168 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:38:41,168 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:38:41,168 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:38:41,173 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:38:41,781 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:38:43,784 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:38:44,174 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:38:45,086 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:38:45,086 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:38:45,086 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:38:45,877 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:38:45,878 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:38:45,878 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:38:45,883 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:38:46,786 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:38:47,789 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:38:49,884 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:38:51,928 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:38:51,928 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:38:51,929 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:38:51,934 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:38:52,793 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:38:53,796 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:38:54,934 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:38:55,870 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 19:38:56,878 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:38:56,878 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:38:56,879 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:38:56,883 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:38:57,800 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:38:57,801 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:39:00,086 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:39:00,086 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:39:00,086 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:39:00,370 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:39:00,471 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:39:00,472 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:39:00,472 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:39:00,477 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:39:00,802 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:39:01,805 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:39:04,371 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:39:04,372 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:39:04,372 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:39:04,377 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:39:04,807 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:39:05,377 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:39:05,809 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:39:08,534 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:39:08,535 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:39:08,535 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:39:08,540 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:39:08,813 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:39:09,815 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:39:10,540 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:39:13,786 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:39:13,787 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:39:13,787 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:39:13,792 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:39:13,818 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:39:15,086 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:39:15,086 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:39:15,087 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:39:15,821 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:39:16,367 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:39:17,821 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:39:17,821 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:39:17,822 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:39:17,827 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:39:18,823 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:39:19,825 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:39:21,697 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:39:21,698 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:39:21,698 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:39:21,698 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:39:21,703 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:39:21,827 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:39:23,830 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:39:25,727 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:39:25,728 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:39:25,728 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:39:25,732 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:39:25,832 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:39:25,875 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 19:39:26,876 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:39:27,835 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:39:30,086 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:39:30,086 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:39:30,087 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:39:30,138 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:39:30,299 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:39:30,299 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:39:30,304 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:39:30,837 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:39:31,839 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:39:32,304 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:39:37,167 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:39:37,168 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:39:37,168 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:39:37,173 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:39:37,843 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:39:38,173 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:39:39,846 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:39:41,181 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:39:41,181 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:39:41,182 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:39:41,187 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:39:41,848 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:39:43,187 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:39:43,851 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:39:44,811 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:39:44,812 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:39:44,812 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:39:44,817 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:39:44,852 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:39:45,086 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:39:45,087 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:39:45,087 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:39:45,854 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:39:48,350 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:39:53,350 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:39:54,531 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:39:54,531 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:39:54,532 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:39:54,537 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:39:54,861 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:39:55,863 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:39:55,874 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 19:39:58,875 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:40:00,086 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:40:00,087 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:40:00,087 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:40:00,673 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:40:00,673 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:40:00,674 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:40:00,678 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:40:00,866 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:40:01,869 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:40:04,361 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:40:04,361 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:40:04,362 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:40:04,362 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:40:04,366 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:40:04,872 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:40:05,872 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:40:09,367 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:40:10,061 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:40:10,062 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:40:10,062 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:40:10,067 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:40:10,877 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:40:11,879 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:40:14,053 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:40:14,053 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:40:14,054 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:40:14,059 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:40:14,881 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:40:15,059 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:40:15,086 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:40:15,086 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:40:15,087 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:40:15,884 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:40:18,638 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:40:18,639 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:40:18,639 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:40:18,644 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:40:18,886 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:40:19,888 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:40:20,681 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:40:22,046 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:40:22,047 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:40:22,047 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:40:22,053 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:40:22,891 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:40:23,891 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:40:25,880 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 19:40:25,883 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:40:27,335 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:40:27,335 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:40:27,336 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:40:27,341 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:40:27,894 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:40:29,897 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:40:30,086 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:40:30,087 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:40:30,087 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:40:31,356 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:40:31,675 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:40:31,675 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:40:31,675 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:40:31,680 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:40:31,898 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:40:33,901 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:40:36,717 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:40:36,814 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:40:36,814 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:40:36,814 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:40:36,819 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:40:36,903 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:40:37,906 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:40:40,668 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:40:40,668 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:40:40,669 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:40:40,673 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:40:40,908 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:40:41,908 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:40:42,674 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:40:44,304 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:40:44,304 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:40:44,305 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:40:44,309 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:40:44,911 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:40:45,086 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:40:45,087 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:40:45,087 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:40:45,913 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:40:48,088 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:40:48,088 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:40:48,089 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:40:48,089 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:40:48,094 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:40:48,917 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:40:49,919 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:40:53,133 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:40:54,173 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:40:54,174 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:40:54,174 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:40:54,179 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:40:54,922 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:40:55,880 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 19:40:55,925 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:40:58,881 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:41:00,086 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:41:00,087 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:41:00,087 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:41:00,563 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:41:00,563 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:41:00,564 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:41:00,568 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:41:00,928 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:41:01,930 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:41:04,605 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:41:05,604 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:41:05,605 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:41:05,605 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:41:05,610 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:41:05,933 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:41:07,937 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:41:09,611 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:41:10,684 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:41:10,684 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:41:10,685 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:41:10,690 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:41:10,939 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:41:11,943 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:41:14,691 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:41:15,087 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:41:15,087 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:41:15,087 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:41:15,808 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:41:15,809 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:41:15,809 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:41:15,814 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:41:15,945 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:41:17,953 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:41:19,604 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:41:19,605 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:41:19,605 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:41:19,610 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:41:19,955 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:41:20,610 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:41:21,959 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:41:24,627 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:41:24,627 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:41:24,628 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:41:24,632 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:41:24,961 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:41:25,633 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:41:25,885 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 19:41:25,963 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:41:30,087 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:41:30,087 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:41:30,087 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:41:30,349 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:41:30,390 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:41:30,390 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:41:30,395 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:41:30,966 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:41:31,395 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:41:31,969 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:41:34,665 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:41:34,666 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:41:34,666 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:41:34,671 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:41:34,971 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:41:35,972 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:41:36,672 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:41:38,126 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:41:38,127 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:41:38,127 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:41:38,132 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:41:38,974 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:41:39,976 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:41:42,132 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:41:44,489 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:41:44,489 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:41:44,490 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:41:44,495 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:41:44,980 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:41:45,087 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:41:45,087 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:41:45,088 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:41:45,983 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:41:47,578 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:41:48,670 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:41:48,671 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:41:48,671 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:41:48,676 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:41:48,986 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:41:49,999 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:41:52,285 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:41:52,285 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:41:52,286 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:41:52,291 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:41:53,001 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:41:53,291 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:41:54,004 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:41:55,589 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:41:55,590 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:41:55,590 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:41:55,595 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:41:55,884 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 19:41:56,005 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:41:58,008 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:41:58,885 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:41:59,915 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:41:59,916 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:41:59,916 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:41:59,921 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:42:00,010 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:42:00,087 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:42:00,088 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:42:00,088 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:42:02,013 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:42:03,516 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:42:03,516 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:42:03,517 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:42:03,524 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:42:04,014 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:42:04,525 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:42:06,018 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:42:08,428 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:42:08,429 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:42:08,429 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:42:08,434 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:42:09,021 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:42:10,024 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:42:10,435 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:42:12,626 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:42:12,627 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:42:12,627 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:42:12,634 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:42:13,026 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:42:14,028 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:42:15,087 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:42:15,088 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:42:15,088 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:42:16,311 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:42:16,511 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:42:16,511 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:42:16,511 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:42:16,516 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:42:17,030 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:42:18,033 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:42:20,507 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:42:20,508 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:42:20,508 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:42:20,513 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:42:21,035 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:42:21,513 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:42:22,038 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:42:24,694 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:42:24,695 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:42:24,695 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:42:24,700 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:42:25,041 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:42:25,887 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 19:42:26,041 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:42:26,888 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:42:29,150 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:42:29,151 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:42:29,151 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:42:29,156 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:42:30,046 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:42:30,046 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:42:30,087 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:42:30,087 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:42:30,088 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:42:32,380 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:42:36,681 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:42:36,682 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:42:36,682 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:42:36,687 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:42:37,050 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:42:37,725 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:42:38,051 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:42:41,292 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:42:41,293 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:42:41,293 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:42:41,298 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:42:42,053 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:42:43,298 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:42:44,056 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:42:45,087 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:42:45,088 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:42:45,088 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:42:48,310 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:42:49,796 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:42:49,800 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:42:49,800 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:42:49,805 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:42:50,062 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:42:52,065 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:42:53,841 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:42:54,269 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:42:54,270 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:42:54,270 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:42:54,275 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:42:55,067 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:42:55,890 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 19:42:56,070 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:42:58,892 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:42:58,911 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:42:58,912 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:42:58,912 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:42:58,917 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:42:59,072 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:43:00,074 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:43:00,088 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:43:00,088 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:43:00,088 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:43:03,322 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:43:03,323 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:43:03,323 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:43:03,328 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:43:04,077 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:43:04,365 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:43:06,080 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:43:09,365 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:43:09,845 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:43:09,846 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:43:09,846 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:43:09,852 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:43:10,084 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:43:12,088 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:43:13,673 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:43:13,674 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:43:13,674 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:43:13,679 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:43:14,089 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:43:14,680 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:43:15,088 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:43:15,088 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:43:15,088 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:43:16,093 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:43:18,187 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:43:18,188 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:43:18,188 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:43:18,193 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:43:19,095 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:43:20,096 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:43:20,233 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:43:21,764 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:43:21,764 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:43:21,765 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:43:21,770 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:43:22,098 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:43:24,102 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:43:25,770 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:43:25,893 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 19:43:26,354 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:43:26,354 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:43:26,354 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:43:26,359 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:43:27,104 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:43:28,106 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:43:29,796 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:43:29,796 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:43:29,797 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:43:29,801 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:43:30,088 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:43:30,088 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:43:30,089 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:43:30,108 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:43:31,378 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:43:32,111 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:43:36,379 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:43:37,315 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:43:37,316 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:43:37,316 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:43:37,321 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:43:38,115 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:43:40,118 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:43:41,287 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:43:41,288 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:43:41,288 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:43:41,294 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:43:42,122 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:43:42,122 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:43:42,294 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:43:45,080 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:43:45,080 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:43:45,081 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:43:45,085 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:43:45,088 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:43:45,088 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:43:45,088 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:43:45,124 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:43:46,127 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:43:47,381 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:43:49,773 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:43:49,773 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:43:49,774 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:43:49,779 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:43:50,131 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:43:52,134 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:43:52,817 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:43:54,916 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:43:54,916 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:43:54,917 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:43:54,921 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:43:55,136 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:43:55,898 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 19:43:56,138 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:43:57,901 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:43:59,485 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:43:59,486 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:43:59,486 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:43:59,491 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:44:00,088 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:44:00,088 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:44:00,089 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:44:00,141 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:44:02,144 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:44:03,337 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:44:03,997 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:44:03,998 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:44:03,998 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:44:04,003 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:44:04,145 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:44:06,148 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:44:07,772 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:44:07,773 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:44:07,773 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:44:07,778 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:44:08,151 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:44:08,778 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:44:10,154 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:44:11,780 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:44:11,781 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:44:11,781 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:44:11,787 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:44:12,156 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:44:13,787 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:44:14,157 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:44:15,088 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:44:15,088 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:44:15,089 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:44:19,391 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:44:20,192 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:44:20,192 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:44:20,193 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:44:20,197 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:44:21,162 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:44:22,164 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:44:25,237 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:44:25,897 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 19:44:26,032 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:44:26,033 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:44:26,033 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:44:26,038 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:44:26,167 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:44:28,170 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:44:30,088 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:44:30,088 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:44:30,089 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:44:30,304 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:44:31,564 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:44:31,564 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:44:31,565 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:44:31,569 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:44:32,173 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:44:34,176 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:44:35,371 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:44:35,372 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:44:35,372 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:44:35,372 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:44:35,377 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:44:36,179 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:44:36,179 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:44:39,210 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:44:39,211 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:44:39,211 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:44:39,216 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:44:40,184 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:44:40,184 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:44:41,217 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:44:45,088 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:44:45,088 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:44:45,089 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:44:46,382 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:44:48,149 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:44:48,149 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:44:48,149 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:44:48,155 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:44:48,189 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:44:50,191 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:44:52,193 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:44:52,675 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:44:52,676 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:44:52,676 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:44:52,681 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:44:53,193 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:44:54,196 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:44:55,903 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 19:44:57,904 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:44:58,217 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:44:58,217 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:44:58,218 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:44:58,223 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:44:59,199 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:45:00,088 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:45:00,089 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:45:00,089 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:45:00,201 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:45:02,185 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:45:02,185 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:45:02,185 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:45:02,191 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:45:02,203 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:45:03,229 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:45:04,206 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:45:08,229 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:45:10,477 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:45:10,478 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:45:10,478 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:45:10,483 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:45:11,211 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:45:12,214 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:45:13,483 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:45:15,088 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:45:15,089 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:45:15,089 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:45:19,401 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:45:19,678 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:45:19,678 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:45:19,679 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:45:19,683 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:45:20,219 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:45:22,222 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:45:23,190 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:45:23,191 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:45:23,191 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:45:23,196 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:45:23,223 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:45:24,226 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:45:25,196 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:45:25,903 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 19:45:26,751 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:45:26,752 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:45:26,752 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:45:26,757 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:45:27,228 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:45:28,230 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:45:30,088 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:45:30,089 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:45:30,089 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:45:30,296 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:45:31,289 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:45:31,289 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:45:31,290 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:45:31,294 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:45:32,235 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:45:32,235 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:45:35,333 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:45:35,406 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:45:35,407 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:45:35,407 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:45:35,412 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:45:36,239 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:45:36,239 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:45:40,413 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:45:40,888 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:45:40,889 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:45:40,889 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:45:40,894 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:45:41,242 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:45:42,245 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:45:44,644 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:45:44,645 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:45:44,645 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:45:44,649 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:45:45,089 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:45:45,089 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:45:45,089 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:45:45,247 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:45:46,250 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:45:46,306 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:45:51,306 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:45:52,899 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:45:52,899 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:45:52,900 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:45:52,905 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:45:53,255 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:45:54,258 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:45:55,910 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 19:45:56,665 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:45:56,666 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:45:56,666 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:45:56,667 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:45:56,671 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:45:57,260 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:45:58,262 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:46:00,089 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:46:00,089 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:46:00,089 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:46:01,327 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:46:01,328 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:46:01,328 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:46:01,333 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:46:02,265 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:46:02,265 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:46:02,373 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:46:05,367 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:46:05,367 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:46:05,368 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:46:05,372 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:46:06,269 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:46:06,270 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:46:08,373 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:46:10,110 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:46:10,110 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:46:10,111 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:46:10,116 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:46:10,273 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:46:12,276 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:46:14,116 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:46:15,089 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:46:15,089 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:46:15,089 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:46:16,872 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:46:16,872 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:46:16,872 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:46:16,877 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:46:17,279 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:46:18,282 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:46:19,917 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:46:20,743 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:46:20,744 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:46:20,744 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:46:20,748 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:46:21,284 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:46:22,287 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:46:25,749 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:46:25,910 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 19:46:28,987 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:46:28,988 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:46:28,988 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:46:28,993 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:46:29,291 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:46:30,089 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:46:30,089 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:46:30,089 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:46:30,294 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:46:31,316 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:46:34,008 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:46:34,009 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:46:34,009 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:46:34,014 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:46:34,296 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:46:36,300 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:46:37,053 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:46:37,957 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:46:37,957 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:46:37,958 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:46:37,962 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:46:38,301 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:46:40,304 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:46:41,926 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:46:41,927 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:46:41,927 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:46:41,932 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:46:42,306 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:46:42,933 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:46:44,309 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:46:45,089 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:46:45,089 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:46:45,090 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:46:45,474 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:46:45,475 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:46:45,475 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:46:45,480 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:46:46,312 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:46:46,313 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:46:48,517 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:46:51,273 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:46:51,273 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:46:51,273 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:46:51,279 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:46:51,317 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:46:52,319 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:46:54,279 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:46:55,913 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 19:46:59,747 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:46:59,748 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:46:59,748 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:46:59,749 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:46:59,753 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:47:00,089 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:47:00,089 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:47:00,090 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:47:00,324 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:47:02,327 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:47:05,385 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:47:08,111 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:47:08,114 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:47:08,114 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:47:08,119 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:47:08,332 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:47:10,335 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:47:11,153 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:47:12,043 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:47:12,044 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:47:12,044 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:47:12,049 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:47:12,336 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:47:14,340 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:47:15,089 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:47:15,089 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:47:15,090 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:47:16,378 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:47:16,985 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:47:16,985 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:47:16,985 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:47:16,990 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:47:17,341 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:47:18,344 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:47:22,029 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:47:22,530 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:47:22,531 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:47:22,531 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:47:22,536 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:47:23,348 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:47:24,350 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:47:25,916 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 19:47:26,674 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:47:26,674 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:47:26,675 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:47:26,680 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:47:27,352 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:47:27,680 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:47:28,355 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:47:30,089 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:47:30,090 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:47:30,090 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:47:30,780 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:47:30,781 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:47:30,781 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:47:30,786 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:47:31,357 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:47:32,359 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:47:32,825 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:47:36,508 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:47:36,509 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:47:36,509 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:47:36,514 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:47:37,362 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:47:38,363 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:47:38,515 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:47:40,698 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:47:40,699 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:47:40,699 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:47:40,704 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:47:41,365 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:47:42,368 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:47:43,705 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:47:44,560 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:47:44,561 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:47:44,561 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:47:44,566 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:47:45,089 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:47:45,090 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:47:45,090 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:47:45,370 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:47:46,372 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:47:48,597 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:47:48,597 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:47:48,597 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:47:48,602 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:47:49,376 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:47:49,641 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:47:50,378 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:47:52,677 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:47:52,678 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:47:52,678 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:47:52,682 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:47:53,380 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:47:54,383 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:47:54,683 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:47:55,918 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 19:47:56,509 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:47:56,509 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:47:56,510 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:47:56,514 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:47:57,385 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:47:58,388 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:48:00,089 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:48:00,090 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:48:00,090 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:48:00,090 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:48:02,675 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:48:02,676 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:48:02,676 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:48:02,681 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:48:03,391 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:48:04,393 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:48:05,717 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:48:06,722 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:48:06,723 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:48:06,723 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:48:06,728 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:48:07,396 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:48:08,399 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:48:10,728 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:48:15,089 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:48:15,090 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:48:15,090 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:48:15,108 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:48:15,349 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:48:15,349 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:48:15,354 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:48:15,403 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:48:16,354 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:48:16,406 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:48:19,115 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:48:19,116 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:48:19,116 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:48:19,121 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:48:19,408 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:48:20,408 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:48:22,122 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:48:22,715 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:48:22,716 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:48:22,716 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:48:22,721 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:48:23,410 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:48:24,413 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:48:25,924 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 19:48:26,262 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:48:26,263 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:48:26,263 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:48:26,268 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:48:26,414 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:48:27,268 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:48:28,418 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:48:30,090 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:48:30,090 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:48:30,090 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:48:30,248 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:48:30,380 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:48:30,380 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:48:30,385 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:48:30,419 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:48:32,386 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:48:32,422 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:48:35,877 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:48:35,878 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:48:35,878 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:48:35,883 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:48:36,425 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:48:37,883 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:48:38,428 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:48:40,256 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:48:40,257 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:48:40,258 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:48:40,262 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:48:40,429 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:48:42,433 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:48:43,263 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:48:45,090 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:48:45,090 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:48:45,090 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:48:45,967 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:48:45,967 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:48:45,967 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:48:45,972 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:48:46,435 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:48:48,439 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:48:49,009 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:48:49,888 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:48:49,889 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:48:49,890 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:48:49,894 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:48:50,441 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:48:52,444 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:48:53,903 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:48:53,904 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:48:53,904 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:48:53,909 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:48:54,446 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:48:54,910 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:48:55,922 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 19:48:56,449 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:48:57,614 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:48:57,614 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:48:57,615 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:48:57,621 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:48:58,452 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:48:58,452 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:49:00,090 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:49:00,090 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:49:00,091 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:49:00,091 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:49:02,926 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:49:02,927 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:49:02,927 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:49:02,933 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:49:03,455 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:49:04,458 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:49:05,969 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:49:08,618 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:49:08,619 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:49:08,619 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:49:08,624 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:49:09,462 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:49:10,465 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:49:11,625 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:49:15,090 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:49:15,090 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:49:15,091 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:49:17,385 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:49:18,267 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:49:18,268 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:49:18,268 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:49:18,273 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:49:18,469 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:49:20,473 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:49:21,832 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:49:21,833 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:49:21,833 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:49:21,838 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:49:22,474 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:49:22,838 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:49:24,478 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:49:25,927 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 19:49:26,942 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:49:26,943 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:49:26,943 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:49:26,948 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:49:27,480 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:49:27,948 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:49:28,480 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:49:30,090 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:49:30,090 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:49:30,091 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:49:31,377 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:49:31,378 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:49:31,378 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:49:31,383 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:49:31,482 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:49:32,485 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:49:33,421 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:49:35,341 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:49:35,342 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:49:35,343 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:49:35,347 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:49:35,487 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:49:36,490 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:49:39,056 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:49:39,057 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:49:39,057 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:49:39,057 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:49:39,062 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:49:39,492 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:49:40,495 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:49:44,063 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:49:44,716 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:49:44,717 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:49:44,717 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:49:44,722 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:49:45,090 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:49:45,090 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:49:45,091 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:49:45,499 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:49:46,501 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:49:49,288 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:49:49,289 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:49:49,289 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:49:49,290 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:49:49,294 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:49:49,504 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:49:50,505 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:49:54,141 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:49:54,142 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:49:54,142 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:49:54,147 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:49:54,508 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:49:55,147 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:49:55,926 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 19:49:56,511 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:49:57,791 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:49:57,791 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:49:57,792 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:49:57,798 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:49:58,512 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:50:00,090 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:50:00,090 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:50:00,091 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:50:00,342 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:50:00,515 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:50:02,296 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:50:02,297 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:50:02,297 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:50:02,302 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:50:02,516 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:50:04,519 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:50:06,341 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:50:08,631 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:50:08,632 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:50:08,633 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:50:08,638 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:50:09,524 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:50:10,527 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:50:11,638 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:50:12,540 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:50:12,541 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:50:12,541 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:50:12,546 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:50:13,529 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:50:14,531 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:50:15,090 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:50:15,090 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:50:15,091 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:50:17,378 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:50:18,208 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:50:18,209 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:50:18,209 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:50:18,214 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:50:18,534 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:50:20,537 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:50:23,253 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:50:25,931 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 19:50:26,565 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:50:26,566 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:50:26,566 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:50:26,571 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:50:27,541 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:50:28,544 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:50:28,572 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:50:30,090 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:50:30,091 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:50:30,091 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:50:33,949 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:50:33,950 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:50:33,950 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:50:33,950 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:50:33,955 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:50:34,547 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:50:36,550 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:50:37,996 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:50:37,997 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:50:37,997 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:50:38,002 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:50:38,552 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:50:39,003 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:50:40,555 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:50:44,003 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:50:44,342 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:50:44,343 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:50:44,344 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:50:44,348 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:50:44,557 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:50:45,090 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:50:45,091 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:50:45,091 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:50:46,561 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:50:48,889 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:50:48,890 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:50:48,890 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:50:48,895 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:50:49,564 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:50:49,933 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:50:50,565 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:50:52,905 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:50:52,906 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:50:52,906 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:50:52,911 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:50:53,566 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:50:54,569 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:50:55,912 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:50:55,930 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 19:50:56,887 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:50:56,888 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:50:56,888 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:50:56,893 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:50:57,571 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:50:58,574 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:51:00,090 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:51:00,091 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:51:00,091 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:51:01,350 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:51:03,067 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:51:03,068 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:51:03,068 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:51:03,074 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:51:03,577 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:51:04,579 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:51:07,113 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:51:07,459 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:51:07,459 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:51:07,460 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:51:07,465 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:51:07,582 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:51:08,585 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:51:12,186 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:51:12,187 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:51:12,187 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:51:12,187 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:51:12,193 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:51:12,588 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:51:14,591 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:51:15,090 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:51:15,091 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:51:15,091 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:51:16,780 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:51:16,781 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:51:16,781 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:51:16,786 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:51:17,593 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:51:17,825 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:51:18,596 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:51:20,577 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:51:20,577 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:51:20,578 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:51:20,583 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:51:20,597 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:51:22,601 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:51:23,584 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:51:25,936 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 19:51:26,257 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:51:26,258 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:51:26,258 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:51:26,263 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:51:26,603 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:51:28,606 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:51:29,264 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:51:30,091 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:51:30,091 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:51:30,091 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:51:33,407 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:51:33,408 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:51:33,408 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:51:33,413 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:51:33,610 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:51:34,453 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:51:34,610 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:51:38,172 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:51:38,173 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:51:38,173 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:51:38,178 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:51:38,613 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:51:40,179 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:51:40,616 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:51:43,503 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:51:43,504 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:51:43,504 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:51:43,509 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:51:43,618 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:51:44,620 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:51:45,091 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:51:45,091 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:51:45,091 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:51:45,310 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:51:48,214 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:51:48,215 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:51:48,215 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:51:48,220 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:51:48,623 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:51:50,626 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:51:51,257 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:51:52,617 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:51:52,618 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:51:52,619 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:51:52,624 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:51:52,627 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:51:54,630 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:51:55,935 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 19:51:56,362 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:51:56,363 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:51:56,364 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:51:56,364 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:51:56,368 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:51:56,631 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:51:58,634 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:52:00,091 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:52:00,091 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:52:00,091 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:52:01,544 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:52:01,545 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:52:01,546 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:52:01,546 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:52:01,550 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:52:01,636 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:52:02,639 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:52:05,681 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:52:05,682 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:52:05,682 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:52:05,687 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:52:06,641 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:52:06,641 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:52:06,688 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:52:09,730 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:52:09,731 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:52:09,731 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:52:09,736 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:52:10,647 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:52:10,647 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:52:11,737 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:52:13,970 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:52:13,971 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:52:13,971 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:52:13,976 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:52:14,649 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:52:15,091 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:52:15,091 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:52:15,092 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:52:16,653 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:52:17,461 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:52:21,120 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:52:21,121 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:52:21,121 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:52:21,126 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:52:21,656 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:52:22,658 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:52:23,165 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:52:25,282 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:52:25,283 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:52:25,284 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:52:25,288 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:52:25,660 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:52:25,937 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 19:52:26,661 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:52:28,939 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:52:30,091 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:52:30,091 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:52:30,092 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:52:31,761 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:52:31,762 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:52:31,762 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:52:31,768 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:52:32,666 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:52:32,667 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:52:34,805 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:52:36,882 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:52:36,886 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:52:36,886 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:52:36,891 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:52:37,669 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:52:38,672 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:52:39,892 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:52:42,086 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:52:42,086 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:52:42,087 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:52:42,091 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:52:42,676 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:52:44,681 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:52:45,091 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:52:45,091 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:52:45,092 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:52:45,092 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:52:50,376 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:52:50,377 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:52:50,377 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:52:50,377 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:52:50,382 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:52:50,687 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:52:52,691 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:52:54,890 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:52:54,891 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:52:54,891 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:52:54,896 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:52:55,694 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:52:55,897 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:52:55,938 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 19:52:56,703 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:52:59,369 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:52:59,370 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:52:59,370 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:52:59,375 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:52:59,705 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:53:00,091 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:53:00,091 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:53:00,092 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:53:00,707 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:53:01,362 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:53:03,802 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:53:03,802 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:53:03,803 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:53:03,809 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:53:04,712 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:53:04,712 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:53:06,845 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:53:08,880 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:53:08,881 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:53:08,881 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:53:08,886 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:53:09,716 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:53:10,719 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:53:11,887 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:53:12,731 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:53:12,732 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:53:12,732 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:53:12,737 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:53:13,722 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:53:14,726 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:53:15,091 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:53:15,092 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:53:15,092 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:53:16,394 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:53:16,394 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:53:16,395 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:53:16,400 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:53:16,727 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:53:17,437 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:53:18,732 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:53:19,730 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:53:19,730 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:53:19,731 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:53:19,740 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:53:20,737 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:53:20,738 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:53:22,741 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:53:24,325 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:53:24,326 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:53:24,326 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:53:24,331 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:53:24,740 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:53:25,940 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 19:53:26,744 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:53:27,942 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:53:28,280 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:53:28,280 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:53:28,281 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:53:28,286 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:53:28,746 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:53:30,091 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:53:30,092 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:53:30,092 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:53:30,751 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:53:32,380 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:53:32,381 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:53:32,381 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:53:32,386 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:53:32,752 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:53:33,425 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:53:34,756 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:53:38,425 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:53:38,659 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:53:38,660 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:53:38,660 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:53:38,665 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:53:38,760 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:53:40,765 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:53:43,522 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:53:43,523 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:53:43,523 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:53:43,523 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:53:43,528 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:53:43,767 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:53:44,771 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:53:45,091 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:53:45,092 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:53:45,092 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:53:47,845 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:53:47,846 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:53:47,846 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:53:47,851 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:53:48,775 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:53:48,775 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:53:48,889 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:53:52,666 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:53:52,667 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:53:52,667 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:53:52,672 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:53:52,780 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:53:54,673 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:53:54,785 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:53:55,945 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 19:53:57,788 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:53:57,788 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:53:57,789 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:53:57,794 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:53:57,796 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:53:58,798 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:53:59,794 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:54:00,092 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:54:00,092 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:54:00,092 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:54:02,504 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:54:02,505 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:54:02,505 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:54:02,510 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:54:02,801 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:54:04,808 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:54:05,549 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:54:08,589 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:54:08,589 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:54:08,590 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:54:08,595 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:54:08,812 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:54:10,596 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:54:10,813 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:54:15,092 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:54:15,092 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:54:15,092 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:54:16,381 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:54:17,108 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:54:17,108 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:54:17,109 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:54:17,113 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:54:17,819 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:54:18,824 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:54:21,076 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:54:21,077 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:54:21,078 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:54:21,083 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:54:21,826 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:54:22,084 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:54:22,830 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:54:24,840 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:54:24,840 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:54:24,841 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:54:24,846 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:54:25,832 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:54:25,944 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 19:54:26,836 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:54:27,946 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:54:30,092 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:54:30,092 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:54:30,093 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:54:31,783 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:54:31,784 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:54:31,784 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:54:31,789 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:54:31,840 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:54:32,845 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:54:33,829 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:54:35,853 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:54:35,854 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:54:35,855 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:54:35,860 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:54:36,851 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:54:36,851 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:54:38,861 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:54:39,974 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:54:39,974 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:54:39,975 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:54:39,980 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:54:40,857 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:54:40,857 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:54:43,980 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:54:44,150 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:54:44,151 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:54:44,151 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:54:44,157 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:54:44,861 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:54:45,092 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:54:45,092 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:54:45,095 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:54:46,865 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:54:47,697 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:54:47,697 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:54:47,698 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:54:47,702 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:54:47,866 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:54:48,868 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:54:49,741 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:54:52,593 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:54:52,593 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:54:52,594 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:54:52,599 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:54:52,873 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:54:54,878 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:54:55,600 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:54:55,949 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 19:54:56,352 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:54:56,353 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:54:56,353 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:54:56,358 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:54:56,879 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:54:58,884 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:55:00,092 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:55:00,092 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:55:00,093 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:55:01,379 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:55:01,789 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:55:01,790 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:55:01,790 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:55:01,795 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:55:01,886 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:55:02,890 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:55:06,833 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:55:07,005 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:55:07,005 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:55:07,006 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:55:07,011 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:55:07,895 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:55:08,899 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:55:11,422 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:55:11,423 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:55:11,423 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:55:11,428 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:55:11,900 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:55:12,428 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:55:12,905 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:55:15,092 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:55:15,092 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:55:15,093 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:55:15,660 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:55:15,661 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:55:15,661 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:55:15,667 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:55:15,907 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:55:16,911 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:55:17,705 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:55:20,509 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:55:20,510 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:55:20,510 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:55:20,515 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:55:20,915 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:55:22,918 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:55:23,516 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:55:24,300 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:55:24,300 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:55:24,301 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:55:24,306 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:55:24,919 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:55:25,949 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 19:55:26,922 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:55:28,711 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:55:28,712 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:55:28,712 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:55:28,712 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:55:28,717 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:55:28,924 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:55:30,092 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:55:30,092 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:55:30,093 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:55:30,926 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:55:32,141 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:55:32,142 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:55:32,142 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:55:32,147 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:55:32,928 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:55:34,185 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:55:34,931 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:55:39,185 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:55:40,627 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:55:40,628 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:55:40,628 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:55:40,636 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:55:40,934 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:55:42,937 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:55:44,637 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:55:45,092 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:55:45,093 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:55:45,093 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:55:45,444 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:55:45,445 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:55:45,445 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:55:45,450 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:55:45,939 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:55:46,942 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:55:49,844 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:55:49,845 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:55:49,845 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:55:49,846 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:55:49,850 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:55:49,945 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:55:52,948 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:55:54,310 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:55:54,311 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:55:54,312 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:55:54,316 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:55:54,949 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:55:55,317 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:55:55,954 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 19:55:56,953 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:55:58,331 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:55:58,332 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:55:58,332 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:55:58,337 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:55:58,954 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:56:00,092 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:56:00,093 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:56:00,093 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:56:00,373 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:56:00,957 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:56:02,075 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:56:02,075 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:56:02,076 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:56:02,081 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:56:02,959 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:56:04,962 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:56:06,117 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:56:06,738 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:56:06,739 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:56:06,739 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:56:06,745 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:56:06,963 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:56:08,967 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:56:11,173 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:56:11,174 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:56:11,174 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:56:11,175 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:56:11,180 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:56:11,969 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:56:12,971 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:56:15,092 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:56:15,093 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:56:15,093 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:56:16,367 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:56:17,103 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:56:17,104 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:56:17,104 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:56:17,109 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:56:17,974 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:56:18,977 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:56:21,059 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:56:21,060 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:56:21,060 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:56:21,065 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:56:21,979 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:56:22,065 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:56:22,981 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:56:25,953 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 19:56:26,461 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:56:26,462 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:56:26,462 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:56:26,467 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:56:26,983 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:56:27,467 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:56:28,986 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:56:30,093 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:56:30,093 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:56:30,093 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:56:30,432 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:56:30,433 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:56:30,433 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:56:30,438 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:56:30,988 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:56:32,477 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:56:32,991 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:56:34,279 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:56:34,279 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:56:34,280 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:56:34,285 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:56:34,992 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:56:36,995 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:56:38,286 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:56:38,769 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:56:38,770 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:56:38,770 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:56:38,775 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:56:38,996 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:56:40,999 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:56:43,529 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:56:43,530 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:56:43,530 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:56:43,531 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:56:43,536 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:56:44,001 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:56:45,003 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:56:45,093 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:56:45,093 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:56:45,093 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:56:49,374 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:56:49,400 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:56:49,401 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:56:49,401 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:56:49,406 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:56:50,008 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:56:51,010 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:56:54,445 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:56:54,518 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:56:54,519 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:56:54,519 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:56:54,524 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:56:55,013 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:56:55,958 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 19:56:57,016 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:56:59,960 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:57:00,093 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:57:00,093 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:57:00,093 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:57:01,591 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:57:01,592 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:57:01,592 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:57:01,597 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:57:02,019 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:57:03,022 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:57:05,637 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:57:06,853 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:57:06,854 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:57:06,854 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:57:06,859 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:57:07,024 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:57:09,028 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:57:10,860 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:57:12,370 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:57:12,371 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:57:12,371 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:57:12,376 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:57:13,031 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:57:15,034 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:57:15,093 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:57:15,093 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:57:15,094 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:57:16,319 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:57:16,488 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:57:16,489 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:57:16,489 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:57:16,494 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:57:17,035 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:57:19,038 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:57:20,314 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:57:20,315 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:57:20,316 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:57:20,322 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:57:21,039 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:57:21,323 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:57:23,042 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:57:23,929 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:57:23,930 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:57:23,930 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:57:23,936 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:57:24,043 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:57:25,957 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 19:57:26,959 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:57:27,047 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:57:30,093 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:57:30,093 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:57:30,094 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:57:32,264 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:57:32,264 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:57:32,265 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:57:32,265 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:57:32,270 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:57:33,050 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:57:35,053 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:57:35,947 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:57:35,947 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:57:35,948 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:57:35,953 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:57:36,054 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:57:37,953 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:57:39,057 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:57:40,317 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:57:40,318 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:57:40,318 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:57:40,323 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:57:41,058 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:57:43,061 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:57:43,323 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:57:45,093 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:57:45,093 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:57:45,094 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:57:46,149 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:57:46,149 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:57:46,150 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:57:46,155 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:57:47,064 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:57:49,067 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:57:49,193 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:57:51,004 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:57:51,005 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:57:51,005 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:57:51,010 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:57:51,070 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:57:53,073 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:57:55,011 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:57:55,959 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 19:57:57,334 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:57:57,334 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:57:57,335 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:57:57,340 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:57:58,076 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:57:59,078 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:58:00,093 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:58:00,093 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:58:00,094 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:58:00,094 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:58:02,364 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:58:02,364 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:58:02,365 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:58:02,370 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:58:03,080 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:58:05,083 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:58:05,409 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:58:09,783 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:58:09,784 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:58:09,784 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:58:09,789 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:58:10,087 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:58:10,790 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:58:11,089 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:58:13,823 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:58:13,827 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:58:13,827 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:58:13,832 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:58:14,091 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:58:15,093 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:58:15,093 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:58:15,094 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:58:16,304 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:58:17,095 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:58:18,066 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:58:18,067 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:58:18,067 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:58:18,072 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:58:18,096 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:58:21,099 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:58:22,109 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:58:22,382 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:58:22,383 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:58:22,383 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:58:22,388 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:58:23,101 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:58:25,104 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:58:25,961 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 19:58:27,963 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:58:30,093 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:58:30,094 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:58:30,094 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:58:30,851 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:58:30,851 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:58:30,852 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:58:30,856 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:58:31,107 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:58:33,110 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:58:33,893 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:58:35,131 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:58:35,132 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:58:35,132 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:58:35,137 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:58:36,112 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:58:37,114 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:58:38,946 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:58:38,947 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:58:38,947 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:58:38,947 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:58:38,952 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:58:39,116 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:58:41,119 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:58:43,953 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:58:44,052 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:58:44,053 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:58:44,053 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:58:44,058 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:58:44,120 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:58:45,093 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:58:45,094 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:58:45,094 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:58:47,124 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:58:48,419 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:58:48,420 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:58:48,420 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:58:48,425 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:58:49,126 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:58:49,465 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:58:51,128 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:58:52,393 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:58:52,394 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:58:52,394 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:58:52,399 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:58:53,129 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:58:55,132 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:58:55,400 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:58:55,963 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 19:58:56,639 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:58:56,640 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:58:56,640 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:58:56,647 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:58:57,133 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:58:59,136 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:59:00,093 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:59:00,094 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:59:00,094 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:59:00,597 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:59:00,597 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:59:00,598 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:59:00,598 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:59:00,603 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:59:01,138 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:59:03,141 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:59:04,560 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:59:04,561 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:59:04,561 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:59:04,566 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:59:05,142 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:59:06,566 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:59:07,145 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:59:08,371 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:59:08,372 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:59:08,372 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:59:08,376 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:59:09,147 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:59:11,150 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:59:12,305 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:59:12,306 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:59:12,306 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:59:12,306 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:59:12,311 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:59:13,152 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:59:15,094 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:59:15,094 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:59:15,094 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:59:15,155 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:59:16,233 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:59:16,234 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:59:16,234 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:59:16,239 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:59:17,156 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:59:18,277 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:59:19,159 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:59:23,277 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:59:24,357 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:59:24,361 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:59:24,362 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:59:24,366 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:59:25,163 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:59:25,965 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 19:59:27,166 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:59:28,091 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:59:28,092 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:59:28,092 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:59:28,097 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:59:28,166 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:59:29,098 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:59:30,094 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:59:30,094 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:59:30,094 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:59:31,170 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:59:32,592 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:59:32,593 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:59:32,593 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:59:32,598 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:59:33,171 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:59:34,637 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:59:35,175 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:59:36,772 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:59:36,773 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:59:36,773 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:59:36,778 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:59:37,176 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:59:39,179 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:59:39,779 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:59:40,689 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:59:40,689 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:59:40,690 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:59:40,695 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:59:41,180 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:59:43,183 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:59:45,094 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:59:45,094 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:59:45,095 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:59:45,095 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:59:45,805 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:59:45,806 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:59:45,806 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:59:45,811 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:59:46,185 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:59:47,188 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:59:49,207 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:59:49,208 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:59:49,208 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:59:49,214 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:59:50,190 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:59:50,215 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:59:51,193 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 19:59:54,425 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:59:54,426 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 19:59:54,426 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:59:54,431 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:59:55,195 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 19:59:55,431 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:59:55,967 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 19:59:57,198 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:00:00,094 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:00:00,094 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:00:00,095 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:00:01,358 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:00:02,790 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:00:02,791 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:00:02,791 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:00:02,796 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:00:03,202 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:00:05,205 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:00:06,833 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:00:08,950 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:00:08,951 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:00:08,952 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:00:08,956 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:00:09,208 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:00:11,212 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:00:11,957 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:00:13,224 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:00:13,225 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:00:13,225 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:00:13,230 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:00:14,213 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:00:15,094 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:00:15,094 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:00:15,095 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:00:15,216 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:00:17,001 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:00:17,001 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:00:17,002 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:00:17,002 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:00:17,007 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:00:17,217 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:00:19,220 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:00:22,045 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:00:22,968 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:00:22,969 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:00:22,969 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:00:22,974 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:00:23,223 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:00:25,226 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:00:25,973 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 20:00:26,976 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:00:26,976 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:00:26,977 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:00:26,981 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:00:27,227 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:00:27,982 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:00:29,230 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:00:30,094 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:00:30,094 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:00:30,095 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:00:30,408 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:00:30,409 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:00:30,409 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:00:30,414 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:00:31,231 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:00:33,234 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:00:33,453 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:00:38,453 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:00:38,775 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:00:38,776 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:00:38,776 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:00:38,782 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:00:39,237 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:00:41,240 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:00:42,985 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:00:42,986 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:00:42,986 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:00:42,991 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:00:43,241 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:00:43,991 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:00:45,094 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:00:45,094 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:00:45,095 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:00:45,244 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:00:49,040 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:00:49,041 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:00:49,041 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:00:49,041 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:00:49,046 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:00:49,247 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:00:51,251 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:00:54,085 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:00:55,972 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 20:00:57,389 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:00:57,390 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:00:57,390 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:00:57,395 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:00:58,255 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:00:59,258 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:00:59,395 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:01:00,094 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:01:00,094 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:01:00,095 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:01:02,604 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:01:02,604 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:01:02,604 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:01:02,609 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:01:03,260 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:01:04,649 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:01:05,264 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:01:06,442 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:01:06,443 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:01:06,443 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:01:06,449 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:01:07,265 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:01:09,269 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:01:10,449 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:01:11,674 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:01:11,675 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:01:11,675 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:01:11,680 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:01:12,272 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:01:13,274 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:01:15,094 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:01:15,095 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:01:15,095 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:01:16,392 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:01:18,776 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:01:18,776 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:01:18,777 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:01:18,781 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:01:19,278 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:01:21,281 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:01:21,821 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:01:25,976 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 20:01:26,978 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:01:27,048 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:01:27,049 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:01:27,049 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:01:27,054 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:01:27,285 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:01:29,288 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:01:30,094 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:01:30,095 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:01:30,095 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:01:32,250 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:01:32,250 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:01:32,251 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:01:32,251 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:01:32,255 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:01:32,289 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:01:35,293 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:01:36,114 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:01:36,115 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:01:36,115 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:01:36,120 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:01:36,294 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:01:38,120 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:01:39,297 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:01:42,529 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:01:42,530 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:01:42,530 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:01:42,535 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:01:43,300 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:01:43,536 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:01:45,094 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:01:45,095 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:01:45,095 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:01:45,303 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:01:46,130 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:01:46,131 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:01:46,131 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:01:46,136 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:01:46,304 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:01:49,173 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:01:49,305 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:01:49,670 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:01:49,671 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:01:49,671 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:01:49,676 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:01:50,308 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:01:51,311 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:01:54,677 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:01:55,292 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:01:55,293 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:01:55,293 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:01:55,298 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:01:55,314 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:01:55,976 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 20:01:57,317 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:01:59,977 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:02:00,095 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:02:00,095 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:02:00,095 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:02:01,225 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:02:01,225 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:02:01,225 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:02:01,230 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:02:01,319 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:02:03,322 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:02:05,269 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:02:07,387 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:02:07,387 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:02:07,387 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:02:07,392 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:02:08,326 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:02:09,329 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:02:10,393 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:02:11,802 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:02:11,803 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:02:11,803 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:02:11,809 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:02:12,331 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:02:13,333 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:02:15,095 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:02:15,095 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:02:15,096 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:02:16,357 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:02:21,013 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:02:21,014 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:02:21,014 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:02:21,019 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:02:21,338 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:02:22,057 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:02:23,342 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:02:25,978 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 20:02:27,979 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:02:29,748 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:02:29,748 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:02:29,748 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:02:29,753 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:02:30,095 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:02:30,095 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:02:30,096 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:02:30,346 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:02:31,348 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:02:33,405 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:02:35,528 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:02:35,528 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:02:35,528 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:02:35,533 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:02:36,351 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:02:37,354 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:02:38,573 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:02:39,820 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:02:39,821 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:02:39,821 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:02:39,826 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:02:40,355 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:02:41,358 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:02:43,826 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:02:44,840 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:02:44,841 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:02:44,841 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:02:44,845 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:02:45,095 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:02:45,095 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:02:45,096 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:02:45,360 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:02:47,364 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:02:49,245 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:02:49,245 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:02:49,246 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:02:49,246 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:02:49,250 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:02:49,365 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:02:51,370 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:02:54,289 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:02:54,977 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:02:54,978 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:02:54,978 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:02:54,983 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:02:55,372 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:02:55,979 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 20:02:57,375 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:02:58,581 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:02:58,582 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:02:58,582 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:02:58,587 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:02:59,377 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:02:59,587 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:03:00,095 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:03:00,095 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:03:00,095 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:03:01,380 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:03:05,391 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:03:06,567 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:03:06,568 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:03:06,568 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:03:06,573 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:03:07,384 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:03:09,388 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:03:10,574 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:03:11,977 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:03:11,978 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:03:11,978 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:03:11,983 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:03:12,390 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:03:13,392 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:03:15,095 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:03:15,095 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:03:15,096 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:03:16,243 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:03:16,244 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:03:16,244 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:03:16,244 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:03:16,249 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:03:16,394 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:03:19,398 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:03:19,769 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:03:19,770 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:03:19,770 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:03:19,775 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:03:20,399 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:03:21,401 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:03:21,775 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:03:23,511 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:03:23,512 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:03:23,512 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:03:23,517 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:03:24,404 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:03:25,406 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:03:25,981 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 20:03:26,983 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:03:29,728 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:03:29,729 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:03:29,729 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:03:29,734 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:03:30,095 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:03:30,096 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:03:30,096 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:03:30,410 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:03:31,412 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:03:32,392 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:03:35,890 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:03:35,891 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:03:35,891 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:03:35,896 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:03:36,415 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:03:37,420 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:03:37,933 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:03:40,094 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:03:40,095 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:03:40,095 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:03:40,101 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:03:40,422 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:03:41,422 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:03:43,101 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:03:45,095 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:03:45,096 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:03:45,096 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:03:45,986 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:03:45,986 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:03:45,986 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:03:45,991 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:03:46,426 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:03:47,428 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:03:49,029 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:03:51,138 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:03:51,139 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:03:51,139 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:03:51,144 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:03:51,432 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:03:53,435 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:03:54,144 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:03:55,987 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 20:03:57,745 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:03:57,746 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:03:57,746 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:03:57,751 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:03:58,438 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:03:59,441 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:03:59,751 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:04:00,095 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:04:00,096 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:04:00,096 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:04:03,102 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:04:03,103 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:04:03,103 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:04:03,108 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:04:03,443 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:04:05,145 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:04:05,446 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:04:06,728 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:04:06,729 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:04:06,729 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:04:06,733 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:04:07,448 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:04:09,452 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:04:10,734 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:04:15,095 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:04:15,096 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:04:15,096 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:04:15,201 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:04:15,326 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:04:15,327 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:04:15,332 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:04:15,456 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:04:16,332 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:04:17,459 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:04:20,593 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:04:20,593 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:04:20,594 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:04:20,599 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:04:21,461 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:04:21,599 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:04:23,465 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:04:25,593 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:04:25,593 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:04:25,594 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:04:25,598 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:04:25,986 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 20:04:26,467 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:04:26,987 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:04:27,469 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:04:30,095 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:04:30,096 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:04:30,096 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:04:30,274 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:04:30,367 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:04:30,367 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:04:30,372 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:04:30,471 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:04:32,372 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:04:33,475 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:04:34,585 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:04:34,586 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:04:34,586 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:04:34,591 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:04:35,476 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:04:37,478 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:04:37,592 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:04:39,088 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:04:39,088 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:04:39,089 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:04:39,093 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:04:39,479 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:04:41,482 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:04:42,878 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:04:42,879 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:04:42,879 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:04:42,879 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:04:42,884 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:04:43,484 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:04:45,096 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:04:45,096 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:04:45,096 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:04:45,487 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:04:48,426 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:04:49,593 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:04:49,593 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:04:49,593 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:04:49,598 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:04:50,492 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:04:51,494 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:04:53,637 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:04:54,536 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:04:54,536 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:04:54,537 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:04:54,542 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:04:55,497 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:04:55,991 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 20:04:57,500 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:04:58,714 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:04:58,715 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:04:58,715 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:04:58,715 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:04:58,721 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:04:59,501 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:05:00,096 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:05:00,096 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:05:00,096 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:05:01,504 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:05:02,873 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:05:02,874 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:05:02,874 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:05:02,879 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:05:03,505 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:05:03,917 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:05:05,509 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:05:07,833 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:05:07,834 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:05:07,834 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:05:07,839 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:05:08,512 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:05:09,514 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:05:09,840 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:05:12,193 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:05:12,194 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:05:12,194 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:05:12,199 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:05:12,516 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:05:13,519 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:05:15,096 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:05:15,096 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:05:15,096 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:05:15,383 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:05:18,019 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:05:18,019 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:05:18,019 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:05:18,024 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:05:18,522 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:05:19,524 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:05:21,061 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:05:23,170 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:05:23,170 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:05:23,171 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:05:23,175 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:05:23,527 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:05:25,530 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:05:25,990 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 20:05:26,991 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:05:27,647 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:05:27,647 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:05:27,648 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:05:27,653 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:05:28,532 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:05:29,535 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:05:30,096 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:05:30,096 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:05:30,097 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:05:31,828 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:05:31,829 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:05:31,829 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:05:31,834 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:05:32,537 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:05:32,873 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:05:33,540 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:05:37,873 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:05:38,006 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:05:38,007 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:05:38,007 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:05:38,012 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:05:38,542 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:05:39,545 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:05:41,778 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:05:41,779 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:05:41,779 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:05:41,784 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:05:42,547 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:05:43,548 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:05:43,784 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:05:45,096 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:05:45,096 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:05:45,097 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:05:45,852 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:05:45,853 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:05:45,853 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:05:45,859 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:05:46,550 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:05:47,552 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:05:48,897 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:05:49,656 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:05:49,657 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:05:49,657 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:05:49,662 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:05:50,555 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:05:51,558 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:05:54,102 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:05:54,103 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:05:54,103 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:05:54,103 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:05:54,108 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:05:54,560 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:05:55,563 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:05:55,995 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 20:05:58,306 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:05:58,307 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:05:58,307 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:05:58,312 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:05:58,565 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:05:59,312 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:05:59,565 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:06:00,096 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:06:00,096 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:06:00,097 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:06:02,791 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:06:02,791 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:06:02,791 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:06:02,796 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:06:03,568 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:06:04,833 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:06:05,571 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:06:06,788 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:06:06,789 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:06:06,789 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:06:06,795 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:06:07,572 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:06:09,577 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:06:10,795 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:06:11,667 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:06:11,668 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:06:11,668 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:06:11,673 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:06:12,578 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:06:13,581 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:06:15,096 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:06:15,096 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:06:15,097 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:06:16,311 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:06:17,059 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:06:17,059 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:06:17,059 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:06:17,064 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:06:17,584 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:06:19,587 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:06:21,385 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:06:21,386 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:06:21,386 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:06:21,386 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:06:21,391 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:06:21,589 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:06:23,594 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:06:25,994 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 20:06:26,995 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:06:27,005 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:06:27,006 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:06:27,006 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:06:27,011 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:06:27,597 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:06:29,600 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:06:30,096 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:06:30,097 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:06:30,097 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:06:31,579 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:06:31,580 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:06:31,580 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:06:31,585 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:06:31,601 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:06:32,625 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:06:33,604 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:06:35,560 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:06:35,561 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:06:35,561 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:06:35,566 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:06:35,605 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:06:37,608 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:06:38,567 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:06:39,508 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:06:39,509 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:06:39,509 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:06:39,514 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:06:39,610 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:06:41,613 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:06:44,514 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:06:45,096 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:06:45,097 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:06:45,097 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:06:45,484 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:06:45,484 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:06:45,485 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:06:45,490 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:06:45,615 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:06:47,619 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:06:49,529 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:06:51,033 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:06:51,034 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:06:51,034 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:06:51,042 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:06:51,623 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:06:53,626 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:06:55,043 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:06:55,448 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:06:55,449 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:06:55,449 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:06:55,455 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:06:55,627 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:06:55,999 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 20:06:57,631 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:06:59,272 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:06:59,273 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:06:59,273 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:06:59,278 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:06:59,632 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:07:00,097 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:07:00,098 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:07:00,098 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:07:00,399 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:07:01,635 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:07:04,676 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:07:04,676 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:07:04,676 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:07:04,681 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:07:05,637 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:07:05,682 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:07:07,640 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:07:09,337 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:07:09,338 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:07:09,338 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:07:09,343 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:07:09,643 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:07:11,343 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:07:11,646 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:07:14,603 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:07:14,604 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:07:14,604 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:07:14,609 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:07:14,648 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:07:15,097 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:07:15,097 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:07:15,098 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:07:16,395 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:07:17,652 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:07:18,765 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:07:18,766 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:07:18,766 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:07:18,771 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:07:19,653 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:07:21,654 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:07:21,809 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:07:22,885 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:07:22,886 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:07:22,886 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:07:22,892 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:07:23,656 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:07:25,659 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:07:25,998 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 20:07:27,000 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:07:27,119 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:07:27,119 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:07:27,119 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:07:27,124 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:07:27,660 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:07:29,664 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:07:30,097 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:07:30,097 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:07:30,098 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:07:31,070 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:07:31,071 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:07:31,071 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:07:31,076 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:07:31,665 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:07:32,113 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:07:33,668 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:07:34,451 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:07:34,451 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:07:34,452 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:07:34,457 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:07:34,669 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:07:37,457 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:07:37,670 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:07:38,943 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:07:38,944 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:07:38,944 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:07:38,949 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:07:39,672 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:07:41,675 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:07:42,949 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:07:43,582 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:07:43,583 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:07:43,583 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:07:43,588 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:07:43,676 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:07:45,097 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:07:45,097 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:07:45,098 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:07:45,679 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:07:47,561 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:07:47,561 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:07:47,561 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:07:47,567 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:07:47,681 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:07:48,605 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:07:49,685 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:07:52,214 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:07:52,215 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:07:52,215 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:07:52,220 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:07:52,688 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:07:53,690 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:07:54,221 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:07:56,000 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 20:07:57,172 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:07:57,172 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:07:57,173 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:07:57,178 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:07:57,693 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:07:59,694 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:08:00,097 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:08:00,098 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:08:00,098 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:08:00,098 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:08:02,027 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:08:02,030 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:08:02,030 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:08:02,036 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:08:02,696 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:08:03,699 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:08:06,069 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:08:07,740 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:08:07,741 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:08:07,741 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:08:07,747 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:08:08,703 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:08:09,706 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:08:11,588 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:08:11,589 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:08:11,589 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:08:11,589 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:08:11,594 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:08:11,707 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:08:13,710 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:08:15,097 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:08:15,098 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:08:15,098 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:08:15,309 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:08:15,359 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:08:15,359 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:08:15,364 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:08:15,711 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:08:17,364 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:08:17,715 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:08:19,379 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:08:19,379 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:08:19,379 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:08:19,384 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:08:19,716 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:08:21,719 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:08:22,421 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:08:23,388 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:08:23,389 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:08:23,390 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:08:23,394 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:08:23,721 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:08:25,724 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:08:26,002 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 20:08:26,974 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:08:26,975 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:08:26,975 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:08:26,979 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:08:27,725 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:08:27,980 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:08:29,729 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:08:30,097 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:08:30,098 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:08:30,098 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:08:32,006 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:08:32,007 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:08:32,007 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:08:32,012 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:08:32,730 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:08:33,049 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:08:33,733 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:08:38,049 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:08:40,207 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:08:40,208 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:08:40,208 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:08:40,213 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:08:40,737 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:08:41,740 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:08:43,213 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:08:44,590 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:08:44,591 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:08:44,591 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:08:44,596 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:08:44,741 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:08:45,097 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:08:45,098 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:08:45,098 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:08:47,745 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:08:48,174 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:08:48,175 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:08:48,175 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:08:48,180 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:08:48,217 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:08:48,746 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:08:49,748 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:08:53,217 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:08:56,004 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 20:08:57,202 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:08:57,203 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:08:57,203 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:08:57,208 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:08:57,754 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:08:59,208 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:08:59,757 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:09:00,097 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:09:00,098 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:09:00,098 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:09:02,457 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:09:02,458 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:09:02,458 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:09:02,463 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:09:02,759 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:09:03,761 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:09:04,501 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:09:05,809 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:09:05,810 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:09:05,810 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:09:05,815 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:09:06,763 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:09:07,766 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:09:09,816 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:09:10,192 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:09:10,192 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:09:10,193 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:09:10,198 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:09:10,769 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:09:11,771 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:09:14,042 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:09:14,045 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:09:14,046 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:09:14,050 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:09:14,773 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:09:15,051 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:09:15,098 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:09:15,098 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:09:15,098 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:09:15,774 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:09:17,818 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:09:17,819 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:09:17,819 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:09:17,824 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:09:18,776 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:09:19,778 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:09:20,861 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:09:25,407 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:09:25,408 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:09:25,408 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:09:25,414 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:09:25,782 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:09:26,007 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 20:09:26,008 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:09:27,785 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:09:30,024 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:09:30,025 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:09:30,025 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:09:30,030 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:09:30,098 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:09:30,098 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:09:30,099 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:09:30,787 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:09:31,376 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:09:31,787 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:09:34,874 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:09:34,875 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:09:34,875 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:09:34,881 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:09:35,790 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:09:36,881 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:09:37,793 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:09:38,479 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:09:38,480 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:09:38,480 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:09:38,485 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:09:38,794 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:09:39,796 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:09:42,486 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:09:43,017 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:09:43,018 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:09:43,018 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:09:43,024 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:09:43,798 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:09:45,098 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:09:45,098 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:09:45,099 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:09:45,802 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:09:47,121 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:09:47,122 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:09:47,122 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:09:47,127 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:09:47,803 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:09:48,165 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:09:49,806 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:09:50,886 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:09:50,887 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:09:50,887 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:09:50,892 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:09:51,809 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:09:53,812 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:09:53,892 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:09:54,992 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:09:54,993 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:09:54,993 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:09:54,998 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:09:55,813 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:09:56,009 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 20:09:57,816 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:09:59,010 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:09:59,832 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:09:59,833 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:09:59,833 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:09:59,838 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:10:00,098 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:10:00,098 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:10:00,099 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:10:00,818 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:10:01,820 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:10:03,584 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:10:03,585 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:10:03,585 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:10:03,590 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:10:03,822 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:10:04,629 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:10:05,825 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:10:07,185 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:10:07,185 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:10:07,185 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:10:07,190 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:10:07,826 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:10:09,829 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:10:10,191 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:10:11,096 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:10:11,096 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:10:11,096 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:10:11,101 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:10:11,830 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:10:13,833 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:10:15,098 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:10:15,098 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:10:15,099 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:10:15,149 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:10:15,309 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:10:15,309 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:10:15,309 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:10:15,316 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:10:15,834 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:10:17,838 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:10:20,316 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:10:21,000 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:10:21,004 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:10:21,004 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:10:21,009 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:10:21,840 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:10:23,843 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:10:26,011 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:10:26,011 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:10:26,012 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 20:10:26,013 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:10:26,013 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:10:26,017 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:10:26,845 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:10:27,848 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:10:30,098 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:10:30,098 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:10:30,099 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:10:31,080 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:10:31,081 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:10:31,081 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:10:31,081 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:10:31,086 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:10:31,850 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:10:33,853 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:10:34,677 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:10:34,678 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:10:34,678 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:10:34,683 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:10:34,854 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:10:36,683 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:10:37,857 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:10:38,968 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:10:38,969 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:10:38,969 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:10:38,974 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:10:39,858 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:10:41,860 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:10:41,974 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:10:42,970 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:10:42,970 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:10:42,971 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:10:42,976 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:10:43,861 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:10:45,098 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:10:45,099 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:10:45,099 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:10:45,864 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:10:47,342 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:10:51,360 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:10:51,360 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:10:51,361 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:10:51,365 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:10:51,869 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:10:52,405 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:10:53,872 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:10:56,013 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 20:10:56,221 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:10:56,221 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:10:56,221 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:10:56,226 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:10:56,874 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:10:57,877 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:10:58,227 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:11:00,098 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:11:00,099 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:11:00,099 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:11:00,552 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:11:00,553 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:11:00,553 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:11:00,558 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:11:00,878 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:11:01,881 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:11:03,597 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:11:06,554 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:11:06,554 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:11:06,554 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:11:06,559 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:11:06,884 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:11:07,886 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:11:09,560 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:11:11,118 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:11:11,118 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:11:11,118 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:11:11,124 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:11:11,890 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:11:13,893 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:11:15,098 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:11:15,099 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:11:15,099 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:11:15,346 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:11:19,190 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:11:19,191 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:11:19,191 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:11:19,196 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:11:19,896 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:11:21,233 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:11:21,899 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:11:23,159 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:11:23,160 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:11:23,160 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:11:23,165 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:11:23,901 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:11:25,904 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:11:26,018 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 20:11:27,019 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:11:28,765 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:11:28,766 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:11:28,766 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:11:28,771 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:11:28,906 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:11:30,098 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:11:30,099 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:11:30,099 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:11:31,909 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:11:32,301 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:11:35,905 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:11:35,905 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:11:35,906 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:11:35,910 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:11:35,911 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:11:37,913 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:11:37,949 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:11:40,771 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:11:40,772 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:11:40,772 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:11:40,777 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:11:40,914 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:11:43,778 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:11:43,918 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:11:45,099 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:11:45,099 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:11:45,099 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:11:47,037 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:11:47,037 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:11:47,038 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:11:47,042 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:11:47,921 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:11:49,043 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:11:49,924 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:11:51,574 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:11:51,575 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:11:51,575 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:11:51,579 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:11:51,926 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:11:53,929 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:11:54,580 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:11:55,827 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:11:55,828 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:11:55,828 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:11:55,833 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:11:55,931 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:11:56,017 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 20:11:57,934 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:12:00,018 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:12:00,099 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:12:00,099 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:12:00,099 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:12:01,032 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:12:01,032 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:12:01,033 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:12:01,037 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:12:01,936 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:12:03,939 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:12:04,836 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:12:04,837 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:12:04,837 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:12:04,842 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:12:04,940 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:12:05,843 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:12:07,944 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:12:09,084 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:12:09,085 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:12:09,085 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:12:09,091 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:12:09,946 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:12:11,092 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:12:11,949 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:12:13,052 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:12:13,053 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:12:13,053 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:12:13,058 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:12:13,950 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:12:15,099 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:12:15,099 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:12:15,099 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:12:15,956 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:12:16,324 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:12:18,356 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:12:18,356 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:12:18,356 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:12:18,361 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:12:18,958 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:12:19,961 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:12:21,401 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:12:22,295 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:12:22,296 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:12:22,296 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:12:22,301 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:12:22,963 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:12:23,966 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:12:26,022 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 20:12:27,023 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:12:27,054 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:12:27,055 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:12:27,055 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:12:27,060 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:12:27,968 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:12:29,971 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:12:30,099 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:12:30,099 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:12:30,100 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:12:30,996 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:12:30,997 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:12:30,997 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:12:31,002 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:12:31,972 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:12:32,041 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:12:33,975 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:12:35,422 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:12:35,423 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:12:35,423 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:12:35,428 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:12:35,977 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:12:37,428 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:12:37,980 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:12:39,952 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:12:39,953 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:12:39,953 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:12:39,958 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:12:39,981 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:12:41,984 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:12:42,959 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:12:43,753 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:12:43,754 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:12:43,754 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:12:43,759 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:12:43,985 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:12:45,099 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:12:45,099 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:12:45,100 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:12:45,988 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:12:48,299 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:12:52,092 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:12:52,092 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:12:52,093 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:12:52,097 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:12:52,993 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:12:53,996 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:12:54,137 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:12:56,021 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 20:12:57,016 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:12:57,017 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:12:57,017 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:12:57,026 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:12:57,998 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:13:00,001 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:13:00,026 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:13:00,099 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:13:00,100 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:13:00,100 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:13:02,908 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:13:02,909 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:13:02,909 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:13:02,914 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:13:03,002 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:13:05,915 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:13:06,006 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:13:07,548 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:13:07,549 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:13:07,549 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:13:07,554 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:13:08,007 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:13:10,012 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:13:11,555 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:13:14,430 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:13:14,431 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:13:14,431 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:13:14,436 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:13:15,014 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:13:15,099 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:13:15,100 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:13:15,100 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:13:16,017 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:13:17,345 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:13:17,941 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:13:17,941 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:13:17,942 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:13:17,947 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:13:18,018 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:13:20,021 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:13:22,058 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:13:22,059 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:13:22,059 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:13:22,064 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:13:23,023 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:13:23,064 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:13:24,026 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:13:26,026 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 20:13:26,920 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:13:26,920 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:13:26,921 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:13:26,925 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:13:27,028 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:13:28,926 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:13:30,032 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:13:30,099 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:13:30,100 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:13:30,100 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:13:32,536 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:13:32,537 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:13:32,537 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:13:32,542 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:13:33,034 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:13:34,036 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:13:34,581 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:13:36,395 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:13:36,395 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:13:36,396 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:13:36,401 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:13:37,038 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:13:38,041 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:13:40,260 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:13:40,261 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:13:40,261 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:13:40,261 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:13:40,266 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:13:41,042 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:13:42,045 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:13:43,981 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:13:43,983 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:13:43,983 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:13:43,988 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:13:44,046 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:13:45,099 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:13:45,100 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:13:45,100 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:13:45,352 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:13:46,047 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:13:47,762 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:13:47,763 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:13:47,763 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:13:47,768 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:13:48,049 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:13:50,052 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:13:50,805 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:13:51,761 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:13:51,762 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:13:51,762 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:13:51,767 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:13:52,054 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:13:54,057 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:13:56,025 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 20:13:56,026 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:13:56,757 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:13:56,757 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:13:56,758 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:13:56,762 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:13:57,059 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:13:58,061 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:14:00,099 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:14:00,100 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:14:00,100 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:14:00,486 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:14:00,486 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:14:00,487 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:14:00,492 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:14:01,063 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:14:01,529 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:14:02,066 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:14:04,349 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:14:04,350 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:14:04,350 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:14:04,355 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:14:05,068 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:14:06,070 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:14:07,355 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:14:09,177 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:14:09,177 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:14:09,178 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:14:09,183 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:14:10,074 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:14:12,077 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:14:13,184 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:14:13,511 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:14:13,512 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:14:13,512 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:14:13,517 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:14:14,078 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:14:15,100 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:14:15,100 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:14:15,100 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:14:16,081 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:14:17,282 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:14:17,283 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:14:17,283 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:14:17,288 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:14:18,082 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:14:18,325 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:14:20,085 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:14:22,497 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:14:22,498 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:14:22,498 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:14:22,502 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:14:23,087 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:14:23,503 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:14:24,090 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:14:26,027 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 20:14:26,233 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:14:26,234 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:14:26,234 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:14:26,239 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:14:27,091 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:14:28,111 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:14:29,239 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:14:30,100 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:14:30,100 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:14:30,100 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:14:34,264 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:14:34,265 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:14:34,265 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:14:34,266 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:14:34,270 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:14:35,115 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:14:36,118 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:14:37,874 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:14:37,875 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:14:37,875 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:14:37,880 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:14:38,119 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:14:39,881 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:14:40,120 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:14:41,947 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:14:41,948 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:14:41,948 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:14:41,953 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:14:42,121 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:14:44,124 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:14:44,954 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:14:45,100 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:14:45,100 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:14:45,100 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:14:45,754 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:14:45,755 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:14:45,755 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:14:45,764 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:14:46,126 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:14:48,129 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:14:50,198 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:14:50,198 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:14:50,199 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:14:50,199 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:14:50,204 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:14:51,132 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:14:52,134 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:14:53,771 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:14:53,772 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:14:53,772 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:14:53,777 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:14:54,135 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:14:55,778 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:14:56,029 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 20:14:56,139 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:14:58,176 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:14:58,177 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:14:58,177 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:14:58,182 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:14:59,141 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:15:00,100 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:15:00,100 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:15:00,101 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:15:00,144 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:15:01,349 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:15:02,882 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:15:02,883 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:15:02,883 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:15:02,888 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:15:03,145 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:15:04,146 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:15:06,641 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:15:06,642 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:15:06,642 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:15:06,642 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:15:06,647 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:15:07,147 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:15:08,150 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:15:11,648 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:15:11,649 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:15:11,650 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:15:11,650 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:15:11,655 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:15:12,154 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:15:14,157 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:15:15,100 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:15:15,100 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:15:15,101 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:15:15,823 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:15:15,823 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:15:15,824 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:15:15,829 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:15:16,158 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:15:16,865 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:15:18,161 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:15:20,160 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:15:20,161 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:15:20,161 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:15:20,166 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:15:21,163 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:15:22,163 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:15:22,166 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:15:24,568 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:15:24,568 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:15:24,569 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:15:24,573 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:15:25,165 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:15:26,032 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 20:15:26,168 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:15:28,033 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:15:28,987 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:15:28,988 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:15:28,988 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:15:28,993 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:15:29,169 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:15:30,100 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:15:30,100 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:15:30,101 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:15:32,173 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:15:32,988 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:15:32,988 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:15:32,988 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:15:32,994 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:15:33,036 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:15:33,174 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:15:36,177 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:15:38,036 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:15:40,573 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:15:40,573 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:15:40,574 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:15:40,578 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:15:41,180 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:15:42,182 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:15:43,579 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:15:45,100 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:15:45,100 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:15:45,101 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:15:49,391 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:15:51,103 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:15:51,104 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:15:51,104 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:15:51,109 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:15:51,189 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:15:54,192 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:15:55,149 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:15:56,034 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 20:15:56,053 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:15:56,054 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:15:56,054 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:15:56,059 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:15:56,194 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:15:58,197 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:16:00,100 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:16:00,101 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:16:00,101 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:16:00,416 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:16:01,938 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:16:01,938 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:16:01,938 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:16:01,943 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:16:02,199 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:16:04,202 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:16:05,981 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:16:06,546 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:16:06,547 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:16:06,547 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:16:06,552 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:16:07,204 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:16:08,206 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:16:10,937 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:16:10,938 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:16:10,938 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:16:10,943 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:16:11,209 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:16:11,944 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:16:12,212 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:16:15,100 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:16:15,101 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:16:15,101 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:16:16,052 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:16:16,053 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:16:16,053 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:16:16,058 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:16:16,214 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:16:17,097 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:16:18,217 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:16:19,940 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:16:19,941 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:16:19,941 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:16:19,946 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:16:20,221 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:16:22,224 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:16:22,947 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:16:23,770 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:16:23,770 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:16:23,771 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:16:23,775 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:16:24,226 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:16:26,036 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 20:16:26,229 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:16:27,519 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:16:27,520 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:16:27,520 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:16:27,525 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:16:28,230 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:16:28,525 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:16:30,100 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:16:30,101 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:16:30,101 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:16:30,233 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:16:31,714 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:16:31,715 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:16:31,715 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:16:31,720 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:16:32,234 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:16:33,757 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:16:34,237 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:16:36,416 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:16:36,416 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:16:36,417 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:16:36,422 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:16:37,239 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:16:38,241 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:16:39,422 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:16:40,884 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:16:40,885 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:16:40,885 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:16:40,890 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:16:41,246 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:16:42,249 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:16:44,891 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:16:45,100 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:16:45,101 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:16:45,101 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:16:47,522 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:16:47,523 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:16:47,523 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:16:47,528 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:16:48,253 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:16:50,254 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:16:50,528 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:16:51,373 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:16:51,374 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:16:51,374 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:16:51,379 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:16:52,256 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:16:54,259 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:16:56,038 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 20:16:56,039 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:16:57,319 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:16:57,319 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:16:57,319 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:16:57,324 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:16:58,261 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:17:00,101 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:17:00,101 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:17:00,101 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:17:00,264 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:17:01,315 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:17:05,614 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:17:05,615 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:17:05,615 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:17:05,620 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:17:06,267 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:17:06,657 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:17:08,271 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:17:09,637 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:17:09,637 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:17:09,637 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:17:09,643 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:17:10,273 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:17:12,274 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:17:12,643 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:17:15,101 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:17:15,101 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:17:15,101 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:17:17,925 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:17:17,925 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:17:17,926 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:17:17,926 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:17:17,931 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:17:18,277 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:17:20,281 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:17:21,612 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:17:21,613 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:17:21,613 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:17:21,619 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:17:22,282 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:17:23,619 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:17:24,286 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:17:26,040 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 20:17:26,374 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:17:26,374 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:17:26,374 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:17:26,379 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:17:27,287 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:17:28,290 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:17:29,380 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:17:30,101 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:17:30,101 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:17:30,101 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:17:30,431 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:17:30,432 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:17:30,432 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:17:30,437 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:17:31,292 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:17:32,294 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:17:34,473 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:17:34,766 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:17:34,767 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:17:34,767 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:17:34,772 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:17:35,296 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:17:36,299 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:17:39,773 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:17:41,678 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:17:41,678 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:17:41,679 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:17:41,683 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:17:42,302 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:17:44,305 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:17:45,101 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:17:45,101 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:17:45,102 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:17:45,332 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:17:46,987 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:17:46,987 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:17:46,987 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:17:46,992 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:17:47,307 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:17:48,310 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:17:51,029 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:17:51,171 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:17:51,171 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:17:51,172 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:17:51,176 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:17:51,312 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:17:54,316 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:17:56,045 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 20:17:56,046 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:17:57,750 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:17:57,751 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:17:57,751 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:17:57,756 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:17:58,319 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:18:00,101 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:18:00,101 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:18:00,102 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:18:00,322 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:18:01,358 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:18:02,314 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:18:02,315 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:18:02,315 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:18:02,320 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:18:02,323 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:18:04,326 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:18:06,371 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:18:06,371 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:18:06,372 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:18:06,372 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:18:06,376 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:18:07,328 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:18:08,330 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:18:11,377 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:18:11,534 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:18:11,535 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:18:11,535 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:18:11,540 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:18:12,334 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:18:14,337 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:18:15,101 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:18:15,101 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:18:15,102 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:18:15,331 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:18:15,378 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:18:15,378 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:18:15,383 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:18:16,338 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:18:16,384 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:18:18,341 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:18:19,390 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:18:19,391 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:18:19,391 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:18:19,395 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:18:20,342 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:18:21,433 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:18:22,346 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:18:23,862 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:18:23,863 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:18:23,863 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:18:23,868 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:18:24,347 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:18:26,044 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 20:18:26,350 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:18:27,045 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:18:27,460 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:18:27,460 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:18:27,460 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:18:27,466 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:18:28,352 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:18:30,101 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:18:30,101 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:18:30,102 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:18:30,355 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:18:32,309 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:18:32,636 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:18:32,637 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:18:32,637 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:18:32,642 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:18:33,356 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:18:34,359 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:18:37,678 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:18:37,679 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:18:37,679 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:18:37,679 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:18:37,684 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:18:38,361 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:18:40,364 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:18:41,568 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:18:41,569 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:18:41,569 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:18:41,574 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:18:42,365 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:18:43,575 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:18:44,368 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:18:45,101 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:18:45,102 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:18:45,102 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:18:46,903 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:18:46,904 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:18:46,904 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:18:46,908 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:18:47,371 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:18:48,373 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:18:48,945 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:18:52,891 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:18:52,892 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:18:52,892 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:18:52,897 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:18:53,377 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:18:54,378 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:18:54,897 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:18:56,048 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 20:19:00,050 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:19:00,101 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:19:00,102 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:19:00,102 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:19:01,276 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:19:01,277 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:19:01,277 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:19:01,282 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:19:01,381 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:19:04,385 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:19:05,321 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:19:05,813 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:19:05,814 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:19:05,814 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:19:05,819 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:19:06,386 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:19:08,389 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:19:10,819 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:19:14,058 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:19:14,058 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:19:14,059 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:19:14,063 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:19:14,394 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:19:15,101 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:19:15,102 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:19:15,102 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:19:16,337 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:19:16,395 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:19:18,044 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:19:18,044 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:19:18,045 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:19:18,050 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:19:18,396 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:19:20,400 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:19:21,872 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:19:21,873 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:19:21,873 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:19:21,873 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:19:21,878 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:19:22,401 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:19:24,404 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:19:25,950 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:19:25,951 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:19:25,951 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:19:25,956 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:19:26,049 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 20:19:26,405 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:19:27,050 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:19:28,408 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:19:30,102 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:19:30,102 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:19:30,102 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:19:31,686 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:19:31,687 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:19:31,687 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:19:31,692 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:19:32,411 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:19:32,729 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:19:34,415 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:19:35,723 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:19:35,724 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:19:35,724 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:19:35,729 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:19:36,416 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:19:37,730 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:19:38,417 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:19:42,350 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:19:42,351 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:19:42,351 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:19:42,356 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:19:42,420 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:19:43,357 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:19:44,423 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:19:45,102 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:19:45,102 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:19:45,102 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:19:46,635 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:19:46,635 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:19:46,635 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:19:46,640 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:19:47,425 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:19:48,428 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:19:48,677 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:19:50,855 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:19:50,856 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:19:50,856 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:19:50,861 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:19:51,431 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:19:52,433 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:19:53,861 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:19:54,929 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:19:54,930 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:19:54,930 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:19:54,935 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:19:55,435 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:19:56,054 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 20:19:56,435 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:19:59,055 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:19:59,263 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:19:59,264 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:19:59,264 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:19:59,269 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:19:59,437 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:20:00,102 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:20:00,102 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:20:00,102 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:20:00,440 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:20:03,494 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:20:03,494 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:20:03,495 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:20:03,500 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:20:04,442 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:20:04,537 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:20:06,445 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:20:08,238 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:20:08,239 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:20:08,239 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:20:08,244 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:20:08,446 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:20:10,245 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:20:10,451 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:20:11,817 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:20:11,818 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:20:11,818 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:20:11,823 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:20:12,452 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:20:14,455 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:20:15,102 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:20:15,102 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:20:15,103 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:20:15,323 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:20:20,299 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:20:20,299 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:20:20,299 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:20:20,305 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:20:20,341 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:20:20,458 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:20:22,461 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:20:24,119 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:20:24,120 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:20:24,120 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:20:24,125 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:20:24,463 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:20:26,053 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 20:20:26,054 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:20:26,466 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:20:28,232 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:20:28,233 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:20:28,233 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:20:28,238 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:20:28,467 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:20:30,102 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:20:30,102 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:20:30,103 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:20:30,470 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:20:31,322 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:20:33,729 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:20:33,730 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:20:33,730 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:20:33,735 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:20:34,472 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:20:36,474 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:20:36,773 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:20:37,813 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:20:37,814 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:20:37,814 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:20:37,819 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:20:38,475 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:20:40,478 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:20:41,819 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:20:42,122 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:20:42,123 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:20:42,123 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:20:42,128 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:20:42,480 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:20:44,483 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:20:45,102 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:20:45,102 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:20:45,103 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:20:46,586 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:20:46,586 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:20:46,586 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:20:46,591 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:20:47,485 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:20:47,629 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:20:48,488 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:20:50,988 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:20:50,989 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:20:50,989 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:20:50,994 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:20:51,491 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:20:52,493 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:20:52,994 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:20:56,055 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 20:20:56,290 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:20:56,291 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:20:56,291 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:20:56,296 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:20:56,496 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:20:58,296 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:20:58,497 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:21:00,102 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:21:00,102 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:21:00,103 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:21:02,796 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:21:02,797 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:21:02,797 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:21:02,802 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:21:03,499 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:21:03,841 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:21:04,502 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:21:08,841 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:21:11,282 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:21:11,283 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:21:11,283 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:21:11,288 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:21:11,508 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:21:12,510 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:21:14,288 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:21:14,895 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:21:14,895 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:21:14,896 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:21:14,901 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:21:15,102 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:21:15,103 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:21:15,103 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:21:15,512 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:21:16,515 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:21:19,101 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:21:19,102 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:21:19,102 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:21:19,107 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:21:19,516 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:21:20,145 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:21:20,519 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:21:23,239 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:21:23,239 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:21:23,240 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:21:23,245 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:21:23,521 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:21:24,523 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:21:25,245 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:21:26,057 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 20:21:28,354 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:21:28,355 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:21:28,355 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:21:28,360 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:21:28,526 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:21:30,102 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:21:30,103 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:21:30,103 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:21:30,351 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:21:30,527 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:21:32,492 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:21:32,492 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:21:32,493 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:21:32,498 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:21:32,528 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:21:34,531 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:21:35,537 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:21:40,537 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:21:40,886 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:21:40,887 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:21:40,887 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:21:40,892 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:21:41,535 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:21:42,538 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:21:45,022 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:21:45,022 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:21:45,023 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:21:45,027 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:21:45,102 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:21:45,103 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:21:45,103 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:21:45,539 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:21:46,351 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:21:46,542 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:21:48,991 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:21:48,992 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:21:48,992 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:21:48,997 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:21:49,544 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:21:50,547 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:21:52,037 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:21:56,059 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 20:21:57,060 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:21:57,407 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:21:57,407 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:21:57,408 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:21:57,412 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:21:57,552 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:21:58,554 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:22:00,102 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:22:00,103 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:22:00,103 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:22:02,364 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:22:03,461 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:22:03,462 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:22:03,462 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:22:03,467 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:22:03,557 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:22:06,561 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:22:07,505 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:22:08,344 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:22:08,345 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:22:08,345 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:22:08,351 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:22:08,562 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:22:10,566 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:22:13,215 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:22:13,215 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:22:13,216 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:22:13,216 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:22:13,221 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:22:13,568 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:22:14,571 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:22:15,103 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:22:15,103 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:22:15,103 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:22:17,691 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:22:17,691 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:22:17,692 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:22:17,696 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:22:18,573 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:22:18,733 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:22:20,574 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:22:22,541 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:22:22,542 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:22:22,542 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:22:22,547 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:22:22,575 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:22:24,547 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:22:24,579 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:22:26,064 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 20:22:30,066 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:22:30,103 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:22:30,103 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:22:30,103 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:22:30,949 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:22:30,949 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:22:30,950 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:22:30,955 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:22:31,583 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:22:32,585 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:22:35,645 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:22:35,646 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:22:35,646 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:22:35,646 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:22:35,651 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:22:36,588 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:22:38,591 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:22:40,150 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:22:40,151 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:22:40,152 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:22:40,157 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:22:40,592 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:22:41,157 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:22:42,595 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:22:45,103 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:22:45,103 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:22:45,103 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:22:45,413 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:22:45,414 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:22:45,414 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:22:45,419 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:22:45,597 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:22:46,457 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:22:46,599 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:22:50,068 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:22:50,069 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:22:50,069 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:22:50,074 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:22:50,602 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:22:52,075 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:22:52,606 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:22:53,701 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:22:53,702 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:22:53,703 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:22:53,708 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:22:54,608 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:22:56,063 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 20:22:56,611 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:22:58,065 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:22:58,912 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:22:58,913 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:22:58,913 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:22:58,919 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:22:59,612 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:23:00,103 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:23:00,103 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:23:00,104 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:23:00,615 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:23:02,556 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:23:02,556 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:23:02,557 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:23:02,562 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:23:02,616 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:23:03,601 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:23:04,619 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:23:08,197 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:23:08,198 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:23:08,198 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:23:08,204 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:23:08,622 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:23:09,204 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:23:10,626 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:23:13,430 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:23:13,431 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:23:13,431 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:23:13,436 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:23:13,628 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:23:14,437 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:23:14,630 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:23:15,103 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:23:15,103 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:23:15,104 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:23:17,821 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:23:17,822 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:23:17,822 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:23:17,827 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:23:18,632 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:23:19,865 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:23:20,636 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:23:23,325 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:23:23,326 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:23:23,326 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:23:23,331 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:23:23,638 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:23:24,638 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:23:25,331 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:23:26,068 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 20:23:27,953 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:23:27,954 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:23:27,954 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:23:27,959 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:23:28,641 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:23:30,103 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:23:30,103 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:23:30,104 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:23:30,402 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:23:30,644 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:23:32,642 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:23:32,643 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:23:32,643 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:23:32,648 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:23:32,648 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:23:34,651 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:23:35,685 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:23:36,606 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:23:36,607 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:23:36,607 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:23:36,612 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:23:36,652 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:23:38,655 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:23:41,613 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:23:42,140 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:23:42,140 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:23:42,141 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:23:42,146 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:23:42,658 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:23:44,661 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:23:45,103 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:23:45,103 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:23:45,104 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:23:46,728 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:23:46,728 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:23:46,728 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:23:46,728 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:23:46,733 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:23:47,663 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:23:48,666 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:23:51,773 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:23:53,639 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:23:53,640 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:23:53,640 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:23:53,645 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:23:53,670 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:23:56,067 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 20:23:56,674 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:23:57,069 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:23:57,805 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:23:57,806 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:23:57,806 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:23:57,811 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:23:58,675 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:24:00,103 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:24:00,104 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:24:00,104 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:24:00,678 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:24:02,309 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:24:02,728 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:24:02,729 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:24:02,729 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:24:02,734 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:24:03,680 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:24:04,682 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:24:06,551 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:24:06,552 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:24:06,552 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:24:06,558 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:24:06,683 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:24:07,558 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:24:08,687 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:24:10,466 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:24:10,467 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:24:10,467 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:24:10,472 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:24:10,689 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:24:12,692 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:24:13,474 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:24:14,493 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:24:14,494 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:24:14,494 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:24:14,499 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:24:14,693 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:24:15,103 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:24:15,104 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:24:15,104 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:24:16,696 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:24:19,118 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:24:19,118 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:24:19,118 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:24:19,119 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:24:19,123 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:24:19,698 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:24:20,700 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:24:22,789 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:24:22,790 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:24:22,790 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:24:22,795 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:24:23,703 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:24:24,703 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:24:24,795 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:24:26,072 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 20:24:27,213 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:24:27,214 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:24:27,214 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:24:27,219 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:24:27,705 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:24:28,707 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:24:30,104 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:24:30,104 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:24:30,104 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:24:30,361 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:24:32,919 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:24:32,919 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:24:32,919 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:24:32,924 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:24:33,710 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:24:34,713 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:24:35,961 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:24:39,685 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:24:39,686 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:24:39,686 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:24:39,691 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:24:39,716 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:24:41,692 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:24:42,719 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:24:44,372 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:24:44,373 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:24:44,373 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:24:44,378 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:24:44,720 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:24:45,103 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:24:45,104 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:24:45,104 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:24:46,724 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:24:47,356 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:24:48,700 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:24:48,700 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:24:48,701 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:24:48,705 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:24:48,725 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:24:50,729 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:24:52,745 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:24:54,385 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:24:54,385 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:24:54,386 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:24:54,391 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:24:54,732 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:24:56,072 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 20:24:56,735 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:24:58,073 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:24:58,533 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:24:58,534 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:24:58,534 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:24:58,539 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:24:58,736 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:25:00,104 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:25:00,104 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:25:00,104 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:25:00,739 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:25:03,369 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:25:08,192 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:25:08,192 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:25:08,192 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:25:08,197 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:25:08,744 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:25:09,237 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:25:10,748 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:25:11,520 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:25:11,521 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:25:11,521 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:25:11,526 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:25:11,749 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:25:12,751 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:25:14,526 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:25:15,104 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:25:15,104 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:25:15,104 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:25:16,439 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:25:16,440 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:25:16,440 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:25:16,444 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:25:16,753 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:25:18,756 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:25:20,390 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:25:20,391 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:25:20,391 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:25:20,391 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:25:20,395 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:25:20,757 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:25:22,760 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:25:24,566 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:25:24,567 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:25:24,567 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:25:24,572 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:25:24,762 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:25:25,572 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:25:26,076 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 20:25:26,765 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:25:28,116 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:25:28,116 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:25:28,117 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:25:28,121 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:25:28,766 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:25:30,104 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:25:30,104 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:25:30,104 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:25:30,767 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:25:31,320 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:25:32,463 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:25:32,464 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:25:32,464 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:25:32,469 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:25:32,768 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:25:34,771 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:25:36,474 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:25:36,475 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:25:36,475 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:25:36,475 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:25:36,480 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:25:36,772 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:25:38,776 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:25:41,480 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:25:42,604 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:25:42,605 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:25:42,605 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:25:42,610 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:25:42,778 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:25:44,781 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:25:45,104 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:25:45,104 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:25:45,105 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:25:46,404 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:25:46,405 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:25:46,405 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:25:46,410 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:25:46,783 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:25:47,449 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:25:48,787 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:25:50,365 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:25:50,366 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:25:50,366 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:25:50,371 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:25:50,788 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:25:52,792 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:25:53,372 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:25:54,235 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:25:54,236 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:25:54,236 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:25:54,241 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:25:54,793 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:25:56,076 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 20:25:56,796 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:25:58,400 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:25:58,401 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:25:58,401 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:25:58,402 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:25:58,407 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:25:58,797 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:26:00,104 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:26:00,104 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:26:00,105 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:26:00,801 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:26:02,369 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:26:02,369 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:26:02,369 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:26:02,374 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:26:02,802 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:26:03,413 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:26:04,805 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:26:06,506 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:26:06,506 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:26:06,507 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:26:06,511 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:26:06,806 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:26:08,512 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:26:08,809 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:26:10,503 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:26:10,504 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:26:10,504 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:26:10,509 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:26:10,811 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:26:12,814 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:26:14,509 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:26:14,907 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:26:14,908 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:26:14,908 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:26:14,912 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:26:15,104 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:26:15,104 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:26:15,105 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:26:15,816 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:26:16,819 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:26:18,871 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:26:18,872 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:26:18,872 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:26:18,877 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:26:19,820 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:26:19,917 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:26:20,823 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:26:23,527 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:26:23,528 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:26:23,528 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:26:23,533 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:26:23,825 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:26:24,828 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:26:25,534 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:26:26,078 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 20:26:29,831 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:26:29,831 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:26:29,832 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:26:29,836 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:26:30,104 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:26:30,104 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:26:30,105 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:26:30,831 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:26:31,394 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:26:32,834 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:26:34,803 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:26:34,804 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:26:34,804 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:26:34,809 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:26:34,835 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:26:36,838 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:26:36,849 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:26:38,771 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:26:38,771 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:26:38,772 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:26:38,777 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:26:38,839 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:26:40,842 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:26:42,777 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:26:45,104 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:26:45,105 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:26:45,105 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:26:45,416 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:26:45,416 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:26:45,416 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:26:45,421 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:26:45,845 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:26:46,848 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:26:48,461 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:26:51,247 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:26:51,248 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:26:51,248 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:26:51,253 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:26:51,852 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:26:52,855 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:26:54,253 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:26:55,028 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:26:55,029 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:26:55,029 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:26:55,034 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:26:55,856 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:26:56,080 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 20:26:56,859 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:26:59,559 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:26:59,560 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:26:59,560 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:26:59,560 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:26:59,565 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:26:59,861 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:27:00,104 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:27:00,105 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:27:00,105 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:27:00,863 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:27:03,247 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:27:03,247 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:27:03,248 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:27:03,252 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:27:03,865 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:27:04,865 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:27:05,289 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:27:08,208 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:27:08,209 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:27:08,209 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:27:08,214 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:27:08,868 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:27:10,872 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:27:11,214 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:27:12,262 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:27:12,263 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:27:12,263 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:27:12,268 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:27:12,873 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:27:14,876 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:27:15,104 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:27:15,105 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:27:15,105 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:27:16,045 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:27:16,046 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:27:16,046 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:27:16,051 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:27:16,877 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:27:17,089 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:27:18,880 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:27:22,089 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:27:24,551 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:27:24,552 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:27:24,552 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:27:24,557 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:27:24,884 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:27:26,082 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 20:27:26,887 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:27:28,083 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:27:29,367 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:27:29,368 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:27:29,368 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:27:29,373 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:27:29,889 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:27:30,104 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:27:30,105 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:27:30,105 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:27:30,891 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:27:33,384 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:27:33,935 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:27:33,936 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:27:33,936 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:27:33,941 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:27:34,893 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:27:36,897 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:27:38,981 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:27:39,672 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:27:39,673 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:27:39,673 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:27:39,677 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:27:39,898 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:27:40,901 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:27:44,678 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:27:45,105 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:27:45,105 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:27:45,105 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:27:45,210 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:27:45,384 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:27:45,384 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:27:45,388 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:27:45,904 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:27:46,906 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:27:49,054 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:27:49,055 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:27:49,055 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:27:49,060 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:27:49,908 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:27:50,061 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:27:50,911 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:27:55,052 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:27:55,053 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:27:55,053 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:27:55,058 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:27:55,915 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:27:56,059 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:27:56,084 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 20:27:56,917 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:28:00,105 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:28:00,105 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:28:00,105 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:28:01,205 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:28:01,205 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:28:01,205 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:28:01,205 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:28:01,210 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:28:01,920 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:28:02,922 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:28:05,946 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:28:05,947 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:28:05,947 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:28:05,951 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:28:06,925 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:28:06,952 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:28:08,928 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:28:11,151 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:28:11,152 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:28:11,152 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:28:11,156 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:28:11,930 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:28:12,157 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:28:12,933 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:28:15,105 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:28:15,105 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:28:15,106 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:28:15,713 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:28:15,714 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:28:15,714 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:28:15,719 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:28:15,935 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:28:16,935 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:28:17,757 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:28:21,761 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:28:21,762 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:28:21,762 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:28:21,767 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:28:21,938 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:28:22,767 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:28:22,938 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:28:25,875 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:28:25,875 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:28:25,876 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:28:25,881 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:28:25,940 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:28:26,086 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 20:28:28,088 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:28:28,944 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:28:29,452 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:28:29,453 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:28:29,453 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:28:29,458 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:28:29,945 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:28:30,105 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:28:30,105 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:28:30,106 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:28:30,947 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:28:33,395 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:28:35,858 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:28:35,858 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:28:35,858 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:28:35,864 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:28:35,950 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:28:38,901 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:28:38,951 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:28:39,438 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:28:39,439 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:28:39,439 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:28:39,444 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:28:39,952 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:28:40,955 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:28:43,979 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:28:43,979 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:28:43,980 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:28:43,980 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:28:43,984 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:28:44,957 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:28:45,105 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:28:45,105 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:28:45,106 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:28:46,960 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:28:47,839 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:28:47,840 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:28:47,840 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:28:47,845 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:28:47,961 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:28:49,882 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:28:50,965 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:28:52,900 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:28:52,900 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:28:52,901 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:28:52,905 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:28:52,967 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:28:54,906 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:28:54,970 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:28:56,091 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 20:28:57,604 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:28:57,605 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:28:57,605 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:28:57,610 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:28:57,971 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:28:58,974 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:29:00,105 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:29:00,105 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:29:00,106 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:29:00,404 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:29:05,405 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:29:06,285 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:29:06,285 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:29:06,285 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:29:06,290 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:29:06,978 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:29:08,981 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:29:11,329 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:29:12,360 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:29:12,361 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:29:12,361 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:29:12,366 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:29:12,985 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:29:14,988 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:29:15,105 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:29:15,105 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:29:15,106 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:29:17,316 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:29:17,545 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:29:17,545 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:29:17,545 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:29:17,550 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:29:17,989 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:29:18,992 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:29:21,610 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:29:21,611 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:29:21,611 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:29:21,616 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:29:21,993 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:29:22,616 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:29:22,994 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:29:26,090 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 20:29:26,546 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:29:26,546 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:29:26,547 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:29:26,551 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:29:26,996 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:29:28,552 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:29:28,999 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:29:30,105 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:29:30,106 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:29:30,106 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:29:34,411 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:29:35,081 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:29:35,081 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:29:35,081 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:29:35,086 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:29:36,003 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:29:37,006 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:29:39,859 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:29:39,860 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:29:39,860 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:29:39,860 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:29:39,865 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:29:40,007 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:29:41,010 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:29:44,541 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:29:44,542 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:29:44,542 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:29:44,547 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:29:45,012 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:29:45,105 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:29:45,106 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:29:45,106 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:29:45,321 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:29:47,015 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:29:49,484 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:29:49,485 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:29:49,485 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:29:49,490 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:29:50,017 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:29:50,529 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:29:51,020 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:29:55,529 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:29:56,098 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 20:29:58,600 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:29:58,601 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:29:58,601 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:29:58,606 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:29:59,025 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:30:00,105 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:30:00,106 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:30:00,106 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:30:01,026 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:30:01,374 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:30:04,933 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:30:04,933 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:30:04,934 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:30:04,938 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:30:05,028 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:30:06,977 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:30:07,032 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:30:09,429 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:30:09,429 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:30:09,430 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:30:09,434 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:30:10,034 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:30:11,037 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:30:12,435 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:30:14,313 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:30:14,314 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:30:14,314 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:30:14,319 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:30:15,039 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:30:15,106 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:30:15,106 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:30:15,106 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:30:17,042 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:30:18,323 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:30:19,606 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:30:19,606 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:30:19,606 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:30:19,612 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:30:20,044 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:30:21,046 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:30:23,479 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:30:23,479 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:30:23,480 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:30:23,480 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:30:23,484 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:30:24,048 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:30:25,051 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:30:26,095 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 20:30:27,515 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:30:27,515 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:30:27,515 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:30:27,520 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:30:28,052 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:30:28,521 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:30:29,053 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:30:30,106 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:30:30,106 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:30:30,106 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:30:31,499 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:30:31,500 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:30:31,500 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:30:31,505 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:30:32,055 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:30:33,057 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:30:33,545 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:30:35,689 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:30:35,690 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:30:35,690 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:30:35,695 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:30:36,059 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:30:37,062 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:30:38,696 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:30:39,556 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:30:39,556 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:30:39,557 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:30:39,561 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:30:40,064 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:30:41,064 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:30:43,416 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:30:43,416 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:30:43,417 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:30:43,421 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:30:44,066 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:30:44,422 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:30:45,068 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:30:45,106 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:30:45,106 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:30:45,106 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:30:47,220 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:30:47,220 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:30:47,220 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:30:47,225 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:30:48,070 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:30:49,073 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:30:50,265 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:30:50,959 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:30:50,960 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:30:50,960 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:30:50,966 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:30:51,074 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:30:53,076 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:30:54,594 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:30:54,595 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:30:54,595 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:30:54,600 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:30:55,077 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:30:55,600 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:30:56,100 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 20:30:57,080 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:30:59,395 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:30:59,396 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:30:59,396 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:30:59,401 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:31:00,082 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:31:00,106 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:31:00,106 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:31:00,106 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:31:01,084 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:31:01,325 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:31:05,036 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:31:05,036 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:31:05,037 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:31:05,041 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:31:05,086 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:31:07,082 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:31:07,089 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:31:09,220 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:31:09,221 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:31:09,221 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:31:09,226 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:31:10,092 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:31:11,095 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:31:12,226 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:31:13,411 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:31:13,411 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:31:13,412 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:31:13,416 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:31:14,097 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:31:15,099 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:31:15,106 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:31:15,106 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:31:15,107 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:31:17,484 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:31:17,949 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:31:17,949 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:31:17,950 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:31:17,954 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:31:18,101 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:31:19,104 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:31:22,993 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:31:23,847 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:31:23,847 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:31:23,848 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:31:23,855 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:31:24,110 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:31:25,112 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:31:26,097 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 20:31:28,098 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:31:29,579 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:31:29,580 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:31:29,580 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:31:29,585 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:31:30,106 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:31:30,106 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:31:30,107 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:31:30,119 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:31:31,121 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:31:33,387 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:31:33,921 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:31:33,922 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:31:33,922 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:31:33,927 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:31:34,125 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:31:35,126 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:31:38,458 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:31:38,459 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:31:38,459 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:31:38,459 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:31:38,464 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:31:39,132 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:31:41,136 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:31:43,465 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:31:45,106 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:31:45,106 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:31:45,107 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:31:47,047 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:31:47,048 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:31:47,048 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:31:47,053 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:31:47,143 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:31:49,093 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:31:49,148 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:31:53,346 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:31:53,347 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:31:53,347 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:31:53,352 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:31:54,154 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:31:54,352 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:31:55,157 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:31:56,100 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 20:31:56,983 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:31:56,983 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:31:56,984 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:31:56,989 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:31:57,159 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:31:59,162 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:31:59,989 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:32:00,106 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:32:00,107 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:32:00,107 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:32:03,305 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:32:03,305 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:32:03,305 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:32:03,310 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:32:04,170 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:32:05,171 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:32:05,349 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:32:08,106 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:32:08,107 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:32:08,107 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:32:08,112 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:32:08,176 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:32:11,113 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:32:11,182 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:32:12,107 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:32:12,108 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:32:12,108 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:32:12,113 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:32:12,183 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:32:15,106 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:32:15,107 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:32:15,107 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:32:15,188 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:32:16,396 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:32:21,397 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:32:21,497 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:32:21,498 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:32:21,498 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:32:21,503 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:32:22,198 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:32:23,200 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:32:26,105 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 20:32:26,421 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:32:26,422 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:32:26,422 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:32:26,422 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:32:26,427 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:32:27,204 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:32:29,209 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:32:30,106 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:32:30,107 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:32:30,107 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:32:31,151 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:32:31,152 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:32:31,152 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:32:31,156 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:32:31,210 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:32:32,193 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:32:33,215 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:32:36,993 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:32:36,993 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:32:36,993 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:32:36,998 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:32:37,219 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:32:37,999 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:32:39,224 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:32:41,286 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:32:41,287 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:32:41,287 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:32:41,292 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:32:42,227 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:32:43,230 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:32:43,292 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:32:45,107 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:32:45,107 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:32:45,107 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:32:45,418 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:32:45,419 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:32:45,419 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:32:45,424 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:32:46,233 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:32:47,236 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:32:48,461 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:32:51,461 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:32:51,462 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:32:51,462 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:32:51,467 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:32:52,243 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:32:53,246 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:32:53,468 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:32:56,104 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 20:32:56,317 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:32:56,318 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:32:56,318 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:32:56,323 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:32:57,249 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:32:59,254 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:32:59,324 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:33:00,107 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:33:00,107 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:33:00,107 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:33:00,418 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:33:00,419 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:33:00,419 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:33:00,424 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:33:01,255 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:33:03,260 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:33:04,461 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:33:04,574 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:33:04,575 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:33:04,575 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:33:04,579 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:33:05,261 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:33:07,266 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:33:08,955 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:33:08,956 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:33:08,956 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:33:08,961 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:33:09,267 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:33:09,962 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:33:11,273 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:33:14,161 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:33:14,162 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:33:14,162 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:33:14,167 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:33:14,276 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:33:15,107 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:33:15,107 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:33:15,107 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:33:15,279 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:33:15,377 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:33:19,900 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:33:19,901 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:33:19,901 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:33:19,906 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:33:20,285 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:33:20,945 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:33:21,288 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:33:23,616 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:33:23,617 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:33:23,617 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:33:23,622 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:33:24,291 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:33:25,291 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:33:26,108 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 20:33:26,109 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:33:28,272 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:33:28,273 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:33:28,273 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:33:28,277 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:33:28,293 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:33:30,107 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:33:30,107 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:33:30,108 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:33:31,297 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:33:31,326 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:33:32,971 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:33:32,971 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:33:32,972 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:33:32,976 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:33:33,298 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:33:35,301 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:33:36,805 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:33:36,806 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:33:36,806 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:33:36,806 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:33:36,811 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:33:37,302 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:33:39,305 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:33:40,750 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:33:40,750 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:33:40,751 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:33:40,755 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:33:41,306 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:33:42,756 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:33:43,309 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:33:44,942 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:33:44,942 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:33:44,943 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:33:44,948 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:33:45,107 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:33:45,107 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:33:45,108 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:33:45,311 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:33:47,314 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:33:48,357 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:33:48,759 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:33:48,759 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:33:48,759 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:33:48,764 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:33:49,315 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:33:51,318 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:33:52,588 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:33:52,589 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:33:52,589 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:33:52,593 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:33:53,321 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:33:53,594 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:33:55,324 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:33:56,108 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 20:33:57,976 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:33:57,976 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:33:57,977 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:33:57,981 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:33:58,325 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:33:58,982 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:33:59,326 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:34:00,107 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:34:00,108 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:34:00,108 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:34:04,330 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:34:06,411 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:34:06,412 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:34:06,412 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:34:06,417 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:34:07,330 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:34:09,333 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:34:09,417 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:34:11,559 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:34:11,560 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:34:11,560 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:34:11,565 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:34:12,336 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:34:13,339 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:34:14,566 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:34:15,107 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:34:15,107 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:34:15,108 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:34:15,856 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:34:15,856 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:34:15,856 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:34:15,861 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:34:16,340 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:34:17,341 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:34:19,862 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:34:21,719 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:34:21,719 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:34:21,720 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:34:21,725 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:34:22,344 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:34:23,344 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:34:25,726 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:34:25,998 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:34:25,999 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:34:25,999 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:34:26,004 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:34:26,113 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 20:34:26,346 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:34:27,349 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:34:30,107 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:34:30,107 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:34:30,108 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:34:31,378 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:34:31,703 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:34:31,704 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:34:31,704 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:34:31,709 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:34:32,352 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:34:33,354 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:34:36,749 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:34:37,314 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:34:37,315 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:34:37,315 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:34:37,320 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:34:37,356 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:34:39,359 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:34:42,321 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:34:42,623 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:34:42,624 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:34:42,624 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:34:42,629 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:34:43,362 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:34:45,107 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:34:45,107 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:34:45,108 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:34:45,365 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:34:46,624 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:34:46,624 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:34:46,625 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:34:46,630 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:34:47,366 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:34:47,669 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:34:49,369 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:34:50,511 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:34:50,511 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:34:50,512 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:34:50,517 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:34:51,370 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:34:53,373 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:34:53,517 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:34:55,888 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:34:55,889 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:34:55,889 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:34:55,894 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:34:56,112 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 20:34:56,374 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:34:57,377 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:34:59,113 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:35:00,107 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:35:00,108 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:35:00,108 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:35:01,530 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:35:01,531 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:35:01,531 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:35:01,536 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:35:02,379 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:35:03,382 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:35:04,573 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:35:07,671 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:35:07,672 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:35:07,672 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:35:07,677 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:35:08,385 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:35:09,387 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:35:09,677 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:35:12,209 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:35:12,210 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:35:12,210 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:35:12,215 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:35:12,390 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:35:13,393 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:35:15,107 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:35:15,108 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:35:15,108 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:35:15,306 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:35:16,065 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:35:16,065 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:35:16,065 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:35:16,070 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:35:16,394 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:35:17,397 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:35:19,676 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:35:19,677 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:35:19,677 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:35:19,682 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:35:20,399 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:35:20,682 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:35:21,401 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:35:24,361 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:35:24,362 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:35:24,362 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:35:24,367 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:35:24,403 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:35:26,114 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 20:35:26,115 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:35:27,406 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:35:28,485 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:35:28,486 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:35:28,486 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:35:28,491 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:35:29,407 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:35:30,107 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:35:30,108 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:35:30,108 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:35:31,409 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:35:31,409 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:35:32,451 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:35:32,452 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:35:32,452 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:35:32,457 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:35:33,410 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:35:35,413 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:35:36,497 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:35:37,476 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:35:37,480 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:35:37,480 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:35:37,485 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:35:38,414 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:35:39,417 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:35:41,088 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:35:41,089 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:35:41,089 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:35:41,094 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:35:41,418 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:35:42,095 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:35:43,420 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:35:44,847 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:35:44,847 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:35:44,848 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:35:44,853 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:35:45,108 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:35:45,108 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:35:45,108 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:35:45,421 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:35:47,341 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:35:47,424 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:35:48,853 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:35:48,853 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:35:48,853 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:35:48,858 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:35:49,425 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:35:51,428 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:35:52,897 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:35:55,148 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:35:55,148 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:35:55,148 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:35:55,153 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:35:55,432 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:35:56,116 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 20:35:57,435 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:35:58,117 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:35:59,684 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:35:59,685 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:35:59,685 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:35:59,690 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:36:00,108 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:36:00,108 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:36:00,109 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:36:00,436 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:36:01,439 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:36:03,309 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:36:03,679 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:36:03,680 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:36:03,680 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:36:03,685 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:36:04,441 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:36:05,443 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:36:07,491 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:36:07,491 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:36:07,492 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:36:07,497 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:36:08,445 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:36:08,497 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:36:09,448 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:36:12,485 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:36:12,486 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:36:12,486 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:36:12,490 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:36:13,451 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:36:14,491 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:36:15,108 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:36:15,108 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:36:15,109 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:36:15,454 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:36:16,986 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:36:16,987 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:36:16,987 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:36:16,993 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:36:17,455 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:36:19,458 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:36:19,993 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:36:21,156 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:36:21,157 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:36:21,157 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:36:21,162 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:36:21,459 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:36:23,463 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:36:25,163 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:36:26,118 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 20:36:27,075 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:36:27,076 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:36:27,076 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:36:27,081 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:36:27,465 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:36:29,468 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:36:30,108 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:36:30,108 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:36:30,109 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:36:30,387 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:36:30,673 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:36:30,674 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:36:30,674 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:36:30,679 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:36:31,469 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:36:33,472 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:36:35,717 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:36:36,877 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:36:36,878 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:36:36,878 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:36:36,883 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:36:37,474 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:36:39,477 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:36:40,884 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:36:40,898 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:36:40,899 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:36:40,899 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:36:40,904 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:36:41,479 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:36:43,482 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:36:44,783 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:36:44,784 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:36:44,784 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:36:44,789 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:36:45,108 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:36:45,109 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:36:45,109 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:36:45,483 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:36:46,336 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:36:47,486 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:36:51,336 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:36:53,343 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:36:53,344 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:36:53,344 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:36:53,349 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:36:53,490 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:36:55,493 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:36:56,120 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 20:36:57,121 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:36:58,018 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:36:58,019 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:36:58,019 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:36:58,024 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:36:58,495 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:36:59,497 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:37:00,108 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:37:00,108 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:37:00,109 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:37:02,416 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:37:03,044 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:37:03,044 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:37:03,045 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:37:03,049 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:37:03,500 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:37:05,503 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:37:08,089 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:37:08,462 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:37:08,463 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:37:08,463 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:37:08,469 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:37:08,504 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:37:11,509 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:37:13,146 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:37:13,146 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:37:13,147 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:37:13,147 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:37:13,152 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:37:13,510 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:37:15,108 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:37:15,109 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:37:15,109 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:37:15,513 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:37:17,556 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:37:17,557 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:37:17,557 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:37:17,562 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:37:18,515 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:37:18,601 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:37:19,518 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:37:22,716 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:37:22,716 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:37:22,717 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:37:22,722 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:37:23,520 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:37:23,722 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:37:25,523 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:37:26,122 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 20:37:27,133 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:37:27,134 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:37:27,135 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:37:27,140 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:37:27,524 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:37:29,140 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:37:29,525 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:37:30,108 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:37:30,109 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:37:30,109 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:37:31,614 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:37:31,615 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:37:31,615 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:37:31,620 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:37:32,527 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:37:33,530 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:37:34,657 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:37:36,340 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:37:36,341 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:37:36,342 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:37:36,346 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:37:36,531 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:37:37,534 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:37:40,347 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:37:41,575 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:37:41,576 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:37:41,576 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:37:41,582 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:37:42,536 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:37:43,540 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:37:45,109 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:37:45,109 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:37:45,109 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:37:45,380 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:37:46,239 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:37:46,240 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:37:46,240 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:37:46,246 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:37:46,541 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:37:47,544 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:37:50,029 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:37:50,029 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:37:50,030 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:37:50,035 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:37:50,546 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:37:51,035 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:37:51,547 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:37:54,909 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:37:54,910 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:37:54,910 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:37:54,915 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:37:55,550 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:37:56,124 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 20:37:56,125 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:37:57,553 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:37:59,555 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:37:59,556 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:37:59,556 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:37:59,561 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:38:00,108 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:38:00,109 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:38:00,109 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:38:00,554 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:38:01,358 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:38:01,557 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:38:04,012 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:38:04,012 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:38:04,013 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:38:04,018 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:38:04,559 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:38:05,561 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:38:07,057 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:38:07,777 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:38:07,777 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:38:07,778 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:38:07,783 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:38:08,563 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:38:09,563 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:38:12,040 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:38:12,041 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:38:12,041 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:38:12,047 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:38:12,567 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:38:13,047 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:38:13,569 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:38:15,109 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:38:15,109 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:38:15,109 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:38:16,610 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:38:16,611 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:38:16,611 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:38:16,616 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:38:17,572 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:38:18,653 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:38:19,575 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:38:19,966 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:38:19,967 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:38:19,967 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:38:19,973 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:38:20,576 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:38:21,578 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:38:23,973 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:38:26,126 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 20:38:28,219 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:38:28,220 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:38:28,221 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:38:28,226 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:38:28,583 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:38:29,226 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:38:29,585 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:38:30,109 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:38:30,109 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:38:30,109 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:38:32,003 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:38:32,003 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:38:32,004 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:38:32,009 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:38:32,587 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:38:33,590 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:38:35,045 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:38:37,031 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:38:37,032 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:38:37,032 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:38:37,037 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:38:37,592 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:38:39,595 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:38:41,004 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:38:41,005 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:38:41,005 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:38:41,005 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:38:41,010 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:38:41,596 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:38:43,600 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:38:45,109 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:38:45,109 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:38:45,109 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:38:46,403 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:38:46,721 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:38:46,721 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:38:46,722 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:38:46,726 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:38:47,602 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:38:49,605 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:38:51,767 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:38:54,697 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:38:54,698 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:38:54,698 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:38:54,703 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:38:55,610 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:38:56,131 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 20:38:57,132 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:38:57,613 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:38:58,989 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:38:58,989 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:38:58,990 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:38:58,995 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:38:59,614 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:39:00,109 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:39:00,109 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:39:00,110 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:39:01,617 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:39:02,396 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:39:02,943 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:39:02,944 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:39:02,944 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:39:02,949 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:39:03,618 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:39:05,619 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:39:06,917 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:39:06,918 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:39:06,919 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:39:06,923 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:39:07,621 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:39:07,924 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:39:09,624 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:39:10,663 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:39:10,664 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:39:10,664 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:39:10,669 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:39:11,626 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:39:13,629 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:39:13,670 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:39:14,823 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:39:14,824 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:39:14,824 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:39:14,829 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:39:15,109 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:39:15,109 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:39:15,110 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:39:15,631 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:39:17,634 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:39:18,579 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:39:18,579 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:39:18,580 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:39:18,585 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:39:18,634 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:39:19,621 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:39:21,638 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:39:22,348 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:39:22,349 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:39:22,349 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:39:22,354 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:39:22,638 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:39:23,641 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:39:25,355 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:39:26,130 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 20:39:26,322 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:39:26,323 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:39:26,323 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:39:26,328 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:39:26,643 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:39:27,644 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:39:30,109 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:39:30,109 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:39:30,110 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:39:30,711 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:39:30,712 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:39:30,712 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:39:30,712 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:39:30,717 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:39:31,646 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:39:33,647 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:39:35,753 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:39:36,175 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:39:36,176 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:39:36,176 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:39:36,181 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:39:36,649 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:39:37,652 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:39:41,181 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:39:41,482 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:39:41,483 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:39:41,483 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:39:41,488 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:39:41,655 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:39:43,658 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:39:45,109 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:39:45,109 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:39:45,110 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:39:45,654 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:39:45,655 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:39:45,655 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:39:45,660 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:39:45,661 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:39:46,697 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:39:47,664 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:39:49,950 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:39:49,951 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:39:49,951 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:39:49,956 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:39:50,666 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:39:51,668 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:39:51,957 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:39:54,163 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:39:54,164 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:39:54,164 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:39:54,169 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:39:54,671 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:39:55,674 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:39:56,135 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 20:39:57,136 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:40:00,109 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:40:00,110 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:40:00,110 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:40:00,623 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:40:00,624 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:40:00,624 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:40:00,629 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:40:00,677 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:40:02,665 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:40:03,681 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:40:04,375 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:40:04,376 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:40:04,376 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:40:04,381 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:40:04,681 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:40:05,682 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:40:08,382 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:40:08,761 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:40:08,762 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:40:08,762 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:40:08,767 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:40:09,684 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:40:11,689 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:40:13,146 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:40:13,147 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:40:13,147 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:40:13,153 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:40:13,690 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:40:14,154 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:40:15,109 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:40:15,110 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:40:15,110 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:40:15,693 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:40:17,736 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:40:17,737 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:40:17,737 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:40:17,742 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:40:18,694 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:40:19,697 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:40:19,781 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:40:23,184 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:40:23,185 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:40:23,185 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:40:23,190 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:40:23,699 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:40:25,190 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:40:25,703 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:40:26,134 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 20:40:28,259 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:40:28,260 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:40:28,260 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:40:28,265 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:40:28,704 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:40:29,707 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:40:30,109 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:40:30,110 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:40:30,110 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:40:30,381 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:40:35,382 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:40:36,748 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:40:36,749 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:40:36,749 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:40:36,754 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:40:37,711 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:40:39,715 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:40:40,793 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:40:41,853 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:40:41,854 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:40:41,854 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:40:41,859 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:40:42,716 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:40:43,719 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:40:45,110 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:40:45,110 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:40:45,110 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:40:46,339 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:40:47,749 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:40:47,750 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:40:47,750 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:40:47,756 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:40:48,722 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:40:49,725 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:40:51,622 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:40:51,623 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:40:51,623 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:40:51,623 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:40:51,628 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:40:51,726 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:40:53,730 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:40:55,412 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:40:55,413 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:40:55,413 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:40:55,418 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:40:55,731 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:40:56,139 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 20:40:57,141 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:40:57,735 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:41:00,110 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:41:00,110 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:41:00,110 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:41:00,243 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:41:00,325 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:41:00,326 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:41:00,331 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:41:00,737 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:41:01,740 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:41:02,331 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:41:07,332 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:41:08,692 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:41:08,693 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:41:08,693 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:41:08,698 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:41:08,743 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:41:11,748 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:41:12,737 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:41:15,110 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:41:15,110 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:41:15,110 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:41:15,299 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:41:15,322 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:41:15,322 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:41:15,327 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:41:15,751 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:41:17,754 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:41:18,328 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:41:19,139 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:41:19,139 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:41:19,140 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:41:19,146 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:41:19,755 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:41:21,758 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:41:23,223 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:41:23,224 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:41:23,225 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:41:23,229 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:41:23,759 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:41:24,230 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:41:25,763 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:41:26,138 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 20:41:27,702 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:41:27,703 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:41:27,703 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:41:27,708 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:41:27,764 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:41:29,709 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:41:29,767 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:41:30,110 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:41:30,110 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:41:30,111 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:41:33,363 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:41:33,363 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:41:33,363 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:41:33,371 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:41:33,769 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:41:35,405 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:41:35,772 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:41:37,545 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:41:37,546 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:41:37,547 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:41:37,551 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:41:37,774 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:41:39,777 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:41:40,552 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:41:41,170 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:41:41,171 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:41:41,171 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:41:41,176 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:41:41,778 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:41:43,781 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:41:45,110 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:41:45,110 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:41:45,111 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:41:46,319 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:41:51,319 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:41:54,377 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:41:54,378 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:41:54,378 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:41:54,384 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:41:54,788 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:41:55,791 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:41:56,140 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 20:41:57,141 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:41:59,475 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:41:59,476 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:41:59,476 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:41:59,481 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:41:59,793 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:42:00,110 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:42:00,110 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:42:00,111 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:42:01,796 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:42:02,392 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:42:03,417 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:42:03,418 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:42:03,418 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:42:03,422 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:42:03,797 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:42:05,800 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:42:07,461 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:42:07,689 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:42:07,692 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:42:07,693 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:42:07,697 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:42:07,801 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:42:09,804 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:42:12,698 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:42:13,159 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:42:13,160 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:42:13,160 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:42:13,164 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:42:13,808 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:42:15,110 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:42:15,110 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:42:15,111 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:42:15,811 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:42:16,926 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:42:16,927 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:42:16,927 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:42:16,931 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:42:17,813 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:42:17,969 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:42:19,816 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:42:20,417 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:42:20,417 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:42:20,418 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:42:20,422 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:42:20,816 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:42:21,819 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:42:23,423 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:42:26,142 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 20:42:26,862 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:42:26,863 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:42:26,863 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:42:26,868 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:42:27,823 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:42:28,869 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:42:29,826 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:42:30,110 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:42:30,110 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:42:30,111 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:42:31,295 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:42:31,296 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:42:31,296 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:42:31,301 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:42:31,827 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:42:33,830 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:42:34,341 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:42:35,547 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:42:35,548 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:42:35,548 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:42:35,553 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:42:35,832 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:42:37,835 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:42:39,554 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:42:41,418 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:42:41,419 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:42:41,419 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:42:41,424 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:42:41,837 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:42:43,840 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:42:45,110 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:42:45,111 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:42:45,111 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:42:45,368 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:42:46,536 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:42:46,536 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:42:46,536 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:42:46,541 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:42:46,842 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:42:47,844 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:42:50,581 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:42:51,243 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:42:51,244 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:42:51,244 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:42:51,249 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:42:51,847 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:42:53,851 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:42:55,847 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:42:55,848 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:42:55,848 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:42:55,848 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:42:55,853 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:42:55,853 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:42:56,144 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 20:42:57,856 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:42:59,999 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:43:00,000 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:43:00,000 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:43:00,005 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:43:00,110 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:43:00,111 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:43:00,111 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:43:00,858 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:43:01,345 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:43:01,861 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:43:06,346 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:43:09,282 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:43:09,282 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:43:09,282 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:43:09,287 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:43:09,865 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:43:11,868 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:43:12,325 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:43:13,308 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:43:13,309 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:43:13,309 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:43:13,314 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:43:13,869 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:43:15,111 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:43:15,111 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:43:15,111 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:43:15,872 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:43:17,389 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:43:17,540 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:43:17,540 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:43:17,541 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:43:17,546 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:43:17,873 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:43:19,876 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:43:21,898 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:43:21,899 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:43:21,899 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:43:21,904 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:43:22,878 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:43:22,905 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:43:23,881 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:43:25,775 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:43:25,776 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:43:25,776 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:43:25,781 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:43:25,882 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:43:26,146 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 20:43:27,885 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:43:28,147 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:43:29,740 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:43:29,741 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:43:29,741 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:43:29,746 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:43:29,887 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:43:30,111 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:43:30,111 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:43:30,111 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:43:31,889 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:43:33,586 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:43:33,992 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:43:33,993 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:43:33,993 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:43:33,997 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:43:34,891 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:43:35,894 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:43:38,125 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:43:38,126 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:43:38,126 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:43:38,131 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:43:38,895 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:43:39,131 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:43:39,898 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:43:44,132 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:43:44,747 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:43:44,747 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:43:44,748 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:43:44,752 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:43:44,901 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:43:45,111 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:43:45,111 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:43:45,111 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:43:45,903 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:43:49,362 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:43:50,150 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:43:50,151 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:43:50,151 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:43:50,156 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:43:50,906 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:43:51,909 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:43:55,193 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:43:56,044 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:43:56,045 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:43:56,045 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:43:56,050 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:43:56,148 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 20:43:56,913 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:43:57,916 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:43:59,789 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:43:59,790 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:43:59,790 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:43:59,795 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:43:59,917 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:44:00,111 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:44:00,112 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:44:00,112 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:44:00,342 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:44:01,920 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:44:05,342 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:44:05,524 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:44:05,525 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:44:05,525 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:44:05,530 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:44:05,922 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:44:07,925 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:44:10,531 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:44:10,641 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:44:10,642 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:44:10,642 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:44:10,647 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:44:10,928 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:44:11,930 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:44:14,822 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:44:14,823 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:44:14,823 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:44:14,828 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:44:14,932 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:44:15,111 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:44:15,111 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:44:15,112 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:44:15,934 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:44:16,370 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:44:19,143 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:44:19,143 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:44:19,144 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:44:19,148 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:44:19,937 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:44:21,940 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:44:22,185 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:44:26,153 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 20:44:27,564 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:44:27,565 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:44:27,565 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:44:27,565 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:44:27,569 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:44:27,943 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:44:29,946 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:44:30,111 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:44:30,111 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:44:30,112 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:44:33,412 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:44:36,043 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:44:36,044 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:44:36,044 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:44:36,049 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:44:36,950 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:44:37,953 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:44:39,089 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:44:42,258 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:44:42,259 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:44:42,260 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:44:42,264 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:44:42,955 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:44:43,958 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:44:44,265 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:44:45,111 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:44:45,111 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:44:45,112 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:44:47,051 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:44:47,052 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:44:47,052 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:44:47,057 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:44:47,961 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:44:49,962 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:44:50,093 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:44:51,150 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:44:51,151 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:44:51,151 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:44:51,156 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:44:51,963 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:44:53,967 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:44:55,157 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:44:55,549 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:44:55,550 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:44:55,550 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:44:55,555 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:44:55,968 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:44:56,152 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 20:44:57,972 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:44:59,743 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:44:59,744 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:44:59,744 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:44:59,750 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:44:59,973 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:45:00,112 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:45:00,113 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:45:00,113 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:45:00,365 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:45:01,976 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:45:04,276 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:45:04,276 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:45:04,277 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:45:04,281 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:45:04,978 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:45:05,978 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:45:06,321 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:45:08,955 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:45:08,956 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:45:08,956 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:45:08,961 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:45:08,981 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:45:09,983 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:45:11,962 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:45:15,111 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:45:15,112 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:45:15,112 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:45:16,008 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:45:16,009 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:45:16,009 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:45:16,014 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:45:16,988 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:45:17,014 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:45:17,989 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:45:20,633 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:45:20,634 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:45:20,634 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:45:20,640 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:45:20,990 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:45:21,993 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:45:22,640 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:45:24,617 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:45:24,617 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:45:24,618 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:45:24,623 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:45:24,995 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:45:25,997 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:45:26,157 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 20:45:28,158 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:45:28,211 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:45:28,212 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:45:28,212 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:45:28,218 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:45:28,999 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:45:30,000 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:45:30,111 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:45:30,112 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:45:30,113 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:45:33,341 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:45:36,491 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:45:36,492 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:45:36,492 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:45:36,497 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:45:37,003 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:45:38,006 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:45:38,498 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:45:41,049 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:45:41,049 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:45:41,050 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:45:41,055 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:45:42,008 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:45:44,011 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:45:44,055 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:45:45,111 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:45:45,113 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:45:45,113 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:45:49,295 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:45:49,296 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:45:49,296 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:45:49,296 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:45:49,301 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:45:50,015 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:45:52,018 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:45:53,225 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:45:53,226 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:45:53,226 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:45:53,231 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:45:54,020 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:45:55,232 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:45:56,023 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:45:56,156 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 20:45:58,099 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:45:58,100 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:45:58,101 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:45:58,106 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:45:59,025 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:46:00,028 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:46:00,111 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:46:00,113 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:46:00,113 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:46:00,413 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:46:02,492 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:46:02,493 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:46:02,493 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:46:02,498 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:46:03,029 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:46:04,032 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:46:05,498 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:46:06,187 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:46:06,188 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:46:06,188 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:46:06,193 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:46:07,034 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:46:08,036 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:46:10,377 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:46:10,378 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:46:10,379 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:46:10,384 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:46:11,039 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:46:11,384 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:46:12,040 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:46:14,514 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:46:14,515 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:46:14,515 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:46:14,520 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:46:15,041 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:46:15,112 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:46:15,113 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:46:15,113 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:46:16,044 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:46:17,364 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:46:19,664 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:46:19,665 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:46:19,666 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:46:19,670 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:46:20,046 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:46:22,049 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:46:22,671 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:46:23,847 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:46:23,848 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:46:23,848 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:46:23,853 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:46:24,050 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:46:26,054 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:46:26,161 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 20:46:28,162 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:46:29,149 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:46:29,150 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:46:29,150 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:46:29,155 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:46:30,056 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:46:30,112 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:46:30,113 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:46:30,113 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:46:32,059 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:46:33,406 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:46:33,588 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:46:33,589 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:46:33,589 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:46:33,594 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:46:34,060 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:46:36,063 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:46:38,595 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:46:42,113 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:46:42,114 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:46:42,114 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:46:42,119 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:46:43,067 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:46:44,068 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:46:44,119 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:46:45,112 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:46:45,113 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:46:45,113 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:46:46,967 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:46:46,968 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:46:46,969 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:46:46,973 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:46:47,070 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:46:48,072 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:46:49,974 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:46:50,811 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:46:50,811 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:46:50,812 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:46:50,817 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:46:51,074 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:46:52,077 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:46:54,797 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:46:54,798 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:46:54,798 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:46:54,803 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:46:55,080 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:46:55,804 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:46:56,082 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:46:56,160 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 20:46:58,730 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:46:58,731 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:46:58,731 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:46:58,736 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:46:59,084 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:47:00,086 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:47:00,112 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:47:00,113 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:47:00,113 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:47:01,364 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:47:02,426 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:47:02,427 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:47:02,427 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:47:02,433 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:47:03,088 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:47:04,091 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:47:06,434 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:47:10,467 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:47:10,468 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:47:10,468 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:47:10,473 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:47:11,096 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:47:11,474 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:47:12,098 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:47:14,216 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:47:14,217 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:47:14,217 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:47:14,222 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:47:15,100 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:47:15,112 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:47:15,113 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:47:15,113 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:47:16,100 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:47:17,339 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:47:21,398 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:47:21,399 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:47:21,400 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:47:21,405 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:47:22,104 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:47:22,405 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:47:24,107 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:47:26,162 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 20:47:26,279 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:47:26,280 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:47:26,280 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:47:26,285 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:47:27,109 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:47:28,112 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:47:28,286 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:47:30,112 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:47:30,113 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:47:30,113 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:47:30,229 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:47:30,381 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:47:30,382 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:47:30,387 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:47:31,114 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:47:32,116 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:47:33,387 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:47:33,889 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:47:33,893 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:47:33,893 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:47:33,898 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:47:34,117 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:47:36,120 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:47:37,876 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:47:37,877 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:47:37,877 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:47:37,882 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:47:38,122 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:47:38,882 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:47:40,125 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:47:42,194 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:47:42,195 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:47:42,195 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:47:42,200 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:47:43,126 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:47:44,127 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:47:44,201 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:47:45,112 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:47:45,113 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:47:45,113 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:47:46,406 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:47:46,407 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:47:46,407 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:47:46,412 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:47:47,129 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:47:48,131 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:47:49,412 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:47:51,268 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:47:51,269 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:47:51,269 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:47:51,274 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:47:52,134 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:47:54,138 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:47:55,197 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:47:55,197 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:47:55,198 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:47:55,198 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:47:55,203 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:47:56,139 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:47:56,164 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 20:47:58,142 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:47:59,901 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:47:59,901 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:47:59,902 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:47:59,907 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:48:00,112 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:48:00,113 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:48:00,114 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:48:00,144 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:48:00,421 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:48:02,148 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:48:04,051 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:48:04,052 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:48:04,052 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:48:04,058 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:48:04,149 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:48:06,059 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:48:06,150 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:48:07,733 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:48:07,733 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:48:07,734 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:48:07,739 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:48:08,151 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:48:10,154 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:48:11,739 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:48:13,122 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:48:13,123 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:48:13,123 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:48:13,128 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:48:13,157 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:48:14,160 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:48:15,112 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:48:15,114 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:48:15,114 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:48:17,339 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:48:21,663 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:48:21,663 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:48:21,664 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:48:21,669 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:48:22,164 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:48:22,669 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:48:24,167 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:48:26,167 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 20:48:26,691 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:48:26,692 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:48:26,692 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:48:26,697 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:48:27,169 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:48:27,697 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:48:28,170 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:48:30,112 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:48:30,114 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:48:30,114 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:48:32,069 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:48:32,070 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:48:32,070 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:48:32,075 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:48:32,172 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:48:33,076 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:48:34,175 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:48:36,411 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:48:36,412 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:48:36,412 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:48:36,418 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:48:37,177 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:48:38,179 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:48:38,418 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:48:41,960 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:48:41,961 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:48:41,961 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:48:41,966 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:48:42,182 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:48:43,967 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:48:44,183 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:48:45,112 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:48:45,114 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:48:45,114 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:48:48,140 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:48:48,141 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:48:48,141 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:48:48,146 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:48:48,185 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:48:49,146 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:48:50,188 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:48:54,147 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:48:56,168 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 20:48:56,219 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:48:56,219 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:48:56,219 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:48:56,225 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:48:57,193 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:48:58,196 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:48:59,225 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:49:00,113 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:49:00,114 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:49:00,114 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:49:01,665 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:49:01,666 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:49:01,666 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:49:01,672 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:49:02,198 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:49:04,201 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:49:04,674 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:49:05,912 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:49:05,912 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:49:05,913 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:49:05,918 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:49:06,202 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:49:08,205 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:49:09,918 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:49:12,195 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:49:12,196 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:49:12,197 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:49:12,201 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:49:12,209 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:49:14,212 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:49:15,113 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:49:15,114 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:49:15,114 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:49:15,114 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:49:17,023 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:49:17,024 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:49:17,024 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:49:17,029 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:49:17,213 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:49:18,216 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:49:21,030 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:49:21,235 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:49:21,236 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:49:21,236 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:49:21,241 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:49:22,220 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:49:22,220 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:49:25,778 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:49:25,778 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:49:25,779 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:49:25,784 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:49:26,171 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 20:49:26,172 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:49:26,223 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:49:28,226 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:49:30,113 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:49:30,114 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:49:30,114 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:49:31,388 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:49:33,211 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:49:33,212 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:49:33,212 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:49:33,217 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:49:33,229 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:49:34,231 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:49:37,218 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:49:37,504 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:49:37,505 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:49:37,506 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:49:37,511 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:49:38,233 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:49:40,236 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:49:42,167 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:49:42,168 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:49:42,168 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:49:42,173 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:49:42,237 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:49:43,173 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:49:44,241 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:49:45,113 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:49:45,114 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:49:45,114 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:49:46,331 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:49:46,332 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:49:46,332 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:49:46,337 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:49:47,242 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:49:48,245 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:49:48,337 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:49:49,899 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:49:49,900 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:49:49,900 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:49:49,905 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:49:50,246 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:49:52,249 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:49:53,906 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:49:56,176 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 20:49:58,411 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:49:58,412 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:49:58,412 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:49:58,418 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:49:59,254 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:49:59,418 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:50:00,113 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:50:00,114 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:50:00,114 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:50:00,257 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:50:02,959 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:50:02,960 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:50:02,960 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:50:02,966 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:50:03,259 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:50:04,259 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:50:04,966 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:50:08,417 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:50:08,418 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:50:08,418 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:50:08,424 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:50:09,262 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:50:10,263 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:50:10,425 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:50:12,823 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:50:12,824 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:50:12,825 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:50:12,829 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:50:13,266 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:50:14,268 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:50:15,113 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:50:15,114 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:50:15,114 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:50:16,321 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:50:17,302 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:50:17,303 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:50:17,303 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:50:17,308 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:50:18,273 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:50:18,273 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:50:21,300 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:50:21,301 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:50:21,301 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:50:21,306 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:50:22,275 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:50:22,275 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:50:22,306 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:50:26,175 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 20:50:27,095 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:50:27,096 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:50:27,096 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:50:27,101 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:50:27,278 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:50:28,102 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:50:28,279 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:50:30,113 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:50:30,114 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:50:30,114 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:50:30,824 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:50:30,824 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:50:30,825 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:50:30,829 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:50:31,281 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:50:32,283 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:50:33,830 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:50:34,766 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:50:34,767 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:50:34,767 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:50:34,772 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:50:35,285 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:50:36,287 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:50:39,773 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:50:43,312 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:50:43,313 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:50:43,313 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:50:43,319 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:50:44,294 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:50:44,294 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:50:45,113 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:50:45,114 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:50:45,115 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:50:45,115 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:50:48,127 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:50:48,127 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:50:48,128 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:50:48,133 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:50:48,296 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:50:50,133 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:50:50,298 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:50:53,206 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:50:53,207 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:50:53,207 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:50:53,212 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:50:53,301 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:50:54,304 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:50:55,213 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:50:56,180 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 20:51:00,113 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:51:00,114 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:51:00,115 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:51:00,334 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:51:02,492 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:51:02,492 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:51:02,493 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:51:02,497 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:51:03,309 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:51:04,311 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:51:05,498 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:51:07,183 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:51:07,184 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:51:07,184 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:51:07,189 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:51:07,313 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:51:08,314 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:51:10,943 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:51:10,944 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:51:10,944 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:51:10,945 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:51:10,949 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:51:11,317 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:51:12,319 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:51:15,113 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:51:15,115 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:51:15,115 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:51:15,186 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:51:15,349 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:51:15,349 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:51:15,355 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:51:16,323 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:51:16,324 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:51:16,355 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:51:21,356 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:51:21,504 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:51:21,505 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:51:21,505 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:51:21,510 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:51:22,327 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:51:24,330 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:51:26,133 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:51:26,134 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:51:26,135 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:51:26,140 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:51:26,179 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 20:51:26,331 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:51:27,181 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:51:28,334 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:51:29,958 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:51:29,959 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:51:29,959 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:51:29,964 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:51:30,114 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:51:30,115 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:51:30,115 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:51:30,336 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:51:32,337 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:51:32,373 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:51:33,673 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:51:33,674 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:51:33,675 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:51:33,679 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:51:34,338 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:51:36,341 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:51:37,419 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:51:37,420 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:51:37,420 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:51:37,420 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:51:37,425 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:51:38,342 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:51:40,345 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:51:41,351 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:51:41,352 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:51:41,352 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:51:41,357 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:51:42,348 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:51:42,349 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:51:43,358 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:51:45,114 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:51:45,115 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:51:45,115 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:51:47,589 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:51:47,590 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:51:47,590 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:51:47,595 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:51:48,352 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:51:48,596 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:51:50,353 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:51:50,783 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:51:50,784 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:51:50,784 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:51:50,789 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:51:51,354 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:51:52,356 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:51:53,789 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:51:56,037 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:51:56,038 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:51:56,038 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:51:56,043 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:51:56,184 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 20:51:56,360 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:51:58,363 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:51:59,186 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:52:00,114 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:52:00,115 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:52:00,115 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:52:04,390 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:52:04,543 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:52:04,544 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:52:04,544 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:52:04,549 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:52:05,366 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:52:06,369 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:52:09,550 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:52:09,727 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:52:09,728 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:52:09,728 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:52:09,733 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:52:10,371 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:52:12,375 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:52:13,148 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:52:13,149 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:52:13,149 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:52:13,154 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:52:13,376 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:52:14,378 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:52:15,114 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:52:15,115 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:52:15,115 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:52:15,375 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:52:19,476 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:52:19,477 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:52:19,477 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:52:19,484 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:52:20,382 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:52:20,484 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:52:22,383 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:52:23,710 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:52:23,711 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:52:23,711 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:52:23,716 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:52:24,384 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:52:25,717 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:52:26,183 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 20:52:26,388 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:52:30,114 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:52:30,115 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:52:30,115 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:52:30,897 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:52:30,897 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:52:30,898 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:52:30,898 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:52:30,903 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:52:31,391 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:52:32,393 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:52:34,796 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:52:34,797 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:52:34,797 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:52:34,802 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:52:35,395 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:52:36,397 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:52:36,803 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:52:38,614 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:52:38,615 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:52:38,615 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:52:38,620 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:52:39,399 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:52:40,401 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:52:42,417 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:52:42,418 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:52:42,418 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:52:42,418 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:52:42,423 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:52:43,403 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:52:44,405 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:52:45,114 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:52:45,115 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:52:45,116 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:52:48,374 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:52:49,575 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:52:49,575 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:52:49,576 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:52:49,581 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:52:50,409 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:52:52,412 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:52:53,582 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:52:53,806 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:52:53,806 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:52:53,807 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:52:53,812 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:52:54,415 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:52:56,185 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 20:52:56,418 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:52:58,221 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:52:58,222 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:52:58,222 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:52:58,227 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:52:58,419 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:52:59,227 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:53:00,114 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:53:00,115 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:53:00,116 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:53:00,422 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:53:04,153 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:53:04,154 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:53:04,155 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:53:04,159 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:53:04,424 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:53:05,160 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:53:06,427 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:53:09,051 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:53:09,052 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:53:09,052 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:53:09,056 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:53:09,429 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:53:10,431 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:53:11,057 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:53:13,463 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:53:13,463 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:53:13,464 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:53:13,469 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:53:14,437 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:53:14,437 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:53:15,114 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:53:15,116 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:53:15,116 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:53:16,383 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:53:18,346 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:53:18,347 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:53:18,347 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:53:18,352 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:53:18,439 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:53:20,440 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:53:22,353 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:53:23,033 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:53:23,034 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:53:23,035 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:53:23,039 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:53:23,442 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:53:24,445 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:53:26,187 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 20:53:28,189 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:53:28,595 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:53:28,596 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:53:28,596 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:53:28,601 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:53:29,448 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:53:30,114 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:53:30,116 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:53:30,116 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:53:30,451 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:53:32,552 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:53:32,553 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:53:32,553 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:53:32,558 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:53:33,452 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:53:33,558 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:53:34,455 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:53:36,308 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:53:36,308 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:53:36,309 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:53:36,314 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:53:36,456 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:53:38,457 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:53:39,314 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:53:39,726 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:53:39,727 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:53:39,727 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:53:39,733 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:53:40,458 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:53:42,461 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:53:44,734 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:53:45,114 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:53:45,115 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:53:45,115 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:53:45,744 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:53:45,745 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:53:45,746 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:53:45,751 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:53:46,463 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:53:48,467 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:53:49,752 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:53:49,922 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:53:49,923 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:53:49,923 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:53:49,929 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:53:50,468 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:53:52,471 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:53:54,044 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:53:54,044 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:53:54,045 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:53:54,050 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:53:54,473 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:53:55,050 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:53:56,189 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 20:53:56,476 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:53:59,674 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:53:59,675 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:53:59,675 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:53:59,680 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:54:00,114 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:54:00,116 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:54:00,116 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:54:00,116 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:54:00,479 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:54:02,482 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:54:03,306 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:54:03,306 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:54:03,307 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:54:03,311 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:54:03,483 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:54:04,485 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:54:05,312 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:54:07,880 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:54:07,881 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:54:07,881 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:54:07,886 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:54:08,487 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:54:10,491 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:54:10,887 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:54:13,187 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:54:13,188 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:54:13,188 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:54:13,193 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:54:13,494 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:54:14,496 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:54:15,115 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:54:15,116 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:54:15,116 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:54:16,394 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:54:16,776 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:54:16,776 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:54:16,777 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:54:16,782 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:54:17,498 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:54:18,499 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:54:21,462 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:54:21,462 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:54:21,463 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:54:21,463 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:54:21,468 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:54:21,500 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:54:22,503 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:54:26,191 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 20:54:26,756 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:54:26,756 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:54:26,757 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:54:26,757 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:54:26,762 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:54:27,506 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:54:28,509 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:54:30,115 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:54:30,116 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:54:30,116 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:54:30,891 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:54:30,892 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:54:30,892 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:54:30,897 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:54:31,511 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:54:31,897 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:54:32,513 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:54:35,319 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:54:35,320 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:54:35,320 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:54:35,325 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:54:35,515 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:54:36,517 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:54:37,326 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:54:39,323 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:54:39,324 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:54:39,324 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:54:39,328 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:54:39,519 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:54:40,521 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:54:42,329 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:54:42,862 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:54:42,863 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:54:42,863 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:54:42,868 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:54:43,523 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:54:44,526 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:54:45,115 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:54:45,116 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:54:45,116 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:54:47,954 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:54:47,955 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:54:47,956 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:54:47,956 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:54:47,961 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:54:48,528 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:54:50,531 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:54:52,238 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:54:52,239 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:54:52,240 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:54:52,245 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:54:52,533 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:54:53,245 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:54:54,537 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:54:55,642 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:54:55,643 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:54:55,643 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:54:55,648 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:54:56,193 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 20:54:56,540 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:54:56,540 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:54:59,195 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:54:59,828 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:54:59,829 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:54:59,829 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:54:59,834 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:55:00,115 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:55:00,116 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:55:00,116 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:55:00,542 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:55:02,546 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:55:04,337 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:55:04,476 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:55:04,476 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:55:04,477 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:55:04,482 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:55:04,547 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:55:06,550 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:55:09,482 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:55:10,644 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:55:10,645 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:55:10,645 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:55:10,650 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:55:11,554 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:55:12,556 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:55:14,220 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:55:14,221 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:55:14,221 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:55:14,226 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:55:14,557 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:55:15,115 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:55:15,116 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:55:15,117 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:55:15,117 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:55:16,560 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:55:19,096 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:55:19,096 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:55:19,097 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:55:19,101 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:55:19,562 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:55:20,564 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:55:21,102 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:55:25,145 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:55:25,146 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:55:25,146 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:55:25,152 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:55:25,568 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:55:26,153 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:55:26,195 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 20:55:26,568 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:55:29,697 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:55:29,698 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:55:29,698 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:55:29,703 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:55:30,115 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:55:30,116 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:55:30,117 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:55:30,570 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:55:31,376 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:55:32,573 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:55:34,622 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:55:34,623 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:55:34,623 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:55:34,628 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:55:35,575 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:55:36,576 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:55:36,628 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:55:39,008 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:55:39,009 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:55:39,009 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:55:39,014 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:55:39,577 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:55:40,580 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:55:42,014 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:55:43,023 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:55:43,024 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:55:43,024 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:55:43,029 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:55:43,582 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:55:44,584 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:55:45,115 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:55:45,117 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:55:45,117 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:55:47,301 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:55:47,301 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:55:47,302 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:55:47,302 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:55:47,307 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:55:47,586 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:55:48,588 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:55:52,307 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:55:53,125 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:55:53,126 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:55:53,126 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:55:53,131 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:55:53,593 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:55:54,595 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:55:56,197 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 20:55:56,661 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:55:56,661 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:55:56,662 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:55:56,666 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:55:57,597 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:55:57,667 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:55:58,598 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:56:00,115 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:56:00,116 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:56:00,117 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:56:03,373 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:56:05,634 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:56:05,635 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:56:05,635 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:56:05,640 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:56:06,604 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:56:06,604 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:56:08,641 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:56:10,660 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:56:10,661 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:56:10,662 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:56:10,666 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:56:11,608 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:56:12,611 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:56:13,667 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:56:14,749 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:56:14,749 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:56:14,750 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:56:14,755 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:56:15,115 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:56:15,117 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:56:15,117 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:56:15,612 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:56:16,614 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:56:18,440 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:56:18,441 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:56:18,442 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:56:18,446 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:56:18,615 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:56:19,447 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:56:20,619 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:56:24,447 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:56:26,141 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:56:26,142 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:56:26,142 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:56:26,147 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:56:26,202 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 20:56:26,622 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:56:28,625 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:56:30,116 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:56:30,117 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:56:30,117 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:56:30,117 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:56:30,807 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:56:30,808 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:56:30,808 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:56:30,814 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:56:31,627 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:56:32,630 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:56:34,840 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:56:34,841 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:56:34,841 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:56:34,847 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:56:35,631 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:56:35,847 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:56:36,634 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:56:38,621 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:56:38,622 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:56:38,622 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:56:38,627 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:56:38,635 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:56:40,638 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:56:41,627 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:56:43,078 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:56:43,079 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:56:43,079 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:56:43,084 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:56:43,640 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:56:44,642 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:56:45,116 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:56:45,117 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:56:45,117 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:56:47,378 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:56:48,465 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:56:48,466 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:56:48,466 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:56:48,471 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:56:48,646 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:56:50,649 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:56:52,045 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:56:52,046 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:56:52,046 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:56:52,051 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:56:52,650 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:56:53,052 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:56:54,654 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:56:56,201 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 20:56:57,157 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:56:57,157 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:56:57,158 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:56:57,163 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:56:57,656 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:56:58,164 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:56:58,657 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:57:00,116 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:57:00,117 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:57:00,117 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:57:00,871 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:57:00,871 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:57:00,872 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:57:00,877 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:57:01,658 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:57:02,661 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:57:03,877 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:57:04,911 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:57:04,912 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:57:04,912 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:57:04,917 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:57:05,663 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:57:06,665 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:57:08,918 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:57:10,622 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:57:10,623 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:57:10,623 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:57:10,628 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:57:10,667 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:57:12,672 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:57:14,613 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:57:14,614 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:57:14,614 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:57:14,614 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:57:14,619 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:57:14,673 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:57:15,116 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:57:15,117 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:57:15,117 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:57:16,676 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:57:19,881 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:57:19,882 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:57:19,882 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:57:19,882 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:57:19,887 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:57:20,679 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:57:22,682 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:57:24,888 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:57:25,014 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:57:25,015 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:57:25,015 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:57:25,020 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:57:25,684 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:57:26,206 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 20:57:26,687 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:57:29,931 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:57:29,932 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:57:29,933 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:57:29,933 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:57:29,938 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:57:30,116 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:57:30,117 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:57:30,117 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:57:30,689 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:57:32,692 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:57:33,937 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:57:33,937 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:57:33,938 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:57:33,942 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:57:34,694 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:57:34,943 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:57:36,697 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:57:38,111 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:57:38,111 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:57:38,112 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:57:38,117 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:57:38,698 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:57:40,117 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:57:40,701 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:57:43,082 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:57:43,082 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:57:43,083 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:57:43,088 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:57:43,702 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:57:44,705 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:57:45,116 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:57:45,117 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:57:45,117 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:57:45,334 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:57:49,232 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:57:49,233 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:57:49,233 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:57:49,238 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:57:49,708 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:57:50,711 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:57:51,239 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:57:55,242 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:57:55,243 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:57:55,243 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:57:55,248 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:57:55,714 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:57:56,205 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 20:57:56,717 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:57:57,207 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:57:59,095 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:57:59,096 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:57:59,096 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:57:59,101 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:57:59,719 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:58:00,116 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:58:00,117 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:58:00,117 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:58:00,719 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:58:02,406 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:58:05,554 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:58:05,555 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:58:05,555 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:58:05,561 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:58:05,722 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:58:06,723 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:58:07,562 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:58:12,562 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:58:12,964 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:58:12,965 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:58:12,966 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:58:12,971 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:58:13,728 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:58:14,730 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:58:15,116 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:58:15,117 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:58:15,117 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:58:17,148 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:58:17,149 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:58:17,149 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:58:17,154 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:58:17,732 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:58:18,154 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:58:18,734 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:58:21,420 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:58:21,421 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:58:21,421 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:58:21,426 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:58:21,736 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:58:22,739 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:58:23,427 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:58:24,905 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:58:24,906 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:58:24,906 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:58:24,911 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:58:25,741 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:58:26,210 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 20:58:26,743 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:58:29,212 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:58:29,283 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:58:29,284 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:58:29,285 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:58:29,290 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:58:29,745 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:58:30,116 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:58:30,117 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:58:30,118 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:58:30,748 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:58:34,410 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:58:36,076 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:58:36,077 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:58:36,077 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:58:36,083 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:58:36,751 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:58:38,754 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:58:40,084 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:58:40,262 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:58:40,263 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:58:40,263 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:58:40,267 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:58:40,755 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:58:42,758 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:58:44,524 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:58:44,525 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:58:44,525 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:58:44,530 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:58:44,760 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:58:45,116 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:58:45,117 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:58:45,118 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:58:45,118 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:58:46,763 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:58:49,215 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:58:49,215 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:58:49,216 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:58:49,221 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:58:49,765 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:58:50,221 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:58:50,767 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:58:54,269 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:58:54,269 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:58:54,270 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:58:54,275 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:58:54,771 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:58:55,275 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:58:56,209 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 20:58:56,774 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:58:58,801 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:58:58,802 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:58:58,802 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:58:58,807 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:58:59,776 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:59:00,116 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:59:00,118 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:59:00,118 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:59:00,353 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:59:00,779 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:59:02,939 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:59:02,940 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:59:02,941 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:59:02,946 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:59:03,780 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:59:04,783 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:59:05,946 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:59:07,331 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:59:07,332 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:59:07,332 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:59:07,338 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:59:07,785 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:59:08,787 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:59:11,338 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:59:11,576 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:59:11,577 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:59:11,577 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:59:11,581 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:59:11,790 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:59:12,793 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:59:15,117 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:59:15,118 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:59:15,118 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:59:15,495 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:59:15,496 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:59:15,496 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:59:15,502 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:59:15,795 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:59:16,502 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:59:16,797 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:59:21,315 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:59:21,316 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:59:21,316 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:59:21,321 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:59:21,800 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:59:22,322 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:59:22,802 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:59:25,459 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:59:25,460 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:59:25,460 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:59:25,465 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:59:25,806 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:59:26,214 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 20:59:26,808 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:59:28,216 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:59:30,117 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:59:30,118 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:59:30,118 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:59:30,357 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:59:30,358 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:59:30,358 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:59:30,363 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:59:30,811 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:59:32,814 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:59:33,364 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:59:34,981 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:59:34,982 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:59:34,982 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:59:34,987 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:59:35,815 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:59:36,818 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:59:38,988 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:59:39,209 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:59:39,209 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:59:39,210 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:59:39,215 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:59:39,820 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:59:40,822 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:59:44,006 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:59:44,007 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:59:44,007 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:59:44,007 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:59:44,013 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:59:44,824 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:59:45,117 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:59:45,118 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:59:45,118 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:59:46,826 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:59:48,287 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:59:48,288 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:59:48,288 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:59:48,293 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:59:48,827 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:59:49,294 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:59:50,830 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:59:53,195 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:59:53,196 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:59:53,196 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:59:53,201 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:59:53,833 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:59:54,835 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 20:59:55,202 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:59:56,213 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 20:59:56,941 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:59:56,942 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 20:59:56,942 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:59:56,947 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:59:57,837 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 20:59:58,840 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:00:00,117 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:00:00,118 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:00:00,118 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:00:00,370 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:00:01,555 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:00:01,556 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:00:01,556 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:00:01,561 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:00:01,841 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:00:04,845 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:00:05,560 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:00:05,560 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:00:05,561 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:00:05,561 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:00:05,566 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:00:05,846 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:00:08,849 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:00:10,566 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:00:11,072 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:00:11,073 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:00:11,073 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:00:11,079 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:00:11,852 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:00:12,854 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:00:14,948 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:00:14,949 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:00:14,950 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:00:14,954 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:00:15,117 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:00:15,118 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:00:15,118 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:00:15,856 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:00:16,320 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:00:16,859 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:00:19,263 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:00:19,264 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:00:19,264 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:00:19,269 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:00:19,861 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:00:20,863 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:00:22,270 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:00:23,843 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:00:23,844 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:00:23,844 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:00:23,849 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:00:23,865 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:00:26,215 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 21:00:26,869 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:00:28,217 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:00:30,000 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:00:30,001 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:00:30,001 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:00:30,006 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:00:30,117 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:00:30,118 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:00:30,118 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:00:30,871 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:00:32,872 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:00:33,410 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:00:34,139 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:00:34,140 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:00:34,140 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:00:34,145 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:00:34,874 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:00:36,877 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:00:39,146 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:00:42,511 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:00:42,511 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:00:42,512 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:00:42,518 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:00:42,880 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:00:44,518 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:00:44,881 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:00:45,117 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:00:45,118 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:00:45,118 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:00:46,431 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:00:46,431 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:00:46,432 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:00:46,437 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:00:46,882 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:00:48,886 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:00:50,438 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:00:51,001 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:00:51,002 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:00:51,002 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:00:51,007 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:00:51,888 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:00:52,890 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:00:56,008 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:00:56,217 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 21:00:56,731 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:00:56,732 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:00:56,732 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:00:56,737 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:00:56,894 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:00:58,897 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:01:00,117 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:01:00,118 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:01:00,118 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:01:01,314 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:01:01,788 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:01:01,789 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:01:01,789 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:01:01,794 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:01:01,899 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:01:04,903 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:01:06,795 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:01:10,207 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:01:10,207 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:01:10,208 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:01:10,213 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:01:10,906 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:01:12,213 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:01:12,910 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:01:14,006 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:01:14,007 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:01:14,007 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:01:14,013 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:01:14,912 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:01:15,117 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:01:15,118 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:01:15,119 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:01:16,915 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:01:17,358 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:01:18,205 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:01:18,206 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:01:18,206 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:01:18,211 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:01:18,916 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:01:20,919 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:01:22,249 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:01:22,250 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:01:22,250 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:01:22,255 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:01:22,920 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:01:23,256 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:01:24,923 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:01:25,941 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:01:25,942 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:01:25,942 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:01:25,947 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:01:26,219 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 21:01:26,924 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:01:28,926 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:01:29,221 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:01:30,117 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:01:30,118 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:01:30,119 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:01:33,463 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:01:33,464 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:01:33,464 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:01:33,469 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:01:33,929 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:01:34,469 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:01:34,931 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:01:38,018 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:01:38,018 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:01:38,019 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:01:38,024 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:01:38,934 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:01:40,024 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:01:40,936 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:01:42,652 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:01:42,653 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:01:42,653 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:01:42,658 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:01:42,938 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:01:44,941 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:01:45,117 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:01:45,119 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:01:45,119 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:01:45,119 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:01:50,421 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:01:51,094 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:01:51,095 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:01:51,095 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:01:51,100 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:01:51,945 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:01:52,948 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:01:55,229 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:01:55,230 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:01:55,230 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:01:55,236 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:01:55,951 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:01:56,222 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 21:01:56,223 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:01:56,953 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:01:58,883 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:01:58,883 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:01:58,884 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:01:58,889 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:01:58,954 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:02:00,118 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:02:00,119 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:02:00,119 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:02:00,958 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:02:01,394 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:02:03,400 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:02:03,400 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:02:03,400 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:02:03,406 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:02:03,960 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:02:04,962 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:02:06,445 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:02:10,738 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:02:10,739 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:02:10,739 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:02:10,744 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:02:10,966 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:02:11,744 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:02:12,970 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:02:15,118 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:02:15,119 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:02:15,119 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:02:15,834 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:02:15,835 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:02:15,835 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:02:15,841 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:02:15,972 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:02:16,842 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:02:18,976 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:02:20,580 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:02:20,581 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:02:20,581 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:02:20,586 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:02:20,977 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:02:22,587 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:02:22,981 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:02:26,224 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 21:02:26,457 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:02:26,458 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:02:26,458 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:02:26,464 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:02:26,984 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:02:28,464 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:02:28,987 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:02:30,118 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:02:30,119 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:02:30,119 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:02:30,579 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:02:30,580 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:02:30,580 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:02:30,585 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:02:30,989 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:02:32,992 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:02:33,586 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:02:34,763 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:02:34,763 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:02:34,764 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:02:34,769 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:02:34,993 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:02:36,996 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:02:38,064 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:02:38,065 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:02:38,065 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:02:38,070 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:02:38,997 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:02:39,071 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:02:41,001 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:02:41,922 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:02:41,923 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:02:41,924 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:02:41,929 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:02:42,001 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:02:44,929 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:02:45,005 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:02:45,118 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:02:45,119 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:02:45,119 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:02:46,498 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:02:46,499 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:02:46,499 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:02:46,505 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:02:47,006 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:02:49,010 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:02:50,506 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:02:50,657 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:02:50,658 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:02:50,658 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:02:50,663 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:02:51,011 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:02:53,015 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:02:55,271 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:02:55,272 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:02:55,272 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:02:55,277 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:02:56,018 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:02:56,226 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 21:02:56,227 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:02:57,020 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:02:59,923 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:02:59,924 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:02:59,924 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:02:59,929 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:03:00,022 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:03:00,118 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:03:00,119 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:03:00,119 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:03:01,328 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:03:03,025 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:03:04,351 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:03:04,352 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:03:04,352 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:03:04,358 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:03:05,027 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:03:06,358 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:03:07,028 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:03:08,262 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:03:08,263 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:03:08,263 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:03:08,268 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:03:09,029 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:03:11,032 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:03:12,269 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:03:12,714 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:03:12,715 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:03:12,715 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:03:12,720 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:03:13,035 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:03:15,038 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:03:15,118 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:03:15,119 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:03:15,119 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:03:16,449 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:03:16,449 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:03:16,450 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:03:16,454 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:03:17,039 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:03:17,455 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:03:19,042 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:03:22,455 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:03:22,924 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:03:22,925 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:03:22,925 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:03:22,930 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:03:23,044 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:03:25,047 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:03:26,228 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 21:03:27,779 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:03:27,780 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:03:27,781 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:03:27,781 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:03:27,786 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:03:28,050 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:03:30,118 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:03:30,119 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:03:30,119 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:03:31,053 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:03:33,417 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:03:36,327 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:03:36,327 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:03:36,328 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:03:36,333 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:03:37,059 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:03:39,062 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:03:39,334 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:03:40,758 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:03:40,759 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:03:40,759 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:03:40,764 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:03:41,063 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:03:43,066 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:03:44,342 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:03:44,343 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:03:44,343 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:03:44,344 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:03:44,348 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:03:45,067 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:03:45,118 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:03:45,119 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:03:45,119 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:03:47,070 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:03:49,385 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:03:50,165 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:03:50,166 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:03:50,166 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:03:50,172 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:03:51,073 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:03:53,076 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:03:54,209 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:03:54,210 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:03:54,211 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:03:54,215 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:03:55,079 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:03:55,216 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:03:56,233 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 21:03:57,083 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:03:59,098 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:03:59,099 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:03:59,099 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:03:59,103 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:04:00,084 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:04:00,118 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:04:00,119 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:04:00,120 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:04:00,338 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:04:01,087 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:04:03,183 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:04:03,184 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:04:03,184 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:04:03,189 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:04:04,088 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:04:05,091 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:04:06,190 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:04:07,592 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:04:07,592 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:04:07,593 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:04:07,598 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:04:08,093 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:04:09,096 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:04:11,571 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:04:11,572 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:04:11,572 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:04:11,572 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:04:11,577 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:04:12,099 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:04:13,102 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:04:15,118 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:04:15,120 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:04:15,120 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:04:16,337 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:04:16,338 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:04:16,338 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:04:16,343 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:04:17,104 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:04:17,344 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:04:19,107 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:04:20,552 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:04:20,553 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:04:20,553 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:04:20,558 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:04:21,108 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:04:22,558 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:04:23,111 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:04:26,232 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 21:04:28,233 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:04:28,525 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:04:28,526 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:04:28,526 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:04:28,532 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:04:29,115 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:04:30,119 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:04:30,120 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:04:30,120 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:04:31,118 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:04:32,715 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:04:32,716 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:04:32,716 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:04:32,721 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:04:33,120 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:04:33,721 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:04:35,123 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:04:37,174 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:04:37,174 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:04:37,175 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:04:37,180 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:04:38,124 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:04:39,125 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:04:39,180 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:04:41,943 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:04:41,944 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:04:41,944 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:04:41,949 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:04:42,127 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:04:44,950 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:04:45,119 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:04:45,120 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:04:45,120 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:04:45,131 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:04:46,912 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:04:46,913 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:04:46,913 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:04:46,918 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:04:47,132 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:04:49,135 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:04:50,918 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:04:52,204 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:04:52,205 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:04:52,205 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:04:52,210 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:04:53,137 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:04:55,142 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:04:56,211 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:04:56,237 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 21:04:56,695 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:04:56,696 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:04:56,696 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:04:56,701 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:04:57,143 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:04:59,146 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:05:00,119 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:05:00,120 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:05:00,120 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:05:01,388 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:05:05,220 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:05:05,220 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:05:05,221 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:05:05,226 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:05:06,150 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:05:07,153 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:05:07,226 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:05:12,227 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:05:12,541 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:05:12,542 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:05:12,542 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:05:12,547 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:05:13,157 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:05:15,119 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:05:15,120 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:05:15,120 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:05:15,161 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:05:17,308 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:05:17,309 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:05:17,310 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:05:17,310 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:05:17,315 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:05:18,162 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:05:19,165 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:05:21,248 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:05:21,249 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:05:21,249 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:05:21,254 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:05:22,166 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:05:23,167 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:05:23,255 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:05:25,026 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:05:25,027 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:05:25,027 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:05:25,032 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:05:25,168 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:05:26,236 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 21:05:27,172 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:05:28,782 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:05:28,783 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:05:28,783 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:05:28,783 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:05:28,788 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:05:29,173 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:05:30,119 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:05:30,120 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:05:30,120 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:05:31,176 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:05:32,579 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:05:32,580 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:05:32,580 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:05:32,585 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:05:33,177 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:05:34,586 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:05:35,180 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:05:36,308 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:05:36,309 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:05:36,310 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:05:36,314 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:05:37,181 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:05:39,184 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:05:40,315 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:05:40,655 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:05:40,656 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:05:40,656 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:05:40,661 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:05:41,185 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:05:43,188 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:05:45,119 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:05:45,120 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:05:45,120 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:05:45,411 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:05:46,167 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:05:46,168 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:05:46,168 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:05:46,172 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:05:46,190 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:05:49,194 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:05:51,173 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:05:51,423 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:05:51,423 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:05:51,424 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:05:51,428 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:05:52,196 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:05:53,198 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:05:56,242 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 21:05:56,243 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:05:57,660 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:05:57,661 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:05:57,661 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:05:57,666 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:05:58,202 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:05:59,205 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:06:00,119 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:06:00,120 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:06:00,120 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:06:01,419 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:06:01,660 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:06:01,660 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:06:01,661 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:06:01,666 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:06:02,207 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:06:03,207 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:06:06,666 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:06:08,208 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:06:08,209 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:06:08,209 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:06:08,214 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:06:09,212 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:06:11,216 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:06:12,029 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:06:12,030 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:06:12,030 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:06:12,030 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:06:12,035 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:06:12,219 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:06:15,119 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:06:15,120 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:06:15,121 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:06:15,220 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:06:17,322 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:06:19,330 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:06:19,331 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:06:19,331 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:06:19,336 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:06:20,227 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:06:21,231 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:06:22,337 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:06:26,240 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 21:06:27,504 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:06:27,505 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:06:27,505 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:06:27,505 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:06:27,510 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:06:28,238 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:06:29,241 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:06:30,119 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:06:30,121 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:06:30,121 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:06:31,082 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:06:31,083 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:06:31,084 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:06:31,088 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:06:31,242 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:06:33,089 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:06:33,245 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:06:38,012 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:06:38,013 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:06:38,013 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:06:38,018 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:06:38,247 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:06:39,019 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:06:41,251 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:06:43,121 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:06:43,122 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:06:43,122 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:06:43,127 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:06:43,252 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:06:44,128 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:06:45,119 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:06:45,120 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:06:45,120 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:06:45,255 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:06:47,566 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:06:47,567 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:06:47,567 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:06:47,572 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:06:48,258 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:06:49,260 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:06:49,573 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:06:54,232 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:06:54,233 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:06:54,233 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:06:54,239 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:06:54,263 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:06:55,240 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:06:56,242 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 21:06:57,267 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:07:00,119 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:07:00,120 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:07:00,120 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:07:00,403 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:07:00,419 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:07:00,419 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:07:00,419 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:07:00,425 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:07:01,269 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:07:03,272 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:07:04,809 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:07:04,809 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:07:04,810 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:07:04,815 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:07:05,273 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:07:05,853 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:07:07,276 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:07:08,772 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:07:08,776 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:07:08,776 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:07:08,781 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:07:09,278 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:07:11,279 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:07:11,782 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:07:12,543 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:07:12,544 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:07:12,544 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:07:12,549 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:07:13,281 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:07:15,120 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:07:15,121 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:07:15,121 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:07:15,285 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:07:16,309 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:07:16,309 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:07:16,310 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:07:16,314 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:07:17,286 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:07:17,315 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:07:19,289 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:07:21,149 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:07:21,150 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:07:21,150 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:07:21,155 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:07:21,290 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:07:23,155 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:07:23,291 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:07:25,234 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:07:25,235 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:07:25,235 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:07:25,240 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:07:25,293 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:07:26,244 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 21:07:27,296 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:07:28,246 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:07:28,780 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:07:28,781 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:07:28,781 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:07:28,786 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:07:29,297 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:07:30,120 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:07:30,121 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:07:30,121 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:07:31,300 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:07:33,350 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:07:34,020 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:07:34,021 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:07:34,021 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:07:34,026 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:07:34,302 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:07:37,306 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:07:37,776 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:07:37,776 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:07:37,777 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:07:37,781 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:07:38,306 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:07:38,782 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:07:39,307 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:07:42,447 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:07:42,448 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:07:42,448 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:07:42,453 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:07:43,309 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:07:44,453 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:07:45,120 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:07:45,121 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:07:45,121 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:07:45,310 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:07:48,307 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:07:48,308 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:07:48,308 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:07:48,313 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:07:48,314 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:07:50,314 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:07:51,318 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:07:53,443 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:07:53,444 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:07:53,444 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:07:53,449 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:07:54,320 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:07:55,323 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:07:55,450 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:07:56,246 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 21:07:57,302 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:07:57,303 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:07:57,303 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:07:57,308 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:07:57,324 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:07:59,327 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:08:00,120 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:08:00,121 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:08:00,121 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:08:01,424 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:08:05,836 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:08:05,837 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:08:05,837 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:08:05,842 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:08:06,331 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:08:06,843 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:08:07,331 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:08:10,546 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:08:10,547 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:08:10,547 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:08:10,552 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:08:11,334 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:08:12,552 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:08:13,338 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:08:15,120 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:08:15,121 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:08:15,121 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:08:16,040 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:08:16,041 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:08:16,041 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:08:16,046 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:08:16,339 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:08:17,342 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:08:18,046 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:08:20,374 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:08:20,375 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:08:20,376 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:08:20,380 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:08:21,344 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:08:23,345 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:08:23,381 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:08:25,624 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:08:25,624 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:08:25,625 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:08:25,629 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:08:26,249 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 21:08:26,348 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:08:27,350 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:08:29,250 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:08:29,474 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:08:29,474 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:08:29,475 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:08:29,479 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:08:30,120 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:08:30,121 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:08:30,121 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:08:30,352 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:08:31,354 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:08:34,421 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:08:35,558 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:08:35,559 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:08:35,559 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:08:35,564 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:08:36,357 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:08:37,359 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:08:39,565 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:08:40,914 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:08:40,915 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:08:40,915 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:08:40,920 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:08:41,362 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:08:43,366 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:08:44,921 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:08:45,120 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:08:45,121 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:08:45,121 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:08:46,320 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:08:46,321 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:08:46,321 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:08:46,326 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:08:46,367 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:08:49,371 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:08:49,635 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:08:49,636 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:08:49,636 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:08:49,641 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:08:50,372 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:08:50,641 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:08:51,375 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:08:53,357 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:08:53,358 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:08:53,358 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:08:53,363 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:08:53,376 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:08:55,380 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:08:56,251 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 21:08:56,253 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:08:57,851 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:08:57,852 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:08:57,852 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:08:57,856 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:08:58,382 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:08:59,384 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:09:00,120 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:09:00,121 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:09:00,121 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:09:01,346 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:09:01,633 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:09:01,634 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:09:01,635 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:09:01,639 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:09:02,386 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:09:03,389 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:09:05,333 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:09:05,333 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:09:05,334 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:09:05,338 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:09:05,390 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:09:07,339 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:09:07,393 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:09:09,660 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:09:09,661 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:09:09,661 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:09:09,666 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:09:10,395 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:09:11,397 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:09:12,667 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:09:14,671 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:09:14,671 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:09:14,672 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:09:14,676 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:09:15,121 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:09:15,122 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:09:15,122 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:09:15,400 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:09:17,404 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:09:18,418 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:09:18,527 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:09:18,528 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:09:18,528 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:09:18,533 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:09:19,405 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:09:21,408 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:09:23,373 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:09:23,377 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:09:23,377 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:09:23,382 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:09:23,409 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:09:24,382 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:09:25,412 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:09:26,253 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 21:09:28,031 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:09:28,031 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:09:28,032 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:09:28,036 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:09:28,414 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:09:29,417 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:09:30,037 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:09:30,120 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:09:30,122 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:09:30,122 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:09:33,110 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:09:33,111 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:09:33,111 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:09:33,116 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:09:33,419 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:09:35,116 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:09:35,420 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:09:38,290 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:09:38,290 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:09:38,291 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:09:38,296 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:09:38,422 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:09:40,296 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:09:41,426 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:09:43,491 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:09:43,492 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:09:43,492 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:09:43,497 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:09:44,427 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:09:45,120 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:09:45,122 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:09:45,122 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:09:45,428 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:09:45,474 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:09:47,890 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:09:47,890 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:09:47,891 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:09:47,896 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:09:48,430 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:09:49,432 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:09:50,896 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:09:54,367 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:09:54,368 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:09:54,368 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:09:54,373 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:09:54,436 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:09:56,255 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 21:09:56,257 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:09:57,440 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:09:58,954 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:09:58,955 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:09:58,955 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:09:58,960 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:09:59,441 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:10:00,121 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:10:00,122 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:10:00,122 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:10:01,316 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:10:01,444 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:10:03,008 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:10:03,009 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:10:03,009 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:10:03,014 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:10:03,445 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:10:05,448 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:10:07,015 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:10:07,249 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:10:07,250 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:10:07,250 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:10:07,255 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:10:07,450 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:10:09,452 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:10:12,256 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:10:13,426 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:10:13,427 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:10:13,427 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:10:13,432 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:10:13,456 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:10:15,121 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:10:15,122 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:10:15,122 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:10:15,459 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:10:17,337 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:10:17,337 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:10:17,338 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:10:17,338 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:10:17,343 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:10:17,460 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:10:19,463 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:10:21,217 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:10:21,217 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:10:21,218 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:10:21,222 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:10:21,464 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:10:23,223 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:10:23,467 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:10:25,867 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:10:25,867 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:10:25,868 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:10:25,872 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:10:26,260 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 21:10:26,469 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:10:27,472 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:10:28,261 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:10:30,121 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:10:30,122 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:10:30,122 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:10:30,712 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:10:30,713 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:10:30,713 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:10:30,718 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:10:31,474 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:10:33,475 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:10:33,719 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:10:34,402 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:10:34,402 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:10:34,403 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:10:34,407 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:10:34,476 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:10:37,480 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:10:38,240 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:10:38,241 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:10:38,241 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:10:38,246 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:10:38,481 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:10:39,247 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:10:41,484 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:10:41,924 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:10:41,925 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:10:41,925 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:10:41,930 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:10:42,485 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:10:43,487 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:10:44,931 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:10:45,121 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:10:45,122 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:10:45,122 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:10:46,116 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:10:46,116 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:10:46,117 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:10:46,121 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:10:46,489 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:10:47,491 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:10:49,962 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:10:49,963 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:10:49,963 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:10:49,963 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:10:49,968 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:10:50,494 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:10:51,496 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:10:54,214 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:10:54,215 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:10:54,215 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:10:54,220 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:10:54,499 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:10:55,221 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:10:55,501 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:10:56,259 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 21:10:58,829 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:10:58,830 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:10:58,830 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:10:58,835 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:10:59,504 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:11:00,121 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:11:00,122 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:11:00,122 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:11:00,332 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:11:01,505 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:11:03,011 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:11:03,012 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:11:03,012 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:11:03,017 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:11:03,506 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:11:05,509 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:11:06,018 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:11:07,829 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:11:07,830 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:11:07,831 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:11:07,835 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:11:08,511 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:11:09,513 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:11:11,836 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:11:14,672 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:11:14,672 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:11:14,673 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:11:14,678 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:11:15,121 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:11:15,123 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:11:15,123 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:11:15,519 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:11:17,383 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:11:17,522 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:11:19,252 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:11:19,252 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:11:19,253 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:11:19,257 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:11:19,523 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:11:21,526 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:11:22,969 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:11:22,970 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:11:22,970 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:11:22,971 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:11:22,975 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:11:23,528 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:11:25,531 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:11:26,264 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 21:11:26,661 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:11:26,661 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:11:26,662 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:11:26,666 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:11:27,532 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:11:28,667 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:11:29,535 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:11:30,121 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:11:30,122 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:11:30,122 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:11:31,403 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:11:31,404 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:11:31,404 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:11:31,409 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:11:31,536 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:11:33,540 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:11:34,410 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:11:36,789 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:11:36,790 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:11:36,790 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:11:36,794 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:11:37,542 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:11:39,545 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:11:39,795 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:11:44,796 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:11:45,085 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:11:45,086 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:11:45,086 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:11:45,091 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:11:45,121 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:11:45,123 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:11:45,123 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:11:45,549 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:11:47,552 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:11:49,051 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:11:49,052 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:11:49,052 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:11:49,057 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:11:49,553 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:11:50,057 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:11:51,556 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:11:53,226 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:11:53,227 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:11:53,227 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:11:53,231 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:11:53,558 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:11:55,232 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:11:55,560 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:11:56,263 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 21:12:00,121 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:12:00,122 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:12:00,123 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:12:00,475 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:12:01,711 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:12:01,711 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:12:01,712 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:12:01,717 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:12:02,564 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:12:03,566 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:12:05,589 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:12:05,590 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:12:05,590 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:12:05,590 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:12:05,595 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:12:06,568 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:12:07,569 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:12:10,227 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:12:10,228 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:12:10,228 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:12:10,234 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:12:10,570 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:12:11,234 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:12:11,573 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:12:13,988 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:12:13,989 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:12:13,989 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:12:13,994 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:12:14,576 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:12:15,121 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:12:15,123 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:12:15,123 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:12:15,579 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:12:16,380 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:12:19,391 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:12:19,391 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:12:19,392 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:12:19,397 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:12:19,581 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:12:21,398 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:12:21,584 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:12:24,045 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:12:24,046 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:12:24,046 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:12:24,051 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:12:24,586 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:12:25,588 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:12:26,268 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 21:12:27,269 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:12:28,049 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:12:28,050 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:12:28,050 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:12:28,055 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:12:28,591 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:12:29,593 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:12:30,122 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:12:30,123 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:12:30,123 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:12:32,338 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:12:32,666 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:12:32,667 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:12:32,667 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:12:32,672 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:12:33,595 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:12:35,597 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:12:36,759 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:12:36,760 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:12:36,760 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:12:36,765 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:12:37,598 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:12:37,766 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:12:39,601 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:12:40,513 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:12:40,514 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:12:40,514 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:12:40,520 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:12:40,601 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:12:43,521 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:12:43,606 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:12:44,337 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:12:44,338 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:12:44,338 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:12:44,343 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:12:44,606 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:12:45,122 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:12:45,123 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:12:45,123 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:12:45,609 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:12:49,391 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:12:51,515 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:12:51,516 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:12:51,516 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:12:51,521 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:12:51,613 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:12:53,616 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:12:54,522 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:12:56,170 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:12:56,171 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:12:56,171 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:12:56,176 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:12:56,267 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 21:12:56,618 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:12:57,621 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:12:59,896 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:12:59,897 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:12:59,897 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:12:59,897 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:12:59,902 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:13:00,122 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:13:00,123 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:13:00,123 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:13:00,623 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:13:01,625 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:13:04,152 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:13:04,153 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:13:04,153 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:13:04,158 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:13:04,627 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:13:05,159 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:13:05,627 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:13:08,003 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:13:08,004 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:13:08,005 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:13:08,009 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:13:08,629 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:13:09,632 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:13:11,010 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:13:13,989 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:13:13,989 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:13:13,990 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:13:13,994 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:13:14,636 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:13:15,122 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:13:15,123 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:13:15,123 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:13:15,639 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:13:16,371 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:13:18,146 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:13:18,147 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:13:18,147 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:13:18,152 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:13:18,641 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:13:19,643 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:13:22,153 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:13:22,404 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:13:22,405 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:13:22,405 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:13:22,409 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:13:22,645 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:13:25,649 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:13:26,272 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 21:13:27,274 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:13:28,433 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:13:28,433 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:13:28,433 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:13:28,439 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:13:28,651 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:13:30,122 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:13:30,123 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:13:30,123 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:13:31,655 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:13:31,983 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:13:31,984 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:13:31,984 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:13:31,989 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:13:32,655 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:13:32,989 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:13:33,658 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:13:37,212 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:13:37,213 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:13:37,213 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:13:37,218 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:13:37,660 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:13:38,218 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:13:39,663 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:13:43,218 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:13:45,122 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:13:45,123 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:13:45,123 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:13:45,534 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:13:45,535 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:13:45,535 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:13:45,540 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:13:45,666 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:13:47,669 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:13:48,541 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:13:49,544 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:13:49,545 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:13:49,545 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:13:49,550 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:13:49,671 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:13:51,674 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:13:53,550 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:13:53,687 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:13:53,688 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:13:53,688 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:13:53,693 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:13:54,677 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:13:55,680 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:13:56,271 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 21:13:58,923 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:13:58,924 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:13:58,924 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:13:58,924 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:13:58,929 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:13:59,682 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:14:00,122 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:14:00,123 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:14:00,123 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:14:01,684 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:14:04,007 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:14:04,008 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:14:04,008 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:14:04,009 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:14:04,013 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:14:04,686 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:14:05,688 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:14:08,093 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:14:08,094 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:14:08,094 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:14:08,100 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:14:08,690 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:14:09,100 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:14:09,693 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:14:12,114 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:14:12,115 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:14:12,115 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:14:12,120 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:14:12,695 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:14:13,696 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:14:14,120 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:14:15,122 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:14:15,123 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:14:15,123 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:14:16,958 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:14:16,959 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:14:16,960 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:14:16,964 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:14:17,698 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:14:19,701 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:14:19,965 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:14:21,338 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:14:21,339 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:14:21,339 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:14:21,344 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:14:21,703 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:14:23,706 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:14:25,321 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:14:25,322 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:14:25,322 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:14:25,322 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:14:25,327 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:14:25,707 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:14:26,273 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 21:14:27,710 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:14:29,822 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:14:29,823 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:14:29,823 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:14:29,828 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:14:30,122 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:14:30,123 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:14:30,124 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:14:30,348 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:14:30,712 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:14:31,715 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:14:35,349 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:14:35,382 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:14:35,386 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:14:35,386 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:14:35,391 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:14:35,717 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:14:37,720 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:14:39,533 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:14:39,534 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:14:39,534 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:14:39,540 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:14:39,721 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:14:40,541 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:14:41,724 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:14:44,307 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:14:44,307 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:14:44,308 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:14:44,312 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:14:44,726 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:14:45,122 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:14:45,123 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:14:45,124 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:14:45,728 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:14:46,375 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:14:48,652 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:14:48,653 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:14:48,653 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:14:48,657 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:14:48,730 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:14:51,658 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:14:51,734 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:14:56,275 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 21:14:57,276 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:15:00,122 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:15:00,124 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:15:00,124 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:15:00,257 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:15:00,374 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:15:00,375 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:15:00,379 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:15:00,740 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:15:01,741 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:15:02,380 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:15:04,685 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:15:04,686 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:15:04,686 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:15:04,691 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:15:04,743 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:15:07,692 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:15:07,746 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:15:08,074 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:15:08,075 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:15:08,075 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:15:08,079 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:15:08,747 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:15:09,749 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:15:12,095 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:15:12,095 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:15:12,096 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:15:12,101 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:15:12,752 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:15:13,101 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:15:13,755 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:15:15,123 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:15:15,124 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:15:15,124 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:15:17,966 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:15:17,967 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:15:17,967 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:15:17,971 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:15:18,757 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:15:18,972 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:15:19,760 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:15:23,972 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:15:26,277 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 21:15:27,226 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:15:27,227 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:15:27,227 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:15:27,232 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:15:27,765 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:15:29,233 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:15:29,766 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:15:30,123 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:15:30,124 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:15:30,124 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:15:31,227 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:15:31,228 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:15:31,228 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:15:31,233 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:15:31,767 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:15:33,770 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:15:34,234 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:15:36,414 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:15:36,415 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:15:36,415 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:15:36,420 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:15:36,772 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:15:37,774 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:15:39,420 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:15:40,252 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:15:40,253 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:15:40,253 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:15:40,258 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:15:40,776 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:15:41,777 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:15:44,618 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:15:44,619 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:15:44,619 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:15:44,620 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:15:44,625 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:15:44,778 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:15:45,123 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:15:45,124 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:15:45,124 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:15:47,782 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:15:48,753 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:15:48,754 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:15:48,754 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:15:48,759 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:15:48,782 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:15:49,759 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:15:51,787 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:15:52,706 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:15:52,707 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:15:52,707 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:15:52,711 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:15:52,787 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:15:55,712 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:15:55,792 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:15:56,279 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 21:15:56,852 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:15:56,853 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:15:56,853 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:15:56,858 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:15:57,793 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:15:59,796 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:16:00,123 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:16:00,124 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:16:00,124 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:16:00,862 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:16:00,862 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:16:00,863 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:16:00,863 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:16:00,868 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:16:01,797 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:16:03,801 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:16:05,081 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:16:05,081 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:16:05,082 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:16:05,086 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:16:05,802 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:16:06,087 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:16:07,805 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:16:10,781 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:16:10,781 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:16:10,782 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:16:10,786 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:16:10,807 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:16:11,787 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:16:13,811 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:16:14,720 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:16:14,721 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:16:14,721 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:16:14,727 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:16:14,812 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:16:15,123 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:16:15,124 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:16:15,124 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:16:17,409 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:16:17,816 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:16:20,197 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:16:20,198 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:16:20,198 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:16:20,202 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:16:20,818 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:16:21,820 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:16:23,203 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:16:26,009 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:16:26,010 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:16:26,010 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:16:26,015 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:16:26,282 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 21:16:26,824 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:16:27,826 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:16:28,283 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:16:29,886 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:16:29,887 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:16:29,887 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:16:29,891 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:16:30,123 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:16:30,124 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:16:30,125 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:16:30,828 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:16:31,830 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:16:33,419 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:16:38,196 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:16:38,197 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:16:38,197 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:16:38,202 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:16:38,834 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:16:39,202 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:16:39,837 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:16:42,382 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:16:42,383 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:16:42,383 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:16:42,388 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:16:42,839 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:16:43,841 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:16:44,389 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:16:45,123 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:16:45,124 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:16:45,124 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:16:46,396 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:16:46,397 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:16:46,397 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:16:46,402 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:16:46,843 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:16:47,846 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:16:49,403 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:16:53,258 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:16:53,259 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:16:53,259 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:16:53,264 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:16:53,849 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:16:55,264 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:16:55,854 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:16:56,288 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 21:16:57,847 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:16:57,848 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:16:57,848 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:16:57,853 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:16:57,855 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:16:59,858 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:17:00,123 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:17:00,124 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:17:00,125 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:17:00,337 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:17:02,535 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:17:02,535 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:17:02,536 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:17:02,540 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:17:02,860 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:17:03,862 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:17:05,541 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:17:08,412 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:17:08,413 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:17:08,413 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:17:08,417 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:17:08,865 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:17:09,868 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:17:11,418 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:17:12,869 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:17:12,870 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:17:12,870 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:17:12,875 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:17:12,875 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:17:15,123 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:17:15,124 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:17:15,125 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:17:15,879 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:17:16,567 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:17:16,568 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:17:16,568 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:17:16,569 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:17:16,573 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:17:16,879 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:17:17,882 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:17:21,574 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:17:24,892 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:17:24,893 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:17:24,893 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:17:24,898 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:17:25,887 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:17:26,286 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 21:17:27,287 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:17:27,890 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:17:30,123 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:17:30,124 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:17:30,124 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:17:30,684 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:17:30,685 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:17:30,685 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:17:30,690 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:17:30,892 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:17:32,690 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:17:33,896 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:17:34,072 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:17:34,072 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:17:34,073 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:17:34,078 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:17:34,896 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:17:35,897 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:17:37,905 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:17:37,906 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:17:37,907 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:17:37,907 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:17:37,911 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:17:38,899 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:17:39,901 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:17:42,229 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:17:42,230 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:17:42,230 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:17:42,235 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:17:42,903 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:17:43,235 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:17:43,906 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:17:45,123 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:17:45,125 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:17:45,125 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:17:46,832 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:17:46,833 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:17:46,833 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:17:46,838 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:17:46,907 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:17:48,839 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:17:49,911 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:17:50,932 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:17:50,933 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:17:50,933 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:17:50,937 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:17:51,913 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:17:53,916 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:17:53,938 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:17:55,732 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:17:55,732 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:17:55,733 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:17:55,737 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:17:55,918 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:17:56,291 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 21:17:57,921 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:17:59,292 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:17:59,519 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:17:59,519 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:17:59,520 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:17:59,524 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:17:59,922 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:18:00,124 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:18:00,125 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:18:00,125 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:18:01,925 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:18:04,343 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:18:06,144 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:18:06,145 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:18:06,145 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:18:06,150 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:18:06,928 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:18:07,931 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:18:10,151 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:18:12,304 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:18:12,305 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:18:12,305 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:18:12,310 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:18:12,935 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:18:13,937 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:18:15,124 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:18:15,125 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:18:15,125 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:18:15,418 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:18:17,127 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:18:17,128 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:18:17,128 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:18:17,134 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:18:17,939 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:18:19,943 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:18:21,135 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:18:21,492 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:18:21,493 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:18:21,493 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:18:21,498 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:18:21,944 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:18:23,947 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:18:25,754 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:18:25,755 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:18:25,755 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:18:25,760 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:18:25,948 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:18:26,290 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 21:18:26,291 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:18:27,952 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:18:29,318 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:18:29,319 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:18:29,319 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:18:29,323 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:18:29,953 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:18:30,124 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:18:30,125 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:18:30,125 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:18:31,321 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:18:31,956 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:18:33,319 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:18:33,320 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:18:33,320 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:18:33,325 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:18:33,957 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:18:35,960 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:18:36,326 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:18:38,051 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:18:38,052 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:18:38,052 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:18:38,057 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:18:38,962 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:18:39,964 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:18:42,058 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:18:45,124 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:18:45,125 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:18:45,125 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:18:46,589 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:18:46,590 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:18:46,590 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:18:46,595 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:18:46,968 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:18:47,595 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:18:47,969 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:18:50,342 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:18:50,343 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:18:50,343 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:18:50,348 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:18:50,971 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:18:51,973 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:18:53,349 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:18:55,890 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:18:55,890 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:18:55,891 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:18:55,896 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:18:55,977 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:18:56,295 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 21:18:57,980 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:18:59,296 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:18:59,710 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:18:59,711 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:18:59,711 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:18:59,716 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:18:59,981 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:19:00,124 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:19:00,126 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:19:00,126 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:19:01,984 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:19:04,433 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:19:04,817 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:19:04,818 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:19:04,818 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:19:04,823 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:19:04,986 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:19:07,989 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:19:08,764 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:19:08,765 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:19:08,765 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:19:08,770 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:19:08,990 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:19:09,771 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:19:09,991 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:19:12,252 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:19:12,253 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:19:12,253 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:19:12,258 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:19:12,994 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:19:13,996 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:19:15,124 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:19:15,125 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:19:15,125 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:19:15,126 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:19:17,311 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:19:17,312 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:19:17,312 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:19:17,318 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:19:17,999 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:19:20,002 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:19:20,318 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:19:21,978 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:19:21,979 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:19:21,979 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:19:21,984 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:19:22,003 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:19:24,006 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:19:25,804 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:19:25,805 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:19:25,805 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:19:25,805 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:19:25,810 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:19:26,007 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:19:26,294 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 21:19:28,011 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:19:30,124 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:19:30,125 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:19:30,126 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:19:30,342 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:19:30,388 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:19:30,388 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:19:30,393 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:19:31,013 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:19:31,394 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:19:32,015 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:19:34,376 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:19:34,377 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:19:34,377 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:19:34,383 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:19:35,017 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:19:36,019 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:19:37,384 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:19:38,877 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:19:38,878 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:19:38,878 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:19:38,882 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:19:39,021 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:19:42,025 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:19:42,883 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:19:43,368 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:19:43,368 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:19:43,369 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:19:43,374 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:19:44,026 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:19:45,124 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:19:45,125 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:19:45,126 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:19:46,029 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:19:47,206 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:19:47,207 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:19:47,207 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:19:47,212 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:19:48,030 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:19:48,213 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:19:50,033 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:19:50,967 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:19:50,968 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:19:50,968 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:19:50,973 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:19:51,034 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:19:53,974 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:19:54,038 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:19:56,299 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 21:19:58,105 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:19:58,106 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:19:58,106 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:19:58,111 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:19:59,042 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:19:59,111 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:20:00,044 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:20:00,124 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:20:00,126 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:20:00,126 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:20:02,497 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:20:02,498 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:20:02,498 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:20:02,503 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:20:03,046 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:20:04,048 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:20:04,504 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:20:08,226 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:20:08,227 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:20:08,227 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:20:08,232 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:20:09,051 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:20:10,052 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:20:10,233 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:20:11,992 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:20:11,993 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:20:11,993 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:20:11,998 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:20:12,053 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:20:14,057 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:20:15,124 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:20:15,126 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:20:15,126 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:20:15,365 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:20:20,365 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:20:20,424 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:20:20,425 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:20:20,425 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:20:20,430 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:20:21,061 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:20:22,065 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:20:24,424 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:20:24,425 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:20:24,425 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:20:24,430 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:20:25,067 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:20:25,431 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:20:26,067 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:20:26,298 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 21:20:27,725 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:20:27,726 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:20:27,726 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:20:27,730 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:20:28,069 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:20:30,072 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:20:30,125 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:20:30,126 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:20:30,126 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:20:31,366 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:20:32,502 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:20:32,503 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:20:32,503 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:20:32,508 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:20:33,074 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:20:34,076 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:20:36,509 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:20:36,884 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:20:36,885 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:20:36,885 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:20:36,890 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:20:37,078 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:20:40,082 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:20:41,891 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:20:42,132 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:20:42,133 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:20:42,134 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:20:42,138 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:20:43,084 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:20:44,087 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:20:45,125 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:20:45,126 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:20:45,126 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:20:45,928 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:20:45,929 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:20:45,929 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:20:45,934 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:20:46,088 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:20:46,934 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:20:48,089 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:20:50,045 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:20:50,046 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:20:50,046 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:20:50,050 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:20:50,090 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:20:52,051 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:20:52,093 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:20:54,187 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:20:54,188 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:20:54,188 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:20:54,193 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:20:55,096 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:20:56,099 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:20:56,303 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 21:20:57,305 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:20:58,036 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:20:58,037 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:20:58,037 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:20:58,042 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:20:58,100 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:21:00,101 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:21:00,125 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:21:00,126 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:21:00,126 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:21:02,424 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:21:02,711 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:21:02,712 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:21:02,712 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:21:02,717 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:21:03,103 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:21:04,105 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:21:06,765 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:21:06,766 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:21:06,766 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:21:06,771 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:21:07,107 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:21:07,772 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:21:08,110 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:21:10,732 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:21:10,733 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:21:10,733 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:21:10,738 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:21:11,111 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:21:12,112 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:21:13,738 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:21:15,125 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:21:15,126 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:21:15,126 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:21:15,328 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:21:15,390 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:21:15,390 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:21:15,395 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:21:16,115 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:21:18,117 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:21:19,396 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:21:19,738 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:21:19,739 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:21:19,739 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:21:19,744 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:21:20,118 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:21:22,121 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:21:24,288 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:21:24,289 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:21:24,289 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:21:24,294 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:21:25,123 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:21:25,294 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:21:26,125 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:21:26,302 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 21:21:27,823 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:21:27,824 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:21:27,825 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:21:27,829 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:21:28,127 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:21:30,125 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:21:30,126 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:21:30,126 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:21:30,128 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:21:30,362 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:21:31,776 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:21:31,777 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:21:31,778 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:21:31,782 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:21:32,129 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:21:34,132 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:21:35,783 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:21:36,574 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:21:36,575 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:21:36,575 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:21:36,580 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:21:37,134 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:21:38,137 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:21:40,930 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:21:40,931 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:21:40,931 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:21:40,931 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:21:40,936 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:21:41,138 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:21:42,141 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:21:45,125 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:21:45,126 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:21:45,126 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:21:46,138 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:21:46,139 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:21:46,139 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:21:46,139 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:21:46,144 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:21:46,145 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:21:48,148 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:21:49,969 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:21:49,970 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:21:49,970 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:21:49,975 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:21:50,149 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:21:51,975 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:21:52,150 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:21:55,420 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:21:55,421 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:21:55,421 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:21:55,426 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:21:56,154 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:21:56,307 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 21:21:57,309 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:21:58,157 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:21:59,761 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:21:59,762 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:21:59,762 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:21:59,768 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:22:00,125 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:22:00,126 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:22:00,127 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:22:00,158 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:22:02,161 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:22:02,384 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:22:05,088 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:22:05,089 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:22:05,090 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:22:05,095 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:22:05,163 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:22:08,095 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:22:08,165 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:22:13,096 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:22:14,035 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:22:14,036 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:22:14,036 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:22:14,041 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:22:14,169 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:22:15,125 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:22:15,126 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:22:15,126 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:22:16,172 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:22:18,373 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:22:22,343 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:22:22,344 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:22:22,344 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:22:22,349 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:22:23,176 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:22:24,179 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:22:24,349 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:22:26,307 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 21:22:26,313 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:22:26,314 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:22:26,314 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:22:26,319 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:22:27,181 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:22:28,181 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:22:30,125 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:22:30,126 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:22:30,127 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:22:30,127 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:22:30,167 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:22:30,331 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:22:30,331 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:22:30,336 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:22:31,183 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:22:32,185 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:22:34,514 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:22:34,515 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:22:34,515 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:22:34,520 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:22:35,187 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:22:35,521 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:22:36,190 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:22:39,963 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:22:39,964 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:22:39,964 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:22:39,969 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:22:40,192 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:22:40,969 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:22:42,195 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:22:45,125 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:22:45,127 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:22:45,127 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:22:45,887 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:22:45,888 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:22:45,888 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:22:45,893 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:22:46,197 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:22:46,894 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:22:48,200 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:22:51,894 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:22:54,225 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:22:54,226 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:22:54,227 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:22:54,231 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:22:55,205 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:22:56,208 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:22:56,310 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 21:22:57,311 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:22:59,975 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:22:59,976 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:22:59,976 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:22:59,981 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:23:00,125 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:23:00,127 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:23:00,127 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:23:00,213 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:23:02,216 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:23:02,415 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:23:04,685 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:23:04,686 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:23:04,687 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:23:04,694 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:23:05,218 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:23:06,220 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:23:07,695 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:23:08,434 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:23:08,435 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:23:08,435 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:23:08,440 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:23:09,222 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:23:10,223 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:23:12,376 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:23:12,377 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:23:12,378 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:23:12,382 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:23:13,226 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:23:13,383 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:23:14,228 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:23:15,126 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:23:15,127 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:23:15,127 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:23:16,766 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:23:16,767 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:23:16,767 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:23:16,772 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:23:17,230 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:23:18,232 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:23:18,773 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:23:21,002 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:23:21,003 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:23:21,003 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:23:21,008 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:23:21,234 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:23:22,235 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:23:24,008 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:23:24,865 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:23:24,865 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:23:24,866 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:23:24,871 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:23:25,237 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:23:26,239 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:23:26,312 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 21:23:28,809 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:23:28,810 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:23:28,810 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:23:28,815 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:23:29,241 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:23:29,816 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:23:30,126 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:23:30,127 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:23:30,127 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:23:30,244 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:23:33,186 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:23:33,187 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:23:33,187 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:23:33,192 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:23:33,245 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:23:35,193 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:23:36,249 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:23:38,533 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:23:38,534 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:23:38,534 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:23:38,539 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:23:39,251 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:23:40,253 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:23:40,540 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:23:42,409 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:23:42,410 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:23:42,410 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:23:42,415 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:23:43,255 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:23:44,257 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:23:45,126 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:23:45,127 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:23:45,127 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:23:46,419 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:23:47,128 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:23:47,129 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:23:47,129 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:23:47,134 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:23:47,259 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:23:50,263 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:23:51,598 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:23:51,599 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:23:51,600 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:23:51,600 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:23:51,605 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:23:52,264 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:23:54,268 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:23:55,726 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:23:55,727 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:23:55,728 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:23:55,732 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:23:56,270 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:23:56,314 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 21:23:57,315 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:23:58,273 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:23:59,893 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:23:59,894 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:23:59,894 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:23:59,899 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:24:00,126 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:24:00,127 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:24:00,127 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:24:00,274 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:24:02,277 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:24:02,342 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:24:03,137 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:24:03,138 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:24:03,139 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:24:03,144 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:24:03,278 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:24:06,281 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:24:07,505 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:24:07,506 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:24:07,506 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:24:07,506 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:24:07,511 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:24:08,283 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:24:10,286 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:24:12,372 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:24:12,373 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:24:12,373 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:24:12,378 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:24:13,289 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:24:13,379 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:24:14,291 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:24:15,126 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:24:15,127 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:24:15,127 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:24:18,326 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:24:18,327 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:24:18,328 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:24:18,332 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:24:19,294 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:24:19,333 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:24:20,297 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:24:22,763 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:24:22,764 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:24:22,765 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:24:22,770 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:24:23,298 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:24:24,301 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:24:24,770 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:24:26,316 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 21:24:27,365 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:24:27,366 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:24:27,366 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:24:27,371 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:24:28,304 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:24:30,126 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:24:30,127 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:24:30,127 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:24:30,127 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:24:30,307 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:24:31,658 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:24:31,659 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:24:31,659 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:24:31,664 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:24:32,308 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:24:34,311 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:24:35,665 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:24:40,665 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:24:40,990 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:24:40,991 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:24:40,991 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:24:40,996 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:24:41,315 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:24:42,317 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:24:44,908 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:24:44,909 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:24:44,909 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:24:44,914 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:24:45,126 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:24:45,127 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:24:45,128 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:24:45,319 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:24:46,320 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:24:46,384 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:24:51,384 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:24:51,437 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:24:51,438 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:24:51,438 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:24:51,445 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:24:52,323 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:24:54,327 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:24:54,878 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:24:54,879 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:24:54,880 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:24:54,884 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:24:55,328 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:24:56,318 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 21:24:56,331 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:24:57,319 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:24:59,055 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:24:59,056 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:24:59,057 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:24:59,061 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:24:59,333 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:25:00,126 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:25:00,127 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:25:00,127 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:25:00,335 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:25:02,334 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:25:02,808 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:25:02,809 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:25:02,809 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:25:02,814 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:25:03,337 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:25:04,338 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:25:06,628 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:25:06,629 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:25:06,630 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:25:06,634 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:25:07,340 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:25:07,635 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:25:08,342 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:25:12,635 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:25:15,039 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:25:15,040 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:25:15,041 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:25:15,045 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:25:15,126 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:25:15,127 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:25:15,127 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:25:15,348 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:25:16,350 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:25:18,388 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:25:20,763 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:25:20,764 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:25:20,764 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:25:20,769 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:25:21,353 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:25:22,355 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:25:23,769 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:25:25,480 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:25:25,481 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:25:25,481 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:25:25,487 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:25:26,323 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 21:25:26,358 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:25:28,361 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:25:29,325 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:25:29,881 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:25:29,882 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:25:29,882 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:25:29,887 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:25:30,127 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:25:30,128 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:25:30,128 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:25:30,362 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:25:32,365 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:25:34,178 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:25:34,179 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:25:34,179 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:25:34,184 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:25:34,367 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:25:35,185 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:25:36,370 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:25:40,185 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:25:41,045 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:25:41,046 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:25:41,046 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:25:41,051 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:25:41,373 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:25:42,375 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:25:44,985 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:25:44,985 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:25:44,986 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:25:44,991 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:25:45,126 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:25:45,128 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:25:45,128 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:25:45,377 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:25:45,400 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:25:46,377 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:25:49,310 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:25:49,311 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:25:49,311 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:25:49,316 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:25:49,379 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:25:51,317 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:25:52,383 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:25:53,399 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:25:53,399 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:25:53,400 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:25:53,404 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:25:54,384 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:25:56,322 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 21:25:56,323 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:25:56,389 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:25:58,761 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:25:58,762 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:25:58,762 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:25:58,767 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:25:59,391 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:26:00,127 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:26:00,127 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:26:00,128 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:26:00,393 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:26:01,368 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:26:06,368 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:26:06,972 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:26:06,973 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:26:06,974 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:26:06,978 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:26:07,397 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:26:08,400 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:26:11,794 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:26:11,795 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:26:11,795 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:26:11,795 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:26:11,800 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:26:12,402 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:26:14,406 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:26:15,127 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:26:15,128 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:26:15,128 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:26:17,113 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:26:17,114 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:26:17,114 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:26:17,114 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:26:17,119 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:26:17,408 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:26:18,411 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:26:22,098 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:26:22,099 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:26:22,099 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:26:22,104 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:26:22,413 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:26:23,104 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:26:24,416 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:26:26,328 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 21:26:28,329 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:26:29,928 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:26:29,929 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:26:29,929 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:26:29,934 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:26:30,127 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:26:30,128 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:26:30,128 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:26:30,420 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:26:32,423 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:26:33,382 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:26:34,389 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:26:34,390 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:26:34,390 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:26:34,395 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:26:34,424 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:26:36,427 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:26:38,396 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:26:38,538 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:26:38,539 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:26:38,539 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:26:38,545 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:26:39,429 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:26:40,432 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:26:42,372 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:26:42,373 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:26:42,373 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:26:42,378 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:26:42,433 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:26:44,379 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:26:44,436 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:26:45,127 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:26:45,128 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:26:45,128 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:26:46,378 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:26:46,379 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:26:46,379 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:26:46,384 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:26:46,437 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:26:48,440 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:26:49,384 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:26:50,404 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:26:50,405 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:26:50,405 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:26:50,410 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:26:50,441 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:26:52,445 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:26:54,411 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:26:54,474 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:26:54,475 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:26:54,476 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:26:54,480 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:26:55,448 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:26:56,326 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 21:26:56,451 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:26:58,043 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:26:58,044 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:26:58,044 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:26:58,049 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:26:58,452 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:27:00,049 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:27:00,127 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:27:00,128 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:27:00,128 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:27:00,455 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:27:02,303 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:27:02,304 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:27:02,304 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:27:02,309 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:27:02,456 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:27:04,459 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:27:05,310 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:27:06,461 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:27:06,462 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:27:06,462 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:27:06,467 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:27:07,461 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:27:08,464 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:27:10,467 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:27:11,209 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:27:11,209 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:27:11,210 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:27:11,215 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:27:11,465 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:27:12,468 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:27:15,127 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:27:15,128 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:27:15,128 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:27:16,402 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:27:16,403 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:27:16,403 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:27:16,404 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:27:16,408 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:27:16,471 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:27:18,474 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:27:20,148 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:27:20,149 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:27:20,150 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:27:20,155 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:27:20,475 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:27:22,155 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:27:22,478 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:27:24,935 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:27:24,936 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:27:24,937 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:27:24,941 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:27:25,480 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:27:26,331 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 21:27:26,481 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:27:27,333 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:27:30,127 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:27:30,128 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:27:30,128 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:27:30,171 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:27:30,366 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:27:30,367 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:27:30,371 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:27:30,483 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:27:32,372 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:27:32,486 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:27:37,372 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:27:37,745 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:27:37,745 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:27:37,746 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:27:37,751 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:27:38,490 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:27:40,493 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:27:42,752 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:27:45,127 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:27:45,128 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:27:45,128 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:27:46,078 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:27:46,079 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:27:46,079 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:27:46,084 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:27:46,496 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:27:48,084 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:27:48,497 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:27:53,085 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:27:54,598 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:27:54,599 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:27:54,599 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:27:54,604 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:27:55,503 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:27:56,330 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 21:27:56,505 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:27:58,332 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:27:58,606 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:27:58,607 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:27:58,607 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:27:58,612 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:27:59,507 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:28:00,127 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:28:00,128 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:28:00,129 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:28:00,507 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:28:01,883 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:28:01,883 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:28:01,884 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:28:01,888 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:28:02,509 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:28:03,889 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:28:04,512 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:28:08,889 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:28:10,179 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:28:10,180 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:28:10,180 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:28:10,184 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:28:10,515 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:28:12,518 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:28:14,185 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:28:14,490 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:28:14,490 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:28:14,491 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:28:14,496 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:28:14,521 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:28:15,127 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:28:15,128 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:28:15,129 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:28:16,523 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:28:18,916 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:28:18,917 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:28:18,917 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:28:18,922 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:28:19,525 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:28:19,922 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:28:20,528 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:28:22,766 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:28:22,767 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:28:22,768 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:28:22,772 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:28:23,530 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:28:24,532 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:28:25,773 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:28:26,333 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 21:28:30,128 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:28:30,128 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:28:30,129 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:28:31,412 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:28:31,879 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:28:31,880 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:28:31,880 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:28:31,885 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:28:32,537 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:28:34,540 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:28:36,287 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:28:36,288 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:28:36,288 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:28:36,293 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:28:36,541 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:28:37,293 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:28:38,544 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:28:39,945 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:28:39,946 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:28:39,946 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:28:39,951 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:28:40,546 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:28:42,547 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:28:42,951 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:28:44,681 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:28:44,682 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:28:44,682 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:28:44,687 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:28:45,128 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:28:45,129 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:28:45,129 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:28:45,548 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:28:46,551 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:28:48,407 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:28:49,184 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:28:49,185 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:28:49,186 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:28:49,190 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:28:49,553 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:28:50,555 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:28:53,308 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:28:53,309 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:28:53,309 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:28:53,313 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:28:53,557 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:28:54,314 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:28:54,560 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:28:56,335 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 21:28:59,336 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:29:00,128 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:29:00,129 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:29:00,129 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:29:01,626 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:29:01,626 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:29:01,627 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:29:01,632 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:29:02,565 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:29:04,567 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:29:04,632 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:29:07,437 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:29:07,438 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:29:07,439 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:29:07,443 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:29:07,568 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:29:08,571 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:29:10,444 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:29:12,589 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:29:12,590 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:29:12,590 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:29:12,595 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:29:13,575 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:29:14,577 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:29:15,128 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:29:15,129 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:29:15,129 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:29:16,326 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:29:21,002 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:29:21,003 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:29:21,004 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:29:21,009 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:29:21,581 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:29:22,009 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:29:22,583 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:29:25,936 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:29:25,937 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:29:25,937 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:29:25,942 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:29:26,337 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 21:29:26,586 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:29:27,338 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:29:28,589 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:29:30,128 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:29:30,129 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:29:30,129 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:29:31,064 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:29:31,065 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:29:31,065 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:29:31,070 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:29:31,591 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:29:32,591 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:29:33,070 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:29:35,484 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:29:35,485 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:29:35,485 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:29:35,489 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:29:35,593 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:29:38,490 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:29:38,597 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:29:39,494 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:29:39,495 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:29:39,495 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:29:39,500 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:29:39,597 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:29:42,601 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:29:43,456 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:29:43,456 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:29:43,457 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:29:43,462 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:29:43,601 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:29:44,462 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:29:44,604 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:29:45,128 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:29:45,129 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:29:45,129 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:29:47,702 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:29:47,702 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:29:47,703 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:29:47,707 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:29:48,606 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:29:49,708 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:29:50,609 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:29:52,951 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:29:52,951 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:29:52,952 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:29:52,957 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:29:53,611 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:29:54,614 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:29:54,957 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:29:56,339 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 21:29:57,242 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:29:57,243 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:29:57,243 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:29:57,248 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:29:57,617 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:29:58,619 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:30:00,128 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:30:00,129 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:30:00,129 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:30:00,337 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:30:02,773 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:30:02,773 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:30:02,774 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:30:02,778 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:30:03,622 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:30:04,625 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:30:05,779 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:30:10,779 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:30:11,103 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:30:11,104 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:30:11,104 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:30:11,109 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:30:11,628 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:30:12,631 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:30:14,964 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:30:14,965 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:30:14,966 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:30:14,971 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:30:15,128 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:30:15,129 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:30:15,129 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:30:15,634 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:30:16,359 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:30:16,634 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:30:19,122 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:30:19,123 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:30:19,124 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:30:19,128 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:30:19,636 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:30:20,639 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:30:22,129 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:30:24,169 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:30:24,170 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:30:24,170 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:30:24,175 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:30:24,641 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:30:26,341 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 21:30:26,644 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:30:27,342 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:30:28,313 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:30:28,314 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:30:28,314 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:30:28,319 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:30:28,645 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:30:30,128 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:30:30,129 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:30:30,129 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:30:30,648 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:30:33,334 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:30:36,159 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:30:36,160 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:30:36,160 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:30:36,165 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:30:36,652 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:30:38,653 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:30:39,165 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:30:40,617 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:30:40,618 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:30:40,618 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:30:40,623 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:30:40,654 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:30:42,657 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:30:44,624 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:30:45,020 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:30:45,021 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:30:45,021 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:30:45,026 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:30:45,128 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:30:45,129 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:30:45,130 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:30:45,659 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:30:46,661 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:30:49,083 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:30:49,084 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:30:49,084 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:30:49,089 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:30:49,663 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:30:50,090 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:30:50,665 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:30:53,684 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:30:53,684 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:30:53,685 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:30:53,689 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:30:54,668 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:30:55,690 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:30:56,346 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 21:30:56,672 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:30:58,588 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:30:58,588 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:30:58,589 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:30:58,593 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:30:58,673 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:31:00,128 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:31:00,129 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:31:00,130 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:31:00,676 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:31:01,396 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:31:02,768 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:31:02,769 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:31:02,769 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:31:02,774 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:31:03,678 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:31:04,681 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:31:06,774 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:31:07,840 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:31:07,840 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:31:07,841 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:31:07,845 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:31:08,683 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:31:10,686 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:31:11,846 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:31:12,400 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:31:12,401 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:31:12,401 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:31:12,406 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:31:12,687 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:31:14,692 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:31:15,129 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:31:15,130 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:31:15,130 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:31:17,419 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:31:20,567 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:31:20,567 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:31:20,568 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:31:20,573 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:31:20,695 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:31:22,574 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:31:22,698 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:31:26,345 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 21:31:28,347 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:31:28,923 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:31:28,924 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:31:28,924 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:31:28,929 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:31:29,702 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:31:30,129 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:31:30,130 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:31:30,130 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:31:30,705 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:31:33,112 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:31:33,113 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:31:33,113 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:31:33,118 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:31:33,706 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:31:34,118 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:31:34,709 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:31:38,909 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:31:38,909 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:31:38,910 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:31:38,915 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:31:39,711 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:31:39,915 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:31:40,714 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:31:43,471 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:31:43,472 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:31:43,472 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:31:43,477 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:31:43,716 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:31:44,716 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:31:45,129 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:31:45,130 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:31:45,130 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:31:45,413 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:31:47,643 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:31:47,643 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:31:47,644 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:31:47,649 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:31:47,718 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:31:50,649 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:31:50,721 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:31:52,151 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:31:52,152 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:31:52,152 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:31:52,158 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:31:52,723 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:31:54,726 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:31:55,951 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:31:55,951 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:31:55,952 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:31:55,952 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:31:55,957 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:31:56,350 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 21:31:56,728 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:31:58,731 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:32:00,129 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:32:00,130 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:32:00,130 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:32:01,422 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:32:01,922 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:32:01,923 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:32:01,923 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:32:01,928 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:32:02,733 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:32:04,736 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:32:06,412 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:32:06,413 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:32:06,413 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:32:06,418 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:32:06,737 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:32:07,418 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:32:08,740 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:32:10,403 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:32:10,404 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:32:10,404 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:32:10,409 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:32:10,742 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:32:12,746 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:32:13,410 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:32:15,129 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:32:15,130 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:32:15,130 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:32:15,699 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:32:15,700 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:32:15,700 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:32:15,705 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:32:15,748 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:32:18,706 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:32:18,753 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:32:20,942 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:32:20,942 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:32:20,943 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:32:20,948 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:32:21,755 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:32:22,757 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:32:23,948 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:32:26,349 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 21:32:26,588 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:32:26,588 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:32:26,589 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:32:26,593 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:32:26,759 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:32:28,763 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:32:29,594 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:32:30,026 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:32:30,026 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:32:30,027 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:32:30,032 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:32:30,129 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:32:30,130 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:32:30,130 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:32:30,764 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:32:32,767 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:32:34,082 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:32:34,083 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:32:34,083 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:32:34,088 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:32:34,768 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:32:35,088 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:32:36,771 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:32:38,794 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:32:38,795 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:32:38,795 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:32:38,800 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:32:39,773 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:32:40,776 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:32:40,801 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:32:43,861 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:32:43,862 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:32:43,862 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:32:43,867 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:32:44,778 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:32:45,129 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:32:45,130 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:32:45,130 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:32:46,342 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:32:46,781 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:32:47,623 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:32:47,624 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:32:47,624 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:32:47,629 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:32:47,782 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:32:48,785 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:32:51,630 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:32:51,769 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:32:51,769 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:32:51,770 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:32:51,775 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:32:51,787 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:32:54,790 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:32:55,654 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:32:55,655 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:32:55,655 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:32:55,660 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:32:55,792 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:32:56,354 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 21:32:56,793 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:32:57,355 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:32:59,466 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:32:59,467 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:32:59,467 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:32:59,472 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:32:59,794 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:33:00,129 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:33:00,130 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:33:00,130 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:33:00,797 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:33:02,430 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:33:04,259 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:33:04,260 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:33:04,260 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:33:04,265 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:33:04,799 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:33:06,803 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:33:08,266 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:33:12,336 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:33:12,337 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:33:12,337 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:33:12,342 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:33:12,806 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:33:13,342 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:33:14,810 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:33:15,129 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:33:15,130 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:33:15,131 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:33:15,912 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:33:15,912 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:33:15,913 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:33:15,918 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:33:16,812 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:33:18,813 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:33:18,918 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:33:19,358 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:33:19,358 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:33:19,359 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:33:19,364 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:33:19,813 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:33:20,816 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:33:24,364 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:33:24,636 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:33:24,637 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:33:24,637 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:33:24,642 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:33:24,818 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:33:26,353 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 21:33:26,821 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:33:29,664 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:33:29,664 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:33:29,665 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:33:29,665 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:33:29,670 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:33:29,823 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:33:30,129 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:33:30,130 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:33:30,131 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:33:30,826 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:33:34,784 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:33:34,785 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:33:34,786 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:33:34,786 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:33:34,790 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:33:34,828 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:33:36,831 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:33:39,791 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:33:39,832 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:33:39,833 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:33:39,833 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:33:39,838 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:33:40,834 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:33:42,837 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:33:44,262 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:33:44,263 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:33:44,263 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:33:44,268 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:33:44,838 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:33:45,129 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:33:45,130 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:33:45,131 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:33:45,364 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:33:46,839 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:33:49,509 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:33:49,510 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:33:49,510 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:33:49,515 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:33:49,841 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:33:50,515 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:33:50,844 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:33:55,346 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:33:55,347 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:33:55,348 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:33:55,353 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:33:55,848 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:33:56,353 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:33:56,358 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 21:33:56,851 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:34:00,130 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:34:00,131 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:34:00,131 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:34:01,361 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:34:03,778 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:34:03,779 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:34:03,779 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:34:03,784 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:34:03,854 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:34:04,857 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:34:06,784 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:34:10,725 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:34:10,726 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:34:10,726 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:34:10,731 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:34:10,860 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:34:12,732 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:34:12,863 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:34:14,840 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:34:14,840 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:34:14,841 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:34:14,845 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:34:14,866 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:34:15,130 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:34:15,131 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:34:15,131 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:34:16,869 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:34:18,381 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:34:20,143 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:34:20,144 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:34:20,144 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:34:20,149 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:34:20,871 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:34:22,874 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:34:24,150 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:34:24,620 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:34:24,621 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:34:24,621 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:34:24,626 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:34:24,876 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:34:26,357 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 21:34:26,879 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:34:29,359 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:34:30,115 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:34:30,116 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:34:30,116 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:34:30,122 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:34:30,130 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:34:30,131 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:34:30,131 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:34:30,881 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:34:32,884 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:34:34,323 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:34:34,324 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:34:34,324 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:34:34,329 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:34:34,885 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:34:35,329 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:34:36,888 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:34:40,255 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:34:40,256 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:34:40,256 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:34:40,261 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:34:40,891 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:34:41,262 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:34:42,894 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:34:45,130 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:34:45,131 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:34:45,131 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:34:46,346 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:34:48,744 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:34:48,744 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:34:48,745 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:34:48,749 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:34:48,897 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:34:50,900 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:34:51,750 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:34:52,718 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:34:52,719 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:34:52,719 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:34:52,724 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:34:52,902 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:34:54,905 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:34:56,359 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 21:34:56,726 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:34:56,727 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:34:56,727 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:34:56,732 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:34:56,907 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:34:57,733 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:34:58,910 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:35:00,130 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:35:00,131 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:35:00,131 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:35:00,847 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:35:00,848 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:35:00,848 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:35:00,853 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:35:00,911 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:35:02,854 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:35:02,913 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:35:05,248 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:35:05,249 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:35:05,249 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:35:05,254 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:35:05,914 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:35:06,917 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:35:08,255 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:35:13,255 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:35:15,130 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:35:15,131 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:35:15,131 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:35:18,389 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:35:23,390 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:35:26,365 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 21:35:29,366 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:35:30,246 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:35:30,932 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:35:30,932 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:35:35,147 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:35:40,148 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:35:45,148 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:35:50,149 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:35:55,149 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:35:56,363 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 21:36:00,365 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:36:03,358 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:36:03,358 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:36:03,358 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:36:05,575 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:36:10,576 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:36:15,576 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:36:18,357 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:36:18,358 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:36:18,358 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:36:20,633 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:36:25,634 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:36:26,365 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 21:36:31,367 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:36:33,357 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:36:33,358 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:36:33,399 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:36:36,631 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:36:41,631 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:36:46,631 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:36:48,358 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:36:48,358 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:36:48,358 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:36:52,601 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:36:56,370 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 21:36:58,372 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:37:03,358 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:37:03,358 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:37:03,399 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:37:03,649 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:37:08,650 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:37:13,650 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:37:18,358 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:37:18,358 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:37:18,358 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:37:19,642 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:37:24,642 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:37:26,369 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 21:37:30,371 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:37:33,358 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:37:33,358 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:37:33,358 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:37:35,575 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:37:40,576 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:37:45,576 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:37:48,358 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:37:48,358 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:37:48,358 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:37:50,587 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:37:55,588 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:37:56,372 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 21:38:01,374 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:38:03,358 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:38:03,358 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:38:03,399 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:38:06,647 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:38:11,648 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:38:16,648 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:38:18,358 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:38:18,358 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:38:18,399 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:38:22,571 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:38:26,376 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 21:38:28,377 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:38:33,358 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:38:33,358 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:38:33,399 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:38:33,606 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:38:38,606 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:38:43,606 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:38:48,358 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:38:48,358 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:38:48,399 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:38:49,578 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:38:54,578 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:38:56,375 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 21:39:00,377 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:39:03,358 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:39:03,358 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:39:03,399 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:39:05,579 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:39:10,579 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:39:13,300 DEBUG SenderThread:1827008 [sender.py:send():382] send: telemetry +2024-03-12 21:39:13,300 DEBUG SenderThread:1827008 [sender.py:send():382] send: exit +2024-03-12 21:39:13,300 INFO SenderThread:1827008 [sender.py:send_exit():589] handling exit code: 0 +2024-03-12 21:39:13,300 INFO SenderThread:1827008 [sender.py:send_exit():591] handling runtime: 25819 +2024-03-12 21:39:13,305 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:39:13,306 INFO SenderThread:1827008 [sender.py:send_exit():597] send defer +2024-03-12 21:39:13,306 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: defer +2024-03-12 21:39:13,306 INFO HandlerThread:1827008 [handler.py:handle_request_defer():172] handle defer: 0 +2024-03-12 21:39:13,306 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: defer +2024-03-12 21:39:13,306 INFO SenderThread:1827008 [sender.py:send_request_defer():613] handle sender defer: 0 +2024-03-12 21:39:13,306 INFO SenderThread:1827008 [sender.py:transition_state():617] send defer: 1 +2024-03-12 21:39:13,306 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: defer +2024-03-12 21:39:13,306 INFO HandlerThread:1827008 [handler.py:handle_request_defer():172] handle defer: 1 +2024-03-12 21:39:13,306 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: defer +2024-03-12 21:39:13,306 INFO SenderThread:1827008 [sender.py:send_request_defer():613] handle sender defer: 1 +2024-03-12 21:39:13,306 INFO SenderThread:1827008 [sender.py:transition_state():617] send defer: 2 +2024-03-12 21:39:13,306 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: defer +2024-03-12 21:39:13,306 INFO HandlerThread:1827008 [handler.py:handle_request_defer():172] handle defer: 2 +2024-03-12 21:39:13,306 INFO HandlerThread:1827008 [system_monitor.py:finish():203] Stopping system monitor +2024-03-12 21:39:13,306 DEBUG SystemMonitor:1827008 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2024-03-12 21:39:13,306 DEBUG SystemMonitor:1827008 [system_monitor.py:_start():183] Publishing last batch of metrics +2024-03-12 21:39:13,309 INFO HandlerThread:1827008 [interfaces.py:finish():202] Joined cpu monitor +2024-03-12 21:39:13,309 INFO HandlerThread:1827008 [interfaces.py:finish():202] Joined disk monitor +2024-03-12 21:39:13,458 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:39:14,462 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:39:15,204 INFO HandlerThread:1827008 [interfaces.py:finish():202] Joined gpu monitor +2024-03-12 21:39:15,204 INFO HandlerThread:1827008 [interfaces.py:finish():202] Joined memory monitor +2024-03-12 21:39:15,204 INFO HandlerThread:1827008 [interfaces.py:finish():202] Joined network monitor +2024-03-12 21:39:15,204 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: poll_exit +2024-03-12 21:39:15,205 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: defer +2024-03-12 21:39:15,205 INFO SenderThread:1827008 [sender.py:send_request_defer():613] handle sender defer: 2 +2024-03-12 21:39:15,205 INFO SenderThread:1827008 [sender.py:transition_state():617] send defer: 3 +2024-03-12 21:39:15,205 DEBUG SenderThread:1827008 [sender.py:send():382] send: stats +2024-03-12 21:39:15,205 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: poll_exit +2024-03-12 21:39:15,206 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: defer +2024-03-12 21:39:15,206 INFO HandlerThread:1827008 [handler.py:handle_request_defer():172] handle defer: 3 +2024-03-12 21:39:15,206 DEBUG SenderThread:1827008 [sender.py:send():382] send: history +2024-03-12 21:39:15,206 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:39:15,211 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:39:15,211 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: defer +2024-03-12 21:39:15,211 INFO SenderThread:1827008 [sender.py:send_request_defer():613] handle sender defer: 3 +2024-03-12 21:39:15,211 INFO SenderThread:1827008 [sender.py:transition_state():617] send defer: 4 +2024-03-12 21:39:15,211 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: defer +2024-03-12 21:39:15,211 INFO HandlerThread:1827008 [handler.py:handle_request_defer():172] handle defer: 4 +2024-03-12 21:39:15,211 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: defer +2024-03-12 21:39:15,211 INFO SenderThread:1827008 [sender.py:send_request_defer():613] handle sender defer: 4 +2024-03-12 21:39:15,211 INFO SenderThread:1827008 [sender.py:transition_state():617] send defer: 5 +2024-03-12 21:39:15,211 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: defer +2024-03-12 21:39:15,211 INFO HandlerThread:1827008 [handler.py:handle_request_defer():172] handle defer: 5 +2024-03-12 21:39:15,212 DEBUG SenderThread:1827008 [sender.py:send():382] send: summary +2024-03-12 21:39:15,216 INFO SenderThread:1827008 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:39:15,216 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: defer +2024-03-12 21:39:15,216 INFO SenderThread:1827008 [sender.py:send_request_defer():613] handle sender defer: 5 +2024-03-12 21:39:15,216 INFO SenderThread:1827008 [sender.py:transition_state():617] send defer: 6 +2024-03-12 21:39:15,216 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: defer +2024-03-12 21:39:15,216 INFO HandlerThread:1827008 [handler.py:handle_request_defer():172] handle defer: 6 +2024-03-12 21:39:15,217 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: defer +2024-03-12 21:39:15,217 INFO SenderThread:1827008 [sender.py:send_request_defer():613] handle sender defer: 6 +2024-03-12 21:39:15,219 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:39:15,276 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: poll_exit +2024-03-12 21:39:15,435 INFO SenderThread:1827008 [sender.py:transition_state():617] send defer: 7 +2024-03-12 21:39:15,435 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: poll_exit +2024-03-12 21:39:15,435 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: defer +2024-03-12 21:39:15,435 INFO HandlerThread:1827008 [handler.py:handle_request_defer():172] handle defer: 7 +2024-03-12 21:39:15,435 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: defer +2024-03-12 21:39:15,435 INFO SenderThread:1827008 [sender.py:send_request_defer():613] handle sender defer: 7 +2024-03-12 21:39:15,464 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/config.yaml +2024-03-12 21:39:15,465 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:39:15,465 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:39:16,276 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: poll_exit +2024-03-12 21:39:18,361 INFO SenderThread:1827008 [sender.py:transition_state():617] send defer: 8 +2024-03-12 21:39:18,361 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: poll_exit +2024-03-12 21:39:18,362 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: defer +2024-03-12 21:39:18,362 INFO HandlerThread:1827008 [handler.py:handle_request_defer():172] handle defer: 8 +2024-03-12 21:39:18,362 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: defer +2024-03-12 21:39:18,362 INFO SenderThread:1827008 [sender.py:send_request_defer():613] handle sender defer: 8 +2024-03-12 21:39:18,362 INFO SenderThread:1827008 [job_builder.py:build():296] Attempting to build job artifact +2024-03-12 21:39:18,363 INFO SenderThread:1827008 [job_builder.py:_get_source_type():426] is repo sourced job +2024-03-12 21:39:18,438 INFO SenderThread:1827008 [job_builder.py:build():402] adding wandb-job metadata file +2024-03-12 21:39:18,466 INFO Thread-12 :1827008 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:39:18,470 INFO SenderThread:1827008 [sender.py:transition_state():617] send defer: 9 +2024-03-12 21:39:18,471 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: defer +2024-03-12 21:39:18,471 INFO HandlerThread:1827008 [handler.py:handle_request_defer():172] handle defer: 9 +2024-03-12 21:39:18,471 DEBUG SenderThread:1827008 [sender.py:send():382] send: artifact +2024-03-12 21:39:19,277 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: poll_exit +2024-03-12 21:39:19,483 INFO SenderThread:1827008 [sender.py:send_artifact():1494] sent artifact job-https___github.com_TRI-ML_prismatic-vlms.git_scripts_pretrain.py - {'id': 'QXJ0aWZhY3Q6NzUzNTUzNzEx', 'state': 'COMMITTED', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjE0MzcwMzk1OA==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6NzUzNTUzNzEx', 'versionIndex': 7}}} +2024-03-12 21:39:19,483 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: defer +2024-03-12 21:39:19,483 INFO SenderThread:1827008 [sender.py:send_request_defer():613] handle sender defer: 9 +2024-03-12 21:39:19,483 INFO SenderThread:1827008 [dir_watcher.py:finish():358] shutting down directory watcher +2024-03-12 21:39:20,467 INFO SenderThread:1827008 [dir_watcher.py:finish():388] scan: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files +2024-03-12 21:39:20,468 INFO SenderThread:1827008 [dir_watcher.py:finish():402] scan save: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/config.yaml config.yaml +2024-03-12 21:39:20,468 INFO SenderThread:1827008 [dir_watcher.py:finish():402] scan save: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/conda-environment.yaml conda-environment.yaml +2024-03-12 21:39:20,468 INFO SenderThread:1827008 [dir_watcher.py:finish():402] scan save: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-metadata.json wandb-metadata.json +2024-03-12 21:39:20,471 INFO SenderThread:1827008 [dir_watcher.py:finish():402] scan save: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/requirements.txt requirements.txt +2024-03-12 21:39:20,471 INFO SenderThread:1827008 [dir_watcher.py:finish():402] scan save: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log output.log +2024-03-12 21:39:20,472 INFO SenderThread:1827008 [dir_watcher.py:finish():402] scan save: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json wandb-summary.json +2024-03-12 21:39:20,472 INFO SenderThread:1827008 [sender.py:transition_state():617] send defer: 10 +2024-03-12 21:39:20,472 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: poll_exit +2024-03-12 21:39:20,475 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: defer +2024-03-12 21:39:20,475 INFO HandlerThread:1827008 [handler.py:handle_request_defer():172] handle defer: 10 +2024-03-12 21:39:20,477 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: defer +2024-03-12 21:39:20,478 INFO SenderThread:1827008 [sender.py:send_request_defer():613] handle sender defer: 10 +2024-03-12 21:39:20,478 INFO SenderThread:1827008 [file_pusher.py:finish():172] shutting down file pusher +2024-03-12 21:39:21,180 INFO wandb-upload_0:1827008 [upload_job.py:push():131] Uploaded file runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/config.yaml +2024-03-12 21:39:21,246 INFO wandb-upload_2:1827008 [upload_job.py:push():131] Uploaded file runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/requirements.txt +2024-03-12 21:39:21,275 INFO wandb-upload_4:1827008 [upload_job.py:push():131] Uploaded file runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/wandb-summary.json +2024-03-12 21:39:21,277 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: poll_exit +2024-03-12 21:39:21,277 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: poll_exit +2024-03-12 21:39:21,279 INFO wandb-upload_1:1827008 [upload_job.py:push():131] Uploaded file runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/conda-environment.yaml +2024-03-12 21:39:21,292 INFO wandb-upload_3:1827008 [upload_job.py:push():131] Uploaded file runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/files/output.log +2024-03-12 21:39:21,492 INFO Thread-11 (_thread_body):1827008 [sender.py:transition_state():617] send defer: 11 +2024-03-12 21:39:21,493 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: defer +2024-03-12 21:39:21,493 INFO HandlerThread:1827008 [handler.py:handle_request_defer():172] handle defer: 11 +2024-03-12 21:39:21,493 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: defer +2024-03-12 21:39:21,493 INFO SenderThread:1827008 [sender.py:send_request_defer():613] handle sender defer: 11 +2024-03-12 21:39:21,493 INFO SenderThread:1827008 [file_pusher.py:join():178] waiting for file pusher +2024-03-12 21:39:21,493 INFO SenderThread:1827008 [sender.py:transition_state():617] send defer: 12 +2024-03-12 21:39:21,494 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: defer +2024-03-12 21:39:21,494 INFO HandlerThread:1827008 [handler.py:handle_request_defer():172] handle defer: 12 +2024-03-12 21:39:21,494 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: defer +2024-03-12 21:39:21,494 INFO SenderThread:1827008 [sender.py:send_request_defer():613] handle sender defer: 12 +2024-03-12 21:39:21,494 INFO SenderThread:1827008 [file_stream.py:finish():595] file stream finish called +2024-03-12 21:39:22,119 INFO SenderThread:1827008 [file_stream.py:finish():599] file stream finish is done +2024-03-12 21:39:22,119 INFO SenderThread:1827008 [sender.py:transition_state():617] send defer: 13 +2024-03-12 21:39:22,119 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: defer +2024-03-12 21:39:22,119 INFO HandlerThread:1827008 [handler.py:handle_request_defer():172] handle defer: 13 +2024-03-12 21:39:22,119 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: defer +2024-03-12 21:39:22,119 INFO SenderThread:1827008 [sender.py:send_request_defer():613] handle sender defer: 13 +2024-03-12 21:39:22,119 INFO SenderThread:1827008 [sender.py:transition_state():617] send defer: 14 +2024-03-12 21:39:22,120 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: defer +2024-03-12 21:39:22,120 INFO HandlerThread:1827008 [handler.py:handle_request_defer():172] handle defer: 14 +2024-03-12 21:39:22,120 DEBUG SenderThread:1827008 [sender.py:send():382] send: final +2024-03-12 21:39:22,120 DEBUG SenderThread:1827008 [sender.py:send():382] send: footer +2024-03-12 21:39:22,120 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: defer +2024-03-12 21:39:22,120 INFO SenderThread:1827008 [sender.py:send_request_defer():613] handle sender defer: 14 +2024-03-12 21:39:22,121 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: poll_exit +2024-03-12 21:39:22,121 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: poll_exit +2024-03-12 21:39:22,121 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:39:22,121 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: server_info +2024-03-12 21:39:22,121 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: server_info +2024-03-12 21:39:22,163 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: get_summary +2024-03-12 21:39:22,164 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: sampled_history +2024-03-12 21:39:22,165 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: job_info +2024-03-12 21:39:22,324 DEBUG SenderThread:1827008 [sender.py:send_request():409] send_request: job_info +2024-03-12 21:39:22,324 DEBUG HandlerThread:1827008 [handler.py:handle_request():146] handle_request: shutdown +2024-03-12 21:39:22,324 INFO HandlerThread:1827008 [handler.py:finish():869] shutting down handler +2024-03-12 21:39:23,165 INFO WriterThread:1827008 [datastore.py:close():296] close: runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/run-zo3nkbji.wandb +2024-03-12 21:39:23,324 INFO SenderThread:1827008 [sender.py:finish():1572] shutting down sender +2024-03-12 21:39:23,324 INFO SenderThread:1827008 [file_pusher.py:finish():172] shutting down file pusher +2024-03-12 21:39:23,324 INFO SenderThread:1827008 [file_pusher.py:join():178] waiting for file pusher diff --git a/qformer2_128/wandb/run-20240312_142853-zo3nkbji/logs/debug.log b/qformer2_128/wandb/run-20240312_142853-zo3nkbji/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..1d37911cd1e7f6adad17d1c2e67ac3eecaba871b --- /dev/null +++ b/qformer2_128/wandb/run-20240312_142853-zo3nkbji/logs/debug.log @@ -0,0 +1,34 @@ +2024-03-12 14:28:53,678 INFO MainThread:1826638 [wandb_setup.py:_flush():76] Current SDK version is 0.16.3 +2024-03-12 14:28:53,678 INFO MainThread:1826638 [wandb_setup.py:_flush():76] Configure stats pid to 1826638 +2024-03-12 14:28:53,678 INFO MainThread:1826638 [wandb_setup.py:_flush():76] Loading settings from /home/lilei/.config/wandb/settings +2024-03-12 14:28:53,678 INFO MainThread:1826638 [wandb_setup.py:_flush():76] Loading settings from /home/lilei/prismatic-vlms/wandb/settings +2024-03-12 14:28:53,678 INFO MainThread:1826638 [wandb_setup.py:_flush():76] Loading settings from environment variables: {} +2024-03-12 14:28:53,678 INFO MainThread:1826638 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False} +2024-03-12 14:28:53,678 INFO MainThread:1826638 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'scripts/pretrain.py', 'program_abspath': '/home/lilei/prismatic-vlms/scripts/pretrain.py', 'program': '/home/lilei/prismatic-vlms/scripts/pretrain.py'} +2024-03-12 14:28:53,678 INFO MainThread:1826638 [wandb_init.py:_log_setup():526] Logging user logs to runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/logs/debug.log +2024-03-12 14:28:53,678 INFO MainThread:1826638 [wandb_init.py:_log_setup():527] Logging internal logs to runs/qformer2_128/wandb/run-20240312_142853-zo3nkbji/logs/debug-internal.log +2024-03-12 14:28:53,678 INFO MainThread:1826638 [wandb_init.py:init():566] calling init triggers +2024-03-12 14:28:53,678 INFO MainThread:1826638 [wandb_init.py:init():573] wandb.init called with sweep_config: {} +config: {'model': {'type': 'one-stage+7b', 'model_id': 'one-stage+7b_qformer2_128', 'arch_specifier': 'qformer2_128', 'vision_backbone_id': 'clip-vit-l-336px', 'llm_backbone_id': 'vicuna-v15-7b', 'image_resize_strategy': 'letterbox', 'llm_max_length': 2048, 'align_epochs': 1, 'align_max_steps': None, 'align_global_batch_size': 256, 'align_per_device_batch_size': 16, 'align_learning_rate': 0.001, 'align_weight_decay': 0.0, 'align_max_grad_norm': 1.0, 'align_lr_scheduler_type': 'linear-warmup+cosine-decay', 'align_warmup_ratio': 0.03, 'align_train_strategy': 'fsdp-shard-grad-op', 'finetune_epochs': 1, 'finetune_max_steps': None, 'finetune_global_batch_size': 128, 'finetune_per_device_batch_size': 16, 'finetune_learning_rate': 2e-05, 'finetune_weight_decay': 0.1, 'finetune_max_grad_norm': 1.0, 'finetune_lr_scheduler_type': 'linear-warmup+cosine-decay', 'finetune_warmup_ratio': 0.03, 'finetune_train_strategy': 'fsdp-full-shard', 'enable_gradient_checkpointing': True, 'enable_mixed_precision_training': True, 'reduce_in_full_precision': False}, 'dataset': {'type': 'llava-v15', 'dataset_id': 'llava-v15', 'align_stage_components': ['download/llava-laion-cc-sbu-558k/chat.json', 'download/llava-laion-cc-sbu-558k'], 'finetune_stage_components': ['download/llava-v1.5-instruct/llava_v1_5_mix665k.json', 'download/llava-v1.5-instruct'], 'dataset_root_dir': 'data'}, 'stage': 'finetune', 'pretrained_checkpoint': None, 'run_id': 'qformer2_128', 'run_root_dir': 'runs', 'seed': 7, 'hf_token': '.hf_token', 'trackers': ['jsonl', 'wandb'], 'wandb_project': 'hf-vlms', 'wandb_entity': 'lilei_stones', 'max_length': 4096} +2024-03-12 14:28:53,678 INFO MainThread:1826638 [wandb_init.py:init():616] starting backend +2024-03-12 14:28:53,678 INFO MainThread:1826638 [wandb_init.py:init():620] setting up manager +2024-03-12 14:28:53,686 INFO MainThread:1826638 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2024-03-12 14:28:53,688 INFO MainThread:1826638 [wandb_init.py:init():628] backend started and connected +2024-03-12 14:28:53,701 INFO MainThread:1826638 [wandb_init.py:init():720] updated telemetry +2024-03-12 14:28:53,719 INFO MainThread:1826638 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout +2024-03-12 14:28:54,229 INFO MainThread:1826638 [wandb_run.py:_on_init():2262] communicating current version +2024-03-12 14:28:54,273 INFO MainThread:1826638 [wandb_run.py:_on_init():2271] got version response upgrade_message: "wandb version 0.16.4 is available! To upgrade, please run:\n $ pip install wandb --upgrade" + +2024-03-12 14:28:54,273 INFO MainThread:1826638 [wandb_init.py:init():804] starting run threads in backend +2024-03-12 14:28:59,052 INFO MainThread:1826638 [wandb_run.py:_console_start():2241] atexit reg +2024-03-12 14:28:59,052 INFO MainThread:1826638 [wandb_run.py:_redirect():2096] redirect: wrap_raw +2024-03-12 14:28:59,052 INFO MainThread:1826638 [wandb_run.py:_redirect():2161] Wrapping output streams. +2024-03-12 14:28:59,052 INFO MainThread:1826638 [wandb_run.py:_redirect():2186] Redirects installed. +2024-03-12 14:28:59,053 INFO MainThread:1826638 [wandb_init.py:init():847] run started, returning control to user process +2024-03-12 21:39:13,274 INFO MainThread:1826638 [wandb_run.py:_finish():1970] finishing run lilei_stones/hf-vlms/zo3nkbji +2024-03-12 21:39:13,275 INFO MainThread:1826638 [wandb_run.py:_atexit_cleanup():2210] got exitcode: 0 +2024-03-12 21:39:13,275 INFO MainThread:1826638 [wandb_run.py:_restore():2193] restore +2024-03-12 21:39:13,275 INFO MainThread:1826638 [wandb_run.py:_restore():2199] restore done +2024-03-12 21:39:23,326 INFO MainThread:1826638 [wandb_run.py:_footer_history_summary_info():3866] rendering history +2024-03-12 21:39:23,326 INFO MainThread:1826638 [wandb_run.py:_footer_history_summary_info():3898] rendering summary +2024-03-12 21:39:23,336 INFO MainThread:1826638 [wandb_run.py:_footer_sync_info():3825] logging synced files diff --git a/qformer2_128/wandb/run-20240312_142853-zo3nkbji/run-zo3nkbji.wandb b/qformer2_128/wandb/run-20240312_142853-zo3nkbji/run-zo3nkbji.wandb new file mode 100644 index 0000000000000000000000000000000000000000..837ee26d0ca52dad23080f171c213faeb4b2f38c --- /dev/null +++ b/qformer2_128/wandb/run-20240312_142853-zo3nkbji/run-zo3nkbji.wandb @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:50613d50b5ac873911c6151848de586e16bc337cd465c8d050a36f2bd8fa66a8 +size 12537234 diff --git a/qformer2_256/checkpoints/latest-checkpoint.pt b/qformer2_256/checkpoints/latest-checkpoint.pt new file mode 100644 index 0000000000000000000000000000000000000000..599018c910720cb8368a10180162308ffdea3d61 --- /dev/null +++ b/qformer2_256/checkpoints/latest-checkpoint.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:71fa610c966fd33aece634123f5d64bb5d771634891037615d56067c9f0fa99d +size 28824506550 diff --git a/qformer2_256/config.json b/qformer2_256/config.json new file mode 100644 index 0000000000000000000000000000000000000000..b6e812aa96698d654a2d24f3ad6c341b4b32b117 --- /dev/null +++ b/qformer2_256/config.json @@ -0,0 +1,60 @@ +{ + "dataset": { + "align_stage_components": [ + "download/llava-laion-cc-sbu-558k/chat.json", + "download/llava-laion-cc-sbu-558k" + ], + "dataset_id": "llava-v15", + "dataset_root_dir": "data", + "finetune_stage_components": [ + "download/llava-v1.5-instruct/llava_v1_5_mix665k.json", + "download/llava-v1.5-instruct" + ], + "type": "llava-v15" + }, + "hf_token": ".hf_token", + "max_length": 4096, + "model": { + "align_epochs": 1, + "align_global_batch_size": 256, + "align_learning_rate": 0.001, + "align_lr_scheduler_type": "linear-warmup+cosine-decay", + "align_max_grad_norm": 1.0, + "align_max_steps": null, + "align_per_device_batch_size": 16, + "align_train_strategy": "fsdp-shard-grad-op", + "align_warmup_ratio": 0.03, + "align_weight_decay": 0.0, + "arch_specifier": "qformer2_256", + "enable_gradient_checkpointing": true, + "enable_mixed_precision_training": true, + "finetune_epochs": 1, + "finetune_global_batch_size": 128, + "finetune_learning_rate": 2e-05, + "finetune_lr_scheduler_type": "linear-warmup+cosine-decay", + "finetune_max_grad_norm": 1.0, + "finetune_max_steps": null, + "finetune_per_device_batch_size": 16, + "finetune_train_strategy": "fsdp-full-shard", + "finetune_warmup_ratio": 0.03, + "finetune_weight_decay": 0.1, + "image_resize_strategy": "letterbox", + "llm_backbone_id": "vicuna-v15-7b", + "llm_max_length": 2048, + "model_id": "one-stage+7b_qformer2_256", + "reduce_in_full_precision": false, + "type": "one-stage+7b", + "vision_backbone_id": "clip-vit-l-336px" + }, + "pretrained_checkpoint": null, + "run_id": "qformer2_256", + "run_root_dir": "runs", + "seed": 7, + "stage": "finetune", + "trackers": [ + "jsonl", + "wandb" + ], + "wandb_entity": "lilei_stones", + "wandb_project": "hf-vlms" +} \ No newline at end of file diff --git a/qformer2_256/config.yaml b/qformer2_256/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..98f313aab9f6716849bb2e477f6299989160900e --- /dev/null +++ b/qformer2_256/config.yaml @@ -0,0 +1,53 @@ +dataset: + align_stage_components: + - download/llava-laion-cc-sbu-558k/chat.json + - download/llava-laion-cc-sbu-558k + dataset_id: llava-v15 + dataset_root_dir: data + finetune_stage_components: + - download/llava-v1.5-instruct/llava_v1_5_mix665k.json + - download/llava-v1.5-instruct + type: llava-v15 +hf_token: .hf_token +max_length: 4096 +model: + align_epochs: 1 + align_global_batch_size: 256 + align_learning_rate: 0.001 + align_lr_scheduler_type: linear-warmup+cosine-decay + align_max_grad_norm: 1.0 + align_max_steps: null + align_per_device_batch_size: 16 + align_train_strategy: fsdp-shard-grad-op + align_warmup_ratio: 0.03 + align_weight_decay: 0.0 + arch_specifier: qformer2_256 + enable_gradient_checkpointing: true + enable_mixed_precision_training: true + finetune_epochs: 1 + finetune_global_batch_size: 128 + finetune_learning_rate: 2.0e-05 + finetune_lr_scheduler_type: linear-warmup+cosine-decay + finetune_max_grad_norm: 1.0 + finetune_max_steps: null + finetune_per_device_batch_size: 16 + finetune_train_strategy: fsdp-full-shard + finetune_warmup_ratio: 0.03 + finetune_weight_decay: 0.1 + image_resize_strategy: letterbox + llm_backbone_id: vicuna-v15-7b + llm_max_length: 2048 + model_id: one-stage+7b_qformer2_256 + reduce_in_full_precision: false + type: one-stage+7b + vision_backbone_id: clip-vit-l-336px +pretrained_checkpoint: null +run_id: qformer2_256 +run_root_dir: runs +seed: 7 +stage: finetune +trackers: +- jsonl +- wandb +wandb_entity: lilei_stones +wandb_project: hf-vlms diff --git a/qformer2_256/qformer2_256.jsonl b/qformer2_256/qformer2_256.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..c3d3d0036254f711bbe0b9ec27f7f1bcdffb65ce --- /dev/null +++ b/qformer2_256/qformer2_256.jsonl @@ -0,0 +1,5198 @@ +{"Finetune/Learning Rate": 1.2903225806451614e-07, "Finetune/Loss": 7.468265533447266, "Finetune/Loss (Raw)": 7.468265533447266, "Finetune/Step": 1, "Finetune/Step Time": 26.131122827529907} +{"Finetune/Learning Rate": 2.580645161290323e-07, "Finetune/Loss": 7.871462345123291, "Finetune/Loss (Raw)": 8.274659156799316, "Finetune/Step": 2, "Finetune/Step Time": 16.93863010406494} +{"Finetune/Learning Rate": 3.870967741935484e-07, "Finetune/Loss": 6.891812801361084, "Finetune/Loss (Raw)": 4.932514190673828, "Finetune/Step": 3, "Finetune/Step Time": 12.69995109240214} +{"Finetune/Learning Rate": 5.161290322580646e-07, "Finetune/Loss": 6.110063552856445, "Finetune/Loss (Raw)": 3.764815330505371, "Finetune/Step": 4, "Finetune/Step Time": 10.624689936637878} +{"Finetune/Learning Rate": 6.451612903225807e-07, "Finetune/Loss": 5.760235786437988, "Finetune/Loss (Raw)": 4.360922813415527, "Finetune/Step": 5, "Finetune/Step Time": 9.56146764755249} +{"Finetune/Learning Rate": 7.741935483870968e-07, "Finetune/Loss": 5.389391899108887, "Finetune/Loss (Raw)": 3.5351722240448, "Finetune/Step": 6, "Finetune/Step Time": 8.805065274238586} +{"Finetune/Learning Rate": 9.032258064516129e-07, "Finetune/Loss": 5.0463175773620605, "Finetune/Loss (Raw)": 2.9878733158111572, "Finetune/Step": 7, "Finetune/Step Time": 8.234421048845563} +{"Finetune/Learning Rate": 1.0322580645161291e-06, "Finetune/Loss": 4.721767425537109, "Finetune/Loss (Raw)": 2.449915885925293, "Finetune/Step": 8, "Finetune/Step Time": 7.985710889101028} +{"Finetune/Learning Rate": 1.1612903225806454e-06, "Finetune/Loss": 4.402499675750732, "Finetune/Loss (Raw)": 1.848355770111084, "Finetune/Step": 9, "Finetune/Step Time": 7.648053910997179} +{"Finetune/Learning Rate": 1.2903225806451614e-06, "Finetune/Loss": 4.190001010894775, "Finetune/Loss (Raw)": 2.277512311935425, "Finetune/Step": 10, "Finetune/Step Time": 7.636433553695679} +{"Finetune/Learning Rate": 1.4193548387096776e-06, "Finetune/Loss": 3.973860025405884, "Finetune/Loss (Raw)": 1.8124505281448364, "Finetune/Step": 11, "Finetune/Step Time": 7.3612329309636895} +{"Finetune/Learning Rate": 1.5483870967741937e-06, "Finetune/Loss": 3.7770395278930664, "Finetune/Loss (Raw)": 1.612015962600708, "Finetune/Step": 12, "Finetune/Step Time": 7.14748873313268} +{"Finetune/Learning Rate": 1.67741935483871e-06, "Finetune/Loss": 3.593334913253784, "Finetune/Loss (Raw)": 1.3888782262802124, "Finetune/Step": 13, "Finetune/Step Time": 6.964043378829956} +{"Finetune/Learning Rate": 1.8064516129032258e-06, "Finetune/Loss": 3.4516496658325195, "Finetune/Loss (Raw)": 1.6097429990768433, "Finetune/Step": 14, "Finetune/Step Time": 6.785414627620152} +{"Finetune/Learning Rate": 1.935483870967742e-06, "Finetune/Loss": 3.346670627593994, "Finetune/Loss (Raw)": 1.8769629001617432, "Finetune/Step": 15, "Finetune/Step Time": 6.7257527828216555} +{"Finetune/Learning Rate": 2.0645161290322582e-06, "Finetune/Loss": 3.2146174907684326, "Finetune/Loss (Raw)": 1.2338210344314575, "Finetune/Step": 16, "Finetune/Step Time": 6.680927202105522} +{"Finetune/Learning Rate": 2.1935483870967745e-06, "Finetune/Loss": 3.1026721000671387, "Finetune/Loss (Raw)": 1.3115472793579102, "Finetune/Step": 17, "Finetune/Step Time": 6.577338330885944} +{"Finetune/Learning Rate": 2.3225806451612907e-06, "Finetune/Loss": 2.9968345165252686, "Finetune/Loss (Raw)": 1.1975919008255005, "Finetune/Step": 18, "Finetune/Step Time": 6.51476158036126} +{"Finetune/Learning Rate": 2.4516129032258066e-06, "Finetune/Loss": 2.905360698699951, "Finetune/Loss (Raw)": 1.258834719657898, "Finetune/Step": 19, "Finetune/Step Time": 6.407304387343557} +{"Finetune/Learning Rate": 2.580645161290323e-06, "Finetune/Loss": 2.8239080905914307, "Finetune/Loss (Raw)": 1.2763094902038574, "Finetune/Step": 20, "Finetune/Step Time": 6.297149682044983} +{"Finetune/Learning Rate": 2.709677419354839e-06, "Finetune/Loss": 2.7700703144073486, "Finetune/Loss (Raw)": 1.6933125257492065, "Finetune/Step": 21, "Finetune/Step Time": 6.28697563353039} +{"Finetune/Learning Rate": 2.8387096774193553e-06, "Finetune/Loss": 2.699847459793091, "Finetune/Loss (Raw)": 1.225165605545044, "Finetune/Step": 22, "Finetune/Step Time": 6.224202947183088} +{"Finetune/Learning Rate": 2.967741935483871e-06, "Finetune/Loss": 2.5925772190093994, "Finetune/Loss (Raw)": 0.23263312876224518, "Finetune/Step": 23, "Finetune/Step Time": 6.314219651014908} +{"Finetune/Learning Rate": 3.0967741935483874e-06, "Finetune/Loss": 2.5404982566833496, "Finetune/Loss (Raw)": 1.3426878452301025, "Finetune/Step": 24, "Finetune/Step Time": 6.2273276547590894} +{"Finetune/Learning Rate": 3.225806451612903e-06, "Finetune/Loss": 2.4856486320495605, "Finetune/Loss (Raw)": 1.1692581176757812, "Finetune/Step": 25, "Finetune/Step Time": 6.174411392211914} +{"Finetune/Learning Rate": 3.35483870967742e-06, "Finetune/Loss": 2.400360584259033, "Finetune/Loss (Raw)": 0.2681545913219452, "Finetune/Step": 26, "Finetune/Step Time": 6.292556157478919} +{"Finetune/Learning Rate": 3.4838709677419357e-06, "Finetune/Loss": 2.361961603164673, "Finetune/Loss (Raw)": 1.3635940551757812, "Finetune/Step": 27, "Finetune/Step Time": 6.208586966549909} +{"Finetune/Learning Rate": 3.6129032258064515e-06, "Finetune/Loss": 2.321213483810425, "Finetune/Loss (Raw)": 1.2210056781768799, "Finetune/Step": 28, "Finetune/Step Time": 6.1402155331202914} +{"Finetune/Learning Rate": 3.741935483870968e-06, "Finetune/Loss": 2.2891640663146973, "Finetune/Loss (Raw)": 1.3917888402938843, "Finetune/Step": 29, "Finetune/Step Time": 6.127956324610217} +{"Finetune/Learning Rate": 3.870967741935484e-06, "Finetune/Loss": 2.2510154247283936, "Finetune/Loss (Raw)": 1.144696593284607, "Finetune/Step": 30, "Finetune/Step Time": 6.119674269358317} +{"Finetune/Learning Rate": 4.000000000000001e-06, "Finetune/Loss": 2.2202486991882324, "Finetune/Loss (Raw)": 1.297253966331482, "Finetune/Step": 31, "Finetune/Step Time": 6.086117767518567} +{"Finetune/Learning Rate": 4.1290322580645165e-06, "Finetune/Loss": 2.188126802444458, "Finetune/Loss (Raw)": 1.1923418045043945, "Finetune/Step": 32, "Finetune/Step Time": 6.109574608504772} +{"Finetune/Learning Rate": 4.258064516129032e-06, "Finetune/Loss": 2.152468681335449, "Finetune/Loss (Raw)": 1.0114058256149292, "Finetune/Step": 33, "Finetune/Step Time": 6.084713206146702} +{"Finetune/Learning Rate": 4.387096774193549e-06, "Finetune/Loss": 2.1205923557281494, "Finetune/Loss (Raw)": 1.0686752796173096, "Finetune/Step": 34, "Finetune/Step Time": 6.04446402016808} +{"Finetune/Learning Rate": 4.516129032258065e-06, "Finetune/Loss": 2.0989291667938232, "Finetune/Loss (Raw)": 1.3623772859573364, "Finetune/Step": 35, "Finetune/Step Time": 6.01783401625497} +{"Finetune/Learning Rate": 4.6451612903225815e-06, "Finetune/Loss": 2.0729572772979736, "Finetune/Loss (Raw)": 1.163948655128479, "Finetune/Step": 36, "Finetune/Step Time": 6.026629613505469} +{"Finetune/Learning Rate": 4.774193548387097e-06, "Finetune/Loss": 2.052586317062378, "Finetune/Loss (Raw)": 1.3192334175109863, "Finetune/Step": 37, "Finetune/Step Time": 6.075744261612764} +{"Finetune/Learning Rate": 4.903225806451613e-06, "Finetune/Loss": 2.028999090194702, "Finetune/Loss (Raw)": 1.156272530555725, "Finetune/Step": 38, "Finetune/Step Time": 6.0291616289239185} +{"Finetune/Learning Rate": 5.032258064516129e-06, "Finetune/Loss": 2.007312774658203, "Finetune/Loss (Raw)": 1.1832287311553955, "Finetune/Step": 39, "Finetune/Step Time": 5.983440820987408} +{"Finetune/Learning Rate": 5.161290322580646e-06, "Finetune/Loss": 1.9866962432861328, "Finetune/Loss (Raw)": 1.1826486587524414, "Finetune/Step": 40, "Finetune/Step Time": 5.989866989850998} +{"Finetune/Learning Rate": 5.290322580645162e-06, "Finetune/Loss": 1.9694899320602417, "Finetune/Loss (Raw)": 1.2812466621398926, "Finetune/Step": 41, "Finetune/Step Time": 5.936317612485188} +{"Finetune/Learning Rate": 5.419354838709678e-06, "Finetune/Loss": 1.9520235061645508, "Finetune/Loss (Raw)": 1.2358945608139038, "Finetune/Step": 42, "Finetune/Step Time": 5.897274233046032} +{"Finetune/Learning Rate": 5.548387096774194e-06, "Finetune/Loss": 1.9341307878494263, "Finetune/Loss (Raw)": 1.1826424598693848, "Finetune/Step": 43, "Finetune/Step Time": 5.894211741380913} +{"Finetune/Learning Rate": 5.677419354838711e-06, "Finetune/Loss": 1.9170427322387695, "Finetune/Loss (Raw)": 1.1822516918182373, "Finetune/Step": 44, "Finetune/Step Time": 5.88202090696855} +{"Finetune/Learning Rate": 5.806451612903226e-06, "Finetune/Loss": 1.898313283920288, "Finetune/Loss (Raw)": 1.0742170810699463, "Finetune/Step": 45, "Finetune/Step Time": 5.941508446799384} +{"Finetune/Learning Rate": 5.935483870967742e-06, "Finetune/Loss": 1.863333821296692, "Finetune/Loss (Raw)": 0.2892591655254364, "Finetune/Step": 46, "Finetune/Step Time": 6.032595427139945} +{"Finetune/Learning Rate": 6.064516129032259e-06, "Finetune/Loss": 1.848137378692627, "Finetune/Loss (Raw)": 1.1490989923477173, "Finetune/Step": 47, "Finetune/Step Time": 6.021208829068123} +{"Finetune/Learning Rate": 6.193548387096775e-06, "Finetune/Loss": 1.8332629203796387, "Finetune/Loss (Raw)": 1.1341673135757446, "Finetune/Step": 48, "Finetune/Step Time": 6.027327741185824} +{"Finetune/Learning Rate": 6.3225806451612906e-06, "Finetune/Loss": 1.819712519645691, "Finetune/Loss (Raw)": 1.1692922115325928, "Finetune/Step": 49, "Finetune/Step Time": 6.001857431567445} +{"Finetune/Learning Rate": 6.451612903225806e-06, "Finetune/Loss": 1.803945541381836, "Finetune/Loss (Raw)": 1.0313671827316284, "Finetune/Step": 50, "Finetune/Step Time": 5.9813041639328} +{"Finetune/Learning Rate": 6.580645161290323e-06, "Finetune/Loss": 1.791102409362793, "Finetune/Loss (Raw)": 1.1489415168762207, "Finetune/Step": 51, "Finetune/Step Time": 5.954180193882363} +{"Finetune/Learning Rate": 6.70967741935484e-06, "Finetune/Loss": 1.7797447443008423, "Finetune/Loss (Raw)": 1.200500726699829, "Finetune/Step": 52, "Finetune/Step Time": 5.945672466204717} +{"Finetune/Learning Rate": 6.838709677419355e-06, "Finetune/Loss": 1.7636029720306396, "Finetune/Loss (Raw)": 0.9242295622825623, "Finetune/Step": 53, "Finetune/Step Time": 5.958210378322962} +{"Finetune/Learning Rate": 6.967741935483871e-06, "Finetune/Loss": 1.748427391052246, "Finetune/Loss (Raw)": 0.9441211223602295, "Finetune/Step": 54, "Finetune/Step Time": 5.951679150263469} +{"Finetune/Learning Rate": 7.096774193548388e-06, "Finetune/Loss": 1.7379436492919922, "Finetune/Loss (Raw)": 1.1718339920043945, "Finetune/Step": 55, "Finetune/Step Time": 5.952269207347523} +{"Finetune/Learning Rate": 7.225806451612903e-06, "Finetune/Loss": 1.7268414497375488, "Finetune/Loss (Raw)": 1.1162109375, "Finetune/Step": 56, "Finetune/Step Time": 5.9624640473297665} +{"Finetune/Learning Rate": 7.35483870967742e-06, "Finetune/Loss": 1.7145155668258667, "Finetune/Loss (Raw)": 1.0242719650268555, "Finetune/Step": 57, "Finetune/Step Time": 5.939692953176666} +{"Finetune/Learning Rate": 7.483870967741936e-06, "Finetune/Loss": 1.7032712697982788, "Finetune/Loss (Raw)": 1.062342882156372, "Finetune/Step": 58, "Finetune/Step Time": 5.9660415279454195} +{"Finetune/Learning Rate": 7.612903225806451e-06, "Finetune/Loss": 1.6930632591247559, "Finetune/Loss (Raw)": 1.1009998321533203, "Finetune/Step": 59, "Finetune/Step Time": 5.972909882917243} +{"Finetune/Learning Rate": 7.741935483870968e-06, "Finetune/Loss": 1.6828837394714355, "Finetune/Loss (Raw)": 1.082289218902588, "Finetune/Step": 60, "Finetune/Step Time": 5.947927618026734} +{"Finetune/Learning Rate": 7.870967741935484e-06, "Finetune/Loss": 1.6741957664489746, "Finetune/Loss (Raw)": 1.1529184579849243, "Finetune/Step": 61, "Finetune/Step Time": 5.931765849473046} +{"Finetune/Learning Rate": 8.000000000000001e-06, "Finetune/Loss": 1.666490912437439, "Finetune/Loss (Raw)": 1.1964969635009766, "Finetune/Step": 62, "Finetune/Step Time": 5.907490568776285} +{"Finetune/Learning Rate": 8.129032258064517e-06, "Finetune/Loss": 1.6593818664550781, "Finetune/Loss (Raw)": 1.2186150550842285, "Finetune/Step": 63, "Finetune/Step Time": 5.88589686817593} +{"Finetune/Learning Rate": 8.258064516129033e-06, "Finetune/Loss": 1.6500803232192993, "Finetune/Loss (Raw)": 1.0640872716903687, "Finetune/Step": 64, "Finetune/Step Time": 5.921369351446629} +{"Finetune/Learning Rate": 8.387096774193549e-06, "Finetune/Loss": 1.6409180164337158, "Finetune/Loss (Raw)": 1.0545285940170288, "Finetune/Step": 65, "Finetune/Step Time": 5.914988844211285} +{"Finetune/Learning Rate": 8.516129032258065e-06, "Finetune/Loss": 1.6313621997833252, "Finetune/Loss (Raw)": 1.0102308988571167, "Finetune/Step": 66, "Finetune/Step Time": 5.900443969350873} +{"Finetune/Learning Rate": 8.64516129032258e-06, "Finetune/Loss": 1.6204649209976196, "Finetune/Loss (Raw)": 0.9012561440467834, "Finetune/Step": 67, "Finetune/Step Time": 5.875971320849746} +{"Finetune/Learning Rate": 8.774193548387098e-06, "Finetune/Loss": 1.6132597923278809, "Finetune/Loss (Raw)": 1.1305180788040161, "Finetune/Step": 68, "Finetune/Step Time": 5.8736394187983345} +{"Finetune/Learning Rate": 8.903225806451614e-06, "Finetune/Loss": 1.6066389083862305, "Finetune/Loss (Raw)": 1.1564117670059204, "Finetune/Step": 69, "Finetune/Step Time": 5.850151694339255} +{"Finetune/Learning Rate": 9.03225806451613e-06, "Finetune/Loss": 1.600831151008606, "Finetune/Loss (Raw)": 1.2000993490219116, "Finetune/Step": 70, "Finetune/Step Time": 5.8305734327861245} +{"Finetune/Learning Rate": 9.161290322580645e-06, "Finetune/Loss": 1.59139084815979, "Finetune/Loss (Raw)": 0.930574893951416, "Finetune/Step": 71, "Finetune/Step Time": 5.813086385458288} +{"Finetune/Learning Rate": 9.290322580645163e-06, "Finetune/Loss": 1.5835644006729126, "Finetune/Loss (Raw)": 1.027884840965271, "Finetune/Step": 72, "Finetune/Step Time": 5.789331568611993} +{"Finetune/Learning Rate": 9.419354838709677e-06, "Finetune/Loss": 1.574595332145691, "Finetune/Loss (Raw)": 0.9288157820701599, "Finetune/Step": 73, "Finetune/Step Time": 5.771751697749307} +{"Finetune/Learning Rate": 9.548387096774195e-06, "Finetune/Loss": 1.566006064414978, "Finetune/Loss (Raw)": 0.9389851093292236, "Finetune/Step": 74, "Finetune/Step Time": 5.754351851102468} +{"Finetune/Learning Rate": 9.67741935483871e-06, "Finetune/Loss": 1.5561952590942383, "Finetune/Loss (Raw)": 0.8302009105682373, "Finetune/Step": 75, "Finetune/Step Time": 5.749417543411255} +{"Finetune/Learning Rate": 9.806451612903226e-06, "Finetune/Loss": 1.5486432313919067, "Finetune/Loss (Raw)": 0.9822400212287903, "Finetune/Step": 76, "Finetune/Step Time": 5.742055547864814} +{"Finetune/Learning Rate": 9.935483870967742e-06, "Finetune/Loss": 1.5429503917694092, "Finetune/Loss (Raw)": 1.1102930307388306, "Finetune/Step": 77, "Finetune/Step Time": 5.721933590901362} +{"Finetune/Learning Rate": 1.0064516129032258e-05, "Finetune/Loss": 1.538803219795227, "Finetune/Loss (Raw)": 1.2194730043411255, "Finetune/Step": 78, "Finetune/Step Time": 5.702040125162173} +{"Finetune/Learning Rate": 1.0193548387096774e-05, "Finetune/Loss": 1.5343513488769531, "Finetune/Loss (Raw)": 1.1870983839035034, "Finetune/Step": 79, "Finetune/Step Time": 5.686922510967979} +{"Finetune/Learning Rate": 1.0322580645161291e-05, "Finetune/Loss": 1.5273540019989014, "Finetune/Loss (Raw)": 0.9745631217956543, "Finetune/Step": 80, "Finetune/Step Time": 5.692722880840302} +{"Finetune/Learning Rate": 1.0451612903225807e-05, "Finetune/Loss": 1.5226012468338013, "Finetune/Loss (Raw)": 1.142380952835083, "Finetune/Step": 81, "Finetune/Step Time": 5.690403935350018} +{"Finetune/Learning Rate": 1.0580645161290325e-05, "Finetune/Loss": 1.5175048112869263, "Finetune/Loss (Raw)": 1.1046953201293945, "Finetune/Step": 82, "Finetune/Step Time": 5.674467656670547} +{"Finetune/Learning Rate": 1.070967741935484e-05, "Finetune/Loss": 1.5108411312103271, "Finetune/Loss (Raw)": 0.9644235372543335, "Finetune/Step": 83, "Finetune/Step Time": 5.683150391980826} +{"Finetune/Learning Rate": 1.0838709677419356e-05, "Finetune/Loss": 1.5070215463638306, "Finetune/Loss (Raw)": 1.1899880170822144, "Finetune/Step": 84, "Finetune/Step Time": 5.685418946402414} +{"Finetune/Learning Rate": 1.096774193548387e-05, "Finetune/Loss": 1.501618504524231, "Finetune/Loss (Raw)": 1.0477615594863892, "Finetune/Step": 85, "Finetune/Step Time": 5.671665612389059} +{"Finetune/Learning Rate": 1.1096774193548388e-05, "Finetune/Loss": 1.4881999492645264, "Finetune/Loss (Raw)": 0.34762799739837646, "Finetune/Step": 86, "Finetune/Step Time": 5.711548153744188} +{"Finetune/Learning Rate": 1.1225806451612904e-05, "Finetune/Loss": 1.482845425605774, "Finetune/Loss (Raw)": 1.0223623514175415, "Finetune/Step": 87, "Finetune/Step Time": 5.69756451146356} +{"Finetune/Learning Rate": 1.1354838709677421e-05, "Finetune/Loss": 1.479845643043518, "Finetune/Loss (Raw)": 1.2188464403152466, "Finetune/Step": 88, "Finetune/Step Time": 5.705306979742917} +{"Finetune/Learning Rate": 1.1483870967741937e-05, "Finetune/Loss": 1.475252628326416, "Finetune/Loss (Raw)": 1.0710783004760742, "Finetune/Step": 89, "Finetune/Step Time": 5.6920014767164595} +{"Finetune/Learning Rate": 1.1612903225806453e-05, "Finetune/Loss": 1.4715322256088257, "Finetune/Loss (Raw)": 1.140405297279358, "Finetune/Step": 90, "Finetune/Step Time": 5.699976754188538} +{"Finetune/Learning Rate": 1.1741935483870967e-05, "Finetune/Loss": 1.4682034254074097, "Finetune/Loss (Raw)": 1.1686192750930786, "Finetune/Step": 91, "Finetune/Step Time": 5.702719203718416} +{"Finetune/Learning Rate": 1.1870967741935484e-05, "Finetune/Loss": 1.4652776718139648, "Finetune/Loss (Raw)": 1.1990360021591187, "Finetune/Step": 92, "Finetune/Step Time": 5.691545779290407} +{"Finetune/Learning Rate": 1.2e-05, "Finetune/Loss": 1.4594136476516724, "Finetune/Loss (Raw)": 0.9199244976043701, "Finetune/Step": 93, "Finetune/Step Time": 5.683369077661986} +{"Finetune/Learning Rate": 1.2129032258064518e-05, "Finetune/Loss": 1.4562257528305054, "Finetune/Loss (Raw)": 1.159747838973999, "Finetune/Step": 94, "Finetune/Step Time": 5.667432681043097} +{"Finetune/Learning Rate": 1.2258064516129034e-05, "Finetune/Loss": 1.452346920967102, "Finetune/Loss (Raw)": 1.08772873878479, "Finetune/Step": 95, "Finetune/Step Time": 5.655676312195627} +{"Finetune/Learning Rate": 1.238709677419355e-05, "Finetune/Loss": 1.4485095739364624, "Finetune/Loss (Raw)": 1.0839828252792358, "Finetune/Step": 96, "Finetune/Step Time": 5.658483870327473} +{"Finetune/Learning Rate": 1.2516129032258067e-05, "Finetune/Loss": 1.4442087411880493, "Finetune/Loss (Raw)": 1.0313365459442139, "Finetune/Step": 97, "Finetune/Step Time": 5.653992222756455} +{"Finetune/Learning Rate": 1.2645161290322581e-05, "Finetune/Loss": 1.4409832954406738, "Finetune/Loss (Raw)": 1.1281002759933472, "Finetune/Step": 98, "Finetune/Step Time": 5.64205002784729} +{"Finetune/Learning Rate": 1.2774193548387097e-05, "Finetune/Loss": 1.428631067276001, "Finetune/Loss (Raw)": 0.2181185930967331, "Finetune/Step": 99, "Finetune/Step Time": 5.669135158712214} +{"Finetune/Learning Rate": 1.2903225806451613e-05, "Finetune/Loss": 1.4264665842056274, "Finetune/Loss (Raw)": 1.212189793586731, "Finetune/Step": 100, "Finetune/Step Time": 5.66154536485672} +{"Finetune/Learning Rate": 1.303225806451613e-05, "Finetune/Loss": 1.422756314277649, "Finetune/Loss (Raw)": 1.051713466644287, "Finetune/Step": 101, "Finetune/Step Time": 5.6634456426790445} +{"Finetune/Learning Rate": 1.3161290322580646e-05, "Finetune/Loss": 1.4206534624099731, "Finetune/Loss (Raw)": 1.2082622051239014, "Finetune/Step": 102, "Finetune/Step Time": 5.647005473866182} +{"Finetune/Learning Rate": 1.3290322580645164e-05, "Finetune/Loss": 1.409700870513916, "Finetune/Loss (Raw)": 0.2925461530685425, "Finetune/Step": 103, "Finetune/Step Time": 5.674309235174679} +{"Finetune/Learning Rate": 1.341935483870968e-05, "Finetune/Loss": 1.4071879386901855, "Finetune/Loss (Raw)": 1.1483336687088013, "Finetune/Step": 104, "Finetune/Step Time": 5.660739387457188} +{"Finetune/Learning Rate": 1.3548387096774194e-05, "Finetune/Loss": 1.4038738012313843, "Finetune/Loss (Raw)": 1.059214472770691, "Finetune/Step": 105, "Finetune/Step Time": 5.644064367385138} +{"Finetune/Learning Rate": 1.367741935483871e-05, "Finetune/Loss": 1.4006341695785522, "Finetune/Loss (Raw)": 1.060482382774353, "Finetune/Step": 106, "Finetune/Step Time": 5.6317348862594026} +{"Finetune/Learning Rate": 1.3806451612903227e-05, "Finetune/Loss": 1.3981188535690308, "Finetune/Loss (Raw)": 1.1315125226974487, "Finetune/Step": 107, "Finetune/Step Time": 5.620287485211809} +{"Finetune/Learning Rate": 1.3935483870967743e-05, "Finetune/Loss": 1.3945579528808594, "Finetune/Loss (Raw)": 1.013527750968933, "Finetune/Step": 108, "Finetune/Step Time": 5.61205275411959} +{"Finetune/Learning Rate": 1.406451612903226e-05, "Finetune/Loss": 1.3922386169433594, "Finetune/Loss (Raw)": 1.1417549848556519, "Finetune/Step": 109, "Finetune/Step Time": 5.601453183987819} +{"Finetune/Learning Rate": 1.4193548387096776e-05, "Finetune/Loss": 1.3891093730926514, "Finetune/Loss (Raw)": 1.048011302947998, "Finetune/Step": 110, "Finetune/Step Time": 5.589387529546564} +{"Finetune/Learning Rate": 1.4322580645161292e-05, "Finetune/Loss": 1.3866208791732788, "Finetune/Loss (Raw)": 1.1128884553909302, "Finetune/Step": 111, "Finetune/Step Time": 5.594901151485272} +{"Finetune/Learning Rate": 1.4451612903225806e-05, "Finetune/Loss": 1.3838858604431152, "Finetune/Loss (Raw)": 1.0802810192108154, "Finetune/Step": 112, "Finetune/Step Time": 5.603074723056385} +{"Finetune/Learning Rate": 1.4580645161290324e-05, "Finetune/Loss": 1.3816020488739014, "Finetune/Loss (Raw)": 1.1258348226547241, "Finetune/Step": 113, "Finetune/Step Time": 5.606291304647395} +{"Finetune/Learning Rate": 1.470967741935484e-05, "Finetune/Loss": 1.3716890811920166, "Finetune/Loss (Raw)": 0.25151562690734863, "Finetune/Step": 114, "Finetune/Step Time": 5.632445699290225} +{"Finetune/Learning Rate": 1.4838709677419357e-05, "Finetune/Loss": 1.3691943883895874, "Finetune/Loss (Raw)": 1.0848197937011719, "Finetune/Step": 115, "Finetune/Step Time": 5.628412912202918} +{"Finetune/Learning Rate": 1.4967741935483873e-05, "Finetune/Loss": 1.3654179573059082, "Finetune/Loss (Raw)": 0.931113600730896, "Finetune/Step": 116, "Finetune/Step Time": 5.638990840007519} +{"Finetune/Learning Rate": 1.5096774193548389e-05, "Finetune/Loss": 1.363224983215332, "Finetune/Loss (Raw)": 1.1088305711746216, "Finetune/Step": 117, "Finetune/Step Time": 5.634112115599152} +{"Finetune/Learning Rate": 1.5225806451612903e-05, "Finetune/Loss": 1.3613226413726807, "Finetune/Loss (Raw)": 1.1387617588043213, "Finetune/Step": 118, "Finetune/Step Time": 5.623911821236045} +{"Finetune/Learning Rate": 1.535483870967742e-05, "Finetune/Loss": 1.358622431755066, "Finetune/Loss (Raw)": 1.0399737358093262, "Finetune/Step": 119, "Finetune/Step Time": 5.6151100487268275} +{"Finetune/Learning Rate": 1.5483870967741936e-05, "Finetune/Loss": 1.3563395738601685, "Finetune/Loss (Raw)": 1.0846832990646362, "Finetune/Step": 120, "Finetune/Step Time": 5.610596841573715} +{"Finetune/Learning Rate": 1.5612903225806454e-05, "Finetune/Loss": 1.3535757064819336, "Finetune/Loss (Raw)": 1.0219284296035767, "Finetune/Step": 121, "Finetune/Step Time": 5.6181626260773205} +{"Finetune/Learning Rate": 1.5741935483870968e-05, "Finetune/Loss": 1.3503972291946411, "Finetune/Loss (Raw)": 0.9658177495002747, "Finetune/Step": 122, "Finetune/Step Time": 5.6094625953768125} +{"Finetune/Learning Rate": 1.5870967741935485e-05, "Finetune/Loss": 1.347657561302185, "Finetune/Loss (Raw)": 1.0134001970291138, "Finetune/Step": 123, "Finetune/Step Time": 5.600236580623844} +{"Finetune/Learning Rate": 1.6000000000000003e-05, "Finetune/Loss": 1.3458713293075562, "Finetune/Loss (Raw)": 1.1261626482009888, "Finetune/Step": 124, "Finetune/Step Time": 5.593318848840652} +{"Finetune/Learning Rate": 1.6129032258064517e-05, "Finetune/Loss": 1.343187928199768, "Finetune/Loss (Raw)": 1.0104392766952515, "Finetune/Step": 125, "Finetune/Step Time": 5.586648616790772} +{"Finetune/Learning Rate": 1.6258064516129034e-05, "Finetune/Loss": 1.3413022756576538, "Finetune/Loss (Raw)": 1.1055861711502075, "Finetune/Step": 126, "Finetune/Step Time": 5.598655299534873} +{"Finetune/Learning Rate": 1.638709677419355e-05, "Finetune/Loss": 1.3385155200958252, "Finetune/Loss (Raw)": 0.9874007105827332, "Finetune/Step": 127, "Finetune/Step Time": 5.588644538338729} +{"Finetune/Learning Rate": 1.6516129032258066e-05, "Finetune/Loss": 1.3369140625, "Finetune/Loss (Raw)": 1.1335322856903076, "Finetune/Step": 128, "Finetune/Step Time": 5.581814382225275} +{"Finetune/Learning Rate": 1.6645161290322583e-05, "Finetune/Loss": 1.2875690460205078, "Finetune/Loss (Raw)": 1.1521122455596924, "Finetune/Step": 129, "Finetune/Step Time": 5.4128291346132755} +{"Finetune/Learning Rate": 1.6774193548387098e-05, "Finetune/Loss": 1.2318007946014404, "Finetune/Loss (Raw)": 1.136315107345581, "Finetune/Step": 130, "Finetune/Step Time": 5.395845411345363} +{"Finetune/Learning Rate": 1.6903225806451615e-05, "Finetune/Loss": 1.2019031047821045, "Finetune/Loss (Raw)": 1.1056175231933594, "Finetune/Step": 131, "Finetune/Step Time": 5.395934451371431} +{"Finetune/Learning Rate": 1.703225806451613e-05, "Finetune/Loss": 1.181183099746704, "Finetune/Loss (Raw)": 1.1126604080200195, "Finetune/Step": 132, "Finetune/Step Time": 5.396320324391127} +{"Finetune/Learning Rate": 1.7161290322580647e-05, "Finetune/Loss": 1.1541677713394165, "Finetune/Loss (Raw)": 0.9029495716094971, "Finetune/Step": 133, "Finetune/Step Time": 5.406433250755072} +{"Finetune/Learning Rate": 1.729032258064516e-05, "Finetune/Loss": 1.1362310647964478, "Finetune/Loss (Raw)": 1.2392711639404297, "Finetune/Step": 134, "Finetune/Step Time": 5.401761947199702} +{"Finetune/Learning Rate": 1.741935483870968e-05, "Finetune/Loss": 1.1206753253936768, "Finetune/Loss (Raw)": 0.9967329502105713, "Finetune/Step": 135, "Finetune/Step Time": 5.403760025277734} +{"Finetune/Learning Rate": 1.7548387096774196e-05, "Finetune/Loss": 1.109955072402954, "Finetune/Loss (Raw)": 1.0777181386947632, "Finetune/Step": 136, "Finetune/Step Time": 5.388735705986619} +{"Finetune/Learning Rate": 1.7677419354838713e-05, "Finetune/Loss": 1.103144884109497, "Finetune/Loss (Raw)": 0.9766666293144226, "Finetune/Step": 137, "Finetune/Step Time": 5.39145771227777} +{"Finetune/Learning Rate": 1.7806451612903228e-05, "Finetune/Loss": 1.094109296798706, "Finetune/Loss (Raw)": 1.1209635734558105, "Finetune/Step": 138, "Finetune/Step Time": 5.362597469240427} +{"Finetune/Learning Rate": 1.7935483870967742e-05, "Finetune/Loss": 1.0870904922485352, "Finetune/Loss (Raw)": 0.9140287041664124, "Finetune/Step": 139, "Finetune/Step Time": 5.364507732912898} +{"Finetune/Learning Rate": 1.806451612903226e-05, "Finetune/Loss": 1.0845646858215332, "Finetune/Loss (Raw)": 1.2887145280838013, "Finetune/Step": 140, "Finetune/Step Time": 5.3585939314216375} +{"Finetune/Learning Rate": 1.8193548387096777e-05, "Finetune/Loss": 1.0818276405334473, "Finetune/Loss (Raw)": 1.0385504961013794, "Finetune/Step": 141, "Finetune/Step Time": 5.352311301976442} +{"Finetune/Learning Rate": 1.832258064516129e-05, "Finetune/Loss": 1.0772595405578613, "Finetune/Loss (Raw)": 1.0250192880630493, "Finetune/Step": 142, "Finetune/Step Time": 5.3923464063555} +{"Finetune/Learning Rate": 1.845161290322581e-05, "Finetune/Loss": 1.070127248764038, "Finetune/Loss (Raw)": 0.9640348553657532, "Finetune/Step": 143, "Finetune/Step Time": 5.406803023070097} +{"Finetune/Learning Rate": 1.8580645161290326e-05, "Finetune/Loss": 1.0634057521820068, "Finetune/Loss (Raw)": 0.3734639883041382, "Finetune/Step": 144, "Finetune/Step Time": 5.425244361162186} +{"Finetune/Learning Rate": 1.870967741935484e-05, "Finetune/Loss": 1.055847406387329, "Finetune/Loss (Raw)": 0.34408262372016907, "Finetune/Step": 145, "Finetune/Step Time": 5.452890856191516} +{"Finetune/Learning Rate": 1.8838709677419354e-05, "Finetune/Loss": 1.0540424585342407, "Finetune/Loss (Raw)": 0.9665518403053284, "Finetune/Step": 146, "Finetune/Step Time": 5.442937163636088} +{"Finetune/Learning Rate": 1.896774193548387e-05, "Finetune/Loss": 1.0512616634368896, "Finetune/Loss (Raw)": 0.9028814435005188, "Finetune/Step": 147, "Finetune/Step Time": 5.449094923213124} +{"Finetune/Learning Rate": 1.909677419354839e-05, "Finetune/Loss": 1.049707055091858, "Finetune/Loss (Raw)": 1.0773378610610962, "Finetune/Step": 148, "Finetune/Step Time": 5.451682956889272} +{"Finetune/Learning Rate": 1.9225806451612907e-05, "Finetune/Loss": 1.0452500581741333, "Finetune/Loss (Raw)": 1.1228134632110596, "Finetune/Step": 149, "Finetune/Step Time": 5.44099104963243} +{"Finetune/Learning Rate": 1.935483870967742e-05, "Finetune/Loss": 1.0445339679718018, "Finetune/Loss (Raw)": 1.1334905624389648, "Finetune/Step": 150, "Finetune/Step Time": 5.445031126961112} +{"Finetune/Learning Rate": 1.948387096774194e-05, "Finetune/Loss": 1.0512604713439941, "Finetune/Loss (Raw)": 1.0936365127563477, "Finetune/Step": 151, "Finetune/Step Time": 5.4190243519842625} +{"Finetune/Learning Rate": 1.9612903225806452e-05, "Finetune/Loss": 1.0490031242370605, "Finetune/Loss (Raw)": 1.053750991821289, "Finetune/Step": 152, "Finetune/Step Time": 5.425213808193803} +{"Finetune/Learning Rate": 1.974193548387097e-05, "Finetune/Loss": 1.0489041805267334, "Finetune/Loss (Raw)": 1.1565797328948975, "Finetune/Step": 153, "Finetune/Step Time": 5.434783019125462} +{"Finetune/Learning Rate": 1.9870967741935484e-05, "Finetune/Loss": 1.055712103843689, "Finetune/Loss (Raw)": 1.1395870447158813, "Finetune/Step": 154, "Finetune/Step Time": 5.398784970864654} +{"Finetune/Learning Rate": 2e-05, "Finetune/Loss": 1.0544798374176025, "Finetune/Loss (Raw)": 1.2058666944503784, "Finetune/Step": 155, "Finetune/Step Time": 5.406365046277642} +{"Finetune/Learning Rate": 1.9999998059597617e-05, "Finetune/Loss": 1.0472681522369385, "Finetune/Loss (Raw)": 0.297910213470459, "Finetune/Step": 156, "Finetune/Step Time": 5.440441584214568} +{"Finetune/Learning Rate": 1.9999992238391222e-05, "Finetune/Loss": 1.0448389053344727, "Finetune/Loss (Raw)": 1.0808390378952026, "Finetune/Step": 157, "Finetune/Step Time": 5.43026727437973} +{"Finetune/Learning Rate": 1.999998253638307e-05, "Finetune/Loss": 1.0446140766143799, "Finetune/Loss (Raw)": 1.1159179210662842, "Finetune/Step": 158, "Finetune/Step Time": 5.421036843210459} +{"Finetune/Learning Rate": 1.999996895357693e-05, "Finetune/Loss": 1.0428869724273682, "Finetune/Loss (Raw)": 1.0761797428131104, "Finetune/Step": 159, "Finetune/Step Time": 5.41374896466732} +{"Finetune/Learning Rate": 1.9999951489978074e-05, "Finetune/Loss": 1.0415127277374268, "Finetune/Loss (Raw)": 1.0164457559585571, "Finetune/Step": 160, "Finetune/Step Time": 5.403204299509525} +{"Finetune/Learning Rate": 1.9999930145593275e-05, "Finetune/Loss": 1.0416827201843262, "Finetune/Loss (Raw)": 1.033155083656311, "Finetune/Step": 161, "Finetune/Step Time": 5.399592375382781} +{"Finetune/Learning Rate": 1.9999904920430818e-05, "Finetune/Loss": 1.0431997776031494, "Finetune/Loss (Raw)": 1.2628639936447144, "Finetune/Step": 162, "Finetune/Step Time": 5.398372123017907} +{"Finetune/Learning Rate": 1.9999875814500492e-05, "Finetune/Loss": 1.0410125255584717, "Finetune/Loss (Raw)": 1.08240807056427, "Finetune/Step": 163, "Finetune/Step Time": 5.398423148319125} +{"Finetune/Learning Rate": 1.99998428278136e-05, "Finetune/Loss": 1.039467692375183, "Finetune/Loss (Raw)": 0.9662144780158997, "Finetune/Step": 164, "Finetune/Step Time": 5.396354950964451} +{"Finetune/Learning Rate": 1.9999805960382928e-05, "Finetune/Loss": 1.037421464920044, "Finetune/Loss (Raw)": 1.057300329208374, "Finetune/Step": 165, "Finetune/Step Time": 5.3732514046132565} +{"Finetune/Learning Rate": 1.99997652122228e-05, "Finetune/Loss": 1.0362002849578857, "Finetune/Loss (Raw)": 0.9999752640724182, "Finetune/Step": 166, "Finetune/Step Time": 5.375239908695221} +{"Finetune/Learning Rate": 1.9999720583349015e-05, "Finetune/Loss": 1.0359035730361938, "Finetune/Loss (Raw)": 1.1452438831329346, "Finetune/Step": 167, "Finetune/Step Time": 5.377141835168004} +{"Finetune/Learning Rate": 1.9999672073778905e-05, "Finetune/Loss": 1.0343677997589111, "Finetune/Loss (Raw)": 0.9860706925392151, "Finetune/Step": 168, "Finetune/Step Time": 5.3653698656708} +{"Finetune/Learning Rate": 1.999961968353129e-05, "Finetune/Loss": 1.0319874286651611, "Finetune/Loss (Raw)": 0.9765651822090149, "Finetune/Step": 169, "Finetune/Step Time": 5.37037599273026} +{"Finetune/Learning Rate": 1.9999563412626503e-05, "Finetune/Loss": 1.0251147747039795, "Finetune/Loss (Raw)": 0.35619068145751953, "Finetune/Step": 170, "Finetune/Step Time": 5.400917366147041} +{"Finetune/Learning Rate": 1.9999503261086378e-05, "Finetune/Loss": 1.0237336158752441, "Finetune/Loss (Raw)": 1.0058619976043701, "Finetune/Step": 171, "Finetune/Step Time": 5.398516310378909} +{"Finetune/Learning Rate": 1.999943922893426e-05, "Finetune/Loss": 1.022249698638916, "Finetune/Loss (Raw)": 0.9923118948936462, "Finetune/Step": 172, "Finetune/Step Time": 5.402004417032003} +{"Finetune/Learning Rate": 1.9999371316195005e-05, "Finetune/Loss": 1.023329496383667, "Finetune/Loss (Raw)": 1.2124285697937012, "Finetune/Step": 173, "Finetune/Step Time": 5.369354015216231} +{"Finetune/Learning Rate": 1.9999299522894957e-05, "Finetune/Loss": 1.0281283855438232, "Finetune/Loss (Raw)": 0.9034967422485352, "Finetune/Step": 174, "Finetune/Step Time": 5.329905696213245} +{"Finetune/Learning Rate": 1.9999223849061987e-05, "Finetune/Loss": 1.0284518003463745, "Finetune/Loss (Raw)": 1.1905115842819214, "Finetune/Step": 175, "Finetune/Step Time": 5.321348151192069} +{"Finetune/Learning Rate": 1.999914429472546e-05, "Finetune/Loss": 1.0223495960235596, "Finetune/Loss (Raw)": 0.353081613779068, "Finetune/Step": 176, "Finetune/Step Time": 5.338018048554659} +{"Finetune/Learning Rate": 1.9999060859916247e-05, "Finetune/Loss": 1.0208501815795898, "Finetune/Loss (Raw)": 0.9773699045181274, "Finetune/Step": 177, "Finetune/Step Time": 5.346024360507727} +{"Finetune/Learning Rate": 1.9998973544666734e-05, "Finetune/Loss": 1.0222407579421997, "Finetune/Loss (Raw)": 1.209365725517273, "Finetune/Step": 178, "Finetune/Step Time": 5.347624082118273} +{"Finetune/Learning Rate": 1.9998882349010798e-05, "Finetune/Loss": 1.0203367471694946, "Finetune/Loss (Raw)": 0.9052232503890991, "Finetune/Step": 179, "Finetune/Step Time": 5.351376861333847} +{"Finetune/Learning Rate": 1.999878727298384e-05, "Finetune/Loss": 1.0192962884902954, "Finetune/Loss (Raw)": 1.0673202276229858, "Finetune/Step": 180, "Finetune/Step Time": 5.3543909676373005} +{"Finetune/Learning Rate": 1.999868831662274e-05, "Finetune/Loss": 1.0200586318969727, "Finetune/Loss (Raw)": 1.0218029022216797, "Finetune/Step": 181, "Finetune/Step Time": 5.340246254578233} +{"Finetune/Learning Rate": 1.9998585479965918e-05, "Finetune/Loss": 1.020941972732544, "Finetune/Loss (Raw)": 1.0571966171264648, "Finetune/Step": 182, "Finetune/Step Time": 5.339201387017965} +{"Finetune/Learning Rate": 1.999847876305328e-05, "Finetune/Loss": 1.0197521448135376, "Finetune/Loss (Raw)": 1.0195382833480835, "Finetune/Step": 183, "Finetune/Step Time": 5.322051515802741} +{"Finetune/Learning Rate": 1.9998368165926237e-05, "Finetune/Loss": 1.0201276540756226, "Finetune/Loss (Raw)": 1.1642769575119019, "Finetune/Step": 184, "Finetune/Step Time": 5.308445625007153} +{"Finetune/Learning Rate": 1.9998253688627706e-05, "Finetune/Loss": 1.019397258758545, "Finetune/Loss (Raw)": 0.9307861924171448, "Finetune/Step": 185, "Finetune/Step Time": 5.30600456520915} +{"Finetune/Learning Rate": 1.9998135331202124e-05, "Finetune/Loss": 1.0190293788909912, "Finetune/Loss (Raw)": 1.0152368545532227, "Finetune/Step": 186, "Finetune/Step Time": 5.296156823635101} +{"Finetune/Learning Rate": 1.9998013093695413e-05, "Finetune/Loss": 1.0183351039886475, "Finetune/Loss (Raw)": 1.0121474266052246, "Finetune/Step": 187, "Finetune/Step Time": 5.281884590163827} +{"Finetune/Learning Rate": 1.9997886976155018e-05, "Finetune/Loss": 1.0187606811523438, "Finetune/Loss (Raw)": 1.1367610692977905, "Finetune/Step": 188, "Finetune/Step Time": 5.280444575473666} +{"Finetune/Learning Rate": 1.9997756978629876e-05, "Finetune/Loss": 1.017640233039856, "Finetune/Loss (Raw)": 1.0094976425170898, "Finetune/Step": 189, "Finetune/Step Time": 5.291464723646641} +{"Finetune/Learning Rate": 1.999762310117044e-05, "Finetune/Loss": 1.0166939496994019, "Finetune/Loss (Raw)": 1.0753668546676636, "Finetune/Step": 190, "Finetune/Step Time": 5.2963424399495125} +{"Finetune/Learning Rate": 1.999748534382867e-05, "Finetune/Loss": 1.015676736831665, "Finetune/Loss (Raw)": 1.0884251594543457, "Finetune/Step": 191, "Finetune/Step Time": 5.301779061555862} +{"Finetune/Learning Rate": 1.9997343706658023e-05, "Finetune/Loss": 1.0157697200775146, "Finetune/Loss (Raw)": 1.075989842414856, "Finetune/Step": 192, "Finetune/Step Time": 5.281697679311037} +{"Finetune/Learning Rate": 1.9997198189713462e-05, "Finetune/Loss": 1.0149023532867432, "Finetune/Loss (Raw)": 0.9435052871704102, "Finetune/Step": 193, "Finetune/Step Time": 5.274644564837217} +{"Finetune/Learning Rate": 1.9997048793051463e-05, "Finetune/Loss": 1.015307068824768, "Finetune/Loss (Raw)": 1.0620267391204834, "Finetune/Step": 194, "Finetune/Step Time": 5.273816995322704} +{"Finetune/Learning Rate": 1.9996895516730005e-05, "Finetune/Loss": 1.0161871910095215, "Finetune/Loss (Raw)": 1.0139116048812866, "Finetune/Step": 195, "Finetune/Step Time": 5.280132383108139} +{"Finetune/Learning Rate": 1.9996738360808566e-05, "Finetune/Loss": 1.0150190591812134, "Finetune/Loss (Raw)": 0.9809898734092712, "Finetune/Step": 196, "Finetune/Step Time": 5.276166755706072} +{"Finetune/Learning Rate": 1.999657732534814e-05, "Finetune/Loss": 1.0142490863800049, "Finetune/Loss (Raw)": 1.0578551292419434, "Finetune/Step": 197, "Finetune/Step Time": 5.282198218628764} +{"Finetune/Learning Rate": 1.9996412410411224e-05, "Finetune/Loss": 1.0128008127212524, "Finetune/Loss (Raw)": 1.0147227048873901, "Finetune/Step": 198, "Finetune/Step Time": 5.281589474529028} +{"Finetune/Learning Rate": 1.9996243616061812e-05, "Finetune/Loss": 1.0133695602416992, "Finetune/Loss (Raw)": 1.003377914428711, "Finetune/Step": 199, "Finetune/Step Time": 5.285946074873209} +{"Finetune/Learning Rate": 1.9996070942365416e-05, "Finetune/Loss": 1.0134289264678955, "Finetune/Loss (Raw)": 1.0354825258255005, "Finetune/Step": 200, "Finetune/Step Time": 5.290556708350778} +{"Finetune/Learning Rate": 1.9995894389389038e-05, "Finetune/Loss": 1.0150086879730225, "Finetune/Loss (Raw)": 1.131021499633789, "Finetune/Step": 201, "Finetune/Step Time": 5.294539636000991} +{"Finetune/Learning Rate": 1.9995713957201208e-05, "Finetune/Loss": 1.0165042877197266, "Finetune/Loss (Raw)": 1.1304171085357666, "Finetune/Step": 202, "Finetune/Step Time": 5.298110080882907} +{"Finetune/Learning Rate": 1.9995529645871938e-05, "Finetune/Loss": 1.0184155702590942, "Finetune/Loss (Raw)": 1.0748533010482788, "Finetune/Step": 203, "Finetune/Step Time": 5.292302690446377} +{"Finetune/Learning Rate": 1.999534145547276e-05, "Finetune/Loss": 1.0190656185150146, "Finetune/Loss (Raw)": 1.0654587745666504, "Finetune/Step": 204, "Finetune/Step Time": 5.286692500114441} +{"Finetune/Learning Rate": 1.99951493860767e-05, "Finetune/Loss": 1.0128777027130127, "Finetune/Loss (Raw)": 0.31823015213012695, "Finetune/Step": 205, "Finetune/Step Time": 5.324981704354286} +{"Finetune/Learning Rate": 1.9994953437758305e-05, "Finetune/Loss": 1.0112547874450684, "Finetune/Loss (Raw)": 1.0117319822311401, "Finetune/Step": 206, "Finetune/Step Time": 5.3336092587560415} +{"Finetune/Learning Rate": 1.999475361059362e-05, "Finetune/Loss": 1.0112521648406982, "Finetune/Loss (Raw)": 1.1867742538452148, "Finetune/Step": 207, "Finetune/Step Time": 5.356871305033565} +{"Finetune/Learning Rate": 1.9994549904660187e-05, "Finetune/Loss": 1.0119105577468872, "Finetune/Loss (Raw)": 1.0588417053222656, "Finetune/Step": 208, "Finetune/Step Time": 5.3420235849916935} +{"Finetune/Learning Rate": 1.9994342320037065e-05, "Finetune/Loss": 1.0109120607376099, "Finetune/Loss (Raw)": 1.0145660638809204, "Finetune/Step": 209, "Finetune/Step Time": 5.336215369403362} +{"Finetune/Learning Rate": 1.9994130856804808e-05, "Finetune/Loss": 1.0047719478607178, "Finetune/Loss (Raw)": 0.3187578022480011, "Finetune/Step": 210, "Finetune/Step Time": 5.367008674889803} +{"Finetune/Learning Rate": 1.9993915515045485e-05, "Finetune/Loss": 1.0049796104431152, "Finetune/Loss (Raw)": 0.9910170435905457, "Finetune/Step": 211, "Finetune/Step Time": 5.359325489029288} +{"Finetune/Learning Rate": 1.999369629484267e-05, "Finetune/Loss": 1.0040873289108276, "Finetune/Loss (Raw)": 1.0757675170898438, "Finetune/Step": 212, "Finetune/Step Time": 5.356718845665455} +{"Finetune/Learning Rate": 1.999347319628143e-05, "Finetune/Loss": 1.0052398443222046, "Finetune/Loss (Raw)": 1.1952837705612183, "Finetune/Step": 213, "Finetune/Step Time": 5.359337797388434} +{"Finetune/Learning Rate": 1.9993246219448347e-05, "Finetune/Loss": 1.0102813243865967, "Finetune/Loss (Raw)": 0.992932915687561, "Finetune/Step": 214, "Finetune/Step Time": 5.341743366792798} +{"Finetune/Learning Rate": 1.999301536443151e-05, "Finetune/Loss": 1.0097484588623047, "Finetune/Loss (Raw)": 0.9541593194007874, "Finetune/Step": 215, "Finetune/Step Time": 5.341418566182256} +{"Finetune/Learning Rate": 1.9992780631320508e-05, "Finetune/Loss": 1.007563591003418, "Finetune/Loss (Raw)": 0.9391830563545227, "Finetune/Step": 216, "Finetune/Step Time": 5.340112056583166} +{"Finetune/Learning Rate": 1.9992542020206432e-05, "Finetune/Loss": 1.0079667568206787, "Finetune/Loss (Raw)": 1.1226881742477417, "Finetune/Step": 217, "Finetune/Step Time": 5.337976021692157} +{"Finetune/Learning Rate": 1.9992299531181892e-05, "Finetune/Loss": 1.0074588060379028, "Finetune/Loss (Raw)": 1.0753740072250366, "Finetune/Step": 218, "Finetune/Step Time": 5.32715293392539} +{"Finetune/Learning Rate": 1.999205316434098e-05, "Finetune/Loss": 1.0078208446502686, "Finetune/Loss (Raw)": 1.2149670124053955, "Finetune/Step": 219, "Finetune/Step Time": 5.315797565504909} +{"Finetune/Learning Rate": 1.9991802919779318e-05, "Finetune/Loss": 1.0060703754425049, "Finetune/Loss (Raw)": 0.9749873280525208, "Finetune/Step": 220, "Finetune/Step Time": 5.312881670892239} +{"Finetune/Learning Rate": 1.9991548797594017e-05, "Finetune/Loss": 1.0076377391815186, "Finetune/Loss (Raw)": 1.1205283403396606, "Finetune/Step": 221, "Finetune/Step Time": 5.315798435360193} +{"Finetune/Learning Rate": 1.9991290797883693e-05, "Finetune/Loss": 1.00608229637146, "Finetune/Loss (Raw)": 0.9606602787971497, "Finetune/Step": 222, "Finetune/Step Time": 5.321862012147903} +{"Finetune/Learning Rate": 1.9991028920748475e-05, "Finetune/Loss": 1.0053391456604004, "Finetune/Loss (Raw)": 0.9926061034202576, "Finetune/Step": 223, "Finetune/Step Time": 5.323111897334456} +{"Finetune/Learning Rate": 1.9990763166289993e-05, "Finetune/Loss": 0.9996378421783447, "Finetune/Loss (Raw)": 0.35421866178512573, "Finetune/Step": 224, "Finetune/Step Time": 5.342781590297818} +{"Finetune/Learning Rate": 1.9990493534611376e-05, "Finetune/Loss": 0.9998375773429871, "Finetune/Loss (Raw)": 1.0568971633911133, "Finetune/Step": 225, "Finetune/Step Time": 5.342016987502575} +{"Finetune/Learning Rate": 1.9990220025817268e-05, "Finetune/Loss": 1.0003300905227661, "Finetune/Loss (Raw)": 1.191135287284851, "Finetune/Step": 226, "Finetune/Step Time": 5.339677359908819} +{"Finetune/Learning Rate": 1.998994264001381e-05, "Finetune/Loss": 1.0079662799835205, "Finetune/Loss (Raw)": 1.1955677270889282, "Finetune/Step": 227, "Finetune/Step Time": 5.308066666126251} +{"Finetune/Learning Rate": 1.9989661377308654e-05, "Finetune/Loss": 1.0075346231460571, "Finetune/Loss (Raw)": 1.15692937374115, "Finetune/Step": 228, "Finetune/Step Time": 5.303257811814547} +{"Finetune/Learning Rate": 1.9989376237810944e-05, "Finetune/Loss": 1.0074057579040527, "Finetune/Loss (Raw)": 1.0352154970169067, "Finetune/Step": 229, "Finetune/Step Time": 5.296868925914168} +{"Finetune/Learning Rate": 1.9989087221631344e-05, "Finetune/Loss": 1.0070253610610962, "Finetune/Loss (Raw)": 1.1595743894577026, "Finetune/Step": 230, "Finetune/Step Time": 5.299175284802914} +{"Finetune/Learning Rate": 1.9988794328882017e-05, "Finetune/Loss": 1.0127452611923218, "Finetune/Loss (Raw)": 1.0246942043304443, "Finetune/Step": 231, "Finetune/Step Time": 5.268574316054583} +{"Finetune/Learning Rate": 1.9988497559676622e-05, "Finetune/Loss": 1.012108564376831, "Finetune/Loss (Raw)": 1.0668466091156006, "Finetune/Step": 232, "Finetune/Step Time": 5.272237341850996} +{"Finetune/Learning Rate": 1.9988196914130332e-05, "Finetune/Loss": 1.0062425136566162, "Finetune/Loss (Raw)": 0.30834782123565674, "Finetune/Step": 233, "Finetune/Step Time": 5.307226387783885} +{"Finetune/Learning Rate": 1.9987892392359827e-05, "Finetune/Loss": 1.0070973634719849, "Finetune/Loss (Raw)": 1.1699000597000122, "Finetune/Step": 234, "Finetune/Step Time": 5.326036609709263} +{"Finetune/Learning Rate": 1.998758399448328e-05, "Finetune/Loss": 1.007683515548706, "Finetune/Loss (Raw)": 1.2065387964248657, "Finetune/Step": 235, "Finetune/Step Time": 5.36697762273252} +{"Finetune/Learning Rate": 1.9987271720620376e-05, "Finetune/Loss": 1.007741928100586, "Finetune/Loss (Raw)": 1.0210016965866089, "Finetune/Step": 236, "Finetune/Step Time": 5.366913167759776} +{"Finetune/Learning Rate": 1.9986955570892302e-05, "Finetune/Loss": 1.0073699951171875, "Finetune/Loss (Raw)": 1.0941507816314697, "Finetune/Step": 237, "Finetune/Step Time": 5.400033041834831} +{"Finetune/Learning Rate": 1.998663554542175e-05, "Finetune/Loss": 1.0071452856063843, "Finetune/Loss (Raw)": 1.0192562341690063, "Finetune/Step": 238, "Finetune/Step Time": 5.403734553605318} +{"Finetune/Learning Rate": 1.9986311644332913e-05, "Finetune/Loss": 1.0054891109466553, "Finetune/Loss (Raw)": 0.900896430015564, "Finetune/Step": 239, "Finetune/Step Time": 5.393737189471722} +{"Finetune/Learning Rate": 1.9985983867751497e-05, "Finetune/Loss": 1.0046024322509766, "Finetune/Loss (Raw)": 0.966780960559845, "Finetune/Step": 240, "Finetune/Step Time": 5.380393054336309} +{"Finetune/Learning Rate": 1.99856522158047e-05, "Finetune/Loss": 1.004443883895874, "Finetune/Loss (Raw)": 1.10553777217865, "Finetune/Step": 241, "Finetune/Step Time": 5.367672825232148} +{"Finetune/Learning Rate": 1.998531668862123e-05, "Finetune/Loss": 1.0108466148376465, "Finetune/Loss (Raw)": 1.071068286895752, "Finetune/Step": 242, "Finetune/Step Time": 5.338476434350014} +{"Finetune/Learning Rate": 1.9984977286331302e-05, "Finetune/Loss": 1.0108637809753418, "Finetune/Loss (Raw)": 1.0870240926742554, "Finetune/Step": 243, "Finetune/Step Time": 5.3540407828986645} +{"Finetune/Learning Rate": 1.998463400906663e-05, "Finetune/Loss": 1.0108222961425781, "Finetune/Loss (Raw)": 0.9257885813713074, "Finetune/Step": 244, "Finetune/Step Time": 5.3339778278023005} +{"Finetune/Learning Rate": 1.998428685696043e-05, "Finetune/Loss": 1.011641502380371, "Finetune/Loss (Raw)": 1.2137025594711304, "Finetune/Step": 245, "Finetune/Step Time": 5.331654503941536} +{"Finetune/Learning Rate": 1.998393583014743e-05, "Finetune/Loss": 1.0118982791900635, "Finetune/Loss (Raw)": 1.1716352701187134, "Finetune/Step": 246, "Finetune/Step Time": 5.34041490405798} +{"Finetune/Learning Rate": 1.9983580928763852e-05, "Finetune/Loss": 1.0120903253555298, "Finetune/Loss (Raw)": 1.0645458698272705, "Finetune/Step": 247, "Finetune/Step Time": 5.3463506400585175} +{"Finetune/Learning Rate": 1.998322215294743e-05, "Finetune/Loss": 1.01149320602417, "Finetune/Loss (Raw)": 1.0082634687423706, "Finetune/Step": 248, "Finetune/Step Time": 5.35497335717082} +{"Finetune/Learning Rate": 1.9982859502837395e-05, "Finetune/Loss": 1.0115447044372559, "Finetune/Loss (Raw)": 1.0285046100616455, "Finetune/Step": 249, "Finetune/Step Time": 5.337069718167186} +{"Finetune/Learning Rate": 1.998249297857449e-05, "Finetune/Loss": 1.0124971866607666, "Finetune/Loss (Raw)": 1.0877488851547241, "Finetune/Step": 250, "Finetune/Step Time": 5.335393121466041} +{"Finetune/Learning Rate": 1.9982122580300947e-05, "Finetune/Loss": 1.013575792312622, "Finetune/Loss (Raw)": 1.151452898979187, "Finetune/Step": 251, "Finetune/Step Time": 5.3502005729824305} +{"Finetune/Learning Rate": 1.9981748308160517e-05, "Finetune/Loss": 1.0129692554473877, "Finetune/Loss (Raw)": 1.0485339164733887, "Finetune/Step": 252, "Finetune/Step Time": 5.349268075078726} +{"Finetune/Learning Rate": 1.9981370162298444e-05, "Finetune/Loss": 1.0135468244552612, "Finetune/Loss (Raw)": 1.0843570232391357, "Finetune/Step": 253, "Finetune/Step Time": 5.34416083060205} +{"Finetune/Learning Rate": 1.9980988142861486e-05, "Finetune/Loss": 1.0125999450683594, "Finetune/Loss (Raw)": 0.984397292137146, "Finetune/Step": 254, "Finetune/Step Time": 5.324251521378756} +{"Finetune/Learning Rate": 1.998060224999789e-05, "Finetune/Loss": 1.0073540210723877, "Finetune/Loss (Raw)": 0.3159051239490509, "Finetune/Step": 255, "Finetune/Step Time": 5.355105005204678} +{"Finetune/Learning Rate": 1.9980212483857412e-05, "Finetune/Loss": 1.005947232246399, "Finetune/Loss (Raw)": 0.9534673094749451, "Finetune/Step": 256, "Finetune/Step Time": 5.355350326746702} +{"Finetune/Learning Rate": 1.9979818844591318e-05, "Finetune/Loss": 1.0061235427856445, "Finetune/Loss (Raw)": 1.1746854782104492, "Finetune/Step": 257, "Finetune/Step Time": 5.355058666318655} +{"Finetune/Learning Rate": 1.9979421332352372e-05, "Finetune/Loss": 1.0059852600097656, "Finetune/Loss (Raw)": 1.1186217069625854, "Finetune/Step": 258, "Finetune/Step Time": 5.354937706142664} +{"Finetune/Learning Rate": 1.9979019947294836e-05, "Finetune/Loss": 1.0058492422103882, "Finetune/Loss (Raw)": 1.0882047414779663, "Finetune/Step": 259, "Finetune/Step Time": 5.356921603903174} +{"Finetune/Learning Rate": 1.9978614689574486e-05, "Finetune/Loss": 1.0057761669158936, "Finetune/Loss (Raw)": 1.1032936573028564, "Finetune/Step": 260, "Finetune/Step Time": 5.3766033463180065} +{"Finetune/Learning Rate": 1.9978205559348586e-05, "Finetune/Loss": 1.005624532699585, "Finetune/Loss (Raw)": 0.8835458755493164, "Finetune/Step": 261, "Finetune/Step Time": 5.359555475413799} +{"Finetune/Learning Rate": 1.997779255677592e-05, "Finetune/Loss": 1.0026044845581055, "Finetune/Loss (Raw)": 0.8527159094810486, "Finetune/Step": 262, "Finetune/Step Time": 5.370865339413285} +{"Finetune/Learning Rate": 1.997737568201676e-05, "Finetune/Loss": 1.002866268157959, "Finetune/Loss (Raw)": 1.0302270650863647, "Finetune/Step": 263, "Finetune/Step Time": 5.369244886562228} +{"Finetune/Learning Rate": 1.9976954935232892e-05, "Finetune/Loss": 1.0021852254867554, "Finetune/Loss (Raw)": 0.990553081035614, "Finetune/Step": 264, "Finetune/Step Time": 5.383903622627258} +{"Finetune/Learning Rate": 1.99765303165876e-05, "Finetune/Loss": 1.00235116481781, "Finetune/Loss (Raw)": 0.9979022145271301, "Finetune/Step": 265, "Finetune/Step Time": 5.380424175411463} +{"Finetune/Learning Rate": 1.997610182624566e-05, "Finetune/Loss": 1.0031347274780273, "Finetune/Loss (Raw)": 1.2212615013122559, "Finetune/Step": 266, "Finetune/Step Time": 5.391654513776302} +{"Finetune/Learning Rate": 1.9975669464373377e-05, "Finetune/Loss": 1.0030478239059448, "Finetune/Loss (Raw)": 0.9029020071029663, "Finetune/Step": 267, "Finetune/Step Time": 5.389050722122192} +{"Finetune/Learning Rate": 1.9975233231138525e-05, "Finetune/Loss": 1.0017045736312866, "Finetune/Loss (Raw)": 1.1167900562286377, "Finetune/Step": 268, "Finetune/Step Time": 5.3926102966070175} +{"Finetune/Learning Rate": 1.997479312671041e-05, "Finetune/Loss": 1.0017393827438354, "Finetune/Loss (Raw)": 1.0429980754852295, "Finetune/Step": 269, "Finetune/Step Time": 5.398438295349479} +{"Finetune/Learning Rate": 1.9974349151259824e-05, "Finetune/Loss": 1.0020709037780762, "Finetune/Loss (Raw)": 1.067448616027832, "Finetune/Step": 270, "Finetune/Step Time": 5.3631992265582085} +{"Finetune/Learning Rate": 1.9973901304959066e-05, "Finetune/Loss": 1.0025830268859863, "Finetune/Loss (Raw)": 1.0295941829681396, "Finetune/Step": 271, "Finetune/Step Time": 5.3539230320602655} +{"Finetune/Learning Rate": 1.9973449587981935e-05, "Finetune/Loss": 1.0084112882614136, "Finetune/Loss (Raw)": 1.1194748878479004, "Finetune/Step": 272, "Finetune/Step Time": 5.327082199975848} +{"Finetune/Learning Rate": 1.9972994000503734e-05, "Finetune/Loss": 1.0140104293823242, "Finetune/Loss (Raw)": 1.0607762336730957, "Finetune/Step": 273, "Finetune/Step Time": 5.299123564735055} +{"Finetune/Learning Rate": 1.997253454270127e-05, "Finetune/Loss": 1.0151005983352661, "Finetune/Loss (Raw)": 1.1061019897460938, "Finetune/Step": 274, "Finetune/Step Time": 5.30562062934041} +{"Finetune/Learning Rate": 1.9972071214752843e-05, "Finetune/Loss": 1.0160353183746338, "Finetune/Loss (Raw)": 1.0225303173065186, "Finetune/Step": 275, "Finetune/Step Time": 5.302927760407329} +{"Finetune/Learning Rate": 1.9971604016838264e-05, "Finetune/Loss": 1.0141396522521973, "Finetune/Loss (Raw)": 0.8346842527389526, "Finetune/Step": 276, "Finetune/Step Time": 5.335862627252936} +{"Finetune/Learning Rate": 1.997113294913885e-05, "Finetune/Loss": 1.0133888721466064, "Finetune/Loss (Raw)": 1.0267151594161987, "Finetune/Step": 277, "Finetune/Step Time": 5.334711154922843} +{"Finetune/Learning Rate": 1.9970658011837404e-05, "Finetune/Loss": 1.012662649154663, "Finetune/Loss (Raw)": 1.0405302047729492, "Finetune/Step": 278, "Finetune/Step Time": 5.327239826321602} +{"Finetune/Learning Rate": 1.997017920511825e-05, "Finetune/Loss": 1.0136725902557373, "Finetune/Loss (Raw)": 1.2229056358337402, "Finetune/Step": 279, "Finetune/Step Time": 5.333583950996399} +{"Finetune/Learning Rate": 1.9969696529167197e-05, "Finetune/Loss": 1.0131819248199463, "Finetune/Loss (Raw)": 0.9909384250640869, "Finetune/Step": 280, "Finetune/Step Time": 5.332187108695507} +{"Finetune/Learning Rate": 1.996920998417156e-05, "Finetune/Loss": 1.012627124786377, "Finetune/Loss (Raw)": 1.085571527481079, "Finetune/Step": 281, "Finetune/Step Time": 5.317586686462164} +{"Finetune/Learning Rate": 1.996871957032016e-05, "Finetune/Loss": 1.0117378234863281, "Finetune/Loss (Raw)": 1.0257728099822998, "Finetune/Step": 282, "Finetune/Step Time": 5.316601051017642} +{"Finetune/Learning Rate": 1.996822528780332e-05, "Finetune/Loss": 1.0112720727920532, "Finetune/Loss (Raw)": 1.1462371349334717, "Finetune/Step": 283, "Finetune/Step Time": 5.315059019252658} +{"Finetune/Learning Rate": 1.996772713681286e-05, "Finetune/Loss": 1.0179810523986816, "Finetune/Loss (Raw)": 1.1566585302352905, "Finetune/Step": 284, "Finetune/Step Time": 5.281084658578038} +{"Finetune/Learning Rate": 1.9967225117542097e-05, "Finetune/Loss": 1.0174239873886108, "Finetune/Loss (Raw)": 1.009538173675537, "Finetune/Step": 285, "Finetune/Step Time": 5.299030188471079} +{"Finetune/Learning Rate": 1.9966719230185864e-05, "Finetune/Loss": 1.0171759128570557, "Finetune/Loss (Raw)": 1.0841621160507202, "Finetune/Step": 286, "Finetune/Step Time": 5.313899576663971} +{"Finetune/Learning Rate": 1.9966209474940477e-05, "Finetune/Loss": 1.0172219276428223, "Finetune/Loss (Raw)": 1.082063913345337, "Finetune/Step": 287, "Finetune/Step Time": 5.3206705916672945} +{"Finetune/Learning Rate": 1.996569585200377e-05, "Finetune/Loss": 1.0163938999176025, "Finetune/Loss (Raw)": 0.9104671478271484, "Finetune/Step": 288, "Finetune/Step Time": 5.328450359404087} +{"Finetune/Learning Rate": 1.9965178361575068e-05, "Finetune/Loss": 1.017124891281128, "Finetune/Loss (Raw)": 1.1267359256744385, "Finetune/Step": 289, "Finetune/Step Time": 5.3326174188405275} +{"Finetune/Learning Rate": 1.9964657003855197e-05, "Finetune/Loss": 1.014812707901001, "Finetune/Loss (Raw)": 0.9668889045715332, "Finetune/Step": 290, "Finetune/Step Time": 5.3436309695243835} +{"Finetune/Learning Rate": 1.9964131779046484e-05, "Finetune/Loss": 1.015154480934143, "Finetune/Loss (Raw)": 1.1261513233184814, "Finetune/Step": 291, "Finetune/Step Time": 5.33762826398015} +{"Finetune/Learning Rate": 1.996360268735276e-05, "Finetune/Loss": 1.0162639617919922, "Finetune/Loss (Raw)": 1.1082336902618408, "Finetune/Step": 292, "Finetune/Step Time": 5.325925000011921} +{"Finetune/Learning Rate": 1.9963069728979357e-05, "Finetune/Loss": 1.0170179605484009, "Finetune/Loss (Raw)": 1.1538121700286865, "Finetune/Step": 293, "Finetune/Step Time": 5.322013553231955} +{"Finetune/Learning Rate": 1.9962532904133107e-05, "Finetune/Loss": 1.0151987075805664, "Finetune/Loss (Raw)": 0.7671210169792175, "Finetune/Step": 294, "Finetune/Step Time": 5.328433308750391} +{"Finetune/Learning Rate": 1.996199221302234e-05, "Finetune/Loss": 1.0147931575775146, "Finetune/Loss (Raw)": 1.0933293104171753, "Finetune/Step": 295, "Finetune/Step Time": 5.334505604580045} +{"Finetune/Learning Rate": 1.9961447655856883e-05, "Finetune/Loss": 1.0150682926177979, "Finetune/Loss (Raw)": 1.0212702751159668, "Finetune/Step": 296, "Finetune/Step Time": 5.339640976861119} +{"Finetune/Learning Rate": 1.9960899232848074e-05, "Finetune/Loss": 1.0163284540176392, "Finetune/Loss (Raw)": 1.1378792524337769, "Finetune/Step": 297, "Finetune/Step Time": 5.342715702950954} +{"Finetune/Learning Rate": 1.9960346944208743e-05, "Finetune/Loss": 1.0206637382507324, "Finetune/Loss (Raw)": 0.9111027121543884, "Finetune/Step": 298, "Finetune/Step Time": 5.337799847126007} +{"Finetune/Learning Rate": 1.995979079015322e-05, "Finetune/Loss": 1.0152077674865723, "Finetune/Loss (Raw)": 0.3074927031993866, "Finetune/Step": 299, "Finetune/Step Time": 5.36821823194623} +{"Finetune/Learning Rate": 1.995923077089734e-05, "Finetune/Loss": 1.0103192329406738, "Finetune/Loss (Raw)": 0.3665814697742462, "Finetune/Step": 300, "Finetune/Step Time": 5.387518763542175} +{"Finetune/Learning Rate": 1.9958666886658436e-05, "Finetune/Loss": 1.0039420127868652, "Finetune/Loss (Raw)": 0.3961431682109833, "Finetune/Step": 301, "Finetune/Step Time": 5.419053154066205} +{"Finetune/Learning Rate": 1.995809913765534e-05, "Finetune/Loss": 1.003856897354126, "Finetune/Loss (Raw)": 0.8926144242286682, "Finetune/Step": 302, "Finetune/Step Time": 5.427525848150253} +{"Finetune/Learning Rate": 1.9957527524108386e-05, "Finetune/Loss": 0.9970688819885254, "Finetune/Loss (Raw)": 0.3216356039047241, "Finetune/Step": 303, "Finetune/Step Time": 5.458318842574954} +{"Finetune/Learning Rate": 1.99569520462394e-05, "Finetune/Loss": 1.0028924942016602, "Finetune/Loss (Raw)": 1.0985137224197388, "Finetune/Step": 304, "Finetune/Step Time": 5.434058232232928} +{"Finetune/Learning Rate": 1.995637270427172e-05, "Finetune/Loss": 1.0034575462341309, "Finetune/Loss (Raw)": 1.0496935844421387, "Finetune/Step": 305, "Finetune/Step Time": 5.420832736417651} +{"Finetune/Learning Rate": 1.9955789498430178e-05, "Finetune/Loss": 1.0014383792877197, "Finetune/Loss (Raw)": 0.9509097933769226, "Finetune/Step": 306, "Finetune/Step Time": 5.421591388061643} +{"Finetune/Learning Rate": 1.99552024289411e-05, "Finetune/Loss": 1.0035239458084106, "Finetune/Loss (Raw)": 1.1721699237823486, "Finetune/Step": 307, "Finetune/Step Time": 5.440710231661797} +{"Finetune/Learning Rate": 1.995461149603232e-05, "Finetune/Loss": 1.0034133195877075, "Finetune/Loss (Raw)": 1.053165316581726, "Finetune/Step": 308, "Finetune/Step Time": 5.43466486223042} +{"Finetune/Learning Rate": 1.9954016699933163e-05, "Finetune/Loss": 1.004432201385498, "Finetune/Loss (Raw)": 1.1522119045257568, "Finetune/Step": 309, "Finetune/Step Time": 5.448789749294519} +{"Finetune/Learning Rate": 1.995341804087446e-05, "Finetune/Loss": 1.0040823221206665, "Finetune/Loss (Raw)": 1.0124194622039795, "Finetune/Step": 310, "Finetune/Step Time": 5.449665242806077} +{"Finetune/Learning Rate": 1.9952815519088544e-05, "Finetune/Loss": 1.004226803779602, "Finetune/Loss (Raw)": 1.0380290746688843, "Finetune/Step": 311, "Finetune/Step Time": 5.456934107467532} +{"Finetune/Learning Rate": 1.9952209134809234e-05, "Finetune/Loss": 1.0028271675109863, "Finetune/Loss (Raw)": 0.9851140379905701, "Finetune/Step": 312, "Finetune/Step Time": 5.455779442563653} +{"Finetune/Learning Rate": 1.995159888827186e-05, "Finetune/Loss": 1.0042868852615356, "Finetune/Loss (Raw)": 1.1176416873931885, "Finetune/Step": 313, "Finetune/Step Time": 5.463650990277529} +{"Finetune/Learning Rate": 1.9950984779713244e-05, "Finetune/Loss": 1.0056285858154297, "Finetune/Loss (Raw)": 1.1869786977767944, "Finetune/Step": 314, "Finetune/Step Time": 5.447484228760004} +{"Finetune/Learning Rate": 1.9950366809371713e-05, "Finetune/Loss": 1.0054925680160522, "Finetune/Loss (Raw)": 0.9947286248207092, "Finetune/Step": 315, "Finetune/Step Time": 5.4534453973174095} +{"Finetune/Learning Rate": 1.9949744977487087e-05, "Finetune/Loss": 0.9996275901794434, "Finetune/Loss (Raw)": 0.3860457241535187, "Finetune/Step": 316, "Finetune/Step Time": 5.485491558909416} +{"Finetune/Learning Rate": 1.994911928430069e-05, "Finetune/Loss": 0.9993482232093811, "Finetune/Loss (Raw)": 0.9737368226051331, "Finetune/Step": 317, "Finetune/Step Time": 5.475615840405226} +{"Finetune/Learning Rate": 1.994848973005534e-05, "Finetune/Loss": 0.9992529153823853, "Finetune/Loss (Raw)": 1.0631648302078247, "Finetune/Step": 318, "Finetune/Step Time": 5.469401491805911} +{"Finetune/Learning Rate": 1.994785631499535e-05, "Finetune/Loss": 0.9985392093658447, "Finetune/Loss (Raw)": 0.9970710873603821, "Finetune/Step": 319, "Finetune/Step Time": 5.4770204462111} +{"Finetune/Learning Rate": 1.994721903936654e-05, "Finetune/Loss": 0.9975506067276001, "Finetune/Loss (Raw)": 0.9494606852531433, "Finetune/Step": 320, "Finetune/Step Time": 5.47221820615232} +{"Finetune/Learning Rate": 1.9946577903416224e-05, "Finetune/Loss": 0.9927514791488647, "Finetune/Loss (Raw)": 0.32921525835990906, "Finetune/Step": 321, "Finetune/Step Time": 5.508443405851722} +{"Finetune/Learning Rate": 1.994593290739322e-05, "Finetune/Loss": 0.9922546744346619, "Finetune/Loss (Raw)": 0.9984363317489624, "Finetune/Step": 322, "Finetune/Step Time": 5.508592948317528} +{"Finetune/Learning Rate": 1.9945284051547823e-05, "Finetune/Loss": 0.9927219748497009, "Finetune/Loss (Raw)": 1.0737165212631226, "Finetune/Step": 323, "Finetune/Step Time": 5.50339006818831} +{"Finetune/Learning Rate": 1.9944631336131856e-05, "Finetune/Loss": 0.9936789274215698, "Finetune/Loss (Raw)": 1.103485345840454, "Finetune/Step": 324, "Finetune/Step Time": 5.502339940518141} +{"Finetune/Learning Rate": 1.994397476139862e-05, "Finetune/Loss": 0.9935925006866455, "Finetune/Loss (Raw)": 1.0467861890792847, "Finetune/Step": 325, "Finetune/Step Time": 5.499740304425359} +{"Finetune/Learning Rate": 1.994331432760292e-05, "Finetune/Loss": 0.9952160120010376, "Finetune/Loss (Raw)": 1.2225338220596313, "Finetune/Step": 326, "Finetune/Step Time": 5.500205295160413} +{"Finetune/Learning Rate": 1.9942650035001053e-05, "Finetune/Loss": 0.9953430891036987, "Finetune/Loss (Raw)": 1.019650936126709, "Finetune/Step": 327, "Finetune/Step Time": 5.494687000289559} +{"Finetune/Learning Rate": 1.994198188385082e-05, "Finetune/Loss": 0.9956580400466919, "Finetune/Loss (Raw)": 1.0757938623428345, "Finetune/Step": 328, "Finetune/Step Time": 5.5012883096933365} +{"Finetune/Learning Rate": 1.9941309874411525e-05, "Finetune/Loss": 0.9942035675048828, "Finetune/Loss (Raw)": 0.9448420405387878, "Finetune/Step": 329, "Finetune/Step Time": 5.517898090183735} +{"Finetune/Learning Rate": 1.9940634006943952e-05, "Finetune/Loss": 0.9922764301300049, "Finetune/Loss (Raw)": 0.8837450742721558, "Finetune/Step": 330, "Finetune/Step Time": 5.521502206102014} +{"Finetune/Learning Rate": 1.9939954281710395e-05, "Finetune/Loss": 0.9921290874481201, "Finetune/Loss (Raw)": 1.0560057163238525, "Finetune/Step": 331, "Finetune/Step Time": 5.5255127642303705} +{"Finetune/Learning Rate": 1.9939270698974643e-05, "Finetune/Loss": 0.990293025970459, "Finetune/Loss (Raw)": 0.8304286003112793, "Finetune/Step": 332, "Finetune/Step Time": 5.542345562949777} +{"Finetune/Learning Rate": 1.9938583259001983e-05, "Finetune/Loss": 0.9959512948989868, "Finetune/Loss (Raw)": 1.0424914360046387, "Finetune/Step": 333, "Finetune/Step Time": 5.511234536767006} +{"Finetune/Learning Rate": 1.993789196205919e-05, "Finetune/Loss": 0.9958749413490295, "Finetune/Loss (Raw)": 1.001956582069397, "Finetune/Step": 334, "Finetune/Step Time": 5.506299119442701} +{"Finetune/Learning Rate": 1.9937196808414552e-05, "Finetune/Loss": 0.9945489764213562, "Finetune/Loss (Raw)": 1.0170494318008423, "Finetune/Step": 335, "Finetune/Step Time": 5.493290901184082} +{"Finetune/Learning Rate": 1.993649779833784e-05, "Finetune/Loss": 0.9948687553405762, "Finetune/Loss (Raw)": 1.099778652191162, "Finetune/Step": 336, "Finetune/Step Time": 5.49646464549005} +{"Finetune/Learning Rate": 1.993579493210032e-05, "Finetune/Loss": 0.9966033101081848, "Finetune/Loss (Raw)": 1.236588716506958, "Finetune/Step": 337, "Finetune/Step Time": 5.491888616234064} +{"Finetune/Learning Rate": 1.9935088209974774e-05, "Finetune/Loss": 1.0019198656082153, "Finetune/Loss (Raw)": 0.9992830157279968, "Finetune/Step": 338, "Finetune/Step Time": 5.469794563949108} +{"Finetune/Learning Rate": 1.9934377632235455e-05, "Finetune/Loss": 1.0017297267913818, "Finetune/Loss (Raw)": 0.9666704535484314, "Finetune/Step": 339, "Finetune/Step Time": 5.472820671275258} +{"Finetune/Learning Rate": 1.9933663199158135e-05, "Finetune/Loss": 1.0005176067352295, "Finetune/Loss (Raw)": 0.9206250905990601, "Finetune/Step": 340, "Finetune/Step Time": 5.471182415261865} +{"Finetune/Learning Rate": 1.993294491102006e-05, "Finetune/Loss": 0.9992250204086304, "Finetune/Loss (Raw)": 1.0298222303390503, "Finetune/Step": 341, "Finetune/Step Time": 5.472861832007766} +{"Finetune/Learning Rate": 1.9932222768099992e-05, "Finetune/Loss": 1.0000275373458862, "Finetune/Loss (Raw)": 1.0956617593765259, "Finetune/Step": 342, "Finetune/Step Time": 5.4543814063072205} +{"Finetune/Learning Rate": 1.9931496770678178e-05, "Finetune/Loss": 1.00056791305542, "Finetune/Loss (Raw)": 1.0233168601989746, "Finetune/Step": 343, "Finetune/Step Time": 5.452870046719909} +{"Finetune/Learning Rate": 1.993076691903636e-05, "Finetune/Loss": 1.0017342567443848, "Finetune/Loss (Raw)": 1.0884953737258911, "Finetune/Step": 344, "Finetune/Step Time": 5.4380511827766895} +{"Finetune/Learning Rate": 1.993003321345779e-05, "Finetune/Loss": 1.0017492771148682, "Finetune/Loss (Raw)": 1.1246072053909302, "Finetune/Step": 345, "Finetune/Step Time": 5.440742939710617} +{"Finetune/Learning Rate": 1.9929295654227193e-05, "Finetune/Loss": 1.002173662185669, "Finetune/Loss (Raw)": 1.1296902894973755, "Finetune/Step": 346, "Finetune/Step Time": 5.44773162715137} +{"Finetune/Learning Rate": 1.9928554241630804e-05, "Finetune/Loss": 1.0017300844192505, "Finetune/Loss (Raw)": 1.1581865549087524, "Finetune/Step": 347, "Finetune/Step Time": 5.451227517798543} +{"Finetune/Learning Rate": 1.9927808975956354e-05, "Finetune/Loss": 1.002590298652649, "Finetune/Loss (Raw)": 1.0850938558578491, "Finetune/Step": 348, "Finetune/Step Time": 5.471468919888139} +{"Finetune/Learning Rate": 1.9927059857493068e-05, "Finetune/Loss": 1.002890706062317, "Finetune/Loss (Raw)": 1.158982753753662, "Finetune/Step": 349, "Finetune/Step Time": 5.4799919333308935} +{"Finetune/Learning Rate": 1.9926306886531658e-05, "Finetune/Loss": 1.0031392574310303, "Finetune/Loss (Raw)": 0.9924761652946472, "Finetune/Step": 350, "Finetune/Step Time": 5.494535863399506} +{"Finetune/Learning Rate": 1.9925550063364337e-05, "Finetune/Loss": 1.003220796585083, "Finetune/Loss (Raw)": 1.0030322074890137, "Finetune/Step": 351, "Finetune/Step Time": 5.507791023701429} +{"Finetune/Learning Rate": 1.992478938828482e-05, "Finetune/Loss": 1.0083122253417969, "Finetune/Loss (Raw)": 1.0059274435043335, "Finetune/Step": 352, "Finetune/Step Time": 5.486898744478822} +{"Finetune/Learning Rate": 1.9924024861588307e-05, "Finetune/Loss": 1.0077481269836426, "Finetune/Loss (Raw)": 0.9846977591514587, "Finetune/Step": 353, "Finetune/Step Time": 5.48660652525723} +{"Finetune/Learning Rate": 1.9923256483571495e-05, "Finetune/Loss": 1.0055456161499023, "Finetune/Loss (Raw)": 0.9092115759849548, "Finetune/Step": 354, "Finetune/Step Time": 5.493302239105105} +{"Finetune/Learning Rate": 1.9922484254532578e-05, "Finetune/Loss": 1.004558801651001, "Finetune/Loss (Raw)": 1.0692507028579712, "Finetune/Step": 355, "Finetune/Step Time": 5.503287328407168} +{"Finetune/Learning Rate": 1.9921708174771237e-05, "Finetune/Loss": 1.00266695022583, "Finetune/Loss (Raw)": 0.9147676229476929, "Finetune/Step": 356, "Finetune/Step Time": 5.510570587590337} +{"Finetune/Learning Rate": 1.9920928244588662e-05, "Finetune/Loss": 1.001806378364563, "Finetune/Loss (Raw)": 0.9250653386116028, "Finetune/Step": 357, "Finetune/Step Time": 5.524246733635664} +{"Finetune/Learning Rate": 1.9920144464287527e-05, "Finetune/Loss": 0.9957609176635742, "Finetune/Loss (Raw)": 0.3857634663581848, "Finetune/Step": 358, "Finetune/Step Time": 5.5547386687248945} +{"Finetune/Learning Rate": 1.9919356834171996e-05, "Finetune/Loss": 0.9944324493408203, "Finetune/Loss (Raw)": 0.8546468019485474, "Finetune/Step": 359, "Finetune/Step Time": 5.5559870060533285} +{"Finetune/Learning Rate": 1.9918565354547738e-05, "Finetune/Loss": 0.9940327405929565, "Finetune/Loss (Raw)": 1.0156925916671753, "Finetune/Step": 360, "Finetune/Step Time": 5.555817283689976} +{"Finetune/Learning Rate": 1.991777002572191e-05, "Finetune/Loss": 0.9986101388931274, "Finetune/Loss (Raw)": 0.8942499756813049, "Finetune/Step": 361, "Finetune/Step Time": 5.5246108956635} +{"Finetune/Learning Rate": 1.9916970848003166e-05, "Finetune/Loss": 0.9971118569374084, "Finetune/Loss (Raw)": 0.9781239628791809, "Finetune/Step": 362, "Finetune/Step Time": 5.517256861552596} +{"Finetune/Learning Rate": 1.9916167821701645e-05, "Finetune/Loss": 0.9961254596710205, "Finetune/Loss (Raw)": 1.0802687406539917, "Finetune/Step": 363, "Finetune/Step Time": 5.4757161140441895} +{"Finetune/Learning Rate": 1.991536094712899e-05, "Finetune/Loss": 0.9967756867408752, "Finetune/Loss (Raw)": 1.1042413711547852, "Finetune/Step": 364, "Finetune/Step Time": 5.474399710074067} +{"Finetune/Learning Rate": 1.991455022459833e-05, "Finetune/Loss": 0.9969966411590576, "Finetune/Loss (Raw)": 1.1224225759506226, "Finetune/Step": 365, "Finetune/Step Time": 5.444017628207803} +{"Finetune/Learning Rate": 1.9913735654424297e-05, "Finetune/Loss": 0.9961472153663635, "Finetune/Loss (Raw)": 0.9105357527732849, "Finetune/Step": 366, "Finetune/Step Time": 5.449352102354169} +{"Finetune/Learning Rate": 1.9912917236923003e-05, "Finetune/Loss": 0.996980607509613, "Finetune/Loss (Raw)": 1.0075684785842896, "Finetune/Step": 367, "Finetune/Step Time": 5.449112251400948} +{"Finetune/Learning Rate": 1.9912094972412067e-05, "Finetune/Loss": 0.9974737167358398, "Finetune/Loss (Raw)": 1.0298995971679688, "Finetune/Step": 368, "Finetune/Step Time": 5.449495282024145} +{"Finetune/Learning Rate": 1.9911268861210584e-05, "Finetune/Loss": 0.996583104133606, "Finetune/Loss (Raw)": 0.9915457367897034, "Finetune/Step": 369, "Finetune/Step Time": 5.4495032001286745} +{"Finetune/Learning Rate": 1.991043890363916e-05, "Finetune/Loss": 0.9973405003547668, "Finetune/Loss (Raw)": 1.16800856590271, "Finetune/Step": 370, "Finetune/Step Time": 5.448579242452979} +{"Finetune/Learning Rate": 1.9909605100019883e-05, "Finetune/Loss": 0.997921347618103, "Finetune/Loss (Raw)": 1.1613726615905762, "Finetune/Step": 371, "Finetune/Step Time": 5.433901382610202} +{"Finetune/Learning Rate": 1.9908767450676336e-05, "Finetune/Loss": 0.9989285469055176, "Finetune/Loss (Raw)": 1.0547078847885132, "Finetune/Step": 372, "Finetune/Step Time": 5.447343826293945} +{"Finetune/Learning Rate": 1.9907925955933593e-05, "Finetune/Loss": 0.9928900003433228, "Finetune/Loss (Raw)": 0.4407692849636078, "Finetune/Step": 373, "Finetune/Step Time": 5.482062503695488} +{"Finetune/Learning Rate": 1.9907080616118223e-05, "Finetune/Loss": 0.9865645170211792, "Finetune/Loss (Raw)": 0.3619769215583801, "Finetune/Step": 374, "Finetune/Step Time": 5.504053326323628} +{"Finetune/Learning Rate": 1.9906231431558283e-05, "Finetune/Loss": 0.9869067668914795, "Finetune/Loss (Raw)": 1.1083520650863647, "Finetune/Step": 375, "Finetune/Step Time": 5.495385266840458} +{"Finetune/Learning Rate": 1.990537840258333e-05, "Finetune/Loss": 0.9860848784446716, "Finetune/Loss (Raw)": 0.9030711054801941, "Finetune/Step": 376, "Finetune/Step Time": 5.4950992204248905} +{"Finetune/Learning Rate": 1.9904521529524402e-05, "Finetune/Loss": 0.9856643676757812, "Finetune/Loss (Raw)": 0.974677562713623, "Finetune/Step": 377, "Finetune/Step Time": 5.500893717631698} +{"Finetune/Learning Rate": 1.990366081271404e-05, "Finetune/Loss": 0.9867140054702759, "Finetune/Loss (Raw)": 1.2220897674560547, "Finetune/Step": 378, "Finetune/Step Time": 5.5056209824979305} +{"Finetune/Learning Rate": 1.990279625248627e-05, "Finetune/Loss": 0.9867218732833862, "Finetune/Loss (Raw)": 1.152470588684082, "Finetune/Step": 379, "Finetune/Step Time": 5.491803923621774} +{"Finetune/Learning Rate": 1.990192784917661e-05, "Finetune/Loss": 0.986662745475769, "Finetune/Loss (Raw)": 1.0409637689590454, "Finetune/Step": 380, "Finetune/Step Time": 5.489798154681921} +{"Finetune/Learning Rate": 1.9901055603122067e-05, "Finetune/Loss": 0.9865138530731201, "Finetune/Loss (Raw)": 1.0652979612350464, "Finetune/Step": 381, "Finetune/Step Time": 5.4979422353208065} +{"Finetune/Learning Rate": 1.9900179514661148e-05, "Finetune/Loss": 0.9860418438911438, "Finetune/Loss (Raw)": 0.92397540807724, "Finetune/Step": 382, "Finetune/Step Time": 5.544955460354686} +{"Finetune/Learning Rate": 1.9899299584133848e-05, "Finetune/Loss": 0.9909330010414124, "Finetune/Loss (Raw)": 0.9419720768928528, "Finetune/Step": 383, "Finetune/Step Time": 5.547495096921921} +{"Finetune/Learning Rate": 1.9898415811881643e-05, "Finetune/Loss": 0.9873530864715576, "Finetune/Loss (Raw)": 0.49524468183517456, "Finetune/Step": 384, "Finetune/Step Time": 5.574777832254767} +{"Finetune/Learning Rate": 1.989752819824751e-05, "Finetune/Loss": 0.9861047863960266, "Finetune/Loss (Raw)": 1.014902114868164, "Finetune/Step": 385, "Finetune/Step Time": 5.571567993611097} +{"Finetune/Learning Rate": 1.989663674357592e-05, "Finetune/Loss": 0.9852758646011353, "Finetune/Loss (Raw)": 1.012515902519226, "Finetune/Step": 386, "Finetune/Step Time": 5.5763241574168205} +{"Finetune/Learning Rate": 1.9895741448212822e-05, "Finetune/Loss": 0.9836090803146362, "Finetune/Loss (Raw)": 0.874856173992157, "Finetune/Step": 387, "Finetune/Step Time": 5.577945975586772} +{"Finetune/Learning Rate": 1.9894842312505673e-05, "Finetune/Loss": 0.9824491739273071, "Finetune/Loss (Raw)": 0.9548258185386658, "Finetune/Step": 388, "Finetune/Step Time": 5.556120442226529} +{"Finetune/Learning Rate": 1.9893939336803396e-05, "Finetune/Loss": 0.9829168319702148, "Finetune/Loss (Raw)": 0.9434069395065308, "Finetune/Step": 389, "Finetune/Step Time": 5.555924575775862} +{"Finetune/Learning Rate": 1.989303252145643e-05, "Finetune/Loss": 0.9828475117683411, "Finetune/Loss (Raw)": 0.8438454866409302, "Finetune/Step": 390, "Finetune/Step Time": 5.542564462870359} +{"Finetune/Learning Rate": 1.9892121866816682e-05, "Finetune/Loss": 0.9826790690422058, "Finetune/Loss (Raw)": 1.0086605548858643, "Finetune/Step": 391, "Finetune/Step Time": 5.537803994491696} +{"Finetune/Learning Rate": 1.989120737323757e-05, "Finetune/Loss": 0.9825848340988159, "Finetune/Loss (Raw)": 0.9784970283508301, "Finetune/Step": 392, "Finetune/Step Time": 5.534185878932476} +{"Finetune/Learning Rate": 1.9890289041073984e-05, "Finetune/Loss": 0.9814715385437012, "Finetune/Loss (Raw)": 0.8554006218910217, "Finetune/Step": 393, "Finetune/Step Time": 5.54617671854794} +{"Finetune/Learning Rate": 1.988936687068231e-05, "Finetune/Loss": 0.9782326221466064, "Finetune/Loss (Raw)": 0.8066777586936951, "Finetune/Step": 394, "Finetune/Step Time": 5.561402732506394} +{"Finetune/Learning Rate": 1.9888440862420433e-05, "Finetune/Loss": 0.9806021451950073, "Finetune/Loss (Raw)": 1.2062077522277832, "Finetune/Step": 395, "Finetune/Step Time": 5.56046324595809} +{"Finetune/Learning Rate": 1.988751101664771e-05, "Finetune/Loss": 0.9805659055709839, "Finetune/Loss (Raw)": 1.112141728401184, "Finetune/Step": 396, "Finetune/Step Time": 5.561146650463343} +{"Finetune/Learning Rate": 1.9886577333724995e-05, "Finetune/Loss": 0.9790744185447693, "Finetune/Loss (Raw)": 0.8520886898040771, "Finetune/Step": 397, "Finetune/Step Time": 5.5769965797662735} +{"Finetune/Learning Rate": 1.9885639814014642e-05, "Finetune/Loss": 0.9799804091453552, "Finetune/Loss (Raw)": 1.18341863155365, "Finetune/Step": 398, "Finetune/Step Time": 5.570659790188074} +{"Finetune/Learning Rate": 1.9884698457880476e-05, "Finetune/Loss": 0.9791778326034546, "Finetune/Loss (Raw)": 0.9268636107444763, "Finetune/Step": 399, "Finetune/Step Time": 5.5584717988967896} +{"Finetune/Learning Rate": 1.988375326568782e-05, "Finetune/Loss": 0.9778597950935364, "Finetune/Loss (Raw)": 0.9507681727409363, "Finetune/Step": 400, "Finetune/Step Time": 5.5562810227274895} +{"Finetune/Learning Rate": 1.9882804237803487e-05, "Finetune/Loss": 0.9783048629760742, "Finetune/Loss (Raw)": 1.1177384853363037, "Finetune/Step": 401, "Finetune/Step Time": 5.558824518695474} +{"Finetune/Learning Rate": 1.9881851374595774e-05, "Finetune/Loss": 0.9775412082672119, "Finetune/Loss (Raw)": 1.0083551406860352, "Finetune/Step": 402, "Finetune/Step Time": 5.5708157531917095} +{"Finetune/Learning Rate": 1.988089467643447e-05, "Finetune/Loss": 0.9771677255630493, "Finetune/Loss (Raw)": 0.9747372269630432, "Finetune/Step": 403, "Finetune/Step Time": 5.5702240858227015} +{"Finetune/Learning Rate": 1.9879934143690848e-05, "Finetune/Loss": 0.9791083931922913, "Finetune/Loss (Raw)": 1.0830821990966797, "Finetune/Step": 404, "Finetune/Step Time": 5.53740524686873} +{"Finetune/Learning Rate": 1.9878969776737675e-05, "Finetune/Loss": 0.9768297672271729, "Finetune/Loss (Raw)": 0.735045850276947, "Finetune/Step": 405, "Finetune/Step Time": 5.549836676567793} +{"Finetune/Learning Rate": 1.9878001575949205e-05, "Finetune/Loss": 0.9724357724189758, "Finetune/Loss (Raw)": 0.4781021475791931, "Finetune/Step": 406, "Finetune/Step Time": 5.581419451162219} +{"Finetune/Learning Rate": 1.9877029541701174e-05, "Finetune/Loss": 0.96616530418396, "Finetune/Loss (Raw)": 0.4202790856361389, "Finetune/Step": 407, "Finetune/Step Time": 5.60048239864409} +{"Finetune/Learning Rate": 1.9876053674370806e-05, "Finetune/Loss": 0.9661160707473755, "Finetune/Loss (Raw)": 0.9846407175064087, "Finetune/Step": 408, "Finetune/Step Time": 5.59969955123961} +{"Finetune/Learning Rate": 1.9875073974336827e-05, "Finetune/Loss": 0.9669605493545532, "Finetune/Loss (Raw)": 1.193666696548462, "Finetune/Step": 409, "Finetune/Step Time": 5.6010581981390715} +{"Finetune/Learning Rate": 1.9874090441979433e-05, "Finetune/Loss": 0.968525767326355, "Finetune/Loss (Raw)": 1.226125717163086, "Finetune/Step": 410, "Finetune/Step Time": 5.599375465884805} +{"Finetune/Learning Rate": 1.987310307768031e-05, "Finetune/Loss": 0.9673736095428467, "Finetune/Loss (Raw)": 0.9987630844116211, "Finetune/Step": 411, "Finetune/Step Time": 5.602008730173111} +{"Finetune/Learning Rate": 1.987211188182264e-05, "Finetune/Loss": 0.9612528085708618, "Finetune/Loss (Raw)": 0.37319400906562805, "Finetune/Step": 412, "Finetune/Step Time": 5.641629846766591} +{"Finetune/Learning Rate": 1.9871116854791085e-05, "Finetune/Loss": 0.9621466994285583, "Finetune/Loss (Raw)": 1.12395179271698, "Finetune/Step": 413, "Finetune/Step Time": 5.619198935106397} +{"Finetune/Learning Rate": 1.9870117996971797e-05, "Finetune/Loss": 0.9610238671302795, "Finetune/Loss (Raw)": 0.94044029712677, "Finetune/Step": 414, "Finetune/Step Time": 5.604396667331457} +{"Finetune/Learning Rate": 1.986911530875241e-05, "Finetune/Loss": 0.9608298540115356, "Finetune/Loss (Raw)": 1.057230830192566, "Finetune/Step": 415, "Finetune/Step Time": 5.60167945548892} +{"Finetune/Learning Rate": 1.9868108790522052e-05, "Finetune/Loss": 0.9625279903411865, "Finetune/Loss (Raw)": 1.1278281211853027, "Finetune/Step": 416, "Finetune/Step Time": 5.587315510958433} +{"Finetune/Learning Rate": 1.986709844267133e-05, "Finetune/Loss": 0.961961030960083, "Finetune/Loss (Raw)": 1.054170846939087, "Finetune/Step": 417, "Finetune/Step Time": 5.582177720963955} +{"Finetune/Learning Rate": 1.9866084265592343e-05, "Finetune/Loss": 0.9630103707313538, "Finetune/Loss (Raw)": 1.1011983156204224, "Finetune/Step": 418, "Finetune/Step Time": 5.56983146071434} +{"Finetune/Learning Rate": 1.986506625967867e-05, "Finetune/Loss": 0.9618641138076782, "Finetune/Loss (Raw)": 0.9794390797615051, "Finetune/Step": 419, "Finetune/Step Time": 5.587465565651655} +{"Finetune/Learning Rate": 1.9864044425325382e-05, "Finetune/Loss": 0.9560549259185791, "Finetune/Loss (Raw)": 0.3646475374698639, "Finetune/Step": 420, "Finetune/Step Time": 5.616985892876983} +{"Finetune/Learning Rate": 1.986301876292903e-05, "Finetune/Loss": 0.9551190137863159, "Finetune/Loss (Raw)": 1.0340145826339722, "Finetune/Step": 421, "Finetune/Step Time": 5.616690326482058} +{"Finetune/Learning Rate": 1.986198927288766e-05, "Finetune/Loss": 0.956997811794281, "Finetune/Loss (Raw)": 1.0076086521148682, "Finetune/Step": 422, "Finetune/Step Time": 5.622418511658907} +{"Finetune/Learning Rate": 1.9860955955600784e-05, "Finetune/Loss": 0.9573294520378113, "Finetune/Loss (Raw)": 1.1357779502868652, "Finetune/Step": 423, "Finetune/Step Time": 5.615974636748433} +{"Finetune/Learning Rate": 1.9859918811469424e-05, "Finetune/Loss": 0.9574867486953735, "Finetune/Loss (Raw)": 1.0414063930511475, "Finetune/Step": 424, "Finetune/Step Time": 5.615220509469509} +{"Finetune/Learning Rate": 1.9858877840896076e-05, "Finetune/Loss": 0.9545997381210327, "Finetune/Loss (Raw)": 0.7683448195457458, "Finetune/Step": 425, "Finetune/Step Time": 5.650341281667352} +{"Finetune/Learning Rate": 1.9857833044284714e-05, "Finetune/Loss": 0.9554500579833984, "Finetune/Loss (Raw)": 1.0199437141418457, "Finetune/Step": 426, "Finetune/Step Time": 5.629090670496225} +{"Finetune/Learning Rate": 1.9856784422040802e-05, "Finetune/Loss": 0.960722804069519, "Finetune/Loss (Raw)": 0.9823999404907227, "Finetune/Step": 427, "Finetune/Step Time": 5.598512634634972} +{"Finetune/Learning Rate": 1.98557319745713e-05, "Finetune/Loss": 0.9666546583175659, "Finetune/Loss (Raw)": 1.1258561611175537, "Finetune/Step": 428, "Finetune/Step Time": 5.56813639216125} +{"Finetune/Learning Rate": 1.985467570228463e-05, "Finetune/Loss": 0.971977174282074, "Finetune/Loss (Raw)": 1.0774297714233398, "Finetune/Step": 429, "Finetune/Step Time": 5.541404295712709} +{"Finetune/Learning Rate": 1.985361560559072e-05, "Finetune/Loss": 0.9720642566680908, "Finetune/Loss (Raw)": 0.903761625289917, "Finetune/Step": 430, "Finetune/Step Time": 5.54058438539505} +{"Finetune/Learning Rate": 1.9852551684900964e-05, "Finetune/Loss": 0.9780740737915039, "Finetune/Loss (Raw)": 1.09089195728302, "Finetune/Step": 431, "Finetune/Step Time": 5.519078681245446} +{"Finetune/Learning Rate": 1.9851483940628256e-05, "Finetune/Loss": 0.9768728613853455, "Finetune/Loss (Raw)": 0.9447579979896545, "Finetune/Step": 432, "Finetune/Step Time": 5.519600970670581} +{"Finetune/Learning Rate": 1.9850412373186967e-05, "Finetune/Loss": 0.9758442640304565, "Finetune/Loss (Raw)": 0.9180371761322021, "Finetune/Step": 433, "Finetune/Step Time": 5.523361349478364} +{"Finetune/Learning Rate": 1.9849336982992948e-05, "Finetune/Loss": 0.9761085510253906, "Finetune/Loss (Raw)": 0.9847357273101807, "Finetune/Step": 434, "Finetune/Step Time": 5.524225419387221} +{"Finetune/Learning Rate": 1.984825777046354e-05, "Finetune/Loss": 0.9735112190246582, "Finetune/Loss (Raw)": 0.839707612991333, "Finetune/Step": 435, "Finetune/Step Time": 5.498276283964515} +{"Finetune/Learning Rate": 1.9847174736017557e-05, "Finetune/Loss": 0.9724993705749512, "Finetune/Loss (Raw)": 0.9236382246017456, "Finetune/Step": 436, "Finetune/Step Time": 5.495152799412608} +{"Finetune/Learning Rate": 1.9846087880075314e-05, "Finetune/Loss": 0.9715425968170166, "Finetune/Loss (Raw)": 1.029752492904663, "Finetune/Step": 437, "Finetune/Step Time": 5.49170933291316} +{"Finetune/Learning Rate": 1.984499720305859e-05, "Finetune/Loss": 0.9712355732917786, "Finetune/Loss (Raw)": 0.9731261730194092, "Finetune/Step": 438, "Finetune/Step Time": 5.483347557485104} +{"Finetune/Learning Rate": 1.984390270539066e-05, "Finetune/Loss": 0.9707913398742676, "Finetune/Loss (Raw)": 0.9811680316925049, "Finetune/Step": 439, "Finetune/Step Time": 5.488119041547179} +{"Finetune/Learning Rate": 1.9842804387496276e-05, "Finetune/Loss": 0.9715656042098999, "Finetune/Loss (Raw)": 1.0842208862304688, "Finetune/Step": 440, "Finetune/Step Time": 5.494674438610673} +{"Finetune/Learning Rate": 1.9841702249801677e-05, "Finetune/Loss": 0.9711032509803772, "Finetune/Loss (Raw)": 1.0584574937820435, "Finetune/Step": 441, "Finetune/Step Time": 5.492348670959473} +{"Finetune/Learning Rate": 1.9840596292734573e-05, "Finetune/Loss": 0.9652789831161499, "Finetune/Loss (Raw)": 0.4414684474468231, "Finetune/Step": 442, "Finetune/Step Time": 5.5256881304085255} +{"Finetune/Learning Rate": 1.9839486516724173e-05, "Finetune/Loss": 0.9659311771392822, "Finetune/Loss (Raw)": 1.0782103538513184, "Finetune/Step": 443, "Finetune/Step Time": 5.529700016602874} +{"Finetune/Learning Rate": 1.983837292220115e-05, "Finetune/Loss": 0.9718356132507324, "Finetune/Loss (Raw)": 1.1418102979660034, "Finetune/Step": 444, "Finetune/Step Time": 5.4989708084613085} +{"Finetune/Learning Rate": 1.983725550959768e-05, "Finetune/Loss": 0.9721622467041016, "Finetune/Loss (Raw)": 1.0155503749847412, "Finetune/Step": 445, "Finetune/Step Time": 5.500775959342718} +{"Finetune/Learning Rate": 1.98361342793474e-05, "Finetune/Loss": 0.972618043422699, "Finetune/Loss (Raw)": 1.1215100288391113, "Finetune/Step": 446, "Finetune/Step Time": 5.508935974910855} +{"Finetune/Learning Rate": 1.9835009231885444e-05, "Finetune/Loss": 0.9729429483413696, "Finetune/Loss (Raw)": 1.0386556386947632, "Finetune/Step": 447, "Finetune/Step Time": 5.498687585815787} +{"Finetune/Learning Rate": 1.9833880367648412e-05, "Finetune/Loss": 0.9730361104011536, "Finetune/Loss (Raw)": 0.9613854885101318, "Finetune/Step": 448, "Finetune/Step Time": 5.504325984045863} +{"Finetune/Learning Rate": 1.9832747687074403e-05, "Finetune/Loss": 0.9793615341186523, "Finetune/Loss (Raw)": 1.1388747692108154, "Finetune/Step": 449, "Finetune/Step Time": 5.471425095573068} +{"Finetune/Learning Rate": 1.9831611190602984e-05, "Finetune/Loss": 0.9789367318153381, "Finetune/Loss (Raw)": 0.9440602660179138, "Finetune/Step": 450, "Finetune/Step Time": 5.470426194369793} +{"Finetune/Learning Rate": 1.983047087867521e-05, "Finetune/Loss": 0.978179931640625, "Finetune/Loss (Raw)": 0.97684645652771, "Finetune/Step": 451, "Finetune/Step Time": 5.474796155467629} +{"Finetune/Learning Rate": 1.9829326751733607e-05, "Finetune/Loss": 0.9774274826049805, "Finetune/Loss (Raw)": 1.007158875465393, "Finetune/Step": 452, "Finetune/Step Time": 5.468221064656973} +{"Finetune/Learning Rate": 1.9828178810222197e-05, "Finetune/Loss": 0.9788376688957214, "Finetune/Loss (Raw)": 1.2272989749908447, "Finetune/Step": 453, "Finetune/Step Time": 5.463177669793367} +{"Finetune/Learning Rate": 1.9827027054586468e-05, "Finetune/Loss": 0.9770433306694031, "Finetune/Loss (Raw)": 0.9928589463233948, "Finetune/Step": 454, "Finetune/Step Time": 5.4779654163867235} +{"Finetune/Learning Rate": 1.9825871485273395e-05, "Finetune/Loss": 0.9776138067245483, "Finetune/Loss (Raw)": 1.0926764011383057, "Finetune/Step": 455, "Finetune/Step Time": 5.476271837949753} +{"Finetune/Learning Rate": 1.9824712102731435e-05, "Finetune/Loss": 0.9718979597091675, "Finetune/Loss (Raw)": 0.344155490398407, "Finetune/Step": 456, "Finetune/Step Time": 5.495367588475347} +{"Finetune/Learning Rate": 1.982354890741052e-05, "Finetune/Loss": 0.9717100262641907, "Finetune/Loss (Raw)": 0.9207944273948669, "Finetune/Step": 457, "Finetune/Step Time": 5.475744696334004} +{"Finetune/Learning Rate": 1.982238189976206e-05, "Finetune/Loss": 0.9743273258209229, "Finetune/Loss (Raw)": 1.2187604904174805, "Finetune/Step": 458, "Finetune/Step Time": 5.473224855959415} +{"Finetune/Learning Rate": 1.9821211080238953e-05, "Finetune/Loss": 0.9738584160804749, "Finetune/Loss (Raw)": 0.9959830641746521, "Finetune/Step": 459, "Finetune/Step Time": 5.476177329197526} +{"Finetune/Learning Rate": 1.9820036449295564e-05, "Finetune/Loss": 0.9740196466445923, "Finetune/Loss (Raw)": 0.851059079170227, "Finetune/Step": 460, "Finetune/Step Time": 5.475097814574838} +{"Finetune/Learning Rate": 1.9818858007387753e-05, "Finetune/Loss": 0.9735827445983887, "Finetune/Loss (Raw)": 0.9865723252296448, "Finetune/Step": 461, "Finetune/Step Time": 5.474438030272722} +{"Finetune/Learning Rate": 1.9817675754972846e-05, "Finetune/Loss": 0.9735022783279419, "Finetune/Loss (Raw)": 0.9916552305221558, "Finetune/Step": 462, "Finetune/Step Time": 5.473911022767425} +{"Finetune/Learning Rate": 1.9816489692509654e-05, "Finetune/Loss": 0.9749219417572021, "Finetune/Loss (Raw)": 1.1987663507461548, "Finetune/Step": 463, "Finetune/Step Time": 5.468088256195188} +{"Finetune/Learning Rate": 1.9815299820458458e-05, "Finetune/Loss": 0.9686051607131958, "Finetune/Loss (Raw)": 0.2912391722202301, "Finetune/Step": 464, "Finetune/Step Time": 5.498168848454952} +{"Finetune/Learning Rate": 1.9814106139281033e-05, "Finetune/Loss": 0.9671251773834229, "Finetune/Loss (Raw)": 1.0471477508544922, "Finetune/Step": 465, "Finetune/Step Time": 5.50516402348876} +{"Finetune/Learning Rate": 1.9812908649440618e-05, "Finetune/Loss": 0.9667388796806335, "Finetune/Loss (Raw)": 0.9498401284217834, "Finetune/Step": 466, "Finetune/Step Time": 5.5100182965397835} +{"Finetune/Learning Rate": 1.981170735140194e-05, "Finetune/Loss": 0.9676635265350342, "Finetune/Loss (Raw)": 1.0850162506103516, "Finetune/Step": 467, "Finetune/Step Time": 5.509965073317289} +{"Finetune/Learning Rate": 1.9810502245631194e-05, "Finetune/Loss": 0.9669963121414185, "Finetune/Loss (Raw)": 0.8352215886116028, "Finetune/Step": 468, "Finetune/Step Time": 5.521691823378205} +{"Finetune/Learning Rate": 1.980929333259606e-05, "Finetune/Loss": 0.9617214202880859, "Finetune/Loss (Raw)": 0.3546407222747803, "Finetune/Step": 469, "Finetune/Step Time": 5.548108199611306} +{"Finetune/Learning Rate": 1.9808080612765694e-05, "Finetune/Loss": 0.960865318775177, "Finetune/Loss (Raw)": 0.9860790371894836, "Finetune/Step": 470, "Finetune/Step Time": 5.546771613880992} +{"Finetune/Learning Rate": 1.980686408661073e-05, "Finetune/Loss": 0.960777997970581, "Finetune/Loss (Raw)": 1.0121357440948486, "Finetune/Step": 471, "Finetune/Step Time": 5.562022736296058} +{"Finetune/Learning Rate": 1.9805643754603275e-05, "Finetune/Loss": 0.9611684679985046, "Finetune/Loss (Raw)": 1.1384788751602173, "Finetune/Step": 472, "Finetune/Step Time": 5.563208280131221} +{"Finetune/Learning Rate": 1.9804419617216922e-05, "Finetune/Loss": 0.9596095085144043, "Finetune/Loss (Raw)": 0.9250655770301819, "Finetune/Step": 473, "Finetune/Step Time": 5.561413329094648} +{"Finetune/Learning Rate": 1.9803191674926727e-05, "Finetune/Loss": 0.9590452909469604, "Finetune/Loss (Raw)": 1.0574668645858765, "Finetune/Step": 474, "Finetune/Step Time": 5.557572009041905} +{"Finetune/Learning Rate": 1.9801959928209236e-05, "Finetune/Loss": 0.9583651423454285, "Finetune/Loss (Raw)": 1.0711232423782349, "Finetune/Step": 475, "Finetune/Step Time": 5.556281346827745} +{"Finetune/Learning Rate": 1.9800724377542463e-05, "Finetune/Loss": 0.9579391479492188, "Finetune/Loss (Raw)": 1.0305699110031128, "Finetune/Step": 476, "Finetune/Step Time": 5.540209703147411} +{"Finetune/Learning Rate": 1.9799485023405904e-05, "Finetune/Loss": 0.9575486183166504, "Finetune/Loss (Raw)": 1.1089959144592285, "Finetune/Step": 477, "Finetune/Step Time": 5.5463000331074} +{"Finetune/Learning Rate": 1.9798241866280523e-05, "Finetune/Loss": 0.9573850035667419, "Finetune/Loss (Raw)": 0.9715277552604675, "Finetune/Step": 478, "Finetune/Step Time": 5.5228969398885965} +{"Finetune/Learning Rate": 1.979699490664877e-05, "Finetune/Loss": 0.9576181173324585, "Finetune/Loss (Raw)": 1.0328741073608398, "Finetune/Step": 479, "Finetune/Step Time": 5.508043387904763} +{"Finetune/Learning Rate": 1.9795744144994562e-05, "Finetune/Loss": 0.9582815170288086, "Finetune/Loss (Raw)": 1.090851068496704, "Finetune/Step": 480, "Finetune/Step Time": 5.500275963917375} +{"Finetune/Learning Rate": 1.97944895818033e-05, "Finetune/Loss": 0.9586353898048401, "Finetune/Loss (Raw)": 1.0299873352050781, "Finetune/Step": 481, "Finetune/Step Time": 5.504773527383804} +{"Finetune/Learning Rate": 1.979323121756185e-05, "Finetune/Loss": 0.9597594738006592, "Finetune/Loss (Raw)": 1.0530928373336792, "Finetune/Step": 482, "Finetune/Step Time": 5.513791298493743} +{"Finetune/Learning Rate": 1.9791969052758563e-05, "Finetune/Loss": 0.9595447778701782, "Finetune/Loss (Raw)": 1.0417637825012207, "Finetune/Step": 483, "Finetune/Step Time": 5.524607660248876} +{"Finetune/Learning Rate": 1.9790703087883258e-05, "Finetune/Loss": 0.9600416421890259, "Finetune/Loss (Raw)": 0.9783771634101868, "Finetune/Step": 484, "Finetune/Step Time": 5.5287020318210125} +{"Finetune/Learning Rate": 1.9789433323427233e-05, "Finetune/Loss": 0.9594997763633728, "Finetune/Loss (Raw)": 0.8557026386260986, "Finetune/Step": 485, "Finetune/Step Time": 5.532451355829835} +{"Finetune/Learning Rate": 1.9788159759883258e-05, "Finetune/Loss": 0.964259147644043, "Finetune/Loss (Raw)": 0.9949623346328735, "Finetune/Step": 486, "Finetune/Step Time": 5.514835400506854} +{"Finetune/Learning Rate": 1.9786882397745573e-05, "Finetune/Loss": 0.9657332301139832, "Finetune/Loss (Raw)": 1.0433316230773926, "Finetune/Step": 487, "Finetune/Step Time": 5.514330744743347} +{"Finetune/Learning Rate": 1.9785601237509907e-05, "Finetune/Loss": 0.965999960899353, "Finetune/Loss (Raw)": 1.0498377084732056, "Finetune/Step": 488, "Finetune/Step Time": 5.5119923911988735} +{"Finetune/Learning Rate": 1.978431627967345e-05, "Finetune/Loss": 0.966437816619873, "Finetune/Loss (Raw)": 0.9502888321876526, "Finetune/Step": 489, "Finetune/Step Time": 5.516444332897663} +{"Finetune/Learning Rate": 1.9783027524734865e-05, "Finetune/Loss": 0.9662042856216431, "Finetune/Loss (Raw)": 0.9482317566871643, "Finetune/Step": 490, "Finetune/Step Time": 5.522617727518082} +{"Finetune/Learning Rate": 1.9781734973194294e-05, "Finetune/Loss": 0.9630603194236755, "Finetune/Loss (Raw)": 0.677840530872345, "Finetune/Step": 491, "Finetune/Step Time": 5.551185702905059} +{"Finetune/Learning Rate": 1.9780438625553352e-05, "Finetune/Loss": 0.9620333313941956, "Finetune/Loss (Raw)": 0.9727904796600342, "Finetune/Step": 492, "Finetune/Step Time": 5.556920548900962} +{"Finetune/Learning Rate": 1.9779138482315128e-05, "Finetune/Loss": 0.9617010951042175, "Finetune/Loss (Raw)": 1.0798957347869873, "Finetune/Step": 493, "Finetune/Step Time": 5.559056680649519} +{"Finetune/Learning Rate": 1.977783454398418e-05, "Finetune/Loss": 0.9622422456741333, "Finetune/Loss (Raw)": 0.9797959327697754, "Finetune/Step": 494, "Finetune/Step Time": 5.569872947409749} +{"Finetune/Learning Rate": 1.9776526811066543e-05, "Finetune/Loss": 0.9623662829399109, "Finetune/Loss (Raw)": 1.023454189300537, "Finetune/Step": 495, "Finetune/Step Time": 5.569962106645107} +{"Finetune/Learning Rate": 1.977521528406972e-05, "Finetune/Loss": 0.9624888896942139, "Finetune/Loss (Raw)": 1.0455888509750366, "Finetune/Step": 496, "Finetune/Step Time": 5.583098758012056} +{"Finetune/Learning Rate": 1.9773899963502693e-05, "Finetune/Loss": 0.962360680103302, "Finetune/Loss (Raw)": 0.9751378297805786, "Finetune/Step": 497, "Finetune/Step Time": 5.58550114184618} +{"Finetune/Learning Rate": 1.9772580849875905e-05, "Finetune/Loss": 0.9614162445068359, "Finetune/Loss (Raw)": 1.047122597694397, "Finetune/Step": 498, "Finetune/Step Time": 5.583665730431676} +{"Finetune/Learning Rate": 1.9771257943701284e-05, "Finetune/Loss": 0.9593298435211182, "Finetune/Loss (Raw)": 0.8943130373954773, "Finetune/Step": 499, "Finetune/Step Time": 5.580509353429079} +{"Finetune/Learning Rate": 1.9769931245492225e-05, "Finetune/Loss": 0.9588698148727417, "Finetune/Loss (Raw)": 0.9958186149597168, "Finetune/Step": 500, "Finetune/Step Time": 5.580596584826708} +{"Finetune/Learning Rate": 1.976860075576359e-05, "Finetune/Loss": 0.9616504907608032, "Finetune/Loss (Raw)": 0.7966972589492798, "Finetune/Step": 501, "Finetune/Step Time": 5.554707117378712} +{"Finetune/Learning Rate": 1.9767266475031716e-05, "Finetune/Loss": 0.9657982587814331, "Finetune/Loss (Raw)": 0.8928964138031006, "Finetune/Step": 502, "Finetune/Step Time": 5.552385816350579} +{"Finetune/Learning Rate": 1.9765928403814416e-05, "Finetune/Loss": 0.9652782678604126, "Finetune/Loss (Raw)": 1.0417946577072144, "Finetune/Step": 503, "Finetune/Step Time": 5.573586851358414} +{"Finetune/Learning Rate": 1.9764586542630965e-05, "Finetune/Loss": 0.9665862321853638, "Finetune/Loss (Raw)": 1.0704935789108276, "Finetune/Step": 504, "Finetune/Step Time": 5.563757210969925} +{"Finetune/Learning Rate": 1.9763240892002113e-05, "Finetune/Loss": 0.9668943285942078, "Finetune/Loss (Raw)": 1.0141032934188843, "Finetune/Step": 505, "Finetune/Step Time": 5.559369117021561} +{"Finetune/Learning Rate": 1.9761891452450082e-05, "Finetune/Loss": 0.9638504981994629, "Finetune/Loss (Raw)": 0.8324795961380005, "Finetune/Step": 506, "Finetune/Step Time": 5.565943710505962} +{"Finetune/Learning Rate": 1.9760538224498563e-05, "Finetune/Loss": 0.9628152847290039, "Finetune/Loss (Raw)": 1.0199670791625977, "Finetune/Step": 507, "Finetune/Step Time": 5.567608525976539} +{"Finetune/Learning Rate": 1.975918120867272e-05, "Finetune/Loss": 0.9624732136726379, "Finetune/Loss (Raw)": 0.9971805810928345, "Finetune/Step": 508, "Finetune/Step Time": 5.567228881642222} +{"Finetune/Learning Rate": 1.975782040549918e-05, "Finetune/Loss": 0.9618006944656372, "Finetune/Loss (Raw)": 0.9792148470878601, "Finetune/Step": 509, "Finetune/Step Time": 5.570481471717358} +{"Finetune/Learning Rate": 1.9756455815506045e-05, "Finetune/Loss": 0.9618558883666992, "Finetune/Loss (Raw)": 0.9310382604598999, "Finetune/Step": 510, "Finetune/Step Time": 5.530071780085564} +{"Finetune/Learning Rate": 1.9755087439222888e-05, "Finetune/Loss": 0.9631383419036865, "Finetune/Loss (Raw)": 1.1061294078826904, "Finetune/Step": 511, "Finetune/Step Time": 5.500493684783578} +{"Finetune/Learning Rate": 1.975371527718075e-05, "Finetune/Loss": 0.9673502445220947, "Finetune/Loss (Raw)": 1.0343687534332275, "Finetune/Step": 512, "Finetune/Step Time": 5.477163406088948} +{"Finetune/Learning Rate": 1.9752339329912137e-05, "Finetune/Loss": 0.9677515029907227, "Finetune/Loss (Raw)": 1.0662542581558228, "Finetune/Step": 513, "Finetune/Step Time": 5.481802314519882} +{"Finetune/Learning Rate": 1.9750959597951026e-05, "Finetune/Loss": 0.9684445858001709, "Finetune/Loss (Raw)": 1.1012338399887085, "Finetune/Step": 514, "Finetune/Step Time": 5.4898520819842815} +{"Finetune/Learning Rate": 1.9749576081832865e-05, "Finetune/Loss": 0.9647516012191772, "Finetune/Loss (Raw)": 0.40215209126472473, "Finetune/Step": 515, "Finetune/Step Time": 5.519364384934306} +{"Finetune/Learning Rate": 1.9748188782094576e-05, "Finetune/Loss": 0.965792179107666, "Finetune/Loss (Raw)": 1.0880331993103027, "Finetune/Step": 516, "Finetune/Step Time": 5.522203858941793} +{"Finetune/Learning Rate": 1.9746797699274535e-05, "Finetune/Loss": 0.9665918946266174, "Finetune/Loss (Raw)": 1.045755386352539, "Finetune/Step": 517, "Finetune/Step Time": 5.521141767501831} +{"Finetune/Learning Rate": 1.9745402833912598e-05, "Finetune/Loss": 0.9683324098587036, "Finetune/Loss (Raw)": 1.0666431188583374, "Finetune/Step": 518, "Finetune/Step Time": 5.528767982497811} +{"Finetune/Learning Rate": 1.9744004186550086e-05, "Finetune/Loss": 0.9688112139701843, "Finetune/Loss (Raw)": 1.0699470043182373, "Finetune/Step": 519, "Finetune/Step Time": 5.5438144113868475} +{"Finetune/Learning Rate": 1.974260175772978e-05, "Finetune/Loss": 0.9691051244735718, "Finetune/Loss (Raw)": 1.0161161422729492, "Finetune/Step": 520, "Finetune/Step Time": 5.54355126991868} +{"Finetune/Learning Rate": 1.9741195547995944e-05, "Finetune/Loss": 0.9703242778778076, "Finetune/Loss (Raw)": 1.0114431381225586, "Finetune/Step": 521, "Finetune/Step Time": 5.534641120582819} +{"Finetune/Learning Rate": 1.9739785557894296e-05, "Finetune/Loss": 0.971847414970398, "Finetune/Loss (Raw)": 1.0016473531723022, "Finetune/Step": 522, "Finetune/Step Time": 5.516529751941562} +{"Finetune/Learning Rate": 1.9738371787972025e-05, "Finetune/Loss": 0.9694982767105103, "Finetune/Loss (Raw)": 0.9055147767066956, "Finetune/Step": 523, "Finetune/Step Time": 5.523562194779515} +{"Finetune/Learning Rate": 1.9736954238777793e-05, "Finetune/Loss": 0.9687694907188416, "Finetune/Loss (Raw)": 1.0188637971878052, "Finetune/Step": 524, "Finetune/Step Time": 5.519643537700176} +{"Finetune/Learning Rate": 1.9735532910861713e-05, "Finetune/Loss": 0.970120906829834, "Finetune/Loss (Raw)": 1.0250636339187622, "Finetune/Step": 525, "Finetune/Step Time": 5.502728745341301} +{"Finetune/Learning Rate": 1.9734107804775385e-05, "Finetune/Loss": 0.9681817293167114, "Finetune/Loss (Raw)": 0.9352092742919922, "Finetune/Step": 526, "Finetune/Step Time": 5.515478793531656} +{"Finetune/Learning Rate": 1.9732678921071858e-05, "Finetune/Loss": 0.9685038328170776, "Finetune/Loss (Raw)": 0.9680888056755066, "Finetune/Step": 527, "Finetune/Step Time": 5.509551241993904} +{"Finetune/Learning Rate": 1.9731246260305657e-05, "Finetune/Loss": 0.9692578911781311, "Finetune/Loss (Raw)": 1.047285795211792, "Finetune/Step": 528, "Finetune/Step Time": 5.504576787352562} +{"Finetune/Learning Rate": 1.9729809823032767e-05, "Finetune/Loss": 0.9691056609153748, "Finetune/Loss (Raw)": 1.098250150680542, "Finetune/Step": 529, "Finetune/Step Time": 5.504250193014741} +{"Finetune/Learning Rate": 1.9728369609810648e-05, "Finetune/Loss": 0.969831109046936, "Finetune/Loss (Raw)": 1.101218819618225, "Finetune/Step": 530, "Finetune/Step Time": 5.4884094297885895} +{"Finetune/Learning Rate": 1.972692562119821e-05, "Finetune/Loss": 0.9713053703308105, "Finetune/Loss (Raw)": 1.163435459136963, "Finetune/Step": 531, "Finetune/Step Time": 5.484696503728628} +{"Finetune/Learning Rate": 1.9725477857755842e-05, "Finetune/Loss": 0.9705547094345093, "Finetune/Loss (Raw)": 0.9870076775550842, "Finetune/Step": 532, "Finetune/Step Time": 5.482698794454336} +{"Finetune/Learning Rate": 1.9724026320045393e-05, "Finetune/Loss": 0.9722232818603516, "Finetune/Loss (Raw)": 0.9486116170883179, "Finetune/Step": 533, "Finetune/Step Time": 5.479144839569926} +{"Finetune/Learning Rate": 1.9722571008630172e-05, "Finetune/Loss": 0.9758692383766174, "Finetune/Loss (Raw)": 0.9447932839393616, "Finetune/Step": 534, "Finetune/Step Time": 5.455611145123839} +{"Finetune/Learning Rate": 1.9721111924074958e-05, "Finetune/Loss": 0.9798709154129028, "Finetune/Loss (Raw)": 0.9324905872344971, "Finetune/Step": 535, "Finetune/Step Time": 5.436957284808159} +{"Finetune/Learning Rate": 1.9719649066946e-05, "Finetune/Loss": 0.9786213040351868, "Finetune/Loss (Raw)": 0.8246901035308838, "Finetune/Step": 536, "Finetune/Step Time": 5.460212778300047} +{"Finetune/Learning Rate": 1.9718182437810996e-05, "Finetune/Loss": 0.9776329398155212, "Finetune/Loss (Raw)": 1.067149043083191, "Finetune/Step": 537, "Finetune/Step Time": 5.461390953511} +{"Finetune/Learning Rate": 1.971671203723912e-05, "Finetune/Loss": 0.9760371446609497, "Finetune/Loss (Raw)": 1.0218733549118042, "Finetune/Step": 538, "Finetune/Step Time": 5.470195164903998} +{"Finetune/Learning Rate": 1.9715237865801007e-05, "Finetune/Loss": 0.9768433570861816, "Finetune/Loss (Raw)": 1.1019597053527832, "Finetune/Step": 539, "Finetune/Step Time": 5.464323880150914} +{"Finetune/Learning Rate": 1.971375992406875e-05, "Finetune/Loss": 0.9815539121627808, "Finetune/Loss (Raw)": 0.9761457443237305, "Finetune/Step": 540, "Finetune/Step Time": 5.424557304009795} +{"Finetune/Learning Rate": 1.9712278212615913e-05, "Finetune/Loss": 0.9806677103042603, "Finetune/Loss (Raw)": 1.0105056762695312, "Finetune/Step": 541, "Finetune/Step Time": 5.431791853159666} +{"Finetune/Learning Rate": 1.9710792732017517e-05, "Finetune/Loss": 0.981337308883667, "Finetune/Loss (Raw)": 1.0261608362197876, "Finetune/Step": 542, "Finetune/Step Time": 5.430698033422232} +{"Finetune/Learning Rate": 1.9709303482850047e-05, "Finetune/Loss": 0.9819143414497375, "Finetune/Loss (Raw)": 1.1310844421386719, "Finetune/Step": 543, "Finetune/Step Time": 5.430589064955711} +{"Finetune/Learning Rate": 1.9707810465691456e-05, "Finetune/Loss": 0.9815623164176941, "Finetune/Loss (Raw)": 1.0827703475952148, "Finetune/Step": 544, "Finetune/Step Time": 5.4309221133589745} +{"Finetune/Learning Rate": 1.970631368112115e-05, "Finetune/Loss": 0.9810578227043152, "Finetune/Loss (Raw)": 0.9895942807197571, "Finetune/Step": 545, "Finetune/Step Time": 5.4286827482283115} +{"Finetune/Learning Rate": 1.970481312972001e-05, "Finetune/Loss": 0.9798226356506348, "Finetune/Loss (Raw)": 0.9431036114692688, "Finetune/Step": 546, "Finetune/Step Time": 5.429907858371735} +{"Finetune/Learning Rate": 1.9703308812070358e-05, "Finetune/Loss": 0.9805882573127747, "Finetune/Loss (Raw)": 1.0774357318878174, "Finetune/Step": 547, "Finetune/Step Time": 5.42072986997664} +{"Finetune/Learning Rate": 1.9701800728756e-05, "Finetune/Loss": 0.9852178692817688, "Finetune/Loss (Raw)": 0.9572376608848572, "Finetune/Step": 548, "Finetune/Step Time": 5.405615584924817} +{"Finetune/Learning Rate": 1.970028888036219e-05, "Finetune/Loss": 0.9837164878845215, "Finetune/Loss (Raw)": 0.8418307304382324, "Finetune/Step": 549, "Finetune/Step Time": 5.414620706811547} +{"Finetune/Learning Rate": 1.9698773267475652e-05, "Finetune/Loss": 0.9836594462394714, "Finetune/Loss (Raw)": 1.0003092288970947, "Finetune/Step": 550, "Finetune/Step Time": 5.396820280700922} +{"Finetune/Learning Rate": 1.9697253890684556e-05, "Finetune/Loss": 0.9824707508087158, "Finetune/Loss (Raw)": 0.9836201071739197, "Finetune/Step": 551, "Finetune/Step Time": 5.398024572059512} +{"Finetune/Learning Rate": 1.969573075057855e-05, "Finetune/Loss": 0.9828921556472778, "Finetune/Loss (Raw)": 1.095353126525879, "Finetune/Step": 552, "Finetune/Step Time": 5.393290420994163} +{"Finetune/Learning Rate": 1.9694203847748734e-05, "Finetune/Loss": 0.9828570485115051, "Finetune/Loss (Raw)": 0.7638501524925232, "Finetune/Step": 553, "Finetune/Step Time": 5.357126299291849} +{"Finetune/Learning Rate": 1.969267318278767e-05, "Finetune/Loss": 0.9837871193885803, "Finetune/Loss (Raw)": 1.1389920711517334, "Finetune/Step": 554, "Finetune/Step Time": 5.354645622894168} +{"Finetune/Learning Rate": 1.969113875628937e-05, "Finetune/Loss": 0.9835413098335266, "Finetune/Loss (Raw)": 0.950939953327179, "Finetune/Step": 555, "Finetune/Step Time": 5.368864884600043} +{"Finetune/Learning Rate": 1.9689600568849328e-05, "Finetune/Loss": 0.9826993346214294, "Finetune/Loss (Raw)": 1.0180844068527222, "Finetune/Step": 556, "Finetune/Step Time": 5.3808369897305965} +{"Finetune/Learning Rate": 1.968805862106448e-05, "Finetune/Loss": 0.9817614555358887, "Finetune/Loss (Raw)": 0.9573763608932495, "Finetune/Step": 557, "Finetune/Step Time": 5.377170344814658} +{"Finetune/Learning Rate": 1.968651291353322e-05, "Finetune/Loss": 0.9766842126846313, "Finetune/Loss (Raw)": 0.2538772523403168, "Finetune/Step": 558, "Finetune/Step Time": 5.396090060472488} +{"Finetune/Learning Rate": 1.968496344685541e-05, "Finetune/Loss": 0.975772500038147, "Finetune/Loss (Raw)": 0.9741920232772827, "Finetune/Step": 559, "Finetune/Step Time": 5.389060594141483} +{"Finetune/Learning Rate": 1.9683410221632374e-05, "Finetune/Loss": 0.9749904870986938, "Finetune/Loss (Raw)": 0.8446539640426636, "Finetune/Step": 560, "Finetune/Step Time": 5.384611079469323} +{"Finetune/Learning Rate": 1.9681853238466883e-05, "Finetune/Loss": 0.974754810333252, "Finetune/Loss (Raw)": 0.887873113155365, "Finetune/Step": 561, "Finetune/Step Time": 5.390792414546013} +{"Finetune/Learning Rate": 1.9680292497963172e-05, "Finetune/Loss": 0.9744409322738647, "Finetune/Loss (Raw)": 0.9445638656616211, "Finetune/Step": 562, "Finetune/Step Time": 5.394002135843039} +{"Finetune/Learning Rate": 1.9678728000726936e-05, "Finetune/Loss": 0.9760759472846985, "Finetune/Loss (Raw)": 1.0489954948425293, "Finetune/Step": 563, "Finetune/Step Time": 5.394494483247399} +{"Finetune/Learning Rate": 1.967715974736532e-05, "Finetune/Loss": 0.9769337773323059, "Finetune/Loss (Raw)": 1.033436894416809, "Finetune/Step": 564, "Finetune/Step Time": 5.404407745227218} +{"Finetune/Learning Rate": 1.9675587738486935e-05, "Finetune/Loss": 0.9763472676277161, "Finetune/Loss (Raw)": 0.9546762108802795, "Finetune/Step": 565, "Finetune/Step Time": 5.403465421870351} +{"Finetune/Learning Rate": 1.9674011974701855e-05, "Finetune/Loss": 0.9771849513053894, "Finetune/Loss (Raw)": 1.0803455114364624, "Finetune/Step": 566, "Finetune/Step Time": 5.405061276629567} +{"Finetune/Learning Rate": 1.9672432456621597e-05, "Finetune/Loss": 0.9773728251457214, "Finetune/Loss (Raw)": 1.0052183866500854, "Finetune/Step": 567, "Finetune/Step Time": 5.407802859321237} +{"Finetune/Learning Rate": 1.9670849184859136e-05, "Finetune/Loss": 0.9711602926254272, "Finetune/Loss (Raw)": 0.289016455411911, "Finetune/Step": 568, "Finetune/Step Time": 5.429147968068719} +{"Finetune/Learning Rate": 1.9669262160028916e-05, "Finetune/Loss": 0.9701632857322693, "Finetune/Loss (Raw)": 0.9308401942253113, "Finetune/Step": 569, "Finetune/Step Time": 5.434699764475226} +{"Finetune/Learning Rate": 1.966767138274683e-05, "Finetune/Loss": 0.974661111831665, "Finetune/Loss (Raw)": 1.0171854496002197, "Finetune/Step": 570, "Finetune/Step Time": 5.413197426125407} +{"Finetune/Learning Rate": 1.966607685363022e-05, "Finetune/Loss": 0.9736930131912231, "Finetune/Loss (Raw)": 0.9542993903160095, "Finetune/Step": 571, "Finetune/Step Time": 5.419311540201306} +{"Finetune/Learning Rate": 1.96644785732979e-05, "Finetune/Loss": 0.9729066491127014, "Finetune/Loss (Raw)": 1.0411574840545654, "Finetune/Step": 572, "Finetune/Step Time": 5.41824908927083} +{"Finetune/Learning Rate": 1.966287654237013e-05, "Finetune/Loss": 0.9733628034591675, "Finetune/Loss (Raw)": 1.0739384889602661, "Finetune/Step": 573, "Finetune/Step Time": 5.416628964245319} +{"Finetune/Learning Rate": 1.9661270761468625e-05, "Finetune/Loss": 0.9732124209403992, "Finetune/Loss (Raw)": 1.1022604703903198, "Finetune/Step": 574, "Finetune/Step Time": 5.408916983753443} +{"Finetune/Learning Rate": 1.965966123121656e-05, "Finetune/Loss": 0.9739720225334167, "Finetune/Loss (Raw)": 1.1358839273452759, "Finetune/Step": 575, "Finetune/Step Time": 5.4315652176737785} +{"Finetune/Learning Rate": 1.965804795223856e-05, "Finetune/Loss": 0.9742000102996826, "Finetune/Loss (Raw)": 0.9905664920806885, "Finetune/Step": 576, "Finetune/Step Time": 5.438236050307751} +{"Finetune/Learning Rate": 1.9656430925160704e-05, "Finetune/Loss": 0.9730368852615356, "Finetune/Loss (Raw)": 0.9900019764900208, "Finetune/Step": 577, "Finetune/Step Time": 5.4340431075543165} +{"Finetune/Learning Rate": 1.9654810150610536e-05, "Finetune/Loss": 0.9731472730636597, "Finetune/Loss (Raw)": 0.9581763744354248, "Finetune/Step": 578, "Finetune/Step Time": 5.429980784654617} +{"Finetune/Learning Rate": 1.9653185629217038e-05, "Finetune/Loss": 0.9721871018409729, "Finetune/Loss (Raw)": 0.8539533019065857, "Finetune/Step": 579, "Finetune/Step Time": 5.4427693877369165} +{"Finetune/Learning Rate": 1.9651557361610663e-05, "Finetune/Loss": 0.9724541902542114, "Finetune/Loss (Raw)": 1.0413455963134766, "Finetune/Step": 580, "Finetune/Step Time": 5.450082121416926} +{"Finetune/Learning Rate": 1.9649925348423308e-05, "Finetune/Loss": 0.9717553853988647, "Finetune/Loss (Raw)": 1.1378575563430786, "Finetune/Step": 581, "Finetune/Step Time": 5.460563069209456} +{"Finetune/Learning Rate": 1.964828959028832e-05, "Finetune/Loss": 0.9721802473068237, "Finetune/Loss (Raw)": 1.0472321510314941, "Finetune/Step": 582, "Finetune/Step Time": 5.455435568466783} +{"Finetune/Learning Rate": 1.964665008784051e-05, "Finetune/Loss": 0.9706659317016602, "Finetune/Loss (Raw)": 0.8988506197929382, "Finetune/Step": 583, "Finetune/Step Time": 5.4616944044828415} +{"Finetune/Learning Rate": 1.9645006841716137e-05, "Finetune/Loss": 0.9764488935470581, "Finetune/Loss (Raw)": 1.0843721628189087, "Finetune/Step": 584, "Finetune/Step Time": 5.4411491341888905} +{"Finetune/Learning Rate": 1.9643359852552912e-05, "Finetune/Loss": 0.9777066111564636, "Finetune/Loss (Raw)": 1.0817797183990479, "Finetune/Step": 585, "Finetune/Step Time": 5.442904099822044} +{"Finetune/Learning Rate": 1.9641709120989996e-05, "Finetune/Loss": 0.9709054231643677, "Finetune/Loss (Raw)": 0.3482094705104828, "Finetune/Step": 586, "Finetune/Step Time": 5.4693668484687805} +{"Finetune/Learning Rate": 1.9640054647668015e-05, "Finetune/Loss": 0.9724332690238953, "Finetune/Loss (Raw)": 1.191548466682434, "Finetune/Step": 587, "Finetune/Step Time": 5.460498459637165} +{"Finetune/Learning Rate": 1.9638396433229025e-05, "Finetune/Loss": 0.9736285209655762, "Finetune/Loss (Raw)": 1.0040476322174072, "Finetune/Step": 588, "Finetune/Step Time": 5.469308955594897} +{"Finetune/Learning Rate": 1.963673447831656e-05, "Finetune/Loss": 0.9743924140930176, "Finetune/Loss (Raw)": 1.0843510627746582, "Finetune/Step": 589, "Finetune/Step Time": 5.464241473004222} +{"Finetune/Learning Rate": 1.9635068783575577e-05, "Finetune/Loss": 0.9751659631729126, "Finetune/Loss (Raw)": 1.0906755924224854, "Finetune/Step": 590, "Finetune/Step Time": 5.465374266728759} +{"Finetune/Learning Rate": 1.9633399349652512e-05, "Finetune/Loss": 0.9731959104537964, "Finetune/Loss (Raw)": 0.9465936422348022, "Finetune/Step": 591, "Finetune/Step Time": 5.467429868876934} +{"Finetune/Learning Rate": 1.9631726177195235e-05, "Finetune/Loss": 0.9785578846931458, "Finetune/Loss (Raw)": 0.9775714874267578, "Finetune/Step": 592, "Finetune/Step Time": 5.435246137902141} +{"Finetune/Learning Rate": 1.9630049266853076e-05, "Finetune/Loss": 0.9782049655914307, "Finetune/Loss (Raw)": 1.001977562904358, "Finetune/Step": 593, "Finetune/Step Time": 5.4505032338202} +{"Finetune/Learning Rate": 1.96283686192768e-05, "Finetune/Loss": 0.9786936044692993, "Finetune/Loss (Raw)": 1.0123862028121948, "Finetune/Step": 594, "Finetune/Step Time": 5.437917245551944} +{"Finetune/Learning Rate": 1.9626684235118646e-05, "Finetune/Loss": 0.9781291484832764, "Finetune/Loss (Raw)": 1.012758493423462, "Finetune/Step": 595, "Finetune/Step Time": 5.428049759939313} +{"Finetune/Learning Rate": 1.9624996115032287e-05, "Finetune/Loss": 0.978887677192688, "Finetune/Loss (Raw)": 0.9323147535324097, "Finetune/Step": 596, "Finetune/Step Time": 5.410874145105481} +{"Finetune/Learning Rate": 1.9623304259672844e-05, "Finetune/Loss": 0.9829940795898438, "Finetune/Loss (Raw)": 0.8802657127380371, "Finetune/Step": 597, "Finetune/Step Time": 5.3826000690460205} +{"Finetune/Learning Rate": 1.96216086696969e-05, "Finetune/Loss": 0.9831576347351074, "Finetune/Loss (Raw)": 1.0070191621780396, "Finetune/Step": 598, "Finetune/Step Time": 5.405543265864253} +{"Finetune/Learning Rate": 1.9619909345762474e-05, "Finetune/Loss": 0.9832092523574829, "Finetune/Loss (Raw)": 1.0187323093414307, "Finetune/Step": 599, "Finetune/Step Time": 5.387423489242792} +{"Finetune/Learning Rate": 1.961820628852905e-05, "Finetune/Loss": 0.9829810857772827, "Finetune/Loss (Raw)": 1.1092790365219116, "Finetune/Step": 600, "Finetune/Step Time": 5.392320428043604} +{"Finetune/Learning Rate": 1.9616499498657537e-05, "Finetune/Loss": 0.9834367632865906, "Finetune/Loss (Raw)": 0.983390748500824, "Finetune/Step": 601, "Finetune/Step Time": 5.416048306971788} +{"Finetune/Learning Rate": 1.961478897681032e-05, "Finetune/Loss": 0.9834900498390198, "Finetune/Loss (Raw)": 1.0642896890640259, "Finetune/Step": 602, "Finetune/Step Time": 5.4205383490771055} +{"Finetune/Learning Rate": 1.9613074723651214e-05, "Finetune/Loss": 0.9824486970901489, "Finetune/Loss (Raw)": 0.9378255009651184, "Finetune/Step": 603, "Finetune/Step Time": 5.425728313624859} +{"Finetune/Learning Rate": 1.9611356739845484e-05, "Finetune/Loss": 0.9827489852905273, "Finetune/Loss (Raw)": 1.0690155029296875, "Finetune/Step": 604, "Finetune/Step Time": 5.425556248053908} +{"Finetune/Learning Rate": 1.9609635026059855e-05, "Finetune/Loss": 0.9814891815185547, "Finetune/Loss (Raw)": 0.9477347135543823, "Finetune/Step": 605, "Finetune/Step Time": 5.420588364824653} +{"Finetune/Learning Rate": 1.9607909582962478e-05, "Finetune/Loss": 0.9811413288116455, "Finetune/Loss (Raw)": 0.9270086288452148, "Finetune/Step": 606, "Finetune/Step Time": 5.437892731279135} +{"Finetune/Learning Rate": 1.9606180411222974e-05, "Finetune/Loss": 0.9795091152191162, "Finetune/Loss (Raw)": 0.8239492177963257, "Finetune/Step": 607, "Finetune/Step Time": 5.467383010312915} +{"Finetune/Learning Rate": 1.9604447511512396e-05, "Finetune/Loss": 0.9794872999191284, "Finetune/Loss (Raw)": 1.0880523920059204, "Finetune/Step": 608, "Finetune/Step Time": 5.464497059583664} +{"Finetune/Learning Rate": 1.960271088450325e-05, "Finetune/Loss": 0.978618860244751, "Finetune/Loss (Raw)": 0.9188361167907715, "Finetune/Step": 609, "Finetune/Step Time": 5.45982495136559} +{"Finetune/Learning Rate": 1.960097053086949e-05, "Finetune/Loss": 0.9779729843139648, "Finetune/Loss (Raw)": 0.9704070687294006, "Finetune/Step": 610, "Finetune/Step Time": 5.450878402218223} +{"Finetune/Learning Rate": 1.9599226451286504e-05, "Finetune/Loss": 0.9779568910598755, "Finetune/Loss (Raw)": 1.0397075414657593, "Finetune/Step": 611, "Finetune/Step Time": 5.42865671031177} +{"Finetune/Learning Rate": 1.9597478646431147e-05, "Finetune/Loss": 0.9798104763031006, "Finetune/Loss (Raw)": 1.215639591217041, "Finetune/Step": 612, "Finetune/Step Time": 5.425813168287277} +{"Finetune/Learning Rate": 1.95957271169817e-05, "Finetune/Loss": 0.9802030324935913, "Finetune/Loss (Raw)": 0.9059450626373291, "Finetune/Step": 613, "Finetune/Step Time": 5.418363939970732} +{"Finetune/Learning Rate": 1.95939718636179e-05, "Finetune/Loss": 0.9801239967346191, "Finetune/Loss (Raw)": 0.9848566651344299, "Finetune/Step": 614, "Finetune/Step Time": 5.403568839654326} +{"Finetune/Learning Rate": 1.9592212887020923e-05, "Finetune/Loss": 0.9800437092781067, "Finetune/Loss (Raw)": 1.0330525636672974, "Finetune/Step": 615, "Finetune/Step Time": 5.410159168764949} +{"Finetune/Learning Rate": 1.9590450187873403e-05, "Finetune/Loss": 0.9796804189682007, "Finetune/Loss (Raw)": 1.0033230781555176, "Finetune/Step": 616, "Finetune/Step Time": 5.412132767960429} +{"Finetune/Learning Rate": 1.95886837668594e-05, "Finetune/Loss": 0.9800745844841003, "Finetune/Loss (Raw)": 1.0007495880126953, "Finetune/Step": 617, "Finetune/Step Time": 5.404214946553111} +{"Finetune/Learning Rate": 1.958691362466443e-05, "Finetune/Loss": 0.980823278427124, "Finetune/Loss (Raw)": 1.0440672636032104, "Finetune/Step": 618, "Finetune/Step Time": 5.3985556326806545} +{"Finetune/Learning Rate": 1.958513976197545e-05, "Finetune/Loss": 0.9827274680137634, "Finetune/Loss (Raw)": 0.9215787649154663, "Finetune/Step": 619, "Finetune/Step Time": 5.392981564626098} +{"Finetune/Learning Rate": 1.9583362179480866e-05, "Finetune/Loss": 0.9831880927085876, "Finetune/Loss (Raw)": 1.0317472219467163, "Finetune/Step": 620, "Finetune/Step Time": 5.383602412417531} +{"Finetune/Learning Rate": 1.9581580877870514e-05, "Finetune/Loss": 0.982730507850647, "Finetune/Loss (Raw)": 1.02131986618042, "Finetune/Step": 621, "Finetune/Step Time": 5.40325690060854} +{"Finetune/Learning Rate": 1.9579795857835692e-05, "Finetune/Loss": 0.9826329350471497, "Finetune/Loss (Raw)": 0.9673109650611877, "Finetune/Step": 622, "Finetune/Step Time": 5.3911522924900055} +{"Finetune/Learning Rate": 1.9578007120069123e-05, "Finetune/Loss": 0.9830021262168884, "Finetune/Loss (Raw)": 1.070716381072998, "Finetune/Step": 623, "Finetune/Step Time": 5.392266688868403} +{"Finetune/Learning Rate": 1.957621466526499e-05, "Finetune/Loss": 0.9826167821884155, "Finetune/Loss (Raw)": 0.996258020401001, "Finetune/Step": 624, "Finetune/Step Time": 5.378347611054778} +{"Finetune/Learning Rate": 1.9574418494118905e-05, "Finetune/Loss": 0.9832669496536255, "Finetune/Loss (Raw)": 1.0583654642105103, "Finetune/Step": 625, "Finetune/Step Time": 5.3792466185987} +{"Finetune/Learning Rate": 1.9572618607327928e-05, "Finetune/Loss": 0.983411431312561, "Finetune/Loss (Raw)": 1.0656051635742188, "Finetune/Step": 626, "Finetune/Step Time": 5.377290861681104} +{"Finetune/Learning Rate": 1.9570815005590556e-05, "Finetune/Loss": 0.9840874075889587, "Finetune/Loss (Raw)": 0.9808501601219177, "Finetune/Step": 627, "Finetune/Step Time": 5.379310535266995} +{"Finetune/Learning Rate": 1.9569007689606735e-05, "Finetune/Loss": 0.9840611219406128, "Finetune/Loss (Raw)": 0.9924459457397461, "Finetune/Step": 628, "Finetune/Step Time": 5.374522645026445} +{"Finetune/Learning Rate": 1.9567196660077848e-05, "Finetune/Loss": 0.9858956933021545, "Finetune/Loss (Raw)": 1.0315226316452026, "Finetune/Step": 629, "Finetune/Step Time": 5.364652328193188} +{"Finetune/Learning Rate": 1.9565381917706726e-05, "Finetune/Loss": 0.9878288507461548, "Finetune/Loss (Raw)": 1.1403391361236572, "Finetune/Step": 630, "Finetune/Step Time": 5.341388834640384} +{"Finetune/Learning Rate": 1.9563563463197625e-05, "Finetune/Loss": 0.9867907762527466, "Finetune/Loss (Raw)": 0.9089280962944031, "Finetune/Step": 631, "Finetune/Step Time": 5.322155207395554} +{"Finetune/Learning Rate": 1.9561741297256257e-05, "Finetune/Loss": 0.9875848293304443, "Finetune/Loss (Raw)": 1.1721266508102417, "Finetune/Step": 632, "Finetune/Step Time": 5.326401825994253} +{"Finetune/Learning Rate": 1.955991542058977e-05, "Finetune/Loss": 0.9895127415657043, "Finetune/Loss (Raw)": 1.2608760595321655, "Finetune/Step": 633, "Finetune/Step Time": 5.327611204236746} +{"Finetune/Learning Rate": 1.9558085833906753e-05, "Finetune/Loss": 0.9904360175132751, "Finetune/Loss (Raw)": 0.9506599307060242, "Finetune/Step": 634, "Finetune/Step Time": 5.318284904584289} +{"Finetune/Learning Rate": 1.9556252537917228e-05, "Finetune/Loss": 0.9903085231781006, "Finetune/Loss (Raw)": 1.0036544799804688, "Finetune/Step": 635, "Finetune/Step Time": 5.31638615205884} +{"Finetune/Learning Rate": 1.955441553333266e-05, "Finetune/Loss": 0.9909411072731018, "Finetune/Loss (Raw)": 1.07814621925354, "Finetune/Step": 636, "Finetune/Step Time": 5.341668773442507} +{"Finetune/Learning Rate": 1.955257482086596e-05, "Finetune/Loss": 0.9918904900550842, "Finetune/Loss (Raw)": 1.1007322072982788, "Finetune/Step": 637, "Finetune/Step Time": 5.337279541417956} +{"Finetune/Learning Rate": 1.955073040123147e-05, "Finetune/Loss": 0.9932126998901367, "Finetune/Loss (Raw)": 1.1002782583236694, "Finetune/Step": 638, "Finetune/Step Time": 5.331675758585334} +{"Finetune/Learning Rate": 1.9548882275144974e-05, "Finetune/Loss": 0.9925476312637329, "Finetune/Loss (Raw)": 1.0210084915161133, "Finetune/Step": 639, "Finetune/Step Time": 5.328850517049432} +{"Finetune/Learning Rate": 1.9547030443323695e-05, "Finetune/Loss": 0.9923051595687866, "Finetune/Loss (Raw)": 1.0033245086669922, "Finetune/Step": 640, "Finetune/Step Time": 5.323503317311406} +{"Finetune/Learning Rate": 1.9545174906486287e-05, "Finetune/Loss": 0.9925694465637207, "Finetune/Loss (Raw)": 1.1000772714614868, "Finetune/Step": 641, "Finetune/Step Time": 5.330657582730055} +{"Finetune/Learning Rate": 1.9543315665352855e-05, "Finetune/Loss": 0.9866033792495728, "Finetune/Loss (Raw)": 0.33758729696273804, "Finetune/Step": 642, "Finetune/Step Time": 5.33976636081934} +{"Finetune/Learning Rate": 1.954145272064493e-05, "Finetune/Loss": 0.9912884831428528, "Finetune/Loss (Raw)": 1.0018510818481445, "Finetune/Step": 643, "Finetune/Step Time": 5.319755597040057} +{"Finetune/Learning Rate": 1.9539586073085482e-05, "Finetune/Loss": 0.991024911403656, "Finetune/Loss (Raw)": 1.0542922019958496, "Finetune/Step": 644, "Finetune/Step Time": 5.324602670967579} +{"Finetune/Learning Rate": 1.9537715723398926e-05, "Finetune/Loss": 0.9913166761398315, "Finetune/Loss (Raw)": 1.0830955505371094, "Finetune/Step": 645, "Finetune/Step Time": 5.322956454008818} +{"Finetune/Learning Rate": 1.9535841672311106e-05, "Finetune/Loss": 0.9909400939941406, "Finetune/Loss (Raw)": 1.0184404850006104, "Finetune/Step": 646, "Finetune/Step Time": 5.327645534649491} +{"Finetune/Learning Rate": 1.9533963920549307e-05, "Finetune/Loss": 0.9900193214416504, "Finetune/Loss (Raw)": 0.9520989060401917, "Finetune/Step": 647, "Finetune/Step Time": 5.328241815790534} +{"Finetune/Learning Rate": 1.9532082468842243e-05, "Finetune/Loss": 0.9901524782180786, "Finetune/Loss (Raw)": 1.0331571102142334, "Finetune/Step": 648, "Finetune/Step Time": 5.326642710715532} +{"Finetune/Learning Rate": 1.953019731792007e-05, "Finetune/Loss": 0.9903250336647034, "Finetune/Loss (Raw)": 1.033522367477417, "Finetune/Step": 649, "Finetune/Step Time": 5.327898466959596} +{"Finetune/Learning Rate": 1.9528308468514383e-05, "Finetune/Loss": 0.9902467727661133, "Finetune/Loss (Raw)": 0.9916276931762695, "Finetune/Step": 650, "Finetune/Step Time": 5.3409564681351185} +{"Finetune/Learning Rate": 1.9526415921358202e-05, "Finetune/Loss": 0.9911848306655884, "Finetune/Loss (Raw)": 1.0256011486053467, "Finetune/Step": 651, "Finetune/Step Time": 5.354070657864213} +{"Finetune/Learning Rate": 1.952451967718599e-05, "Finetune/Loss": 0.9910832643508911, "Finetune/Loss (Raw)": 1.005852222442627, "Finetune/Step": 652, "Finetune/Step Time": 5.358501970767975} +{"Finetune/Learning Rate": 1.952261973673364e-05, "Finetune/Loss": 0.9902514219284058, "Finetune/Loss (Raw)": 0.9185947179794312, "Finetune/Step": 653, "Finetune/Step Time": 5.360895508900285} +{"Finetune/Learning Rate": 1.9520716100738487e-05, "Finetune/Loss": 0.9924284219741821, "Finetune/Loss (Raw)": 1.2138630151748657, "Finetune/Step": 654, "Finetune/Step Time": 5.352240709587932} +{"Finetune/Learning Rate": 1.9518808769939287e-05, "Finetune/Loss": 0.9925566911697388, "Finetune/Loss (Raw)": 0.9845088124275208, "Finetune/Step": 655, "Finetune/Step Time": 5.36160696297884} +{"Finetune/Learning Rate": 1.9516897745076243e-05, "Finetune/Loss": 0.99196457862854, "Finetune/Loss (Raw)": 0.9714926481246948, "Finetune/Step": 656, "Finetune/Step Time": 5.36475789360702} +{"Finetune/Learning Rate": 1.9514983026890988e-05, "Finetune/Loss": 0.9909034371376038, "Finetune/Loss (Raw)": 0.9624165892601013, "Finetune/Step": 657, "Finetune/Step Time": 5.385686589404941} +{"Finetune/Learning Rate": 1.9513064616126588e-05, "Finetune/Loss": 0.989993691444397, "Finetune/Loss (Raw)": 0.984774649143219, "Finetune/Step": 658, "Finetune/Step Time": 5.387186931446195} +{"Finetune/Learning Rate": 1.951114251352753e-05, "Finetune/Loss": 0.9872218370437622, "Finetune/Loss (Raw)": 0.8086425065994263, "Finetune/Step": 659, "Finetune/Step Time": 5.409503139555454} +{"Finetune/Learning Rate": 1.950921671983976e-05, "Finetune/Loss": 0.9883711338043213, "Finetune/Loss (Raw)": 1.1341173648834229, "Finetune/Step": 660, "Finetune/Step Time": 5.414052858948708} +{"Finetune/Learning Rate": 1.9507287235810625e-05, "Finetune/Loss": 0.9889273643493652, "Finetune/Loss (Raw)": 1.0198091268539429, "Finetune/Step": 661, "Finetune/Step Time": 5.406510891392827} +{"Finetune/Learning Rate": 1.9505354062188934e-05, "Finetune/Loss": 0.9898484945297241, "Finetune/Loss (Raw)": 1.0626938343048096, "Finetune/Step": 662, "Finetune/Step Time": 5.395597403869033} +{"Finetune/Learning Rate": 1.9503417199724904e-05, "Finetune/Loss": 0.9912480115890503, "Finetune/Loss (Raw)": 1.1116348505020142, "Finetune/Step": 663, "Finetune/Step Time": 5.384725345298648} +{"Finetune/Learning Rate": 1.95014766491702e-05, "Finetune/Loss": 0.9928194284439087, "Finetune/Loss (Raw)": 1.0258312225341797, "Finetune/Step": 664, "Finetune/Step Time": 5.3721649665385485} +{"Finetune/Learning Rate": 1.9499532411277907e-05, "Finetune/Loss": 0.9934418797492981, "Finetune/Loss (Raw)": 1.1468167304992676, "Finetune/Step": 665, "Finetune/Step Time": 5.380043717101216} +{"Finetune/Learning Rate": 1.9497584486802548e-05, "Finetune/Loss": 0.9945977926254272, "Finetune/Loss (Raw)": 1.169826865196228, "Finetune/Step": 666, "Finetune/Step Time": 5.3723493330180645} +{"Finetune/Learning Rate": 1.9495632876500076e-05, "Finetune/Loss": 0.9947460889816284, "Finetune/Loss (Raw)": 1.1209503412246704, "Finetune/Step": 667, "Finetune/Step Time": 5.385915085673332} +{"Finetune/Learning Rate": 1.949367758112787e-05, "Finetune/Loss": 0.9938839673995972, "Finetune/Loss (Raw)": 0.8657891750335693, "Finetune/Step": 668, "Finetune/Step Time": 5.391071103513241} +{"Finetune/Learning Rate": 1.949171860144474e-05, "Finetune/Loss": 0.9941056370735168, "Finetune/Loss (Raw)": 1.0388857126235962, "Finetune/Step": 669, "Finetune/Step Time": 5.385369123890996} +{"Finetune/Learning Rate": 1.9489755938210936e-05, "Finetune/Loss": 0.9932870864868164, "Finetune/Loss (Raw)": 0.9213862419128418, "Finetune/Step": 670, "Finetune/Step Time": 5.3877076506614685} +{"Finetune/Learning Rate": 1.9487789592188123e-05, "Finetune/Loss": 0.9922596216201782, "Finetune/Loss (Raw)": 0.9995776414871216, "Finetune/Step": 671, "Finetune/Step Time": 5.386616267263889} +{"Finetune/Learning Rate": 1.9485819564139404e-05, "Finetune/Loss": 0.9921278357505798, "Finetune/Loss (Raw)": 1.065889596939087, "Finetune/Step": 672, "Finetune/Step Time": 5.383408093824983} +{"Finetune/Learning Rate": 1.9483845854829304e-05, "Finetune/Loss": 0.9931477308273315, "Finetune/Loss (Raw)": 1.1201380491256714, "Finetune/Step": 673, "Finetune/Step Time": 5.383939642459154} +{"Finetune/Learning Rate": 1.9481868465023788e-05, "Finetune/Loss": 0.994540810585022, "Finetune/Loss (Raw)": 1.1214240789413452, "Finetune/Step": 674, "Finetune/Step Time": 5.384624008089304} +{"Finetune/Learning Rate": 1.9479887395490234e-05, "Finetune/Loss": 0.9944663047790527, "Finetune/Loss (Raw)": 1.067898154258728, "Finetune/Step": 675, "Finetune/Step Time": 5.38656179793179} +{"Finetune/Learning Rate": 1.9477902646997464e-05, "Finetune/Loss": 0.9950981736183167, "Finetune/Loss (Raw)": 1.0381158590316772, "Finetune/Step": 676, "Finetune/Step Time": 5.380404828116298} +{"Finetune/Learning Rate": 1.947591422031572e-05, "Finetune/Loss": 0.9957793951034546, "Finetune/Loss (Raw)": 0.9290226697921753, "Finetune/Step": 677, "Finetune/Step Time": 5.412839310243726} +{"Finetune/Learning Rate": 1.9473922116216664e-05, "Finetune/Loss": 0.9968534111976624, "Finetune/Loss (Raw)": 1.1377894878387451, "Finetune/Step": 678, "Finetune/Step Time": 5.42454307153821} +{"Finetune/Learning Rate": 1.9471926335473396e-05, "Finetune/Loss": 0.9981650710105896, "Finetune/Loss (Raw)": 1.1515151262283325, "Finetune/Step": 679, "Finetune/Step Time": 5.429892597720027} +{"Finetune/Learning Rate": 1.9469926878860447e-05, "Finetune/Loss": 0.9971500039100647, "Finetune/Loss (Raw)": 0.9654184579849243, "Finetune/Step": 680, "Finetune/Step Time": 5.428847389295697} +{"Finetune/Learning Rate": 1.9467923747153756e-05, "Finetune/Loss": 0.9995514750480652, "Finetune/Loss (Raw)": 1.0712393522262573, "Finetune/Step": 681, "Finetune/Step Time": 5.426951818168163} +{"Finetune/Learning Rate": 1.9465916941130705e-05, "Finetune/Loss": 0.999110221862793, "Finetune/Loss (Raw)": 1.082503318786621, "Finetune/Step": 682, "Finetune/Step Time": 5.423833150416613} +{"Finetune/Learning Rate": 1.9463906461570098e-05, "Finetune/Loss": 1.0003567934036255, "Finetune/Loss (Raw)": 1.1105108261108398, "Finetune/Step": 683, "Finetune/Step Time": 5.404438301920891} +{"Finetune/Learning Rate": 1.9461892309252158e-05, "Finetune/Loss": 0.9999299049377441, "Finetune/Loss (Raw)": 0.9634470343589783, "Finetune/Step": 684, "Finetune/Step Time": 5.3935206308960915} +{"Finetune/Learning Rate": 1.945987448495854e-05, "Finetune/Loss": 0.9984444379806519, "Finetune/Loss (Raw)": 0.7672289609909058, "Finetune/Step": 685, "Finetune/Step Time": 5.4090834595263} +{"Finetune/Learning Rate": 1.945785298947233e-05, "Finetune/Loss": 1.0037915706634521, "Finetune/Loss (Raw)": 0.9383217692375183, "Finetune/Step": 686, "Finetune/Step Time": 5.3917016461491585} +{"Finetune/Learning Rate": 1.9455827823578017e-05, "Finetune/Loss": 1.0037853717803955, "Finetune/Loss (Raw)": 0.9733876585960388, "Finetune/Step": 687, "Finetune/Step Time": 5.38902840949595} +{"Finetune/Learning Rate": 1.9453798988061535e-05, "Finetune/Loss": 1.0051324367523193, "Finetune/Loss (Raw)": 1.017075777053833, "Finetune/Step": 688, "Finetune/Step Time": 5.387454219162464} +{"Finetune/Learning Rate": 1.9451766483710236e-05, "Finetune/Loss": 1.0039222240447998, "Finetune/Loss (Raw)": 0.7329630255699158, "Finetune/Step": 689, "Finetune/Step Time": 5.395199626684189} +{"Finetune/Learning Rate": 1.94497303113129e-05, "Finetune/Loss": 1.003150463104248, "Finetune/Loss (Raw)": 0.8457889556884766, "Finetune/Step": 690, "Finetune/Step Time": 5.432698851451278} +{"Finetune/Learning Rate": 1.9447690471659715e-05, "Finetune/Loss": 1.003255844116211, "Finetune/Loss (Raw)": 1.0624784231185913, "Finetune/Step": 691, "Finetune/Step Time": 5.434461848810315} +{"Finetune/Learning Rate": 1.944564696554231e-05, "Finetune/Loss": 1.002448320388794, "Finetune/Loss (Raw)": 0.9300810694694519, "Finetune/Step": 692, "Finetune/Step Time": 5.452111883088946} +{"Finetune/Learning Rate": 1.944359979375373e-05, "Finetune/Loss": 1.0027190446853638, "Finetune/Loss (Raw)": 0.9893260598182678, "Finetune/Step": 693, "Finetune/Step Time": 5.456275073811412} +{"Finetune/Learning Rate": 1.9441548957088438e-05, "Finetune/Loss": 0.9974205493927002, "Finetune/Loss (Raw)": 0.4021354019641876, "Finetune/Step": 694, "Finetune/Step Time": 5.485586982220411} +{"Finetune/Learning Rate": 1.9439494456342327e-05, "Finetune/Loss": 0.996362566947937, "Finetune/Loss (Raw)": 0.8698021769523621, "Finetune/Step": 695, "Finetune/Step Time": 5.491596104577184} +{"Finetune/Learning Rate": 1.943743629231271e-05, "Finetune/Loss": 1.002833366394043, "Finetune/Loss (Raw)": 1.1172659397125244, "Finetune/Step": 696, "Finetune/Step Time": 5.466005556285381} +{"Finetune/Learning Rate": 1.9435374465798318e-05, "Finetune/Loss": 1.0029321908950806, "Finetune/Loss (Raw)": 0.9434981942176819, "Finetune/Step": 697, "Finetune/Step Time": 5.472681205719709} +{"Finetune/Learning Rate": 1.9433308977599306e-05, "Finetune/Loss": 1.0043222904205322, "Finetune/Loss (Raw)": 1.1951192617416382, "Finetune/Step": 698, "Finetune/Step Time": 5.460946671664715} +{"Finetune/Learning Rate": 1.943123982851725e-05, "Finetune/Loss": 1.0062897205352783, "Finetune/Loss (Raw)": 1.206124186515808, "Finetune/Step": 699, "Finetune/Step Time": 5.457766907289624} +{"Finetune/Learning Rate": 1.9429167019355143e-05, "Finetune/Loss": 1.0059261322021484, "Finetune/Loss (Raw)": 0.9946330785751343, "Finetune/Step": 700, "Finetune/Step Time": 5.477456225082278} +{"Finetune/Learning Rate": 1.9427090550917407e-05, "Finetune/Loss": 0.9999833106994629, "Finetune/Loss (Raw)": 0.3132472634315491, "Finetune/Step": 701, "Finetune/Step Time": 5.5152864791452885} +{"Finetune/Learning Rate": 1.9425010424009876e-05, "Finetune/Loss": 0.9990497827529907, "Finetune/Loss (Raw)": 0.9827738404273987, "Finetune/Step": 702, "Finetune/Step Time": 5.521874206140637} +{"Finetune/Learning Rate": 1.9422926639439806e-05, "Finetune/Loss": 0.9970524311065674, "Finetune/Loss (Raw)": 0.8802216053009033, "Finetune/Step": 703, "Finetune/Step Time": 5.512935230508447} +{"Finetune/Learning Rate": 1.9420839198015878e-05, "Finetune/Loss": 0.9971148371696472, "Finetune/Loss (Raw)": 0.998549222946167, "Finetune/Step": 704, "Finetune/Step Time": 5.514387305825949} +{"Finetune/Learning Rate": 1.941874810054818e-05, "Finetune/Loss": 0.9973074197769165, "Finetune/Loss (Raw)": 1.014658808708191, "Finetune/Step": 705, "Finetune/Step Time": 5.519387597218156} +{"Finetune/Learning Rate": 1.941665334784823e-05, "Finetune/Loss": 0.9959948062896729, "Finetune/Loss (Raw)": 0.7901569604873657, "Finetune/Step": 706, "Finetune/Step Time": 5.522014839574695} +{"Finetune/Learning Rate": 1.9414554940728963e-05, "Finetune/Loss": 0.9964239597320557, "Finetune/Loss (Raw)": 0.9088848829269409, "Finetune/Step": 707, "Finetune/Step Time": 5.507152196019888} +{"Finetune/Learning Rate": 1.9412452880004728e-05, "Finetune/Loss": 0.9973894357681274, "Finetune/Loss (Raw)": 1.1649225950241089, "Finetune/Step": 708, "Finetune/Step Time": 5.511610699817538} +{"Finetune/Learning Rate": 1.941034716649129e-05, "Finetune/Loss": 0.9960837364196777, "Finetune/Loss (Raw)": 0.9707304835319519, "Finetune/Step": 709, "Finetune/Step Time": 5.5067139863967896} +{"Finetune/Learning Rate": 1.9408237801005837e-05, "Finetune/Loss": 0.9948145151138306, "Finetune/Loss (Raw)": 0.8847770094871521, "Finetune/Step": 710, "Finetune/Step Time": 5.501360146328807} +{"Finetune/Learning Rate": 1.9406124784366976e-05, "Finetune/Loss": 0.9962102174758911, "Finetune/Loss (Raw)": 1.07749342918396, "Finetune/Step": 711, "Finetune/Step Time": 5.514579392969608} +{"Finetune/Learning Rate": 1.9404008117394726e-05, "Finetune/Loss": 0.9934619665145874, "Finetune/Loss (Raw)": 0.7326021790504456, "Finetune/Step": 712, "Finetune/Step Time": 5.530496114864945} +{"Finetune/Learning Rate": 1.9401887800910525e-05, "Finetune/Loss": 0.9926601648330688, "Finetune/Loss (Raw)": 0.9791526198387146, "Finetune/Step": 713, "Finetune/Step Time": 5.531144913285971} +{"Finetune/Learning Rate": 1.939976383573722e-05, "Finetune/Loss": 0.9972589015960693, "Finetune/Loss (Raw)": 0.93683922290802, "Finetune/Step": 714, "Finetune/Step Time": 5.511170350015163} +{"Finetune/Learning Rate": 1.9397636222699088e-05, "Finetune/Loss": 0.9967231750488281, "Finetune/Loss (Raw)": 1.1229780912399292, "Finetune/Step": 715, "Finetune/Step Time": 5.5103980135172606} +{"Finetune/Learning Rate": 1.939550496262181e-05, "Finetune/Loss": 0.9956620931625366, "Finetune/Loss (Raw)": 0.8682321906089783, "Finetune/Step": 716, "Finetune/Step Time": 5.484419064596295} +{"Finetune/Learning Rate": 1.939337005633249e-05, "Finetune/Loss": 0.9960874319076538, "Finetune/Loss (Raw)": 1.1387895345687866, "Finetune/Step": 717, "Finetune/Step Time": 5.4965391512960196} +{"Finetune/Learning Rate": 1.939123150465964e-05, "Finetune/Loss": 0.9958058595657349, "Finetune/Loss (Raw)": 1.0546305179595947, "Finetune/Step": 718, "Finetune/Step Time": 5.500649850815535} +{"Finetune/Learning Rate": 1.938908930843319e-05, "Finetune/Loss": 0.9965465068817139, "Finetune/Loss (Raw)": 1.0413991212844849, "Finetune/Step": 719, "Finetune/Step Time": 5.501138988882303} +{"Finetune/Learning Rate": 1.9386943468484487e-05, "Finetune/Loss": 0.998401939868927, "Finetune/Loss (Raw)": 1.2150661945343018, "Finetune/Step": 720, "Finetune/Step Time": 5.503247892484069} +{"Finetune/Learning Rate": 1.938479398564629e-05, "Finetune/Loss": 0.9978669881820679, "Finetune/Loss (Raw)": 0.9335042238235474, "Finetune/Step": 721, "Finetune/Step Time": 5.507000111043453} +{"Finetune/Learning Rate": 1.938264086075277e-05, "Finetune/Loss": 0.997775673866272, "Finetune/Loss (Raw)": 1.0006998777389526, "Finetune/Step": 722, "Finetune/Step Time": 5.533564656972885} +{"Finetune/Learning Rate": 1.938048409463951e-05, "Finetune/Loss": 0.9985907077789307, "Finetune/Loss (Raw)": 1.117089867591858, "Finetune/Step": 723, "Finetune/Step Time": 5.530565060675144} +{"Finetune/Learning Rate": 1.9378323688143515e-05, "Finetune/Loss": 0.9997521042823792, "Finetune/Loss (Raw)": 1.080962896347046, "Finetune/Step": 724, "Finetune/Step Time": 5.545819379389286} +{"Finetune/Learning Rate": 1.937615964210319e-05, "Finetune/Loss": 1.0017645359039307, "Finetune/Loss (Raw)": 1.1378459930419922, "Finetune/Step": 725, "Finetune/Step Time": 5.544757900759578} +{"Finetune/Learning Rate": 1.9373991957358362e-05, "Finetune/Loss": 0.9998491406440735, "Finetune/Loss (Raw)": 0.7618641257286072, "Finetune/Step": 726, "Finetune/Step Time": 5.5327138137072325} +{"Finetune/Learning Rate": 1.9371820634750266e-05, "Finetune/Loss": 0.9999504685401917, "Finetune/Loss (Raw)": 1.0317035913467407, "Finetune/Step": 727, "Finetune/Step Time": 5.5393538773059845} +{"Finetune/Learning Rate": 1.9369645675121554e-05, "Finetune/Loss": 1.000641107559204, "Finetune/Loss (Raw)": 1.1976816654205322, "Finetune/Step": 728, "Finetune/Step Time": 5.538899851962924} +{"Finetune/Learning Rate": 1.936746707931628e-05, "Finetune/Loss": 0.9980588555335999, "Finetune/Loss (Raw)": 0.6528623104095459, "Finetune/Step": 729, "Finetune/Step Time": 5.528117028996348} +{"Finetune/Learning Rate": 1.9365284848179917e-05, "Finetune/Loss": 0.9975879788398743, "Finetune/Loss (Raw)": 1.004018783569336, "Finetune/Step": 730, "Finetune/Step Time": 5.520346816629171} +{"Finetune/Learning Rate": 1.9363098982559346e-05, "Finetune/Loss": 0.9987983703613281, "Finetune/Loss (Raw)": 1.0927451848983765, "Finetune/Step": 731, "Finetune/Step Time": 5.5159486047923565} +{"Finetune/Learning Rate": 1.9360909483302862e-05, "Finetune/Loss": 0.9975630044937134, "Finetune/Loss (Raw)": 0.9108969569206238, "Finetune/Step": 732, "Finetune/Step Time": 5.5479566603899} +{"Finetune/Learning Rate": 1.9358716351260164e-05, "Finetune/Loss": 0.9976381659507751, "Finetune/Loss (Raw)": 0.9573534727096558, "Finetune/Step": 733, "Finetune/Step Time": 5.545596798881888} +{"Finetune/Learning Rate": 1.935651958728236e-05, "Finetune/Loss": 0.9972137808799744, "Finetune/Loss (Raw)": 0.8726890087127686, "Finetune/Step": 734, "Finetune/Step Time": 5.533076407387853} +{"Finetune/Learning Rate": 1.9354319192221975e-05, "Finetune/Loss": 0.9978352785110474, "Finetune/Loss (Raw)": 0.903501033782959, "Finetune/Step": 735, "Finetune/Step Time": 5.510444210842252} +{"Finetune/Learning Rate": 1.9352115166932943e-05, "Finetune/Loss": 0.9966089725494385, "Finetune/Loss (Raw)": 0.9310784339904785, "Finetune/Step": 736, "Finetune/Step Time": 5.546147195622325} +{"Finetune/Learning Rate": 1.9349907512270597e-05, "Finetune/Loss": 0.9982012510299683, "Finetune/Loss (Raw)": 1.122649908065796, "Finetune/Step": 737, "Finetune/Step Time": 5.541565520688891} +{"Finetune/Learning Rate": 1.9347696229091685e-05, "Finetune/Loss": 0.9986006021499634, "Finetune/Loss (Raw)": 1.021523356437683, "Finetune/Step": 738, "Finetune/Step Time": 5.537501091137528} +{"Finetune/Learning Rate": 1.934548131825437e-05, "Finetune/Loss": 0.9940738677978516, "Finetune/Loss (Raw)": 0.46029043197631836, "Finetune/Step": 739, "Finetune/Step Time": 5.573144994676113} +{"Finetune/Learning Rate": 1.9343262780618206e-05, "Finetune/Loss": 0.9935307502746582, "Finetune/Loss (Raw)": 1.1461125612258911, "Finetune/Step": 740, "Finetune/Step Time": 5.5753678027540445} +{"Finetune/Learning Rate": 1.934104061704417e-05, "Finetune/Loss": 0.9927996397018433, "Finetune/Loss (Raw)": 0.8123726844787598, "Finetune/Step": 741, "Finetune/Step Time": 5.593470910564065} +{"Finetune/Learning Rate": 1.9338814828394638e-05, "Finetune/Loss": 0.9927855730056763, "Finetune/Loss (Raw)": 0.9830597043037415, "Finetune/Step": 742, "Finetune/Step Time": 5.598843736574054} +{"Finetune/Learning Rate": 1.9336585415533398e-05, "Finetune/Loss": 0.9930993914604187, "Finetune/Loss (Raw)": 1.0732152462005615, "Finetune/Step": 743, "Finetune/Step Time": 5.591900190338492} +{"Finetune/Learning Rate": 1.9334352379325638e-05, "Finetune/Loss": 0.9930429458618164, "Finetune/Loss (Raw)": 0.9961006045341492, "Finetune/Step": 744, "Finetune/Step Time": 5.593414871022105} +{"Finetune/Learning Rate": 1.9332115720637957e-05, "Finetune/Loss": 0.9922153949737549, "Finetune/Loss (Raw)": 0.894822895526886, "Finetune/Step": 745, "Finetune/Step Time": 5.611718498170376} +{"Finetune/Learning Rate": 1.9329875440338363e-05, "Finetune/Loss": 0.9926583170890808, "Finetune/Loss (Raw)": 1.1007657051086426, "Finetune/Step": 746, "Finetune/Step Time": 5.602203086018562} +{"Finetune/Learning Rate": 1.9327631539296258e-05, "Finetune/Loss": 0.9942089319229126, "Finetune/Loss (Raw)": 1.1200453042984009, "Finetune/Step": 747, "Finetune/Step Time": 5.581929966807365} +{"Finetune/Learning Rate": 1.932538401838246e-05, "Finetune/Loss": 0.99474036693573, "Finetune/Loss (Raw)": 1.0997809171676636, "Finetune/Step": 748, "Finetune/Step Time": 5.583659026771784} +{"Finetune/Learning Rate": 1.9323132878469185e-05, "Finetune/Loss": 0.9946444034576416, "Finetune/Loss (Raw)": 1.009034276008606, "Finetune/Step": 749, "Finetune/Step Time": 5.572400527074933} +{"Finetune/Learning Rate": 1.9320878120430063e-05, "Finetune/Loss": 0.9951450228691101, "Finetune/Loss (Raw)": 1.0313924551010132, "Finetune/Step": 750, "Finetune/Step Time": 5.577852223068476} +{"Finetune/Learning Rate": 1.9318619745140115e-05, "Finetune/Loss": 0.9937905073165894, "Finetune/Loss (Raw)": 0.8973377346992493, "Finetune/Step": 751, "Finetune/Step Time": 5.580932023003697} +{"Finetune/Learning Rate": 1.9316357753475776e-05, "Finetune/Loss": 0.9944484233856201, "Finetune/Loss (Raw)": 1.0804632902145386, "Finetune/Step": 752, "Finetune/Step Time": 5.587034609168768} +{"Finetune/Learning Rate": 1.9314092146314872e-05, "Finetune/Loss": 0.9892828464508057, "Finetune/Loss (Raw)": 0.39717549085617065, "Finetune/Step": 753, "Finetune/Step Time": 5.615864314138889} +{"Finetune/Learning Rate": 1.9311822924536656e-05, "Finetune/Loss": 0.9878145456314087, "Finetune/Loss (Raw)": 0.8776652812957764, "Finetune/Step": 754, "Finetune/Step Time": 5.615564519539475} +{"Finetune/Learning Rate": 1.930955008902176e-05, "Finetune/Loss": 0.987476110458374, "Finetune/Loss (Raw)": 0.937527060508728, "Finetune/Step": 755, "Finetune/Step Time": 5.63720634393394} +{"Finetune/Learning Rate": 1.9307273640652228e-05, "Finetune/Loss": 0.9872015714645386, "Finetune/Loss (Raw)": 0.9573088884353638, "Finetune/Step": 756, "Finetune/Step Time": 5.628720527514815} +{"Finetune/Learning Rate": 1.9304993580311505e-05, "Finetune/Loss": 0.9868754148483276, "Finetune/Loss (Raw)": 0.9897739887237549, "Finetune/Step": 757, "Finetune/Step Time": 5.6403716169297695} +{"Finetune/Learning Rate": 1.9302709908884437e-05, "Finetune/Loss": 0.9870321154594421, "Finetune/Loss (Raw)": 1.1603989601135254, "Finetune/Step": 758, "Finetune/Step Time": 5.643340500071645} +{"Finetune/Learning Rate": 1.9300422627257275e-05, "Finetune/Loss": 0.988318920135498, "Finetune/Loss (Raw)": 1.0736311674118042, "Finetune/Step": 759, "Finetune/Step Time": 5.6431990042328835} +{"Finetune/Learning Rate": 1.9298131736317666e-05, "Finetune/Loss": 0.9871007800102234, "Finetune/Loss (Raw)": 1.0162091255187988, "Finetune/Step": 760, "Finetune/Step Time": 5.630442552268505} +{"Finetune/Learning Rate": 1.929583723695466e-05, "Finetune/Loss": 0.9845659732818604, "Finetune/Loss (Raw)": 0.9364197850227356, "Finetune/Step": 761, "Finetune/Step Time": 5.633587388321757} +{"Finetune/Learning Rate": 1.9293539130058708e-05, "Finetune/Loss": 0.9801222085952759, "Finetune/Loss (Raw)": 0.3818659782409668, "Finetune/Step": 762, "Finetune/Step Time": 5.66500248759985} +{"Finetune/Learning Rate": 1.9291237416521662e-05, "Finetune/Loss": 0.9792963266372681, "Finetune/Loss (Raw)": 0.8979324698448181, "Finetune/Step": 763, "Finetune/Step Time": 5.668590785935521} +{"Finetune/Learning Rate": 1.9288932097236772e-05, "Finetune/Loss": 0.9787169694900513, "Finetune/Loss (Raw)": 1.0039983987808228, "Finetune/Step": 764, "Finetune/Step Time": 5.645514972507954} +{"Finetune/Learning Rate": 1.9286623173098683e-05, "Finetune/Loss": 0.9778342247009277, "Finetune/Loss (Raw)": 0.9877369999885559, "Finetune/Step": 765, "Finetune/Step Time": 5.643714869394898} +{"Finetune/Learning Rate": 1.928431064500345e-05, "Finetune/Loss": 0.977816641330719, "Finetune/Loss (Raw)": 1.0980281829833984, "Finetune/Step": 766, "Finetune/Step Time": 5.661764185875654} +{"Finetune/Learning Rate": 1.9281994513848515e-05, "Finetune/Loss": 0.9770535230636597, "Finetune/Loss (Raw)": 0.9233296513557434, "Finetune/Step": 767, "Finetune/Step Time": 5.670417131856084} +{"Finetune/Learning Rate": 1.9279674780532726e-05, "Finetune/Loss": 0.9764868021011353, "Finetune/Loss (Raw)": 0.930780827999115, "Finetune/Step": 768, "Finetune/Step Time": 5.685291919857264} +{"Finetune/Learning Rate": 1.9277351445956323e-05, "Finetune/Loss": 0.9743490219116211, "Finetune/Loss (Raw)": 0.8264361619949341, "Finetune/Step": 769, "Finetune/Step Time": 5.693015016615391} +{"Finetune/Learning Rate": 1.927502451102095e-05, "Finetune/Loss": 0.9798297882080078, "Finetune/Loss (Raw)": 1.0391336679458618, "Finetune/Step": 770, "Finetune/Step Time": 5.676260959357023} +{"Finetune/Learning Rate": 1.9272693976629642e-05, "Finetune/Loss": 0.9804299473762512, "Finetune/Loss (Raw)": 1.078670859336853, "Finetune/Step": 771, "Finetune/Step Time": 5.670211175456643} +{"Finetune/Learning Rate": 1.9270359843686837e-05, "Finetune/Loss": 0.9800267219543457, "Finetune/Loss (Raw)": 1.002677083015442, "Finetune/Step": 772, "Finetune/Step Time": 5.663811638951302} +{"Finetune/Learning Rate": 1.9268022113098366e-05, "Finetune/Loss": 0.9798836708068848, "Finetune/Loss (Raw)": 1.0647897720336914, "Finetune/Step": 773, "Finetune/Step Time": 5.674114223569632} +{"Finetune/Learning Rate": 1.9265680785771455e-05, "Finetune/Loss": 0.9787572622299194, "Finetune/Loss (Raw)": 0.8742567300796509, "Finetune/Step": 774, "Finetune/Step Time": 5.685689041391015} +{"Finetune/Learning Rate": 1.926333586261473e-05, "Finetune/Loss": 0.9794251918792725, "Finetune/Loss (Raw)": 1.03759765625, "Finetune/Step": 775, "Finetune/Step Time": 5.6877613719552755} +{"Finetune/Learning Rate": 1.9260987344538205e-05, "Finetune/Loss": 0.9776651263237, "Finetune/Loss (Raw)": 0.8078632354736328, "Finetune/Step": 776, "Finetune/Step Time": 5.694300681352615} +{"Finetune/Learning Rate": 1.9258635232453296e-05, "Finetune/Loss": 0.9784930944442749, "Finetune/Loss (Raw)": 1.1395046710968018, "Finetune/Step": 777, "Finetune/Step Time": 5.70552590675652} +{"Finetune/Learning Rate": 1.9256279527272815e-05, "Finetune/Loss": 0.9793045520782471, "Finetune/Loss (Raw)": 1.0954993963241577, "Finetune/Step": 778, "Finetune/Step Time": 5.690494759008288} +{"Finetune/Learning Rate": 1.9253920229910966e-05, "Finetune/Loss": 0.978617787361145, "Finetune/Loss (Raw)": 0.9376922845840454, "Finetune/Step": 779, "Finetune/Step Time": 5.68230689689517} +{"Finetune/Learning Rate": 1.925155734128334e-05, "Finetune/Loss": 0.9733948111534119, "Finetune/Loss (Raw)": 0.3373101055622101, "Finetune/Step": 780, "Finetune/Step Time": 5.712345372885466} +{"Finetune/Learning Rate": 1.9249190862306935e-05, "Finetune/Loss": 0.9735264778137207, "Finetune/Loss (Raw)": 0.9354416131973267, "Finetune/Step": 781, "Finetune/Step Time": 5.723300278186798} +{"Finetune/Learning Rate": 1.9246820793900127e-05, "Finetune/Loss": 0.9698275327682495, "Finetune/Loss (Raw)": 0.7404043674468994, "Finetune/Step": 782, "Finetune/Step Time": 5.753531463444233} +{"Finetune/Learning Rate": 1.92444471369827e-05, "Finetune/Loss": 0.9693800210952759, "Finetune/Loss (Raw)": 0.9272271394729614, "Finetune/Step": 783, "Finetune/Step Time": 5.800486860796809} +{"Finetune/Learning Rate": 1.924206989247582e-05, "Finetune/Loss": 0.967660129070282, "Finetune/Loss (Raw)": 0.7513449788093567, "Finetune/Step": 784, "Finetune/Step Time": 5.817554634064436} +{"Finetune/Learning Rate": 1.9239689061302053e-05, "Finetune/Loss": 0.9685866832733154, "Finetune/Loss (Raw)": 1.081022024154663, "Finetune/Step": 785, "Finetune/Step Time": 5.812442105263472} +{"Finetune/Learning Rate": 1.923730464438535e-05, "Finetune/Loss": 0.9697692394256592, "Finetune/Loss (Raw)": 1.1361349821090698, "Finetune/Step": 786, "Finetune/Step Time": 5.810548763722181} +{"Finetune/Learning Rate": 1.923491664265106e-05, "Finetune/Loss": 0.9721916913986206, "Finetune/Loss (Raw)": 1.118722915649414, "Finetune/Step": 787, "Finetune/Step Time": 5.814817618578672} +{"Finetune/Learning Rate": 1.9232525057025915e-05, "Finetune/Loss": 0.9721770286560059, "Finetune/Loss (Raw)": 1.1322376728057861, "Finetune/Step": 788, "Finetune/Step Time": 5.811353353783488} +{"Finetune/Learning Rate": 1.9230129888438046e-05, "Finetune/Loss": 0.9720755219459534, "Finetune/Loss (Raw)": 1.006813883781433, "Finetune/Step": 789, "Finetune/Step Time": 5.809760868549347} +{"Finetune/Learning Rate": 1.922773113781697e-05, "Finetune/Loss": 0.9697807431221008, "Finetune/Loss (Raw)": 0.7689642906188965, "Finetune/Step": 790, "Finetune/Step Time": 5.8402402959764} +{"Finetune/Learning Rate": 1.9225328806093596e-05, "Finetune/Loss": 0.9684181213378906, "Finetune/Loss (Raw)": 0.937213659286499, "Finetune/Step": 791, "Finetune/Step Time": 5.841635018587112} +{"Finetune/Learning Rate": 1.922292289420022e-05, "Finetune/Loss": 0.9670423865318298, "Finetune/Loss (Raw)": 0.8497417569160461, "Finetune/Step": 792, "Finetune/Step Time": 5.832161374390125} +{"Finetune/Learning Rate": 1.9220513403070534e-05, "Finetune/Loss": 0.9665772318840027, "Finetune/Loss (Raw)": 1.0872737169265747, "Finetune/Step": 793, "Finetune/Step Time": 5.834768870845437} +{"Finetune/Learning Rate": 1.9218100333639608e-05, "Finetune/Loss": 0.9647691249847412, "Finetune/Loss (Raw)": 0.9383875131607056, "Finetune/Step": 794, "Finetune/Step Time": 5.837239755317569} +{"Finetune/Learning Rate": 1.9215683686843912e-05, "Finetune/Loss": 0.9637304544448853, "Finetune/Loss (Raw)": 0.9880048036575317, "Finetune/Step": 795, "Finetune/Step Time": 5.840096326544881} +{"Finetune/Learning Rate": 1.92132634636213e-05, "Finetune/Loss": 0.9644135236740112, "Finetune/Loss (Raw)": 0.9532221555709839, "Finetune/Step": 796, "Finetune/Step Time": 5.8363798800855875} +{"Finetune/Learning Rate": 1.9210839664911013e-05, "Finetune/Loss": 0.9638753533363342, "Finetune/Loss (Raw)": 0.9699934124946594, "Finetune/Step": 797, "Finetune/Step Time": 5.842802034690976} +{"Finetune/Learning Rate": 1.9208412291653674e-05, "Finetune/Loss": 0.9646044969558716, "Finetune/Loss (Raw)": 1.0147241353988647, "Finetune/Step": 798, "Finetune/Step Time": 5.84520960226655} +{"Finetune/Learning Rate": 1.9205981344791307e-05, "Finetune/Loss": 0.9639286994934082, "Finetune/Loss (Raw)": 0.9130727052688599, "Finetune/Step": 799, "Finetune/Step Time": 5.851569989696145} +{"Finetune/Learning Rate": 1.9203546825267313e-05, "Finetune/Loss": 0.963901162147522, "Finetune/Loss (Raw)": 1.062364101409912, "Finetune/Step": 800, "Finetune/Step Time": 5.857250355184078} +{"Finetune/Learning Rate": 1.9201108734026477e-05, "Finetune/Loss": 0.9635345935821533, "Finetune/Loss (Raw)": 1.0732213258743286, "Finetune/Step": 801, "Finetune/Step Time": 5.882625080645084} +{"Finetune/Learning Rate": 1.9198667072014978e-05, "Finetune/Loss": 0.962486982345581, "Finetune/Loss (Raw)": 0.9873248338699341, "Finetune/Step": 802, "Finetune/Step Time": 5.886158529669046} +{"Finetune/Learning Rate": 1.9196221840180378e-05, "Finetune/Loss": 0.9603921175003052, "Finetune/Loss (Raw)": 0.7997627854347229, "Finetune/Step": 803, "Finetune/Step Time": 5.89246934838593} +{"Finetune/Learning Rate": 1.9193773039471622e-05, "Finetune/Loss": 0.9600275754928589, "Finetune/Loss (Raw)": 0.9914525747299194, "Finetune/Step": 804, "Finetune/Step Time": 5.88372478634119} +{"Finetune/Learning Rate": 1.9191320670839047e-05, "Finetune/Loss": 0.9609121084213257, "Finetune/Loss (Raw)": 1.0422344207763672, "Finetune/Step": 805, "Finetune/Step Time": 5.853380708023906} +{"Finetune/Learning Rate": 1.918886473523436e-05, "Finetune/Loss": 0.9586402177810669, "Finetune/Loss (Raw)": 0.846989631652832, "Finetune/Step": 806, "Finetune/Step Time": 5.867847951129079} +{"Finetune/Learning Rate": 1.918640523361067e-05, "Finetune/Loss": 0.957358181476593, "Finetune/Loss (Raw)": 0.9874146580696106, "Finetune/Step": 807, "Finetune/Step Time": 5.859889946877956} +{"Finetune/Learning Rate": 1.9183942166922455e-05, "Finetune/Loss": 0.957987904548645, "Finetune/Loss (Raw)": 1.046034336090088, "Finetune/Step": 808, "Finetune/Step Time": 5.879032708704472} +{"Finetune/Learning Rate": 1.9181475536125588e-05, "Finetune/Loss": 0.957055389881134, "Finetune/Loss (Raw)": 0.9518662691116333, "Finetune/Step": 809, "Finetune/Step Time": 5.883117746561766} +{"Finetune/Learning Rate": 1.917900534217732e-05, "Finetune/Loss": 0.957175076007843, "Finetune/Loss (Raw)": 1.0978291034698486, "Finetune/Step": 810, "Finetune/Step Time": 5.8864526730030775} +{"Finetune/Learning Rate": 1.9176531586036282e-05, "Finetune/Loss": 0.9551163911819458, "Finetune/Loss (Raw)": 0.8469897508621216, "Finetune/Step": 811, "Finetune/Step Time": 5.883961744606495} +{"Finetune/Learning Rate": 1.917405426866249e-05, "Finetune/Loss": 0.9572058916091919, "Finetune/Loss (Raw)": 1.2309077978134155, "Finetune/Step": 812, "Finetune/Step Time": 5.881037777289748} +{"Finetune/Learning Rate": 1.9171573391017348e-05, "Finetune/Loss": 0.9588927030563354, "Finetune/Loss (Raw)": 0.9831353425979614, "Finetune/Step": 813, "Finetune/Step Time": 5.867689734324813} +{"Finetune/Learning Rate": 1.9169088954063634e-05, "Finetune/Loss": 0.9590824842453003, "Finetune/Loss (Raw)": 0.962626039981842, "Finetune/Step": 814, "Finetune/Step Time": 5.857370495796204} +{"Finetune/Learning Rate": 1.9166600958765508e-05, "Finetune/Loss": 0.9591141939163208, "Finetune/Loss (Raw)": 0.9774408936500549, "Finetune/Step": 815, "Finetune/Step Time": 5.861578863114119} +{"Finetune/Learning Rate": 1.916410940608851e-05, "Finetune/Loss": 0.9595212340354919, "Finetune/Loss (Raw)": 1.0691794157028198, "Finetune/Step": 816, "Finetune/Step Time": 5.861606378108263} +{"Finetune/Learning Rate": 1.9161614296999565e-05, "Finetune/Loss": 0.9622206687927246, "Finetune/Loss (Raw)": 1.078493356704712, "Finetune/Step": 817, "Finetune/Step Time": 5.850523108616471} +{"Finetune/Learning Rate": 1.9159115632466978e-05, "Finetune/Loss": 0.9581708908081055, "Finetune/Loss (Raw)": 0.3274155855178833, "Finetune/Step": 818, "Finetune/Step Time": 5.833572326228023} +{"Finetune/Learning Rate": 1.9156613413460432e-05, "Finetune/Loss": 0.9563425183296204, "Finetune/Loss (Raw)": 0.8284473419189453, "Finetune/Step": 819, "Finetune/Step Time": 5.833404345437884} +{"Finetune/Learning Rate": 1.9154107640950986e-05, "Finetune/Loss": 0.9548848867416382, "Finetune/Loss (Raw)": 0.7435054779052734, "Finetune/Step": 820, "Finetune/Step Time": 5.813995840027928} +{"Finetune/Learning Rate": 1.9151598315911088e-05, "Finetune/Loss": 0.9549428224563599, "Finetune/Loss (Raw)": 0.996738612651825, "Finetune/Step": 821, "Finetune/Step Time": 5.814386086538434} +{"Finetune/Learning Rate": 1.9149085439314548e-05, "Finetune/Loss": 0.9590287208557129, "Finetune/Loss (Raw)": 0.9251306056976318, "Finetune/Step": 822, "Finetune/Step Time": 5.787608582526445} +{"Finetune/Learning Rate": 1.9146569012136572e-05, "Finetune/Loss": 0.9608176946640015, "Finetune/Loss (Raw)": 1.0987906455993652, "Finetune/Step": 823, "Finetune/Step Time": 5.773678030818701} +{"Finetune/Learning Rate": 1.914404903535373e-05, "Finetune/Loss": 0.9605594873428345, "Finetune/Loss (Raw)": 1.0842094421386719, "Finetune/Step": 824, "Finetune/Step Time": 5.772152308374643} +{"Finetune/Learning Rate": 1.9141525509943987e-05, "Finetune/Loss": 0.9603610038757324, "Finetune/Loss (Raw)": 0.9180947542190552, "Finetune/Step": 825, "Finetune/Step Time": 5.762274084612727} +{"Finetune/Learning Rate": 1.9138998436886664e-05, "Finetune/Loss": 0.9577284455299377, "Finetune/Loss (Raw)": 0.8581583499908447, "Finetune/Step": 826, "Finetune/Step Time": 5.768444759771228} +{"Finetune/Learning Rate": 1.913646781716247e-05, "Finetune/Loss": 0.9561642408370972, "Finetune/Loss (Raw)": 1.0058966875076294, "Finetune/Step": 827, "Finetune/Step Time": 5.754973793402314} +{"Finetune/Learning Rate": 1.9133933651753493e-05, "Finetune/Loss": 0.9513675570487976, "Finetune/Loss (Raw)": 0.3806626796722412, "Finetune/Step": 828, "Finetune/Step Time": 5.765443155542016} +{"Finetune/Learning Rate": 1.913139594164319e-05, "Finetune/Loss": 0.9573187828063965, "Finetune/Loss (Raw)": 1.0750023126602173, "Finetune/Step": 829, "Finetune/Step Time": 5.735930867493153} +{"Finetune/Learning Rate": 1.9128854687816394e-05, "Finetune/Loss": 0.9574163556098938, "Finetune/Loss (Raw)": 0.9952653050422668, "Finetune/Step": 830, "Finetune/Step Time": 5.733211379498243} +{"Finetune/Learning Rate": 1.9126309891259324e-05, "Finetune/Loss": 0.9586997032165527, "Finetune/Loss (Raw)": 1.0444958209991455, "Finetune/Step": 831, "Finetune/Step Time": 5.744761899113655} +{"Finetune/Learning Rate": 1.9123761552959562e-05, "Finetune/Loss": 0.959378719329834, "Finetune/Loss (Raw)": 1.0854570865631104, "Finetune/Step": 832, "Finetune/Step Time": 5.731741661205888} +{"Finetune/Learning Rate": 1.9121209673906064e-05, "Finetune/Loss": 0.9547812938690186, "Finetune/Loss (Raw)": 0.4261833429336548, "Finetune/Step": 833, "Finetune/Step Time": 5.758158287033439} +{"Finetune/Learning Rate": 1.911865425508917e-05, "Finetune/Loss": 0.9563703536987305, "Finetune/Loss (Raw)": 0.9935636520385742, "Finetune/Step": 834, "Finetune/Step Time": 5.766158426180482} +{"Finetune/Learning Rate": 1.9116095297500588e-05, "Finetune/Loss": 0.9569858908653259, "Finetune/Loss (Raw)": 0.9876717329025269, "Finetune/Step": 835, "Finetune/Step Time": 5.7765484023839235} +{"Finetune/Learning Rate": 1.9113532802133394e-05, "Finetune/Loss": 0.9558354616165161, "Finetune/Loss (Raw)": 1.0176599025726318, "Finetune/Step": 836, "Finetune/Step Time": 5.766204785555601} +{"Finetune/Learning Rate": 1.911096676998205e-05, "Finetune/Loss": 0.9555566310882568, "Finetune/Loss (Raw)": 0.9350476264953613, "Finetune/Step": 837, "Finetune/Step Time": 5.775348134338856} +{"Finetune/Learning Rate": 1.9108397202042376e-05, "Finetune/Loss": 0.9569185972213745, "Finetune/Loss (Raw)": 1.059108018875122, "Finetune/Step": 838, "Finetune/Step Time": 5.779681898653507} +{"Finetune/Learning Rate": 1.910582409931158e-05, "Finetune/Loss": 0.9555752277374268, "Finetune/Loss (Raw)": 0.9055472016334534, "Finetune/Step": 839, "Finetune/Step Time": 5.771137025207281} +{"Finetune/Learning Rate": 1.910324746278822e-05, "Finetune/Loss": 0.9590002298355103, "Finetune/Loss (Raw)": 1.1709903478622437, "Finetune/Step": 840, "Finetune/Step Time": 5.754198115319014} +{"Finetune/Learning Rate": 1.9100667293472252e-05, "Finetune/Loss": 0.9583667516708374, "Finetune/Loss (Raw)": 0.8980797529220581, "Finetune/Step": 841, "Finetune/Step Time": 5.758273037150502} +{"Finetune/Learning Rate": 1.9098083592364976e-05, "Finetune/Loss": 0.9594107866287231, "Finetune/Loss (Raw)": 1.070475459098816, "Finetune/Step": 842, "Finetune/Step Time": 5.751826351508498} +{"Finetune/Learning Rate": 1.9095496360469087e-05, "Finetune/Loss": 0.9588460922241211, "Finetune/Loss (Raw)": 1.0506874322891235, "Finetune/Step": 843, "Finetune/Step Time": 5.76640897244215} +{"Finetune/Learning Rate": 1.9092905598788633e-05, "Finetune/Loss": 0.9598512649536133, "Finetune/Loss (Raw)": 0.9968987107276917, "Finetune/Step": 844, "Finetune/Step Time": 5.768046228215098} +{"Finetune/Learning Rate": 1.9090311308329045e-05, "Finetune/Loss": 0.9593856930732727, "Finetune/Loss (Raw)": 1.0791966915130615, "Finetune/Step": 845, "Finetune/Step Time": 5.754364436492324} +{"Finetune/Learning Rate": 1.9087713490097108e-05, "Finetune/Loss": 0.9592993259429932, "Finetune/Loss (Raw)": 1.0435690879821777, "Finetune/Step": 846, "Finetune/Step Time": 5.7465724516659975} +{"Finetune/Learning Rate": 1.9085112145100987e-05, "Finetune/Loss": 0.9548013210296631, "Finetune/Loss (Raw)": 0.46565860509872437, "Finetune/Step": 847, "Finetune/Step Time": 5.777145462110639} +{"Finetune/Learning Rate": 1.9082507274350216e-05, "Finetune/Loss": 0.9519474506378174, "Finetune/Loss (Raw)": 0.8497663140296936, "Finetune/Step": 848, "Finetune/Step Time": 5.779824310913682} +{"Finetune/Learning Rate": 1.9079898878855695e-05, "Finetune/Loss": 0.9525455236434937, "Finetune/Loss (Raw)": 1.0100677013397217, "Finetune/Step": 849, "Finetune/Step Time": 5.757304776459932} +{"Finetune/Learning Rate": 1.9077286959629684e-05, "Finetune/Loss": 0.9517004489898682, "Finetune/Loss (Raw)": 0.8925266861915588, "Finetune/Step": 850, "Finetune/Step Time": 5.741661801934242} +{"Finetune/Learning Rate": 1.907467151768583e-05, "Finetune/Loss": 0.9509503841400146, "Finetune/Loss (Raw)": 1.0210816860198975, "Finetune/Step": 851, "Finetune/Step Time": 5.753466168418527} +{"Finetune/Learning Rate": 1.9072052554039123e-05, "Finetune/Loss": 0.9494063258171082, "Finetune/Loss (Raw)": 0.8833205699920654, "Finetune/Step": 852, "Finetune/Step Time": 5.738220611587167} +{"Finetune/Learning Rate": 1.9069430069705936e-05, "Finetune/Loss": 0.9479752779006958, "Finetune/Loss (Raw)": 0.9546767473220825, "Finetune/Step": 853, "Finetune/Step Time": 5.740110028535128} +{"Finetune/Learning Rate": 1.9066804065704006e-05, "Finetune/Loss": 0.9493793249130249, "Finetune/Loss (Raw)": 0.9415764808654785, "Finetune/Step": 854, "Finetune/Step Time": 5.742422709241509} +{"Finetune/Learning Rate": 1.9064174543052435e-05, "Finetune/Loss": 0.9475452899932861, "Finetune/Loss (Raw)": 0.7969515323638916, "Finetune/Step": 855, "Finetune/Step Time": 5.7786574475467205} +{"Finetune/Learning Rate": 1.9061541502771682e-05, "Finetune/Loss": 0.9459433555603027, "Finetune/Loss (Raw)": 0.9926326274871826, "Finetune/Step": 856, "Finetune/Step Time": 5.775734355673194} +{"Finetune/Learning Rate": 1.905890494588359e-05, "Finetune/Loss": 0.9482570886611938, "Finetune/Loss (Raw)": 0.9490153193473816, "Finetune/Step": 857, "Finetune/Step Time": 5.765748573467135} +{"Finetune/Learning Rate": 1.9056264873411343e-05, "Finetune/Loss": 0.9497004151344299, "Finetune/Loss (Raw)": 1.188768744468689, "Finetune/Step": 858, "Finetune/Step Time": 5.763271614909172} +{"Finetune/Learning Rate": 1.905362128637951e-05, "Finetune/Loss": 0.9498046636581421, "Finetune/Loss (Raw)": 1.1060876846313477, "Finetune/Step": 859, "Finetune/Step Time": 5.778168840333819} +{"Finetune/Learning Rate": 1.905097418581401e-05, "Finetune/Loss": 0.9501222968101501, "Finetune/Loss (Raw)": 0.9515563249588013, "Finetune/Step": 860, "Finetune/Step Time": 5.745861418545246} +{"Finetune/Learning Rate": 1.904832357274214e-05, "Finetune/Loss": 0.9483348727226257, "Finetune/Loss (Raw)": 0.7285627126693726, "Finetune/Step": 861, "Finetune/Step Time": 5.735893419012427} +{"Finetune/Learning Rate": 1.9045669448192537e-05, "Finetune/Loss": 0.9494737386703491, "Finetune/Loss (Raw)": 1.0184614658355713, "Finetune/Step": 862, "Finetune/Step Time": 5.738686198368669} +{"Finetune/Learning Rate": 1.904301181319523e-05, "Finetune/Loss": 0.9490472674369812, "Finetune/Loss (Raw)": 0.848917543888092, "Finetune/Step": 863, "Finetune/Step Time": 5.744826437905431} +{"Finetune/Learning Rate": 1.9040350668781584e-05, "Finetune/Loss": 0.949902355670929, "Finetune/Loss (Raw)": 1.0405303239822388, "Finetune/Step": 864, "Finetune/Step Time": 5.712293615564704} +{"Finetune/Learning Rate": 1.9037686015984343e-05, "Finetune/Loss": 0.9486240148544312, "Finetune/Loss (Raw)": 0.9590223431587219, "Finetune/Step": 865, "Finetune/Step Time": 5.734221106395125} +{"Finetune/Learning Rate": 1.9035017855837604e-05, "Finetune/Loss": 0.9494045972824097, "Finetune/Loss (Raw)": 1.1214367151260376, "Finetune/Step": 866, "Finetune/Step Time": 5.73603056743741} +{"Finetune/Learning Rate": 1.9032346189376828e-05, "Finetune/Loss": 0.9541184902191162, "Finetune/Loss (Raw)": 1.0636712312698364, "Finetune/Step": 867, "Finetune/Step Time": 5.704033339396119} +{"Finetune/Learning Rate": 1.902967101763884e-05, "Finetune/Loss": 0.9525473713874817, "Finetune/Loss (Raw)": 0.9450041651725769, "Finetune/Step": 868, "Finetune/Step Time": 5.697424981743097} +{"Finetune/Learning Rate": 1.902699234166182e-05, "Finetune/Loss": 0.9547240138053894, "Finetune/Loss (Raw)": 1.090983271598816, "Finetune/Step": 869, "Finetune/Step Time": 5.68832023255527} +{"Finetune/Learning Rate": 1.9024310162485308e-05, "Finetune/Loss": 0.954100489616394, "Finetune/Loss (Raw)": 0.903249204158783, "Finetune/Step": 870, "Finetune/Step Time": 5.69180154427886} +{"Finetune/Learning Rate": 1.9021624481150208e-05, "Finetune/Loss": 0.9523829221725464, "Finetune/Loss (Raw)": 0.8533643484115601, "Finetune/Step": 871, "Finetune/Step Time": 5.695630941540003} +{"Finetune/Learning Rate": 1.9018935298698777e-05, "Finetune/Loss": 0.9524106979370117, "Finetune/Loss (Raw)": 0.9996557235717773, "Finetune/Step": 872, "Finetune/Step Time": 5.695046577602625} +{"Finetune/Learning Rate": 1.9016242616174637e-05, "Finetune/Loss": 0.9535677433013916, "Finetune/Loss (Raw)": 1.0429280996322632, "Finetune/Step": 873, "Finetune/Step Time": 5.678507836535573} +{"Finetune/Learning Rate": 1.9013546434622766e-05, "Finetune/Loss": 0.951858639717102, "Finetune/Loss (Raw)": 0.8820000886917114, "Finetune/Step": 874, "Finetune/Step Time": 5.683650378137827} +{"Finetune/Learning Rate": 1.90108467550895e-05, "Finetune/Loss": 0.9505031704902649, "Finetune/Loss (Raw)": 0.9465407729148865, "Finetune/Step": 875, "Finetune/Step Time": 5.6950955875217915} +{"Finetune/Learning Rate": 1.9008143578622527e-05, "Finetune/Loss": 0.9492502808570862, "Finetune/Loss (Raw)": 0.9394135475158691, "Finetune/Step": 876, "Finetune/Step Time": 5.6961763352155685} +{"Finetune/Learning Rate": 1.9005436906270902e-05, "Finetune/Loss": 0.9494372606277466, "Finetune/Loss (Raw)": 1.0329689979553223, "Finetune/Step": 877, "Finetune/Step Time": 5.694633262231946} +{"Finetune/Learning Rate": 1.9002726739085027e-05, "Finetune/Loss": 0.9491522312164307, "Finetune/Loss (Raw)": 0.9949043989181519, "Finetune/Step": 878, "Finetune/Step Time": 5.687184300273657} +{"Finetune/Learning Rate": 1.900001307811667e-05, "Finetune/Loss": 0.949128270149231, "Finetune/Loss (Raw)": 0.8942701816558838, "Finetune/Step": 879, "Finetune/Step Time": 5.708008613437414} +{"Finetune/Learning Rate": 1.899729592441895e-05, "Finetune/Loss": 0.9486576318740845, "Finetune/Loss (Raw)": 1.02022385597229, "Finetune/Step": 880, "Finetune/Step Time": 5.706729989498854} +{"Finetune/Learning Rate": 1.899457527904634e-05, "Finetune/Loss": 0.9525023102760315, "Finetune/Loss (Raw)": 0.8892912268638611, "Finetune/Step": 881, "Finetune/Step Time": 5.694171320647001} +{"Finetune/Learning Rate": 1.8991851143054662e-05, "Finetune/Loss": 0.9539927840232849, "Finetune/Loss (Raw)": 1.0684521198272705, "Finetune/Step": 882, "Finetune/Step Time": 5.694312326610088} +{"Finetune/Learning Rate": 1.8989123517501112e-05, "Finetune/Loss": 0.9549551606178284, "Finetune/Loss (Raw)": 1.0607103109359741, "Finetune/Step": 883, "Finetune/Step Time": 5.665231708437204} +{"Finetune/Learning Rate": 1.8986392403444222e-05, "Finetune/Loss": 0.9558565616607666, "Finetune/Loss (Raw)": 1.0726832151412964, "Finetune/Step": 884, "Finetune/Step Time": 5.666043847799301} +{"Finetune/Learning Rate": 1.8983657801943884e-05, "Finetune/Loss": 0.9499167799949646, "Finetune/Loss (Raw)": 0.22948431968688965, "Finetune/Step": 885, "Finetune/Step Time": 5.684031143784523} +{"Finetune/Learning Rate": 1.8980919714061343e-05, "Finetune/Loss": 0.948788046836853, "Finetune/Loss (Raw)": 1.0159187316894531, "Finetune/Step": 886, "Finetune/Step Time": 5.687270577996969} +{"Finetune/Learning Rate": 1.89781781408592e-05, "Finetune/Loss": 0.9435603618621826, "Finetune/Loss (Raw)": 0.4044935405254364, "Finetune/Step": 887, "Finetune/Step Time": 5.718286909162998} +{"Finetune/Learning Rate": 1.8975433083401403e-05, "Finetune/Loss": 0.9441100358963013, "Finetune/Loss (Raw)": 1.0865633487701416, "Finetune/Step": 888, "Finetune/Step Time": 5.7239570785313845} +{"Finetune/Learning Rate": 1.897268454275326e-05, "Finetune/Loss": 0.9449632167816162, "Finetune/Loss (Raw)": 1.0456303358078003, "Finetune/Step": 889, "Finetune/Step Time": 5.72973146289587} +{"Finetune/Learning Rate": 1.896993251998142e-05, "Finetune/Loss": 0.9495867490768433, "Finetune/Loss (Raw)": 0.9736764430999756, "Finetune/Step": 890, "Finetune/Step Time": 5.699548684060574} +{"Finetune/Learning Rate": 1.8967177016153895e-05, "Finetune/Loss": 0.9503129720687866, "Finetune/Loss (Raw)": 0.9908855557441711, "Finetune/Step": 891, "Finetune/Step Time": 5.702767465263605} +{"Finetune/Learning Rate": 1.8964418032340037e-05, "Finetune/Loss": 0.9494298696517944, "Finetune/Loss (Raw)": 0.8909663558006287, "Finetune/Step": 892, "Finetune/Step Time": 5.706754231825471} +{"Finetune/Learning Rate": 1.8961655569610557e-05, "Finetune/Loss": 0.9505109786987305, "Finetune/Loss (Raw)": 1.1261204481124878, "Finetune/Step": 893, "Finetune/Step Time": 5.70335097797215} +{"Finetune/Learning Rate": 1.8958889629037514e-05, "Finetune/Loss": 0.9495522975921631, "Finetune/Loss (Raw)": 0.9753090739250183, "Finetune/Step": 894, "Finetune/Step Time": 5.68771624006331} +{"Finetune/Learning Rate": 1.895612021169431e-05, "Finetune/Loss": 0.9499754905700684, "Finetune/Loss (Raw)": 0.9775033593177795, "Finetune/Step": 895, "Finetune/Step Time": 5.699633441865444} +{"Finetune/Learning Rate": 1.8953347318655704e-05, "Finetune/Loss": 0.951034665107727, "Finetune/Loss (Raw)": 1.0663576126098633, "Finetune/Step": 896, "Finetune/Step Time": 5.686900481581688} +{"Finetune/Learning Rate": 1.8950570950997808e-05, "Finetune/Loss": 0.9512935280799866, "Finetune/Loss (Raw)": 0.8595654368400574, "Finetune/Step": 897, "Finetune/Step Time": 5.690395699813962} +{"Finetune/Learning Rate": 1.894779110979807e-05, "Finetune/Loss": 0.9459930658340454, "Finetune/Loss (Raw)": 0.3606853187084198, "Finetune/Step": 898, "Finetune/Step Time": 5.707315307110548} +{"Finetune/Learning Rate": 1.894500779613529e-05, "Finetune/Loss": 0.944014310836792, "Finetune/Loss (Raw)": 0.8253757357597351, "Finetune/Step": 899, "Finetune/Step Time": 5.7914235927164555} +{"Finetune/Learning Rate": 1.8942221011089623e-05, "Finetune/Loss": 0.9448684453964233, "Finetune/Loss (Raw)": 1.1120187044143677, "Finetune/Step": 900, "Finetune/Step Time": 5.796020373702049} +{"Finetune/Learning Rate": 1.8939430755742564e-05, "Finetune/Loss": 0.945164680480957, "Finetune/Loss (Raw)": 1.1027013063430786, "Finetune/Step": 901, "Finetune/Step Time": 5.790677418932319} +{"Finetune/Learning Rate": 1.8936637031176957e-05, "Finetune/Loss": 0.9466168880462646, "Finetune/Loss (Raw)": 1.0601364374160767, "Finetune/Step": 902, "Finetune/Step Time": 5.778866171836853} +{"Finetune/Learning Rate": 1.893383983847699e-05, "Finetune/Loss": 0.9469568729400635, "Finetune/Loss (Raw)": 1.0811105966567993, "Finetune/Step": 903, "Finetune/Step Time": 5.788868948817253} +{"Finetune/Learning Rate": 1.89310391787282e-05, "Finetune/Loss": 0.9481558799743652, "Finetune/Loss (Raw)": 0.9613369703292847, "Finetune/Step": 904, "Finetune/Step Time": 5.77958133071661} +{"Finetune/Learning Rate": 1.892823505301747e-05, "Finetune/Loss": 0.9475514888763428, "Finetune/Loss (Raw)": 1.062157154083252, "Finetune/Step": 905, "Finetune/Step Time": 5.764783700928092} +{"Finetune/Learning Rate": 1.8925427462433027e-05, "Finetune/Loss": 0.946342408657074, "Finetune/Loss (Raw)": 0.9407365322113037, "Finetune/Step": 906, "Finetune/Step Time": 5.762244960293174} +{"Finetune/Learning Rate": 1.8922616408064436e-05, "Finetune/Loss": 0.9472441673278809, "Finetune/Loss (Raw)": 1.0531061887741089, "Finetune/Step": 907, "Finetune/Step Time": 5.7495528142899275} +{"Finetune/Learning Rate": 1.8919801891002615e-05, "Finetune/Loss": 0.9522340297698975, "Finetune/Loss (Raw)": 0.9760183095932007, "Finetune/Step": 908, "Finetune/Step Time": 5.718993911519647} +{"Finetune/Learning Rate": 1.891698391233983e-05, "Finetune/Loss": 0.9527314901351929, "Finetune/Loss (Raw)": 0.9991192817687988, "Finetune/Step": 909, "Finetune/Step Time": 5.712831089273095} +{"Finetune/Learning Rate": 1.8914162473169676e-05, "Finetune/Loss": 0.9545455574989319, "Finetune/Loss (Raw)": 0.9726027846336365, "Finetune/Step": 910, "Finetune/Step Time": 5.684516018256545} +{"Finetune/Learning Rate": 1.8911337574587098e-05, "Finetune/Loss": 0.9546182155609131, "Finetune/Loss (Raw)": 0.9365196824073792, "Finetune/Step": 911, "Finetune/Step Time": 5.632182888686657} +{"Finetune/Learning Rate": 1.8908509217688387e-05, "Finetune/Loss": 0.9575190544128418, "Finetune/Loss (Raw)": 1.1226511001586914, "Finetune/Step": 912, "Finetune/Step Time": 5.623294830322266} +{"Finetune/Learning Rate": 1.8905677403571177e-05, "Finetune/Loss": 0.9547210931777954, "Finetune/Loss (Raw)": 0.7228900194168091, "Finetune/Step": 913, "Finetune/Step Time": 5.618154879659414} +{"Finetune/Learning Rate": 1.890284213333443e-05, "Finetune/Loss": 0.9546337127685547, "Finetune/Loss (Raw)": 1.1249526739120483, "Finetune/Step": 914, "Finetune/Step Time": 5.622128617018461} +{"Finetune/Learning Rate": 1.8900003408078468e-05, "Finetune/Loss": 0.9538254737854004, "Finetune/Loss (Raw)": 1.0152662992477417, "Finetune/Step": 915, "Finetune/Step Time": 5.594336738809943} +{"Finetune/Learning Rate": 1.889716122890494e-05, "Finetune/Loss": 0.9530707597732544, "Finetune/Loss (Raw)": 1.0356419086456299, "Finetune/Step": 916, "Finetune/Step Time": 5.601145731285214} +{"Finetune/Learning Rate": 1.889431559691684e-05, "Finetune/Loss": 0.9545041918754578, "Finetune/Loss (Raw)": 1.1902856826782227, "Finetune/Step": 917, "Finetune/Step Time": 5.65089731477201} +{"Finetune/Learning Rate": 1.8891466513218507e-05, "Finetune/Loss": 0.9561272859573364, "Finetune/Loss (Raw)": 0.9767169952392578, "Finetune/Step": 918, "Finetune/Step Time": 5.627483958378434} +{"Finetune/Learning Rate": 1.888861397891561e-05, "Finetune/Loss": 0.9547762870788574, "Finetune/Loss (Raw)": 0.7642858624458313, "Finetune/Step": 919, "Finetune/Step Time": 5.646841267123818} +{"Finetune/Learning Rate": 1.888575799511516e-05, "Finetune/Loss": 0.9547419548034668, "Finetune/Loss (Raw)": 0.8453455567359924, "Finetune/Step": 920, "Finetune/Step Time": 5.652832193300128} +{"Finetune/Learning Rate": 1.888289856292552e-05, "Finetune/Loss": 0.953973650932312, "Finetune/Loss (Raw)": 0.9889432191848755, "Finetune/Step": 921, "Finetune/Step Time": 5.64261974580586} +{"Finetune/Learning Rate": 1.8880035683456365e-05, "Finetune/Loss": 0.9538547992706299, "Finetune/Loss (Raw)": 0.9231769442558289, "Finetune/Step": 922, "Finetune/Step Time": 5.641412142664194} +{"Finetune/Learning Rate": 1.887716935781873e-05, "Finetune/Loss": 0.9542455673217773, "Finetune/Loss (Raw)": 1.0380144119262695, "Finetune/Step": 923, "Finetune/Step Time": 5.633371319621801} +{"Finetune/Learning Rate": 1.887429958712498e-05, "Finetune/Loss": 0.9541158676147461, "Finetune/Loss (Raw)": 0.9366298913955688, "Finetune/Step": 924, "Finetune/Step Time": 5.641811154782772} +{"Finetune/Learning Rate": 1.8871426372488818e-05, "Finetune/Loss": 0.9529006481170654, "Finetune/Loss (Raw)": 0.8144370913505554, "Finetune/Step": 925, "Finetune/Step Time": 5.661323960870504} +{"Finetune/Learning Rate": 1.886854971502528e-05, "Finetune/Loss": 0.9523799419403076, "Finetune/Loss (Raw)": 0.9480724930763245, "Finetune/Step": 926, "Finetune/Step Time": 5.665311930701137} +{"Finetune/Learning Rate": 1.886566961585074e-05, "Finetune/Loss": 0.9533966779708862, "Finetune/Loss (Raw)": 1.0432199239730835, "Finetune/Step": 927, "Finetune/Step Time": 5.65825704857707} +{"Finetune/Learning Rate": 1.886278607608291e-05, "Finetune/Loss": 0.9529907703399658, "Finetune/Loss (Raw)": 1.0104047060012817, "Finetune/Step": 928, "Finetune/Step Time": 5.65039910748601} +{"Finetune/Learning Rate": 1.8859899096840836e-05, "Finetune/Loss": 0.9526765942573547, "Finetune/Loss (Raw)": 1.0330028533935547, "Finetune/Step": 929, "Finetune/Step Time": 5.625792747363448} +{"Finetune/Learning Rate": 1.8857008679244897e-05, "Finetune/Loss": 0.9520328044891357, "Finetune/Loss (Raw)": 0.9049280285835266, "Finetune/Step": 930, "Finetune/Step Time": 5.6302754655480385} +{"Finetune/Learning Rate": 1.885411482441681e-05, "Finetune/Loss": 0.953829288482666, "Finetune/Loss (Raw)": 1.0297107696533203, "Finetune/Step": 931, "Finetune/Step Time": 5.611634284257889} +{"Finetune/Learning Rate": 1.885121753347962e-05, "Finetune/Loss": 0.9544006586074829, "Finetune/Loss (Raw)": 1.0645811557769775, "Finetune/Step": 932, "Finetune/Step Time": 5.61791724152863} +{"Finetune/Learning Rate": 1.884831680755771e-05, "Finetune/Loss": 0.9544331431388855, "Finetune/Loss (Raw)": 1.0463939905166626, "Finetune/Step": 933, "Finetune/Step Time": 5.627650620415807} +{"Finetune/Learning Rate": 1.8845412647776795e-05, "Finetune/Loss": 0.9557383060455322, "Finetune/Loss (Raw)": 1.0140540599822998, "Finetune/Step": 934, "Finetune/Step Time": 5.613681772723794} +{"Finetune/Learning Rate": 1.8842505055263928e-05, "Finetune/Loss": 0.9547593593597412, "Finetune/Loss (Raw)": 0.8621057271957397, "Finetune/Step": 935, "Finetune/Step Time": 5.6144904512912035} +{"Finetune/Learning Rate": 1.883959403114748e-05, "Finetune/Loss": 0.9539796113967896, "Finetune/Loss (Raw)": 0.9462268352508545, "Finetune/Step": 936, "Finetune/Step Time": 5.606623012572527} +{"Finetune/Learning Rate": 1.8836679576557165e-05, "Finetune/Loss": 0.9553732872009277, "Finetune/Loss (Raw)": 1.1302598714828491, "Finetune/Step": 937, "Finetune/Step Time": 5.601554377004504} +{"Finetune/Learning Rate": 1.8833761692624034e-05, "Finetune/Loss": 0.9556781053543091, "Finetune/Loss (Raw)": 1.1368560791015625, "Finetune/Step": 938, "Finetune/Step Time": 5.605373656377196} +{"Finetune/Learning Rate": 1.883084038048045e-05, "Finetune/Loss": 0.9563285112380981, "Finetune/Loss (Raw)": 0.9302242994308472, "Finetune/Step": 939, "Finetune/Step Time": 5.606570651754737} +{"Finetune/Learning Rate": 1.8827915641260127e-05, "Finetune/Loss": 0.9547032117843628, "Finetune/Loss (Raw)": 1.0228763818740845, "Finetune/Step": 940, "Finetune/Step Time": 5.609393950551748} +{"Finetune/Learning Rate": 1.882498747609809e-05, "Finetune/Loss": 0.9546447992324829, "Finetune/Loss (Raw)": 0.9756515026092529, "Finetune/Step": 941, "Finetune/Step Time": 5.608707673847675} +{"Finetune/Learning Rate": 1.8822055886130708e-05, "Finetune/Loss": 0.954472541809082, "Finetune/Loss (Raw)": 0.9405805468559265, "Finetune/Step": 942, "Finetune/Step Time": 5.614642353728414} +{"Finetune/Learning Rate": 1.8819120872495675e-05, "Finetune/Loss": 0.9542518854141235, "Finetune/Loss (Raw)": 0.9492045044898987, "Finetune/Step": 943, "Finetune/Step Time": 5.613501682877541} +{"Finetune/Learning Rate": 1.8816182436332006e-05, "Finetune/Loss": 0.9537100195884705, "Finetune/Loss (Raw)": 0.9998152256011963, "Finetune/Step": 944, "Finetune/Step Time": 5.608641721308231} +{"Finetune/Learning Rate": 1.881324057878006e-05, "Finetune/Loss": 0.952082633972168, "Finetune/Loss (Raw)": 0.8701916337013245, "Finetune/Step": 945, "Finetune/Step Time": 5.618347687646747} +{"Finetune/Learning Rate": 1.8810295300981504e-05, "Finetune/Loss": 0.9565973281860352, "Finetune/Loss (Raw)": 0.9052979946136475, "Finetune/Step": 946, "Finetune/Step Time": 5.604860257357359} +{"Finetune/Learning Rate": 1.8807346604079352e-05, "Finetune/Loss": 0.9577032327651978, "Finetune/Loss (Raw)": 0.9699999094009399, "Finetune/Step": 947, "Finetune/Step Time": 5.624508339911699} +{"Finetune/Learning Rate": 1.880439448921793e-05, "Finetune/Loss": 0.954919159412384, "Finetune/Loss (Raw)": 0.3871409595012665, "Finetune/Step": 948, "Finetune/Step Time": 5.649701464921236} +{"Finetune/Learning Rate": 1.88014389575429e-05, "Finetune/Loss": 0.9559962749481201, "Finetune/Loss (Raw)": 1.134610891342163, "Finetune/Step": 949, "Finetune/Step Time": 5.629586894065142} +{"Finetune/Learning Rate": 1.879848001020124e-05, "Finetune/Loss": 0.9558782577514648, "Finetune/Loss (Raw)": 0.9100271463394165, "Finetune/Step": 950, "Finetune/Step Time": 5.6272456757724285} +{"Finetune/Learning Rate": 1.8795517648341267e-05, "Finetune/Loss": 0.9565582275390625, "Finetune/Loss (Raw)": 1.1858209371566772, "Finetune/Step": 951, "Finetune/Step Time": 5.620673973113298} +{"Finetune/Learning Rate": 1.8792551873112614e-05, "Finetune/Loss": 0.9547631144523621, "Finetune/Loss (Raw)": 0.85444176197052, "Finetune/Step": 952, "Finetune/Step Time": 5.625770373269916} +{"Finetune/Learning Rate": 1.8789582685666237e-05, "Finetune/Loss": 0.9552834630012512, "Finetune/Loss (Raw)": 0.984695315361023, "Finetune/Step": 953, "Finetune/Step Time": 5.618554670363665} +{"Finetune/Learning Rate": 1.878661008715442e-05, "Finetune/Loss": 0.9557409882545471, "Finetune/Loss (Raw)": 0.9167280793190002, "Finetune/Step": 954, "Finetune/Step Time": 5.615148784592748} +{"Finetune/Learning Rate": 1.8783634078730774e-05, "Finetune/Loss": 0.9556378126144409, "Finetune/Loss (Raw)": 0.992682933807373, "Finetune/Step": 955, "Finetune/Step Time": 5.61460816860199} +{"Finetune/Learning Rate": 1.878065466155023e-05, "Finetune/Loss": 0.9607291221618652, "Finetune/Loss (Raw)": 1.0323588848114014, "Finetune/Step": 956, "Finetune/Step Time": 5.586322268471122} +{"Finetune/Learning Rate": 1.877767183676904e-05, "Finetune/Loss": 0.9593889713287354, "Finetune/Loss (Raw)": 0.903459906578064, "Finetune/Step": 957, "Finetune/Step Time": 5.582776945084333} +{"Finetune/Learning Rate": 1.8774685605544776e-05, "Finetune/Loss": 0.9592523574829102, "Finetune/Loss (Raw)": 0.9777779579162598, "Finetune/Step": 958, "Finetune/Step Time": 5.5919893607497215} +{"Finetune/Learning Rate": 1.8771695969036342e-05, "Finetune/Loss": 0.9584015607833862, "Finetune/Loss (Raw)": 0.9355936050415039, "Finetune/Step": 959, "Finetune/Step Time": 5.5622820649296045} +{"Finetune/Learning Rate": 1.8768702928403954e-05, "Finetune/Loss": 0.9573906660079956, "Finetune/Loss (Raw)": 0.956057608127594, "Finetune/Step": 960, "Finetune/Step Time": 5.5588211845606565} +{"Finetune/Learning Rate": 1.8765706484809155e-05, "Finetune/Loss": 0.9609633684158325, "Finetune/Loss (Raw)": 0.883491575717926, "Finetune/Step": 961, "Finetune/Step Time": 5.528788918629289} +{"Finetune/Learning Rate": 1.8762706639414806e-05, "Finetune/Loss": 0.9618860483169556, "Finetune/Loss (Raw)": 1.1116715669631958, "Finetune/Step": 962, "Finetune/Step Time": 5.526148281991482} +{"Finetune/Learning Rate": 1.8759703393385087e-05, "Finetune/Loss": 0.9613014459609985, "Finetune/Loss (Raw)": 0.9128325581550598, "Finetune/Step": 963, "Finetune/Step Time": 5.5139347948133945} +{"Finetune/Learning Rate": 1.87566967478855e-05, "Finetune/Loss": 0.9606294631958008, "Finetune/Loss (Raw)": 0.9316551685333252, "Finetune/Step": 964, "Finetune/Step Time": 5.518683917820454} +{"Finetune/Learning Rate": 1.8753686704082868e-05, "Finetune/Loss": 0.9596496820449829, "Finetune/Loss (Raw)": 0.8096365928649902, "Finetune/Step": 965, "Finetune/Step Time": 5.513804627582431} +{"Finetune/Learning Rate": 1.8750673263145323e-05, "Finetune/Loss": 0.9587982892990112, "Finetune/Loss (Raw)": 0.9501333236694336, "Finetune/Step": 966, "Finetune/Step Time": 5.514318006113172} +{"Finetune/Learning Rate": 1.874765642624233e-05, "Finetune/Loss": 0.9595357179641724, "Finetune/Loss (Raw)": 0.9999299645423889, "Finetune/Step": 967, "Finetune/Step Time": 5.529632177203894} +{"Finetune/Learning Rate": 1.874463619454466e-05, "Finetune/Loss": 0.9586837887763977, "Finetune/Loss (Raw)": 1.0619466304779053, "Finetune/Step": 968, "Finetune/Step Time": 5.5278716422617435} +{"Finetune/Learning Rate": 1.874161256922441e-05, "Finetune/Loss": 0.9589571952819824, "Finetune/Loss (Raw)": 0.9330783486366272, "Finetune/Step": 969, "Finetune/Step Time": 5.539835376664996} +{"Finetune/Learning Rate": 1.8738585551454982e-05, "Finetune/Loss": 0.9561231732368469, "Finetune/Loss (Raw)": 0.707710862159729, "Finetune/Step": 970, "Finetune/Step Time": 5.547412756830454} +{"Finetune/Learning Rate": 1.873555514241111e-05, "Finetune/Loss": 0.955765962600708, "Finetune/Loss (Raw)": 1.0049691200256348, "Finetune/Step": 971, "Finetune/Step Time": 5.560418916866183} +{"Finetune/Learning Rate": 1.8732521343268838e-05, "Finetune/Loss": 0.9510668516159058, "Finetune/Loss (Raw)": 0.39541304111480713, "Finetune/Step": 972, "Finetune/Step Time": 5.589917920529842} +{"Finetune/Learning Rate": 1.8729484155205517e-05, "Finetune/Loss": 0.9495958685874939, "Finetune/Loss (Raw)": 0.890916109085083, "Finetune/Step": 973, "Finetune/Step Time": 5.59524804726243} +{"Finetune/Learning Rate": 1.8726443579399828e-05, "Finetune/Loss": 0.9490082859992981, "Finetune/Loss (Raw)": 0.9683544039726257, "Finetune/Step": 974, "Finetune/Step Time": 5.610709644854069} +{"Finetune/Learning Rate": 1.8723399617031754e-05, "Finetune/Loss": 0.9540534019470215, "Finetune/Loss (Raw)": 1.111440658569336, "Finetune/Step": 975, "Finetune/Step Time": 5.592273747548461} +{"Finetune/Learning Rate": 1.8720352269282592e-05, "Finetune/Loss": 0.9551138877868652, "Finetune/Loss (Raw)": 0.9854990243911743, "Finetune/Step": 976, "Finetune/Step Time": 5.594648679718375} +{"Finetune/Learning Rate": 1.8717301537334975e-05, "Finetune/Loss": 0.9556278586387634, "Finetune/Loss (Raw)": 1.0758603811264038, "Finetune/Step": 977, "Finetune/Step Time": 5.600183978676796} +{"Finetune/Learning Rate": 1.8714247422372816e-05, "Finetune/Loss": 0.9560569524765015, "Finetune/Loss (Raw)": 0.9474515318870544, "Finetune/Step": 978, "Finetune/Step Time": 5.585183687508106} +{"Finetune/Learning Rate": 1.8711189925581366e-05, "Finetune/Loss": 0.9549919962882996, "Finetune/Loss (Raw)": 0.8847652077674866, "Finetune/Step": 979, "Finetune/Step Time": 5.58060809597373} +{"Finetune/Learning Rate": 1.8708129048147176e-05, "Finetune/Loss": 0.9571841359138489, "Finetune/Loss (Raw)": 1.163917899131775, "Finetune/Step": 980, "Finetune/Step Time": 5.578998984768987} +{"Finetune/Learning Rate": 1.8705064791258118e-05, "Finetune/Loss": 0.9574860334396362, "Finetune/Loss (Raw)": 0.9933233857154846, "Finetune/Step": 981, "Finetune/Step Time": 5.574350515380502} +{"Finetune/Learning Rate": 1.870199715610336e-05, "Finetune/Loss": 0.9581658244132996, "Finetune/Loss (Raw)": 1.0285803079605103, "Finetune/Step": 982, "Finetune/Step Time": 5.562485435977578} +{"Finetune/Learning Rate": 1.8698926143873404e-05, "Finetune/Loss": 0.9599288702011108, "Finetune/Loss (Raw)": 1.0226258039474487, "Finetune/Step": 983, "Finetune/Step Time": 5.531385824084282} +{"Finetune/Learning Rate": 1.8695851755760042e-05, "Finetune/Loss": 0.9598338603973389, "Finetune/Loss (Raw)": 0.9804624319076538, "Finetune/Step": 984, "Finetune/Step Time": 5.542238445952535} +{"Finetune/Learning Rate": 1.8692773992956383e-05, "Finetune/Loss": 0.9598535299301147, "Finetune/Loss (Raw)": 0.9515353441238403, "Finetune/Step": 985, "Finetune/Step Time": 5.540795659646392} +{"Finetune/Learning Rate": 1.8689692856656854e-05, "Finetune/Loss": 0.9574871063232422, "Finetune/Loss (Raw)": 0.8858680129051208, "Finetune/Step": 986, "Finetune/Step Time": 5.544165940955281} +{"Finetune/Learning Rate": 1.8686608348057178e-05, "Finetune/Loss": 0.956744372844696, "Finetune/Loss (Raw)": 1.0110158920288086, "Finetune/Step": 987, "Finetune/Step Time": 5.541317189112306} +{"Finetune/Learning Rate": 1.868352046835439e-05, "Finetune/Loss": 0.9560838937759399, "Finetune/Loss (Raw)": 0.8670253157615662, "Finetune/Step": 988, "Finetune/Step Time": 5.54404478892684} +{"Finetune/Learning Rate": 1.8680429218746842e-05, "Finetune/Loss": 0.9584865570068359, "Finetune/Loss (Raw)": 1.0360976457595825, "Finetune/Step": 989, "Finetune/Step Time": 5.547318315133452} +{"Finetune/Learning Rate": 1.867733460043419e-05, "Finetune/Loss": 0.9564234614372253, "Finetune/Loss (Raw)": 0.754381537437439, "Finetune/Step": 990, "Finetune/Step Time": 5.552805168554187} +{"Finetune/Learning Rate": 1.8674236614617386e-05, "Finetune/Loss": 0.9579852223396301, "Finetune/Loss (Raw)": 1.0488251447677612, "Finetune/Step": 991, "Finetune/Step Time": 5.543162589892745} +{"Finetune/Learning Rate": 1.86711352624987e-05, "Finetune/Loss": 0.9575346112251282, "Finetune/Loss (Raw)": 0.9828497767448425, "Finetune/Step": 992, "Finetune/Step Time": 5.540753003209829} +{"Finetune/Learning Rate": 1.866803054528171e-05, "Finetune/Loss": 0.9577955007553101, "Finetune/Loss (Raw)": 0.9924139976501465, "Finetune/Step": 993, "Finetune/Step Time": 5.551387500017881} +{"Finetune/Learning Rate": 1.8664922464171295e-05, "Finetune/Loss": 0.9550889730453491, "Finetune/Loss (Raw)": 0.7750037908554077, "Finetune/Step": 994, "Finetune/Step Time": 5.559519900009036} +{"Finetune/Learning Rate": 1.866181102037364e-05, "Finetune/Loss": 0.9541558027267456, "Finetune/Loss (Raw)": 0.9442246556282043, "Finetune/Step": 995, "Finetune/Step Time": 5.558168608695269} +{"Finetune/Learning Rate": 1.8658696215096235e-05, "Finetune/Loss": 0.9543287754058838, "Finetune/Loss (Raw)": 0.9671469330787659, "Finetune/Step": 996, "Finetune/Step Time": 5.561883797869086} +{"Finetune/Learning Rate": 1.8655578049547878e-05, "Finetune/Loss": 0.9531277418136597, "Finetune/Loss (Raw)": 0.9372463226318359, "Finetune/Step": 997, "Finetune/Step Time": 5.541647415608168} +{"Finetune/Learning Rate": 1.8652456524938665e-05, "Finetune/Loss": 0.9535812139511108, "Finetune/Loss (Raw)": 0.961298942565918, "Finetune/Step": 998, "Finetune/Step Time": 5.554607147350907} +{"Finetune/Learning Rate": 1.864933164248e-05, "Finetune/Loss": 0.9556711316108704, "Finetune/Loss (Raw)": 1.1208738088607788, "Finetune/Step": 999, "Finetune/Step Time": 5.547780070453882} +{"Finetune/Learning Rate": 1.8646203403384583e-05, "Finetune/Loss": 0.9567956924438477, "Finetune/Loss (Raw)": 1.143601655960083, "Finetune/Step": 1000, "Finetune/Step Time": 5.541447930037975} +{"Finetune/Learning Rate": 1.8643071808866434e-05, "Finetune/Loss": 0.9512094259262085, "Finetune/Loss (Raw)": 0.3278841972351074, "Finetune/Step": 1001, "Finetune/Step Time": 5.575628656893969} +{"Finetune/Learning Rate": 1.8639936860140856e-05, "Finetune/Loss": 0.9520056843757629, "Finetune/Loss (Raw)": 0.983919084072113, "Finetune/Step": 1002, "Finetune/Step Time": 5.57590638846159} +{"Finetune/Learning Rate": 1.8636798558424464e-05, "Finetune/Loss": 0.9528481960296631, "Finetune/Loss (Raw)": 1.054376482963562, "Finetune/Step": 1003, "Finetune/Step Time": 5.578917836770415} +{"Finetune/Learning Rate": 1.8633656904935168e-05, "Finetune/Loss": 0.9538471102714539, "Finetune/Loss (Raw)": 1.0672820806503296, "Finetune/Step": 1004, "Finetune/Step Time": 5.578992849215865} +{"Finetune/Learning Rate": 1.863051190089219e-05, "Finetune/Loss": 0.9537537097930908, "Finetune/Loss (Raw)": 1.0210131406784058, "Finetune/Step": 1005, "Finetune/Step Time": 5.5714228339493275} +{"Finetune/Learning Rate": 1.8627363547516032e-05, "Finetune/Loss": 0.9528306126594543, "Finetune/Loss (Raw)": 0.876747190952301, "Finetune/Step": 1006, "Finetune/Step Time": 5.58757015503943} +{"Finetune/Learning Rate": 1.8624211846028522e-05, "Finetune/Loss": 0.9531583189964294, "Finetune/Loss (Raw)": 0.9362167119979858, "Finetune/Step": 1007, "Finetune/Step Time": 5.568713705986738} +{"Finetune/Learning Rate": 1.862105679765277e-05, "Finetune/Loss": 0.9528864026069641, "Finetune/Loss (Raw)": 0.9854170680046082, "Finetune/Step": 1008, "Finetune/Step Time": 5.56812752597034} +{"Finetune/Learning Rate": 1.8617898403613183e-05, "Finetune/Loss": 0.9492952823638916, "Finetune/Loss (Raw)": 0.4296268820762634, "Finetune/Step": 1009, "Finetune/Step Time": 5.5804444048553705} +{"Finetune/Learning Rate": 1.8614736665135476e-05, "Finetune/Loss": 0.9480085372924805, "Finetune/Loss (Raw)": 0.9037494659423828, "Finetune/Step": 1010, "Finetune/Step Time": 5.582600694149733} +{"Finetune/Learning Rate": 1.861157158344666e-05, "Finetune/Loss": 0.946927547454834, "Finetune/Loss (Raw)": 0.9223387241363525, "Finetune/Step": 1011, "Finetune/Step Time": 5.581924393773079} +{"Finetune/Learning Rate": 1.8608403159775037e-05, "Finetune/Loss": 0.9464793801307678, "Finetune/Loss (Raw)": 1.0153210163116455, "Finetune/Step": 1012, "Finetune/Step Time": 5.592168025672436} +{"Finetune/Learning Rate": 1.8605231395350213e-05, "Finetune/Loss": 0.951859176158905, "Finetune/Loss (Raw)": 0.9181011915206909, "Finetune/Step": 1013, "Finetune/Step Time": 5.5609169490635395} +{"Finetune/Learning Rate": 1.860205629140309e-05, "Finetune/Loss": 0.9520982503890991, "Finetune/Loss (Raw)": 1.0465216636657715, "Finetune/Step": 1014, "Finetune/Step Time": 5.55607845634222} +{"Finetune/Learning Rate": 1.8598877849165857e-05, "Finetune/Loss": 0.9513859748840332, "Finetune/Loss (Raw)": 0.3133176267147064, "Finetune/Step": 1015, "Finetune/Step Time": 5.5581401493400335} +{"Finetune/Learning Rate": 1.8595696069872013e-05, "Finetune/Loss": 0.9507780075073242, "Finetune/Loss (Raw)": 1.008743166923523, "Finetune/Step": 1016, "Finetune/Step Time": 5.556388357654214} +{"Finetune/Learning Rate": 1.859251095475634e-05, "Finetune/Loss": 0.9510581493377686, "Finetune/Loss (Raw)": 1.0814908742904663, "Finetune/Step": 1017, "Finetune/Step Time": 5.5477222464978695} +{"Finetune/Learning Rate": 1.858932250505492e-05, "Finetune/Loss": 0.9512727856636047, "Finetune/Loss (Raw)": 1.0011494159698486, "Finetune/Step": 1018, "Finetune/Step Time": 5.586757220327854} +{"Finetune/Learning Rate": 1.8586130722005128e-05, "Finetune/Loss": 0.9507371187210083, "Finetune/Loss (Raw)": 0.9223136305809021, "Finetune/Step": 1019, "Finetune/Step Time": 5.5847001280635595} +{"Finetune/Learning Rate": 1.8582935606845636e-05, "Finetune/Loss": 0.9514979124069214, "Finetune/Loss (Raw)": 0.9883541464805603, "Finetune/Step": 1020, "Finetune/Step Time": 5.5803914796561} +{"Finetune/Learning Rate": 1.8579737160816397e-05, "Finetune/Loss": 0.9494878053665161, "Finetune/Loss (Raw)": 0.8688320517539978, "Finetune/Step": 1021, "Finetune/Step Time": 5.591670874506235} +{"Finetune/Learning Rate": 1.8576535385158675e-05, "Finetune/Loss": 0.9498579502105713, "Finetune/Loss (Raw)": 1.0226824283599854, "Finetune/Step": 1022, "Finetune/Step Time": 5.597897846251726} +{"Finetune/Learning Rate": 1.8573330281115008e-05, "Finetune/Loss": 0.9488831758499146, "Finetune/Loss (Raw)": 0.8527313470840454, "Finetune/Step": 1023, "Finetune/Step Time": 5.596555586904287} +{"Finetune/Learning Rate": 1.8570121849929245e-05, "Finetune/Loss": 0.9478403925895691, "Finetune/Loss (Raw)": 0.9328843951225281, "Finetune/Step": 1024, "Finetune/Step Time": 5.598004445433617} +{"Finetune/Learning Rate": 1.8566910092846508e-05, "Finetune/Loss": 0.9480361938476562, "Finetune/Loss (Raw)": 0.8846287131309509, "Finetune/Step": 1025, "Finetune/Step Time": 5.588087540119886} +{"Finetune/Learning Rate": 1.8563695011113217e-05, "Finetune/Loss": 0.9532113075256348, "Finetune/Loss (Raw)": 1.0230965614318848, "Finetune/Step": 1026, "Finetune/Step Time": 5.566126182675362} +{"Finetune/Learning Rate": 1.8560476605977085e-05, "Finetune/Loss": 0.9496818780899048, "Finetune/Loss (Raw)": 0.37360531091690063, "Finetune/Step": 1027, "Finetune/Step Time": 5.508696153759956} +{"Finetune/Learning Rate": 1.8557254878687114e-05, "Finetune/Loss": 0.9492202997207642, "Finetune/Loss (Raw)": 1.0529447793960571, "Finetune/Step": 1028, "Finetune/Step Time": 5.507291862741113} +{"Finetune/Learning Rate": 1.8554029830493587e-05, "Finetune/Loss": 0.9474751949310303, "Finetune/Loss (Raw)": 0.8793216943740845, "Finetune/Step": 1029, "Finetune/Step Time": 5.52483269944787} +{"Finetune/Learning Rate": 1.855080146264809e-05, "Finetune/Loss": 0.9472715854644775, "Finetune/Loss (Raw)": 1.0340863466262817, "Finetune/Step": 1030, "Finetune/Step Time": 5.523421252146363} +{"Finetune/Learning Rate": 1.854756977640348e-05, "Finetune/Loss": 0.9465791583061218, "Finetune/Loss (Raw)": 0.9924725890159607, "Finetune/Step": 1031, "Finetune/Step Time": 5.503581197932363} +{"Finetune/Learning Rate": 1.8544334773013924e-05, "Finetune/Loss": 0.9418049454689026, "Finetune/Loss (Raw)": 0.3502351641654968, "Finetune/Step": 1032, "Finetune/Step Time": 5.537055518478155} +{"Finetune/Learning Rate": 1.8541096453734852e-05, "Finetune/Loss": 0.9414036273956299, "Finetune/Loss (Raw)": 1.0107871294021606, "Finetune/Step": 1033, "Finetune/Step Time": 5.558091251179576} +{"Finetune/Learning Rate": 1.8537854819822998e-05, "Finetune/Loss": 0.9406359195709229, "Finetune/Loss (Raw)": 0.8424713611602783, "Finetune/Step": 1034, "Finetune/Step Time": 5.565635206177831} +{"Finetune/Learning Rate": 1.853460987253638e-05, "Finetune/Loss": 0.940284013748169, "Finetune/Loss (Raw)": 1.008060097694397, "Finetune/Step": 1035, "Finetune/Step Time": 5.578031808137894} +{"Finetune/Learning Rate": 1.8531361613134286e-05, "Finetune/Loss": 0.9402666687965393, "Finetune/Loss (Raw)": 0.973802387714386, "Finetune/Step": 1036, "Finetune/Step Time": 5.582935338839889} +{"Finetune/Learning Rate": 1.8528110042877316e-05, "Finetune/Loss": 0.9393556118011475, "Finetune/Loss (Raw)": 0.8824963569641113, "Finetune/Step": 1037, "Finetune/Step Time": 5.585956649854779} +{"Finetune/Learning Rate": 1.8524855163027337e-05, "Finetune/Loss": 0.9408807754516602, "Finetune/Loss (Raw)": 1.1678264141082764, "Finetune/Step": 1038, "Finetune/Step Time": 5.579311029985547} +{"Finetune/Learning Rate": 1.85215969748475e-05, "Finetune/Loss": 0.941497802734375, "Finetune/Loss (Raw)": 1.0155034065246582, "Finetune/Step": 1039, "Finetune/Step Time": 5.578978143632412} +{"Finetune/Learning Rate": 1.851833547960225e-05, "Finetune/Loss": 0.9400120973587036, "Finetune/Loss (Raw)": 0.9324761033058167, "Finetune/Step": 1040, "Finetune/Step Time": 5.574973221868277} +{"Finetune/Learning Rate": 1.8515070678557304e-05, "Finetune/Loss": 0.942844033241272, "Finetune/Loss (Raw)": 1.0853816270828247, "Finetune/Step": 1041, "Finetune/Step Time": 5.556641044095159} +{"Finetune/Learning Rate": 1.8511802572979672e-05, "Finetune/Loss": 0.942518949508667, "Finetune/Loss (Raw)": 1.0833433866500854, "Finetune/Step": 1042, "Finetune/Step Time": 5.557663030922413} +{"Finetune/Learning Rate": 1.8508531164137644e-05, "Finetune/Loss": 0.9375852346420288, "Finetune/Loss (Raw)": 0.38375064730644226, "Finetune/Step": 1043, "Finetune/Step Time": 5.59590719640255} +{"Finetune/Learning Rate": 1.8505256453300778e-05, "Finetune/Loss": 0.9362043142318726, "Finetune/Loss (Raw)": 0.8588843941688538, "Finetune/Step": 1044, "Finetune/Step Time": 5.596005082130432} +{"Finetune/Learning Rate": 1.850197844173994e-05, "Finetune/Loss": 0.9351764917373657, "Finetune/Loss (Raw)": 1.058717966079712, "Finetune/Step": 1045, "Finetune/Step Time": 5.5642557721585035} +{"Finetune/Learning Rate": 1.8498697130727255e-05, "Finetune/Loss": 0.9349623322486877, "Finetune/Loss (Raw)": 0.949309766292572, "Finetune/Step": 1046, "Finetune/Step Time": 5.566959964111447} +{"Finetune/Learning Rate": 1.8495412521536135e-05, "Finetune/Loss": 0.9368150234222412, "Finetune/Loss (Raw)": 1.0014252662658691, "Finetune/Step": 1047, "Finetune/Step Time": 5.553795797750354} +{"Finetune/Learning Rate": 1.8492124615441276e-05, "Finetune/Loss": 0.9393823146820068, "Finetune/Loss (Raw)": 1.1739634275436401, "Finetune/Step": 1048, "Finetune/Step Time": 5.545634170994163} +{"Finetune/Learning Rate": 1.8488833413718646e-05, "Finetune/Loss": 0.9385336637496948, "Finetune/Loss (Raw)": 0.8803184032440186, "Finetune/Step": 1049, "Finetune/Step Time": 5.54322912171483} +{"Finetune/Learning Rate": 1.84855389176455e-05, "Finetune/Loss": 0.9376872777938843, "Finetune/Loss (Raw)": 0.8148438930511475, "Finetune/Step": 1050, "Finetune/Step Time": 5.555333983153105} +{"Finetune/Learning Rate": 1.8482241128500366e-05, "Finetune/Loss": 0.9370126128196716, "Finetune/Loss (Raw)": 0.9516491889953613, "Finetune/Step": 1051, "Finetune/Step Time": 5.559106629341841} +{"Finetune/Learning Rate": 1.847894004756305e-05, "Finetune/Loss": 0.9387089610099792, "Finetune/Loss (Raw)": 1.153769850730896, "Finetune/Step": 1052, "Finetune/Step Time": 5.5572952050715685} +{"Finetune/Learning Rate": 1.8475635676114643e-05, "Finetune/Loss": 0.9400495290756226, "Finetune/Loss (Raw)": 0.9860239624977112, "Finetune/Step": 1053, "Finetune/Step Time": 5.549565734341741} +{"Finetune/Learning Rate": 1.8472328015437503e-05, "Finetune/Loss": 0.9413827061653137, "Finetune/Loss (Raw)": 1.1187148094177246, "Finetune/Step": 1054, "Finetune/Step Time": 5.544633250683546} +{"Finetune/Learning Rate": 1.8469017066815265e-05, "Finetune/Loss": 0.9402822256088257, "Finetune/Loss (Raw)": 0.9023594260215759, "Finetune/Step": 1055, "Finetune/Step Time": 5.54578335210681} +{"Finetune/Learning Rate": 1.8465702831532848e-05, "Finetune/Loss": 0.9409269094467163, "Finetune/Loss (Raw)": 1.0929248332977295, "Finetune/Step": 1056, "Finetune/Step Time": 5.547115618363023} +{"Finetune/Learning Rate": 1.8462385310876444e-05, "Finetune/Loss": 0.9400447010993958, "Finetune/Loss (Raw)": 0.920088529586792, "Finetune/Step": 1057, "Finetune/Step Time": 5.541813915595412} +{"Finetune/Learning Rate": 1.845906450613351e-05, "Finetune/Loss": 0.9402545690536499, "Finetune/Loss (Raw)": 0.9317847490310669, "Finetune/Step": 1058, "Finetune/Step Time": 5.534243227913976} +{"Finetune/Learning Rate": 1.8455740418592793e-05, "Finetune/Loss": 0.9396075010299683, "Finetune/Loss (Raw)": 0.9468852877616882, "Finetune/Step": 1059, "Finetune/Step Time": 5.552726317197084} +{"Finetune/Learning Rate": 1.8452413049544304e-05, "Finetune/Loss": 0.9389398097991943, "Finetune/Loss (Raw)": 0.9791261553764343, "Finetune/Step": 1060, "Finetune/Step Time": 5.550857443362474} +{"Finetune/Learning Rate": 1.844908240027933e-05, "Finetune/Loss": 0.9373916983604431, "Finetune/Loss (Raw)": 0.8482286334037781, "Finetune/Step": 1061, "Finetune/Step Time": 5.531548663973808} +{"Finetune/Learning Rate": 1.8445748472090432e-05, "Finetune/Loss": 0.9372230172157288, "Finetune/Loss (Raw)": 0.9924598336219788, "Finetune/Step": 1062, "Finetune/Step Time": 5.52700318582356} +{"Finetune/Learning Rate": 1.844241126627144e-05, "Finetune/Loss": 0.938166618347168, "Finetune/Loss (Raw)": 0.9828869104385376, "Finetune/Step": 1063, "Finetune/Step Time": 5.534716689959168} +{"Finetune/Learning Rate": 1.843907078411746e-05, "Finetune/Loss": 0.9400490522384644, "Finetune/Loss (Raw)": 1.1871784925460815, "Finetune/Step": 1064, "Finetune/Step Time": 5.52209303714335} +{"Finetune/Learning Rate": 1.8435727026924864e-05, "Finetune/Loss": 0.9401332139968872, "Finetune/Loss (Raw)": 1.141035795211792, "Finetune/Step": 1065, "Finetune/Step Time": 5.523571835830808} +{"Finetune/Learning Rate": 1.8432379995991307e-05, "Finetune/Loss": 0.9383305907249451, "Finetune/Loss (Raw)": 0.9061251878738403, "Finetune/Step": 1066, "Finetune/Step Time": 5.517606334760785} +{"Finetune/Learning Rate": 1.8429029692615702e-05, "Finetune/Loss": 0.9397442936897278, "Finetune/Loss (Raw)": 1.1111743450164795, "Finetune/Step": 1067, "Finetune/Step Time": 5.52455703727901} +{"Finetune/Learning Rate": 1.8425676118098236e-05, "Finetune/Loss": 0.9388279318809509, "Finetune/Loss (Raw)": 0.9055812954902649, "Finetune/Step": 1068, "Finetune/Step Time": 5.528987927362323} +{"Finetune/Learning Rate": 1.8422319273740364e-05, "Finetune/Loss": 0.9388080835342407, "Finetune/Loss (Raw)": 0.9731176495552063, "Finetune/Step": 1069, "Finetune/Step Time": 5.555269081145525} +{"Finetune/Learning Rate": 1.8418959160844813e-05, "Finetune/Loss": 0.9395058155059814, "Finetune/Loss (Raw)": 1.0298815965652466, "Finetune/Step": 1070, "Finetune/Step Time": 5.544215802103281} +{"Finetune/Learning Rate": 1.8415595780715583e-05, "Finetune/Loss": 0.9402318000793457, "Finetune/Loss (Raw)": 1.042136549949646, "Finetune/Step": 1071, "Finetune/Step Time": 5.542248856276274} +{"Finetune/Learning Rate": 1.8412229134657926e-05, "Finetune/Loss": 0.940590500831604, "Finetune/Loss (Raw)": 1.04572331905365, "Finetune/Step": 1072, "Finetune/Step Time": 5.55199852026999} +{"Finetune/Learning Rate": 1.8408859223978378e-05, "Finetune/Loss": 0.9413025379180908, "Finetune/Loss (Raw)": 0.9613409638404846, "Finetune/Step": 1073, "Finetune/Step Time": 5.540962710976601} +{"Finetune/Learning Rate": 1.840548604998474e-05, "Finetune/Loss": 0.9424487948417664, "Finetune/Loss (Raw)": 1.0520169734954834, "Finetune/Step": 1074, "Finetune/Step Time": 5.526249324902892} +{"Finetune/Learning Rate": 1.840210961398606e-05, "Finetune/Loss": 0.9431081414222717, "Finetune/Loss (Raw)": 1.0543923377990723, "Finetune/Step": 1075, "Finetune/Step Time": 5.5209826324135065} +{"Finetune/Learning Rate": 1.8398729917292683e-05, "Finetune/Loss": 0.9484838843345642, "Finetune/Loss (Raw)": 1.075236439704895, "Finetune/Step": 1076, "Finetune/Step Time": 5.4861676171422005} +{"Finetune/Learning Rate": 1.8395346961216193e-05, "Finetune/Loss": 0.9481632709503174, "Finetune/Loss (Raw)": 1.0935693979263306, "Finetune/Step": 1077, "Finetune/Step Time": 5.490302266553044} +{"Finetune/Learning Rate": 1.8391960747069453e-05, "Finetune/Loss": 0.9493699073791504, "Finetune/Loss (Raw)": 1.0644760131835938, "Finetune/Step": 1078, "Finetune/Step Time": 5.492563670501113} +{"Finetune/Learning Rate": 1.8388571276166588e-05, "Finetune/Loss": 0.9469202160835266, "Finetune/Loss (Raw)": 0.8722589612007141, "Finetune/Step": 1079, "Finetune/Step Time": 5.496676314622164} +{"Finetune/Learning Rate": 1.8385178549822982e-05, "Finetune/Loss": 0.9478566646575928, "Finetune/Loss (Raw)": 0.9743132591247559, "Finetune/Step": 1080, "Finetune/Step Time": 5.496375985443592} +{"Finetune/Learning Rate": 1.8381782569355288e-05, "Finetune/Loss": 0.9486076831817627, "Finetune/Loss (Raw)": 1.0808161497116089, "Finetune/Step": 1081, "Finetune/Step Time": 5.499975115060806} +{"Finetune/Learning Rate": 1.837838333608142e-05, "Finetune/Loss": 0.9481104612350464, "Finetune/Loss (Raw)": 0.8530986309051514, "Finetune/Step": 1082, "Finetune/Step Time": 5.520685901865363} +{"Finetune/Learning Rate": 1.8374980851320555e-05, "Finetune/Loss": 0.9483335614204407, "Finetune/Loss (Raw)": 1.0212323665618896, "Finetune/Step": 1083, "Finetune/Step Time": 5.519359966740012} +{"Finetune/Learning Rate": 1.8371575116393125e-05, "Finetune/Loss": 0.9486683011054993, "Finetune/Loss (Raw)": 1.075204849243164, "Finetune/Step": 1084, "Finetune/Step Time": 5.533839579671621} +{"Finetune/Learning Rate": 1.8368166132620838e-05, "Finetune/Loss": 0.9495699405670166, "Finetune/Loss (Raw)": 1.0188734531402588, "Finetune/Step": 1085, "Finetune/Step Time": 5.5239827409386635} +{"Finetune/Learning Rate": 1.8364753901326647e-05, "Finetune/Loss": 0.9500408172607422, "Finetune/Loss (Raw)": 1.0380462408065796, "Finetune/Step": 1086, "Finetune/Step Time": 5.510530818253756} +{"Finetune/Learning Rate": 1.8361338423834776e-05, "Finetune/Loss": 0.9507074356079102, "Finetune/Loss (Raw)": 1.020914912223816, "Finetune/Step": 1087, "Finetune/Step Time": 5.512977942824364} +{"Finetune/Learning Rate": 1.8357919701470702e-05, "Finetune/Loss": 0.9498717784881592, "Finetune/Loss (Raw)": 0.8491007089614868, "Finetune/Step": 1088, "Finetune/Step Time": 5.52642715908587} +{"Finetune/Learning Rate": 1.835449773556117e-05, "Finetune/Loss": 0.9508291482925415, "Finetune/Loss (Raw)": 1.006035566329956, "Finetune/Step": 1089, "Finetune/Step Time": 5.522532491013408} +{"Finetune/Learning Rate": 1.8351072527434173e-05, "Finetune/Loss": 0.9506987929344177, "Finetune/Loss (Raw)": 1.0949864387512207, "Finetune/Step": 1090, "Finetune/Step Time": 5.532581960782409} +{"Finetune/Learning Rate": 1.8347644078418968e-05, "Finetune/Loss": 0.9509237408638, "Finetune/Loss (Raw)": 0.9416211247444153, "Finetune/Step": 1091, "Finetune/Step Time": 5.531283875927329} +{"Finetune/Learning Rate": 1.834421238984607e-05, "Finetune/Loss": 0.9517609477043152, "Finetune/Loss (Raw)": 1.038820743560791, "Finetune/Step": 1092, "Finetune/Step Time": 5.535921715199947} +{"Finetune/Learning Rate": 1.8340777463047253e-05, "Finetune/Loss": 0.9523696303367615, "Finetune/Loss (Raw)": 0.8875486254692078, "Finetune/Step": 1093, "Finetune/Step Time": 5.533324282616377} +{"Finetune/Learning Rate": 1.833733929935554e-05, "Finetune/Loss": 0.9527379274368286, "Finetune/Loss (Raw)": 0.9972752332687378, "Finetune/Step": 1094, "Finetune/Step Time": 5.522815542295575} +{"Finetune/Learning Rate": 1.833389790010522e-05, "Finetune/Loss": 0.9530490636825562, "Finetune/Loss (Raw)": 1.0397546291351318, "Finetune/Step": 1095, "Finetune/Step Time": 5.501382153481245} +{"Finetune/Learning Rate": 1.8330453266631828e-05, "Finetune/Loss": 0.9530867338180542, "Finetune/Loss (Raw)": 1.0667731761932373, "Finetune/Step": 1096, "Finetune/Step Time": 5.497346181422472} +{"Finetune/Learning Rate": 1.8327005400272165e-05, "Finetune/Loss": 0.9535658359527588, "Finetune/Loss (Raw)": 0.9944010972976685, "Finetune/Step": 1097, "Finetune/Step Time": 5.4934490621089935} +{"Finetune/Learning Rate": 1.8323554302364273e-05, "Finetune/Loss": 0.9557439088821411, "Finetune/Loss (Raw)": 0.9864967465400696, "Finetune/Step": 1098, "Finetune/Step Time": 5.500691512599587} +{"Finetune/Learning Rate": 1.8320099974247462e-05, "Finetune/Loss": 0.955714762210846, "Finetune/Loss (Raw)": 1.0012398958206177, "Finetune/Step": 1099, "Finetune/Step Time": 5.478842925280333} +{"Finetune/Learning Rate": 1.831664241726229e-05, "Finetune/Loss": 0.9601829051971436, "Finetune/Loss (Raw)": 0.9673431515693665, "Finetune/Step": 1100, "Finetune/Step Time": 5.452850919216871} +{"Finetune/Learning Rate": 1.831318163275056e-05, "Finetune/Loss": 0.9616952538490295, "Finetune/Loss (Raw)": 1.0844905376434326, "Finetune/Step": 1101, "Finetune/Step Time": 5.452406223863363} +{"Finetune/Learning Rate": 1.8309717622055343e-05, "Finetune/Loss": 0.9603943228721619, "Finetune/Loss (Raw)": 0.8018364906311035, "Finetune/Step": 1102, "Finetune/Step Time": 5.43888789601624} +{"Finetune/Learning Rate": 1.830625038652095e-05, "Finetune/Loss": 0.9605211019515991, "Finetune/Loss (Raw)": 1.1276675462722778, "Finetune/Step": 1103, "Finetune/Step Time": 5.425205007195473} +{"Finetune/Learning Rate": 1.8302779927492945e-05, "Finetune/Loss": 0.9611762762069702, "Finetune/Loss (Raw)": 1.069364309310913, "Finetune/Step": 1104, "Finetune/Step Time": 5.428695006296039} +{"Finetune/Learning Rate": 1.829930624631815e-05, "Finetune/Loss": 0.9614752531051636, "Finetune/Loss (Raw)": 1.1141239404678345, "Finetune/Step": 1105, "Finetune/Step Time": 5.434007748961449} +{"Finetune/Learning Rate": 1.829582934434463e-05, "Finetune/Loss": 0.961906909942627, "Finetune/Loss (Raw)": 1.00270676612854, "Finetune/Step": 1106, "Finetune/Step Time": 5.446162017062306} +{"Finetune/Learning Rate": 1.829234922292171e-05, "Finetune/Loss": 0.963005006313324, "Finetune/Loss (Raw)": 1.0253193378448486, "Finetune/Step": 1107, "Finetune/Step Time": 5.441238071769476} +{"Finetune/Learning Rate": 1.8288865883399946e-05, "Finetune/Loss": 0.9624707698822021, "Finetune/Loss (Raw)": 1.0955418348312378, "Finetune/Step": 1108, "Finetune/Step Time": 5.441711759194732} +{"Finetune/Learning Rate": 1.8285379327131157e-05, "Finetune/Loss": 0.9610904455184937, "Finetune/Loss (Raw)": 0.816633403301239, "Finetune/Step": 1109, "Finetune/Step Time": 5.453328305855393} +{"Finetune/Learning Rate": 1.828188955546841e-05, "Finetune/Loss": 0.9607871770858765, "Finetune/Loss (Raw)": 0.9897692203521729, "Finetune/Step": 1110, "Finetune/Step Time": 5.451814180240035} +{"Finetune/Learning Rate": 1.827839656976602e-05, "Finetune/Loss": 0.9616529941558838, "Finetune/Loss (Raw)": 1.133448600769043, "Finetune/Step": 1111, "Finetune/Step Time": 5.441737569868565} +{"Finetune/Learning Rate": 1.8274900371379543e-05, "Finetune/Loss": 0.9612350463867188, "Finetune/Loss (Raw)": 0.9269662499427795, "Finetune/Step": 1112, "Finetune/Step Time": 5.429607575759292} +{"Finetune/Learning Rate": 1.8271400961665784e-05, "Finetune/Loss": 0.9612659215927124, "Finetune/Loss (Raw)": 0.9554851651191711, "Finetune/Step": 1113, "Finetune/Step Time": 5.426097245886922} +{"Finetune/Learning Rate": 1.8267898341982796e-05, "Finetune/Loss": 0.9624375104904175, "Finetune/Loss (Raw)": 1.0358269214630127, "Finetune/Step": 1114, "Finetune/Step Time": 5.454865841194987} +{"Finetune/Learning Rate": 1.8264392513689882e-05, "Finetune/Loss": 0.9625129699707031, "Finetune/Loss (Raw)": 1.0206794738769531, "Finetune/Step": 1115, "Finetune/Step Time": 5.4522011280059814} +{"Finetune/Learning Rate": 1.8260883478147578e-05, "Finetune/Loss": 0.9642925262451172, "Finetune/Loss (Raw)": 1.0948017835617065, "Finetune/Step": 1116, "Finetune/Step Time": 5.446831621229649} +{"Finetune/Learning Rate": 1.8257371236717682e-05, "Finetune/Loss": 0.9631181955337524, "Finetune/Loss (Raw)": 0.8857957124710083, "Finetune/Step": 1117, "Finetune/Step Time": 5.446367267519236} +{"Finetune/Learning Rate": 1.8253855790763215e-05, "Finetune/Loss": 0.9657779932022095, "Finetune/Loss (Raw)": 1.0948288440704346, "Finetune/Step": 1118, "Finetune/Step Time": 5.438608547672629} +{"Finetune/Learning Rate": 1.8250337141648464e-05, "Finetune/Loss": 0.9633212089538574, "Finetune/Loss (Raw)": 0.7343598008155823, "Finetune/Step": 1119, "Finetune/Step Time": 5.448397029191256} +{"Finetune/Learning Rate": 1.8246815290738937e-05, "Finetune/Loss": 0.9632762670516968, "Finetune/Loss (Raw)": 0.9770969748497009, "Finetune/Step": 1120, "Finetune/Step Time": 5.451273050159216} +{"Finetune/Learning Rate": 1.8243290239401402e-05, "Finetune/Loss": 0.9645459651947021, "Finetune/Loss (Raw)": 1.1549264192581177, "Finetune/Step": 1121, "Finetune/Step Time": 5.420339936390519} +{"Finetune/Learning Rate": 1.8239761989003866e-05, "Finetune/Loss": 0.9666510820388794, "Finetune/Loss (Raw)": 1.0444631576538086, "Finetune/Step": 1122, "Finetune/Step Time": 5.4266041070222855} +{"Finetune/Learning Rate": 1.8236230540915567e-05, "Finetune/Loss": 0.9666375517845154, "Finetune/Loss (Raw)": 0.9424960613250732, "Finetune/Step": 1123, "Finetune/Step Time": 5.444734295830131} +{"Finetune/Learning Rate": 1.8232695896506995e-05, "Finetune/Loss": 0.9687089920043945, "Finetune/Loss (Raw)": 1.2322885990142822, "Finetune/Step": 1124, "Finetune/Step Time": 5.4409814942628145} +{"Finetune/Learning Rate": 1.8229158057149875e-05, "Finetune/Loss": 0.9687857627868652, "Finetune/Loss (Raw)": 0.94707852602005, "Finetune/Step": 1125, "Finetune/Step Time": 5.4373299311846495} +{"Finetune/Learning Rate": 1.8225617024217174e-05, "Finetune/Loss": 0.968965470790863, "Finetune/Loss (Raw)": 0.9843001961708069, "Finetune/Step": 1126, "Finetune/Step Time": 5.419156359508634} +{"Finetune/Learning Rate": 1.8222072799083096e-05, "Finetune/Loss": 0.9681921005249023, "Finetune/Loss (Raw)": 1.0218756198883057, "Finetune/Step": 1127, "Finetune/Step Time": 5.418617084622383} +{"Finetune/Learning Rate": 1.821852538312309e-05, "Finetune/Loss": 0.9669003486633301, "Finetune/Loss (Raw)": 0.9782571196556091, "Finetune/Step": 1128, "Finetune/Step Time": 5.430701047182083} +{"Finetune/Learning Rate": 1.8214974777713834e-05, "Finetune/Loss": 0.9730439782142639, "Finetune/Loss (Raw)": 1.114270806312561, "Finetune/Step": 1129, "Finetune/Step Time": 5.393711566925049} +{"Finetune/Learning Rate": 1.821142098423325e-05, "Finetune/Loss": 0.9715858697891235, "Finetune/Loss (Raw)": 0.7972841858863831, "Finetune/Step": 1130, "Finetune/Step Time": 5.402119068428874} +{"Finetune/Learning Rate": 1.8207864004060497e-05, "Finetune/Loss": 0.9718274474143982, "Finetune/Loss (Raw)": 1.0852973461151123, "Finetune/Step": 1131, "Finetune/Step Time": 5.384509552270174} +{"Finetune/Learning Rate": 1.820430383857597e-05, "Finetune/Loss": 0.9715324640274048, "Finetune/Loss (Raw)": 1.0295217037200928, "Finetune/Step": 1132, "Finetune/Step Time": 5.386103538796306} +{"Finetune/Learning Rate": 1.8200740489161296e-05, "Finetune/Loss": 0.9712566137313843, "Finetune/Loss (Raw)": 0.9857127070426941, "Finetune/Step": 1133, "Finetune/Step Time": 5.381399592384696} +{"Finetune/Learning Rate": 1.819717395719935e-05, "Finetune/Loss": 0.9718687534332275, "Finetune/Loss (Raw)": 0.9550923109054565, "Finetune/Step": 1134, "Finetune/Step Time": 5.3631441574543715} +{"Finetune/Learning Rate": 1.819360424407422e-05, "Finetune/Loss": 0.9731813669204712, "Finetune/Loss (Raw)": 1.1042348146438599, "Finetune/Step": 1135, "Finetune/Step Time": 5.358335509896278} +{"Finetune/Learning Rate": 1.8190031351171256e-05, "Finetune/Loss": 0.9731225967407227, "Finetune/Loss (Raw)": 0.9778941869735718, "Finetune/Step": 1136, "Finetune/Step Time": 5.355566209182143} +{"Finetune/Learning Rate": 1.818645527987702e-05, "Finetune/Loss": 0.9779435396194458, "Finetune/Loss (Raw)": 1.046706199645996, "Finetune/Step": 1137, "Finetune/Step Time": 5.325234245508909} +{"Finetune/Learning Rate": 1.8182876031579312e-05, "Finetune/Loss": 0.9788938164710999, "Finetune/Loss (Raw)": 1.0253857374191284, "Finetune/Step": 1138, "Finetune/Step Time": 5.33460140414536} +{"Finetune/Learning Rate": 1.8179293607667177e-05, "Finetune/Loss": 0.9792331457138062, "Finetune/Loss (Raw)": 0.9657769799232483, "Finetune/Step": 1139, "Finetune/Step Time": 5.34738477319479} +{"Finetune/Learning Rate": 1.817570800953088e-05, "Finetune/Loss": 0.9793167114257812, "Finetune/Loss (Raw)": 1.026005744934082, "Finetune/Step": 1140, "Finetune/Step Time": 5.335282895714045} +{"Finetune/Learning Rate": 1.8172119238561924e-05, "Finetune/Loss": 0.9796015024185181, "Finetune/Loss (Raw)": 0.9545606374740601, "Finetune/Step": 1141, "Finetune/Step Time": 5.3462293446063995} +{"Finetune/Learning Rate": 1.8168527296153037e-05, "Finetune/Loss": 0.9738184809684753, "Finetune/Loss (Raw)": 0.3063012957572937, "Finetune/Step": 1142, "Finetune/Step Time": 5.370724968612194} +{"Finetune/Learning Rate": 1.8164932183698184e-05, "Finetune/Loss": 0.9786887168884277, "Finetune/Loss (Raw)": 0.936698853969574, "Finetune/Step": 1143, "Finetune/Step Time": 5.351270550861955} +{"Finetune/Learning Rate": 1.8161333902592554e-05, "Finetune/Loss": 0.978728711605072, "Finetune/Loss (Raw)": 1.0138612985610962, "Finetune/Step": 1144, "Finetune/Step Time": 5.355472611263394} +{"Finetune/Learning Rate": 1.8157732454232577e-05, "Finetune/Loss": 0.9777851104736328, "Finetune/Loss (Raw)": 0.9607192277908325, "Finetune/Step": 1145, "Finetune/Step Time": 5.355797305703163} +{"Finetune/Learning Rate": 1.81541278400159e-05, "Finetune/Loss": 0.9782371520996094, "Finetune/Loss (Raw)": 1.0590027570724487, "Finetune/Step": 1146, "Finetune/Step Time": 5.320771288126707} +{"Finetune/Learning Rate": 1.81505200613414e-05, "Finetune/Loss": 0.9774060249328613, "Finetune/Loss (Raw)": 0.8159366250038147, "Finetune/Step": 1147, "Finetune/Step Time": 5.323892544955015} +{"Finetune/Learning Rate": 1.8146909119609196e-05, "Finetune/Loss": 0.9723961353302002, "Finetune/Loss (Raw)": 0.34708482027053833, "Finetune/Step": 1148, "Finetune/Step Time": 5.355488575994968} +{"Finetune/Learning Rate": 1.8143295016220612e-05, "Finetune/Loss": 0.9742581844329834, "Finetune/Loss (Raw)": 1.107174277305603, "Finetune/Step": 1149, "Finetune/Step Time": 5.355600943788886} +{"Finetune/Learning Rate": 1.813967775257822e-05, "Finetune/Loss": 0.9739038944244385, "Finetune/Loss (Raw)": 0.9773304462432861, "Finetune/Step": 1150, "Finetune/Step Time": 5.3459861017763615} +{"Finetune/Learning Rate": 1.8136057330085805e-05, "Finetune/Loss": 0.9741676449775696, "Finetune/Loss (Raw)": 0.8864953517913818, "Finetune/Step": 1151, "Finetune/Step Time": 5.335089260712266} +{"Finetune/Learning Rate": 1.8132433750148383e-05, "Finetune/Loss": 0.9747005701065063, "Finetune/Loss (Raw)": 1.0010960102081299, "Finetune/Step": 1152, "Finetune/Step Time": 5.352516850456595} +{"Finetune/Learning Rate": 1.8128807014172197e-05, "Finetune/Loss": 0.9763676524162292, "Finetune/Loss (Raw)": 1.0980217456817627, "Finetune/Step": 1153, "Finetune/Step Time": 5.361060041934252} +{"Finetune/Learning Rate": 1.812517712356471e-05, "Finetune/Loss": 0.9709715843200684, "Finetune/Loss (Raw)": 0.3323972523212433, "Finetune/Step": 1154, "Finetune/Step Time": 5.3839031383395195} +{"Finetune/Learning Rate": 1.812154407973461e-05, "Finetune/Loss": 0.9707581996917725, "Finetune/Loss (Raw)": 0.3462907373905182, "Finetune/Step": 1155, "Finetune/Step Time": 5.382040176540613} +{"Finetune/Learning Rate": 1.8117907884091815e-05, "Finetune/Loss": 0.9715552926063538, "Finetune/Loss (Raw)": 1.1549718379974365, "Finetune/Step": 1156, "Finetune/Step Time": 5.378223778679967} +{"Finetune/Learning Rate": 1.8114268538047457e-05, "Finetune/Loss": 0.9728623032569885, "Finetune/Loss (Raw)": 1.046619176864624, "Finetune/Step": 1157, "Finetune/Step Time": 5.3598074875772} +{"Finetune/Learning Rate": 1.81106260430139e-05, "Finetune/Loss": 0.9719408750534058, "Finetune/Loss (Raw)": 0.9161407947540283, "Finetune/Step": 1158, "Finetune/Step Time": 5.35614082030952} +{"Finetune/Learning Rate": 1.810698040040472e-05, "Finetune/Loss": 0.9708962440490723, "Finetune/Loss (Raw)": 0.8587587475776672, "Finetune/Step": 1159, "Finetune/Step Time": 5.350330803543329} +{"Finetune/Learning Rate": 1.810333161163472e-05, "Finetune/Loss": 0.9757067561149597, "Finetune/Loss (Raw)": 0.9659866094589233, "Finetune/Step": 1160, "Finetune/Step Time": 5.344237906858325} +{"Finetune/Learning Rate": 1.809967967811993e-05, "Finetune/Loss": 0.9756277203559875, "Finetune/Loss (Raw)": 1.0006723403930664, "Finetune/Step": 1161, "Finetune/Step Time": 5.337022775784135} +{"Finetune/Learning Rate": 1.8096024601277586e-05, "Finetune/Loss": 0.9769691824913025, "Finetune/Loss (Raw)": 1.014176368713379, "Finetune/Step": 1162, "Finetune/Step Time": 5.339253818616271} +{"Finetune/Learning Rate": 1.8092366382526157e-05, "Finetune/Loss": 0.9754084348678589, "Finetune/Loss (Raw)": 0.8082781434059143, "Finetune/Step": 1163, "Finetune/Step Time": 5.348015358671546} +{"Finetune/Learning Rate": 1.8088705023285324e-05, "Finetune/Loss": 0.9772002696990967, "Finetune/Loss (Raw)": 1.203162431716919, "Finetune/Step": 1164, "Finetune/Step Time": 5.345978364348412} +{"Finetune/Learning Rate": 1.8085040524975994e-05, "Finetune/Loss": 0.973178505897522, "Finetune/Loss (Raw)": 0.36771273612976074, "Finetune/Step": 1165, "Finetune/Step Time": 5.365735685452819} +{"Finetune/Learning Rate": 1.8081372889020282e-05, "Finetune/Loss": 0.9709919095039368, "Finetune/Loss (Raw)": 0.8879372477531433, "Finetune/Step": 1166, "Finetune/Step Time": 5.374224804341793} +{"Finetune/Learning Rate": 1.8077702116841524e-05, "Finetune/Loss": 0.9695912003517151, "Finetune/Loss (Raw)": 0.8362118601799011, "Finetune/Step": 1167, "Finetune/Step Time": 5.3887756783515215} +{"Finetune/Learning Rate": 1.807402820986428e-05, "Finetune/Loss": 0.9698197245597839, "Finetune/Loss (Raw)": 0.9617310762405396, "Finetune/Step": 1168, "Finetune/Step Time": 5.387734856456518} +{"Finetune/Learning Rate": 1.8070351169514324e-05, "Finetune/Loss": 0.9692019820213318, "Finetune/Loss (Raw)": 1.0063132047653198, "Finetune/Step": 1169, "Finetune/Step Time": 5.389512045308948} +{"Finetune/Learning Rate": 1.8066670997218633e-05, "Finetune/Loss": 0.9644380211830139, "Finetune/Loss (Raw)": 0.47355198860168457, "Finetune/Step": 1170, "Finetune/Step Time": 5.415818117558956} +{"Finetune/Learning Rate": 1.806298769440542e-05, "Finetune/Loss": 0.9701942205429077, "Finetune/Loss (Raw)": 1.1205470561981201, "Finetune/Step": 1171, "Finetune/Step Time": 5.376256670802832} +{"Finetune/Learning Rate": 1.8059301262504097e-05, "Finetune/Loss": 0.9716986417770386, "Finetune/Loss (Raw)": 1.0514556169509888, "Finetune/Step": 1172, "Finetune/Step Time": 5.371051399037242} +{"Finetune/Learning Rate": 1.80556117029453e-05, "Finetune/Loss": 0.971198558807373, "Finetune/Loss (Raw)": 0.9947003126144409, "Finetune/Step": 1173, "Finetune/Step Time": 5.354637527838349} +{"Finetune/Learning Rate": 1.8051919017160875e-05, "Finetune/Loss": 0.966636061668396, "Finetune/Loss (Raw)": 0.36530348658561707, "Finetune/Step": 1174, "Finetune/Step Time": 5.3769398126751184} +{"Finetune/Learning Rate": 1.804822320658388e-05, "Finetune/Loss": 0.9662700891494751, "Finetune/Loss (Raw)": 0.9545890092849731, "Finetune/Step": 1175, "Finetune/Step Time": 5.385037077590823} +{"Finetune/Learning Rate": 1.8044524272648588e-05, "Finetune/Loss": 0.9644339084625244, "Finetune/Loss (Raw)": 0.9389352202415466, "Finetune/Step": 1176, "Finetune/Step Time": 5.385741299018264} +{"Finetune/Learning Rate": 1.8040822216790475e-05, "Finetune/Loss": 0.9598344564437866, "Finetune/Loss (Raw)": 0.29158341884613037, "Finetune/Step": 1177, "Finetune/Step Time": 5.4189027938991785} +{"Finetune/Learning Rate": 1.803711704044625e-05, "Finetune/Loss": 0.9560171365737915, "Finetune/Loss (Raw)": 0.32622581720352173, "Finetune/Step": 1178, "Finetune/Step Time": 5.43513061478734} +{"Finetune/Learning Rate": 1.803340874505381e-05, "Finetune/Loss": 0.9576468467712402, "Finetune/Loss (Raw)": 1.1602543592453003, "Finetune/Step": 1179, "Finetune/Step Time": 5.4279304426163435} +{"Finetune/Learning Rate": 1.8029697332052277e-05, "Finetune/Loss": 0.9559148550033569, "Finetune/Loss (Raw)": 0.9320741295814514, "Finetune/Step": 1180, "Finetune/Step Time": 5.444473205134273} +{"Finetune/Learning Rate": 1.8025982802881976e-05, "Finetune/Loss": 0.9567630290985107, "Finetune/Loss (Raw)": 1.0945873260498047, "Finetune/Step": 1181, "Finetune/Step Time": 5.427306963130832} +{"Finetune/Learning Rate": 1.802226515898444e-05, "Finetune/Loss": 0.9535905122756958, "Finetune/Loss (Raw)": 0.7126331925392151, "Finetune/Step": 1182, "Finetune/Step Time": 5.4292900785803795} +{"Finetune/Learning Rate": 1.8018544401802422e-05, "Finetune/Loss": 0.9546846151351929, "Finetune/Loss (Raw)": 1.0424033403396606, "Finetune/Step": 1183, "Finetune/Step Time": 5.4306495897471905} +{"Finetune/Learning Rate": 1.801482053277987e-05, "Finetune/Loss": 0.9525876045227051, "Finetune/Loss (Raw)": 0.8245091438293457, "Finetune/Step": 1184, "Finetune/Step Time": 5.460231468081474} +{"Finetune/Learning Rate": 1.8011093553361944e-05, "Finetune/Loss": 0.9533684253692627, "Finetune/Loss (Raw)": 1.0200363397598267, "Finetune/Step": 1185, "Finetune/Step Time": 5.468537433072925} +{"Finetune/Learning Rate": 1.8007363464995015e-05, "Finetune/Loss": 0.9548672437667847, "Finetune/Loss (Raw)": 1.123631238937378, "Finetune/Step": 1186, "Finetune/Step Time": 5.46894670650363} +{"Finetune/Learning Rate": 1.800363026912665e-05, "Finetune/Loss": 0.9502241611480713, "Finetune/Loss (Raw)": 0.35257261991500854, "Finetune/Step": 1187, "Finetune/Step Time": 5.486041801050305} +{"Finetune/Learning Rate": 1.7999893967205643e-05, "Finetune/Loss": 0.9508653879165649, "Finetune/Loss (Raw)": 1.0612037181854248, "Finetune/Step": 1188, "Finetune/Step Time": 5.485121339559555} +{"Finetune/Learning Rate": 1.7996154560681963e-05, "Finetune/Loss": 0.9525817036628723, "Finetune/Loss (Raw)": 1.067915916442871, "Finetune/Step": 1189, "Finetune/Step Time": 5.490207821130753} +{"Finetune/Learning Rate": 1.7992412051006817e-05, "Finetune/Loss": 0.9524804353713989, "Finetune/Loss (Raw)": 0.9794926643371582, "Finetune/Step": 1190, "Finetune/Step Time": 5.488720877096057} +{"Finetune/Learning Rate": 1.7988666439632593e-05, "Finetune/Loss": 0.9530198574066162, "Finetune/Loss (Raw)": 1.0519453287124634, "Finetune/Step": 1191, "Finetune/Step Time": 5.482280593365431} +{"Finetune/Learning Rate": 1.7984917728012886e-05, "Finetune/Loss": 0.9517914652824402, "Finetune/Loss (Raw)": 1.0299381017684937, "Finetune/Step": 1192, "Finetune/Step Time": 5.489711141213775} +{"Finetune/Learning Rate": 1.7981165917602502e-05, "Finetune/Loss": 0.9508926868438721, "Finetune/Loss (Raw)": 1.02599036693573, "Finetune/Step": 1193, "Finetune/Step Time": 5.488387702032924} +{"Finetune/Learning Rate": 1.7977411009857446e-05, "Finetune/Loss": 0.9521580934524536, "Finetune/Loss (Raw)": 1.068097472190857, "Finetune/Step": 1194, "Finetune/Step Time": 5.502066060900688} +{"Finetune/Learning Rate": 1.797365300623492e-05, "Finetune/Loss": 0.9513049125671387, "Finetune/Loss (Raw)": 1.0019748210906982, "Finetune/Step": 1195, "Finetune/Step Time": 5.491850595921278} +{"Finetune/Learning Rate": 1.7969891908193338e-05, "Finetune/Loss": 0.9519526362419128, "Finetune/Loss (Raw)": 0.9884850382804871, "Finetune/Step": 1196, "Finetune/Step Time": 5.488652385771275} +{"Finetune/Learning Rate": 1.7966127717192305e-05, "Finetune/Loss": 0.9530357122421265, "Finetune/Loss (Raw)": 1.1117491722106934, "Finetune/Step": 1197, "Finetune/Step Time": 5.455580735579133} +{"Finetune/Learning Rate": 1.796236043469263e-05, "Finetune/Loss": 0.9535274505615234, "Finetune/Loss (Raw)": 1.0928184986114502, "Finetune/Step": 1198, "Finetune/Step Time": 5.4612278416752815} +{"Finetune/Learning Rate": 1.7958590062156318e-05, "Finetune/Loss": 0.9534977674484253, "Finetune/Loss (Raw)": 1.0383409261703491, "Finetune/Step": 1199, "Finetune/Step Time": 5.457070948556066} +{"Finetune/Learning Rate": 1.7954816601046587e-05, "Finetune/Loss": 0.9535949230194092, "Finetune/Loss (Raw)": 1.0581567287445068, "Finetune/Step": 1200, "Finetune/Step Time": 5.453833727166057} +{"Finetune/Learning Rate": 1.7951040052827836e-05, "Finetune/Loss": 0.9542863368988037, "Finetune/Loss (Raw)": 1.0498480796813965, "Finetune/Step": 1201, "Finetune/Step Time": 5.461401911452413} +{"Finetune/Learning Rate": 1.794726041896567e-05, "Finetune/Loss": 0.9551345109939575, "Finetune/Loss (Raw)": 1.1605815887451172, "Finetune/Step": 1202, "Finetune/Step Time": 5.464986450970173} +{"Finetune/Learning Rate": 1.7943477700926895e-05, "Finetune/Loss": 0.9557249546051025, "Finetune/Loss (Raw)": 1.1299666166305542, "Finetune/Step": 1203, "Finetune/Step Time": 5.46082054823637} +{"Finetune/Learning Rate": 1.7939691900179507e-05, "Finetune/Loss": 0.9557754993438721, "Finetune/Loss (Raw)": 1.081709384918213, "Finetune/Step": 1204, "Finetune/Step Time": 5.469139184802771} +{"Finetune/Learning Rate": 1.79359030181927e-05, "Finetune/Loss": 0.9548402428627014, "Finetune/Loss (Raw)": 0.9738581776618958, "Finetune/Step": 1205, "Finetune/Step Time": 5.486695561558008} +{"Finetune/Learning Rate": 1.7932111056436868e-05, "Finetune/Loss": 0.9555623531341553, "Finetune/Loss (Raw)": 1.156901240348816, "Finetune/Step": 1206, "Finetune/Step Time": 5.478891618549824} +{"Finetune/Learning Rate": 1.79283160163836e-05, "Finetune/Loss": 0.9562475681304932, "Finetune/Loss (Raw)": 0.9599697589874268, "Finetune/Step": 1207, "Finetune/Step Time": 5.485819336026907} +{"Finetune/Learning Rate": 1.792451789950567e-05, "Finetune/Loss": 0.9566276669502258, "Finetune/Loss (Raw)": 1.0229631662368774, "Finetune/Step": 1208, "Finetune/Step Time": 5.4861817713826895} +{"Finetune/Learning Rate": 1.7920716707277054e-05, "Finetune/Loss": 0.9564380645751953, "Finetune/Loss (Raw)": 1.0565524101257324, "Finetune/Step": 1209, "Finetune/Step Time": 5.484461158514023} +{"Finetune/Learning Rate": 1.7916912441172924e-05, "Finetune/Loss": 0.9575832486152649, "Finetune/Loss (Raw)": 0.9996851682662964, "Finetune/Step": 1210, "Finetune/Step Time": 5.4622085597366095} +{"Finetune/Learning Rate": 1.7913105102669643e-05, "Finetune/Loss": 0.9570304751396179, "Finetune/Loss (Raw)": 0.9504724740982056, "Finetune/Step": 1211, "Finetune/Step Time": 5.46507366001606} +{"Finetune/Learning Rate": 1.790929469324476e-05, "Finetune/Loss": 0.9574158191680908, "Finetune/Loss (Raw)": 1.1245352029800415, "Finetune/Step": 1212, "Finetune/Step Time": 5.453109763562679} +{"Finetune/Learning Rate": 1.7905481214377027e-05, "Finetune/Loss": 0.9569221138954163, "Finetune/Loss (Raw)": 0.9556716084480286, "Finetune/Step": 1213, "Finetune/Step Time": 5.452438447624445} +{"Finetune/Learning Rate": 1.790166466754637e-05, "Finetune/Loss": 0.955507755279541, "Finetune/Loss (Raw)": 0.857004702091217, "Finetune/Step": 1214, "Finetune/Step Time": 5.466199118643999} +{"Finetune/Learning Rate": 1.7897845054233927e-05, "Finetune/Loss": 0.9552589654922485, "Finetune/Loss (Raw)": 0.9890698790550232, "Finetune/Step": 1215, "Finetune/Step Time": 5.466895854100585} +{"Finetune/Learning Rate": 1.7894022375922006e-05, "Finetune/Loss": 0.9513753652572632, "Finetune/Loss (Raw)": 0.35200440883636475, "Finetune/Step": 1216, "Finetune/Step Time": 5.482460306957364} +{"Finetune/Learning Rate": 1.7890196634094122e-05, "Finetune/Loss": 0.9521881937980652, "Finetune/Loss (Raw)": 1.1100836992263794, "Finetune/Step": 1217, "Finetune/Step Time": 5.485377149656415} +{"Finetune/Learning Rate": 1.7886367830234968e-05, "Finetune/Loss": 0.9518623352050781, "Finetune/Loss (Raw)": 1.0532702207565308, "Finetune/Step": 1218, "Finetune/Step Time": 5.470185251906514} +{"Finetune/Learning Rate": 1.788253596583043e-05, "Finetune/Loss": 0.9468963146209717, "Finetune/Loss (Raw)": 0.30597037076950073, "Finetune/Step": 1219, "Finetune/Step Time": 5.508226869627833} +{"Finetune/Learning Rate": 1.787870104236757e-05, "Finetune/Loss": 0.9465543627738953, "Finetune/Loss (Raw)": 0.9950510263442993, "Finetune/Step": 1220, "Finetune/Step Time": 5.495900979265571} +{"Finetune/Learning Rate": 1.7874863061334658e-05, "Finetune/Loss": 0.9477444291114807, "Finetune/Loss (Raw)": 1.0398808717727661, "Finetune/Step": 1221, "Finetune/Step Time": 5.505819356068969} +{"Finetune/Learning Rate": 1.7871022024221133e-05, "Finetune/Loss": 0.947762668132782, "Finetune/Loss (Raw)": 0.9996054172515869, "Finetune/Step": 1222, "Finetune/Step Time": 5.521492246538401} +{"Finetune/Learning Rate": 1.7867177932517634e-05, "Finetune/Loss": 0.9476273059844971, "Finetune/Loss (Raw)": 1.0224260091781616, "Finetune/Step": 1223, "Finetune/Step Time": 5.516776267439127} +{"Finetune/Learning Rate": 1.786333078771597e-05, "Finetune/Loss": 0.947176456451416, "Finetune/Loss (Raw)": 1.0090662240982056, "Finetune/Step": 1224, "Finetune/Step Time": 5.523200308904052} +{"Finetune/Learning Rate": 1.785948059130915e-05, "Finetune/Loss": 0.946286678314209, "Finetune/Loss (Raw)": 0.8805186748504639, "Finetune/Step": 1225, "Finetune/Step Time": 5.508183654397726} +{"Finetune/Learning Rate": 1.785562734479135e-05, "Finetune/Loss": 0.9456183314323425, "Finetune/Loss (Raw)": 0.9009431600570679, "Finetune/Step": 1226, "Finetune/Step Time": 5.498584670946002} +{"Finetune/Learning Rate": 1.785177104965795e-05, "Finetune/Loss": 0.9445050954818726, "Finetune/Loss (Raw)": 0.8587384223937988, "Finetune/Step": 1227, "Finetune/Step Time": 5.50129553489387} +{"Finetune/Learning Rate": 1.7847911707405494e-05, "Finetune/Loss": 0.9451999664306641, "Finetune/Loss (Raw)": 1.0562944412231445, "Finetune/Step": 1228, "Finetune/Step Time": 5.512342821806669} +{"Finetune/Learning Rate": 1.7844049319531725e-05, "Finetune/Loss": 0.945176362991333, "Finetune/Loss (Raw)": 1.081462025642395, "Finetune/Step": 1229, "Finetune/Step Time": 5.514040371403098} +{"Finetune/Learning Rate": 1.7840183887535558e-05, "Finetune/Loss": 0.9420888423919678, "Finetune/Loss (Raw)": 0.4066356420516968, "Finetune/Step": 1230, "Finetune/Step Time": 5.551792494952679} +{"Finetune/Learning Rate": 1.7836315412917088e-05, "Finetune/Loss": 0.9420139789581299, "Finetune/Loss (Raw)": 1.118086338043213, "Finetune/Step": 1231, "Finetune/Step Time": 5.545588621869683} +{"Finetune/Learning Rate": 1.78324438971776e-05, "Finetune/Loss": 0.941551685333252, "Finetune/Loss (Raw)": 1.0101888179779053, "Finetune/Step": 1232, "Finetune/Step Time": 5.534251969307661} +{"Finetune/Learning Rate": 1.782856934181955e-05, "Finetune/Loss": 0.9409582614898682, "Finetune/Loss (Raw)": 1.0381745100021362, "Finetune/Step": 1233, "Finetune/Step Time": 5.518973097205162} +{"Finetune/Learning Rate": 1.782469174834658e-05, "Finetune/Loss": 0.9394651055335999, "Finetune/Loss (Raw)": 0.8115758299827576, "Finetune/Step": 1234, "Finetune/Step Time": 5.52837124094367} +{"Finetune/Learning Rate": 1.7820811118263503e-05, "Finetune/Loss": 0.9388972520828247, "Finetune/Loss (Raw)": 0.9526355266571045, "Finetune/Step": 1235, "Finetune/Step Time": 5.527005923911929} +{"Finetune/Learning Rate": 1.7816927453076324e-05, "Finetune/Loss": 0.9390252232551575, "Finetune/Loss (Raw)": 1.1119205951690674, "Finetune/Step": 1236, "Finetune/Step Time": 5.523139992728829} +{"Finetune/Learning Rate": 1.781304075429221e-05, "Finetune/Loss": 0.9406265616416931, "Finetune/Loss (Raw)": 1.021604061126709, "Finetune/Step": 1237, "Finetune/Step Time": 5.525761723518372} +{"Finetune/Learning Rate": 1.7809151023419517e-05, "Finetune/Loss": 0.9420285224914551, "Finetune/Loss (Raw)": 1.1692250967025757, "Finetune/Step": 1238, "Finetune/Step Time": 5.528523864224553} +{"Finetune/Learning Rate": 1.7805258261967774e-05, "Finetune/Loss": 0.9408729076385498, "Finetune/Loss (Raw)": 0.9855207204818726, "Finetune/Step": 1239, "Finetune/Step Time": 5.542476298287511} +{"Finetune/Learning Rate": 1.780136247144768e-05, "Finetune/Loss": 0.941275417804718, "Finetune/Loss (Raw)": 0.9784908294677734, "Finetune/Step": 1240, "Finetune/Step Time": 5.54074346460402} +{"Finetune/Learning Rate": 1.7797463653371125e-05, "Finetune/Loss": 0.940842866897583, "Finetune/Loss (Raw)": 0.900124728679657, "Finetune/Step": 1241, "Finetune/Step Time": 5.544967360794544} +{"Finetune/Learning Rate": 1.7793561809251153e-05, "Finetune/Loss": 0.9403115510940552, "Finetune/Loss (Raw)": 0.9678104519844055, "Finetune/Step": 1242, "Finetune/Step Time": 5.512680511921644} +{"Finetune/Learning Rate": 1.7789656940602003e-05, "Finetune/Loss": 0.9405840039253235, "Finetune/Loss (Raw)": 1.0555531978607178, "Finetune/Step": 1243, "Finetune/Step Time": 5.506393652409315} +{"Finetune/Learning Rate": 1.7785749048939076e-05, "Finetune/Loss": 0.9396321177482605, "Finetune/Loss (Raw)": 0.9729628562927246, "Finetune/Step": 1244, "Finetune/Step Time": 5.510161757469177} +{"Finetune/Learning Rate": 1.7781838135778945e-05, "Finetune/Loss": 0.9402998685836792, "Finetune/Loss (Raw)": 0.9712667465209961, "Finetune/Step": 1245, "Finetune/Step Time": 5.50052691437304} +{"Finetune/Learning Rate": 1.777792420263936e-05, "Finetune/Loss": 0.9390115737915039, "Finetune/Loss (Raw)": 0.9299311637878418, "Finetune/Step": 1246, "Finetune/Step Time": 5.517640640959144} +{"Finetune/Learning Rate": 1.7774007251039246e-05, "Finetune/Loss": 0.9416776895523071, "Finetune/Loss (Raw)": 1.0756210088729858, "Finetune/Step": 1247, "Finetune/Step Time": 5.503599841147661} +{"Finetune/Learning Rate": 1.777008728249869e-05, "Finetune/Loss": 0.9404804110527039, "Finetune/Loss (Raw)": 0.8238458633422852, "Finetune/Step": 1248, "Finetune/Step Time": 5.508038014173508} +{"Finetune/Learning Rate": 1.776616429853896e-05, "Finetune/Loss": 0.938657283782959, "Finetune/Loss (Raw)": 0.9215617179870605, "Finetune/Step": 1249, "Finetune/Step Time": 5.50875242613256} +{"Finetune/Learning Rate": 1.7762238300682484e-05, "Finetune/Loss": 0.9386074542999268, "Finetune/Loss (Raw)": 1.0380808115005493, "Finetune/Step": 1250, "Finetune/Step Time": 5.488976784050465} +{"Finetune/Learning Rate": 1.775830929045287e-05, "Finetune/Loss": 0.939294695854187, "Finetune/Loss (Raw)": 1.0304685831069946, "Finetune/Step": 1251, "Finetune/Step Time": 5.47446546703577} +{"Finetune/Learning Rate": 1.7754377269374893e-05, "Finetune/Loss": 0.9366645216941833, "Finetune/Loss (Raw)": 0.8956285715103149, "Finetune/Step": 1252, "Finetune/Step Time": 5.470613410696387} +{"Finetune/Learning Rate": 1.7750442238974483e-05, "Finetune/Loss": 0.9365060925483704, "Finetune/Loss (Raw)": 0.9267999529838562, "Finetune/Step": 1253, "Finetune/Step Time": 5.48923802934587} +{"Finetune/Learning Rate": 1.7746504200778756e-05, "Finetune/Loss": 0.9365106821060181, "Finetune/Loss (Raw)": 0.9848839640617371, "Finetune/Step": 1254, "Finetune/Step Time": 5.498458918184042} +{"Finetune/Learning Rate": 1.7742563156315987e-05, "Finetune/Loss": 0.9371525645256042, "Finetune/Loss (Raw)": 1.104037880897522, "Finetune/Step": 1255, "Finetune/Step Time": 5.501055754721165} +{"Finetune/Learning Rate": 1.773861910711562e-05, "Finetune/Loss": 0.9374477863311768, "Finetune/Loss (Raw)": 1.016045331954956, "Finetune/Step": 1256, "Finetune/Step Time": 5.497088709846139} +{"Finetune/Learning Rate": 1.773467205470826e-05, "Finetune/Loss": 0.9342421293258667, "Finetune/Loss (Raw)": 0.703953742980957, "Finetune/Step": 1257, "Finetune/Step Time": 5.523648431524634} +{"Finetune/Learning Rate": 1.7730722000625683e-05, "Finetune/Loss": 0.9354932308197021, "Finetune/Loss (Raw)": 0.9574152231216431, "Finetune/Step": 1258, "Finetune/Step Time": 5.512137362733483} +{"Finetune/Learning Rate": 1.7726768946400825e-05, "Finetune/Loss": 0.9340265989303589, "Finetune/Loss (Raw)": 0.8975715041160583, "Finetune/Step": 1259, "Finetune/Step Time": 5.523433867841959} +{"Finetune/Learning Rate": 1.772281289356779e-05, "Finetune/Loss": 0.9351575374603271, "Finetune/Loss (Raw)": 1.1742844581604004, "Finetune/Step": 1260, "Finetune/Step Time": 5.525603910908103} +{"Finetune/Learning Rate": 1.771885384366185e-05, "Finetune/Loss": 0.9340852499008179, "Finetune/Loss (Raw)": 0.8484653830528259, "Finetune/Step": 1261, "Finetune/Step Time": 5.525311013683677} +{"Finetune/Learning Rate": 1.7714891798219432e-05, "Finetune/Loss": 0.9343923330307007, "Finetune/Loss (Raw)": 0.9943978190422058, "Finetune/Step": 1262, "Finetune/Step Time": 5.520729038864374} +{"Finetune/Learning Rate": 1.7710926758778123e-05, "Finetune/Loss": 0.9340460300445557, "Finetune/Loss (Raw)": 1.05989670753479, "Finetune/Step": 1263, "Finetune/Step Time": 5.517357585951686} +{"Finetune/Learning Rate": 1.7706958726876685e-05, "Finetune/Loss": 0.9342249631881714, "Finetune/Loss (Raw)": 1.0008094310760498, "Finetune/Step": 1264, "Finetune/Step Time": 5.515646241605282} +{"Finetune/Learning Rate": 1.770298770405503e-05, "Finetune/Loss": 0.929028332233429, "Finetune/Loss (Raw)": 0.3815348744392395, "Finetune/Step": 1265, "Finetune/Step Time": 5.551232125610113} +{"Finetune/Learning Rate": 1.7699013691854238e-05, "Finetune/Loss": 0.9285755157470703, "Finetune/Loss (Raw)": 0.9674192667007446, "Finetune/Step": 1266, "Finetune/Step Time": 5.5562760308384895} +{"Finetune/Learning Rate": 1.769503669181654e-05, "Finetune/Loss": 0.9291965365409851, "Finetune/Loss (Raw)": 1.0452706813812256, "Finetune/Step": 1267, "Finetune/Step Time": 5.542516300454736} +{"Finetune/Learning Rate": 1.7691056705485333e-05, "Finetune/Loss": 0.9295504689216614, "Finetune/Loss (Raw)": 1.0713093280792236, "Finetune/Step": 1268, "Finetune/Step Time": 5.54623213596642} +{"Finetune/Learning Rate": 1.7687073734405177e-05, "Finetune/Loss": 0.9295821785926819, "Finetune/Loss (Raw)": 0.9586197733879089, "Finetune/Step": 1269, "Finetune/Step Time": 5.532333945855498} +{"Finetune/Learning Rate": 1.768308778012178e-05, "Finetune/Loss": 0.9360872507095337, "Finetune/Loss (Raw)": 1.1389508247375488, "Finetune/Step": 1270, "Finetune/Step Time": 5.5107759311795235} +{"Finetune/Learning Rate": 1.767909884418202e-05, "Finetune/Loss": 0.937372624874115, "Finetune/Loss (Raw)": 1.1012262105941772, "Finetune/Step": 1271, "Finetune/Step Time": 5.496032796800137} +{"Finetune/Learning Rate": 1.7675106928133915e-05, "Finetune/Loss": 0.9358516335487366, "Finetune/Loss (Raw)": 0.8191765546798706, "Finetune/Step": 1272, "Finetune/Step Time": 5.504793342202902} +{"Finetune/Learning Rate": 1.7671112033526655e-05, "Finetune/Loss": 0.9356902837753296, "Finetune/Loss (Raw)": 0.9400694966316223, "Finetune/Step": 1273, "Finetune/Step Time": 5.51469892449677} +{"Finetune/Learning Rate": 1.7667114161910586e-05, "Finetune/Loss": 0.9342777729034424, "Finetune/Loss (Raw)": 0.8781907558441162, "Finetune/Step": 1274, "Finetune/Step Time": 5.506111424416304} +{"Finetune/Learning Rate": 1.7663113314837192e-05, "Finetune/Loss": 0.9370337724685669, "Finetune/Loss (Raw)": 1.1687071323394775, "Finetune/Step": 1275, "Finetune/Step Time": 5.497606908902526} +{"Finetune/Learning Rate": 1.7659109493859134e-05, "Finetune/Loss": 0.9417750239372253, "Finetune/Loss (Raw)": 0.9539673328399658, "Finetune/Step": 1276, "Finetune/Step Time": 5.467477589845657} +{"Finetune/Learning Rate": 1.7655102700530212e-05, "Finetune/Loss": 0.9407004714012146, "Finetune/Loss (Raw)": 0.9696308374404907, "Finetune/Step": 1277, "Finetune/Step Time": 5.471059627830982} +{"Finetune/Learning Rate": 1.7651092936405385e-05, "Finetune/Loss": 0.9389071464538574, "Finetune/Loss (Raw)": 0.7477889657020569, "Finetune/Step": 1278, "Finetune/Step Time": 5.477496733888984} +{"Finetune/Learning Rate": 1.7647080203040765e-05, "Finetune/Loss": 0.9398014545440674, "Finetune/Loss (Raw)": 1.0009678602218628, "Finetune/Step": 1279, "Finetune/Step Time": 5.4743424486368895} +{"Finetune/Learning Rate": 1.7643064501993614e-05, "Finetune/Loss": 0.9410390257835388, "Finetune/Loss (Raw)": 1.15950345993042, "Finetune/Step": 1280, "Finetune/Step Time": 5.45323964767158} +{"Finetune/Learning Rate": 1.763904583482235e-05, "Finetune/Loss": 0.9399220943450928, "Finetune/Loss (Raw)": 0.9550572037696838, "Finetune/Step": 1281, "Finetune/Step Time": 5.44345529563725} +{"Finetune/Learning Rate": 1.7635024203086532e-05, "Finetune/Loss": 0.9450727105140686, "Finetune/Loss (Raw)": 0.9916688799858093, "Finetune/Step": 1282, "Finetune/Step Time": 5.413529168814421} +{"Finetune/Learning Rate": 1.7630999608346888e-05, "Finetune/Loss": 0.9492964148521423, "Finetune/Loss (Raw)": 0.8869261741638184, "Finetune/Step": 1283, "Finetune/Step Time": 5.3947216384112835} +{"Finetune/Learning Rate": 1.7626972052165275e-05, "Finetune/Loss": 0.9478814005851746, "Finetune/Loss (Raw)": 0.9738509058952332, "Finetune/Step": 1284, "Finetune/Step Time": 5.398005047813058} +{"Finetune/Learning Rate": 1.7622941536104715e-05, "Finetune/Loss": 0.9479803442955017, "Finetune/Loss (Raw)": 1.0592825412750244, "Finetune/Step": 1285, "Finetune/Step Time": 5.398069133982062} +{"Finetune/Learning Rate": 1.7618908061729373e-05, "Finetune/Loss": 0.9483669400215149, "Finetune/Loss (Raw)": 0.9656290411949158, "Finetune/Step": 1286, "Finetune/Step Time": 5.413274792954326} +{"Finetune/Learning Rate": 1.7614871630604554e-05, "Finetune/Loss": 0.9493193030357361, "Finetune/Loss (Raw)": 0.9806572794914246, "Finetune/Step": 1287, "Finetune/Step Time": 5.412751415744424} +{"Finetune/Learning Rate": 1.761083224429672e-05, "Finetune/Loss": 0.9488941431045532, "Finetune/Loss (Raw)": 0.9115712642669678, "Finetune/Step": 1288, "Finetune/Step Time": 5.395024869590998} +{"Finetune/Learning Rate": 1.760678990437349e-05, "Finetune/Loss": 0.9482215642929077, "Finetune/Loss (Raw)": 0.9145808815956116, "Finetune/Step": 1289, "Finetune/Step Time": 5.379838604480028} +{"Finetune/Learning Rate": 1.76027446124036e-05, "Finetune/Loss": 0.9489989280700684, "Finetune/Loss (Raw)": 1.113673210144043, "Finetune/Step": 1290, "Finetune/Step Time": 5.3691766038537025} +{"Finetune/Learning Rate": 1.759869636995696e-05, "Finetune/Loss": 0.9504566192626953, "Finetune/Loss (Raw)": 0.9948632121086121, "Finetune/Step": 1291, "Finetune/Step Time": 5.365826331079006} +{"Finetune/Learning Rate": 1.7594645178604614e-05, "Finetune/Loss": 0.948602557182312, "Finetune/Loss (Raw)": 0.9658480286598206, "Finetune/Step": 1292, "Finetune/Step Time": 5.36609903909266} +{"Finetune/Learning Rate": 1.7590591039918744e-05, "Finetune/Loss": 0.9531986713409424, "Finetune/Loss (Raw)": 0.9560185074806213, "Finetune/Step": 1293, "Finetune/Step Time": 5.3351640943437815} +{"Finetune/Learning Rate": 1.7586533955472686e-05, "Finetune/Loss": 0.9540093541145325, "Finetune/Loss (Raw)": 0.9917002320289612, "Finetune/Step": 1294, "Finetune/Step Time": 5.338064698502421} +{"Finetune/Learning Rate": 1.7582473926840916e-05, "Finetune/Loss": 0.9547262191772461, "Finetune/Loss (Raw)": 0.9279658198356628, "Finetune/Step": 1295, "Finetune/Step Time": 5.3368403557688} +{"Finetune/Learning Rate": 1.7578410955599046e-05, "Finetune/Loss": 0.9547934532165527, "Finetune/Loss (Raw)": 0.9703414440155029, "Finetune/Step": 1296, "Finetune/Step Time": 5.340690223500133} +{"Finetune/Learning Rate": 1.7574345043323843e-05, "Finetune/Loss": 0.9548059701919556, "Finetune/Loss (Raw)": 1.0079087018966675, "Finetune/Step": 1297, "Finetune/Step Time": 5.3558290507644415} +{"Finetune/Learning Rate": 1.7570276191593202e-05, "Finetune/Loss": 0.9578502178192139, "Finetune/Loss (Raw)": 0.8632248640060425, "Finetune/Step": 1298, "Finetune/Step Time": 5.321129156276584} +{"Finetune/Learning Rate": 1.7566204401986172e-05, "Finetune/Loss": 0.9549541473388672, "Finetune/Loss (Raw)": 0.7498477697372437, "Finetune/Step": 1299, "Finetune/Step Time": 5.325598852708936} +{"Finetune/Learning Rate": 1.7562129676082933e-05, "Finetune/Loss": 0.9537479877471924, "Finetune/Loss (Raw)": 0.8970710635185242, "Finetune/Step": 1300, "Finetune/Step Time": 5.323494495823979} +{"Finetune/Learning Rate": 1.75580520154648e-05, "Finetune/Loss": 0.9539597034454346, "Finetune/Loss (Raw)": 1.0217922925949097, "Finetune/Step": 1301, "Finetune/Step Time": 5.341351307928562} +{"Finetune/Learning Rate": 1.755397142171424e-05, "Finetune/Loss": 0.958486795425415, "Finetune/Loss (Raw)": 0.9447715282440186, "Finetune/Step": 1302, "Finetune/Step Time": 5.33185407333076} +{"Finetune/Learning Rate": 1.7549887896414853e-05, "Finetune/Loss": 0.9587386846542358, "Finetune/Loss (Raw)": 0.9868363738059998, "Finetune/Step": 1303, "Finetune/Step Time": 5.315308617427945} +{"Finetune/Learning Rate": 1.7545801441151368e-05, "Finetune/Loss": 0.9592537879943848, "Finetune/Loss (Raw)": 1.0048705339431763, "Finetune/Step": 1304, "Finetune/Step Time": 5.315694121643901} +{"Finetune/Learning Rate": 1.7541712057509666e-05, "Finetune/Loss": 0.964561939239502, "Finetune/Loss (Raw)": 0.9710200428962708, "Finetune/Step": 1305, "Finetune/Step Time": 5.287932764738798} +{"Finetune/Learning Rate": 1.7537619747076752e-05, "Finetune/Loss": 0.9698275923728943, "Finetune/Loss (Raw)": 1.0002319812774658, "Finetune/Step": 1306, "Finetune/Step Time": 5.279846724122763} +{"Finetune/Learning Rate": 1.7533524511440777e-05, "Finetune/Loss": 0.9692223072052002, "Finetune/Loss (Raw)": 1.0827720165252686, "Finetune/Step": 1307, "Finetune/Step Time": 5.2735656928271055} +{"Finetune/Learning Rate": 1.7529426352191014e-05, "Finetune/Loss": 0.9703298211097717, "Finetune/Loss (Raw)": 1.0738446712493896, "Finetune/Step": 1308, "Finetune/Step Time": 5.256088038906455} +{"Finetune/Learning Rate": 1.752532527091789e-05, "Finetune/Loss": 0.9692332744598389, "Finetune/Loss (Raw)": 0.9542303681373596, "Finetune/Step": 1309, "Finetune/Step Time": 5.274560196325183} +{"Finetune/Learning Rate": 1.7521221269212944e-05, "Finetune/Loss": 0.9707931280136108, "Finetune/Loss (Raw)": 0.9122909903526306, "Finetune/Step": 1310, "Finetune/Step Time": 5.27271762676537} +{"Finetune/Learning Rate": 1.751711434866886e-05, "Finetune/Loss": 0.9705449342727661, "Finetune/Loss (Raw)": 1.0106340646743774, "Finetune/Step": 1311, "Finetune/Step Time": 5.274732714518905} +{"Finetune/Learning Rate": 1.751300451087946e-05, "Finetune/Loss": 0.9701189994812012, "Finetune/Loss (Raw)": 0.7699903845787048, "Finetune/Step": 1312, "Finetune/Step Time": 5.26939420029521} +{"Finetune/Learning Rate": 1.7508891757439687e-05, "Finetune/Loss": 0.970095157623291, "Finetune/Loss (Raw)": 1.0169821977615356, "Finetune/Step": 1313, "Finetune/Step Time": 5.270706862211227} +{"Finetune/Learning Rate": 1.7504776089945626e-05, "Finetune/Loss": 0.9691588878631592, "Finetune/Loss (Raw)": 1.0037914514541626, "Finetune/Step": 1314, "Finetune/Step Time": 5.268314629793167} +{"Finetune/Learning Rate": 1.7500657509994478e-05, "Finetune/Loss": 0.9741748571395874, "Finetune/Loss (Raw)": 0.9946111440658569, "Finetune/Step": 1315, "Finetune/Step Time": 5.238431299105287} +{"Finetune/Learning Rate": 1.7496536019184588e-05, "Finetune/Loss": 0.9750053882598877, "Finetune/Loss (Raw)": 1.1675183773040771, "Finetune/Step": 1316, "Finetune/Step Time": 5.243420930579305} +{"Finetune/Learning Rate": 1.749241161911543e-05, "Finetune/Loss": 0.9749605655670166, "Finetune/Loss (Raw)": 1.0621802806854248, "Finetune/Step": 1317, "Finetune/Step Time": 5.236187299713492} +{"Finetune/Learning Rate": 1.7488284311387597e-05, "Finetune/Loss": 0.9747950434684753, "Finetune/Loss (Raw)": 0.9583005905151367, "Finetune/Step": 1318, "Finetune/Step Time": 5.236363364383578} +{"Finetune/Learning Rate": 1.7484154097602822e-05, "Finetune/Loss": 0.974390983581543, "Finetune/Loss (Raw)": 1.0002299547195435, "Finetune/Step": 1319, "Finetune/Step Time": 5.234222134575248} +{"Finetune/Learning Rate": 1.7480020979363956e-05, "Finetune/Loss": 0.9739774465560913, "Finetune/Loss (Raw)": 0.9769973754882812, "Finetune/Step": 1320, "Finetune/Step Time": 5.235647330060601} +{"Finetune/Learning Rate": 1.747588495827498e-05, "Finetune/Loss": 0.9738641977310181, "Finetune/Loss (Raw)": 1.0115050077438354, "Finetune/Step": 1321, "Finetune/Step Time": 5.244124639779329} +{"Finetune/Learning Rate": 1.7471746035941007e-05, "Finetune/Loss": 0.9732243418693542, "Finetune/Loss (Raw)": 0.9861896634101868, "Finetune/Step": 1322, "Finetune/Step Time": 5.233549742028117} +{"Finetune/Learning Rate": 1.7467604213968274e-05, "Finetune/Loss": 0.9736315011978149, "Finetune/Loss (Raw)": 1.0540872812271118, "Finetune/Step": 1323, "Finetune/Step Time": 5.234369739890099} +{"Finetune/Learning Rate": 1.746345949396414e-05, "Finetune/Loss": 0.9739174246788025, "Finetune/Loss (Raw)": 1.0250879526138306, "Finetune/Step": 1324, "Finetune/Step Time": 5.236247815191746} +{"Finetune/Learning Rate": 1.7459311877537085e-05, "Finetune/Loss": 0.9735549688339233, "Finetune/Loss (Raw)": 1.0653486251831055, "Finetune/Step": 1325, "Finetune/Step Time": 5.254028093069792} +{"Finetune/Learning Rate": 1.745516136629672e-05, "Finetune/Loss": 0.9731359481811523, "Finetune/Loss (Raw)": 1.0391889810562134, "Finetune/Step": 1326, "Finetune/Step Time": 5.246783735230565} +{"Finetune/Learning Rate": 1.7451007961853778e-05, "Finetune/Loss": 0.9722812175750732, "Finetune/Loss (Raw)": 0.9289328455924988, "Finetune/Step": 1327, "Finetune/Step Time": 5.284537801519036} +{"Finetune/Learning Rate": 1.7446851665820118e-05, "Finetune/Loss": 0.9729701280593872, "Finetune/Loss (Raw)": 1.1463385820388794, "Finetune/Step": 1328, "Finetune/Step Time": 5.279822094365954} +{"Finetune/Learning Rate": 1.744269247980871e-05, "Finetune/Loss": 0.9739865660667419, "Finetune/Loss (Raw)": 1.1799532175064087, "Finetune/Step": 1329, "Finetune/Step Time": 5.277581909671426} +{"Finetune/Learning Rate": 1.7438530405433663e-05, "Finetune/Loss": 0.9709980487823486, "Finetune/Loss (Raw)": 0.7780522704124451, "Finetune/Step": 1330, "Finetune/Step Time": 5.271170483902097} +{"Finetune/Learning Rate": 1.7434365444310185e-05, "Finetune/Loss": 0.9704210758209229, "Finetune/Loss (Raw)": 1.0561171770095825, "Finetune/Step": 1331, "Finetune/Step Time": 5.259388092905283} +{"Finetune/Learning Rate": 1.7430197598054622e-05, "Finetune/Loss": 0.9688031673431396, "Finetune/Loss (Raw)": 0.8746152520179749, "Finetune/Step": 1332, "Finetune/Step Time": 5.262385703623295} +{"Finetune/Learning Rate": 1.7426026868284438e-05, "Finetune/Loss": 0.9687502384185791, "Finetune/Loss (Raw)": 0.9670819640159607, "Finetune/Step": 1333, "Finetune/Step Time": 5.249046651646495} +{"Finetune/Learning Rate": 1.7421853256618204e-05, "Finetune/Loss": 0.9676380157470703, "Finetune/Loss (Raw)": 1.0145328044891357, "Finetune/Step": 1334, "Finetune/Step Time": 5.253031890839338} +{"Finetune/Learning Rate": 1.741767676467562e-05, "Finetune/Loss": 0.9682767391204834, "Finetune/Loss (Raw)": 1.0417265892028809, "Finetune/Step": 1335, "Finetune/Step Time": 5.245309770107269} +{"Finetune/Learning Rate": 1.7413497394077502e-05, "Finetune/Loss": 0.9683519601821899, "Finetune/Loss (Raw)": 1.032598614692688, "Finetune/Step": 1336, "Finetune/Step Time": 5.240808391943574} +{"Finetune/Learning Rate": 1.7409315146445783e-05, "Finetune/Loss": 0.9689470529556274, "Finetune/Loss (Raw)": 1.1327221393585205, "Finetune/Step": 1337, "Finetune/Step Time": 5.243303209543228} +{"Finetune/Learning Rate": 1.7405130023403513e-05, "Finetune/Loss": 0.9635974168777466, "Finetune/Loss (Raw)": 0.3149299621582031, "Finetune/Step": 1338, "Finetune/Step Time": 5.276259172707796} +{"Finetune/Learning Rate": 1.740094202657485e-05, "Finetune/Loss": 0.9633667469024658, "Finetune/Loss (Raw)": 0.9209450483322144, "Finetune/Step": 1339, "Finetune/Step Time": 5.281784234568477} +{"Finetune/Learning Rate": 1.7396751157585078e-05, "Finetune/Loss": 0.9624056816101074, "Finetune/Loss (Raw)": 1.0015195608139038, "Finetune/Step": 1340, "Finetune/Step Time": 5.288231981918216} +{"Finetune/Learning Rate": 1.739255741806059e-05, "Finetune/Loss": 0.9623790979385376, "Finetune/Loss (Raw)": 0.9522697329521179, "Finetune/Step": 1341, "Finetune/Step Time": 5.289699353277683} +{"Finetune/Learning Rate": 1.73883608096289e-05, "Finetune/Loss": 0.9643654227256775, "Finetune/Loss (Raw)": 1.1112557649612427, "Finetune/Step": 1342, "Finetune/Step Time": 5.287163786590099} +{"Finetune/Learning Rate": 1.738416133391862e-05, "Finetune/Loss": 0.9638007283210754, "Finetune/Loss (Raw)": 0.9167889952659607, "Finetune/Step": 1343, "Finetune/Step Time": 5.309712564572692} +{"Finetune/Learning Rate": 1.7379958992559494e-05, "Finetune/Loss": 0.9684925079345703, "Finetune/Loss (Raw)": 0.952549397945404, "Finetune/Step": 1344, "Finetune/Step Time": 5.284821594133973} +{"Finetune/Learning Rate": 1.7375753787182364e-05, "Finetune/Loss": 0.9681844115257263, "Finetune/Loss (Raw)": 1.0706523656845093, "Finetune/Step": 1345, "Finetune/Step Time": 5.284880727529526} +{"Finetune/Learning Rate": 1.7371545719419188e-05, "Finetune/Loss": 0.967418909072876, "Finetune/Loss (Raw)": 0.9552749395370483, "Finetune/Step": 1346, "Finetune/Step Time": 5.287061233073473} +{"Finetune/Learning Rate": 1.7367334790903034e-05, "Finetune/Loss": 0.9741089940071106, "Finetune/Loss (Raw)": 1.1623066663742065, "Finetune/Step": 1347, "Finetune/Step Time": 5.267954122275114} +{"Finetune/Learning Rate": 1.736312100326808e-05, "Finetune/Loss": 0.969700276851654, "Finetune/Loss (Raw)": 0.43073347210884094, "Finetune/Step": 1348, "Finetune/Step Time": 5.301977436989546} +{"Finetune/Learning Rate": 1.7358904358149625e-05, "Finetune/Loss": 0.9700781106948853, "Finetune/Loss (Raw)": 1.0882469415664673, "Finetune/Step": 1349, "Finetune/Step Time": 5.282302146777511} +{"Finetune/Learning Rate": 1.7354684857184056e-05, "Finetune/Loss": 0.9705196022987366, "Finetune/Loss (Raw)": 1.0561176538467407, "Finetune/Step": 1350, "Finetune/Step Time": 5.275770042091608} +{"Finetune/Learning Rate": 1.735046250200888e-05, "Finetune/Loss": 0.9706753492355347, "Finetune/Loss (Raw)": 1.0423532724380493, "Finetune/Step": 1351, "Finetune/Step Time": 5.279666937887669} +{"Finetune/Learning Rate": 1.7346237294262717e-05, "Finetune/Loss": 0.9661216139793396, "Finetune/Loss (Raw)": 0.4261987507343292, "Finetune/Step": 1352, "Finetune/Step Time": 5.310671254992485} +{"Finetune/Learning Rate": 1.734200923558528e-05, "Finetune/Loss": 0.9666874408721924, "Finetune/Loss (Raw)": 0.9529393315315247, "Finetune/Step": 1353, "Finetune/Step Time": 5.310688689351082} +{"Finetune/Learning Rate": 1.7337778327617397e-05, "Finetune/Loss": 0.9675206542015076, "Finetune/Loss (Raw)": 1.0075981616973877, "Finetune/Step": 1354, "Finetune/Step Time": 5.318163692951202} +{"Finetune/Learning Rate": 1.733354457200101e-05, "Finetune/Loss": 0.9694593548774719, "Finetune/Loss (Raw)": 1.106888771057129, "Finetune/Step": 1355, "Finetune/Step Time": 5.317537218332291} +{"Finetune/Learning Rate": 1.7329307970379145e-05, "Finetune/Loss": 0.9695873260498047, "Finetune/Loss (Raw)": 1.0726748704910278, "Finetune/Step": 1356, "Finetune/Step Time": 5.3063386008143425} +{"Finetune/Learning Rate": 1.732506852439595e-05, "Finetune/Loss": 0.9694454073905945, "Finetune/Loss (Raw)": 1.0632939338684082, "Finetune/Step": 1357, "Finetune/Step Time": 5.3134552501142025} +{"Finetune/Learning Rate": 1.7320826235696672e-05, "Finetune/Loss": 0.9739511609077454, "Finetune/Loss (Raw)": 0.9833766222000122, "Finetune/Step": 1358, "Finetune/Step Time": 5.275599937886} +{"Finetune/Learning Rate": 1.7316581105927658e-05, "Finetune/Loss": 0.971954345703125, "Finetune/Loss (Raw)": 0.8624901175498962, "Finetune/Step": 1359, "Finetune/Step Time": 5.289718611165881} +{"Finetune/Learning Rate": 1.731233313673636e-05, "Finetune/Loss": 0.9719012379646301, "Finetune/Loss (Raw)": 1.0033986568450928, "Finetune/Step": 1360, "Finetune/Step Time": 5.298091419041157} +{"Finetune/Learning Rate": 1.7308082329771335e-05, "Finetune/Loss": 0.9723050594329834, "Finetune/Loss (Raw)": 1.0898648500442505, "Finetune/Step": 1361, "Finetune/Step Time": 5.299070879817009} +{"Finetune/Learning Rate": 1.7303828686682233e-05, "Finetune/Loss": 0.9735196828842163, "Finetune/Loss (Raw)": 0.9670443534851074, "Finetune/Step": 1362, "Finetune/Step Time": 5.281495317816734} +{"Finetune/Learning Rate": 1.7299572209119813e-05, "Finetune/Loss": 0.9738496541976929, "Finetune/Loss (Raw)": 0.9948670864105225, "Finetune/Step": 1363, "Finetune/Step Time": 5.282188806682825} +{"Finetune/Learning Rate": 1.7295312898735933e-05, "Finetune/Loss": 0.972684383392334, "Finetune/Loss (Raw)": 0.9627660512924194, "Finetune/Step": 1364, "Finetune/Step Time": 5.311153391376138} +{"Finetune/Learning Rate": 1.7291050757183546e-05, "Finetune/Loss": 0.9717989563941956, "Finetune/Loss (Raw)": 0.9082710146903992, "Finetune/Step": 1365, "Finetune/Step Time": 5.301979802548885} +{"Finetune/Learning Rate": 1.7286785786116707e-05, "Finetune/Loss": 0.9701768159866333, "Finetune/Loss (Raw)": 0.9615908265113831, "Finetune/Step": 1366, "Finetune/Step Time": 5.296520680189133} +{"Finetune/Learning Rate": 1.7282517987190567e-05, "Finetune/Loss": 0.9697154760360718, "Finetune/Loss (Raw)": 0.9264681935310364, "Finetune/Step": 1367, "Finetune/Step Time": 5.30986382253468} +{"Finetune/Learning Rate": 1.7278247362061372e-05, "Finetune/Loss": 0.9701409339904785, "Finetune/Loss (Raw)": 1.0329480171203613, "Finetune/Step": 1368, "Finetune/Step Time": 5.310063678771257} +{"Finetune/Learning Rate": 1.7273973912386473e-05, "Finetune/Loss": 0.971320390701294, "Finetune/Loss (Raw)": 1.051098108291626, "Finetune/Step": 1369, "Finetune/Step Time": 5.324476838111877} +{"Finetune/Learning Rate": 1.726969763982431e-05, "Finetune/Loss": 0.9704326391220093, "Finetune/Loss (Raw)": 0.8541861772537231, "Finetune/Step": 1370, "Finetune/Step Time": 5.327083386480808} +{"Finetune/Learning Rate": 1.7265418546034424e-05, "Finetune/Loss": 0.9698233008384705, "Finetune/Loss (Raw)": 0.9775475263595581, "Finetune/Step": 1371, "Finetune/Step Time": 5.327609306201339} +{"Finetune/Learning Rate": 1.7261136632677445e-05, "Finetune/Loss": 0.9651974439620972, "Finetune/Loss (Raw)": 0.38085639476776123, "Finetune/Step": 1372, "Finetune/Step Time": 5.355785993859172} +{"Finetune/Learning Rate": 1.7256851901415102e-05, "Finetune/Loss": 0.9664318561553955, "Finetune/Loss (Raw)": 1.1292644739151, "Finetune/Step": 1373, "Finetune/Step Time": 5.355435764417052} +{"Finetune/Learning Rate": 1.725256435391021e-05, "Finetune/Loss": 0.9664933085441589, "Finetune/Loss (Raw)": 0.9378056526184082, "Finetune/Step": 1374, "Finetune/Step Time": 5.346449138596654} +{"Finetune/Learning Rate": 1.7248273991826692e-05, "Finetune/Loss": 0.9662913084030151, "Finetune/Loss (Raw)": 1.0497609376907349, "Finetune/Step": 1375, "Finetune/Step Time": 5.3438014555722475} +{"Finetune/Learning Rate": 1.7243980816829543e-05, "Finetune/Loss": 0.9623392820358276, "Finetune/Loss (Raw)": 0.31799182295799255, "Finetune/Step": 1376, "Finetune/Step Time": 5.366965766996145} +{"Finetune/Learning Rate": 1.723968483058487e-05, "Finetune/Loss": 0.963552713394165, "Finetune/Loss (Raw)": 1.0768769979476929, "Finetune/Step": 1377, "Finetune/Step Time": 5.375168114900589} +{"Finetune/Learning Rate": 1.7235386034759853e-05, "Finetune/Loss": 0.9644498825073242, "Finetune/Loss (Raw)": 1.1529157161712646, "Finetune/Step": 1378, "Finetune/Step Time": 5.393923528492451} +{"Finetune/Learning Rate": 1.723108443102278e-05, "Finetune/Loss": 0.9621458053588867, "Finetune/Loss (Raw)": 0.735548198223114, "Finetune/Step": 1379, "Finetune/Step Time": 5.41051060706377} +{"Finetune/Learning Rate": 1.722678002104301e-05, "Finetune/Loss": 0.9625418186187744, "Finetune/Loss (Raw)": 0.9463216066360474, "Finetune/Step": 1380, "Finetune/Step Time": 5.410220177844167} +{"Finetune/Learning Rate": 1.722247280649101e-05, "Finetune/Loss": 0.9635615348815918, "Finetune/Loss (Raw)": 1.0573146343231201, "Finetune/Step": 1381, "Finetune/Step Time": 5.395376307889819} +{"Finetune/Learning Rate": 1.7218162789038312e-05, "Finetune/Loss": 0.9636081457138062, "Finetune/Loss (Raw)": 0.9908574223518372, "Finetune/Step": 1382, "Finetune/Step Time": 5.389086540788412} +{"Finetune/Learning Rate": 1.7213849970357565e-05, "Finetune/Loss": 0.9621567726135254, "Finetune/Loss (Raw)": 0.9182654023170471, "Finetune/Step": 1383, "Finetune/Step Time": 5.392943635582924} +{"Finetune/Learning Rate": 1.7209534352122484e-05, "Finetune/Loss": 0.9617853164672852, "Finetune/Loss (Raw)": 0.968496561050415, "Finetune/Step": 1384, "Finetune/Step Time": 5.395573198795319} +{"Finetune/Learning Rate": 1.720521593600787e-05, "Finetune/Loss": 0.964264452457428, "Finetune/Loss (Raw)": 1.0212846994400024, "Finetune/Step": 1385, "Finetune/Step Time": 5.37196721509099} +{"Finetune/Learning Rate": 1.7200894723689626e-05, "Finetune/Loss": 0.9640229940414429, "Finetune/Loss (Raw)": 0.9265052080154419, "Finetune/Step": 1386, "Finetune/Step Time": 5.378164634108543} +{"Finetune/Learning Rate": 1.7196570716844724e-05, "Finetune/Loss": 0.9637973308563232, "Finetune/Loss (Raw)": 0.8686924576759338, "Finetune/Step": 1387, "Finetune/Step Time": 5.372430374845862} +{"Finetune/Learning Rate": 1.7192243917151228e-05, "Finetune/Loss": 0.9626718163490295, "Finetune/Loss (Raw)": 1.0302114486694336, "Finetune/Step": 1388, "Finetune/Step Time": 5.367735970765352} +{"Finetune/Learning Rate": 1.7187914326288286e-05, "Finetune/Loss": 0.9639309644699097, "Finetune/Loss (Raw)": 1.009629726409912, "Finetune/Step": 1389, "Finetune/Step Time": 5.368663469329476} +{"Finetune/Learning Rate": 1.718358194593612e-05, "Finetune/Loss": 0.9653140306472778, "Finetune/Loss (Raw)": 1.1714417934417725, "Finetune/Step": 1390, "Finetune/Step Time": 5.376557687297463} +{"Finetune/Learning Rate": 1.7179246777776053e-05, "Finetune/Loss": 0.9639329314231873, "Finetune/Loss (Raw)": 0.8831139206886292, "Finetune/Step": 1391, "Finetune/Step Time": 5.392097158357501} +{"Finetune/Learning Rate": 1.7174908823490475e-05, "Finetune/Loss": 0.9632008075714111, "Finetune/Loss (Raw)": 0.9071025848388672, "Finetune/Step": 1392, "Finetune/Step Time": 5.4254830330610275} +{"Finetune/Learning Rate": 1.717056808476286e-05, "Finetune/Loss": 0.9678940773010254, "Finetune/Loss (Raw)": 0.9822589159011841, "Finetune/Step": 1393, "Finetune/Step Time": 5.4273325353860855} +{"Finetune/Learning Rate": 1.716622456327776e-05, "Finetune/Loss": 0.9684723019599915, "Finetune/Loss (Raw)": 1.0414377450942993, "Finetune/Step": 1394, "Finetune/Step Time": 5.414136363193393} +{"Finetune/Learning Rate": 1.7161878260720826e-05, "Finetune/Loss": 0.9678118228912354, "Finetune/Loss (Raw)": 0.9607327580451965, "Finetune/Step": 1395, "Finetune/Step Time": 5.41666323877871} +{"Finetune/Learning Rate": 1.7157529178778757e-05, "Finetune/Loss": 0.9653161764144897, "Finetune/Loss (Raw)": 0.7518686056137085, "Finetune/Step": 1396, "Finetune/Step Time": 5.429191380739212} +{"Finetune/Learning Rate": 1.7153177319139352e-05, "Finetune/Loss": 0.96610426902771, "Finetune/Loss (Raw)": 1.059483528137207, "Finetune/Step": 1397, "Finetune/Step Time": 5.4531920328736305} +{"Finetune/Learning Rate": 1.7148822683491483e-05, "Finetune/Loss": 0.9650518298149109, "Finetune/Loss (Raw)": 1.0042489767074585, "Finetune/Step": 1398, "Finetune/Step Time": 5.442920669913292} +{"Finetune/Learning Rate": 1.7144465273525105e-05, "Finetune/Loss": 0.9629197716712952, "Finetune/Loss (Raw)": 0.828326404094696, "Finetune/Step": 1399, "Finetune/Step Time": 5.449311373755336} +{"Finetune/Learning Rate": 1.7140105090931232e-05, "Finetune/Loss": 0.9647293090820312, "Finetune/Loss (Raw)": 1.0507941246032715, "Finetune/Step": 1400, "Finetune/Step Time": 5.444272829219699} +{"Finetune/Learning Rate": 1.7135742137401976e-05, "Finetune/Loss": 0.9646178483963013, "Finetune/Loss (Raw)": 0.9257989525794983, "Finetune/Step": 1401, "Finetune/Step Time": 5.442588269710541} +{"Finetune/Learning Rate": 1.7131376414630506e-05, "Finetune/Loss": 0.9608428478240967, "Finetune/Loss (Raw)": 0.39499568939208984, "Finetune/Step": 1402, "Finetune/Step Time": 5.475265860557556} +{"Finetune/Learning Rate": 1.712700792431108e-05, "Finetune/Loss": 0.9590622186660767, "Finetune/Loss (Raw)": 0.9407876133918762, "Finetune/Step": 1403, "Finetune/Step Time": 5.47562725096941} +{"Finetune/Learning Rate": 1.7122636668139022e-05, "Finetune/Loss": 0.95925372838974, "Finetune/Loss (Raw)": 0.9784747958183289, "Finetune/Step": 1404, "Finetune/Step Time": 5.472161645069718} +{"Finetune/Learning Rate": 1.711826264781073e-05, "Finetune/Loss": 0.9582728147506714, "Finetune/Loss (Raw)": 0.8440743684768677, "Finetune/Step": 1405, "Finetune/Step Time": 5.470423728227615} +{"Finetune/Learning Rate": 1.711388586502368e-05, "Finetune/Loss": 0.9599730968475342, "Finetune/Loss (Raw)": 0.965429425239563, "Finetune/Step": 1406, "Finetune/Step Time": 5.463653527200222} +{"Finetune/Learning Rate": 1.710950632147641e-05, "Finetune/Loss": 0.955074667930603, "Finetune/Loss (Raw)": 0.3739655017852783, "Finetune/Step": 1407, "Finetune/Step Time": 5.489166526123881} +{"Finetune/Learning Rate": 1.710512401886854e-05, "Finetune/Loss": 0.9528756141662598, "Finetune/Loss (Raw)": 0.8780291676521301, "Finetune/Step": 1408, "Finetune/Step Time": 5.491733426228166} +{"Finetune/Learning Rate": 1.7100738958900752e-05, "Finetune/Loss": 0.9532384872436523, "Finetune/Loss (Raw)": 1.0015056133270264, "Finetune/Step": 1409, "Finetune/Step Time": 5.483572270721197} +{"Finetune/Learning Rate": 1.7096351143274804e-05, "Finetune/Loss": 0.9525653719902039, "Finetune/Loss (Raw)": 0.9055018424987793, "Finetune/Step": 1410, "Finetune/Step Time": 5.512244151905179} +{"Finetune/Learning Rate": 1.7091960573693523e-05, "Finetune/Loss": 0.9539371132850647, "Finetune/Loss (Raw)": 1.062514066696167, "Finetune/Step": 1411, "Finetune/Step Time": 5.50088868662715} +{"Finetune/Learning Rate": 1.70875672518608e-05, "Finetune/Loss": 0.9521996974945068, "Finetune/Loss (Raw)": 0.7514623403549194, "Finetune/Step": 1412, "Finetune/Step Time": 5.5009983610361814} +{"Finetune/Learning Rate": 1.7083171179481603e-05, "Finetune/Loss": 0.9519748687744141, "Finetune/Loss (Raw)": 1.0304958820343018, "Finetune/Step": 1413, "Finetune/Step Time": 5.504599217325449} +{"Finetune/Learning Rate": 1.7078772358261957e-05, "Finetune/Loss": 0.953067421913147, "Finetune/Loss (Raw)": 1.1054805517196655, "Finetune/Step": 1414, "Finetune/Step Time": 5.494267230853438} +{"Finetune/Learning Rate": 1.707437078990896e-05, "Finetune/Loss": 0.9532543420791626, "Finetune/Loss (Raw)": 1.0045912265777588, "Finetune/Step": 1415, "Finetune/Step Time": 5.499187059700489} +{"Finetune/Learning Rate": 1.7069966476130773e-05, "Finetune/Loss": 0.9546486139297485, "Finetune/Loss (Raw)": 1.090035080909729, "Finetune/Step": 1416, "Finetune/Step Time": 5.4866486098617315} +{"Finetune/Learning Rate": 1.706555941863663e-05, "Finetune/Loss": 0.9547478556632996, "Finetune/Loss (Raw)": 0.9272820949554443, "Finetune/Step": 1417, "Finetune/Step Time": 5.48559895157814} +{"Finetune/Learning Rate": 1.7061149619136818e-05, "Finetune/Loss": 0.9528847336769104, "Finetune/Loss (Raw)": 0.8751897811889648, "Finetune/Step": 1418, "Finetune/Step Time": 5.497800901532173} +{"Finetune/Learning Rate": 1.7056737079342694e-05, "Finetune/Loss": 0.9520059823989868, "Finetune/Loss (Raw)": 0.8823877573013306, "Finetune/Step": 1419, "Finetune/Step Time": 5.486641522496939} +{"Finetune/Learning Rate": 1.7052321800966684e-05, "Finetune/Loss": 0.9523258209228516, "Finetune/Loss (Raw)": 1.0067894458770752, "Finetune/Step": 1420, "Finetune/Step Time": 5.48127381503582} +{"Finetune/Learning Rate": 1.7047903785722268e-05, "Finetune/Loss": 0.9520699977874756, "Finetune/Loss (Raw)": 0.9232710003852844, "Finetune/Step": 1421, "Finetune/Step Time": 5.486880540847778} +{"Finetune/Learning Rate": 1.704348303532399e-05, "Finetune/Loss": 0.9516713619232178, "Finetune/Loss (Raw)": 0.9406738877296448, "Finetune/Step": 1422, "Finetune/Step Time": 5.483343852683902} +{"Finetune/Learning Rate": 1.7039059551487457e-05, "Finetune/Loss": 0.9526427984237671, "Finetune/Loss (Raw)": 1.052307367324829, "Finetune/Step": 1423, "Finetune/Step Time": 5.473573824390769} +{"Finetune/Learning Rate": 1.703463333592934e-05, "Finetune/Loss": 0.9517687559127808, "Finetune/Loss (Raw)": 0.8584707379341125, "Finetune/Step": 1424, "Finetune/Step Time": 5.473477449268103} +{"Finetune/Learning Rate": 1.7030204390367368e-05, "Finetune/Loss": 0.9517574310302734, "Finetune/Loss (Raw)": 1.0064514875411987, "Finetune/Step": 1425, "Finetune/Step Time": 5.467247158288956} +{"Finetune/Learning Rate": 1.7025772716520324e-05, "Finetune/Loss": 0.9519649147987366, "Finetune/Loss (Raw)": 0.8897795677185059, "Finetune/Step": 1426, "Finetune/Step Time": 5.480328151956201} +{"Finetune/Learning Rate": 1.702133831610805e-05, "Finetune/Loss": 0.9534825086593628, "Finetune/Loss (Raw)": 0.9441075325012207, "Finetune/Step": 1427, "Finetune/Step Time": 5.4797364473342896} +{"Finetune/Learning Rate": 1.701690119085146e-05, "Finetune/Loss": 0.9538236856460571, "Finetune/Loss (Raw)": 0.9407430291175842, "Finetune/Step": 1428, "Finetune/Step Time": 5.485974041745067} +{"Finetune/Learning Rate": 1.701246134247251e-05, "Finetune/Loss": 0.9526858329772949, "Finetune/Loss (Raw)": 0.8761405944824219, "Finetune/Step": 1429, "Finetune/Step Time": 5.476508295163512} +{"Finetune/Learning Rate": 1.700801877269422e-05, "Finetune/Loss": 0.9527936577796936, "Finetune/Loss (Raw)": 0.9585750699043274, "Finetune/Step": 1430, "Finetune/Step Time": 5.46777443960309} +{"Finetune/Learning Rate": 1.7003573483240666e-05, "Finetune/Loss": 0.9521173238754272, "Finetune/Loss (Raw)": 0.9002642631530762, "Finetune/Step": 1431, "Finetune/Step Time": 5.4789253901690245} +{"Finetune/Learning Rate": 1.6999125475836973e-05, "Finetune/Loss": 0.9518146514892578, "Finetune/Loss (Raw)": 0.9661290645599365, "Finetune/Step": 1432, "Finetune/Step Time": 5.481910930946469} +{"Finetune/Learning Rate": 1.6994674752209334e-05, "Finetune/Loss": 0.9520424008369446, "Finetune/Loss (Raw)": 1.0001739263534546, "Finetune/Step": 1433, "Finetune/Step Time": 5.483225932344794} +{"Finetune/Learning Rate": 1.6990221314084976e-05, "Finetune/Loss": 0.9471695423126221, "Finetune/Loss (Raw)": 0.3765051066875458, "Finetune/Step": 1434, "Finetune/Step Time": 5.492473689839244} +{"Finetune/Learning Rate": 1.6985765163192204e-05, "Finetune/Loss": 0.946678102016449, "Finetune/Loss (Raw)": 1.0198686122894287, "Finetune/Step": 1435, "Finetune/Step Time": 5.499592710286379} +{"Finetune/Learning Rate": 1.698130630126036e-05, "Finetune/Loss": 0.944638729095459, "Finetune/Loss (Raw)": 0.8128087520599365, "Finetune/Step": 1436, "Finetune/Step Time": 5.5157616920769215} +{"Finetune/Learning Rate": 1.697684473001983e-05, "Finetune/Loss": 0.9454118013381958, "Finetune/Loss (Raw)": 1.0531810522079468, "Finetune/Step": 1437, "Finetune/Step Time": 5.498495265841484} +{"Finetune/Learning Rate": 1.6972380451202083e-05, "Finetune/Loss": 0.9459570646286011, "Finetune/Loss (Raw)": 0.9820831418037415, "Finetune/Step": 1438, "Finetune/Step Time": 5.493847759440541} +{"Finetune/Learning Rate": 1.69679134665396e-05, "Finetune/Loss": 0.9461930394172668, "Finetune/Loss (Raw)": 1.040840983390808, "Finetune/Step": 1439, "Finetune/Step Time": 5.490802515298128} +{"Finetune/Learning Rate": 1.696344377776594e-05, "Finetune/Loss": 0.945784866809845, "Finetune/Loss (Raw)": 0.7177424430847168, "Finetune/Step": 1440, "Finetune/Step Time": 5.495925676077604} +{"Finetune/Learning Rate": 1.69589713866157e-05, "Finetune/Loss": 0.945226788520813, "Finetune/Loss (Raw)": 0.9455518126487732, "Finetune/Step": 1441, "Finetune/Step Time": 5.492080802097917} +{"Finetune/Learning Rate": 1.6954496294824526e-05, "Finetune/Loss": 0.9452136754989624, "Finetune/Loss (Raw)": 1.0021060705184937, "Finetune/Step": 1442, "Finetune/Step Time": 5.497732393443584} +{"Finetune/Learning Rate": 1.6950018504129112e-05, "Finetune/Loss": 0.9445878267288208, "Finetune/Loss (Raw)": 0.9145077466964722, "Finetune/Step": 1443, "Finetune/Step Time": 5.501313738524914} +{"Finetune/Learning Rate": 1.6945538016267206e-05, "Finetune/Loss": 0.9426528811454773, "Finetune/Loss (Raw)": 0.9198429584503174, "Finetune/Step": 1444, "Finetune/Step Time": 5.498861478641629} +{"Finetune/Learning Rate": 1.6941054832977598e-05, "Finetune/Loss": 0.9415732622146606, "Finetune/Loss (Raw)": 0.9239839315414429, "Finetune/Step": 1445, "Finetune/Step Time": 5.501198723912239} +{"Finetune/Learning Rate": 1.6936568956000123e-05, "Finetune/Loss": 0.9407859444618225, "Finetune/Loss (Raw)": 0.8575277328491211, "Finetune/Step": 1446, "Finetune/Step Time": 5.5160631109029055} +{"Finetune/Learning Rate": 1.693208038707566e-05, "Finetune/Loss": 0.9405394196510315, "Finetune/Loss (Raw)": 0.968680739402771, "Finetune/Step": 1447, "Finetune/Step Time": 5.525787353515625} +{"Finetune/Learning Rate": 1.6927589127946136e-05, "Finetune/Loss": 0.9412367343902588, "Finetune/Loss (Raw)": 1.0662484169006348, "Finetune/Step": 1448, "Finetune/Step Time": 5.516887344419956} +{"Finetune/Learning Rate": 1.692309518035452e-05, "Finetune/Loss": 0.9400484561920166, "Finetune/Loss (Raw)": 0.859403669834137, "Finetune/Step": 1449, "Finetune/Step Time": 5.514057816937566} +{"Finetune/Learning Rate": 1.691859854604483e-05, "Finetune/Loss": 0.9392412304878235, "Finetune/Loss (Raw)": 0.8828645348548889, "Finetune/Step": 1450, "Finetune/Step Time": 5.549982028082013} +{"Finetune/Learning Rate": 1.6914099226762116e-05, "Finetune/Loss": 0.9381608963012695, "Finetune/Loss (Raw)": 0.9158074855804443, "Finetune/Step": 1451, "Finetune/Step Time": 5.553000707179308} +{"Finetune/Learning Rate": 1.6909597224252478e-05, "Finetune/Loss": 0.938494861125946, "Finetune/Loss (Raw)": 1.0678343772888184, "Finetune/Step": 1452, "Finetune/Step Time": 5.544824989512563} +{"Finetune/Learning Rate": 1.6905092540263055e-05, "Finetune/Loss": 0.9372855424880981, "Finetune/Loss (Raw)": 0.9105522632598877, "Finetune/Step": 1453, "Finetune/Step Time": 5.539685931056738} +{"Finetune/Learning Rate": 1.690058517654203e-05, "Finetune/Loss": 0.938575029373169, "Finetune/Loss (Raw)": 1.2042546272277832, "Finetune/Step": 1454, "Finetune/Step Time": 5.542350213974714} +{"Finetune/Learning Rate": 1.689607513483862e-05, "Finetune/Loss": 0.9392253160476685, "Finetune/Loss (Raw)": 1.0121699571609497, "Finetune/Step": 1455, "Finetune/Step Time": 5.515211230143905} +{"Finetune/Learning Rate": 1.6891562416903083e-05, "Finetune/Loss": 0.9376212358474731, "Finetune/Loss (Raw)": 0.9410024881362915, "Finetune/Step": 1456, "Finetune/Step Time": 5.5169864892959595} +{"Finetune/Learning Rate": 1.688704702448672e-05, "Finetune/Loss": 0.9313420057296753, "Finetune/Loss (Raw)": 0.3762179911136627, "Finetune/Step": 1457, "Finetune/Step Time": 5.540608879178762} +{"Finetune/Learning Rate": 1.6882528959341865e-05, "Finetune/Loss": 0.9326943159103394, "Finetune/Loss (Raw)": 0.9511494040489197, "Finetune/Step": 1458, "Finetune/Step Time": 5.53758660890162} +{"Finetune/Learning Rate": 1.6878008223221895e-05, "Finetune/Loss": 0.9330030083656311, "Finetune/Loss (Raw)": 1.0956283807754517, "Finetune/Step": 1459, "Finetune/Step Time": 5.546062748879194} +{"Finetune/Learning Rate": 1.687348481788121e-05, "Finetune/Loss": 0.9347617626190186, "Finetune/Loss (Raw)": 1.0997346639633179, "Finetune/Step": 1460, "Finetune/Step Time": 5.535617189481854} +{"Finetune/Learning Rate": 1.6868958745075262e-05, "Finetune/Loss": 0.9359601736068726, "Finetune/Loss (Raw)": 1.1204735040664673, "Finetune/Step": 1461, "Finetune/Step Time": 5.528879519551992} +{"Finetune/Learning Rate": 1.6864430006560527e-05, "Finetune/Loss": 0.9362070560455322, "Finetune/Loss (Raw)": 1.0461398363113403, "Finetune/Step": 1462, "Finetune/Step Time": 5.531347222626209} +{"Finetune/Learning Rate": 1.685989860409453e-05, "Finetune/Loss": 0.9363159537315369, "Finetune/Loss (Raw)": 1.0556694269180298, "Finetune/Step": 1463, "Finetune/Step Time": 5.532421819865704} +{"Finetune/Learning Rate": 1.6855364539435805e-05, "Finetune/Loss": 0.9352075457572937, "Finetune/Loss (Raw)": 0.8907225131988525, "Finetune/Step": 1464, "Finetune/Step Time": 5.53752700611949} +{"Finetune/Learning Rate": 1.685082781434395e-05, "Finetune/Loss": 0.9340133666992188, "Finetune/Loss (Raw)": 0.9798634052276611, "Finetune/Step": 1465, "Finetune/Step Time": 5.530493004247546} +{"Finetune/Learning Rate": 1.6846288430579565e-05, "Finetune/Loss": 0.9391683340072632, "Finetune/Loss (Raw)": 0.9747637510299683, "Finetune/Step": 1466, "Finetune/Step Time": 5.50434915907681} +{"Finetune/Learning Rate": 1.6841746389904306e-05, "Finetune/Loss": 0.9394626617431641, "Finetune/Loss (Raw)": 0.9586236476898193, "Finetune/Step": 1467, "Finetune/Step Time": 5.522436276078224} +{"Finetune/Learning Rate": 1.683720169408085e-05, "Finetune/Loss": 0.9396812915802002, "Finetune/Loss (Raw)": 1.0295007228851318, "Finetune/Step": 1468, "Finetune/Step Time": 5.514738176018} +{"Finetune/Learning Rate": 1.6832654344872893e-05, "Finetune/Loss": 0.9352874755859375, "Finetune/Loss (Raw)": 0.3898600935935974, "Finetune/Step": 1469, "Finetune/Step Time": 5.553345123305917} +{"Finetune/Learning Rate": 1.6828104344045186e-05, "Finetune/Loss": 0.9343891739845276, "Finetune/Loss (Raw)": 0.9962789416313171, "Finetune/Step": 1470, "Finetune/Step Time": 5.54070570319891} +{"Finetune/Learning Rate": 1.682355169336349e-05, "Finetune/Loss": 0.9350355863571167, "Finetune/Loss (Raw)": 0.9995272755622864, "Finetune/Step": 1471, "Finetune/Step Time": 5.5144858453422785} +{"Finetune/Learning Rate": 1.6818996394594603e-05, "Finetune/Loss": 0.9343751668930054, "Finetune/Loss (Raw)": 0.8680136203765869, "Finetune/Step": 1472, "Finetune/Step Time": 5.5077871810644865} +{"Finetune/Learning Rate": 1.681443844950634e-05, "Finetune/Loss": 0.9315610527992249, "Finetune/Loss (Raw)": 0.7104480862617493, "Finetune/Step": 1473, "Finetune/Step Time": 5.514121377840638} +{"Finetune/Learning Rate": 1.6809877859867557e-05, "Finetune/Loss": 0.9329257607460022, "Finetune/Loss (Raw)": 1.1299604177474976, "Finetune/Step": 1474, "Finetune/Step Time": 5.508562013506889} +{"Finetune/Learning Rate": 1.680531462744813e-05, "Finetune/Loss": 0.9298492670059204, "Finetune/Loss (Raw)": 0.7685053944587708, "Finetune/Step": 1475, "Finetune/Step Time": 5.501115724444389} +{"Finetune/Learning Rate": 1.680074875401896e-05, "Finetune/Loss": 0.9343629479408264, "Finetune/Loss (Raw)": 1.0084946155548096, "Finetune/Step": 1476, "Finetune/Step Time": 5.472652839496732} +{"Finetune/Learning Rate": 1.679618024135197e-05, "Finetune/Loss": 0.9283162355422974, "Finetune/Loss (Raw)": 0.3142560124397278, "Finetune/Step": 1477, "Finetune/Step Time": 5.509238878265023} +{"Finetune/Learning Rate": 1.679160909122011e-05, "Finetune/Loss": 0.9284560680389404, "Finetune/Loss (Raw)": 1.0740277767181396, "Finetune/Step": 1478, "Finetune/Step Time": 5.500704390928149} +{"Finetune/Learning Rate": 1.6787035305397363e-05, "Finetune/Loss": 0.9273690581321716, "Finetune/Loss (Raw)": 0.9032157063484192, "Finetune/Step": 1479, "Finetune/Step Time": 5.515234442427754} +{"Finetune/Learning Rate": 1.6782458885658716e-05, "Finetune/Loss": 0.9316307902336121, "Finetune/Loss (Raw)": 0.9717029929161072, "Finetune/Step": 1480, "Finetune/Step Time": 5.478010829538107} +{"Finetune/Learning Rate": 1.6777879833780192e-05, "Finetune/Loss": 0.9319651126861572, "Finetune/Loss (Raw)": 0.9957219362258911, "Finetune/Step": 1481, "Finetune/Step Time": 5.483934642747045} +{"Finetune/Learning Rate": 1.677329815153883e-05, "Finetune/Loss": 0.9316450357437134, "Finetune/Loss (Raw)": 0.9666322469711304, "Finetune/Step": 1482, "Finetune/Step Time": 5.470728686079383} +{"Finetune/Learning Rate": 1.6768713840712692e-05, "Finetune/Loss": 0.9302794933319092, "Finetune/Loss (Raw)": 0.9320995211601257, "Finetune/Step": 1483, "Finetune/Step Time": 5.494979744777083} +{"Finetune/Learning Rate": 1.6764126903080865e-05, "Finetune/Loss": 0.9300082325935364, "Finetune/Loss (Raw)": 1.037955641746521, "Finetune/Step": 1484, "Finetune/Step Time": 5.491223093122244} +{"Finetune/Learning Rate": 1.675953734042344e-05, "Finetune/Loss": 0.9288586378097534, "Finetune/Loss (Raw)": 0.9161445498466492, "Finetune/Step": 1485, "Finetune/Step Time": 5.479716692119837} +{"Finetune/Learning Rate": 1.6754945154521548e-05, "Finetune/Loss": 0.9236121773719788, "Finetune/Loss (Raw)": 0.31182894110679626, "Finetune/Step": 1486, "Finetune/Step Time": 5.510442057624459} +{"Finetune/Learning Rate": 1.6750350347157315e-05, "Finetune/Loss": 0.925473153591156, "Finetune/Loss (Raw)": 1.1006938219070435, "Finetune/Step": 1487, "Finetune/Step Time": 5.51068851724267} +{"Finetune/Learning Rate": 1.6745752920113907e-05, "Finetune/Loss": 0.9270182847976685, "Finetune/Loss (Raw)": 1.201174259185791, "Finetune/Step": 1488, "Finetune/Step Time": 5.5031004045158625} +{"Finetune/Learning Rate": 1.6741152875175485e-05, "Finetune/Loss": 0.9249107241630554, "Finetune/Loss (Raw)": 0.8201001882553101, "Finetune/Step": 1489, "Finetune/Step Time": 5.517626482993364} +{"Finetune/Learning Rate": 1.6736550214127247e-05, "Finetune/Loss": 0.9247695803642273, "Finetune/Loss (Raw)": 0.9489836096763611, "Finetune/Step": 1490, "Finetune/Step Time": 5.543536743149161} +{"Finetune/Learning Rate": 1.673194493875539e-05, "Finetune/Loss": 0.9235997200012207, "Finetune/Loss (Raw)": 0.8451249003410339, "Finetune/Step": 1491, "Finetune/Step Time": 5.562387079000473} +{"Finetune/Learning Rate": 1.672733705084713e-05, "Finetune/Loss": 0.9245234727859497, "Finetune/Loss (Raw)": 1.0810073614120483, "Finetune/Step": 1492, "Finetune/Step Time": 5.533749110996723} +{"Finetune/Learning Rate": 1.67227265521907e-05, "Finetune/Loss": 0.9200387001037598, "Finetune/Loss (Raw)": 0.3342149257659912, "Finetune/Step": 1493, "Finetune/Step Time": 5.560468407347798} +{"Finetune/Learning Rate": 1.671811344457535e-05, "Finetune/Loss": 0.9198985695838928, "Finetune/Loss (Raw)": 0.9436521530151367, "Finetune/Step": 1494, "Finetune/Step Time": 5.564085533842444} +{"Finetune/Learning Rate": 1.671349772979133e-05, "Finetune/Loss": 0.920843780040741, "Finetune/Loss (Raw)": 1.0474607944488525, "Finetune/Step": 1495, "Finetune/Step Time": 5.549667853862047} +{"Finetune/Learning Rate": 1.6708879409629908e-05, "Finetune/Loss": 0.9204662442207336, "Finetune/Loss (Raw)": 0.9846211671829224, "Finetune/Step": 1496, "Finetune/Step Time": 5.549842938780785} +{"Finetune/Learning Rate": 1.670425848588337e-05, "Finetune/Loss": 0.9189022779464722, "Finetune/Loss (Raw)": 0.8509084582328796, "Finetune/Step": 1497, "Finetune/Step Time": 5.538655420765281} +{"Finetune/Learning Rate": 1.6699634960345005e-05, "Finetune/Loss": 0.920634388923645, "Finetune/Loss (Raw)": 1.0758984088897705, "Finetune/Step": 1498, "Finetune/Step Time": 5.541626950725913} +{"Finetune/Learning Rate": 1.6695008834809107e-05, "Finetune/Loss": 0.9163506031036377, "Finetune/Loss (Raw)": 0.42922088503837585, "Finetune/Step": 1499, "Finetune/Step Time": 5.566888969391584} +{"Finetune/Learning Rate": 1.669038011107099e-05, "Finetune/Loss": 0.9161414504051208, "Finetune/Loss (Raw)": 0.3540843427181244, "Finetune/Step": 1500, "Finetune/Step Time": 5.570781722664833} +{"Finetune/Learning Rate": 1.6685748790926975e-05, "Finetune/Loss": 0.9154812097549438, "Finetune/Loss (Raw)": 1.0447556972503662, "Finetune/Step": 1501, "Finetune/Step Time": 5.603448385372758} +{"Finetune/Learning Rate": 1.6681114876174377e-05, "Finetune/Loss": 0.9168257713317871, "Finetune/Loss (Raw)": 1.1099120378494263, "Finetune/Step": 1502, "Finetune/Step Time": 5.5958218071609735} +{"Finetune/Learning Rate": 1.6676478368611536e-05, "Finetune/Loss": 0.9154261350631714, "Finetune/Loss (Raw)": 0.870603084564209, "Finetune/Step": 1503, "Finetune/Step Time": 5.598685430362821} +{"Finetune/Learning Rate": 1.667183927003779e-05, "Finetune/Loss": 0.9208073616027832, "Finetune/Loss (Raw)": 1.0067898035049438, "Finetune/Step": 1504, "Finetune/Step Time": 5.579107325524092} +{"Finetune/Learning Rate": 1.6667197582253474e-05, "Finetune/Loss": 0.920100212097168, "Finetune/Loss (Raw)": 0.9863669872283936, "Finetune/Step": 1505, "Finetune/Step Time": 5.565380467101932} +{"Finetune/Learning Rate": 1.6662553307059947e-05, "Finetune/Loss": 0.9180178642272949, "Finetune/Loss (Raw)": 0.8863650560379028, "Finetune/Step": 1506, "Finetune/Step Time": 5.554419655352831} +{"Finetune/Learning Rate": 1.6657906446259554e-05, "Finetune/Loss": 0.9206615686416626, "Finetune/Loss (Raw)": 1.073947548866272, "Finetune/Step": 1507, "Finetune/Step Time": 5.533214846625924} +{"Finetune/Learning Rate": 1.6653257001655652e-05, "Finetune/Loss": 0.9212139248847961, "Finetune/Loss (Raw)": 1.0170212984085083, "Finetune/Step": 1508, "Finetune/Step Time": 5.537247929722071} +{"Finetune/Learning Rate": 1.664860497505261e-05, "Finetune/Loss": 0.9209239482879639, "Finetune/Loss (Raw)": 1.0202000141143799, "Finetune/Step": 1509, "Finetune/Step Time": 5.531117416918278} +{"Finetune/Learning Rate": 1.664395036825577e-05, "Finetune/Loss": 0.9216779470443726, "Finetune/Loss (Raw)": 1.0873634815216064, "Finetune/Step": 1510, "Finetune/Step Time": 5.530270580202341} +{"Finetune/Learning Rate": 1.6639293183071514e-05, "Finetune/Loss": 0.9232282042503357, "Finetune/Loss (Raw)": 1.1167036294937134, "Finetune/Step": 1511, "Finetune/Step Time": 5.529566016048193} +{"Finetune/Learning Rate": 1.6634633421307187e-05, "Finetune/Loss": 0.922838032245636, "Finetune/Loss (Raw)": 0.9185535311698914, "Finetune/Step": 1512, "Finetune/Step Time": 5.533352004364133} +{"Finetune/Learning Rate": 1.6629971084771165e-05, "Finetune/Loss": 0.9239193201065063, "Finetune/Loss (Raw)": 1.1596896648406982, "Finetune/Step": 1513, "Finetune/Step Time": 5.535301109775901} +{"Finetune/Learning Rate": 1.66253061752728e-05, "Finetune/Loss": 0.9243209362030029, "Finetune/Loss (Raw)": 0.9779122471809387, "Finetune/Step": 1514, "Finetune/Step Time": 5.539717132225633} +{"Finetune/Learning Rate": 1.662063869462246e-05, "Finetune/Loss": 0.9250341057777405, "Finetune/Loss (Raw)": 0.9599776864051819, "Finetune/Step": 1515, "Finetune/Step Time": 5.537779258564115} +{"Finetune/Learning Rate": 1.6615968644631498e-05, "Finetune/Loss": 0.9244623184204102, "Finetune/Loss (Raw)": 0.9570226669311523, "Finetune/Step": 1516, "Finetune/Step Time": 5.540127640590072} +{"Finetune/Learning Rate": 1.661129602711227e-05, "Finetune/Loss": 0.9234260320663452, "Finetune/Loss (Raw)": 0.8769887685775757, "Finetune/Step": 1517, "Finetune/Step Time": 5.547011995688081} +{"Finetune/Learning Rate": 1.6606620843878128e-05, "Finetune/Loss": 0.9227506518363953, "Finetune/Loss (Raw)": 1.0849908590316772, "Finetune/Step": 1518, "Finetune/Step Time": 5.541328001767397} +{"Finetune/Learning Rate": 1.6601943096743426e-05, "Finetune/Loss": 0.9224773645401001, "Finetune/Loss (Raw)": 0.8481303453445435, "Finetune/Step": 1519, "Finetune/Step Time": 5.526565242558718} +{"Finetune/Learning Rate": 1.6597262787523493e-05, "Finetune/Loss": 0.9184664487838745, "Finetune/Loss (Raw)": 0.39370298385620117, "Finetune/Step": 1520, "Finetune/Step Time": 5.521219041198492} +{"Finetune/Learning Rate": 1.6592579918034678e-05, "Finetune/Loss": 0.9190324544906616, "Finetune/Loss (Raw)": 1.054710030555725, "Finetune/Step": 1521, "Finetune/Step Time": 5.489220408722758} +{"Finetune/Learning Rate": 1.65878944900943e-05, "Finetune/Loss": 0.9181309938430786, "Finetune/Loss (Raw)": 0.9260449409484863, "Finetune/Step": 1522, "Finetune/Step Time": 5.500913120806217} +{"Finetune/Learning Rate": 1.6583206505520695e-05, "Finetune/Loss": 0.9170666933059692, "Finetune/Loss (Raw)": 0.8245097398757935, "Finetune/Step": 1523, "Finetune/Step Time": 5.515034347772598} +{"Finetune/Learning Rate": 1.6578515966133167e-05, "Finetune/Loss": 0.9183034896850586, "Finetune/Loss (Raw)": 0.9101837277412415, "Finetune/Step": 1524, "Finetune/Step Time": 5.527899164706469} +{"Finetune/Learning Rate": 1.6573822873752026e-05, "Finetune/Loss": 0.916856586933136, "Finetune/Loss (Raw)": 0.8742707371711731, "Finetune/Step": 1525, "Finetune/Step Time": 5.5160259790718555} +{"Finetune/Learning Rate": 1.6569127230198575e-05, "Finetune/Loss": 0.917522668838501, "Finetune/Loss (Raw)": 1.0895179510116577, "Finetune/Step": 1526, "Finetune/Step Time": 5.5154591128230095} +{"Finetune/Learning Rate": 1.6564429037295097e-05, "Finetune/Loss": 0.9184598922729492, "Finetune/Loss (Raw)": 0.9482883214950562, "Finetune/Step": 1527, "Finetune/Step Time": 5.510145699605346} +{"Finetune/Learning Rate": 1.6559728296864868e-05, "Finetune/Loss": 0.9135230779647827, "Finetune/Loss (Raw)": 0.4188740849494934, "Finetune/Step": 1528, "Finetune/Step Time": 5.535233216360211} +{"Finetune/Learning Rate": 1.6555025010732156e-05, "Finetune/Loss": 0.9145026206970215, "Finetune/Loss (Raw)": 1.051184058189392, "Finetune/Step": 1529, "Finetune/Step Time": 5.525610659271479} +{"Finetune/Learning Rate": 1.6550319180722213e-05, "Finetune/Loss": 0.9174245595932007, "Finetune/Loss (Raw)": 0.7690069675445557, "Finetune/Step": 1530, "Finetune/Step Time": 5.513084819540381} +{"Finetune/Learning Rate": 1.6545610808661278e-05, "Finetune/Loss": 0.9179363250732422, "Finetune/Loss (Raw)": 1.0062874555587769, "Finetune/Step": 1531, "Finetune/Step Time": 5.5167020708322525} +{"Finetune/Learning Rate": 1.654089989637658e-05, "Finetune/Loss": 0.9178508520126343, "Finetune/Loss (Raw)": 0.967535138130188, "Finetune/Step": 1532, "Finetune/Step Time": 5.525988131761551} +{"Finetune/Learning Rate": 1.6536186445696337e-05, "Finetune/Loss": 0.9183127880096436, "Finetune/Loss (Raw)": 0.9032063484191895, "Finetune/Step": 1533, "Finetune/Step Time": 5.519351039081812} +{"Finetune/Learning Rate": 1.653147045844974e-05, "Finetune/Loss": 0.9184197187423706, "Finetune/Loss (Raw)": 0.9791144728660583, "Finetune/Step": 1534, "Finetune/Step Time": 5.5272156819701195} +{"Finetune/Learning Rate": 1.6526751936466974e-05, "Finetune/Loss": 0.922491192817688, "Finetune/Loss (Raw)": 0.895109236240387, "Finetune/Step": 1535, "Finetune/Step Time": 5.498355161398649} +{"Finetune/Learning Rate": 1.6522030881579207e-05, "Finetune/Loss": 0.9226996898651123, "Finetune/Loss (Raw)": 0.9047202467918396, "Finetune/Step": 1536, "Finetune/Step Time": 5.504565645009279} +{"Finetune/Learning Rate": 1.6517307295618582e-05, "Finetune/Loss": 0.9219423532485962, "Finetune/Loss (Raw)": 0.9045741558074951, "Finetune/Step": 1537, "Finetune/Step Time": 5.504848826676607} +{"Finetune/Learning Rate": 1.6512581180418245e-05, "Finetune/Loss": 0.9182938933372498, "Finetune/Loss (Raw)": 0.438489705324173, "Finetune/Step": 1538, "Finetune/Step Time": 5.504902491346002} +{"Finetune/Learning Rate": 1.650785253781229e-05, "Finetune/Loss": 0.9177243113517761, "Finetune/Loss (Raw)": 0.9896128177642822, "Finetune/Step": 1539, "Finetune/Step Time": 5.5059121530503035} +{"Finetune/Learning Rate": 1.6503121369635822e-05, "Finetune/Loss": 0.9195741415023804, "Finetune/Loss (Raw)": 0.9882450699806213, "Finetune/Step": 1540, "Finetune/Step Time": 5.5055554155260324} +{"Finetune/Learning Rate": 1.6498387677724917e-05, "Finetune/Loss": 0.9199981689453125, "Finetune/Loss (Raw)": 1.0847669839859009, "Finetune/Step": 1541, "Finetune/Step Time": 5.506266640499234} +{"Finetune/Learning Rate": 1.6493651463916622e-05, "Finetune/Loss": 0.9185308218002319, "Finetune/Loss (Raw)": 0.9176641702651978, "Finetune/Step": 1542, "Finetune/Step Time": 5.508579725399613} +{"Finetune/Learning Rate": 1.6488912730048973e-05, "Finetune/Loss": 0.9173474311828613, "Finetune/Loss (Raw)": 0.8531051278114319, "Finetune/Step": 1543, "Finetune/Step Time": 5.5041982643306255} +{"Finetune/Learning Rate": 1.6484171477960978e-05, "Finetune/Loss": 0.9165585041046143, "Finetune/Loss (Raw)": 0.9890583157539368, "Finetune/Step": 1544, "Finetune/Step Time": 5.502764536067843} +{"Finetune/Learning Rate": 1.6479427709492622e-05, "Finetune/Loss": 0.9164499044418335, "Finetune/Loss (Raw)": 0.913388729095459, "Finetune/Step": 1545, "Finetune/Step Time": 5.505344973877072} +{"Finetune/Learning Rate": 1.6474681426484878e-05, "Finetune/Loss": 0.9180272817611694, "Finetune/Loss (Raw)": 1.077082872390747, "Finetune/Step": 1546, "Finetune/Step Time": 5.499217454344034} +{"Finetune/Learning Rate": 1.6469932630779677e-05, "Finetune/Loss": 0.9187748432159424, "Finetune/Loss (Raw)": 0.9780743718147278, "Finetune/Step": 1547, "Finetune/Step Time": 5.4985766764730215} +{"Finetune/Learning Rate": 1.6465181324219937e-05, "Finetune/Loss": 0.9190162420272827, "Finetune/Loss (Raw)": 1.037699580192566, "Finetune/Step": 1548, "Finetune/Step Time": 5.508233206346631} +{"Finetune/Learning Rate": 1.6460427508649546e-05, "Finetune/Loss": 0.9149467945098877, "Finetune/Loss (Raw)": 0.40237048268318176, "Finetune/Step": 1549, "Finetune/Step Time": 5.5335019286721945} +{"Finetune/Learning Rate": 1.6455671185913367e-05, "Finetune/Loss": 0.9154955148696899, "Finetune/Loss (Raw)": 1.0109153985977173, "Finetune/Step": 1550, "Finetune/Step Time": 5.536812216043472} +{"Finetune/Learning Rate": 1.645091235785724e-05, "Finetune/Loss": 0.9140337109565735, "Finetune/Loss (Raw)": 0.8651959896087646, "Finetune/Step": 1551, "Finetune/Step Time": 5.531102364882827} +{"Finetune/Learning Rate": 1.644615102632797e-05, "Finetune/Loss": 0.9152445793151855, "Finetune/Loss (Raw)": 1.013463020324707, "Finetune/Step": 1552, "Finetune/Step Time": 5.526208641007543} +{"Finetune/Learning Rate": 1.6441387193173337e-05, "Finetune/Loss": 0.9144502878189087, "Finetune/Loss (Raw)": 0.9047799706459045, "Finetune/Step": 1553, "Finetune/Step Time": 5.542547378689051} +{"Finetune/Learning Rate": 1.6436620860242093e-05, "Finetune/Loss": 0.9142636060714722, "Finetune/Loss (Raw)": 0.8658815026283264, "Finetune/Step": 1554, "Finetune/Step Time": 5.538251282647252} +{"Finetune/Learning Rate": 1.6431852029383955e-05, "Finetune/Loss": 0.9151782393455505, "Finetune/Loss (Raw)": 1.061181902885437, "Finetune/Step": 1555, "Finetune/Step Time": 5.541142243891954} +{"Finetune/Learning Rate": 1.6427080702449616e-05, "Finetune/Loss": 0.9147424697875977, "Finetune/Loss (Raw)": 0.884960949420929, "Finetune/Step": 1556, "Finetune/Step Time": 5.547331182286143} +{"Finetune/Learning Rate": 1.6422306881290734e-05, "Finetune/Loss": 0.9145762920379639, "Finetune/Loss (Raw)": 0.8548837900161743, "Finetune/Step": 1557, "Finetune/Step Time": 5.5515729784965515} +{"Finetune/Learning Rate": 1.641753056775994e-05, "Finetune/Loss": 0.9143568277359009, "Finetune/Loss (Raw)": 0.9304760098457336, "Finetune/Step": 1558, "Finetune/Step Time": 5.5410589426755905} +{"Finetune/Learning Rate": 1.641275176371082e-05, "Finetune/Loss": 0.9164025783538818, "Finetune/Loss (Raw)": 1.1621140241622925, "Finetune/Step": 1559, "Finetune/Step Time": 5.5376771576702595} +{"Finetune/Learning Rate": 1.6407970470997943e-05, "Finetune/Loss": 0.916978657245636, "Finetune/Loss (Raw)": 1.0398752689361572, "Finetune/Step": 1560, "Finetune/Step Time": 5.542808853089809} +{"Finetune/Learning Rate": 1.6403186691476828e-05, "Finetune/Loss": 0.9170935153961182, "Finetune/Loss (Raw)": 1.0148802995681763, "Finetune/Step": 1561, "Finetune/Step Time": 5.54009417258203} +{"Finetune/Learning Rate": 1.639840042700397e-05, "Finetune/Loss": 0.922027051448822, "Finetune/Loss (Raw)": 1.0079928636550903, "Finetune/Step": 1562, "Finetune/Step Time": 5.518112087622285} +{"Finetune/Learning Rate": 1.6393611679436828e-05, "Finetune/Loss": 0.9213858842849731, "Finetune/Loss (Raw)": 0.937790036201477, "Finetune/Step": 1563, "Finetune/Step Time": 5.541894730180502} +{"Finetune/Learning Rate": 1.6388820450633813e-05, "Finetune/Loss": 0.9221986532211304, "Finetune/Loss (Raw)": 0.9168513417243958, "Finetune/Step": 1564, "Finetune/Step Time": 5.521805005148053} +{"Finetune/Learning Rate": 1.6384026742454315e-05, "Finetune/Loss": 0.9206516742706299, "Finetune/Loss (Raw)": 0.8551651835441589, "Finetune/Step": 1565, "Finetune/Step Time": 5.52127162925899} +{"Finetune/Learning Rate": 1.6379230556758676e-05, "Finetune/Loss": 0.9205442667007446, "Finetune/Loss (Raw)": 0.9683321118354797, "Finetune/Step": 1566, "Finetune/Step Time": 5.528105014935136} +{"Finetune/Learning Rate": 1.63744318954082e-05, "Finetune/Loss": 0.9201823472976685, "Finetune/Loss (Raw)": 0.9945147037506104, "Finetune/Step": 1567, "Finetune/Step Time": 5.534454423934221} +{"Finetune/Learning Rate": 1.636963076026516e-05, "Finetune/Loss": 0.9221432209014893, "Finetune/Loss (Raw)": 0.9687373638153076, "Finetune/Step": 1568, "Finetune/Step Time": 5.506097637116909} +{"Finetune/Learning Rate": 1.6364827153192773e-05, "Finetune/Loss": 0.9213591814041138, "Finetune/Loss (Raw)": 0.8452048897743225, "Finetune/Step": 1569, "Finetune/Step Time": 5.518506146967411} +{"Finetune/Learning Rate": 1.6360021076055234e-05, "Finetune/Loss": 0.9217010736465454, "Finetune/Loss (Raw)": 1.0458614826202393, "Finetune/Step": 1570, "Finetune/Step Time": 5.515479568392038} +{"Finetune/Learning Rate": 1.6355212530717683e-05, "Finetune/Loss": 0.9221001863479614, "Finetune/Loss (Raw)": 0.9655861854553223, "Finetune/Step": 1571, "Finetune/Step Time": 5.511473476886749} +{"Finetune/Learning Rate": 1.6350401519046226e-05, "Finetune/Loss": 0.9228917360305786, "Finetune/Loss (Raw)": 1.0211694240570068, "Finetune/Step": 1572, "Finetune/Step Time": 5.505309730768204} +{"Finetune/Learning Rate": 1.634558804290792e-05, "Finetune/Loss": 0.9234537482261658, "Finetune/Loss (Raw)": 0.9959200620651245, "Finetune/Step": 1573, "Finetune/Step Time": 5.521212754771113} +{"Finetune/Learning Rate": 1.634077210417078e-05, "Finetune/Loss": 0.9232566952705383, "Finetune/Loss (Raw)": 0.8323063850402832, "Finetune/Step": 1574, "Finetune/Step Time": 5.504905242472887} +{"Finetune/Learning Rate": 1.633595370470378e-05, "Finetune/Loss": 0.925064206123352, "Finetune/Loss (Raw)": 1.2000397443771362, "Finetune/Step": 1575, "Finetune/Step Time": 5.5095756240189075} +{"Finetune/Learning Rate": 1.6331132846376843e-05, "Finetune/Loss": 0.9243155717849731, "Finetune/Loss (Raw)": 0.9704256653785706, "Finetune/Step": 1576, "Finetune/Step Time": 5.512884765863419} +{"Finetune/Learning Rate": 1.632630953106086e-05, "Finetune/Loss": 0.9251406788825989, "Finetune/Loss (Raw)": 0.9650242328643799, "Finetune/Step": 1577, "Finetune/Step Time": 5.503953201696277} +{"Finetune/Learning Rate": 1.6321483760627658e-05, "Finetune/Loss": 0.9260040521621704, "Finetune/Loss (Raw)": 0.9933687448501587, "Finetune/Step": 1578, "Finetune/Step Time": 5.46153911575675} +{"Finetune/Learning Rate": 1.6316655536950017e-05, "Finetune/Loss": 0.9264422059059143, "Finetune/Loss (Raw)": 0.971889853477478, "Finetune/Step": 1579, "Finetune/Step Time": 5.467748740687966} +{"Finetune/Learning Rate": 1.6311824861901693e-05, "Finetune/Loss": 0.9241567254066467, "Finetune/Loss (Raw)": 0.7752957940101624, "Finetune/Step": 1580, "Finetune/Step Time": 5.502783356234431} +{"Finetune/Learning Rate": 1.6306991737357366e-05, "Finetune/Loss": 0.9237030744552612, "Finetune/Loss (Raw)": 0.8524821996688843, "Finetune/Step": 1581, "Finetune/Step Time": 5.497142590582371} +{"Finetune/Learning Rate": 1.630215616519268e-05, "Finetune/Loss": 0.9218666553497314, "Finetune/Loss (Raw)": 0.9691996574401855, "Finetune/Step": 1582, "Finetune/Step Time": 5.503645345568657} +{"Finetune/Learning Rate": 1.6297318147284223e-05, "Finetune/Loss": 0.9218031167984009, "Finetune/Loss (Raw)": 1.0040295124053955, "Finetune/Step": 1583, "Finetune/Step Time": 5.5010137520730495} +{"Finetune/Learning Rate": 1.6292477685509538e-05, "Finetune/Loss": 0.9220997095108032, "Finetune/Loss (Raw)": 0.9789649248123169, "Finetune/Step": 1584, "Finetune/Step Time": 5.502725383266807} +{"Finetune/Learning Rate": 1.6287634781747113e-05, "Finetune/Loss": 0.9274787306785583, "Finetune/Loss (Raw)": 1.06473708152771, "Finetune/Step": 1585, "Finetune/Step Time": 5.472052205353975} +{"Finetune/Learning Rate": 1.628278943787639e-05, "Finetune/Loss": 0.9288478493690491, "Finetune/Loss (Raw)": 1.1264015436172485, "Finetune/Step": 1586, "Finetune/Step Time": 5.53038932941854} +{"Finetune/Learning Rate": 1.6277941655777744e-05, "Finetune/Loss": 0.9285261631011963, "Finetune/Loss (Raw)": 1.0544482469558716, "Finetune/Step": 1587, "Finetune/Step Time": 5.523112341761589} +{"Finetune/Learning Rate": 1.6273091437332506e-05, "Finetune/Loss": 0.9274277687072754, "Finetune/Loss (Raw)": 0.9591430425643921, "Finetune/Step": 1588, "Finetune/Step Time": 5.533347774296999} +{"Finetune/Learning Rate": 1.6268238784422954e-05, "Finetune/Loss": 0.9259763956069946, "Finetune/Loss (Raw)": 0.9346917867660522, "Finetune/Step": 1589, "Finetune/Step Time": 5.549267694354057} +{"Finetune/Learning Rate": 1.6263383698932307e-05, "Finetune/Loss": 0.9258400201797485, "Finetune/Loss (Raw)": 1.0286840200424194, "Finetune/Step": 1590, "Finetune/Step Time": 5.559840448200703} +{"Finetune/Learning Rate": 1.6258526182744727e-05, "Finetune/Loss": 0.9241986274719238, "Finetune/Loss (Raw)": 0.8455795049667358, "Finetune/Step": 1591, "Finetune/Step Time": 5.575283112004399} +{"Finetune/Learning Rate": 1.6253666237745326e-05, "Finetune/Loss": 0.9241703748703003, "Finetune/Loss (Raw)": 0.8870994448661804, "Finetune/Step": 1592, "Finetune/Step Time": 5.577036378905177} +{"Finetune/Learning Rate": 1.6248803865820152e-05, "Finetune/Loss": 0.9237561821937561, "Finetune/Loss (Raw)": 0.9268510341644287, "Finetune/Step": 1593, "Finetune/Step Time": 5.580171089619398} +{"Finetune/Learning Rate": 1.624393906885619e-05, "Finetune/Loss": 0.9234791994094849, "Finetune/Loss (Raw)": 0.939312219619751, "Finetune/Step": 1594, "Finetune/Step Time": 5.585762195289135} +{"Finetune/Learning Rate": 1.6239071848741385e-05, "Finetune/Loss": 0.9227358102798462, "Finetune/Loss (Raw)": 0.8634570837020874, "Finetune/Step": 1595, "Finetune/Step Time": 5.563373629003763} +{"Finetune/Learning Rate": 1.6234202207364596e-05, "Finetune/Loss": 0.9234011173248291, "Finetune/Loss (Raw)": 1.1146687269210815, "Finetune/Step": 1596, "Finetune/Step Time": 5.568243505433202} +{"Finetune/Learning Rate": 1.6229330146615646e-05, "Finetune/Loss": 0.9269959330558777, "Finetune/Loss (Raw)": 0.849990725517273, "Finetune/Step": 1597, "Finetune/Step Time": 5.53521135635674} +{"Finetune/Learning Rate": 1.6224455668385283e-05, "Finetune/Loss": 0.9269554018974304, "Finetune/Loss (Raw)": 0.9910955429077148, "Finetune/Step": 1598, "Finetune/Step Time": 5.54857936874032} +{"Finetune/Learning Rate": 1.62195787745652e-05, "Finetune/Loss": 0.9221040606498718, "Finetune/Loss (Raw)": 0.37855085730552673, "Finetune/Step": 1599, "Finetune/Step Time": 5.580913575366139} +{"Finetune/Learning Rate": 1.6214699467048017e-05, "Finetune/Loss": 0.92312091588974, "Finetune/Loss (Raw)": 0.998173713684082, "Finetune/Step": 1600, "Finetune/Step Time": 5.583812991157174} +{"Finetune/Learning Rate": 1.6209817747727304e-05, "Finetune/Loss": 0.9205957651138306, "Finetune/Loss (Raw)": 0.38722288608551025, "Finetune/Step": 1601, "Finetune/Step Time": 5.6166262328624725} +{"Finetune/Learning Rate": 1.620493361849756e-05, "Finetune/Loss": 0.9202598333358765, "Finetune/Loss (Raw)": 1.0869718790054321, "Finetune/Step": 1602, "Finetune/Step Time": 5.62617108784616} +{"Finetune/Learning Rate": 1.6200047081254223e-05, "Finetune/Loss": 0.9222772121429443, "Finetune/Loss (Raw)": 1.0267199277877808, "Finetune/Step": 1603, "Finetune/Step Time": 5.621596639975905} +{"Finetune/Learning Rate": 1.6195158137893656e-05, "Finetune/Loss": 0.9214063882827759, "Finetune/Loss (Raw)": 0.8970391750335693, "Finetune/Step": 1604, "Finetune/Step Time": 5.62158134020865} +{"Finetune/Learning Rate": 1.6190266790313165e-05, "Finetune/Loss": 0.9254648685455322, "Finetune/Loss (Raw)": 0.8337419033050537, "Finetune/Step": 1605, "Finetune/Step Time": 5.599507011473179} +{"Finetune/Learning Rate": 1.618537304041099e-05, "Finetune/Loss": 0.9231153130531311, "Finetune/Loss (Raw)": 0.7732797265052795, "Finetune/Step": 1606, "Finetune/Step Time": 5.63124499283731} +{"Finetune/Learning Rate": 1.6180476890086297e-05, "Finetune/Loss": 0.9231332540512085, "Finetune/Loss (Raw)": 0.9055176377296448, "Finetune/Step": 1607, "Finetune/Step Time": 5.617747103795409} +{"Finetune/Learning Rate": 1.6175578341239188e-05, "Finetune/Loss": 0.9185765981674194, "Finetune/Loss (Raw)": 0.38844582438468933, "Finetune/Step": 1608, "Finetune/Step Time": 5.649111691862345} +{"Finetune/Learning Rate": 1.617067739577069e-05, "Finetune/Loss": 0.9183148741722107, "Finetune/Loss (Raw)": 0.9622207880020142, "Finetune/Step": 1609, "Finetune/Step Time": 5.64325862005353} +{"Finetune/Learning Rate": 1.616577405558277e-05, "Finetune/Loss": 0.9181007146835327, "Finetune/Loss (Raw)": 0.9392226934432983, "Finetune/Step": 1610, "Finetune/Step Time": 5.639567343518138} +{"Finetune/Learning Rate": 1.616086832257831e-05, "Finetune/Loss": 0.9175525903701782, "Finetune/Loss (Raw)": 0.8619273900985718, "Finetune/Step": 1611, "Finetune/Step Time": 5.625614730641246} +{"Finetune/Learning Rate": 1.615596019866114e-05, "Finetune/Loss": 0.9166728258132935, "Finetune/Loss (Raw)": 0.9253512620925903, "Finetune/Step": 1612, "Finetune/Step Time": 5.63365994207561} +{"Finetune/Learning Rate": 1.6151049685736e-05, "Finetune/Loss": 0.916244626045227, "Finetune/Loss (Raw)": 0.8613433241844177, "Finetune/Step": 1613, "Finetune/Step Time": 5.645770963281393} +{"Finetune/Learning Rate": 1.6146136785708564e-05, "Finetune/Loss": 0.9215723276138306, "Finetune/Loss (Raw)": 0.993770956993103, "Finetune/Step": 1614, "Finetune/Step Time": 5.614152576774359} +{"Finetune/Learning Rate": 1.6141221500485437e-05, "Finetune/Loss": 0.9212992787361145, "Finetune/Loss (Raw)": 1.0657386779785156, "Finetune/Step": 1615, "Finetune/Step Time": 5.602714983746409} +{"Finetune/Learning Rate": 1.6136303831974146e-05, "Finetune/Loss": 0.9194790124893188, "Finetune/Loss (Raw)": 0.9681823253631592, "Finetune/Step": 1616, "Finetune/Step Time": 5.601981047540903} +{"Finetune/Learning Rate": 1.6131383782083135e-05, "Finetune/Loss": 0.9201641082763672, "Finetune/Loss (Raw)": 0.9078009724617004, "Finetune/Step": 1617, "Finetune/Step Time": 5.598494898527861} +{"Finetune/Learning Rate": 1.6126461352721784e-05, "Finetune/Loss": 0.9198250770568848, "Finetune/Loss (Raw)": 0.9055823683738708, "Finetune/Step": 1618, "Finetune/Step Time": 5.572734527289867} +{"Finetune/Learning Rate": 1.612153654580039e-05, "Finetune/Loss": 0.9197351932525635, "Finetune/Loss (Raw)": 0.8336116075515747, "Finetune/Step": 1619, "Finetune/Step Time": 5.5672487784177065} +{"Finetune/Learning Rate": 1.6116609363230176e-05, "Finetune/Loss": 0.9202606678009033, "Finetune/Loss (Raw)": 1.1482748985290527, "Finetune/Step": 1620, "Finetune/Step Time": 5.569907059893012} +{"Finetune/Learning Rate": 1.6111679806923287e-05, "Finetune/Loss": 0.9243767261505127, "Finetune/Loss (Raw)": 0.8610736727714539, "Finetune/Step": 1621, "Finetune/Step Time": 5.544339442625642} +{"Finetune/Learning Rate": 1.6106747878792784e-05, "Finetune/Loss": 0.925383448600769, "Finetune/Loss (Raw)": 1.072511911392212, "Finetune/Step": 1622, "Finetune/Step Time": 5.548342837020755} +{"Finetune/Learning Rate": 1.610181358075265e-05, "Finetune/Loss": 0.9261263012886047, "Finetune/Loss (Raw)": 1.1425460577011108, "Finetune/Step": 1623, "Finetune/Step Time": 5.542068822309375} +{"Finetune/Learning Rate": 1.60968769147178e-05, "Finetune/Loss": 0.9256280660629272, "Finetune/Loss (Raw)": 0.9208433032035828, "Finetune/Step": 1624, "Finetune/Step Time": 5.544171309098601} +{"Finetune/Learning Rate": 1.6091937882604047e-05, "Finetune/Loss": 0.9267206192016602, "Finetune/Loss (Raw)": 0.9907587170600891, "Finetune/Step": 1625, "Finetune/Step Time": 5.544251434504986} +{"Finetune/Learning Rate": 1.608699648632814e-05, "Finetune/Loss": 0.9261852502822876, "Finetune/Loss (Raw)": 1.007373571395874, "Finetune/Step": 1626, "Finetune/Step Time": 5.5403260588645935} +{"Finetune/Learning Rate": 1.6082052727807732e-05, "Finetune/Loss": 0.9309772253036499, "Finetune/Loss (Raw)": 1.0425841808319092, "Finetune/Step": 1627, "Finetune/Step Time": 5.505859622731805} +{"Finetune/Learning Rate": 1.60771066089614e-05, "Finetune/Loss": 0.935942530632019, "Finetune/Loss (Raw)": 0.989647626876831, "Finetune/Step": 1628, "Finetune/Step Time": 5.469291593879461} +{"Finetune/Learning Rate": 1.6072158131708643e-05, "Finetune/Loss": 0.9352365732192993, "Finetune/Loss (Raw)": 0.9543968439102173, "Finetune/Step": 1629, "Finetune/Step Time": 5.445800147950649} +{"Finetune/Learning Rate": 1.6067207297969864e-05, "Finetune/Loss": 0.9346847534179688, "Finetune/Loss (Raw)": 1.0392756462097168, "Finetune/Step": 1630, "Finetune/Step Time": 5.443432178348303} +{"Finetune/Learning Rate": 1.6062254109666383e-05, "Finetune/Loss": 0.9353086948394775, "Finetune/Loss (Raw)": 0.9504682421684265, "Finetune/Step": 1631, "Finetune/Step Time": 5.4458760432899} +{"Finetune/Learning Rate": 1.6057298568720436e-05, "Finetune/Loss": 0.9347878694534302, "Finetune/Loss (Raw)": 0.9401240944862366, "Finetune/Step": 1632, "Finetune/Step Time": 5.436959909275174} +{"Finetune/Learning Rate": 1.6052340677055175e-05, "Finetune/Loss": 0.933902382850647, "Finetune/Loss (Raw)": 0.8730281591415405, "Finetune/Step": 1633, "Finetune/Step Time": 5.4372694082558155} +{"Finetune/Learning Rate": 1.6047380436594663e-05, "Finetune/Loss": 0.9348639249801636, "Finetune/Loss (Raw)": 1.0094397068023682, "Finetune/Step": 1634, "Finetune/Step Time": 5.429791362956166} +{"Finetune/Learning Rate": 1.6042417849263863e-05, "Finetune/Loss": 0.9341540336608887, "Finetune/Loss (Raw)": 0.9830800890922546, "Finetune/Step": 1635, "Finetune/Step Time": 5.451326327398419} +{"Finetune/Learning Rate": 1.6037452916988665e-05, "Finetune/Loss": 0.9339234828948975, "Finetune/Loss (Raw)": 0.9875041842460632, "Finetune/Step": 1636, "Finetune/Step Time": 5.447881622239947} +{"Finetune/Learning Rate": 1.6032485641695862e-05, "Finetune/Loss": 0.9334880709648132, "Finetune/Loss (Raw)": 0.9644745588302612, "Finetune/Step": 1637, "Finetune/Step Time": 5.449308617040515} +{"Finetune/Learning Rate": 1.602751602531316e-05, "Finetune/Loss": 0.9320909976959229, "Finetune/Loss (Raw)": 0.9085298776626587, "Finetune/Step": 1638, "Finetune/Step Time": 5.451040683314204} +{"Finetune/Learning Rate": 1.602254406976916e-05, "Finetune/Loss": 0.9299452304840088, "Finetune/Loss (Raw)": 0.8420575261116028, "Finetune/Step": 1639, "Finetune/Step Time": 5.448202660307288} +{"Finetune/Learning Rate": 1.6017569776993388e-05, "Finetune/Loss": 0.9307980537414551, "Finetune/Loss (Raw)": 1.0277093648910522, "Finetune/Step": 1640, "Finetune/Step Time": 5.436179284006357} +{"Finetune/Learning Rate": 1.601259314891627e-05, "Finetune/Loss": 0.9298311471939087, "Finetune/Loss (Raw)": 1.035923719406128, "Finetune/Step": 1641, "Finetune/Step Time": 5.433494485914707} +{"Finetune/Learning Rate": 1.6007614187469137e-05, "Finetune/Loss": 0.9299165606498718, "Finetune/Loss (Raw)": 0.9888473749160767, "Finetune/Step": 1642, "Finetune/Step Time": 5.422313133254647} +{"Finetune/Learning Rate": 1.6002632894584227e-05, "Finetune/Loss": 0.9308440685272217, "Finetune/Loss (Raw)": 1.0787038803100586, "Finetune/Step": 1643, "Finetune/Step Time": 5.4239745978266} +{"Finetune/Learning Rate": 1.599764927219468e-05, "Finetune/Loss": 0.9311045408248901, "Finetune/Loss (Raw)": 0.9903613924980164, "Finetune/Step": 1644, "Finetune/Step Time": 5.449436482042074} +{"Finetune/Learning Rate": 1.5992663322234548e-05, "Finetune/Loss": 0.9314554929733276, "Finetune/Loss (Raw)": 0.9219010472297668, "Finetune/Step": 1645, "Finetune/Step Time": 5.450985908508301} +{"Finetune/Learning Rate": 1.5987675046638777e-05, "Finetune/Loss": 0.931029200553894, "Finetune/Loss (Raw)": 1.030431866645813, "Finetune/Step": 1646, "Finetune/Step Time": 5.454144038259983} +{"Finetune/Learning Rate": 1.5982684447343214e-05, "Finetune/Loss": 0.931496262550354, "Finetune/Loss (Raw)": 0.9079132080078125, "Finetune/Step": 1647, "Finetune/Step Time": 5.467298174276948} +{"Finetune/Learning Rate": 1.5977691526284623e-05, "Finetune/Loss": 0.9366124868392944, "Finetune/Loss (Raw)": 1.048583984375, "Finetune/Step": 1648, "Finetune/Step Time": 5.440676610916853} +{"Finetune/Learning Rate": 1.5972696285400656e-05, "Finetune/Loss": 0.9357503652572632, "Finetune/Loss (Raw)": 0.9443597197532654, "Finetune/Step": 1649, "Finetune/Step Time": 5.437366735190153} +{"Finetune/Learning Rate": 1.596769872662987e-05, "Finetune/Loss": 0.9360578656196594, "Finetune/Loss (Raw)": 0.9654051661491394, "Finetune/Step": 1650, "Finetune/Step Time": 5.425920119509101} +{"Finetune/Learning Rate": 1.5962698851911715e-05, "Finetune/Loss": 0.9365195035934448, "Finetune/Loss (Raw)": 0.8835987448692322, "Finetune/Step": 1651, "Finetune/Step Time": 5.427845932543278} +{"Finetune/Learning Rate": 1.5957696663186547e-05, "Finetune/Loss": 0.9364233016967773, "Finetune/Loss (Raw)": 0.8978621363639832, "Finetune/Step": 1652, "Finetune/Step Time": 5.410768987610936} +{"Finetune/Learning Rate": 1.595269216239562e-05, "Finetune/Loss": 0.937203049659729, "Finetune/Loss (Raw)": 0.9740852117538452, "Finetune/Step": 1653, "Finetune/Step Time": 5.402988068759441} +{"Finetune/Learning Rate": 1.594768535148108e-05, "Finetune/Loss": 0.9366686344146729, "Finetune/Loss (Raw)": 1.0211055278778076, "Finetune/Step": 1654, "Finetune/Step Time": 5.421383656561375} +{"Finetune/Learning Rate": 1.5942676232385973e-05, "Finetune/Loss": 0.9322596192359924, "Finetune/Loss (Raw)": 0.3839384615421295, "Finetune/Step": 1655, "Finetune/Step Time": 5.451377805322409} +{"Finetune/Learning Rate": 1.5937664807054242e-05, "Finetune/Loss": 0.9366089105606079, "Finetune/Loss (Raw)": 0.9755837917327881, "Finetune/Step": 1656, "Finetune/Step Time": 5.422854740172625} +{"Finetune/Learning Rate": 1.593265107743072e-05, "Finetune/Loss": 0.935335099697113, "Finetune/Loss (Raw)": 0.8881371021270752, "Finetune/Step": 1657, "Finetune/Step Time": 5.421603916212916} +{"Finetune/Learning Rate": 1.5927635045461145e-05, "Finetune/Loss": 0.9369117021560669, "Finetune/Loss (Raw)": 0.9708144664764404, "Finetune/Step": 1658, "Finetune/Step Time": 5.405967639759183} +{"Finetune/Learning Rate": 1.5922616713092135e-05, "Finetune/Loss": 0.9375302791595459, "Finetune/Loss (Raw)": 1.0854594707489014, "Finetune/Step": 1659, "Finetune/Step Time": 5.4068511836230755} +{"Finetune/Learning Rate": 1.5917596082271206e-05, "Finetune/Loss": 0.9375910758972168, "Finetune/Loss (Raw)": 0.9753166437149048, "Finetune/Step": 1660, "Finetune/Step Time": 5.403772043064237} +{"Finetune/Learning Rate": 1.5912573154946768e-05, "Finetune/Loss": 0.9378160238265991, "Finetune/Loss (Raw)": 0.9320008158683777, "Finetune/Step": 1661, "Finetune/Step Time": 5.3991715759038925} +{"Finetune/Learning Rate": 1.5907547933068125e-05, "Finetune/Loss": 0.938503086566925, "Finetune/Loss (Raw)": 1.0670597553253174, "Finetune/Step": 1662, "Finetune/Step Time": 5.398693732917309} +{"Finetune/Learning Rate": 1.590252041858546e-05, "Finetune/Loss": 0.9370920062065125, "Finetune/Loss (Raw)": 0.714491069316864, "Finetune/Step": 1663, "Finetune/Step Time": 5.408390745520592} +{"Finetune/Learning Rate": 1.5897490613449864e-05, "Finetune/Loss": 0.9362694025039673, "Finetune/Loss (Raw)": 0.7994306087493896, "Finetune/Step": 1664, "Finetune/Step Time": 5.401080597192049} +{"Finetune/Learning Rate": 1.5892458519613296e-05, "Finetune/Loss": 0.9375563859939575, "Finetune/Loss (Raw)": 1.0693106651306152, "Finetune/Step": 1665, "Finetune/Step Time": 5.400905303657055} +{"Finetune/Learning Rate": 1.588742413902862e-05, "Finetune/Loss": 0.9412081837654114, "Finetune/Loss (Raw)": 0.9059189558029175, "Finetune/Step": 1666, "Finetune/Step Time": 5.365372339263558} +{"Finetune/Learning Rate": 1.5882387473649572e-05, "Finetune/Loss": 0.9413350224494934, "Finetune/Loss (Raw)": 1.005845308303833, "Finetune/Step": 1667, "Finetune/Step Time": 5.363129699602723} +{"Finetune/Learning Rate": 1.5877348525430793e-05, "Finetune/Loss": 0.9419341683387756, "Finetune/Loss (Raw)": 1.0649302005767822, "Finetune/Step": 1668, "Finetune/Step Time": 5.368015417829156} +{"Finetune/Learning Rate": 1.5872307296327795e-05, "Finetune/Loss": 0.9409784078598022, "Finetune/Loss (Raw)": 0.9624360799789429, "Finetune/Step": 1669, "Finetune/Step Time": 5.364740354940295} +{"Finetune/Learning Rate": 1.5867263788296984e-05, "Finetune/Loss": 0.941626787185669, "Finetune/Loss (Raw)": 1.000649094581604, "Finetune/Step": 1670, "Finetune/Step Time": 5.351088158786297} +{"Finetune/Learning Rate": 1.5862218003295647e-05, "Finetune/Loss": 0.9420187473297119, "Finetune/Loss (Raw)": 0.9032813310623169, "Finetune/Step": 1671, "Finetune/Step Time": 5.359054960310459} +{"Finetune/Learning Rate": 1.5857169943281948e-05, "Finetune/Loss": 0.9424716234207153, "Finetune/Loss (Raw)": 1.0470229387283325, "Finetune/Step": 1672, "Finetune/Step Time": 5.375834664329886} +{"Finetune/Learning Rate": 1.585211961021495e-05, "Finetune/Loss": 0.9419228434562683, "Finetune/Loss (Raw)": 0.8431516289710999, "Finetune/Step": 1673, "Finetune/Step Time": 5.38082080706954} +{"Finetune/Learning Rate": 1.5847067006054584e-05, "Finetune/Loss": 0.9397711157798767, "Finetune/Loss (Raw)": 0.8016579747200012, "Finetune/Step": 1674, "Finetune/Step Time": 5.387245558202267} +{"Finetune/Learning Rate": 1.584201213276167e-05, "Finetune/Loss": 0.939495861530304, "Finetune/Loss (Raw)": 0.9428367614746094, "Finetune/Step": 1675, "Finetune/Step Time": 5.39151624776423} +{"Finetune/Learning Rate": 1.5836954992297897e-05, "Finetune/Loss": 0.939866304397583, "Finetune/Loss (Raw)": 1.0851200819015503, "Finetune/Step": 1676, "Finetune/Step Time": 5.385324869304895} +{"Finetune/Learning Rate": 1.583189558662585e-05, "Finetune/Loss": 0.9446533918380737, "Finetune/Loss (Raw)": 1.0151249170303345, "Finetune/Step": 1677, "Finetune/Step Time": 5.354377828538418} +{"Finetune/Learning Rate": 1.5826833917708983e-05, "Finetune/Loss": 0.9434833526611328, "Finetune/Loss (Raw)": 0.8611443638801575, "Finetune/Step": 1678, "Finetune/Step Time": 5.346893539652228} +{"Finetune/Learning Rate": 1.5821769987511634e-05, "Finetune/Loss": 0.9397690892219543, "Finetune/Loss (Raw)": 0.38976824283599854, "Finetune/Step": 1679, "Finetune/Step Time": 5.3793214075267315} +{"Finetune/Learning Rate": 1.5816703797999015e-05, "Finetune/Loss": 0.9392553567886353, "Finetune/Loss (Raw)": 0.9477047920227051, "Finetune/Step": 1680, "Finetune/Step Time": 5.386001853272319} +{"Finetune/Learning Rate": 1.5811635351137212e-05, "Finetune/Loss": 0.9398118257522583, "Finetune/Loss (Raw)": 0.9760069847106934, "Finetune/Step": 1681, "Finetune/Step Time": 5.362702263519168} +{"Finetune/Learning Rate": 1.580656464889319e-05, "Finetune/Loss": 0.9417663812637329, "Finetune/Loss (Raw)": 1.116066813468933, "Finetune/Step": 1682, "Finetune/Step Time": 5.365597248077393} +{"Finetune/Learning Rate": 1.5801491693234793e-05, "Finetune/Loss": 0.940351128578186, "Finetune/Loss (Raw)": 0.8800274729728699, "Finetune/Step": 1683, "Finetune/Step Time": 5.378424974158406} +{"Finetune/Learning Rate": 1.5796416486130735e-05, "Finetune/Loss": 0.9405822157859802, "Finetune/Loss (Raw)": 0.9145401120185852, "Finetune/Step": 1684, "Finetune/Step Time": 5.369769746437669} +{"Finetune/Learning Rate": 1.5791339029550604e-05, "Finetune/Loss": 0.940834105014801, "Finetune/Loss (Raw)": 0.8871254920959473, "Finetune/Step": 1685, "Finetune/Step Time": 5.373538402840495} +{"Finetune/Learning Rate": 1.578625932546486e-05, "Finetune/Loss": 0.9399954080581665, "Finetune/Loss (Raw)": 0.8231291770935059, "Finetune/Step": 1686, "Finetune/Step Time": 5.3850635420531034} +{"Finetune/Learning Rate": 1.578117737584484e-05, "Finetune/Loss": 0.9389611482620239, "Finetune/Loss (Raw)": 1.0297263860702515, "Finetune/Step": 1687, "Finetune/Step Time": 5.38186914101243} +{"Finetune/Learning Rate": 1.577609318266275e-05, "Finetune/Loss": 0.9390276074409485, "Finetune/Loss (Raw)": 1.0483800172805786, "Finetune/Step": 1688, "Finetune/Step Time": 5.384410621598363} +{"Finetune/Learning Rate": 1.5771006747891663e-05, "Finetune/Loss": 0.9335193634033203, "Finetune/Loss (Raw)": 0.3098300099372864, "Finetune/Step": 1689, "Finetune/Step Time": 5.412551309913397} +{"Finetune/Learning Rate": 1.576591807350553e-05, "Finetune/Loss": 0.9340300559997559, "Finetune/Loss (Raw)": 1.0733627080917358, "Finetune/Step": 1690, "Finetune/Step Time": 5.403163641691208} +{"Finetune/Learning Rate": 1.5760827161479158e-05, "Finetune/Loss": 0.9341722726821899, "Finetune/Loss (Raw)": 0.9559853672981262, "Finetune/Step": 1691, "Finetune/Step Time": 5.37949238717556} +{"Finetune/Learning Rate": 1.5755734013788235e-05, "Finetune/Loss": 0.9340565800666809, "Finetune/Loss (Raw)": 0.9020489454269409, "Finetune/Step": 1692, "Finetune/Step Time": 5.401696773245931} +{"Finetune/Learning Rate": 1.5750638632409315e-05, "Finetune/Loss": 0.9340541362762451, "Finetune/Loss (Raw)": 0.8548510074615479, "Finetune/Step": 1693, "Finetune/Step Time": 5.420384651049972} +{"Finetune/Learning Rate": 1.574554101931981e-05, "Finetune/Loss": 0.9323306679725647, "Finetune/Loss (Raw)": 0.7477304339408875, "Finetune/Step": 1694, "Finetune/Step Time": 5.428160574287176} +{"Finetune/Learning Rate": 1.5740441176498015e-05, "Finetune/Loss": 0.9316424131393433, "Finetune/Loss (Raw)": 0.9064087271690369, "Finetune/Step": 1695, "Finetune/Step Time": 5.438904462382197} +{"Finetune/Learning Rate": 1.5735339105923068e-05, "Finetune/Loss": 0.9317363500595093, "Finetune/Loss (Raw)": 0.9807730317115784, "Finetune/Step": 1696, "Finetune/Step Time": 5.4368780348449945} +{"Finetune/Learning Rate": 1.5730234809574986e-05, "Finetune/Loss": 0.9330122470855713, "Finetune/Loss (Raw)": 1.0085136890411377, "Finetune/Step": 1697, "Finetune/Step Time": 5.423155130818486} +{"Finetune/Learning Rate": 1.5725128289434647e-05, "Finetune/Loss": 0.9322572350502014, "Finetune/Loss (Raw)": 0.9492195248603821, "Finetune/Step": 1698, "Finetune/Step Time": 5.4253652431070805} +{"Finetune/Learning Rate": 1.5720019547483795e-05, "Finetune/Loss": 0.932852029800415, "Finetune/Loss (Raw)": 1.0417201519012451, "Finetune/Step": 1699, "Finetune/Step Time": 5.423155497759581} +{"Finetune/Learning Rate": 1.571490858570503e-05, "Finetune/Loss": 0.9329123497009277, "Finetune/Loss (Raw)": 1.0288913249969482, "Finetune/Step": 1700, "Finetune/Step Time": 5.432907754555345} +{"Finetune/Learning Rate": 1.5709795406081818e-05, "Finetune/Loss": 0.9334753751754761, "Finetune/Loss (Raw)": 1.067988395690918, "Finetune/Step": 1701, "Finetune/Step Time": 5.41793755069375} +{"Finetune/Learning Rate": 1.570468001059848e-05, "Finetune/Loss": 0.9347537755966187, "Finetune/Loss (Raw)": 0.9959470629692078, "Finetune/Step": 1702, "Finetune/Step Time": 5.417392410337925} +{"Finetune/Learning Rate": 1.5699562401240206e-05, "Finetune/Loss": 0.9322429299354553, "Finetune/Loss (Raw)": 0.8786414265632629, "Finetune/Step": 1703, "Finetune/Step Time": 5.424355918541551} +{"Finetune/Learning Rate": 1.5694442579993042e-05, "Finetune/Loss": 0.9273720383644104, "Finetune/Loss (Raw)": 0.3469568192958832, "Finetune/Step": 1704, "Finetune/Step Time": 5.456137062981725} +{"Finetune/Learning Rate": 1.5689320548843882e-05, "Finetune/Loss": 0.9272452592849731, "Finetune/Loss (Raw)": 0.9487932920455933, "Finetune/Step": 1705, "Finetune/Step Time": 5.458373628556728} +{"Finetune/Learning Rate": 1.5684196309780495e-05, "Finetune/Loss": 0.9276772737503052, "Finetune/Loss (Raw)": 1.048661708831787, "Finetune/Step": 1706, "Finetune/Step Time": 5.460861019790173} +{"Finetune/Learning Rate": 1.5679069864791493e-05, "Finetune/Loss": 0.9275193810462952, "Finetune/Loss (Raw)": 0.9516834616661072, "Finetune/Step": 1707, "Finetune/Step Time": 5.457738256081939} +{"Finetune/Learning Rate": 1.567394121586635e-05, "Finetune/Loss": 0.9289724826812744, "Finetune/Loss (Raw)": 0.9613007307052612, "Finetune/Step": 1708, "Finetune/Step Time": 5.437868485227227} +{"Finetune/Learning Rate": 1.56688103649954e-05, "Finetune/Loss": 0.9309325814247131, "Finetune/Loss (Raw)": 1.1033686399459839, "Finetune/Step": 1709, "Finetune/Step Time": 5.436088846996427} +{"Finetune/Learning Rate": 1.566367731416982e-05, "Finetune/Loss": 0.9311816096305847, "Finetune/Loss (Raw)": 1.0010792016983032, "Finetune/Step": 1710, "Finetune/Step Time": 5.440094040706754} +{"Finetune/Learning Rate": 1.5658542065381644e-05, "Finetune/Loss": 0.9312927722930908, "Finetune/Loss (Raw)": 1.018256664276123, "Finetune/Step": 1711, "Finetune/Step Time": 5.443257970735431} +{"Finetune/Learning Rate": 1.565340462062377e-05, "Finetune/Loss": 0.932540774345398, "Finetune/Loss (Raw)": 1.1387072801589966, "Finetune/Step": 1712, "Finetune/Step Time": 5.443543277680874} +{"Finetune/Learning Rate": 1.5648264981889936e-05, "Finetune/Loss": 0.9321002960205078, "Finetune/Loss (Raw)": 1.0083591938018799, "Finetune/Step": 1713, "Finetune/Step Time": 5.443586502224207} +{"Finetune/Learning Rate": 1.5643123151174736e-05, "Finetune/Loss": 0.9311068058013916, "Finetune/Loss (Raw)": 0.9992374181747437, "Finetune/Step": 1714, "Finetune/Step Time": 5.393349213525653} +{"Finetune/Learning Rate": 1.5637979130473612e-05, "Finetune/Loss": 0.9293715357780457, "Finetune/Loss (Raw)": 0.8323290348052979, "Finetune/Step": 1715, "Finetune/Step Time": 5.402396436780691} +{"Finetune/Learning Rate": 1.563283292178286e-05, "Finetune/Loss": 0.9291056990623474, "Finetune/Loss (Raw)": 0.9251158237457275, "Finetune/Step": 1716, "Finetune/Step Time": 5.390487100929022} +{"Finetune/Learning Rate": 1.5627684527099624e-05, "Finetune/Loss": 0.9301726818084717, "Finetune/Loss (Raw)": 1.0712683200836182, "Finetune/Step": 1717, "Finetune/Step Time": 5.395803170278668} +{"Finetune/Learning Rate": 1.5622533948421895e-05, "Finetune/Loss": 0.929847776889801, "Finetune/Loss (Raw)": 0.9870929718017578, "Finetune/Step": 1718, "Finetune/Step Time": 5.380932006984949} +{"Finetune/Learning Rate": 1.5617381187748507e-05, "Finetune/Loss": 0.9307733774185181, "Finetune/Loss (Raw)": 0.9640608429908752, "Finetune/Step": 1719, "Finetune/Step Time": 5.367225706577301} +{"Finetune/Learning Rate": 1.5612226247079155e-05, "Finetune/Loss": 0.9319889545440674, "Finetune/Loss (Raw)": 1.0426920652389526, "Finetune/Step": 1720, "Finetune/Step Time": 5.365309888496995} +{"Finetune/Learning Rate": 1.5607069128414366e-05, "Finetune/Loss": 0.9325262308120728, "Finetune/Loss (Raw)": 0.9956234097480774, "Finetune/Step": 1721, "Finetune/Step Time": 5.3668074030429125} +{"Finetune/Learning Rate": 1.5601909833755514e-05, "Finetune/Loss": 0.9329907894134521, "Finetune/Loss (Raw)": 0.9987711906433105, "Finetune/Step": 1722, "Finetune/Step Time": 5.357418265193701} +{"Finetune/Learning Rate": 1.559674836510482e-05, "Finetune/Loss": 0.934539794921875, "Finetune/Loss (Raw)": 1.0617293119430542, "Finetune/Step": 1723, "Finetune/Step Time": 5.373571138828993} +{"Finetune/Learning Rate": 1.5591584724465363e-05, "Finetune/Loss": 0.9328221082687378, "Finetune/Loss (Raw)": 0.8948126435279846, "Finetune/Step": 1724, "Finetune/Step Time": 5.366368064656854} +{"Finetune/Learning Rate": 1.5586418913841034e-05, "Finetune/Loss": 0.9338682293891907, "Finetune/Loss (Raw)": 0.9838882684707642, "Finetune/Step": 1725, "Finetune/Step Time": 5.356305692344904} +{"Finetune/Learning Rate": 1.558125093523659e-05, "Finetune/Loss": 0.9331784248352051, "Finetune/Loss (Raw)": 0.9027968645095825, "Finetune/Step": 1726, "Finetune/Step Time": 5.349035756662488} +{"Finetune/Learning Rate": 1.5576080790657626e-05, "Finetune/Loss": 0.9379631280899048, "Finetune/Loss (Raw)": 0.9910002946853638, "Finetune/Step": 1727, "Finetune/Step Time": 5.334246516227722} +{"Finetune/Learning Rate": 1.5570908482110566e-05, "Finetune/Loss": 0.9385700225830078, "Finetune/Loss (Raw)": 1.0758533477783203, "Finetune/Step": 1728, "Finetune/Step Time": 5.331446433439851} +{"Finetune/Learning Rate": 1.5565734011602693e-05, "Finetune/Loss": 0.9435646533966064, "Finetune/Loss (Raw)": 1.0265294313430786, "Finetune/Step": 1729, "Finetune/Step Time": 5.294841405004263} +{"Finetune/Learning Rate": 1.556055738114211e-05, "Finetune/Loss": 0.9415508508682251, "Finetune/Loss (Raw)": 0.8292055726051331, "Finetune/Step": 1730, "Finetune/Step Time": 5.2873819172382355} +{"Finetune/Learning Rate": 1.555537859273777e-05, "Finetune/Loss": 0.9422444105148315, "Finetune/Loss (Raw)": 1.1154999732971191, "Finetune/Step": 1731, "Finetune/Step Time": 5.284570250660181} +{"Finetune/Learning Rate": 1.5550197648399453e-05, "Finetune/Loss": 0.9438985586166382, "Finetune/Loss (Raw)": 1.1087677478790283, "Finetune/Step": 1732, "Finetune/Step Time": 5.299139151349664} +{"Finetune/Learning Rate": 1.554501455013779e-05, "Finetune/Loss": 0.9451471567153931, "Finetune/Loss (Raw)": 0.9935628175735474, "Finetune/Step": 1733, "Finetune/Step Time": 5.2904592622071505} +{"Finetune/Learning Rate": 1.5539829299964236e-05, "Finetune/Loss": 0.9471040964126587, "Finetune/Loss (Raw)": 1.0237659215927124, "Finetune/Step": 1734, "Finetune/Step Time": 5.274125115945935} +{"Finetune/Learning Rate": 1.5534641899891084e-05, "Finetune/Loss": 0.9472545385360718, "Finetune/Loss (Raw)": 0.9247750639915466, "Finetune/Step": 1735, "Finetune/Step Time": 5.274027995765209} +{"Finetune/Learning Rate": 1.5529452351931473e-05, "Finetune/Loss": 0.9518604874610901, "Finetune/Loss (Raw)": 0.9780147671699524, "Finetune/Step": 1736, "Finetune/Step Time": 5.242678863927722} +{"Finetune/Learning Rate": 1.5524260658099352e-05, "Finetune/Loss": 0.9514508843421936, "Finetune/Loss (Raw)": 0.9097893834114075, "Finetune/Step": 1737, "Finetune/Step Time": 5.242207702249289} +{"Finetune/Learning Rate": 1.551906682040952e-05, "Finetune/Loss": 0.9468991756439209, "Finetune/Loss (Raw)": 0.356606662273407, "Finetune/Step": 1738, "Finetune/Step Time": 5.269776325672865} +{"Finetune/Learning Rate": 1.5513870840877608e-05, "Finetune/Loss": 0.9475266933441162, "Finetune/Loss (Raw)": 0.9422475695610046, "Finetune/Step": 1739, "Finetune/Step Time": 5.247975192964077} +{"Finetune/Learning Rate": 1.5508672721520076e-05, "Finetune/Loss": 0.9472522735595703, "Finetune/Loss (Raw)": 0.8902267813682556, "Finetune/Step": 1740, "Finetune/Step Time": 5.258523393422365} +{"Finetune/Learning Rate": 1.5503472464354207e-05, "Finetune/Loss": 0.9493223428726196, "Finetune/Loss (Raw)": 1.1263054609298706, "Finetune/Step": 1741, "Finetune/Step Time": 5.248657064512372} +{"Finetune/Learning Rate": 1.5498270071398116e-05, "Finetune/Loss": 0.948304295539856, "Finetune/Loss (Raw)": 0.863463819026947, "Finetune/Step": 1742, "Finetune/Step Time": 5.256982568651438} +{"Finetune/Learning Rate": 1.549306554467076e-05, "Finetune/Loss": 0.9475181102752686, "Finetune/Loss (Raw)": 0.9651132822036743, "Finetune/Step": 1743, "Finetune/Step Time": 5.2725027315318584} +{"Finetune/Learning Rate": 1.5487858886191903e-05, "Finetune/Loss": 0.9471791982650757, "Finetune/Loss (Raw)": 0.9247970581054688, "Finetune/Step": 1744, "Finetune/Step Time": 5.273801166564226} +{"Finetune/Learning Rate": 1.548265009798216e-05, "Finetune/Loss": 0.9477676153182983, "Finetune/Loss (Raw)": 0.9831193089485168, "Finetune/Step": 1745, "Finetune/Step Time": 5.269266311079264} +{"Finetune/Learning Rate": 1.547743918206295e-05, "Finetune/Loss": 0.9488414525985718, "Finetune/Loss (Raw)": 1.0430266857147217, "Finetune/Step": 1746, "Finetune/Step Time": 5.269643122330308} +{"Finetune/Learning Rate": 1.5472226140456537e-05, "Finetune/Loss": 0.9457412958145142, "Finetune/Loss (Raw)": 0.4367966651916504, "Finetune/Step": 1747, "Finetune/Step Time": 5.294020084664226} +{"Finetune/Learning Rate": 1.546701097518599e-05, "Finetune/Loss": 0.9434504508972168, "Finetune/Loss (Raw)": 0.8550461530685425, "Finetune/Step": 1748, "Finetune/Step Time": 5.304251881316304} +{"Finetune/Learning Rate": 1.546179368827522e-05, "Finetune/Loss": 0.9443544745445251, "Finetune/Loss (Raw)": 0.97678542137146, "Finetune/Step": 1749, "Finetune/Step Time": 5.302734324708581} +{"Finetune/Learning Rate": 1.545657428174895e-05, "Finetune/Loss": 0.9446250200271606, "Finetune/Loss (Raw)": 1.107140302658081, "Finetune/Step": 1750, "Finetune/Step Time": 5.321265891194344} +{"Finetune/Learning Rate": 1.5451352757632734e-05, "Finetune/Loss": 0.9438270926475525, "Finetune/Loss (Raw)": 1.040414810180664, "Finetune/Step": 1751, "Finetune/Step Time": 5.325450714677572} +{"Finetune/Learning Rate": 1.544612911795294e-05, "Finetune/Loss": 0.9458655118942261, "Finetune/Loss (Raw)": 1.1817594766616821, "Finetune/Step": 1752, "Finetune/Step Time": 5.329854417592287} +{"Finetune/Learning Rate": 1.5440903364736763e-05, "Finetune/Loss": 0.945502758026123, "Finetune/Loss (Raw)": 0.9443274140357971, "Finetune/Step": 1753, "Finetune/Step Time": 5.328007452189922} +{"Finetune/Learning Rate": 1.5435675500012212e-05, "Finetune/Loss": 0.9454452991485596, "Finetune/Loss (Raw)": 1.0000146627426147, "Finetune/Step": 1754, "Finetune/Step Time": 5.33001696690917} +{"Finetune/Learning Rate": 1.5430445525808123e-05, "Finetune/Loss": 0.944990873336792, "Finetune/Loss (Raw)": 0.98442542552948, "Finetune/Step": 1755, "Finetune/Step Time": 5.342318439856172} +{"Finetune/Learning Rate": 1.5425213444154152e-05, "Finetune/Loss": 0.9457921981811523, "Finetune/Loss (Raw)": 1.0922136306762695, "Finetune/Step": 1756, "Finetune/Step Time": 5.35147251188755} +{"Finetune/Learning Rate": 1.5419979257080755e-05, "Finetune/Loss": 0.9411746859550476, "Finetune/Loss (Raw)": 0.36335641145706177, "Finetune/Step": 1757, "Finetune/Step Time": 5.377355415374041} +{"Finetune/Learning Rate": 1.541474296661923e-05, "Finetune/Loss": 0.9397830963134766, "Finetune/Loss (Raw)": 0.8611419200897217, "Finetune/Step": 1758, "Finetune/Step Time": 5.381240447983146} +{"Finetune/Learning Rate": 1.5409504574801667e-05, "Finetune/Loss": 0.9404866695404053, "Finetune/Loss (Raw)": 1.0405436754226685, "Finetune/Step": 1759, "Finetune/Step Time": 5.386868234723806} +{"Finetune/Learning Rate": 1.5404264083660993e-05, "Finetune/Loss": 0.9413511753082275, "Finetune/Loss (Raw)": 1.0507667064666748, "Finetune/Step": 1760, "Finetune/Step Time": 5.384909262880683} +{"Finetune/Learning Rate": 1.539902149523094e-05, "Finetune/Loss": 0.9412773251533508, "Finetune/Loss (Raw)": 0.8635789752006531, "Finetune/Step": 1761, "Finetune/Step Time": 5.389636619016528} +{"Finetune/Learning Rate": 1.5393776811546047e-05, "Finetune/Loss": 0.9400172233581543, "Finetune/Loss (Raw)": 0.8481395840644836, "Finetune/Step": 1762, "Finetune/Step Time": 5.399187559261918} +{"Finetune/Learning Rate": 1.538853003464168e-05, "Finetune/Loss": 0.9408963918685913, "Finetune/Loss (Raw)": 1.0956193208694458, "Finetune/Step": 1763, "Finetune/Step Time": 5.399572592228651} +{"Finetune/Learning Rate": 1.538328116655401e-05, "Finetune/Loss": 0.9414427280426025, "Finetune/Loss (Raw)": 1.0574400424957275, "Finetune/Step": 1764, "Finetune/Step Time": 5.419926492497325} +{"Finetune/Learning Rate": 1.5378030209320018e-05, "Finetune/Loss": 0.940977156162262, "Finetune/Loss (Raw)": 0.904880702495575, "Finetune/Step": 1765, "Finetune/Step Time": 5.432810511440039} +{"Finetune/Learning Rate": 1.53727771649775e-05, "Finetune/Loss": 0.9417513012886047, "Finetune/Loss (Raw)": 1.0076185464859009, "Finetune/Step": 1766, "Finetune/Step Time": 5.434389002621174} +{"Finetune/Learning Rate": 1.5367522035565055e-05, "Finetune/Loss": 0.9416406750679016, "Finetune/Loss (Raw)": 0.8278974294662476, "Finetune/Step": 1767, "Finetune/Step Time": 5.435150381177664} +{"Finetune/Learning Rate": 1.5362264823122107e-05, "Finetune/Loss": 0.9406566619873047, "Finetune/Loss (Raw)": 0.9017525911331177, "Finetune/Step": 1768, "Finetune/Step Time": 5.436223220080137} +{"Finetune/Learning Rate": 1.5357005529688868e-05, "Finetune/Loss": 0.9388484358787537, "Finetune/Loss (Raw)": 0.8044726252555847, "Finetune/Step": 1769, "Finetune/Step Time": 5.451093262061477} +{"Finetune/Learning Rate": 1.5351744157306366e-05, "Finetune/Loss": 0.939120888710022, "Finetune/Loss (Raw)": 1.023726463317871, "Finetune/Step": 1770, "Finetune/Step Time": 5.45784011669457} +{"Finetune/Learning Rate": 1.5346480708016445e-05, "Finetune/Loss": 0.9377779364585876, "Finetune/Loss (Raw)": 0.90680330991745, "Finetune/Step": 1771, "Finetune/Step Time": 5.456051506102085} +{"Finetune/Learning Rate": 1.5341215183861738e-05, "Finetune/Loss": 0.9386181831359863, "Finetune/Loss (Raw)": 1.0979126691818237, "Finetune/Step": 1772, "Finetune/Step Time": 5.430228672921658} +{"Finetune/Learning Rate": 1.53359475868857e-05, "Finetune/Loss": 0.9389988780021667, "Finetune/Loss (Raw)": 0.9706320762634277, "Finetune/Step": 1773, "Finetune/Step Time": 5.433352537453175} +{"Finetune/Learning Rate": 1.5330677919132577e-05, "Finetune/Loss": 0.9384906888008118, "Finetune/Loss (Raw)": 0.9653750658035278, "Finetune/Step": 1774, "Finetune/Step Time": 5.446671202778816} +{"Finetune/Learning Rate": 1.5325406182647426e-05, "Finetune/Loss": 0.9395327568054199, "Finetune/Loss (Raw)": 1.0413050651550293, "Finetune/Step": 1775, "Finetune/Step Time": 5.4335746094584465} +{"Finetune/Learning Rate": 1.5320132379476107e-05, "Finetune/Loss": 0.9388195276260376, "Finetune/Loss (Raw)": 0.9572868943214417, "Finetune/Step": 1776, "Finetune/Step Time": 5.432652730494738} +{"Finetune/Learning Rate": 1.531485651166528e-05, "Finetune/Loss": 0.9394333362579346, "Finetune/Loss (Raw)": 1.0229246616363525, "Finetune/Step": 1777, "Finetune/Step Time": 5.4272397849708796} +{"Finetune/Learning Rate": 1.5309578581262404e-05, "Finetune/Loss": 0.939335286617279, "Finetune/Loss (Raw)": 0.9528588056564331, "Finetune/Step": 1778, "Finetune/Step Time": 5.428927604109049} +{"Finetune/Learning Rate": 1.530429859031574e-05, "Finetune/Loss": 0.9410293698310852, "Finetune/Loss (Raw)": 1.1004445552825928, "Finetune/Step": 1779, "Finetune/Step Time": 5.413086891174316} +{"Finetune/Learning Rate": 1.5299016540874346e-05, "Finetune/Loss": 0.9413089156150818, "Finetune/Loss (Raw)": 0.9336414933204651, "Finetune/Step": 1780, "Finetune/Step Time": 5.405125934630632} +{"Finetune/Learning Rate": 1.5293732434988095e-05, "Finetune/Loss": 0.9401447176933289, "Finetune/Loss (Raw)": 0.8250693678855896, "Finetune/Step": 1781, "Finetune/Step Time": 5.406282126903534} +{"Finetune/Learning Rate": 1.5288446274707635e-05, "Finetune/Loss": 0.9394856691360474, "Finetune/Loss (Raw)": 0.9367430806159973, "Finetune/Step": 1782, "Finetune/Step Time": 5.38809241913259} +{"Finetune/Learning Rate": 1.5283158062084423e-05, "Finetune/Loss": 0.9432801008224487, "Finetune/Loss (Raw)": 0.8696329593658447, "Finetune/Step": 1783, "Finetune/Step Time": 5.358276903629303} +{"Finetune/Learning Rate": 1.5277867799170708e-05, "Finetune/Loss": 0.9440394639968872, "Finetune/Loss (Raw)": 1.0727766752243042, "Finetune/Step": 1784, "Finetune/Step Time": 5.362385107204318} +{"Finetune/Learning Rate": 1.527257548801955e-05, "Finetune/Loss": 0.9432061910629272, "Finetune/Loss (Raw)": 0.7814800143241882, "Finetune/Step": 1785, "Finetune/Step Time": 5.3772212360054255} +{"Finetune/Learning Rate": 1.526728113068478e-05, "Finetune/Loss": 0.9428466558456421, "Finetune/Loss (Raw)": 0.9247891902923584, "Finetune/Step": 1786, "Finetune/Step Time": 5.375273151323199} +{"Finetune/Learning Rate": 1.526198472922104e-05, "Finetune/Loss": 0.9406567811965942, "Finetune/Loss (Raw)": 0.8051621317863464, "Finetune/Step": 1787, "Finetune/Step Time": 5.3855095617473125} +{"Finetune/Learning Rate": 1.5256686285683753e-05, "Finetune/Loss": 0.9398899078369141, "Finetune/Loss (Raw)": 0.8771497011184692, "Finetune/Step": 1788, "Finetune/Step Time": 5.381089905276895} +{"Finetune/Learning Rate": 1.5251385802129149e-05, "Finetune/Loss": 0.9349095821380615, "Finetune/Loss (Raw)": 0.2945246994495392, "Finetune/Step": 1789, "Finetune/Step Time": 5.412677405402064} +{"Finetune/Learning Rate": 1.5246083280614241e-05, "Finetune/Loss": 0.9347864389419556, "Finetune/Loss (Raw)": 1.0512940883636475, "Finetune/Step": 1790, "Finetune/Step Time": 5.414956294000149} +{"Finetune/Learning Rate": 1.5240778723196833e-05, "Finetune/Loss": 0.937751054763794, "Finetune/Loss (Raw)": 1.093965768814087, "Finetune/Step": 1791, "Finetune/Step Time": 5.407228218391538} +{"Finetune/Learning Rate": 1.523547213193552e-05, "Finetune/Loss": 0.9393662810325623, "Finetune/Loss (Raw)": 1.0061768293380737, "Finetune/Step": 1792, "Finetune/Step Time": 5.407370613887906} +{"Finetune/Learning Rate": 1.5230163508889686e-05, "Finetune/Loss": 0.9374057054519653, "Finetune/Loss (Raw)": 0.818352460861206, "Finetune/Step": 1793, "Finetune/Step Time": 5.422491887584329} +{"Finetune/Learning Rate": 1.5224852856119503e-05, "Finetune/Loss": 0.9385585784912109, "Finetune/Loss (Raw)": 1.0534889698028564, "Finetune/Step": 1794, "Finetune/Step Time": 5.428435293957591} +{"Finetune/Learning Rate": 1.5219540175685938e-05, "Finetune/Loss": 0.9336110949516296, "Finetune/Loss (Raw)": 0.37257540225982666, "Finetune/Step": 1795, "Finetune/Step Time": 5.460369173437357} +{"Finetune/Learning Rate": 1.5214225469650727e-05, "Finetune/Loss": 0.9324564933776855, "Finetune/Loss (Raw)": 0.9171351790428162, "Finetune/Step": 1796, "Finetune/Step Time": 5.452800737693906} +{"Finetune/Learning Rate": 1.5208908740076416e-05, "Finetune/Loss": 0.9329019784927368, "Finetune/Loss (Raw)": 1.0194567441940308, "Finetune/Step": 1797, "Finetune/Step Time": 5.452536506578326} +{"Finetune/Learning Rate": 1.5203589989026318e-05, "Finetune/Loss": 0.9323819875717163, "Finetune/Loss (Raw)": 0.934090256690979, "Finetune/Step": 1798, "Finetune/Step Time": 5.456374330446124} +{"Finetune/Learning Rate": 1.5198269218564535e-05, "Finetune/Loss": 0.9326109886169434, "Finetune/Loss (Raw)": 0.9325987696647644, "Finetune/Step": 1799, "Finetune/Step Time": 5.447883889079094} +{"Finetune/Learning Rate": 1.5192946430755955e-05, "Finetune/Loss": 0.9316229224205017, "Finetune/Loss (Raw)": 0.9205474257469177, "Finetune/Step": 1800, "Finetune/Step Time": 5.441759781911969} +{"Finetune/Learning Rate": 1.518762162766625e-05, "Finetune/Loss": 0.933434247970581, "Finetune/Loss (Raw)": 1.074997901916504, "Finetune/Step": 1801, "Finetune/Step Time": 5.446029739454389} +{"Finetune/Learning Rate": 1.5182294811361871e-05, "Finetune/Loss": 0.9347311854362488, "Finetune/Loss (Raw)": 0.9676709175109863, "Finetune/Step": 1802, "Finetune/Step Time": 5.448377877473831} +{"Finetune/Learning Rate": 1.5176965983910053e-05, "Finetune/Loss": 0.9346707463264465, "Finetune/Loss (Raw)": 0.9351047873497009, "Finetune/Step": 1803, "Finetune/Step Time": 5.460843041539192} +{"Finetune/Learning Rate": 1.5171635147378804e-05, "Finetune/Loss": 0.933950662612915, "Finetune/Loss (Raw)": 0.9929425716400146, "Finetune/Step": 1804, "Finetune/Step Time": 5.464344834908843} +{"Finetune/Learning Rate": 1.5166302303836928e-05, "Finetune/Loss": 0.9311069846153259, "Finetune/Loss (Raw)": 0.6511359214782715, "Finetune/Step": 1805, "Finetune/Step Time": 5.4730110373348} +{"Finetune/Learning Rate": 1.5160967455353985e-05, "Finetune/Loss": 0.9311913251876831, "Finetune/Loss (Raw)": 0.8719456195831299, "Finetune/Step": 1806, "Finetune/Step Time": 5.4849238861352205} +{"Finetune/Learning Rate": 1.5155630604000338e-05, "Finetune/Loss": 0.9356203079223633, "Finetune/Loss (Raw)": 0.9566804766654968, "Finetune/Step": 1807, "Finetune/Step Time": 5.453291600570083} +{"Finetune/Learning Rate": 1.5150291751847106e-05, "Finetune/Loss": 0.9358380436897278, "Finetune/Loss (Raw)": 0.975564181804657, "Finetune/Step": 1808, "Finetune/Step Time": 5.4429088439792395} +{"Finetune/Learning Rate": 1.5144950900966198e-05, "Finetune/Loss": 0.9357414841651917, "Finetune/Loss (Raw)": 0.9636505842208862, "Finetune/Step": 1809, "Finetune/Step Time": 5.4424188900738955} +{"Finetune/Learning Rate": 1.5139608053430288e-05, "Finetune/Loss": 0.934612512588501, "Finetune/Loss (Raw)": 0.9715601801872253, "Finetune/Step": 1810, "Finetune/Step Time": 5.439065884798765} +{"Finetune/Learning Rate": 1.5134263211312837e-05, "Finetune/Loss": 0.9351421594619751, "Finetune/Loss (Raw)": 0.9478240013122559, "Finetune/Step": 1811, "Finetune/Step Time": 5.426896266639233} +{"Finetune/Learning Rate": 1.5128916376688071e-05, "Finetune/Loss": 0.935320258140564, "Finetune/Loss (Raw)": 0.9373323321342468, "Finetune/Step": 1812, "Finetune/Step Time": 5.434123661369085} +{"Finetune/Learning Rate": 1.5123567551630993e-05, "Finetune/Loss": 0.9357483386993408, "Finetune/Loss (Raw)": 0.941921055316925, "Finetune/Step": 1813, "Finetune/Step Time": 5.429605858400464} +{"Finetune/Learning Rate": 1.511821673821738e-05, "Finetune/Loss": 0.9371245503425598, "Finetune/Loss (Raw)": 0.9992856979370117, "Finetune/Step": 1814, "Finetune/Step Time": 5.417602516710758} +{"Finetune/Learning Rate": 1.5112863938523774e-05, "Finetune/Loss": 0.936098575592041, "Finetune/Loss (Raw)": 0.8984012007713318, "Finetune/Step": 1815, "Finetune/Step Time": 5.416248805820942} +{"Finetune/Learning Rate": 1.5107509154627491e-05, "Finetune/Loss": 0.9361835718154907, "Finetune/Loss (Raw)": 1.059249758720398, "Finetune/Step": 1816, "Finetune/Step Time": 5.411422286182642} +{"Finetune/Learning Rate": 1.5102152388606625e-05, "Finetune/Loss": 0.9409810304641724, "Finetune/Loss (Raw)": 0.9239087700843811, "Finetune/Step": 1817, "Finetune/Step Time": 5.38262490555644} +{"Finetune/Learning Rate": 1.5096793642540027e-05, "Finetune/Loss": 0.9397644996643066, "Finetune/Loss (Raw)": 0.9176505208015442, "Finetune/Step": 1818, "Finetune/Step Time": 5.389354864135385} +{"Finetune/Learning Rate": 1.5091432918507322e-05, "Finetune/Loss": 0.9400668144226074, "Finetune/Loss (Raw)": 0.9946808815002441, "Finetune/Step": 1819, "Finetune/Step Time": 5.391957111656666} +{"Finetune/Learning Rate": 1.5086070218588904e-05, "Finetune/Loss": 0.9397902488708496, "Finetune/Loss (Raw)": 0.8666439652442932, "Finetune/Step": 1820, "Finetune/Step Time": 5.370121540501714} +{"Finetune/Learning Rate": 1.5080705544865932e-05, "Finetune/Loss": 0.9407642483711243, "Finetune/Loss (Raw)": 0.9795272946357727, "Finetune/Step": 1821, "Finetune/Step Time": 5.354864068329334} +{"Finetune/Learning Rate": 1.5075338899420326e-05, "Finetune/Loss": 0.9442759156227112, "Finetune/Loss (Raw)": 1.1972273588180542, "Finetune/Step": 1822, "Finetune/Step Time": 5.341340117156506} +{"Finetune/Learning Rate": 1.5069970284334785e-05, "Finetune/Loss": 0.9447354674339294, "Finetune/Loss (Raw)": 0.9652328491210938, "Finetune/Step": 1823, "Finetune/Step Time": 5.327425200492144} +{"Finetune/Learning Rate": 1.506459970169276e-05, "Finetune/Loss": 0.9437954425811768, "Finetune/Loss (Raw)": 0.8604455590248108, "Finetune/Step": 1824, "Finetune/Step Time": 5.331917054951191} +{"Finetune/Learning Rate": 1.5059227153578466e-05, "Finetune/Loss": 0.9431266784667969, "Finetune/Loss (Raw)": 0.9229123592376709, "Finetune/Step": 1825, "Finetune/Step Time": 5.333236979320645} +{"Finetune/Learning Rate": 1.505385264207689e-05, "Finetune/Loss": 0.943739116191864, "Finetune/Loss (Raw)": 1.0276139974594116, "Finetune/Step": 1826, "Finetune/Step Time": 5.334036128595471} +{"Finetune/Learning Rate": 1.5048476169273767e-05, "Finetune/Loss": 0.943067729473114, "Finetune/Loss (Raw)": 0.9557825922966003, "Finetune/Step": 1827, "Finetune/Step Time": 5.3666282538324594} +{"Finetune/Learning Rate": 1.5043097737255607e-05, "Finetune/Loss": 0.9436337947845459, "Finetune/Loss (Raw)": 1.1013457775115967, "Finetune/Step": 1828, "Finetune/Step Time": 5.364468304440379} +{"Finetune/Learning Rate": 1.5037717348109677e-05, "Finetune/Loss": 0.9441941976547241, "Finetune/Loss (Raw)": 1.1397151947021484, "Finetune/Step": 1829, "Finetune/Step Time": 5.368225922808051} +{"Finetune/Learning Rate": 1.5032335003923994e-05, "Finetune/Loss": 0.9447206854820251, "Finetune/Loss (Raw)": 1.0633482933044434, "Finetune/Step": 1830, "Finetune/Step Time": 5.38542402535677} +{"Finetune/Learning Rate": 1.5026950706787346e-05, "Finetune/Loss": 0.9451239109039307, "Finetune/Loss (Raw)": 0.9302437901496887, "Finetune/Step": 1831, "Finetune/Step Time": 5.370332362130284} +{"Finetune/Learning Rate": 1.502156445878927e-05, "Finetune/Loss": 0.9506247043609619, "Finetune/Loss (Raw)": 1.0510575771331787, "Finetune/Step": 1832, "Finetune/Step Time": 5.338352905586362} +{"Finetune/Learning Rate": 1.5016176262020069e-05, "Finetune/Loss": 0.9499599933624268, "Finetune/Loss (Raw)": 0.8637136220932007, "Finetune/Step": 1833, "Finetune/Step Time": 5.357844114303589} +{"Finetune/Learning Rate": 1.5010786118570791e-05, "Finetune/Loss": 0.9503780603408813, "Finetune/Loss (Raw)": 1.1021729707717896, "Finetune/Step": 1834, "Finetune/Step Time": 5.362582799047232} +{"Finetune/Learning Rate": 1.5005394030533246e-05, "Finetune/Loss": 0.9505693912506104, "Finetune/Loss (Raw)": 0.9761735796928406, "Finetune/Step": 1835, "Finetune/Step Time": 5.371102785691619} +{"Finetune/Learning Rate": 1.5000000000000002e-05, "Finetune/Loss": 0.9507933259010315, "Finetune/Loss (Raw)": 0.9899657368659973, "Finetune/Step": 1836, "Finetune/Step Time": 5.3662036675959826} +{"Finetune/Learning Rate": 1.4994604029064372e-05, "Finetune/Loss": 0.9496126770973206, "Finetune/Loss (Raw)": 0.952247679233551, "Finetune/Step": 1837, "Finetune/Step Time": 5.368196917697787} +{"Finetune/Learning Rate": 1.4989206119820432e-05, "Finetune/Loss": 0.94964998960495, "Finetune/Loss (Raw)": 1.005859613418579, "Finetune/Step": 1838, "Finetune/Step Time": 5.360854845494032} +{"Finetune/Learning Rate": 1.4983806274363005e-05, "Finetune/Loss": 0.94952392578125, "Finetune/Loss (Raw)": 1.002116084098816, "Finetune/Step": 1839, "Finetune/Step Time": 5.363227749243379} +{"Finetune/Learning Rate": 1.497840449478766e-05, "Finetune/Loss": 0.9487757682800293, "Finetune/Loss (Raw)": 1.0429418087005615, "Finetune/Step": 1840, "Finetune/Step Time": 5.373177260160446} +{"Finetune/Learning Rate": 1.4973000783190728e-05, "Finetune/Loss": 0.9475104212760925, "Finetune/Loss (Raw)": 0.8463947176933289, "Finetune/Step": 1841, "Finetune/Step Time": 5.374554427340627} +{"Finetune/Learning Rate": 1.4967595141669278e-05, "Finetune/Loss": 0.948177695274353, "Finetune/Loss (Raw)": 1.0846545696258545, "Finetune/Step": 1842, "Finetune/Step Time": 5.373012617230415} +{"Finetune/Learning Rate": 1.4962187572321143e-05, "Finetune/Loss": 0.9494438171386719, "Finetune/Loss (Raw)": 0.9943938255310059, "Finetune/Step": 1843, "Finetune/Step Time": 5.3783762995153666} +{"Finetune/Learning Rate": 1.4956778077244884e-05, "Finetune/Loss": 0.9497250318527222, "Finetune/Loss (Raw)": 0.9611026048660278, "Finetune/Step": 1844, "Finetune/Step Time": 5.389939216896892} +{"Finetune/Learning Rate": 1.4951366658539832e-05, "Finetune/Loss": 0.9488368034362793, "Finetune/Loss (Raw)": 0.9575793147087097, "Finetune/Step": 1845, "Finetune/Step Time": 5.37844018638134} +{"Finetune/Learning Rate": 1.4945953318306044e-05, "Finetune/Loss": 0.9495880603790283, "Finetune/Loss (Raw)": 1.0832587480545044, "Finetune/Step": 1846, "Finetune/Step Time": 5.379972148686647} +{"Finetune/Learning Rate": 1.4940538058644332e-05, "Finetune/Loss": 0.9500797986984253, "Finetune/Loss (Raw)": 1.0270031690597534, "Finetune/Step": 1847, "Finetune/Step Time": 5.388396808877587} +{"Finetune/Learning Rate": 1.4935120881656263e-05, "Finetune/Loss": 0.9495987892150879, "Finetune/Loss (Raw)": 0.9811142086982727, "Finetune/Step": 1848, "Finetune/Step Time": 5.380892073735595} +{"Finetune/Learning Rate": 1.4929701789444122e-05, "Finetune/Loss": 0.950023353099823, "Finetune/Loss (Raw)": 1.049973487854004, "Finetune/Step": 1849, "Finetune/Step Time": 5.374304711818695} +{"Finetune/Learning Rate": 1.4924280784110965e-05, "Finetune/Loss": 0.9504493474960327, "Finetune/Loss (Raw)": 1.0533006191253662, "Finetune/Step": 1850, "Finetune/Step Time": 5.380325593054295} +{"Finetune/Learning Rate": 1.4918857867760571e-05, "Finetune/Loss": 0.9488598108291626, "Finetune/Loss (Raw)": 0.8582696914672852, "Finetune/Step": 1851, "Finetune/Step Time": 5.383533900603652} +{"Finetune/Learning Rate": 1.4913433042497473e-05, "Finetune/Loss": 0.9498039484024048, "Finetune/Loss (Raw)": 1.015659213066101, "Finetune/Step": 1852, "Finetune/Step Time": 5.384538758546114} +{"Finetune/Learning Rate": 1.4908006310426942e-05, "Finetune/Loss": 0.9499237537384033, "Finetune/Loss (Raw)": 0.9992256164550781, "Finetune/Step": 1853, "Finetune/Step Time": 5.385221425443888} +{"Finetune/Learning Rate": 1.4902577673654977e-05, "Finetune/Loss": 0.9507479071617126, "Finetune/Loss (Raw)": 1.0082839727401733, "Finetune/Step": 1854, "Finetune/Step Time": 5.378836235031486} +{"Finetune/Learning Rate": 1.4897147134288334e-05, "Finetune/Loss": 0.9512629508972168, "Finetune/Loss (Raw)": 1.0569225549697876, "Finetune/Step": 1855, "Finetune/Step Time": 5.369597485288978} +{"Finetune/Learning Rate": 1.4891714694434495e-05, "Finetune/Loss": 0.9514450430870056, "Finetune/Loss (Raw)": 1.09916353225708, "Finetune/Step": 1856, "Finetune/Step Time": 5.369808800518513} +{"Finetune/Learning Rate": 1.4886280356201688e-05, "Finetune/Loss": 0.9506717920303345, "Finetune/Loss (Raw)": 0.9275499582290649, "Finetune/Step": 1857, "Finetune/Step Time": 5.4063611179590225} +{"Finetune/Learning Rate": 1.4880844121698872e-05, "Finetune/Loss": 0.9521660208702087, "Finetune/Loss (Raw)": 1.0204697847366333, "Finetune/Step": 1858, "Finetune/Step Time": 5.413922604173422} +{"Finetune/Learning Rate": 1.487540599303574e-05, "Finetune/Loss": 0.9518609642982483, "Finetune/Loss (Raw)": 1.076456069946289, "Finetune/Step": 1859, "Finetune/Step Time": 5.408177211880684} +{"Finetune/Learning Rate": 1.4869965972322728e-05, "Finetune/Loss": 0.9504219889640808, "Finetune/Loss (Raw)": 0.9245761036872864, "Finetune/Step": 1860, "Finetune/Step Time": 5.396741129457951} +{"Finetune/Learning Rate": 1.4864524061670999e-05, "Finetune/Loss": 0.9494182467460632, "Finetune/Loss (Raw)": 0.8650851845741272, "Finetune/Step": 1861, "Finetune/Step Time": 5.397041916847229} +{"Finetune/Learning Rate": 1.4859080263192456e-05, "Finetune/Loss": 0.9482621550559998, "Finetune/Loss (Raw)": 0.8757873773574829, "Finetune/Step": 1862, "Finetune/Step Time": 5.381218317896128} +{"Finetune/Learning Rate": 1.4853634578999728e-05, "Finetune/Loss": 0.9476487636566162, "Finetune/Loss (Raw)": 0.846265971660614, "Finetune/Step": 1863, "Finetune/Step Time": 5.380713369697332} +{"Finetune/Learning Rate": 1.4848187011206177e-05, "Finetune/Loss": 0.9467176198959351, "Finetune/Loss (Raw)": 0.8588173985481262, "Finetune/Step": 1864, "Finetune/Step Time": 5.3856346514075994} +{"Finetune/Learning Rate": 1.4842737561925901e-05, "Finetune/Loss": 0.9478000402450562, "Finetune/Loss (Raw)": 1.0483479499816895, "Finetune/Step": 1865, "Finetune/Step Time": 5.387983649969101} +{"Finetune/Learning Rate": 1.4837286233273722e-05, "Finetune/Loss": 0.9533411264419556, "Finetune/Loss (Raw)": 1.0658609867095947, "Finetune/Step": 1866, "Finetune/Step Time": 5.370632914826274} +{"Finetune/Learning Rate": 1.48318330273652e-05, "Finetune/Loss": 0.9546639919281006, "Finetune/Loss (Raw)": 1.1115758419036865, "Finetune/Step": 1867, "Finetune/Step Time": 5.377207914367318} +{"Finetune/Learning Rate": 1.4826377946316611e-05, "Finetune/Loss": 0.9555017948150635, "Finetune/Loss (Raw)": 0.9974673986434937, "Finetune/Step": 1868, "Finetune/Step Time": 5.378773096948862} +{"Finetune/Learning Rate": 1.4820920992244964e-05, "Finetune/Loss": 0.9550576210021973, "Finetune/Loss (Raw)": 1.0694466829299927, "Finetune/Step": 1869, "Finetune/Step Time": 5.376010285690427} +{"Finetune/Learning Rate": 1.481546216726801e-05, "Finetune/Loss": 0.9554197788238525, "Finetune/Loss (Raw)": 0.9098233580589294, "Finetune/Step": 1870, "Finetune/Step Time": 5.36872766725719} +{"Finetune/Learning Rate": 1.4810001473504194e-05, "Finetune/Loss": 0.9555076360702515, "Finetune/Loss (Raw)": 0.9763656258583069, "Finetune/Step": 1871, "Finetune/Step Time": 5.382407136261463} +{"Finetune/Learning Rate": 1.4804538913072719e-05, "Finetune/Loss": 0.9550065994262695, "Finetune/Loss (Raw)": 0.8606560826301575, "Finetune/Step": 1872, "Finetune/Step Time": 5.386742075905204} +{"Finetune/Learning Rate": 1.479907448809349e-05, "Finetune/Loss": 0.9501986503601074, "Finetune/Loss (Raw)": 0.36769646406173706, "Finetune/Step": 1873, "Finetune/Step Time": 5.412078874185681} +{"Finetune/Learning Rate": 1.4793608200687146e-05, "Finetune/Loss": 0.9503861665725708, "Finetune/Loss (Raw)": 1.0670349597930908, "Finetune/Step": 1874, "Finetune/Step Time": 5.412717990577221} +{"Finetune/Learning Rate": 1.478814005297505e-05, "Finetune/Loss": 0.9542011022567749, "Finetune/Loss (Raw)": 0.9251059293746948, "Finetune/Step": 1875, "Finetune/Step Time": 5.387417733669281} +{"Finetune/Learning Rate": 1.4782670047079275e-05, "Finetune/Loss": 0.9550931453704834, "Finetune/Loss (Raw)": 0.9692337512969971, "Finetune/Step": 1876, "Finetune/Step Time": 5.374486815184355} +{"Finetune/Learning Rate": 1.477719818512263e-05, "Finetune/Loss": 0.9550283551216125, "Finetune/Loss (Raw)": 0.968490481376648, "Finetune/Step": 1877, "Finetune/Step Time": 5.3788976818323135} +{"Finetune/Learning Rate": 1.4771724469228638e-05, "Finetune/Loss": 0.9528007507324219, "Finetune/Loss (Raw)": 0.8220067620277405, "Finetune/Step": 1878, "Finetune/Step Time": 5.366496359929442} +{"Finetune/Learning Rate": 1.4766248901521539e-05, "Finetune/Loss": 0.9530773162841797, "Finetune/Loss (Raw)": 1.0758143663406372, "Finetune/Step": 1879, "Finetune/Step Time": 5.357590055093169} +{"Finetune/Learning Rate": 1.4760771484126296e-05, "Finetune/Loss": 0.9503685235977173, "Finetune/Loss (Raw)": 0.8350292444229126, "Finetune/Step": 1880, "Finetune/Step Time": 5.383783705532551} +{"Finetune/Learning Rate": 1.4755292219168581e-05, "Finetune/Loss": 0.9496668577194214, "Finetune/Loss (Raw)": 0.8545149564743042, "Finetune/Step": 1881, "Finetune/Step Time": 5.396399844437838} +{"Finetune/Learning Rate": 1.4749811108774798e-05, "Finetune/Loss": 0.9496626853942871, "Finetune/Loss (Raw)": 0.9994807243347168, "Finetune/Step": 1882, "Finetune/Step Time": 5.392578259110451} +{"Finetune/Learning Rate": 1.4744328155072058e-05, "Finetune/Loss": 0.9490009546279907, "Finetune/Loss (Raw)": 0.8997185230255127, "Finetune/Step": 1883, "Finetune/Step Time": 5.395506788045168} +{"Finetune/Learning Rate": 1.4738843360188184e-05, "Finetune/Loss": 0.9432587623596191, "Finetune/Loss (Raw)": 0.3572183847427368, "Finetune/Step": 1884, "Finetune/Step Time": 5.4189876858145} +{"Finetune/Learning Rate": 1.4733356726251722e-05, "Finetune/Loss": 0.9481638669967651, "Finetune/Loss (Raw)": 0.9912094473838806, "Finetune/Step": 1885, "Finetune/Step Time": 5.394474882632494} +{"Finetune/Learning Rate": 1.4727868255391924e-05, "Finetune/Loss": 0.9499396681785583, "Finetune/Loss (Raw)": 1.0884478092193604, "Finetune/Step": 1886, "Finetune/Step Time": 5.398979915305972} +{"Finetune/Learning Rate": 1.472237794973876e-05, "Finetune/Loss": 0.9483392834663391, "Finetune/Loss (Raw)": 0.8357003331184387, "Finetune/Step": 1887, "Finetune/Step Time": 5.399525186046958} +{"Finetune/Learning Rate": 1.471688581142291e-05, "Finetune/Loss": 0.9462459087371826, "Finetune/Loss (Raw)": 0.7828104496002197, "Finetune/Step": 1888, "Finetune/Step Time": 5.404621770605445} +{"Finetune/Learning Rate": 1.4711391842575765e-05, "Finetune/Loss": 0.9468725919723511, "Finetune/Loss (Raw)": 0.9437929391860962, "Finetune/Step": 1889, "Finetune/Step Time": 5.4105914533138275} +{"Finetune/Learning Rate": 1.4705896045329432e-05, "Finetune/Loss": 0.9476009607315063, "Finetune/Loss (Raw)": 0.941368579864502, "Finetune/Step": 1890, "Finetune/Step Time": 5.40425493940711} +{"Finetune/Learning Rate": 1.4700398421816714e-05, "Finetune/Loss": 0.947014570236206, "Finetune/Loss (Raw)": 1.020565390586853, "Finetune/Step": 1891, "Finetune/Step Time": 5.382798872888088} +{"Finetune/Learning Rate": 1.469489897417114e-05, "Finetune/Loss": 0.9476040601730347, "Finetune/Loss (Raw)": 1.1328904628753662, "Finetune/Step": 1892, "Finetune/Step Time": 5.3632295075803995} +{"Finetune/Learning Rate": 1.468939770452693e-05, "Finetune/Loss": 0.9478861689567566, "Finetune/Loss (Raw)": 0.9409939050674438, "Finetune/Step": 1893, "Finetune/Step Time": 5.364809924736619} +{"Finetune/Learning Rate": 1.4683894615019028e-05, "Finetune/Loss": 0.9465315341949463, "Finetune/Loss (Raw)": 0.8342294096946716, "Finetune/Step": 1894, "Finetune/Step Time": 5.3563372194767} +{"Finetune/Learning Rate": 1.4678389707783072e-05, "Finetune/Loss": 0.9472824931144714, "Finetune/Loss (Raw)": 0.9240165948867798, "Finetune/Step": 1895, "Finetune/Step Time": 5.355881325900555} +{"Finetune/Learning Rate": 1.4672882984955404e-05, "Finetune/Loss": 0.947844922542572, "Finetune/Loss (Raw)": 0.9737424850463867, "Finetune/Step": 1896, "Finetune/Step Time": 5.363278530538082} +{"Finetune/Learning Rate": 1.4667374448673081e-05, "Finetune/Loss": 0.9503070116043091, "Finetune/Loss (Raw)": 1.1196165084838867, "Finetune/Step": 1897, "Finetune/Step Time": 5.355431506410241} +{"Finetune/Learning Rate": 1.4661864101073857e-05, "Finetune/Loss": 0.9502294063568115, "Finetune/Loss (Raw)": 1.013797402381897, "Finetune/Step": 1898, "Finetune/Step Time": 5.355282047763467} +{"Finetune/Learning Rate": 1.465635194429619e-05, "Finetune/Loss": 0.9497700333595276, "Finetune/Loss (Raw)": 0.8480003476142883, "Finetune/Step": 1899, "Finetune/Step Time": 5.374412743374705} +{"Finetune/Learning Rate": 1.4650837980479246e-05, "Finetune/Loss": 0.9482203722000122, "Finetune/Loss (Raw)": 0.8995512127876282, "Finetune/Step": 1900, "Finetune/Step Time": 5.375539703294635} +{"Finetune/Learning Rate": 1.4645322211762876e-05, "Finetune/Loss": 0.9478682279586792, "Finetune/Loss (Raw)": 0.9255640506744385, "Finetune/Step": 1901, "Finetune/Step Time": 5.371943198144436} +{"Finetune/Learning Rate": 1.4639804640287652e-05, "Finetune/Loss": 0.9481651186943054, "Finetune/Loss (Raw)": 1.0033719539642334, "Finetune/Step": 1902, "Finetune/Step Time": 5.36263682320714} +{"Finetune/Learning Rate": 1.4634285268194828e-05, "Finetune/Loss": 0.94673091173172, "Finetune/Loss (Raw)": 0.8577322363853455, "Finetune/Step": 1903, "Finetune/Step Time": 5.382475210353732} +{"Finetune/Learning Rate": 1.462876409762637e-05, "Finetune/Loss": 0.9467155933380127, "Finetune/Loss (Raw)": 0.9553236961364746, "Finetune/Step": 1904, "Finetune/Step Time": 5.386946387588978} +{"Finetune/Learning Rate": 1.4623241130724937e-05, "Finetune/Loss": 0.9412389993667603, "Finetune/Loss (Raw)": 0.3219192922115326, "Finetune/Step": 1905, "Finetune/Step Time": 5.420663869008422} +{"Finetune/Learning Rate": 1.4617716369633876e-05, "Finetune/Loss": 0.9413154125213623, "Finetune/Loss (Raw)": 0.962645947933197, "Finetune/Step": 1906, "Finetune/Step Time": 5.432445857673883} +{"Finetune/Learning Rate": 1.461218981649725e-05, "Finetune/Loss": 0.9403276443481445, "Finetune/Loss (Raw)": 0.9740082025527954, "Finetune/Step": 1907, "Finetune/Step Time": 5.448161941021681} +{"Finetune/Learning Rate": 1.4606661473459801e-05, "Finetune/Loss": 0.9401067495346069, "Finetune/Loss (Raw)": 0.9053664207458496, "Finetune/Step": 1908, "Finetune/Step Time": 5.445842083543539} +{"Finetune/Learning Rate": 1.4601131342666973e-05, "Finetune/Loss": 0.9395493268966675, "Finetune/Loss (Raw)": 0.753720223903656, "Finetune/Step": 1909, "Finetune/Step Time": 5.446658140048385} +{"Finetune/Learning Rate": 1.45955994262649e-05, "Finetune/Loss": 0.939598560333252, "Finetune/Loss (Raw)": 0.9430367946624756, "Finetune/Step": 1910, "Finetune/Step Time": 5.449008775874972} +{"Finetune/Learning Rate": 1.459006572640041e-05, "Finetune/Loss": 0.9406335949897766, "Finetune/Loss (Raw)": 1.00212562084198, "Finetune/Step": 1911, "Finetune/Step Time": 5.454100778326392} +{"Finetune/Learning Rate": 1.4584530245221025e-05, "Finetune/Loss": 0.9399675130844116, "Finetune/Loss (Raw)": 0.9875209331512451, "Finetune/Step": 1912, "Finetune/Step Time": 5.448120139539242} +{"Finetune/Learning Rate": 1.4578992984874956e-05, "Finetune/Loss": 0.9415338039398193, "Finetune/Loss (Raw)": 0.9819661974906921, "Finetune/Step": 1913, "Finetune/Step Time": 5.432492058724165} +{"Finetune/Learning Rate": 1.457345394751111e-05, "Finetune/Loss": 0.942933201789856, "Finetune/Loss (Raw)": 1.103899598121643, "Finetune/Step": 1914, "Finetune/Step Time": 5.429946009069681} +{"Finetune/Learning Rate": 1.4567913135279075e-05, "Finetune/Loss": 0.9450575709342957, "Finetune/Loss (Raw)": 1.077090859413147, "Finetune/Step": 1915, "Finetune/Step Time": 5.44703639857471} +{"Finetune/Learning Rate": 1.456237055032913e-05, "Finetune/Loss": 0.9454861283302307, "Finetune/Loss (Raw)": 0.931998074054718, "Finetune/Step": 1916, "Finetune/Step Time": 5.446946224197745} +{"Finetune/Learning Rate": 1.455682619481225e-05, "Finetune/Loss": 0.9499173164367676, "Finetune/Loss (Raw)": 0.8617222309112549, "Finetune/Step": 1917, "Finetune/Step Time": 5.415211461484432} +{"Finetune/Learning Rate": 1.4551280070880089e-05, "Finetune/Loss": 0.9487286806106567, "Finetune/Loss (Raw)": 0.8991473913192749, "Finetune/Step": 1918, "Finetune/Step Time": 5.403721679002047} +{"Finetune/Learning Rate": 1.4545732180684989e-05, "Finetune/Loss": 0.9476985931396484, "Finetune/Loss (Raw)": 0.9621113538742065, "Finetune/Step": 1919, "Finetune/Step Time": 5.400026548653841} +{"Finetune/Learning Rate": 1.4540182526379976e-05, "Finetune/Loss": 0.9465910196304321, "Finetune/Loss (Raw)": 0.8644111752510071, "Finetune/Step": 1920, "Finetune/Step Time": 5.410431243479252} +{"Finetune/Learning Rate": 1.4534631110118763e-05, "Finetune/Loss": 0.9482418298721313, "Finetune/Loss (Raw)": 1.029654622077942, "Finetune/Step": 1921, "Finetune/Step Time": 5.394300287589431} +{"Finetune/Learning Rate": 1.4529077934055754e-05, "Finetune/Loss": 0.9423980712890625, "Finetune/Loss (Raw)": 0.3054847717285156, "Finetune/Step": 1922, "Finetune/Step Time": 5.429342865943909} +{"Finetune/Learning Rate": 1.4523523000346012e-05, "Finetune/Loss": 0.9471064805984497, "Finetune/Loss (Raw)": 0.9752575159072876, "Finetune/Step": 1923, "Finetune/Step Time": 5.398972971364856} +{"Finetune/Learning Rate": 1.4517966311145313e-05, "Finetune/Loss": 0.9481894373893738, "Finetune/Loss (Raw)": 1.0557500123977661, "Finetune/Step": 1924, "Finetune/Step Time": 5.397673701867461} +{"Finetune/Learning Rate": 1.4512407868610094e-05, "Finetune/Loss": 0.9493653774261475, "Finetune/Loss (Raw)": 1.1699796915054321, "Finetune/Step": 1925, "Finetune/Step Time": 5.394891567528248} +{"Finetune/Learning Rate": 1.450684767489748e-05, "Finetune/Loss": 0.9447150230407715, "Finetune/Loss (Raw)": 0.3388385474681854, "Finetune/Step": 1926, "Finetune/Step Time": 5.420869167894125} +{"Finetune/Learning Rate": 1.4501285732165271e-05, "Finetune/Loss": 0.9405582547187805, "Finetune/Loss (Raw)": 0.4005410969257355, "Finetune/Step": 1927, "Finetune/Step Time": 5.453126499429345} +{"Finetune/Learning Rate": 1.4495722042571946e-05, "Finetune/Loss": 0.9415066242218018, "Finetune/Loss (Raw)": 1.0419323444366455, "Finetune/Step": 1928, "Finetune/Step Time": 5.441028432920575} +{"Finetune/Learning Rate": 1.4490156608276674e-05, "Finetune/Loss": 0.9396286606788635, "Finetune/Loss (Raw)": 0.8346238732337952, "Finetune/Step": 1929, "Finetune/Step Time": 5.432814778760076} +{"Finetune/Learning Rate": 1.4484589431439283e-05, "Finetune/Loss": 0.9391567707061768, "Finetune/Loss (Raw)": 0.9072596430778503, "Finetune/Step": 1930, "Finetune/Step Time": 5.418359380215406} +{"Finetune/Learning Rate": 1.4479020514220285e-05, "Finetune/Loss": 0.9395033121109009, "Finetune/Loss (Raw)": 0.9794701933860779, "Finetune/Step": 1931, "Finetune/Step Time": 5.396451339125633} +{"Finetune/Learning Rate": 1.4473449858780874e-05, "Finetune/Loss": 0.93842613697052, "Finetune/Loss (Raw)": 0.8550562858581543, "Finetune/Step": 1932, "Finetune/Step Time": 5.410813542082906} +{"Finetune/Learning Rate": 1.4467877467282906e-05, "Finetune/Loss": 0.941034197807312, "Finetune/Loss (Raw)": 0.9849768877029419, "Finetune/Step": 1933, "Finetune/Step Time": 5.403850607573986} +{"Finetune/Learning Rate": 1.4462303341888925e-05, "Finetune/Loss": 0.9421926736831665, "Finetune/Loss (Raw)": 1.020232915878296, "Finetune/Step": 1934, "Finetune/Step Time": 5.3900989927351475} +{"Finetune/Learning Rate": 1.4456727484762137e-05, "Finetune/Loss": 0.9429413080215454, "Finetune/Loss (Raw)": 1.0524988174438477, "Finetune/Step": 1935, "Finetune/Step Time": 5.396301636472344} +{"Finetune/Learning Rate": 1.4451149898066418e-05, "Finetune/Loss": 0.941900908946991, "Finetune/Loss (Raw)": 0.8423939347267151, "Finetune/Step": 1936, "Finetune/Step Time": 5.404902242124081} +{"Finetune/Learning Rate": 1.4445570583966329e-05, "Finetune/Loss": 0.9420669078826904, "Finetune/Loss (Raw)": 0.9849027991294861, "Finetune/Step": 1937, "Finetune/Step Time": 5.403737111017108} +{"Finetune/Learning Rate": 1.4439989544627085e-05, "Finetune/Loss": 0.9376245737075806, "Finetune/Loss (Raw)": 0.40294063091278076, "Finetune/Step": 1938, "Finetune/Step Time": 5.429096732288599} +{"Finetune/Learning Rate": 1.4434406782214584e-05, "Finetune/Loss": 0.9376834034919739, "Finetune/Loss (Raw)": 0.9553512334823608, "Finetune/Step": 1939, "Finetune/Step Time": 5.424383617937565} +{"Finetune/Learning Rate": 1.4428822298895386e-05, "Finetune/Loss": 0.9351300001144409, "Finetune/Loss (Raw)": 0.6105027794837952, "Finetune/Step": 1940, "Finetune/Step Time": 5.430741496384144} +{"Finetune/Learning Rate": 1.442323609683672e-05, "Finetune/Loss": 0.9349603652954102, "Finetune/Loss (Raw)": 0.9202030897140503, "Finetune/Step": 1941, "Finetune/Step Time": 5.418581617996097} +{"Finetune/Learning Rate": 1.4417648178206481e-05, "Finetune/Loss": 0.9360966086387634, "Finetune/Loss (Raw)": 1.144718050956726, "Finetune/Step": 1942, "Finetune/Step Time": 5.420721482485533} +{"Finetune/Learning Rate": 1.4412058545173228e-05, "Finetune/Loss": 0.9373053312301636, "Finetune/Loss (Raw)": 1.0531301498413086, "Finetune/Step": 1943, "Finetune/Step Time": 5.425527084618807} +{"Finetune/Learning Rate": 1.4406467199906194e-05, "Finetune/Loss": 0.9365854263305664, "Finetune/Loss (Raw)": 0.9670996069908142, "Finetune/Step": 1944, "Finetune/Step Time": 5.424502657726407} +{"Finetune/Learning Rate": 1.440087414457527e-05, "Finetune/Loss": 0.9323865175247192, "Finetune/Loss (Raw)": 0.38643786311149597, "Finetune/Step": 1945, "Finetune/Step Time": 5.455070689320564} +{"Finetune/Learning Rate": 1.4395279381351008e-05, "Finetune/Loss": 0.9326151609420776, "Finetune/Loss (Raw)": 0.9469231367111206, "Finetune/Step": 1946, "Finetune/Step Time": 5.461963998153806} +{"Finetune/Learning Rate": 1.438968291240463e-05, "Finetune/Loss": 0.9337615370750427, "Finetune/Loss (Raw)": 1.141413688659668, "Finetune/Step": 1947, "Finetune/Step Time": 5.455071318894625} +{"Finetune/Learning Rate": 1.4384084739908016e-05, "Finetune/Loss": 0.935539722442627, "Finetune/Loss (Raw)": 1.0942577123641968, "Finetune/Step": 1948, "Finetune/Step Time": 5.459822621196508} +{"Finetune/Learning Rate": 1.4378484866033705e-05, "Finetune/Loss": 0.9362627267837524, "Finetune/Loss (Raw)": 1.0720680952072144, "Finetune/Step": 1949, "Finetune/Step Time": 5.45849840156734} +{"Finetune/Learning Rate": 1.43728832929549e-05, "Finetune/Loss": 0.9353960752487183, "Finetune/Loss (Raw)": 1.0863031148910522, "Finetune/Step": 1950, "Finetune/Step Time": 5.4547198340296745} +{"Finetune/Learning Rate": 1.4367280022845462e-05, "Finetune/Loss": 0.9362219572067261, "Finetune/Loss (Raw)": 1.0709396600723267, "Finetune/Step": 1951, "Finetune/Step Time": 5.459241127595305} +{"Finetune/Learning Rate": 1.4361675057879913e-05, "Finetune/Loss": 0.9380794763565063, "Finetune/Loss (Raw)": 1.0982118844985962, "Finetune/Step": 1952, "Finetune/Step Time": 5.453459372743964} +{"Finetune/Learning Rate": 1.4356068400233429e-05, "Finetune/Loss": 0.9366835951805115, "Finetune/Loss (Raw)": 0.7442300915718079, "Finetune/Step": 1953, "Finetune/Step Time": 5.470163239166141} +{"Finetune/Learning Rate": 1.435046005208184e-05, "Finetune/Loss": 0.9354543685913086, "Finetune/Loss (Raw)": 0.8702812790870667, "Finetune/Step": 1954, "Finetune/Step Time": 5.478472150862217} +{"Finetune/Learning Rate": 1.4344850015601644e-05, "Finetune/Loss": 0.9346739053726196, "Finetune/Loss (Raw)": 0.8558804988861084, "Finetune/Step": 1955, "Finetune/Step Time": 5.463283706456423} +{"Finetune/Learning Rate": 1.4339238292969981e-05, "Finetune/Loss": 0.932873010635376, "Finetune/Loss (Raw)": 0.8708371520042419, "Finetune/Step": 1956, "Finetune/Step Time": 5.468363106250763} +{"Finetune/Learning Rate": 1.4333624886364655e-05, "Finetune/Loss": 0.9324896335601807, "Finetune/Loss (Raw)": 1.0906356573104858, "Finetune/Step": 1957, "Finetune/Step Time": 5.462022624909878} +{"Finetune/Learning Rate": 1.4328009797964115e-05, "Finetune/Loss": 0.9326058626174927, "Finetune/Loss (Raw)": 1.0782253742218018, "Finetune/Step": 1958, "Finetune/Step Time": 5.4421498738229275} +{"Finetune/Learning Rate": 1.432239302994747e-05, "Finetune/Loss": 0.9320109486579895, "Finetune/Loss (Raw)": 0.8540976643562317, "Finetune/Step": 1959, "Finetune/Step Time": 5.444517612457275} +{"Finetune/Learning Rate": 1.431677458449448e-05, "Finetune/Loss": 0.9327211380004883, "Finetune/Loss (Raw)": 1.1419663429260254, "Finetune/Step": 1960, "Finetune/Step Time": 5.449527118355036} +{"Finetune/Learning Rate": 1.431115446378555e-05, "Finetune/Loss": 0.9324671030044556, "Finetune/Loss (Raw)": 0.8311821818351746, "Finetune/Step": 1961, "Finetune/Step Time": 5.442845800891519} +{"Finetune/Learning Rate": 1.4305532670001743e-05, "Finetune/Loss": 0.9270573854446411, "Finetune/Loss (Raw)": 0.4097360670566559, "Finetune/Step": 1962, "Finetune/Step Time": 5.471030389890075} +{"Finetune/Learning Rate": 1.4299909205324762e-05, "Finetune/Loss": 0.9225366115570068, "Finetune/Loss (Raw)": 0.3975188136100769, "Finetune/Step": 1963, "Finetune/Step Time": 5.488459089770913} +{"Finetune/Learning Rate": 1.4294284071936968e-05, "Finetune/Loss": 0.9228748083114624, "Finetune/Loss (Raw)": 1.0332564115524292, "Finetune/Step": 1964, "Finetune/Step Time": 5.495298946276307} +{"Finetune/Learning Rate": 1.4288657272021367e-05, "Finetune/Loss": 0.9219543933868408, "Finetune/Loss (Raw)": 0.8344356417655945, "Finetune/Step": 1965, "Finetune/Step Time": 5.4942192658782005} +{"Finetune/Learning Rate": 1.4283028807761603e-05, "Finetune/Loss": 0.9210604429244995, "Finetune/Loss (Raw)": 0.8914265632629395, "Finetune/Step": 1966, "Finetune/Step Time": 5.489044364541769} +{"Finetune/Learning Rate": 1.4277398681341982e-05, "Finetune/Loss": 0.9198318123817444, "Finetune/Loss (Raw)": 0.8448576331138611, "Finetune/Step": 1967, "Finetune/Step Time": 5.476545127108693} +{"Finetune/Learning Rate": 1.4271766894947441e-05, "Finetune/Loss": 0.9196904897689819, "Finetune/Loss (Raw)": 1.0248467922210693, "Finetune/Step": 1968, "Finetune/Step Time": 5.4627276081591845} +{"Finetune/Learning Rate": 1.4266133450763564e-05, "Finetune/Loss": 0.9165675640106201, "Finetune/Loss (Raw)": 0.4466623365879059, "Finetune/Step": 1969, "Finetune/Step Time": 5.4922015853226185} +{"Finetune/Learning Rate": 1.426049835097659e-05, "Finetune/Loss": 0.9153328537940979, "Finetune/Loss (Raw)": 0.9266140460968018, "Finetune/Step": 1970, "Finetune/Step Time": 5.487062223255634} +{"Finetune/Learning Rate": 1.425486159777338e-05, "Finetune/Loss": 0.9107352495193481, "Finetune/Loss (Raw)": 0.40589699149131775, "Finetune/Step": 1971, "Finetune/Step Time": 5.502081660553813} +{"Finetune/Learning Rate": 1.4249223193341456e-05, "Finetune/Loss": 0.9103996157646179, "Finetune/Loss (Raw)": 0.918143093585968, "Finetune/Step": 1972, "Finetune/Step Time": 5.490194838494062} +{"Finetune/Learning Rate": 1.4243583139868969e-05, "Finetune/Loss": 0.9098610877990723, "Finetune/Loss (Raw)": 0.8886532783508301, "Finetune/Step": 1973, "Finetune/Step Time": 5.517143525183201} +{"Finetune/Learning Rate": 1.423794143954471e-05, "Finetune/Loss": 0.9098871946334839, "Finetune/Loss (Raw)": 1.0865906476974487, "Finetune/Step": 1974, "Finetune/Step Time": 5.516295516863465} +{"Finetune/Learning Rate": 1.4232298094558125e-05, "Finetune/Loss": 0.9091428518295288, "Finetune/Loss (Raw)": 0.9317376613616943, "Finetune/Step": 1975, "Finetune/Step Time": 5.506041890010238} +{"Finetune/Learning Rate": 1.4226653107099273e-05, "Finetune/Loss": 0.9090142846107483, "Finetune/Loss (Raw)": 0.9646547436714172, "Finetune/Step": 1976, "Finetune/Step Time": 5.508884161710739} +{"Finetune/Learning Rate": 1.422100647935887e-05, "Finetune/Loss": 0.9086304306983948, "Finetune/Loss (Raw)": 1.0008348226547241, "Finetune/Step": 1977, "Finetune/Step Time": 5.517247760668397} +{"Finetune/Learning Rate": 1.421535821352826e-05, "Finetune/Loss": 0.9086964130401611, "Finetune/Loss (Raw)": 1.0617475509643555, "Finetune/Step": 1978, "Finetune/Step Time": 5.510257627815008} +{"Finetune/Learning Rate": 1.4209708311799424e-05, "Finetune/Loss": 0.9049551486968994, "Finetune/Loss (Raw)": 0.37939536571502686, "Finetune/Step": 1979, "Finetune/Step Time": 5.517739957198501} +{"Finetune/Learning Rate": 1.4204056776364985e-05, "Finetune/Loss": 0.9036829471588135, "Finetune/Loss (Raw)": 0.8528051376342773, "Finetune/Step": 1980, "Finetune/Step Time": 5.530769532546401} +{"Finetune/Learning Rate": 1.4198403609418184e-05, "Finetune/Loss": 0.9038840532302856, "Finetune/Loss (Raw)": 1.024966835975647, "Finetune/Step": 1981, "Finetune/Step Time": 5.53133593313396} +{"Finetune/Learning Rate": 1.419274881315291e-05, "Finetune/Loss": 0.9036164283752441, "Finetune/Loss (Raw)": 0.9740381240844727, "Finetune/Step": 1982, "Finetune/Step Time": 5.5389082580804825} +{"Finetune/Learning Rate": 1.418709238976368e-05, "Finetune/Loss": 0.902451753616333, "Finetune/Loss (Raw)": 0.907841682434082, "Finetune/Step": 1983, "Finetune/Step Time": 5.53143941052258} +{"Finetune/Learning Rate": 1.4181434341445634e-05, "Finetune/Loss": 0.9025331139564514, "Finetune/Loss (Raw)": 1.1095751523971558, "Finetune/Step": 1984, "Finetune/Step Time": 5.531099380925298} +{"Finetune/Learning Rate": 1.4175774670394561e-05, "Finetune/Loss": 0.9032373428344727, "Finetune/Loss (Raw)": 1.0176889896392822, "Finetune/Step": 1985, "Finetune/Step Time": 5.491530789062381} +{"Finetune/Learning Rate": 1.4170113378806862e-05, "Finetune/Loss": 0.9045698642730713, "Finetune/Loss (Raw)": 1.1910367012023926, "Finetune/Step": 1986, "Finetune/Step Time": 5.482183625921607} +{"Finetune/Learning Rate": 1.4164450468879578e-05, "Finetune/Loss": 0.9036691188812256, "Finetune/Loss (Raw)": 0.9611617922782898, "Finetune/Step": 1987, "Finetune/Step Time": 5.513824101537466} +{"Finetune/Learning Rate": 1.4158785942810367e-05, "Finetune/Loss": 0.9033786654472351, "Finetune/Loss (Raw)": 0.8873968124389648, "Finetune/Step": 1988, "Finetune/Step Time": 5.511948969215155} +{"Finetune/Learning Rate": 1.4153119802797526e-05, "Finetune/Loss": 0.9042679071426392, "Finetune/Loss (Raw)": 0.9789031147956848, "Finetune/Step": 1989, "Finetune/Step Time": 5.513691930100322} +{"Finetune/Learning Rate": 1.4147452051039974e-05, "Finetune/Loss": 0.9053431749343872, "Finetune/Loss (Raw)": 1.0134201049804688, "Finetune/Step": 1990, "Finetune/Step Time": 5.514341125264764} +{"Finetune/Learning Rate": 1.4141782689737252e-05, "Finetune/Loss": 0.9052144289016724, "Finetune/Loss (Raw)": 0.8297936916351318, "Finetune/Step": 1991, "Finetune/Step Time": 5.510346880182624} +{"Finetune/Learning Rate": 1.413611172108953e-05, "Finetune/Loss": 0.9062238931655884, "Finetune/Loss (Raw)": 0.9880244731903076, "Finetune/Step": 1992, "Finetune/Step Time": 5.505248799920082} +{"Finetune/Learning Rate": 1.41304391472976e-05, "Finetune/Loss": 0.9049603343009949, "Finetune/Loss (Raw)": 0.8866167664527893, "Finetune/Step": 1993, "Finetune/Step Time": 5.509576760232449} +{"Finetune/Learning Rate": 1.4124764970562874e-05, "Finetune/Loss": 0.9026374816894531, "Finetune/Loss (Raw)": 0.7685279846191406, "Finetune/Step": 1994, "Finetune/Step Time": 5.505599582567811} +{"Finetune/Learning Rate": 1.4119089193087396e-05, "Finetune/Loss": 0.900273323059082, "Finetune/Loss (Raw)": 0.8089695572853088, "Finetune/Step": 1995, "Finetune/Step Time": 5.509190345183015} +{"Finetune/Learning Rate": 1.4113411817073818e-05, "Finetune/Loss": 0.8991373777389526, "Finetune/Loss (Raw)": 0.8520628809928894, "Finetune/Step": 1996, "Finetune/Step Time": 5.496643358841538} +{"Finetune/Learning Rate": 1.4107732844725424e-05, "Finetune/Loss": 0.8991689682006836, "Finetune/Loss (Raw)": 1.0734879970550537, "Finetune/Step": 1997, "Finetune/Step Time": 5.498784551396966} +{"Finetune/Learning Rate": 1.4102052278246107e-05, "Finetune/Loss": 0.8992428779602051, "Finetune/Loss (Raw)": 0.9192841649055481, "Finetune/Step": 1998, "Finetune/Step Time": 5.499167136847973} +{"Finetune/Learning Rate": 1.4096370119840389e-05, "Finetune/Loss": 0.8948450088500977, "Finetune/Loss (Raw)": 0.4134410619735718, "Finetune/Step": 1999, "Finetune/Step Time": 5.498964156955481} +{"Finetune/Learning Rate": 1.4090686371713403e-05, "Finetune/Loss": 0.8951154947280884, "Finetune/Loss (Raw)": 0.89527827501297, "Finetune/Step": 2000, "Finetune/Step Time": 5.522452335804701} +{"Finetune/Learning Rate": 1.4085001036070896e-05, "Finetune/Loss": 0.899823784828186, "Finetune/Loss (Raw)": 0.9703611731529236, "Finetune/Step": 2001, "Finetune/Step Time": 5.496112447232008} +{"Finetune/Learning Rate": 1.4079314115119242e-05, "Finetune/Loss": 0.8989048004150391, "Finetune/Loss (Raw)": 0.9494075775146484, "Finetune/Step": 2002, "Finetune/Step Time": 5.494293576106429} +{"Finetune/Learning Rate": 1.4073625611065422e-05, "Finetune/Loss": 0.8986327648162842, "Finetune/Loss (Raw)": 0.8902763724327087, "Finetune/Step": 2003, "Finetune/Step Time": 5.493445590138435} +{"Finetune/Learning Rate": 1.4067935526117033e-05, "Finetune/Loss": 0.8981353640556335, "Finetune/Loss (Raw)": 0.905571460723877, "Finetune/Step": 2004, "Finetune/Step Time": 5.4979906268417835} +{"Finetune/Learning Rate": 1.4062243862482286e-05, "Finetune/Loss": 0.8978217840194702, "Finetune/Loss (Raw)": 0.9283447861671448, "Finetune/Step": 2005, "Finetune/Step Time": 5.491803312674165} +{"Finetune/Learning Rate": 1.4056550622370004e-05, "Finetune/Loss": 0.8987905383110046, "Finetune/Loss (Raw)": 0.9460150003433228, "Finetune/Step": 2006, "Finetune/Step Time": 5.481339877471328} +{"Finetune/Learning Rate": 1.4050855807989626e-05, "Finetune/Loss": 0.8993357419967651, "Finetune/Loss (Raw)": 1.145597219467163, "Finetune/Step": 2007, "Finetune/Step Time": 5.487499127164483} +{"Finetune/Learning Rate": 1.4045159421551191e-05, "Finetune/Loss": 0.9005956649780273, "Finetune/Loss (Raw)": 0.9962993860244751, "Finetune/Step": 2008, "Finetune/Step Time": 5.4615730326622725} +{"Finetune/Learning Rate": 1.4039461465265362e-05, "Finetune/Loss": 0.9009616374969482, "Finetune/Loss (Raw)": 0.9013638496398926, "Finetune/Step": 2009, "Finetune/Step Time": 5.445724934339523} +{"Finetune/Learning Rate": 1.4033761941343403e-05, "Finetune/Loss": 0.9013320207595825, "Finetune/Loss (Raw)": 1.0468944311141968, "Finetune/Step": 2010, "Finetune/Step Time": 5.456727894023061} +{"Finetune/Learning Rate": 1.4028060851997183e-05, "Finetune/Loss": 0.9030016660690308, "Finetune/Loss (Raw)": 1.1134262084960938, "Finetune/Step": 2011, "Finetune/Step Time": 5.449790371581912} +{"Finetune/Learning Rate": 1.4022358199439193e-05, "Finetune/Loss": 0.907599687576294, "Finetune/Loss (Raw)": 0.9457677006721497, "Finetune/Step": 2012, "Finetune/Step Time": 5.417681563645601} +{"Finetune/Learning Rate": 1.4016653985882512e-05, "Finetune/Loss": 0.9079368710517883, "Finetune/Loss (Raw)": 1.034364104270935, "Finetune/Step": 2013, "Finetune/Step Time": 5.434024916961789} +{"Finetune/Learning Rate": 1.4010948213540838e-05, "Finetune/Loss": 0.9057780504226685, "Finetune/Loss (Raw)": 0.8121246695518494, "Finetune/Step": 2014, "Finetune/Step Time": 5.443050915375352} +{"Finetune/Learning Rate": 1.4005240884628473e-05, "Finetune/Loss": 0.9023228883743286, "Finetune/Loss (Raw)": 0.393437922000885, "Finetune/Step": 2015, "Finetune/Step Time": 5.465887852013111} +{"Finetune/Learning Rate": 1.3999532001360309e-05, "Finetune/Loss": 0.8993592262268066, "Finetune/Loss (Raw)": 0.4034639596939087, "Finetune/Step": 2016, "Finetune/Step Time": 5.49193543754518} +{"Finetune/Learning Rate": 1.3993821565951868e-05, "Finetune/Loss": 0.8987035751342773, "Finetune/Loss (Raw)": 0.8598635792732239, "Finetune/Step": 2017, "Finetune/Step Time": 5.49730184301734} +{"Finetune/Learning Rate": 1.3988109580619244e-05, "Finetune/Loss": 0.8989372253417969, "Finetune/Loss (Raw)": 0.9712821841239929, "Finetune/Step": 2018, "Finetune/Step Time": 5.498632589355111} +{"Finetune/Learning Rate": 1.3982396047579156e-05, "Finetune/Loss": 0.897792398929596, "Finetune/Loss (Raw)": 0.8740193247795105, "Finetune/Step": 2019, "Finetune/Step Time": 5.505387289449573} +{"Finetune/Learning Rate": 1.3976680969048912e-05, "Finetune/Loss": 0.8966067433357239, "Finetune/Loss (Raw)": 0.9811282157897949, "Finetune/Step": 2020, "Finetune/Step Time": 5.508594162762165} +{"Finetune/Learning Rate": 1.3970964347246419e-05, "Finetune/Loss": 0.8967868089675903, "Finetune/Loss (Raw)": 0.964046061038971, "Finetune/Step": 2021, "Finetune/Step Time": 5.510527027770877} +{"Finetune/Learning Rate": 1.3965246184390193e-05, "Finetune/Loss": 0.8974958658218384, "Finetune/Loss (Raw)": 0.9249903559684753, "Finetune/Step": 2022, "Finetune/Step Time": 5.5174578335136175} +{"Finetune/Learning Rate": 1.3959526482699335e-05, "Finetune/Loss": 0.8963437080383301, "Finetune/Loss (Raw)": 0.7765439748764038, "Finetune/Step": 2023, "Finetune/Step Time": 5.542462754994631} +{"Finetune/Learning Rate": 1.3953805244393554e-05, "Finetune/Loss": 0.8961580991744995, "Finetune/Loss (Raw)": 0.9499746561050415, "Finetune/Step": 2024, "Finetune/Step Time": 5.535969767719507} +{"Finetune/Learning Rate": 1.394808247169315e-05, "Finetune/Loss": 0.894328773021698, "Finetune/Loss (Raw)": 0.8854697346687317, "Finetune/Step": 2025, "Finetune/Step Time": 5.5481017883867025} +{"Finetune/Learning Rate": 1.3942358166819016e-05, "Finetune/Loss": 0.8939656019210815, "Finetune/Loss (Raw)": 0.9673156142234802, "Finetune/Step": 2026, "Finetune/Step Time": 5.534536128863692} +{"Finetune/Learning Rate": 1.3936632331992648e-05, "Finetune/Loss": 0.8941092491149902, "Finetune/Loss (Raw)": 0.8663790822029114, "Finetune/Step": 2027, "Finetune/Step Time": 5.5142190009355545} +{"Finetune/Learning Rate": 1.3930904969436125e-05, "Finetune/Loss": 0.8946694135665894, "Finetune/Loss (Raw)": 0.9712589979171753, "Finetune/Step": 2028, "Finetune/Step Time": 5.528362547978759} +{"Finetune/Learning Rate": 1.3925176081372129e-05, "Finetune/Loss": 0.8956788182258606, "Finetune/Loss (Raw)": 1.054762363433838, "Finetune/Step": 2029, "Finetune/Step Time": 5.521126005798578} +{"Finetune/Learning Rate": 1.3919445670023933e-05, "Finetune/Loss": 0.8905766010284424, "Finetune/Loss (Raw)": 0.35028886795043945, "Finetune/Step": 2030, "Finetune/Step Time": 5.550561608746648} +{"Finetune/Learning Rate": 1.3913713737615387e-05, "Finetune/Loss": 0.8909722566604614, "Finetune/Loss (Raw)": 0.9083735346794128, "Finetune/Step": 2031, "Finetune/Step Time": 5.529390128329396} +{"Finetune/Learning Rate": 1.3907980286370952e-05, "Finetune/Loss": 0.8917772173881531, "Finetune/Loss (Raw)": 1.0583583116531372, "Finetune/Step": 2032, "Finetune/Step Time": 5.525009276345372} +{"Finetune/Learning Rate": 1.3902245318515661e-05, "Finetune/Loss": 0.8974062204360962, "Finetune/Loss (Raw)": 1.042431116104126, "Finetune/Step": 2033, "Finetune/Step Time": 5.497769370675087} +{"Finetune/Learning Rate": 1.389650883627515e-05, "Finetune/Loss": 0.8982868790626526, "Finetune/Loss (Raw)": 1.0753718614578247, "Finetune/Step": 2034, "Finetune/Step Time": 5.4858301635831594} +{"Finetune/Learning Rate": 1.3890770841875632e-05, "Finetune/Loss": 0.897340714931488, "Finetune/Loss (Raw)": 0.8528960347175598, "Finetune/Step": 2035, "Finetune/Step Time": 5.495560921728611} +{"Finetune/Learning Rate": 1.388503133754391e-05, "Finetune/Loss": 0.8980788588523865, "Finetune/Loss (Raw)": 0.9998537302017212, "Finetune/Step": 2036, "Finetune/Step Time": 5.508911371231079} +{"Finetune/Learning Rate": 1.3879290325507378e-05, "Finetune/Loss": 0.8997623920440674, "Finetune/Loss (Raw)": 0.969214677810669, "Finetune/Step": 2037, "Finetune/Step Time": 5.523401567712426} +{"Finetune/Learning Rate": 1.3873547807994e-05, "Finetune/Loss": 0.8988882303237915, "Finetune/Loss (Raw)": 0.831138551235199, "Finetune/Step": 2038, "Finetune/Step Time": 5.525712022557855} +{"Finetune/Learning Rate": 1.386780378723235e-05, "Finetune/Loss": 0.8994026780128479, "Finetune/Loss (Raw)": 1.067978024482727, "Finetune/Step": 2039, "Finetune/Step Time": 5.520226065069437} +{"Finetune/Learning Rate": 1.3862058265451561e-05, "Finetune/Loss": 0.899635910987854, "Finetune/Loss (Raw)": 1.0173677206039429, "Finetune/Step": 2040, "Finetune/Step Time": 5.518643420189619} +{"Finetune/Learning Rate": 1.385631124488136e-05, "Finetune/Loss": 0.8997405171394348, "Finetune/Loss (Raw)": 0.9953666925430298, "Finetune/Step": 2041, "Finetune/Step Time": 5.5620743203908205} +{"Finetune/Learning Rate": 1.385056272775205e-05, "Finetune/Loss": 0.8975790143013, "Finetune/Loss (Raw)": 0.8272233009338379, "Finetune/Step": 2042, "Finetune/Step Time": 5.5632912162691355} +{"Finetune/Learning Rate": 1.3844812716294523e-05, "Finetune/Loss": 0.8965755105018616, "Finetune/Loss (Raw)": 0.9486384987831116, "Finetune/Step": 2043, "Finetune/Step Time": 5.533300869166851} +{"Finetune/Learning Rate": 1.3839061212740244e-05, "Finetune/Loss": 0.8966479301452637, "Finetune/Loss (Raw)": 0.9412693977355957, "Finetune/Step": 2044, "Finetune/Step Time": 5.532943936064839} +{"Finetune/Learning Rate": 1.383330821932126e-05, "Finetune/Loss": 0.8980702757835388, "Finetune/Loss (Raw)": 1.0437861680984497, "Finetune/Step": 2045, "Finetune/Step Time": 5.5449876226484776} +{"Finetune/Learning Rate": 1.3827553738270196e-05, "Finetune/Loss": 0.8993213772773743, "Finetune/Loss (Raw)": 1.0592867136001587, "Finetune/Step": 2046, "Finetune/Step Time": 5.5448581874370575} +{"Finetune/Learning Rate": 1.382179777182025e-05, "Finetune/Loss": 0.8994052410125732, "Finetune/Loss (Raw)": 0.9728507399559021, "Finetune/Step": 2047, "Finetune/Step Time": 5.548150157555938} +{"Finetune/Learning Rate": 1.3816040322205206e-05, "Finetune/Loss": 0.8989975452423096, "Finetune/Loss (Raw)": 0.812221109867096, "Finetune/Step": 2048, "Finetune/Step Time": 5.537047944962978} +{"Finetune/Learning Rate": 1.3810281391659414e-05, "Finetune/Loss": 0.8965821862220764, "Finetune/Loss (Raw)": 0.7204908728599548, "Finetune/Step": 2049, "Finetune/Step Time": 5.551344903185964} +{"Finetune/Learning Rate": 1.3804520982417802e-05, "Finetune/Loss": 0.900850772857666, "Finetune/Loss (Raw)": 0.8518644571304321, "Finetune/Step": 2050, "Finetune/Step Time": 5.528003027662635} +{"Finetune/Learning Rate": 1.3798759096715877e-05, "Finetune/Loss": 0.9017311334609985, "Finetune/Loss (Raw)": 1.0879361629486084, "Finetune/Step": 2051, "Finetune/Step Time": 5.532257430255413} +{"Finetune/Learning Rate": 1.379299573678971e-05, "Finetune/Loss": 0.9011707305908203, "Finetune/Loss (Raw)": 0.9840214848518372, "Finetune/Step": 2052, "Finetune/Step Time": 5.544080253690481} +{"Finetune/Learning Rate": 1.3787230904875947e-05, "Finetune/Loss": 0.8994475603103638, "Finetune/Loss (Raw)": 0.9494102597236633, "Finetune/Step": 2053, "Finetune/Step Time": 5.546480407938361} +{"Finetune/Learning Rate": 1.3781464603211811e-05, "Finetune/Loss": 0.9050402641296387, "Finetune/Loss (Raw)": 1.054709553718567, "Finetune/Step": 2054, "Finetune/Step Time": 5.517854357138276} +{"Finetune/Learning Rate": 1.3775696834035088e-05, "Finetune/Loss": 0.9098919034004211, "Finetune/Loss (Raw)": 1.0215485095977783, "Finetune/Step": 2055, "Finetune/Step Time": 5.491718199104071} +{"Finetune/Learning Rate": 1.3769927599584141e-05, "Finetune/Loss": 0.909710168838501, "Finetune/Loss (Raw)": 1.0186737775802612, "Finetune/Step": 2056, "Finetune/Step Time": 5.492448812350631} +{"Finetune/Learning Rate": 1.376415690209789e-05, "Finetune/Loss": 0.9101616144180298, "Finetune/Loss (Raw)": 0.8924062252044678, "Finetune/Step": 2057, "Finetune/Step Time": 5.5008839797228575} +{"Finetune/Learning Rate": 1.3758384743815838e-05, "Finetune/Loss": 0.9101994037628174, "Finetune/Loss (Raw)": 0.912094235420227, "Finetune/Step": 2058, "Finetune/Step Time": 5.51224641315639} +{"Finetune/Learning Rate": 1.3752611126978045e-05, "Finetune/Loss": 0.9091268181800842, "Finetune/Loss (Raw)": 0.8421810269355774, "Finetune/Step": 2059, "Finetune/Step Time": 5.5186837036162615} +{"Finetune/Learning Rate": 1.3746836053825133e-05, "Finetune/Loss": 0.9098740220069885, "Finetune/Loss (Raw)": 0.9507033824920654, "Finetune/Step": 2060, "Finetune/Step Time": 5.498519530519843} +{"Finetune/Learning Rate": 1.3741059526598299e-05, "Finetune/Loss": 0.910322904586792, "Finetune/Loss (Raw)": 1.042433500289917, "Finetune/Step": 2061, "Finetune/Step Time": 5.497373769059777} +{"Finetune/Learning Rate": 1.3735281547539304e-05, "Finetune/Loss": 0.9090893268585205, "Finetune/Loss (Raw)": 0.8623354434967041, "Finetune/Step": 2062, "Finetune/Step Time": 5.497721632942557} +{"Finetune/Learning Rate": 1.3729502118890463e-05, "Finetune/Loss": 0.9088717103004456, "Finetune/Loss (Raw)": 1.0246456861495972, "Finetune/Step": 2063, "Finetune/Step Time": 5.4898443054407835} +{"Finetune/Learning Rate": 1.3723721242894663e-05, "Finetune/Loss": 0.9097039699554443, "Finetune/Loss (Raw)": 0.9489127397537231, "Finetune/Step": 2064, "Finetune/Step Time": 5.504013726487756} +{"Finetune/Learning Rate": 1.3717938921795347e-05, "Finetune/Loss": 0.9097426533699036, "Finetune/Loss (Raw)": 0.9898627400398254, "Finetune/Step": 2065, "Finetune/Step Time": 5.51203435100615} +{"Finetune/Learning Rate": 1.3712155157836522e-05, "Finetune/Loss": 0.9092482924461365, "Finetune/Loss (Raw)": 0.3396607041358948, "Finetune/Step": 2066, "Finetune/Step Time": 5.512415751814842} +{"Finetune/Learning Rate": 1.3706369953262754e-05, "Finetune/Loss": 0.9092721939086914, "Finetune/Loss (Raw)": 0.9584121704101562, "Finetune/Step": 2067, "Finetune/Step Time": 5.513656010851264} +{"Finetune/Learning Rate": 1.3700583310319171e-05, "Finetune/Loss": 0.9117723703384399, "Finetune/Loss (Raw)": 0.9305296540260315, "Finetune/Step": 2068, "Finetune/Step Time": 5.49705021828413} +{"Finetune/Learning Rate": 1.3694795231251453e-05, "Finetune/Loss": 0.9117891788482666, "Finetune/Loss (Raw)": 0.9223447442054749, "Finetune/Step": 2069, "Finetune/Step Time": 5.4933533519506454} +{"Finetune/Learning Rate": 1.368900571830584e-05, "Finetune/Loss": 0.9084149599075317, "Finetune/Loss (Raw)": 0.7128241062164307, "Finetune/Step": 2070, "Finetune/Step Time": 5.505004080012441} +{"Finetune/Learning Rate": 1.3683214773729123e-05, "Finetune/Loss": 0.907828688621521, "Finetune/Loss (Raw)": 0.9780866503715515, "Finetune/Step": 2071, "Finetune/Step Time": 5.505209309980273} +{"Finetune/Learning Rate": 1.3677422399768667e-05, "Finetune/Loss": 0.9090117812156677, "Finetune/Loss (Raw)": 1.1185327768325806, "Finetune/Step": 2072, "Finetune/Step Time": 5.497256992384791} +{"Finetune/Learning Rate": 1.3671628598672373e-05, "Finetune/Loss": 0.9131942987442017, "Finetune/Loss (Raw)": 0.9218018651008606, "Finetune/Step": 2073, "Finetune/Step Time": 5.481673337519169} +{"Finetune/Learning Rate": 1.3665833372688702e-05, "Finetune/Loss": 0.9139162302017212, "Finetune/Loss (Raw)": 1.0393335819244385, "Finetune/Step": 2074, "Finetune/Step Time": 5.472483342513442} +{"Finetune/Learning Rate": 1.366003672406667e-05, "Finetune/Loss": 0.9119873046875, "Finetune/Loss (Raw)": 0.8945024013519287, "Finetune/Step": 2075, "Finetune/Step Time": 5.471852896735072} +{"Finetune/Learning Rate": 1.3654238655055836e-05, "Finetune/Loss": 0.9096019268035889, "Finetune/Loss (Raw)": 0.7889309525489807, "Finetune/Step": 2076, "Finetune/Step Time": 5.484451310709119} +{"Finetune/Learning Rate": 1.364843916790633e-05, "Finetune/Loss": 0.9081015586853027, "Finetune/Loss (Raw)": 0.8800244927406311, "Finetune/Step": 2077, "Finetune/Step Time": 5.4842894077301025} +{"Finetune/Learning Rate": 1.3642638264868808e-05, "Finetune/Loss": 0.9065964818000793, "Finetune/Loss (Raw)": 0.8936552405357361, "Finetune/Step": 2078, "Finetune/Step Time": 5.48411825299263} +{"Finetune/Learning Rate": 1.3636835948194494e-05, "Finetune/Loss": 0.905641496181488, "Finetune/Loss (Raw)": 0.9486979842185974, "Finetune/Step": 2079, "Finetune/Step Time": 5.4845701567828655} +{"Finetune/Learning Rate": 1.3631032220135152e-05, "Finetune/Loss": 0.9031906127929688, "Finetune/Loss (Raw)": 0.784500241279602, "Finetune/Step": 2080, "Finetune/Step Time": 5.492399916052818} +{"Finetune/Learning Rate": 1.3625227082943093e-05, "Finetune/Loss": 0.904670238494873, "Finetune/Loss (Raw)": 0.9336201548576355, "Finetune/Step": 2081, "Finetune/Step Time": 5.492208665236831} +{"Finetune/Learning Rate": 1.361942053887118e-05, "Finetune/Loss": 0.9046132564544678, "Finetune/Loss (Raw)": 0.8629894256591797, "Finetune/Step": 2082, "Finetune/Step Time": 5.479233782738447} +{"Finetune/Learning Rate": 1.361361259017282e-05, "Finetune/Loss": 0.9056046605110168, "Finetune/Loss (Raw)": 0.982779324054718, "Finetune/Step": 2083, "Finetune/Step Time": 5.461779111996293} +{"Finetune/Learning Rate": 1.3607803239101965e-05, "Finetune/Loss": 0.9075939655303955, "Finetune/Loss (Raw)": 1.1254700422286987, "Finetune/Step": 2084, "Finetune/Step Time": 5.451676273718476} +{"Finetune/Learning Rate": 1.360199248791311e-05, "Finetune/Loss": 0.9015762209892273, "Finetune/Loss (Raw)": 0.32036545872688293, "Finetune/Step": 2085, "Finetune/Step Time": 5.482053801417351} +{"Finetune/Learning Rate": 1.359618033886129e-05, "Finetune/Loss": 0.9008952975273132, "Finetune/Loss (Raw)": 0.9910662174224854, "Finetune/Step": 2086, "Finetune/Step Time": 5.486735386773944} +{"Finetune/Learning Rate": 1.359036679420209e-05, "Finetune/Loss": 0.9025830030441284, "Finetune/Loss (Raw)": 1.0701245069503784, "Finetune/Step": 2087, "Finetune/Step Time": 5.476693047210574} +{"Finetune/Learning Rate": 1.3584551856191635e-05, "Finetune/Loss": 0.9019124507904053, "Finetune/Loss (Raw)": 1.0561268329620361, "Finetune/Step": 2088, "Finetune/Step Time": 5.466923478990793} +{"Finetune/Learning Rate": 1.3578735527086585e-05, "Finetune/Loss": 0.9032255411148071, "Finetune/Loss (Raw)": 0.9992632269859314, "Finetune/Step": 2089, "Finetune/Step Time": 5.466641630977392} +{"Finetune/Learning Rate": 1.357291780914415e-05, "Finetune/Loss": 0.9021637439727783, "Finetune/Loss (Raw)": 0.27382659912109375, "Finetune/Step": 2090, "Finetune/Step Time": 5.467551352456212} +{"Finetune/Learning Rate": 1.3567098704622064e-05, "Finetune/Loss": 0.9057137966156006, "Finetune/Loss (Raw)": 0.8519271612167358, "Finetune/Step": 2091, "Finetune/Step Time": 5.439576331526041} +{"Finetune/Learning Rate": 1.3561278215778618e-05, "Finetune/Loss": 0.9056234359741211, "Finetune/Loss (Raw)": 1.0216903686523438, "Finetune/Step": 2092, "Finetune/Step Time": 5.427460599690676} +{"Finetune/Learning Rate": 1.355545634487262e-05, "Finetune/Loss": 0.9067201614379883, "Finetune/Loss (Raw)": 0.9748137593269348, "Finetune/Step": 2093, "Finetune/Step Time": 5.42538957670331} +{"Finetune/Learning Rate": 1.354963309416343e-05, "Finetune/Loss": 0.9081407785415649, "Finetune/Loss (Raw)": 1.0732698440551758, "Finetune/Step": 2094, "Finetune/Step Time": 5.426122764125466} +{"Finetune/Learning Rate": 1.3543808465910938e-05, "Finetune/Loss": 0.9091470241546631, "Finetune/Loss (Raw)": 0.9736552238464355, "Finetune/Step": 2095, "Finetune/Step Time": 5.430953962728381} +{"Finetune/Learning Rate": 1.3537982462375566e-05, "Finetune/Loss": 0.9085726737976074, "Finetune/Loss (Raw)": 0.9513291716575623, "Finetune/Step": 2096, "Finetune/Step Time": 5.433087915182114} +{"Finetune/Learning Rate": 1.3532155085818276e-05, "Finetune/Loss": 0.913395881652832, "Finetune/Loss (Raw)": 1.0640329122543335, "Finetune/Step": 2097, "Finetune/Step Time": 5.400845665484667} +{"Finetune/Learning Rate": 1.3526326338500553e-05, "Finetune/Loss": 0.9133599996566772, "Finetune/Loss (Raw)": 0.9220165610313416, "Finetune/Step": 2098, "Finetune/Step Time": 5.4096514247357845} +{"Finetune/Learning Rate": 1.3520496222684424e-05, "Finetune/Loss": 0.9130268096923828, "Finetune/Loss (Raw)": 0.3632466197013855, "Finetune/Step": 2099, "Finetune/Step Time": 5.412153804674745} +{"Finetune/Learning Rate": 1.3514664740632446e-05, "Finetune/Loss": 0.9121346473693848, "Finetune/Loss (Raw)": 0.8039537072181702, "Finetune/Step": 2100, "Finetune/Step Time": 5.417944448068738} +{"Finetune/Learning Rate": 1.3508831894607696e-05, "Finetune/Loss": 0.9132299423217773, "Finetune/Loss (Raw)": 1.0288548469543457, "Finetune/Step": 2101, "Finetune/Step Time": 5.3826092425733805} +{"Finetune/Learning Rate": 1.3502997686873797e-05, "Finetune/Loss": 0.9118078947067261, "Finetune/Loss (Raw)": 0.9045619964599609, "Finetune/Step": 2102, "Finetune/Step Time": 5.385905290022492} +{"Finetune/Learning Rate": 1.3497162119694883e-05, "Finetune/Loss": 0.907427966594696, "Finetune/Loss (Raw)": 0.37110814452171326, "Finetune/Step": 2103, "Finetune/Step Time": 5.418713318184018} +{"Finetune/Learning Rate": 1.3491325195335627e-05, "Finetune/Loss": 0.907027006149292, "Finetune/Loss (Raw)": 0.9133284091949463, "Finetune/Step": 2104, "Finetune/Step Time": 5.425595851615071} +{"Finetune/Learning Rate": 1.3485486916061228e-05, "Finetune/Loss": 0.9065175652503967, "Finetune/Loss (Raw)": 0.9356340169906616, "Finetune/Step": 2105, "Finetune/Step Time": 5.435577208176255} +{"Finetune/Learning Rate": 1.3479647284137398e-05, "Finetune/Loss": 0.9055613279342651, "Finetune/Loss (Raw)": 0.9393498301506042, "Finetune/Step": 2106, "Finetune/Step Time": 5.43422837741673} +{"Finetune/Learning Rate": 1.3473806301830398e-05, "Finetune/Loss": 0.9099600315093994, "Finetune/Loss (Raw)": 0.9424183964729309, "Finetune/Step": 2107, "Finetune/Step Time": 5.40284164622426} +{"Finetune/Learning Rate": 1.346796397140699e-05, "Finetune/Loss": 0.9103657007217407, "Finetune/Loss (Raw)": 0.9047448635101318, "Finetune/Step": 2108, "Finetune/Step Time": 5.390971792861819} +{"Finetune/Learning Rate": 1.3462120295134476e-05, "Finetune/Loss": 0.9098237156867981, "Finetune/Loss (Raw)": 0.9555871486663818, "Finetune/Step": 2109, "Finetune/Step Time": 5.394277548417449} +{"Finetune/Learning Rate": 1.3456275275280661e-05, "Finetune/Loss": 0.9098553657531738, "Finetune/Loss (Raw)": 0.9780911207199097, "Finetune/Step": 2110, "Finetune/Step Time": 5.40251680277288} +{"Finetune/Learning Rate": 1.3450428914113892e-05, "Finetune/Loss": 0.9100764989852905, "Finetune/Loss (Raw)": 0.9361495971679688, "Finetune/Step": 2111, "Finetune/Step Time": 5.417463045567274} +{"Finetune/Learning Rate": 1.3444581213903026e-05, "Finetune/Loss": 0.904992938041687, "Finetune/Loss (Raw)": 0.4588748812675476, "Finetune/Step": 2112, "Finetune/Step Time": 5.449079459533095} +{"Finetune/Learning Rate": 1.3438732176917437e-05, "Finetune/Loss": 0.9044994711875916, "Finetune/Loss (Raw)": 0.9545280337333679, "Finetune/Step": 2113, "Finetune/Step Time": 5.446915531530976} +{"Finetune/Learning Rate": 1.3432881805427032e-05, "Finetune/Loss": 0.9040055274963379, "Finetune/Loss (Raw)": 1.1278115510940552, "Finetune/Step": 2114, "Finetune/Step Time": 5.46471743658185} +{"Finetune/Learning Rate": 1.3427030101702215e-05, "Finetune/Loss": 0.9044172167778015, "Finetune/Loss (Raw)": 1.0138499736785889, "Finetune/Step": 2115, "Finetune/Step Time": 5.435905944555998} +{"Finetune/Learning Rate": 1.3421177068013925e-05, "Finetune/Loss": 0.9007793068885803, "Finetune/Loss (Raw)": 0.4217487573623657, "Finetune/Step": 2116, "Finetune/Step Time": 5.463937196880579} +{"Finetune/Learning Rate": 1.3415322706633606e-05, "Finetune/Loss": 0.899118185043335, "Finetune/Loss (Raw)": 0.7662778496742249, "Finetune/Step": 2117, "Finetune/Step Time": 5.479744225740433} +{"Finetune/Learning Rate": 1.3409467019833223e-05, "Finetune/Loss": 0.8980727195739746, "Finetune/Loss (Raw)": 0.8796008229255676, "Finetune/Step": 2118, "Finetune/Step Time": 5.475434556603432} +{"Finetune/Learning Rate": 1.3403610009885258e-05, "Finetune/Loss": 0.8984848856925964, "Finetune/Loss (Raw)": 0.8825517296791077, "Finetune/Step": 2119, "Finetune/Step Time": 5.488771030679345} +{"Finetune/Learning Rate": 1.3397751679062692e-05, "Finetune/Loss": 0.8933985233306885, "Finetune/Loss (Raw)": 0.33696451783180237, "Finetune/Step": 2120, "Finetune/Step Time": 5.520572889596224} +{"Finetune/Learning Rate": 1.3391892029639037e-05, "Finetune/Loss": 0.8938771486282349, "Finetune/Loss (Raw)": 0.9478914737701416, "Finetune/Step": 2121, "Finetune/Step Time": 5.512361157685518} +{"Finetune/Learning Rate": 1.3386031063888309e-05, "Finetune/Loss": 0.8957852125167847, "Finetune/Loss (Raw)": 1.012752890586853, "Finetune/Step": 2122, "Finetune/Step Time": 5.5118048917502165} +{"Finetune/Learning Rate": 1.3380168784085028e-05, "Finetune/Loss": 0.8974214792251587, "Finetune/Loss (Raw)": 1.0184109210968018, "Finetune/Step": 2123, "Finetune/Step Time": 5.503316102549434} +{"Finetune/Learning Rate": 1.337430519250424e-05, "Finetune/Loss": 0.8985714912414551, "Finetune/Loss (Raw)": 0.9992672801017761, "Finetune/Step": 2124, "Finetune/Step Time": 5.4995578192174435} +{"Finetune/Learning Rate": 1.3368440291421475e-05, "Finetune/Loss": 0.8981090784072876, "Finetune/Loss (Raw)": 1.0142995119094849, "Finetune/Step": 2125, "Finetune/Step Time": 5.513610517606139} +{"Finetune/Learning Rate": 1.3362574083112804e-05, "Finetune/Loss": 0.8988938927650452, "Finetune/Loss (Raw)": 1.0197412967681885, "Finetune/Step": 2126, "Finetune/Step Time": 5.510358069092035} +{"Finetune/Learning Rate": 1.3356706569854778e-05, "Finetune/Loss": 0.9030567407608032, "Finetune/Loss (Raw)": 0.9462923407554626, "Finetune/Step": 2127, "Finetune/Step Time": 5.481400568038225} +{"Finetune/Learning Rate": 1.3350837753924466e-05, "Finetune/Loss": 0.8988298773765564, "Finetune/Loss (Raw)": 0.3542279005050659, "Finetune/Step": 2128, "Finetune/Step Time": 5.4851506017148495} +{"Finetune/Learning Rate": 1.3344967637599445e-05, "Finetune/Loss": 0.8934323787689209, "Finetune/Loss (Raw)": 0.27948787808418274, "Finetune/Step": 2129, "Finetune/Step Time": 5.511604398488998} +{"Finetune/Learning Rate": 1.3339096223157782e-05, "Finetune/Loss": 0.8937407732009888, "Finetune/Loss (Raw)": 0.988875150680542, "Finetune/Step": 2130, "Finetune/Step Time": 5.509978095069528} +{"Finetune/Learning Rate": 1.3333223512878068e-05, "Finetune/Loss": 0.8945594429969788, "Finetune/Loss (Raw)": 0.9950670003890991, "Finetune/Step": 2131, "Finetune/Step Time": 5.499859377741814} +{"Finetune/Learning Rate": 1.3327349509039387e-05, "Finetune/Loss": 0.8944436311721802, "Finetune/Loss (Raw)": 0.890746533870697, "Finetune/Step": 2132, "Finetune/Step Time": 5.5040196515619755} +{"Finetune/Learning Rate": 1.3321474213921317e-05, "Finetune/Loss": 0.8962756395339966, "Finetune/Loss (Raw)": 1.1628450155258179, "Finetune/Step": 2133, "Finetune/Step Time": 5.501300795003772} +{"Finetune/Learning Rate": 1.3315597629803959e-05, "Finetune/Loss": 0.8959403038024902, "Finetune/Loss (Raw)": 0.9030979871749878, "Finetune/Step": 2134, "Finetune/Step Time": 5.512051811441779} +{"Finetune/Learning Rate": 1.3309719758967885e-05, "Finetune/Loss": 0.8938963413238525, "Finetune/Loss (Raw)": 0.8839627504348755, "Finetune/Step": 2135, "Finetune/Step Time": 5.501899553462863} +{"Finetune/Learning Rate": 1.3303840603694192e-05, "Finetune/Loss": 0.8938440084457397, "Finetune/Loss (Raw)": 0.9896035194396973, "Finetune/Step": 2136, "Finetune/Step Time": 5.494485994800925} +{"Finetune/Learning Rate": 1.3297960166264462e-05, "Finetune/Loss": 0.8901050090789795, "Finetune/Loss (Raw)": 0.4227783679962158, "Finetune/Step": 2137, "Finetune/Step Time": 5.526421125978231} +{"Finetune/Learning Rate": 1.3292078448960778e-05, "Finetune/Loss": 0.8891675472259521, "Finetune/Loss (Raw)": 0.9268865585327148, "Finetune/Step": 2138, "Finetune/Step Time": 5.518517272546887} +{"Finetune/Learning Rate": 1.3286195454065726e-05, "Finetune/Loss": 0.8874120712280273, "Finetune/Loss (Raw)": 0.8887356519699097, "Finetune/Step": 2139, "Finetune/Step Time": 5.533076833933592} +{"Finetune/Learning Rate": 1.3280311183862371e-05, "Finetune/Loss": 0.8877115249633789, "Finetune/Loss (Raw)": 0.9840905070304871, "Finetune/Step": 2140, "Finetune/Step Time": 5.535770932212472} +{"Finetune/Learning Rate": 1.3274425640634287e-05, "Finetune/Loss": 0.888131856918335, "Finetune/Loss (Raw)": 1.088167428970337, "Finetune/Step": 2141, "Finetune/Step Time": 5.516221361234784} +{"Finetune/Learning Rate": 1.3268538826665544e-05, "Finetune/Loss": 0.8885520100593567, "Finetune/Loss (Raw)": 0.8659065365791321, "Finetune/Step": 2142, "Finetune/Step Time": 5.510126210749149} +{"Finetune/Learning Rate": 1.326265074424069e-05, "Finetune/Loss": 0.892858624458313, "Finetune/Loss (Raw)": 0.9446800351142883, "Finetune/Step": 2143, "Finetune/Step Time": 5.495804145932198} +{"Finetune/Learning Rate": 1.3256761395644787e-05, "Finetune/Loss": 0.8969354629516602, "Finetune/Loss (Raw)": 0.9253056645393372, "Finetune/Step": 2144, "Finetune/Step Time": 5.476090464740992} +{"Finetune/Learning Rate": 1.3250870783163363e-05, "Finetune/Loss": 0.8958088159561157, "Finetune/Loss (Raw)": 0.7156559228897095, "Finetune/Step": 2145, "Finetune/Step Time": 5.481125762686133} +{"Finetune/Learning Rate": 1.3244978909082457e-05, "Finetune/Loss": 0.8955142498016357, "Finetune/Loss (Raw)": 0.9335724115371704, "Finetune/Step": 2146, "Finetune/Step Time": 5.496102932840586} +{"Finetune/Learning Rate": 1.3239085775688591e-05, "Finetune/Loss": 0.8962864279747009, "Finetune/Loss (Raw)": 0.9728546142578125, "Finetune/Step": 2147, "Finetune/Step Time": 5.498384630307555} +{"Finetune/Learning Rate": 1.3233191385268771e-05, "Finetune/Loss": 0.8970769643783569, "Finetune/Loss (Raw)": 1.0823267698287964, "Finetune/Step": 2148, "Finetune/Step Time": 5.500243676826358} +{"Finetune/Learning Rate": 1.3227295740110502e-05, "Finetune/Loss": 0.8973737955093384, "Finetune/Loss (Raw)": 1.0020365715026855, "Finetune/Step": 2149, "Finetune/Step Time": 5.483836743980646} +{"Finetune/Learning Rate": 1.3221398842501758e-05, "Finetune/Loss": 0.8973948955535889, "Finetune/Loss (Raw)": 0.9276880025863647, "Finetune/Step": 2150, "Finetune/Step Time": 5.486837267875671} +{"Finetune/Learning Rate": 1.321550069473102e-05, "Finetune/Loss": 0.9004037976264954, "Finetune/Loss (Raw)": 1.1616830825805664, "Finetune/Step": 2151, "Finetune/Step Time": 5.471060838550329} +{"Finetune/Learning Rate": 1.3209601299087238e-05, "Finetune/Loss": 0.8994947671890259, "Finetune/Loss (Raw)": 0.8336227536201477, "Finetune/Step": 2152, "Finetune/Step Time": 5.482793848961592} +{"Finetune/Learning Rate": 1.3203700657859855e-05, "Finetune/Loss": 0.8992729187011719, "Finetune/Loss (Raw)": 0.8570703268051147, "Finetune/Step": 2153, "Finetune/Step Time": 5.466969503089786} +{"Finetune/Learning Rate": 1.3197798773338793e-05, "Finetune/Loss": 0.8986895680427551, "Finetune/Loss (Raw)": 0.8926470279693604, "Finetune/Step": 2154, "Finetune/Step Time": 5.476360663771629} +{"Finetune/Learning Rate": 1.3191895647814457e-05, "Finetune/Loss": 0.8991119861602783, "Finetune/Loss (Raw)": 0.9204431772232056, "Finetune/Step": 2155, "Finetune/Step Time": 5.4832990150898695} +{"Finetune/Learning Rate": 1.318599128357774e-05, "Finetune/Loss": 0.8975315093994141, "Finetune/Loss (Raw)": 0.768966019153595, "Finetune/Step": 2156, "Finetune/Step Time": 5.484232984483242} +{"Finetune/Learning Rate": 1.3180085682920005e-05, "Finetune/Loss": 0.8963817358016968, "Finetune/Loss (Raw)": 0.9075931906700134, "Finetune/Step": 2157, "Finetune/Step Time": 5.493843974545598} +{"Finetune/Learning Rate": 1.31741788481331e-05, "Finetune/Loss": 0.9012585878372192, "Finetune/Loss (Raw)": 0.974526584148407, "Finetune/Step": 2158, "Finetune/Step Time": 5.457209737971425} +{"Finetune/Learning Rate": 1.316827078150936e-05, "Finetune/Loss": 0.9019873738288879, "Finetune/Loss (Raw)": 1.0016509294509888, "Finetune/Step": 2159, "Finetune/Step Time": 5.473953317850828} +{"Finetune/Learning Rate": 1.316236148534158e-05, "Finetune/Loss": 0.9012390971183777, "Finetune/Loss (Raw)": 0.9625855088233948, "Finetune/Step": 2160, "Finetune/Step Time": 5.469625044614077} +{"Finetune/Learning Rate": 1.3156450961923051e-05, "Finetune/Loss": 0.8998736143112183, "Finetune/Loss (Raw)": 0.8676406741142273, "Finetune/Step": 2161, "Finetune/Step Time": 5.463451184332371} +{"Finetune/Learning Rate": 1.3150539213547528e-05, "Finetune/Loss": 0.8988364934921265, "Finetune/Loss (Raw)": 0.9426229000091553, "Finetune/Step": 2162, "Finetune/Step Time": 5.468278242275119} +{"Finetune/Learning Rate": 1.3144626242509243e-05, "Finetune/Loss": 0.8991565704345703, "Finetune/Loss (Raw)": 0.893861711025238, "Finetune/Step": 2163, "Finetune/Step Time": 5.452360866591334} +{"Finetune/Learning Rate": 1.3138712051102908e-05, "Finetune/Loss": 0.8996180295944214, "Finetune/Loss (Raw)": 1.0589312314987183, "Finetune/Step": 2164, "Finetune/Step Time": 5.439045296981931} +{"Finetune/Learning Rate": 1.3132796641623703e-05, "Finetune/Loss": 0.8996193408966064, "Finetune/Loss (Raw)": 0.969375491142273, "Finetune/Step": 2165, "Finetune/Step Time": 5.448965486139059} +{"Finetune/Learning Rate": 1.3126880016367287e-05, "Finetune/Loss": 0.9007465839385986, "Finetune/Loss (Raw)": 0.9754235148429871, "Finetune/Step": 2166, "Finetune/Step Time": 5.448974601924419} +{"Finetune/Learning Rate": 1.3120962177629783e-05, "Finetune/Loss": 0.9009184837341309, "Finetune/Loss (Raw)": 1.0899814367294312, "Finetune/Step": 2167, "Finetune/Step Time": 5.44876167178154} +{"Finetune/Learning Rate": 1.311504312770779e-05, "Finetune/Loss": 0.9002410769462585, "Finetune/Loss (Raw)": 0.9306591749191284, "Finetune/Step": 2168, "Finetune/Step Time": 5.448039453476667} +{"Finetune/Learning Rate": 1.3109122868898374e-05, "Finetune/Loss": 0.9000676870346069, "Finetune/Loss (Raw)": 0.9731817245483398, "Finetune/Step": 2169, "Finetune/Step Time": 5.405182085931301} +{"Finetune/Learning Rate": 1.310320140349907e-05, "Finetune/Loss": 0.9002127051353455, "Finetune/Loss (Raw)": 0.8457788228988647, "Finetune/Step": 2170, "Finetune/Step Time": 5.4065292458981276} +{"Finetune/Learning Rate": 1.3097278733807892e-05, "Finetune/Loss": 0.9000802040100098, "Finetune/Loss (Raw)": 0.9316796660423279, "Finetune/Step": 2171, "Finetune/Step Time": 5.4161006063222885} +{"Finetune/Learning Rate": 1.3091354862123304e-05, "Finetune/Loss": 0.8998361229896545, "Finetune/Loss (Raw)": 0.9100306630134583, "Finetune/Step": 2172, "Finetune/Step Time": 5.41635780967772} +{"Finetune/Learning Rate": 1.3085429790744248e-05, "Finetune/Loss": 0.8990488052368164, "Finetune/Loss (Raw)": 0.9430059790611267, "Finetune/Step": 2173, "Finetune/Step Time": 5.420003160834312} +{"Finetune/Learning Rate": 1.3079503521970127e-05, "Finetune/Loss": 0.896990180015564, "Finetune/Loss (Raw)": 0.7957755923271179, "Finetune/Step": 2174, "Finetune/Step Time": 5.432777613401413} +{"Finetune/Learning Rate": 1.3073576058100815e-05, "Finetune/Loss": 0.8967863321304321, "Finetune/Loss (Raw)": 0.9467712640762329, "Finetune/Step": 2175, "Finetune/Step Time": 5.44875162281096} +{"Finetune/Learning Rate": 1.3067647401436636e-05, "Finetune/Loss": 0.8982303142547607, "Finetune/Loss (Raw)": 0.9970501661300659, "Finetune/Step": 2176, "Finetune/Step Time": 5.448517004027963} +{"Finetune/Learning Rate": 1.3061717554278393e-05, "Finetune/Loss": 0.9010798931121826, "Finetune/Loss (Raw)": 1.085227370262146, "Finetune/Step": 2177, "Finetune/Step Time": 5.437707705423236} +{"Finetune/Learning Rate": 1.3055786518927342e-05, "Finetune/Loss": 0.9007538557052612, "Finetune/Loss (Raw)": 0.8101335167884827, "Finetune/Step": 2178, "Finetune/Step Time": 5.425469541922212} +{"Finetune/Learning Rate": 1.3049854297685206e-05, "Finetune/Loss": 0.8999600410461426, "Finetune/Loss (Raw)": 0.9863350987434387, "Finetune/Step": 2179, "Finetune/Step Time": 5.434857722371817} +{"Finetune/Learning Rate": 1.3043920892854155e-05, "Finetune/Loss": 0.8999366760253906, "Finetune/Loss (Raw)": 0.9810253977775574, "Finetune/Step": 2180, "Finetune/Step Time": 5.428838292136788} +{"Finetune/Learning Rate": 1.3037986306736839e-05, "Finetune/Loss": 0.9002528190612793, "Finetune/Loss (Raw)": 0.989881157875061, "Finetune/Step": 2181, "Finetune/Step Time": 5.4268744848668575} +{"Finetune/Learning Rate": 1.3032050541636343e-05, "Finetune/Loss": 0.8998606204986572, "Finetune/Loss (Raw)": 1.0044996738433838, "Finetune/Step": 2182, "Finetune/Step Time": 5.430230803787708} +{"Finetune/Learning Rate": 1.3026113599856229e-05, "Finetune/Loss": 0.8981149196624756, "Finetune/Loss (Raw)": 0.7981110215187073, "Finetune/Step": 2183, "Finetune/Step Time": 5.446770705282688} +{"Finetune/Learning Rate": 1.302017548370051e-05, "Finetune/Loss": 0.8967186212539673, "Finetune/Loss (Raw)": 0.8399396538734436, "Finetune/Step": 2184, "Finetune/Step Time": 5.466409806162119} +{"Finetune/Learning Rate": 1.3014236195473649e-05, "Finetune/Loss": 0.8971000909805298, "Finetune/Loss (Raw)": 0.9412376880645752, "Finetune/Step": 2185, "Finetune/Step Time": 5.453003333881497} +{"Finetune/Learning Rate": 1.3008295737480567e-05, "Finetune/Loss": 0.8971773386001587, "Finetune/Loss (Raw)": 0.9219731688499451, "Finetune/Step": 2186, "Finetune/Step Time": 5.4382070209831} +{"Finetune/Learning Rate": 1.300235411202664e-05, "Finetune/Loss": 0.8965988159179688, "Finetune/Loss (Raw)": 0.7681381106376648, "Finetune/Step": 2187, "Finetune/Step Time": 5.466460099443793} +{"Finetune/Learning Rate": 1.2996411321417699e-05, "Finetune/Loss": 0.8965303897857666, "Finetune/Loss (Raw)": 0.9419496059417725, "Finetune/Step": 2188, "Finetune/Step Time": 5.499720523133874} +{"Finetune/Learning Rate": 1.2990467367960027e-05, "Finetune/Loss": 0.8950817584991455, "Finetune/Loss (Raw)": 0.8570065498352051, "Finetune/Step": 2189, "Finetune/Step Time": 5.507478315383196} +{"Finetune/Learning Rate": 1.298452225396035e-05, "Finetune/Loss": 0.8950993418693542, "Finetune/Loss (Raw)": 0.8645786046981812, "Finetune/Step": 2190, "Finetune/Step Time": 5.516582265496254} +{"Finetune/Learning Rate": 1.2978575981725858e-05, "Finetune/Loss": 0.8947501182556152, "Finetune/Loss (Raw)": 0.9799465537071228, "Finetune/Step": 2191, "Finetune/Step Time": 5.531723070889711} +{"Finetune/Learning Rate": 1.2972628553564178e-05, "Finetune/Loss": 0.8937116861343384, "Finetune/Loss (Raw)": 0.8159984350204468, "Finetune/Step": 2192, "Finetune/Step Time": 5.512892471626401} +{"Finetune/Learning Rate": 1.2966679971783387e-05, "Finetune/Loss": 0.893872857093811, "Finetune/Loss (Raw)": 1.0104937553405762, "Finetune/Step": 2193, "Finetune/Step Time": 5.507537769153714} +{"Finetune/Learning Rate": 1.2960730238692023e-05, "Finetune/Loss": 0.8989289402961731, "Finetune/Loss (Raw)": 0.9868359565734863, "Finetune/Step": 2194, "Finetune/Step Time": 5.479955295100808} +{"Finetune/Learning Rate": 1.2954779356599056e-05, "Finetune/Loss": 0.8992774486541748, "Finetune/Loss (Raw)": 1.0030176639556885, "Finetune/Step": 2195, "Finetune/Step Time": 5.479002779349685} +{"Finetune/Learning Rate": 1.2948827327813908e-05, "Finetune/Loss": 0.8987878561019897, "Finetune/Loss (Raw)": 0.8678690195083618, "Finetune/Step": 2196, "Finetune/Step Time": 5.4886304792016745} +{"Finetune/Learning Rate": 1.2942874154646447e-05, "Finetune/Loss": 0.8994491100311279, "Finetune/Loss (Raw)": 1.0069867372512817, "Finetune/Step": 2197, "Finetune/Step Time": 5.4900995790958405} +{"Finetune/Learning Rate": 1.2936919839406978e-05, "Finetune/Loss": 0.900773286819458, "Finetune/Loss (Raw)": 0.8823119401931763, "Finetune/Step": 2198, "Finetune/Step Time": 5.484032407402992} +{"Finetune/Learning Rate": 1.2930964384406259e-05, "Finetune/Loss": 0.8998955488204956, "Finetune/Loss (Raw)": 0.8657344579696655, "Finetune/Step": 2199, "Finetune/Step Time": 5.475726865231991} +{"Finetune/Learning Rate": 1.2925007791955487e-05, "Finetune/Loss": 0.8981723785400391, "Finetune/Loss (Raw)": 0.8979638814926147, "Finetune/Step": 2200, "Finetune/Step Time": 5.493399750441313} +{"Finetune/Learning Rate": 1.2919050064366294e-05, "Finetune/Loss": 0.8937324285507202, "Finetune/Loss (Raw)": 0.3534947633743286, "Finetune/Step": 2201, "Finetune/Step Time": 5.506807940080762} +{"Finetune/Learning Rate": 1.2913091203950764e-05, "Finetune/Loss": 0.8928781747817993, "Finetune/Loss (Raw)": 0.9299904704093933, "Finetune/Step": 2202, "Finetune/Step Time": 5.4981442876160145} +{"Finetune/Learning Rate": 1.2907131213021404e-05, "Finetune/Loss": 0.8927983045578003, "Finetune/Loss (Raw)": 0.8842760920524597, "Finetune/Step": 2203, "Finetune/Step Time": 5.5043316427618265} +{"Finetune/Learning Rate": 1.2901170093891182e-05, "Finetune/Loss": 0.8933452367782593, "Finetune/Loss (Raw)": 0.8589342832565308, "Finetune/Step": 2204, "Finetune/Step Time": 5.496228242293} +{"Finetune/Learning Rate": 1.2895207848873488e-05, "Finetune/Loss": 0.8898283839225769, "Finetune/Loss (Raw)": 0.4298681914806366, "Finetune/Step": 2205, "Finetune/Step Time": 5.52597339078784} +{"Finetune/Learning Rate": 1.288924448028215e-05, "Finetune/Loss": 0.8882768750190735, "Finetune/Loss (Raw)": 0.6950642466545105, "Finetune/Step": 2206, "Finetune/Step Time": 5.583048293367028} +{"Finetune/Learning Rate": 1.288327999043144e-05, "Finetune/Loss": 0.888466477394104, "Finetune/Loss (Raw)": 0.9729706645011902, "Finetune/Step": 2207, "Finetune/Step Time": 5.578324379399419} +{"Finetune/Learning Rate": 1.2877314381636052e-05, "Finetune/Loss": 0.8892030119895935, "Finetune/Loss (Raw)": 0.878773033618927, "Finetune/Step": 2208, "Finetune/Step Time": 5.5822932701557875} +{"Finetune/Learning Rate": 1.287134765621113e-05, "Finetune/Loss": 0.8890206813812256, "Finetune/Loss (Raw)": 0.9102838635444641, "Finetune/Step": 2209, "Finetune/Step Time": 5.567055217921734} +{"Finetune/Learning Rate": 1.2865379816472244e-05, "Finetune/Loss": 0.8853525519371033, "Finetune/Loss (Raw)": 0.39347222447395325, "Finetune/Step": 2210, "Finetune/Step Time": 5.597089122980833} +{"Finetune/Learning Rate": 1.2859410864735389e-05, "Finetune/Loss": 0.8856673240661621, "Finetune/Loss (Raw)": 1.0230650901794434, "Finetune/Step": 2211, "Finetune/Step Time": 5.621547043323517} +{"Finetune/Learning Rate": 1.2853440803317003e-05, "Finetune/Loss": 0.8825796842575073, "Finetune/Loss (Raw)": 0.7302621006965637, "Finetune/Step": 2212, "Finetune/Step Time": 5.638891518115997} +{"Finetune/Learning Rate": 1.2847469634533952e-05, "Finetune/Loss": 0.8865638375282288, "Finetune/Loss (Raw)": 0.8303323984146118, "Finetune/Step": 2213, "Finetune/Step Time": 5.614262128248811} +{"Finetune/Learning Rate": 1.2841497360703528e-05, "Finetune/Loss": 0.884467601776123, "Finetune/Loss (Raw)": 0.722743809223175, "Finetune/Step": 2214, "Finetune/Step Time": 5.6208301987499} +{"Finetune/Learning Rate": 1.283552398414345e-05, "Finetune/Loss": 0.8837903738021851, "Finetune/Loss (Raw)": 0.9834420084953308, "Finetune/Step": 2215, "Finetune/Step Time": 5.626166930422187} +{"Finetune/Learning Rate": 1.2829549507171877e-05, "Finetune/Loss": 0.8830336332321167, "Finetune/Loss (Raw)": 0.9592572450637817, "Finetune/Step": 2216, "Finetune/Step Time": 5.628492770716548} +{"Finetune/Learning Rate": 1.2823573932107381e-05, "Finetune/Loss": 0.8832678198814392, "Finetune/Loss (Raw)": 1.0292418003082275, "Finetune/Step": 2217, "Finetune/Step Time": 5.624581286683679} +{"Finetune/Learning Rate": 1.2817597261268965e-05, "Finetune/Loss": 0.8870220184326172, "Finetune/Loss (Raw)": 0.7543618083000183, "Finetune/Step": 2218, "Finetune/Step Time": 5.603407185524702} +{"Finetune/Learning Rate": 1.2811619496976066e-05, "Finetune/Loss": 0.8871358633041382, "Finetune/Loss (Raw)": 0.8664965629577637, "Finetune/Step": 2219, "Finetune/Step Time": 5.639083694666624} +{"Finetune/Learning Rate": 1.2805640641548529e-05, "Finetune/Loss": 0.8869768381118774, "Finetune/Loss (Raw)": 1.0013395547866821, "Finetune/Step": 2220, "Finetune/Step Time": 5.638785887509584} +{"Finetune/Learning Rate": 1.2799660697306636e-05, "Finetune/Loss": 0.886009931564331, "Finetune/Loss (Raw)": 0.8510496616363525, "Finetune/Step": 2221, "Finetune/Step Time": 5.655406089499593} +{"Finetune/Learning Rate": 1.2793679666571084e-05, "Finetune/Loss": 0.8853296041488647, "Finetune/Loss (Raw)": 0.9861853718757629, "Finetune/Step": 2222, "Finetune/Step Time": 5.664591545239091} +{"Finetune/Learning Rate": 1.2787697551662994e-05, "Finetune/Loss": 0.88518226146698, "Finetune/Loss (Raw)": 0.9547972679138184, "Finetune/Step": 2223, "Finetune/Step Time": 5.6681660246104} +{"Finetune/Learning Rate": 1.2781714354903913e-05, "Finetune/Loss": 0.8849607110023499, "Finetune/Loss (Raw)": 0.9229744672775269, "Finetune/Step": 2224, "Finetune/Step Time": 5.669478852301836} +{"Finetune/Learning Rate": 1.2775730078615798e-05, "Finetune/Loss": 0.8837409019470215, "Finetune/Loss (Raw)": 0.9078974723815918, "Finetune/Step": 2225, "Finetune/Step Time": 5.680816628038883} +{"Finetune/Learning Rate": 1.276974472512103e-05, "Finetune/Loss": 0.883895218372345, "Finetune/Loss (Raw)": 0.9417710900306702, "Finetune/Step": 2226, "Finetune/Step Time": 5.680734263733029} +{"Finetune/Learning Rate": 1.276375829674241e-05, "Finetune/Loss": 0.8888164162635803, "Finetune/Loss (Raw)": 0.9931569695472717, "Finetune/Step": 2227, "Finetune/Step Time": 5.65370081923902} +{"Finetune/Learning Rate": 1.275777079580315e-05, "Finetune/Loss": 0.8896442651748657, "Finetune/Loss (Raw)": 0.9099210500717163, "Finetune/Step": 2228, "Finetune/Step Time": 5.652137663215399} +{"Finetune/Learning Rate": 1.2751782224626891e-05, "Finetune/Loss": 0.8889241218566895, "Finetune/Loss (Raw)": 0.9366779923439026, "Finetune/Step": 2229, "Finetune/Step Time": 5.6510460283607244} +{"Finetune/Learning Rate": 1.274579258553767e-05, "Finetune/Loss": 0.8887119293212891, "Finetune/Loss (Raw)": 0.8774039149284363, "Finetune/Step": 2230, "Finetune/Step Time": 5.651481445878744} +{"Finetune/Learning Rate": 1.2739801880859954e-05, "Finetune/Loss": 0.8936629295349121, "Finetune/Loss (Raw)": 1.0048270225524902, "Finetune/Step": 2231, "Finetune/Step Time": 5.619719645008445} +{"Finetune/Learning Rate": 1.2733810112918618e-05, "Finetune/Loss": 0.894639790058136, "Finetune/Loss (Raw)": 1.038371205329895, "Finetune/Step": 2232, "Finetune/Step Time": 5.611277271062136} +{"Finetune/Learning Rate": 1.2727817284038953e-05, "Finetune/Loss": 0.8937041163444519, "Finetune/Loss (Raw)": 0.8158636689186096, "Finetune/Step": 2233, "Finetune/Step Time": 5.609401430934668} +{"Finetune/Learning Rate": 1.2721823396546655e-05, "Finetune/Loss": 0.8940441608428955, "Finetune/Loss (Raw)": 0.9828805327415466, "Finetune/Step": 2234, "Finetune/Step Time": 5.613107204437256} +{"Finetune/Learning Rate": 1.2715828452767833e-05, "Finetune/Loss": 0.894889235496521, "Finetune/Loss (Raw)": 1.0505861043930054, "Finetune/Step": 2235, "Finetune/Step Time": 5.624225942417979} +{"Finetune/Learning Rate": 1.2709832455029017e-05, "Finetune/Loss": 0.8956630229949951, "Finetune/Loss (Raw)": 1.003792643547058, "Finetune/Step": 2236, "Finetune/Step Time": 5.624997580423951} +{"Finetune/Learning Rate": 1.2703835405657124e-05, "Finetune/Loss": 0.8953970670700073, "Finetune/Loss (Raw)": 0.9215385913848877, "Finetune/Step": 2237, "Finetune/Step Time": 5.625697948038578} +{"Finetune/Learning Rate": 1.2697837306979499e-05, "Finetune/Loss": 0.8938510417938232, "Finetune/Loss (Raw)": 0.7801998853683472, "Finetune/Step": 2238, "Finetune/Step Time": 5.629979215562344} +{"Finetune/Learning Rate": 1.2691838161323888e-05, "Finetune/Loss": 0.8957593441009521, "Finetune/Loss (Raw)": 1.1804145574569702, "Finetune/Step": 2239, "Finetune/Step Time": 5.615789758041501} +{"Finetune/Learning Rate": 1.2685837971018438e-05, "Finetune/Loss": 0.9005522727966309, "Finetune/Loss (Raw)": 1.0723696947097778, "Finetune/Step": 2240, "Finetune/Step Time": 5.586571739986539} +{"Finetune/Learning Rate": 1.2679836738391714e-05, "Finetune/Loss": 0.9009621143341064, "Finetune/Loss (Raw)": 1.0069851875305176, "Finetune/Step": 2241, "Finetune/Step Time": 5.59507942199707} +{"Finetune/Learning Rate": 1.2673834465772662e-05, "Finetune/Loss": 0.8992840051651001, "Finetune/Loss (Raw)": 0.9130130410194397, "Finetune/Step": 2242, "Finetune/Step Time": 5.578045921400189} +{"Finetune/Learning Rate": 1.2667831155490662e-05, "Finetune/Loss": 0.8983402252197266, "Finetune/Loss (Raw)": 0.8930456638336182, "Finetune/Step": 2243, "Finetune/Step Time": 5.58057139441371} +{"Finetune/Learning Rate": 1.2661826809875474e-05, "Finetune/Loss": 0.9042172431945801, "Finetune/Loss (Raw)": 1.1740121841430664, "Finetune/Step": 2244, "Finetune/Step Time": 5.547158982604742} +{"Finetune/Learning Rate": 1.265582143125727e-05, "Finetune/Loss": 0.90580153465271, "Finetune/Loss (Raw)": 0.9690701365470886, "Finetune/Step": 2245, "Finetune/Step Time": 5.527842918410897} +{"Finetune/Learning Rate": 1.264981502196662e-05, "Finetune/Loss": 0.907583475112915, "Finetune/Loss (Raw)": 1.107688546180725, "Finetune/Step": 2246, "Finetune/Step Time": 5.5308459885418415} +{"Finetune/Learning Rate": 1.2643807584334491e-05, "Finetune/Loss": 0.9074576497077942, "Finetune/Loss (Raw)": 0.8664405941963196, "Finetune/Step": 2247, "Finetune/Step Time": 5.529503893107176} +{"Finetune/Learning Rate": 1.2637799120692253e-05, "Finetune/Loss": 0.9109351634979248, "Finetune/Loss (Raw)": 0.7820903658866882, "Finetune/Step": 2248, "Finetune/Step Time": 5.4943264201283455} +{"Finetune/Learning Rate": 1.2631789633371678e-05, "Finetune/Loss": 0.9104214906692505, "Finetune/Loss (Raw)": 0.8821331858634949, "Finetune/Step": 2249, "Finetune/Step Time": 5.499849800020456} +{"Finetune/Learning Rate": 1.2625779124704925e-05, "Finetune/Loss": 0.9099863767623901, "Finetune/Loss (Raw)": 0.9570642113685608, "Finetune/Step": 2250, "Finetune/Step Time": 5.515410451218486} +{"Finetune/Learning Rate": 1.2619767597024561e-05, "Finetune/Loss": 0.9095298051834106, "Finetune/Loss (Raw)": 0.9599687457084656, "Finetune/Step": 2251, "Finetune/Step Time": 5.516817342489958} +{"Finetune/Learning Rate": 1.2613755052663536e-05, "Finetune/Loss": 0.9042320251464844, "Finetune/Loss (Raw)": 0.3211514353752136, "Finetune/Step": 2252, "Finetune/Step Time": 5.5434695184230804} +{"Finetune/Learning Rate": 1.2607741493955208e-05, "Finetune/Loss": 0.904433012008667, "Finetune/Loss (Raw)": 1.040027379989624, "Finetune/Step": 2253, "Finetune/Step Time": 5.524378774687648} +{"Finetune/Learning Rate": 1.2601726923233315e-05, "Finetune/Loss": 0.904715895652771, "Finetune/Loss (Raw)": 1.0559481382369995, "Finetune/Step": 2254, "Finetune/Step Time": 5.530623538419604} +{"Finetune/Learning Rate": 1.2595711342831998e-05, "Finetune/Loss": 0.9049137830734253, "Finetune/Loss (Raw)": 0.9716277122497559, "Finetune/Step": 2255, "Finetune/Step Time": 5.52839046344161} +{"Finetune/Learning Rate": 1.2589694755085789e-05, "Finetune/Loss": 0.9091027975082397, "Finetune/Loss (Raw)": 0.8904165029525757, "Finetune/Step": 2256, "Finetune/Step Time": 5.499391715973616} +{"Finetune/Learning Rate": 1.25836771623296e-05, "Finetune/Loss": 0.9146600365638733, "Finetune/Loss (Raw)": 0.9908146262168884, "Finetune/Step": 2257, "Finetune/Step Time": 5.470912586897612} +{"Finetune/Learning Rate": 1.257765856689875e-05, "Finetune/Loss": 0.9129201173782349, "Finetune/Loss (Raw)": 0.7661709785461426, "Finetune/Step": 2258, "Finetune/Step Time": 5.471405977383256} +{"Finetune/Learning Rate": 1.2571638971128933e-05, "Finetune/Loss": 0.9118098020553589, "Finetune/Loss (Raw)": 0.8529407382011414, "Finetune/Step": 2259, "Finetune/Step Time": 5.469059878960252} +{"Finetune/Learning Rate": 1.2565618377356237e-05, "Finetune/Loss": 0.9116624593734741, "Finetune/Loss (Raw)": 0.8718878626823425, "Finetune/Step": 2260, "Finetune/Step Time": 5.474178932607174} +{"Finetune/Learning Rate": 1.2559596787917144e-05, "Finetune/Loss": 0.9101940393447876, "Finetune/Loss (Raw)": 0.9748890995979309, "Finetune/Step": 2261, "Finetune/Step Time": 5.476152500137687} +{"Finetune/Learning Rate": 1.2553574205148502e-05, "Finetune/Loss": 0.9113764762878418, "Finetune/Loss (Raw)": 1.0544421672821045, "Finetune/Step": 2262, "Finetune/Step Time": 5.469882087782025} +{"Finetune/Learning Rate": 1.2547550631387571e-05, "Finetune/Loss": 0.9111422896385193, "Finetune/Loss (Raw)": 0.8539983034133911, "Finetune/Step": 2263, "Finetune/Step Time": 5.474343236535788} +{"Finetune/Learning Rate": 1.2541526068971975e-05, "Finetune/Loss": 0.9109975099563599, "Finetune/Loss (Raw)": 0.9710685014724731, "Finetune/Step": 2264, "Finetune/Step Time": 5.477496551349759} +{"Finetune/Learning Rate": 1.2535500520239727e-05, "Finetune/Loss": 0.9148572087287903, "Finetune/Loss (Raw)": 0.916817307472229, "Finetune/Step": 2265, "Finetune/Step Time": 5.448484517633915} +{"Finetune/Learning Rate": 1.2529473987529235e-05, "Finetune/Loss": 0.9136551022529602, "Finetune/Loss (Raw)": 0.7730129957199097, "Finetune/Step": 2266, "Finetune/Step Time": 5.450197348371148} +{"Finetune/Learning Rate": 1.2523446473179267e-05, "Finetune/Loss": 0.9136868715286255, "Finetune/Loss (Raw)": 0.8928042054176331, "Finetune/Step": 2267, "Finetune/Step Time": 5.4299811236560345} +{"Finetune/Learning Rate": 1.2517417979528991e-05, "Finetune/Loss": 0.9126121997833252, "Finetune/Loss (Raw)": 0.846537709236145, "Finetune/Step": 2268, "Finetune/Step Time": 5.427373895421624} +{"Finetune/Learning Rate": 1.2511388508917946e-05, "Finetune/Loss": 0.9115399122238159, "Finetune/Loss (Raw)": 0.9509058594703674, "Finetune/Step": 2269, "Finetune/Step Time": 5.4316211566329} +{"Finetune/Learning Rate": 1.250535806368605e-05, "Finetune/Loss": 0.9111218452453613, "Finetune/Loss (Raw)": 0.8123971223831177, "Finetune/Step": 2270, "Finetune/Step Time": 5.435260526835918} +{"Finetune/Learning Rate": 1.2499326646173606e-05, "Finetune/Loss": 0.9113744497299194, "Finetune/Loss (Raw)": 0.9770132899284363, "Finetune/Step": 2271, "Finetune/Step Time": 5.420187823474407} +{"Finetune/Learning Rate": 1.2493294258721282e-05, "Finetune/Loss": 0.9118751883506775, "Finetune/Loss (Raw)": 0.9894030094146729, "Finetune/Step": 2272, "Finetune/Step Time": 5.414393339306116} +{"Finetune/Learning Rate": 1.2487260903670136e-05, "Finetune/Loss": 0.9136148691177368, "Finetune/Loss (Raw)": 0.9383329153060913, "Finetune/Step": 2273, "Finetune/Step Time": 5.394724279642105} +{"Finetune/Learning Rate": 1.2481226583361592e-05, "Finetune/Loss": 0.912960410118103, "Finetune/Loss (Raw)": 0.8498062491416931, "Finetune/Step": 2274, "Finetune/Step Time": 5.4058387614786625} +{"Finetune/Learning Rate": 1.2475191300137452e-05, "Finetune/Loss": 0.9134963154792786, "Finetune/Loss (Raw)": 1.0414496660232544, "Finetune/Step": 2275, "Finetune/Step Time": 5.397323478013277} +{"Finetune/Learning Rate": 1.2469155056339894e-05, "Finetune/Loss": 0.9124031066894531, "Finetune/Loss (Raw)": 0.9424000382423401, "Finetune/Step": 2276, "Finetune/Step Time": 5.394358363002539} +{"Finetune/Learning Rate": 1.2463117854311463e-05, "Finetune/Loss": 0.9105144143104553, "Finetune/Loss (Raw)": 0.7602771520614624, "Finetune/Step": 2277, "Finetune/Step Time": 5.398989068344235} +{"Finetune/Learning Rate": 1.2457079696395082e-05, "Finetune/Loss": 0.9109897613525391, "Finetune/Loss (Raw)": 0.9885337352752686, "Finetune/Step": 2278, "Finetune/Step Time": 5.391968794167042} +{"Finetune/Learning Rate": 1.2451040584934042e-05, "Finetune/Loss": 0.9101226329803467, "Finetune/Loss (Raw)": 1.0506926774978638, "Finetune/Step": 2279, "Finetune/Step Time": 5.380902856588364} +{"Finetune/Learning Rate": 1.2445000522272005e-05, "Finetune/Loss": 0.9106294512748718, "Finetune/Loss (Raw)": 0.8984919190406799, "Finetune/Step": 2280, "Finetune/Step Time": 5.370498802512884} +{"Finetune/Learning Rate": 1.2438959510752997e-05, "Finetune/Loss": 0.9110803604125977, "Finetune/Loss (Raw)": 0.9147911667823792, "Finetune/Step": 2281, "Finetune/Step Time": 5.371882816776633} +{"Finetune/Learning Rate": 1.243291755272142e-05, "Finetune/Loss": 0.9112809896469116, "Finetune/Loss (Raw)": 0.9183251857757568, "Finetune/Step": 2282, "Finetune/Step Time": 5.365724550560117} +{"Finetune/Learning Rate": 1.2426874650522039e-05, "Finetune/Loss": 0.912310004234314, "Finetune/Loss (Raw)": 1.052154779434204, "Finetune/Step": 2283, "Finetune/Step Time": 5.364379085600376} +{"Finetune/Learning Rate": 1.2420830806499988e-05, "Finetune/Loss": 0.9133695363998413, "Finetune/Loss (Raw)": 0.9045916199684143, "Finetune/Step": 2284, "Finetune/Step Time": 5.373365361243486} +{"Finetune/Learning Rate": 1.2414786023000765e-05, "Finetune/Loss": 0.9139161705970764, "Finetune/Loss (Raw)": 0.9775598645210266, "Finetune/Step": 2285, "Finetune/Step Time": 5.373407395556569} +{"Finetune/Learning Rate": 1.240874030237023e-05, "Finetune/Loss": 0.9138954877853394, "Finetune/Loss (Raw)": 0.9718790650367737, "Finetune/Step": 2286, "Finetune/Step Time": 5.389371311292052} +{"Finetune/Learning Rate": 1.2402693646954607e-05, "Finetune/Loss": 0.9122610092163086, "Finetune/Loss (Raw)": 0.7924330234527588, "Finetune/Step": 2287, "Finetune/Step Time": 5.373956413939595} +{"Finetune/Learning Rate": 1.2396646059100492e-05, "Finetune/Loss": 0.9122723340988159, "Finetune/Loss (Raw)": 0.9640340209007263, "Finetune/Step": 2288, "Finetune/Step Time": 5.383668035268784} +{"Finetune/Learning Rate": 1.2390597541154832e-05, "Finetune/Loss": 0.9114260077476501, "Finetune/Loss (Raw)": 0.7593194246292114, "Finetune/Step": 2289, "Finetune/Step Time": 5.397337198257446} +{"Finetune/Learning Rate": 1.238454809546494e-05, "Finetune/Loss": 0.911989152431488, "Finetune/Loss (Raw)": 1.014704942703247, "Finetune/Step": 2290, "Finetune/Step Time": 5.389884911477566} +{"Finetune/Learning Rate": 1.2378497724378484e-05, "Finetune/Loss": 0.9116154909133911, "Finetune/Loss (Raw)": 0.846027672290802, "Finetune/Step": 2291, "Finetune/Step Time": 5.40241558291018} +{"Finetune/Learning Rate": 1.2372446430243497e-05, "Finetune/Loss": 0.9100614786148071, "Finetune/Loss (Raw)": 0.8600254058837891, "Finetune/Step": 2292, "Finetune/Step Time": 5.410082750022411} +{"Finetune/Learning Rate": 1.2366394215408369e-05, "Finetune/Loss": 0.908367931842804, "Finetune/Loss (Raw)": 0.7525917291641235, "Finetune/Step": 2293, "Finetune/Step Time": 5.3945063557475805} +{"Finetune/Learning Rate": 1.2360341082221847e-05, "Finetune/Loss": 0.9081484079360962, "Finetune/Loss (Raw)": 0.9473342895507812, "Finetune/Step": 2294, "Finetune/Step Time": 5.3927269876003265} +{"Finetune/Learning Rate": 1.2354287033033032e-05, "Finetune/Loss": 0.9070867896080017, "Finetune/Loss (Raw)": 0.9540895819664001, "Finetune/Step": 2295, "Finetune/Step Time": 5.4108060877770185} +{"Finetune/Learning Rate": 1.2348232070191383e-05, "Finetune/Loss": 0.9058164358139038, "Finetune/Loss (Raw)": 0.7680491209030151, "Finetune/Step": 2296, "Finetune/Step Time": 5.411381598562002} +{"Finetune/Learning Rate": 1.2342176196046715e-05, "Finetune/Loss": 0.9056538343429565, "Finetune/Loss (Raw)": 0.9523683190345764, "Finetune/Step": 2297, "Finetune/Step Time": 5.413495654240251} +{"Finetune/Learning Rate": 1.2336119412949193e-05, "Finetune/Loss": 0.905958890914917, "Finetune/Loss (Raw)": 0.8848346471786499, "Finetune/Step": 2298, "Finetune/Step Time": 5.414648286998272} +{"Finetune/Learning Rate": 1.2330061723249328e-05, "Finetune/Loss": 0.906394362449646, "Finetune/Loss (Raw)": 0.9874213933944702, "Finetune/Step": 2299, "Finetune/Step Time": 5.412636052817106} +{"Finetune/Learning Rate": 1.2324003129298006e-05, "Finetune/Loss": 0.9069659113883972, "Finetune/Loss (Raw)": 0.9831848740577698, "Finetune/Step": 2300, "Finetune/Step Time": 5.418177055194974} +{"Finetune/Learning Rate": 1.2317943633446441e-05, "Finetune/Loss": 0.9061760902404785, "Finetune/Loss (Raw)": 0.8419027924537659, "Finetune/Step": 2301, "Finetune/Step Time": 5.404912123456597} +{"Finetune/Learning Rate": 1.2311883238046204e-05, "Finetune/Loss": 0.9062790870666504, "Finetune/Loss (Raw)": 0.8089596033096313, "Finetune/Step": 2302, "Finetune/Step Time": 5.397722667083144} +{"Finetune/Learning Rate": 1.2305821945449222e-05, "Finetune/Loss": 0.9061310291290283, "Finetune/Loss (Raw)": 0.9278233051300049, "Finetune/Step": 2303, "Finetune/Step Time": 5.3798613250255585} +{"Finetune/Learning Rate": 1.2299759758007755e-05, "Finetune/Loss": 0.9040286540985107, "Finetune/Loss (Raw)": 0.7279407978057861, "Finetune/Step": 2304, "Finetune/Step Time": 5.388324184343219} +{"Finetune/Learning Rate": 1.2293696678074428e-05, "Finetune/Loss": 0.9018988013267517, "Finetune/Loss (Raw)": 0.812611997127533, "Finetune/Step": 2305, "Finetune/Step Time": 5.38394914008677} +{"Finetune/Learning Rate": 1.22876327080022e-05, "Finetune/Loss": 0.9038723707199097, "Finetune/Loss (Raw)": 1.062756061553955, "Finetune/Step": 2306, "Finetune/Step Time": 5.386255471035838} +{"Finetune/Learning Rate": 1.2281567850144383e-05, "Finetune/Loss": 0.9038764238357544, "Finetune/Loss (Raw)": 0.9868542551994324, "Finetune/Step": 2307, "Finetune/Step Time": 5.370771259069443} +{"Finetune/Learning Rate": 1.2275502106854624e-05, "Finetune/Loss": 0.9011017084121704, "Finetune/Loss (Raw)": 0.6258559823036194, "Finetune/Step": 2308, "Finetune/Step Time": 5.373246509581804} +{"Finetune/Learning Rate": 1.2269435480486923e-05, "Finetune/Loss": 0.9007840752601624, "Finetune/Loss (Raw)": 0.9492220282554626, "Finetune/Step": 2309, "Finetune/Step Time": 5.373383414000273} +{"Finetune/Learning Rate": 1.226336797339562e-05, "Finetune/Loss": 0.9004862308502197, "Finetune/Loss (Raw)": 0.9663776159286499, "Finetune/Step": 2310, "Finetune/Step Time": 5.368188314139843} +{"Finetune/Learning Rate": 1.2257299587935394e-05, "Finetune/Loss": 0.9023426175117493, "Finetune/Loss (Raw)": 1.0357346534729004, "Finetune/Step": 2311, "Finetune/Step Time": 5.348130920901895} +{"Finetune/Learning Rate": 1.2251230326461268e-05, "Finetune/Loss": 0.9027111530303955, "Finetune/Loss (Raw)": 0.8871009945869446, "Finetune/Step": 2312, "Finetune/Step Time": 5.352731531485915} +{"Finetune/Learning Rate": 1.2245160191328605e-05, "Finetune/Loss": 0.9028249382972717, "Finetune/Loss (Raw)": 0.955808162689209, "Finetune/Step": 2313, "Finetune/Step Time": 5.357946524396539} +{"Finetune/Learning Rate": 1.2239089184893102e-05, "Finetune/Loss": 0.9024494290351868, "Finetune/Loss (Raw)": 0.8739088773727417, "Finetune/Step": 2314, "Finetune/Step Time": 5.366228401660919} +{"Finetune/Learning Rate": 1.2233017309510804e-05, "Finetune/Loss": 0.9032710790634155, "Finetune/Loss (Raw)": 0.8733042478561401, "Finetune/Step": 2315, "Finetune/Step Time": 5.332564925774932} +{"Finetune/Learning Rate": 1.2226944567538084e-05, "Finetune/Loss": 0.9036484956741333, "Finetune/Loss (Raw)": 0.9902580380439758, "Finetune/Step": 2316, "Finetune/Step Time": 5.297695990651846} +{"Finetune/Learning Rate": 1.2220870961331654e-05, "Finetune/Loss": 0.9038047790527344, "Finetune/Loss (Raw)": 0.8770104050636292, "Finetune/Step": 2317, "Finetune/Step Time": 5.294017683714628} +{"Finetune/Learning Rate": 1.2214796493248564e-05, "Finetune/Loss": 0.904039204120636, "Finetune/Loss (Raw)": 0.8945889472961426, "Finetune/Step": 2318, "Finetune/Step Time": 5.281996538862586} +{"Finetune/Learning Rate": 1.2208721165646193e-05, "Finetune/Loss": 0.9049357771873474, "Finetune/Loss (Raw)": 1.0947095155715942, "Finetune/Step": 2319, "Finetune/Step Time": 5.273144135251641} +{"Finetune/Learning Rate": 1.2202644980882258e-05, "Finetune/Loss": 0.9060158133506775, "Finetune/Loss (Raw)": 0.9542367458343506, "Finetune/Step": 2320, "Finetune/Step Time": 5.2773151863366365} +{"Finetune/Learning Rate": 1.2196567941314815e-05, "Finetune/Loss": 0.905502438545227, "Finetune/Loss (Raw)": 0.944787323474884, "Finetune/Step": 2321, "Finetune/Step Time": 5.2847288120538} +{"Finetune/Learning Rate": 1.2190490049302233e-05, "Finetune/Loss": 0.905695915222168, "Finetune/Loss (Raw)": 1.0116069316864014, "Finetune/Step": 2322, "Finetune/Step Time": 5.284768441691995} +{"Finetune/Learning Rate": 1.218441130720323e-05, "Finetune/Loss": 0.9042215347290039, "Finetune/Loss (Raw)": 0.8142931461334229, "Finetune/Step": 2323, "Finetune/Step Time": 5.298729931935668} +{"Finetune/Learning Rate": 1.2178331717376848e-05, "Finetune/Loss": 0.9042421579360962, "Finetune/Loss (Raw)": 0.8705035448074341, "Finetune/Step": 2324, "Finetune/Step Time": 5.288502339273691} +{"Finetune/Learning Rate": 1.217225128218245e-05, "Finetune/Loss": 0.9044692516326904, "Finetune/Loss (Raw)": 1.0360581874847412, "Finetune/Step": 2325, "Finetune/Step Time": 5.295461194589734} +{"Finetune/Learning Rate": 1.2166170003979742e-05, "Finetune/Loss": 0.9009833335876465, "Finetune/Loss (Raw)": 0.4361185133457184, "Finetune/Step": 2326, "Finetune/Step Time": 5.320111025124788} +{"Finetune/Learning Rate": 1.2160087885128745e-05, "Finetune/Loss": 0.9013842344284058, "Finetune/Loss (Raw)": 0.9170466661453247, "Finetune/Step": 2327, "Finetune/Step Time": 5.335737569257617} +{"Finetune/Learning Rate": 1.2154004927989815e-05, "Finetune/Loss": 0.9011372327804565, "Finetune/Loss (Raw)": 0.8663379549980164, "Finetune/Step": 2328, "Finetune/Step Time": 5.324158325791359} +{"Finetune/Learning Rate": 1.2147921134923624e-05, "Finetune/Loss": 0.9061163067817688, "Finetune/Loss (Raw)": 0.9908246397972107, "Finetune/Step": 2329, "Finetune/Step Time": 5.295560322701931} +{"Finetune/Learning Rate": 1.2141836508291175e-05, "Finetune/Loss": 0.905268132686615, "Finetune/Loss (Raw)": 0.821424126625061, "Finetune/Step": 2330, "Finetune/Step Time": 5.308814410120249} +{"Finetune/Learning Rate": 1.2135751050453793e-05, "Finetune/Loss": 0.9057611227035522, "Finetune/Loss (Raw)": 0.9473836421966553, "Finetune/Step": 2331, "Finetune/Step Time": 5.311343850567937} +{"Finetune/Learning Rate": 1.2129664763773125e-05, "Finetune/Loss": 0.9079338908195496, "Finetune/Loss (Raw)": 1.1370422840118408, "Finetune/Step": 2332, "Finetune/Step Time": 5.312667666003108} +{"Finetune/Learning Rate": 1.212357765061114e-05, "Finetune/Loss": 0.9124459028244019, "Finetune/Loss (Raw)": 1.007404088973999, "Finetune/Step": 2333, "Finetune/Step Time": 5.300342021510005} +{"Finetune/Learning Rate": 1.211748971333013e-05, "Finetune/Loss": 0.914486289024353, "Finetune/Loss (Raw)": 0.9562349319458008, "Finetune/Step": 2334, "Finetune/Step Time": 5.257699221372604} +{"Finetune/Learning Rate": 1.21114009542927e-05, "Finetune/Loss": 0.9137825965881348, "Finetune/Loss (Raw)": 0.8828959465026855, "Finetune/Step": 2335, "Finetune/Step Time": 5.254503820091486} +{"Finetune/Learning Rate": 1.2105311375861786e-05, "Finetune/Loss": 0.913912296295166, "Finetune/Loss (Raw)": 0.8953771591186523, "Finetune/Step": 2336, "Finetune/Step Time": 5.264251012355089} +{"Finetune/Learning Rate": 1.2099220980400625e-05, "Finetune/Loss": 0.9143803715705872, "Finetune/Loss (Raw)": 0.9701993465423584, "Finetune/Step": 2337, "Finetune/Step Time": 5.263656532391906} +{"Finetune/Learning Rate": 1.2093129770272787e-05, "Finetune/Loss": 0.9190087914466858, "Finetune/Loss (Raw)": 0.9859133362770081, "Finetune/Step": 2338, "Finetune/Step Time": 5.233293954282999} +{"Finetune/Learning Rate": 1.208703774784215e-05, "Finetune/Loss": 0.9176405668258667, "Finetune/Loss (Raw)": 0.8479219675064087, "Finetune/Step": 2339, "Finetune/Step Time": 5.22862141020596} +{"Finetune/Learning Rate": 1.2080944915472908e-05, "Finetune/Loss": 0.9190056920051575, "Finetune/Loss (Raw)": 0.9050039649009705, "Finetune/Step": 2340, "Finetune/Step Time": 5.219353334978223} +{"Finetune/Learning Rate": 1.2074851275529575e-05, "Finetune/Loss": 0.9207661151885986, "Finetune/Loss (Raw)": 1.0556684732437134, "Finetune/Step": 2341, "Finetune/Step Time": 5.2462525852024555} +{"Finetune/Learning Rate": 1.2068756830376966e-05, "Finetune/Loss": 0.9179061651229858, "Finetune/Loss (Raw)": 0.356668621301651, "Finetune/Step": 2342, "Finetune/Step Time": 5.268160006031394} +{"Finetune/Learning Rate": 1.2062661582380222e-05, "Finetune/Loss": 0.9180835485458374, "Finetune/Loss (Raw)": 1.0061476230621338, "Finetune/Step": 2343, "Finetune/Step Time": 5.280153963714838} +{"Finetune/Learning Rate": 1.2056565533904787e-05, "Finetune/Loss": 0.9177755117416382, "Finetune/Loss (Raw)": 0.9198306798934937, "Finetune/Step": 2344, "Finetune/Step Time": 5.303932515904307} +{"Finetune/Learning Rate": 1.2050468687316418e-05, "Finetune/Loss": 0.9188224077224731, "Finetune/Loss (Raw)": 1.1632366180419922, "Finetune/Step": 2345, "Finetune/Step Time": 5.292315240949392} +{"Finetune/Learning Rate": 1.2044371044981188e-05, "Finetune/Loss": 0.9200934767723083, "Finetune/Loss (Raw)": 0.9170660972595215, "Finetune/Step": 2346, "Finetune/Step Time": 5.282666394487023} +{"Finetune/Learning Rate": 1.2038272609265464e-05, "Finetune/Loss": 0.9203916788101196, "Finetune/Loss (Raw)": 0.9046679139137268, "Finetune/Step": 2347, "Finetune/Step Time": 5.245880207046866} +{"Finetune/Learning Rate": 1.2032173382535936e-05, "Finetune/Loss": 0.9205397963523865, "Finetune/Loss (Raw)": 1.020297646522522, "Finetune/Step": 2348, "Finetune/Step Time": 5.25902021676302} +{"Finetune/Learning Rate": 1.2026073367159597e-05, "Finetune/Loss": 0.9211801290512085, "Finetune/Loss (Raw)": 0.9330141544342041, "Finetune/Step": 2349, "Finetune/Step Time": 5.250196872279048} +{"Finetune/Learning Rate": 1.2019972565503733e-05, "Finetune/Loss": 0.9206663370132446, "Finetune/Loss (Raw)": 0.9204146265983582, "Finetune/Step": 2350, "Finetune/Step Time": 5.248428484424949} +{"Finetune/Learning Rate": 1.2013870979935957e-05, "Finetune/Loss": 0.9213516712188721, "Finetune/Loss (Raw)": 1.0425128936767578, "Finetune/Step": 2351, "Finetune/Step Time": 5.245917761698365} +{"Finetune/Learning Rate": 1.200776861282417e-05, "Finetune/Loss": 0.9226400256156921, "Finetune/Loss (Raw)": 1.087892770767212, "Finetune/Step": 2352, "Finetune/Step Time": 5.25949951633811} +{"Finetune/Learning Rate": 1.2001665466536582e-05, "Finetune/Loss": 0.9227097034454346, "Finetune/Loss (Raw)": 0.916816234588623, "Finetune/Step": 2353, "Finetune/Step Time": 5.253203995525837} +{"Finetune/Learning Rate": 1.199556154344171e-05, "Finetune/Loss": 0.9226809144020081, "Finetune/Loss (Raw)": 0.9380869269371033, "Finetune/Step": 2354, "Finetune/Step Time": 5.245174672454596} +{"Finetune/Learning Rate": 1.1989456845908356e-05, "Finetune/Loss": 0.921399712562561, "Finetune/Loss (Raw)": 0.8291593790054321, "Finetune/Step": 2355, "Finetune/Step Time": 5.242296874523163} +{"Finetune/Learning Rate": 1.1983351376305648e-05, "Finetune/Loss": 0.9207046031951904, "Finetune/Loss (Raw)": 0.8209457993507385, "Finetune/Step": 2356, "Finetune/Step Time": 5.2427349500358105} +{"Finetune/Learning Rate": 1.197724513700299e-05, "Finetune/Loss": 0.9205354452133179, "Finetune/Loss (Raw)": 0.9150299429893494, "Finetune/Step": 2357, "Finetune/Step Time": 5.24720605276525} +{"Finetune/Learning Rate": 1.1971138130370097e-05, "Finetune/Loss": 0.9217935800552368, "Finetune/Loss (Raw)": 1.0384474992752075, "Finetune/Step": 2358, "Finetune/Step Time": 5.244210381060839} +{"Finetune/Learning Rate": 1.1965030358776975e-05, "Finetune/Loss": 0.9162246584892273, "Finetune/Loss (Raw)": 0.29200291633605957, "Finetune/Step": 2359, "Finetune/Step Time": 5.274296494200826} +{"Finetune/Learning Rate": 1.1958921824593936e-05, "Finetune/Loss": 0.9153336882591248, "Finetune/Loss (Raw)": 0.9243246912956238, "Finetune/Step": 2360, "Finetune/Step Time": 5.272965913638473} +{"Finetune/Learning Rate": 1.1952812530191586e-05, "Finetune/Loss": 0.9163788557052612, "Finetune/Loss (Raw)": 0.9496410489082336, "Finetune/Step": 2361, "Finetune/Step Time": 5.266767278313637} +{"Finetune/Learning Rate": 1.1946702477940815e-05, "Finetune/Loss": 0.9161105751991272, "Finetune/Loss (Raw)": 0.9485482573509216, "Finetune/Step": 2362, "Finetune/Step Time": 5.259651707485318} +{"Finetune/Learning Rate": 1.194059167021282e-05, "Finetune/Loss": 0.9137451648712158, "Finetune/Loss (Raw)": 0.7478088736534119, "Finetune/Step": 2363, "Finetune/Step Time": 5.265639944002032} +{"Finetune/Learning Rate": 1.1934480109379083e-05, "Finetune/Loss": 0.9126142263412476, "Finetune/Loss (Raw)": 0.8590327501296997, "Finetune/Step": 2364, "Finetune/Step Time": 5.2629221472889185} +{"Finetune/Learning Rate": 1.1928367797811384e-05, "Finetune/Loss": 0.9135700464248657, "Finetune/Loss (Raw)": 1.0438904762268066, "Finetune/Step": 2365, "Finetune/Step Time": 5.2584772650152445} +{"Finetune/Learning Rate": 1.1922254737881792e-05, "Finetune/Loss": 0.9134910106658936, "Finetune/Loss (Raw)": 0.770075798034668, "Finetune/Step": 2366, "Finetune/Step Time": 5.254837164655328} +{"Finetune/Learning Rate": 1.1916140931962661e-05, "Finetune/Loss": 0.911910891532898, "Finetune/Loss (Raw)": 0.9781571626663208, "Finetune/Step": 2367, "Finetune/Step Time": 5.260133791714907} +{"Finetune/Learning Rate": 1.191002638242665e-05, "Finetune/Loss": 0.910759687423706, "Finetune/Loss (Raw)": 0.9250202775001526, "Finetune/Step": 2368, "Finetune/Step Time": 5.255856443196535} +{"Finetune/Learning Rate": 1.1903911091646684e-05, "Finetune/Loss": 0.9108313322067261, "Finetune/Loss (Raw)": 1.016155481338501, "Finetune/Step": 2369, "Finetune/Step Time": 5.252703094854951} +{"Finetune/Learning Rate": 1.1897795061995998e-05, "Finetune/Loss": 0.9108228087425232, "Finetune/Loss (Raw)": 0.9119250774383545, "Finetune/Step": 2370, "Finetune/Step Time": 5.254671007394791} +{"Finetune/Learning Rate": 1.18916782958481e-05, "Finetune/Loss": 0.9119154214859009, "Finetune/Loss (Raw)": 1.0328974723815918, "Finetune/Step": 2371, "Finetune/Step Time": 5.253283683210611} +{"Finetune/Learning Rate": 1.1885560795576782e-05, "Finetune/Loss": 0.9098426699638367, "Finetune/Loss (Raw)": 0.9087010025978088, "Finetune/Step": 2372, "Finetune/Step Time": 5.255979992449284} +{"Finetune/Learning Rate": 1.1879442563556141e-05, "Finetune/Loss": 0.9102849364280701, "Finetune/Loss (Raw)": 1.0256781578063965, "Finetune/Step": 2373, "Finetune/Step Time": 5.278104705736041} +{"Finetune/Learning Rate": 1.1873323602160526e-05, "Finetune/Loss": 0.908082127571106, "Finetune/Loss (Raw)": 0.8257269859313965, "Finetune/Step": 2374, "Finetune/Step Time": 5.27935079485178} +{"Finetune/Learning Rate": 1.1867203913764599e-05, "Finetune/Loss": 0.9073704481124878, "Finetune/Loss (Raw)": 0.7753462195396423, "Finetune/Step": 2375, "Finetune/Step Time": 5.282594358548522} +{"Finetune/Learning Rate": 1.1861083500743284e-05, "Finetune/Loss": 0.9087437987327576, "Finetune/Loss (Raw)": 0.9578769207000732, "Finetune/Step": 2376, "Finetune/Step Time": 5.2875904478132725} +{"Finetune/Learning Rate": 1.1854962365471798e-05, "Finetune/Loss": 0.9102581739425659, "Finetune/Loss (Raw)": 1.075969934463501, "Finetune/Step": 2377, "Finetune/Step Time": 5.28231137059629} +{"Finetune/Learning Rate": 1.1848840510325635e-05, "Finetune/Loss": 0.9106968641281128, "Finetune/Loss (Raw)": 1.0132237672805786, "Finetune/Step": 2378, "Finetune/Step Time": 5.2607468999922276} +{"Finetune/Learning Rate": 1.184271793768056e-05, "Finetune/Loss": 0.9105781316757202, "Finetune/Loss (Raw)": 0.9447762966156006, "Finetune/Step": 2379, "Finetune/Step Time": 5.277829356491566} +{"Finetune/Learning Rate": 1.1836594649912636e-05, "Finetune/Loss": 0.9143403768539429, "Finetune/Loss (Raw)": 0.8027123212814331, "Finetune/Step": 2380, "Finetune/Step Time": 5.25162192247808} +{"Finetune/Learning Rate": 1.1830470649398183e-05, "Finetune/Loss": 0.9134050607681274, "Finetune/Loss (Raw)": 0.9203067421913147, "Finetune/Step": 2381, "Finetune/Step Time": 5.254620473831892} +{"Finetune/Learning Rate": 1.1824345938513806e-05, "Finetune/Loss": 0.9114041328430176, "Finetune/Loss (Raw)": 0.799827516078949, "Finetune/Step": 2382, "Finetune/Step Time": 5.266627456992865} +{"Finetune/Learning Rate": 1.181822051963639e-05, "Finetune/Loss": 0.9091832637786865, "Finetune/Loss (Raw)": 0.6873624920845032, "Finetune/Step": 2383, "Finetune/Step Time": 5.279926899820566} +{"Finetune/Learning Rate": 1.1812094395143086e-05, "Finetune/Loss": 0.9108960032463074, "Finetune/Loss (Raw)": 1.1096469163894653, "Finetune/Step": 2384, "Finetune/Step Time": 5.275635873898864} +{"Finetune/Learning Rate": 1.1805967567411325e-05, "Finetune/Loss": 0.910297155380249, "Finetune/Loss (Raw)": 0.9141584634780884, "Finetune/Step": 2385, "Finetune/Step Time": 5.2808660212904215} +{"Finetune/Learning Rate": 1.1799840038818813e-05, "Finetune/Loss": 0.9124762415885925, "Finetune/Loss (Raw)": 1.045098900794983, "Finetune/Step": 2386, "Finetune/Step Time": 5.2873309925198555} +{"Finetune/Learning Rate": 1.1793711811743515e-05, "Finetune/Loss": 0.9127136468887329, "Finetune/Loss (Raw)": 0.8833255171775818, "Finetune/Step": 2387, "Finetune/Step Time": 5.301241332665086} +{"Finetune/Learning Rate": 1.1787582888563689e-05, "Finetune/Loss": 0.91361403465271, "Finetune/Loss (Raw)": 0.9871428608894348, "Finetune/Step": 2388, "Finetune/Step Time": 5.311499822884798} +{"Finetune/Learning Rate": 1.1781453271657839e-05, "Finetune/Loss": 0.9141786694526672, "Finetune/Loss (Raw)": 1.0471558570861816, "Finetune/Step": 2389, "Finetune/Step Time": 5.310189006850123} +{"Finetune/Learning Rate": 1.1775322963404757e-05, "Finetune/Loss": 0.9140899777412415, "Finetune/Loss (Raw)": 1.0430870056152344, "Finetune/Step": 2390, "Finetune/Step Time": 5.314331090077758} +{"Finetune/Learning Rate": 1.176919196618349e-05, "Finetune/Loss": 0.9152112007141113, "Finetune/Loss (Raw)": 0.9975161552429199, "Finetune/Step": 2391, "Finetune/Step Time": 5.313384236767888} +{"Finetune/Learning Rate": 1.1763060282373364e-05, "Finetune/Loss": 0.9150153398513794, "Finetune/Loss (Raw)": 0.9459972381591797, "Finetune/Step": 2392, "Finetune/Step Time": 5.313554473221302} +{"Finetune/Learning Rate": 1.1756927914353963e-05, "Finetune/Loss": 0.9159907698631287, "Finetune/Loss (Raw)": 1.0416746139526367, "Finetune/Step": 2393, "Finetune/Step Time": 5.326658315956593} +{"Finetune/Learning Rate": 1.175079486450514e-05, "Finetune/Loss": 0.9130516648292542, "Finetune/Loss (Raw)": 0.39680683612823486, "Finetune/Step": 2394, "Finetune/Step Time": 5.360374728217721} +{"Finetune/Learning Rate": 1.1744661135207009e-05, "Finetune/Loss": 0.9135541915893555, "Finetune/Loss (Raw)": 0.9571247696876526, "Finetune/Step": 2395, "Finetune/Step Time": 5.364980874583125} +{"Finetune/Learning Rate": 1.1738526728839956e-05, "Finetune/Loss": 0.9138327836990356, "Finetune/Loss (Raw)": 0.8822071552276611, "Finetune/Step": 2396, "Finetune/Step Time": 5.362221244722605} +{"Finetune/Learning Rate": 1.1732391647784618e-05, "Finetune/Loss": 0.9137463569641113, "Finetune/Loss (Raw)": 0.9398461580276489, "Finetune/Step": 2397, "Finetune/Step Time": 5.348982099443674} +{"Finetune/Learning Rate": 1.1726255894421908e-05, "Finetune/Loss": 0.9158706665039062, "Finetune/Loss (Raw)": 1.0842992067337036, "Finetune/Step": 2398, "Finetune/Step Time": 5.339863169938326} +{"Finetune/Learning Rate": 1.1720119471132985e-05, "Finetune/Loss": 0.9161017537117004, "Finetune/Loss (Raw)": 1.0065938234329224, "Finetune/Step": 2399, "Finetune/Step Time": 5.341227576136589} +{"Finetune/Learning Rate": 1.1713982380299278e-05, "Finetune/Loss": 0.914849579334259, "Finetune/Loss (Raw)": 0.8291245102882385, "Finetune/Step": 2400, "Finetune/Step Time": 5.33325850404799} +{"Finetune/Learning Rate": 1.1707844624302471e-05, "Finetune/Loss": 0.9150171279907227, "Finetune/Loss (Raw)": 0.959776759147644, "Finetune/Step": 2401, "Finetune/Step Time": 5.342279005795717} +{"Finetune/Learning Rate": 1.170170620552451e-05, "Finetune/Loss": 0.9148691296577454, "Finetune/Loss (Raw)": 0.8308618664741516, "Finetune/Step": 2402, "Finetune/Step Time": 5.331269336864352} +{"Finetune/Learning Rate": 1.1695567126347593e-05, "Finetune/Loss": 0.9140811562538147, "Finetune/Loss (Raw)": 0.9405889511108398, "Finetune/Step": 2403, "Finetune/Step Time": 5.340146137401462} +{"Finetune/Learning Rate": 1.1689427389154176e-05, "Finetune/Loss": 0.9142274856567383, "Finetune/Loss (Raw)": 0.9611347317695618, "Finetune/Step": 2404, "Finetune/Step Time": 5.339168399572372} +{"Finetune/Learning Rate": 1.1683286996326975e-05, "Finetune/Loss": 0.9157280325889587, "Finetune/Loss (Raw)": 0.9523450136184692, "Finetune/Step": 2405, "Finetune/Step Time": 5.3349884785711765} +{"Finetune/Learning Rate": 1.1677145950248954e-05, "Finetune/Loss": 0.9158188104629517, "Finetune/Loss (Raw)": 1.0001567602157593, "Finetune/Step": 2406, "Finetune/Step Time": 5.336157200857997} +{"Finetune/Learning Rate": 1.167100425330333e-05, "Finetune/Loss": 0.9131426811218262, "Finetune/Loss (Raw)": 0.7081494331359863, "Finetune/Step": 2407, "Finetune/Step Time": 5.35745963267982} +{"Finetune/Learning Rate": 1.1664861907873583e-05, "Finetune/Loss": 0.9149011373519897, "Finetune/Loss (Raw)": 1.1235740184783936, "Finetune/Step": 2408, "Finetune/Step Time": 5.3658600971102715} +{"Finetune/Learning Rate": 1.165871891634343e-05, "Finetune/Loss": 0.9147847890853882, "Finetune/Loss (Raw)": 0.8998903632164001, "Finetune/Step": 2409, "Finetune/Step Time": 5.365848882123828} +{"Finetune/Learning Rate": 1.165257528109685e-05, "Finetune/Loss": 0.9149411916732788, "Finetune/Loss (Raw)": 0.9383525252342224, "Finetune/Step": 2410, "Finetune/Step Time": 5.374855022877455} +{"Finetune/Learning Rate": 1.1646431004518068e-05, "Finetune/Loss": 0.9127418994903564, "Finetune/Loss (Raw)": 0.7706416845321655, "Finetune/Step": 2411, "Finetune/Step Time": 5.3764268066734076} +{"Finetune/Learning Rate": 1.1640286088991558e-05, "Finetune/Loss": 0.9126541018486023, "Finetune/Loss (Raw)": 0.8933537006378174, "Finetune/Step": 2412, "Finetune/Step Time": 5.359711669385433} +{"Finetune/Learning Rate": 1.1634140536902039e-05, "Finetune/Loss": 0.9114645719528198, "Finetune/Loss (Raw)": 0.8253061175346375, "Finetune/Step": 2413, "Finetune/Step Time": 5.34941516071558} +{"Finetune/Learning Rate": 1.1627994350634482e-05, "Finetune/Loss": 0.9113938808441162, "Finetune/Loss (Raw)": 0.9628269672393799, "Finetune/Step": 2414, "Finetune/Step Time": 5.329625507816672} +{"Finetune/Learning Rate": 1.1621847532574101e-05, "Finetune/Loss": 0.9121701717376709, "Finetune/Loss (Raw)": 0.8917984962463379, "Finetune/Step": 2415, "Finetune/Step Time": 5.329700533300638} +{"Finetune/Learning Rate": 1.1615700085106357e-05, "Finetune/Loss": 0.9121999740600586, "Finetune/Loss (Raw)": 0.9678394198417664, "Finetune/Step": 2416, "Finetune/Step Time": 5.332827804610133} +{"Finetune/Learning Rate": 1.1609552010616953e-05, "Finetune/Loss": 0.9134775996208191, "Finetune/Loss (Raw)": 0.922861635684967, "Finetune/Step": 2417, "Finetune/Step Time": 5.330627456307411} +{"Finetune/Learning Rate": 1.1603403311491841e-05, "Finetune/Loss": 0.9090138673782349, "Finetune/Loss (Raw)": 0.4433441460132599, "Finetune/Step": 2418, "Finetune/Step Time": 5.360656384378672} +{"Finetune/Learning Rate": 1.1597253990117205e-05, "Finetune/Loss": 0.9094542264938354, "Finetune/Loss (Raw)": 0.9023926854133606, "Finetune/Step": 2419, "Finetune/Step Time": 5.367698602378368} +{"Finetune/Learning Rate": 1.1591104048879479e-05, "Finetune/Loss": 0.9111316204071045, "Finetune/Loss (Raw)": 1.074733018875122, "Finetune/Step": 2420, "Finetune/Step Time": 5.357359146699309} +{"Finetune/Learning Rate": 1.1584953490165334e-05, "Finetune/Loss": 0.9113491177558899, "Finetune/Loss (Raw)": 0.7804355621337891, "Finetune/Step": 2421, "Finetune/Step Time": 5.366858085617423} +{"Finetune/Learning Rate": 1.1578802316361685e-05, "Finetune/Loss": 0.9101160764694214, "Finetune/Loss (Raw)": 0.7895036935806274, "Finetune/Step": 2422, "Finetune/Step Time": 5.376749876886606} +{"Finetune/Learning Rate": 1.157265052985568e-05, "Finetune/Loss": 0.9085124731063843, "Finetune/Loss (Raw)": 0.748834490776062, "Finetune/Step": 2423, "Finetune/Step Time": 5.369595475494862} +{"Finetune/Learning Rate": 1.156649813303471e-05, "Finetune/Loss": 0.9099006652832031, "Finetune/Loss (Raw)": 0.9457353353500366, "Finetune/Step": 2424, "Finetune/Step Time": 5.369383629411459} +{"Finetune/Learning Rate": 1.1560345128286396e-05, "Finetune/Loss": 0.9091275930404663, "Finetune/Loss (Raw)": 0.8534107804298401, "Finetune/Step": 2425, "Finetune/Step Time": 5.380125571042299} +{"Finetune/Learning Rate": 1.1554191517998598e-05, "Finetune/Loss": 0.9051223993301392, "Finetune/Loss (Raw)": 0.3721688985824585, "Finetune/Step": 2426, "Finetune/Step Time": 5.409920943900943} +{"Finetune/Learning Rate": 1.1548037304559419e-05, "Finetune/Loss": 0.9050224423408508, "Finetune/Loss (Raw)": 0.9746338129043579, "Finetune/Step": 2427, "Finetune/Step Time": 5.40013288334012} +{"Finetune/Learning Rate": 1.1541882490357183e-05, "Finetune/Loss": 0.9036917686462402, "Finetune/Loss (Raw)": 0.8128553032875061, "Finetune/Step": 2428, "Finetune/Step Time": 5.39468534104526} +{"Finetune/Learning Rate": 1.1535727077780456e-05, "Finetune/Loss": 0.9047317504882812, "Finetune/Loss (Raw)": 0.9750121831893921, "Finetune/Step": 2429, "Finetune/Step Time": 5.406365621834993} +{"Finetune/Learning Rate": 1.1529571069218033e-05, "Finetune/Loss": 0.9043250679969788, "Finetune/Loss (Raw)": 0.7569093108177185, "Finetune/Step": 2430, "Finetune/Step Time": 5.430712077766657} +{"Finetune/Learning Rate": 1.152341446705894e-05, "Finetune/Loss": 0.9050132632255554, "Finetune/Loss (Raw)": 1.0159109830856323, "Finetune/Step": 2431, "Finetune/Step Time": 5.428997479379177} +{"Finetune/Learning Rate": 1.1517257273692432e-05, "Finetune/Loss": 0.9065506458282471, "Finetune/Loss (Raw)": 0.9247218370437622, "Finetune/Step": 2432, "Finetune/Step Time": 5.43138163164258} +{"Finetune/Learning Rate": 1.1511099491508003e-05, "Finetune/Loss": 0.9070672988891602, "Finetune/Loss (Raw)": 0.8787568211555481, "Finetune/Step": 2433, "Finetune/Step Time": 5.431007903069258} +{"Finetune/Learning Rate": 1.1504941122895361e-05, "Finetune/Loss": 0.9060081243515015, "Finetune/Loss (Raw)": 0.9271729588508606, "Finetune/Step": 2434, "Finetune/Step Time": 5.430057777091861} +{"Finetune/Learning Rate": 1.1498782170244449e-05, "Finetune/Loss": 0.9052722454071045, "Finetune/Loss (Raw)": 0.8926671743392944, "Finetune/Step": 2435, "Finetune/Step Time": 5.439286332577467} +{"Finetune/Learning Rate": 1.1492622635945438e-05, "Finetune/Loss": 0.9080911874771118, "Finetune/Loss (Raw)": 0.9866775870323181, "Finetune/Step": 2436, "Finetune/Step Time": 5.4323877319693565} +{"Finetune/Learning Rate": 1.1486462522388724e-05, "Finetune/Loss": 0.9075231552124023, "Finetune/Loss (Raw)": 0.8765173554420471, "Finetune/Step": 2437, "Finetune/Step Time": 5.437494993209839} +{"Finetune/Learning Rate": 1.1480301831964926e-05, "Finetune/Loss": 0.907947838306427, "Finetune/Loss (Raw)": 1.0207362174987793, "Finetune/Step": 2438, "Finetune/Step Time": 5.441758342087269} +{"Finetune/Learning Rate": 1.1474140567064889e-05, "Finetune/Loss": 0.9076694250106812, "Finetune/Loss (Raw)": 1.0000994205474854, "Finetune/Step": 2439, "Finetune/Step Time": 5.441958328709006} +{"Finetune/Learning Rate": 1.1467978730079677e-05, "Finetune/Loss": 0.9085770845413208, "Finetune/Loss (Raw)": 1.0032782554626465, "Finetune/Step": 2440, "Finetune/Step Time": 5.421332327648997} +{"Finetune/Learning Rate": 1.1461816323400579e-05, "Finetune/Loss": 0.9086962342262268, "Finetune/Loss (Raw)": 0.9710566997528076, "Finetune/Step": 2441, "Finetune/Step Time": 5.4171895533800125} +{"Finetune/Learning Rate": 1.1455653349419105e-05, "Finetune/Loss": 0.9091470241546631, "Finetune/Loss (Raw)": 0.9316105842590332, "Finetune/Step": 2442, "Finetune/Step Time": 5.413046091794968} +{"Finetune/Learning Rate": 1.1449489810526985e-05, "Finetune/Loss": 0.9099999666213989, "Finetune/Loss (Raw)": 0.9824807643890381, "Finetune/Step": 2443, "Finetune/Step Time": 5.4179807640612125} +{"Finetune/Learning Rate": 1.144332570911617e-05, "Finetune/Loss": 0.9098906517028809, "Finetune/Loss (Raw)": 0.9762612581253052, "Finetune/Step": 2444, "Finetune/Step Time": 5.424836628139019} +{"Finetune/Learning Rate": 1.1437161047578826e-05, "Finetune/Loss": 0.9060671925544739, "Finetune/Loss (Raw)": 0.38761088252067566, "Finetune/Step": 2445, "Finetune/Step Time": 5.4524336233735085} +{"Finetune/Learning Rate": 1.1430995828307337e-05, "Finetune/Loss": 0.9063916206359863, "Finetune/Loss (Raw)": 0.9361229538917542, "Finetune/Step": 2446, "Finetune/Step Time": 5.453958222642541} +{"Finetune/Learning Rate": 1.1424830053694303e-05, "Finetune/Loss": 0.904983401298523, "Finetune/Loss (Raw)": 0.9144508242607117, "Finetune/Step": 2447, "Finetune/Step Time": 5.458453977480531} +{"Finetune/Learning Rate": 1.1418663726132543e-05, "Finetune/Loss": 0.9054123163223267, "Finetune/Loss (Raw)": 1.0091348886489868, "Finetune/Step": 2448, "Finetune/Step Time": 5.454644605517387} +{"Finetune/Learning Rate": 1.141249684801509e-05, "Finetune/Loss": 0.9044145345687866, "Finetune/Loss (Raw)": 0.8170782923698425, "Finetune/Step": 2449, "Finetune/Step Time": 5.446688516065478} +{"Finetune/Learning Rate": 1.140632942173519e-05, "Finetune/Loss": 0.9003170728683472, "Finetune/Loss (Raw)": 0.48713117837905884, "Finetune/Step": 2450, "Finetune/Step Time": 5.474756443873048} +{"Finetune/Learning Rate": 1.1400161449686293e-05, "Finetune/Loss": 0.8964733481407166, "Finetune/Loss (Raw)": 0.3222925066947937, "Finetune/Step": 2451, "Finetune/Step Time": 5.493371654301882} +{"Finetune/Learning Rate": 1.1393992934262073e-05, "Finetune/Loss": 0.8972039222717285, "Finetune/Loss (Raw)": 0.9640176296234131, "Finetune/Step": 2452, "Finetune/Step Time": 5.499830700457096} +{"Finetune/Learning Rate": 1.1387823877856413e-05, "Finetune/Loss": 0.8964552879333496, "Finetune/Loss (Raw)": 0.9402398467063904, "Finetune/Step": 2453, "Finetune/Step Time": 5.517671098932624} +{"Finetune/Learning Rate": 1.1381654282863399e-05, "Finetune/Loss": 0.9003371000289917, "Finetune/Loss (Raw)": 0.9329918026924133, "Finetune/Step": 2454, "Finetune/Step Time": 5.49674067273736} +{"Finetune/Learning Rate": 1.1375484151677334e-05, "Finetune/Loss": 0.9010765552520752, "Finetune/Loss (Raw)": 1.011695146560669, "Finetune/Step": 2455, "Finetune/Step Time": 5.484003737568855} +{"Finetune/Learning Rate": 1.1369313486692723e-05, "Finetune/Loss": 0.9018573760986328, "Finetune/Loss (Raw)": 0.9662818908691406, "Finetune/Step": 2456, "Finetune/Step Time": 5.473993161693215} +{"Finetune/Learning Rate": 1.136314229030428e-05, "Finetune/Loss": 0.9016425013542175, "Finetune/Loss (Raw)": 0.963318407535553, "Finetune/Step": 2457, "Finetune/Step Time": 5.471008136868477} +{"Finetune/Learning Rate": 1.1356970564906928e-05, "Finetune/Loss": 0.9027251601219177, "Finetune/Loss (Raw)": 0.9600039720535278, "Finetune/Step": 2458, "Finetune/Step Time": 5.476956833153963} +{"Finetune/Learning Rate": 1.1350798312895795e-05, "Finetune/Loss": 0.9022372961044312, "Finetune/Loss (Raw)": 0.8849363327026367, "Finetune/Step": 2459, "Finetune/Step Time": 5.470982486382127} +{"Finetune/Learning Rate": 1.1344625536666208e-05, "Finetune/Loss": 0.9006965160369873, "Finetune/Loss (Raw)": 0.939825713634491, "Finetune/Step": 2460, "Finetune/Step Time": 5.465309217572212} +{"Finetune/Learning Rate": 1.13384522386137e-05, "Finetune/Loss": 0.9012386202812195, "Finetune/Loss (Raw)": 1.0767898559570312, "Finetune/Step": 2461, "Finetune/Step Time": 5.461561230942607} +{"Finetune/Learning Rate": 1.1332278421134007e-05, "Finetune/Loss": 0.9013017416000366, "Finetune/Loss (Raw)": 0.9643115997314453, "Finetune/Step": 2462, "Finetune/Step Time": 5.448363801464438} +{"Finetune/Learning Rate": 1.1326104086623075e-05, "Finetune/Loss": 0.9018175601959229, "Finetune/Loss (Raw)": 0.9489200115203857, "Finetune/Step": 2463, "Finetune/Step Time": 5.453590972349048} +{"Finetune/Learning Rate": 1.1319929237477033e-05, "Finetune/Loss": 0.9028851985931396, "Finetune/Loss (Raw)": 1.0320420265197754, "Finetune/Step": 2464, "Finetune/Step Time": 5.433610321953893} +{"Finetune/Learning Rate": 1.1313753876092223e-05, "Finetune/Loss": 0.902416467666626, "Finetune/Loss (Raw)": 0.9101946353912354, "Finetune/Step": 2465, "Finetune/Step Time": 5.436365300789475} +{"Finetune/Learning Rate": 1.1307578004865188e-05, "Finetune/Loss": 0.901117205619812, "Finetune/Loss (Raw)": 0.8196164965629578, "Finetune/Step": 2466, "Finetune/Step Time": 5.437586039304733} +{"Finetune/Learning Rate": 1.130140162619265e-05, "Finetune/Loss": 0.9017014503479004, "Finetune/Loss (Raw)": 0.9226970076560974, "Finetune/Step": 2467, "Finetune/Step Time": 5.417831102386117} +{"Finetune/Learning Rate": 1.1295224742471555e-05, "Finetune/Loss": 0.9033890962600708, "Finetune/Loss (Raw)": 1.121031403541565, "Finetune/Step": 2468, "Finetune/Step Time": 5.422937523573637} +{"Finetune/Learning Rate": 1.128904735609902e-05, "Finetune/Loss": 0.9030247926712036, "Finetune/Loss (Raw)": 1.0090320110321045, "Finetune/Step": 2469, "Finetune/Step Time": 5.3943184819072485} +{"Finetune/Learning Rate": 1.1282869469472375e-05, "Finetune/Loss": 0.9068472385406494, "Finetune/Loss (Raw)": 0.8459492921829224, "Finetune/Step": 2470, "Finetune/Step Time": 5.370256502181292} +{"Finetune/Learning Rate": 1.1276691084989135e-05, "Finetune/Loss": 0.9067834615707397, "Finetune/Loss (Raw)": 0.997981607913971, "Finetune/Step": 2471, "Finetune/Step Time": 5.35421940125525} +{"Finetune/Learning Rate": 1.1270512205047008e-05, "Finetune/Loss": 0.9070289134979248, "Finetune/Loss (Raw)": 0.9512398838996887, "Finetune/Step": 2472, "Finetune/Step Time": 5.328428782522678} +{"Finetune/Learning Rate": 1.1264332832043902e-05, "Finetune/Loss": 0.9053449630737305, "Finetune/Loss (Raw)": 0.9476969242095947, "Finetune/Step": 2473, "Finetune/Step Time": 5.3347725085914135} +{"Finetune/Learning Rate": 1.1258152968377907e-05, "Finetune/Loss": 0.905593991279602, "Finetune/Loss (Raw)": 0.9489380121231079, "Finetune/Step": 2474, "Finetune/Step Time": 5.339174527674913} +{"Finetune/Learning Rate": 1.125197261644731e-05, "Finetune/Loss": 0.9054757952690125, "Finetune/Loss (Raw)": 0.889542818069458, "Finetune/Step": 2475, "Finetune/Step Time": 5.338996708393097} +{"Finetune/Learning Rate": 1.1245791778650578e-05, "Finetune/Loss": 0.9048446416854858, "Finetune/Loss (Raw)": 0.9395122528076172, "Finetune/Step": 2476, "Finetune/Step Time": 5.325565477833152} +{"Finetune/Learning Rate": 1.1239610457386381e-05, "Finetune/Loss": 0.9048461318016052, "Finetune/Loss (Raw)": 0.9332044720649719, "Finetune/Step": 2477, "Finetune/Step Time": 5.328069446608424} +{"Finetune/Learning Rate": 1.1233428655053572e-05, "Finetune/Loss": 0.9048818349838257, "Finetune/Loss (Raw)": 0.9249764680862427, "Finetune/Step": 2478, "Finetune/Step Time": 5.322362795472145} +{"Finetune/Learning Rate": 1.1227246374051175e-05, "Finetune/Loss": 0.9032964706420898, "Finetune/Loss (Raw)": 0.8395953178405762, "Finetune/Step": 2479, "Finetune/Step Time": 5.333852309733629} +{"Finetune/Learning Rate": 1.1221063616778425e-05, "Finetune/Loss": 0.9014483690261841, "Finetune/Loss (Raw)": 0.8513249754905701, "Finetune/Step": 2480, "Finetune/Step Time": 5.327139332890511} +{"Finetune/Learning Rate": 1.1214880385634723e-05, "Finetune/Loss": 0.8972326517105103, "Finetune/Loss (Raw)": 0.3772088587284088, "Finetune/Step": 2481, "Finetune/Step Time": 5.361327409744263} +{"Finetune/Learning Rate": 1.120869668301966e-05, "Finetune/Loss": 0.8985154628753662, "Finetune/Loss (Raw)": 1.1022930145263672, "Finetune/Step": 2482, "Finetune/Step Time": 5.363643089309335} +{"Finetune/Learning Rate": 1.1202512511333017e-05, "Finetune/Loss": 0.8992936611175537, "Finetune/Loss (Raw)": 0.9287676215171814, "Finetune/Step": 2483, "Finetune/Step Time": 5.3692374136298895} +{"Finetune/Learning Rate": 1.119632787297474e-05, "Finetune/Loss": 0.8998578786849976, "Finetune/Loss (Raw)": 0.8931645750999451, "Finetune/Step": 2484, "Finetune/Step Time": 5.374794306233525} +{"Finetune/Learning Rate": 1.1190142770344976e-05, "Finetune/Loss": 0.9000811576843262, "Finetune/Loss (Raw)": 0.9436091184616089, "Finetune/Step": 2485, "Finetune/Step Time": 5.375088235363364} +{"Finetune/Learning Rate": 1.1183957205844037e-05, "Finetune/Loss": 0.8984577655792236, "Finetune/Loss (Raw)": 0.8306483030319214, "Finetune/Step": 2486, "Finetune/Step Time": 5.385603554546833} +{"Finetune/Learning Rate": 1.1177771181872416e-05, "Finetune/Loss": 0.8986365795135498, "Finetune/Loss (Raw)": 0.3148890733718872, "Finetune/Step": 2487, "Finetune/Step Time": 5.384057039394975} +{"Finetune/Learning Rate": 1.1171584700830803e-05, "Finetune/Loss": 0.8995537757873535, "Finetune/Loss (Raw)": 1.0417306423187256, "Finetune/Step": 2488, "Finetune/Step Time": 5.3832072503864765} +{"Finetune/Learning Rate": 1.1165397765120035e-05, "Finetune/Loss": 0.8997707366943359, "Finetune/Loss (Raw)": 0.9774096012115479, "Finetune/Step": 2489, "Finetune/Step Time": 5.401961296796799} +{"Finetune/Learning Rate": 1.115921037714115e-05, "Finetune/Loss": 0.8948583602905273, "Finetune/Loss (Raw)": 0.3197668492794037, "Finetune/Step": 2490, "Finetune/Step Time": 5.437329845502973} +{"Finetune/Learning Rate": 1.1153022539295345e-05, "Finetune/Loss": 0.8960973024368286, "Finetune/Loss (Raw)": 0.90639328956604, "Finetune/Step": 2491, "Finetune/Step Time": 5.430477490648627} +{"Finetune/Learning Rate": 1.1146834253984008e-05, "Finetune/Loss": 0.8956621885299683, "Finetune/Loss (Raw)": 0.8033416867256165, "Finetune/Step": 2492, "Finetune/Step Time": 5.431971672922373} +{"Finetune/Learning Rate": 1.1140645523608684e-05, "Finetune/Loss": 0.8952006101608276, "Finetune/Loss (Raw)": 0.9848055839538574, "Finetune/Step": 2493, "Finetune/Step Time": 5.440204337239265} +{"Finetune/Learning Rate": 1.1134456350571099e-05, "Finetune/Loss": 0.8972122669219971, "Finetune/Loss (Raw)": 1.0275611877441406, "Finetune/Step": 2494, "Finetune/Step Time": 5.428769003599882} +{"Finetune/Learning Rate": 1.112826673727316e-05, "Finetune/Loss": 0.8969500064849854, "Finetune/Loss (Raw)": 0.9445969462394714, "Finetune/Step": 2495, "Finetune/Step Time": 5.421838991343975} +{"Finetune/Learning Rate": 1.112207668611692e-05, "Finetune/Loss": 0.8970269560813904, "Finetune/Loss (Raw)": 0.9348642826080322, "Finetune/Step": 2496, "Finetune/Step Time": 5.4432339034974575} +{"Finetune/Learning Rate": 1.1115886199504631e-05, "Finetune/Loss": 0.8954033851623535, "Finetune/Loss (Raw)": 0.8083460927009583, "Finetune/Step": 2497, "Finetune/Step Time": 5.441889941692352} +{"Finetune/Learning Rate": 1.110969527983869e-05, "Finetune/Loss": 0.8962352275848389, "Finetune/Loss (Raw)": 1.0183990001678467, "Finetune/Step": 2498, "Finetune/Step Time": 5.442693181335926} +{"Finetune/Learning Rate": 1.1103503929521677e-05, "Finetune/Loss": 0.8933910131454468, "Finetune/Loss (Raw)": 0.6688385009765625, "Finetune/Step": 2499, "Finetune/Step Time": 5.455440543591976} +{"Finetune/Learning Rate": 1.1097312150956336e-05, "Finetune/Loss": 0.8936938047409058, "Finetune/Loss (Raw)": 0.947453498840332, "Finetune/Step": 2500, "Finetune/Step Time": 5.455266570672393} +{"Finetune/Learning Rate": 1.1091119946545568e-05, "Finetune/Loss": 0.8930600881576538, "Finetune/Loss (Raw)": 0.9445575475692749, "Finetune/Step": 2501, "Finetune/Step Time": 5.435765700414777} +{"Finetune/Learning Rate": 1.1084927318692453e-05, "Finetune/Loss": 0.8940283060073853, "Finetune/Loss (Raw)": 0.9496614933013916, "Finetune/Step": 2502, "Finetune/Step Time": 5.436247704550624} +{"Finetune/Learning Rate": 1.1078734269800227e-05, "Finetune/Loss": 0.894988477230072, "Finetune/Loss (Raw)": 0.8982462882995605, "Finetune/Step": 2503, "Finetune/Step Time": 5.425218939781189} +{"Finetune/Learning Rate": 1.1072540802272291e-05, "Finetune/Loss": 0.894513726234436, "Finetune/Loss (Raw)": 0.8971080183982849, "Finetune/Step": 2504, "Finetune/Step Time": 5.429617412388325} +{"Finetune/Learning Rate": 1.1066346918512211e-05, "Finetune/Loss": 0.8932258486747742, "Finetune/Loss (Raw)": 0.9111287593841553, "Finetune/Step": 2505, "Finetune/Step Time": 5.429550714790821} +{"Finetune/Learning Rate": 1.1060152620923707e-05, "Finetune/Loss": 0.8920793533325195, "Finetune/Loss (Raw)": 0.8664745688438416, "Finetune/Step": 2506, "Finetune/Step Time": 5.433732762932777} +{"Finetune/Learning Rate": 1.105395791191067e-05, "Finetune/Loss": 0.892583429813385, "Finetune/Loss (Raw)": 1.0092946290969849, "Finetune/Step": 2507, "Finetune/Step Time": 5.4207385163754225} +{"Finetune/Learning Rate": 1.1047762793877146e-05, "Finetune/Loss": 0.8929561376571655, "Finetune/Loss (Raw)": 0.8504120111465454, "Finetune/Step": 2508, "Finetune/Step Time": 5.422468151897192} +{"Finetune/Learning Rate": 1.1041567269227336e-05, "Finetune/Loss": 0.8923702239990234, "Finetune/Loss (Raw)": 0.8453168272972107, "Finetune/Step": 2509, "Finetune/Step Time": 5.425583386793733} +{"Finetune/Learning Rate": 1.1035371340365606e-05, "Finetune/Loss": 0.8943480253219604, "Finetune/Loss (Raw)": 1.0529803037643433, "Finetune/Step": 2510, "Finetune/Step Time": 5.408765679225326} +{"Finetune/Learning Rate": 1.1029175009696468e-05, "Finetune/Loss": 0.896683394908905, "Finetune/Loss (Raw)": 0.9862916469573975, "Finetune/Step": 2511, "Finetune/Step Time": 5.405331775546074} +{"Finetune/Learning Rate": 1.1022978279624606e-05, "Finetune/Loss": 0.8943613171577454, "Finetune/Loss (Raw)": 0.8124211430549622, "Finetune/Step": 2512, "Finetune/Step Time": 5.4144227784127} +{"Finetune/Learning Rate": 1.1016781152554846e-05, "Finetune/Loss": 0.8938959836959839, "Finetune/Loss (Raw)": 0.8546017408370972, "Finetune/Step": 2513, "Finetune/Step Time": 5.4139397498220205} +{"Finetune/Learning Rate": 1.1010583630892166e-05, "Finetune/Loss": 0.893744707107544, "Finetune/Loss (Raw)": 1.0257335901260376, "Finetune/Step": 2514, "Finetune/Step Time": 5.420150745660067} +{"Finetune/Learning Rate": 1.1004385717041716e-05, "Finetune/Loss": 0.8929421901702881, "Finetune/Loss (Raw)": 0.7806056141853333, "Finetune/Step": 2515, "Finetune/Step Time": 5.425238758325577} +{"Finetune/Learning Rate": 1.0998187413408774e-05, "Finetune/Loss": 0.8926048278808594, "Finetune/Loss (Raw)": 0.9439578056335449, "Finetune/Step": 2516, "Finetune/Step Time": 5.408330902457237} +{"Finetune/Learning Rate": 1.0991988722398786e-05, "Finetune/Loss": 0.8917196393013, "Finetune/Loss (Raw)": 0.933849573135376, "Finetune/Step": 2517, "Finetune/Step Time": 5.411703955382109} +{"Finetune/Learning Rate": 1.0985789646417341e-05, "Finetune/Loss": 0.8904160261154175, "Finetune/Loss (Raw)": 0.8762214779853821, "Finetune/Step": 2518, "Finetune/Step Time": 5.41719002649188} +{"Finetune/Learning Rate": 1.097959018787018e-05, "Finetune/Loss": 0.8895326256752014, "Finetune/Loss (Raw)": 0.8844416737556458, "Finetune/Step": 2519, "Finetune/Step Time": 5.421469539403915} +{"Finetune/Learning Rate": 1.0973390349163195e-05, "Finetune/Loss": 0.8899587392807007, "Finetune/Loss (Raw)": 1.0005370378494263, "Finetune/Step": 2520, "Finetune/Step Time": 5.420748246833682} +{"Finetune/Learning Rate": 1.0967190132702414e-05, "Finetune/Loss": 0.884971022605896, "Finetune/Loss (Raw)": 0.4032527506351471, "Finetune/Step": 2521, "Finetune/Step Time": 5.435737036168575} +{"Finetune/Learning Rate": 1.096098954089403e-05, "Finetune/Loss": 0.8893834352493286, "Finetune/Loss (Raw)": 0.9615994095802307, "Finetune/Step": 2522, "Finetune/Step Time": 5.41350881010294} +{"Finetune/Learning Rate": 1.0954788576144365e-05, "Finetune/Loss": 0.8874906301498413, "Finetune/Loss (Raw)": 0.7148400545120239, "Finetune/Step": 2523, "Finetune/Step Time": 5.415263179689646} +{"Finetune/Learning Rate": 1.0948587240859893e-05, "Finetune/Loss": 0.8874940872192383, "Finetune/Loss (Raw)": 0.8826537132263184, "Finetune/Step": 2524, "Finetune/Step Time": 5.414178052917123} +{"Finetune/Learning Rate": 1.0942385537447236e-05, "Finetune/Loss": 0.8869049549102783, "Finetune/Loss (Raw)": 0.8644328117370605, "Finetune/Step": 2525, "Finetune/Step Time": 5.427310911938548} +{"Finetune/Learning Rate": 1.0936183468313147e-05, "Finetune/Loss": 0.8858760595321655, "Finetune/Loss (Raw)": 0.9526028037071228, "Finetune/Step": 2526, "Finetune/Step Time": 5.429874015972018} +{"Finetune/Learning Rate": 1.0929981035864532e-05, "Finetune/Loss": 0.885851263999939, "Finetune/Loss (Raw)": 1.0034198760986328, "Finetune/Step": 2527, "Finetune/Step Time": 5.440059792250395} +{"Finetune/Learning Rate": 1.0923778242508437e-05, "Finetune/Loss": 0.8872168064117432, "Finetune/Loss (Raw)": 1.0039093494415283, "Finetune/Step": 2528, "Finetune/Step Time": 5.456759627908468} +{"Finetune/Learning Rate": 1.0917575090652038e-05, "Finetune/Loss": 0.8818504810333252, "Finetune/Loss (Raw)": 0.2728908061981201, "Finetune/Step": 2529, "Finetune/Step Time": 5.480254603549838} +{"Finetune/Learning Rate": 1.091137158270266e-05, "Finetune/Loss": 0.8826687335968018, "Finetune/Loss (Raw)": 0.9356006979942322, "Finetune/Step": 2530, "Finetune/Step Time": 5.466401860117912} +{"Finetune/Learning Rate": 1.0905167721067762e-05, "Finetune/Loss": 0.8818479180335999, "Finetune/Loss (Raw)": 0.8355234265327454, "Finetune/Step": 2531, "Finetune/Step Time": 5.465282401069999} +{"Finetune/Learning Rate": 1.0898963508154947e-05, "Finetune/Loss": 0.8820509910583496, "Finetune/Loss (Raw)": 0.9871264100074768, "Finetune/Step": 2532, "Finetune/Step Time": 5.482196751981974} +{"Finetune/Learning Rate": 1.0892758946371943e-05, "Finetune/Loss": 0.8815571069717407, "Finetune/Loss (Raw)": 0.889135479927063, "Finetune/Step": 2533, "Finetune/Step Time": 5.484564978629351} +{"Finetune/Learning Rate": 1.0886554038126624e-05, "Finetune/Loss": 0.8766940832138062, "Finetune/Loss (Raw)": 0.3776812255382538, "Finetune/Step": 2534, "Finetune/Step Time": 5.514418927952647} +{"Finetune/Learning Rate": 1.0880348785826991e-05, "Finetune/Loss": 0.8791581392288208, "Finetune/Loss (Raw)": 1.023545265197754, "Finetune/Step": 2535, "Finetune/Step Time": 5.4982323087751865} +{"Finetune/Learning Rate": 1.087414319188118e-05, "Finetune/Loss": 0.876118540763855, "Finetune/Loss (Raw)": 0.7345141172409058, "Finetune/Step": 2536, "Finetune/Step Time": 5.506245046854019} +{"Finetune/Learning Rate": 1.0867937258697463e-05, "Finetune/Loss": 0.8761880397796631, "Finetune/Loss (Raw)": 0.9087871313095093, "Finetune/Step": 2537, "Finetune/Step Time": 5.518663363531232} +{"Finetune/Learning Rate": 1.086173098868424e-05, "Finetune/Loss": 0.8766666650772095, "Finetune/Loss (Raw)": 0.9996148943901062, "Finetune/Step": 2538, "Finetune/Step Time": 5.522544806823134} +{"Finetune/Learning Rate": 1.0855524384250048e-05, "Finetune/Loss": 0.8775641322135925, "Finetune/Loss (Raw)": 0.8855128884315491, "Finetune/Step": 2539, "Finetune/Step Time": 5.515322208404541} +{"Finetune/Learning Rate": 1.0849317447803542e-05, "Finetune/Loss": 0.8785228133201599, "Finetune/Loss (Raw)": 1.016064167022705, "Finetune/Step": 2540, "Finetune/Step Time": 5.5101633705198765} +{"Finetune/Learning Rate": 1.0843110181753513e-05, "Finetune/Loss": 0.8787621259689331, "Finetune/Loss (Raw)": 0.8559420704841614, "Finetune/Step": 2541, "Finetune/Step Time": 5.531982466578484} +{"Finetune/Learning Rate": 1.0836902588508888e-05, "Finetune/Loss": 0.8784427642822266, "Finetune/Loss (Raw)": 0.921951949596405, "Finetune/Step": 2542, "Finetune/Step Time": 5.5364610850811005} +{"Finetune/Learning Rate": 1.0830694670478706e-05, "Finetune/Loss": 0.8787816762924194, "Finetune/Loss (Raw)": 0.9351744055747986, "Finetune/Step": 2543, "Finetune/Step Time": 5.536764727905393} +{"Finetune/Learning Rate": 1.082448643007214e-05, "Finetune/Loss": 0.8769334554672241, "Finetune/Loss (Raw)": 0.7312647104263306, "Finetune/Step": 2544, "Finetune/Step Time": 5.541564458981156} +{"Finetune/Learning Rate": 1.0818277869698488e-05, "Finetune/Loss": 0.8766014575958252, "Finetune/Loss (Raw)": 0.880372941493988, "Finetune/Step": 2545, "Finetune/Step Time": 5.535015679895878} +{"Finetune/Learning Rate": 1.0812068991767171e-05, "Finetune/Loss": 0.8753217458724976, "Finetune/Loss (Raw)": 0.2795421779155731, "Finetune/Step": 2546, "Finetune/Step Time": 5.534761015325785} +{"Finetune/Learning Rate": 1.0805859798687727e-05, "Finetune/Loss": 0.8751349449157715, "Finetune/Loss (Raw)": 0.878478467464447, "Finetune/Step": 2547, "Finetune/Step Time": 5.511289430782199} +{"Finetune/Learning Rate": 1.0799650292869835e-05, "Finetune/Loss": 0.8748586177825928, "Finetune/Loss (Raw)": 1.0393695831298828, "Finetune/Step": 2548, "Finetune/Step Time": 5.513810265809298} +{"Finetune/Learning Rate": 1.0793440476723274e-05, "Finetune/Loss": 0.8713013529777527, "Finetune/Loss (Raw)": 0.32509732246398926, "Finetune/Step": 2549, "Finetune/Step Time": 5.524532036855817} +{"Finetune/Learning Rate": 1.0787230352657954e-05, "Finetune/Loss": 0.8724749684333801, "Finetune/Loss (Raw)": 0.9397284984588623, "Finetune/Step": 2550, "Finetune/Step Time": 5.51380062662065} +{"Finetune/Learning Rate": 1.0781019923083905e-05, "Finetune/Loss": 0.8741545677185059, "Finetune/Loss (Raw)": 0.9638147950172424, "Finetune/Step": 2551, "Finetune/Step Time": 5.509985344484448} +{"Finetune/Learning Rate": 1.077480919041127e-05, "Finetune/Loss": 0.8752931356430054, "Finetune/Loss (Raw)": 1.091478943824768, "Finetune/Step": 2552, "Finetune/Step Time": 5.510759182274342} +{"Finetune/Learning Rate": 1.0768598157050313e-05, "Finetune/Loss": 0.8754597902297974, "Finetune/Loss (Raw)": 0.8747392892837524, "Finetune/Step": 2553, "Finetune/Step Time": 5.512136222794652} +{"Finetune/Learning Rate": 1.0762386825411422e-05, "Finetune/Loss": 0.8800339698791504, "Finetune/Loss (Raw)": 0.9576650857925415, "Finetune/Step": 2554, "Finetune/Step Time": 5.510178266093135} +{"Finetune/Learning Rate": 1.0756175197905086e-05, "Finetune/Loss": 0.8750096559524536, "Finetune/Loss (Raw)": 0.33152127265930176, "Finetune/Step": 2555, "Finetune/Step Time": 5.54086784645915} +{"Finetune/Learning Rate": 1.0749963276941919e-05, "Finetune/Loss": 0.8757339715957642, "Finetune/Loss (Raw)": 0.9055688381195068, "Finetune/Step": 2556, "Finetune/Step Time": 5.540697414427996} +{"Finetune/Learning Rate": 1.0743751064932648e-05, "Finetune/Loss": 0.8741024732589722, "Finetune/Loss (Raw)": 0.766184389591217, "Finetune/Step": 2557, "Finetune/Step Time": 5.524350102990866} +{"Finetune/Learning Rate": 1.0737538564288104e-05, "Finetune/Loss": 0.8750447034835815, "Finetune/Loss (Raw)": 0.8775116205215454, "Finetune/Step": 2558, "Finetune/Step Time": 5.494270348921418} +{"Finetune/Learning Rate": 1.0731325777419246e-05, "Finetune/Loss": 0.8742382526397705, "Finetune/Loss (Raw)": 0.9126843214035034, "Finetune/Step": 2559, "Finetune/Step Time": 5.49308492615819} +{"Finetune/Learning Rate": 1.0725112706737133e-05, "Finetune/Loss": 0.8739923238754272, "Finetune/Loss (Raw)": 0.8932523131370544, "Finetune/Step": 2560, "Finetune/Step Time": 5.5019024927169085} +{"Finetune/Learning Rate": 1.0718899354652931e-05, "Finetune/Loss": 0.8741672039031982, "Finetune/Loss (Raw)": 0.9011396169662476, "Finetune/Step": 2561, "Finetune/Step Time": 5.505318535491824} +{"Finetune/Learning Rate": 1.071268572357793e-05, "Finetune/Loss": 0.8749165534973145, "Finetune/Loss (Raw)": 1.0230883359909058, "Finetune/Step": 2562, "Finetune/Step Time": 5.504300719127059} +{"Finetune/Learning Rate": 1.0706471815923511e-05, "Finetune/Loss": 0.8730552792549133, "Finetune/Loss (Raw)": 0.6544205546379089, "Finetune/Step": 2563, "Finetune/Step Time": 5.501976517960429} +{"Finetune/Learning Rate": 1.070025763410117e-05, "Finetune/Loss": 0.8726341724395752, "Finetune/Loss (Raw)": 0.9327793717384338, "Finetune/Step": 2564, "Finetune/Step Time": 5.504808519035578} +{"Finetune/Learning Rate": 1.0694043180522515e-05, "Finetune/Loss": 0.8734846115112305, "Finetune/Loss (Raw)": 0.9853676557540894, "Finetune/Step": 2565, "Finetune/Step Time": 5.502532349899411} +{"Finetune/Learning Rate": 1.0687828457599254e-05, "Finetune/Loss": 0.8728289604187012, "Finetune/Loss (Raw)": 0.9368127584457397, "Finetune/Step": 2566, "Finetune/Step Time": 5.50530999340117} +{"Finetune/Learning Rate": 1.0681613467743195e-05, "Finetune/Loss": 0.8713432550430298, "Finetune/Loss (Raw)": 0.8099319338798523, "Finetune/Step": 2567, "Finetune/Step Time": 5.509784858673811} +{"Finetune/Learning Rate": 1.0675398213366257e-05, "Finetune/Loss": 0.8697991371154785, "Finetune/Loss (Raw)": 0.8056299090385437, "Finetune/Step": 2568, "Finetune/Step Time": 5.513318656012416} +{"Finetune/Learning Rate": 1.0669182696880459e-05, "Finetune/Loss": 0.871052622795105, "Finetune/Loss (Raw)": 1.1315075159072876, "Finetune/Step": 2569, "Finetune/Step Time": 5.5111011397093534} +{"Finetune/Learning Rate": 1.0662966920697918e-05, "Finetune/Loss": 0.8710187673568726, "Finetune/Loss (Raw)": 0.9272759556770325, "Finetune/Step": 2570, "Finetune/Step Time": 5.515737956389785} +{"Finetune/Learning Rate": 1.0656750887230862e-05, "Finetune/Loss": 0.8716375827789307, "Finetune/Loss (Raw)": 1.0616904497146606, "Finetune/Step": 2571, "Finetune/Step Time": 5.514211442321539} +{"Finetune/Learning Rate": 1.0650534598891608e-05, "Finetune/Loss": 0.870932936668396, "Finetune/Loss (Raw)": 0.8860651254653931, "Finetune/Step": 2572, "Finetune/Step Time": 5.5115282870829105} +{"Finetune/Learning Rate": 1.0644318058092577e-05, "Finetune/Loss": 0.8750053644180298, "Finetune/Loss (Raw)": 0.9088742136955261, "Finetune/Step": 2573, "Finetune/Step Time": 5.486717918887734} +{"Finetune/Learning Rate": 1.0638101267246283e-05, "Finetune/Loss": 0.8746227025985718, "Finetune/Loss (Raw)": 0.8871493935585022, "Finetune/Step": 2574, "Finetune/Step Time": 5.509931115433574} +{"Finetune/Learning Rate": 1.063188422876535e-05, "Finetune/Loss": 0.874008059501648, "Finetune/Loss (Raw)": 0.8357701301574707, "Finetune/Step": 2575, "Finetune/Step Time": 5.502195429056883} +{"Finetune/Learning Rate": 1.0625666945062484e-05, "Finetune/Loss": 0.8735249042510986, "Finetune/Loss (Raw)": 0.9472990036010742, "Finetune/Step": 2576, "Finetune/Step Time": 5.503172945231199} +{"Finetune/Learning Rate": 1.061944941855049e-05, "Finetune/Loss": 0.8748297095298767, "Finetune/Loss (Raw)": 0.9840932488441467, "Finetune/Step": 2577, "Finetune/Step Time": 5.507205987349153} +{"Finetune/Learning Rate": 1.061323165164227e-05, "Finetune/Loss": 0.8790878653526306, "Finetune/Loss (Raw)": 1.0321706533432007, "Finetune/Step": 2578, "Finetune/Step Time": 5.476613936945796} +{"Finetune/Learning Rate": 1.0607013646750818e-05, "Finetune/Loss": 0.8845524787902832, "Finetune/Loss (Raw)": 1.0217697620391846, "Finetune/Step": 2579, "Finetune/Step Time": 5.4450951144099236} +{"Finetune/Learning Rate": 1.0600795406289221e-05, "Finetune/Loss": 0.8825992345809937, "Finetune/Loss (Raw)": 0.7139965891838074, "Finetune/Step": 2580, "Finetune/Step Time": 5.45092848688364} +{"Finetune/Learning Rate": 1.059457693267066e-05, "Finetune/Loss": 0.8835158348083496, "Finetune/Loss (Raw)": 1.0575624704360962, "Finetune/Step": 2581, "Finetune/Step Time": 5.4330000672489405} +{"Finetune/Learning Rate": 1.0588358228308395e-05, "Finetune/Loss": 0.883167564868927, "Finetune/Loss (Raw)": 0.888418436050415, "Finetune/Step": 2582, "Finetune/Step Time": 5.422456923872232} +{"Finetune/Learning Rate": 1.0582139295615793e-05, "Finetune/Loss": 0.8817059993743896, "Finetune/Loss (Raw)": 0.8246114253997803, "Finetune/Step": 2583, "Finetune/Step Time": 5.430999116972089} +{"Finetune/Learning Rate": 1.0575920137006288e-05, "Finetune/Loss": 0.8823201656341553, "Finetune/Loss (Raw)": 1.044899821281433, "Finetune/Step": 2584, "Finetune/Step Time": 5.431423796340823} +{"Finetune/Learning Rate": 1.056970075489343e-05, "Finetune/Loss": 0.8811813592910767, "Finetune/Loss (Raw)": 0.8175502419471741, "Finetune/Step": 2585, "Finetune/Step Time": 5.433947654440999} +{"Finetune/Learning Rate": 1.0563481151690827e-05, "Finetune/Loss": 0.8813348412513733, "Finetune/Loss (Raw)": 0.9796475768089294, "Finetune/Step": 2586, "Finetune/Step Time": 5.4203030448406935} +{"Finetune/Learning Rate": 1.055726132981219e-05, "Finetune/Loss": 0.8818427324295044, "Finetune/Loss (Raw)": 0.9499537944793701, "Finetune/Step": 2587, "Finetune/Step Time": 5.420536536723375} +{"Finetune/Learning Rate": 1.0551041291671311e-05, "Finetune/Loss": 0.8812514543533325, "Finetune/Loss (Raw)": 0.8641293048858643, "Finetune/Step": 2588, "Finetune/Step Time": 5.423768654465675} +{"Finetune/Learning Rate": 1.0544821039682066e-05, "Finetune/Loss": 0.8806498646736145, "Finetune/Loss (Raw)": 0.999792218208313, "Finetune/Step": 2589, "Finetune/Step Time": 5.427799887955189} +{"Finetune/Learning Rate": 1.0538600576258413e-05, "Finetune/Loss": 0.8803733587265015, "Finetune/Loss (Raw)": 0.928916871547699, "Finetune/Step": 2590, "Finetune/Step Time": 5.430716088041663} +{"Finetune/Learning Rate": 1.0532379903814387e-05, "Finetune/Loss": 0.8793111443519592, "Finetune/Loss (Raw)": 0.8129569292068481, "Finetune/Step": 2591, "Finetune/Step Time": 5.464251713827252} +{"Finetune/Learning Rate": 1.0526159024764118e-05, "Finetune/Loss": 0.8784050941467285, "Finetune/Loss (Raw)": 0.91606205701828, "Finetune/Step": 2592, "Finetune/Step Time": 5.472298221662641} +{"Finetune/Learning Rate": 1.0519937941521805e-05, "Finetune/Loss": 0.8778387308120728, "Finetune/Loss (Raw)": 0.8377040028572083, "Finetune/Step": 2593, "Finetune/Step Time": 5.492753352969885} +{"Finetune/Learning Rate": 1.0513716656501723e-05, "Finetune/Loss": 0.8779388070106506, "Finetune/Loss (Raw)": 0.8324270248413086, "Finetune/Step": 2594, "Finetune/Step Time": 5.504065593704581} +{"Finetune/Learning Rate": 1.0507495172118242e-05, "Finetune/Loss": 0.8777581453323364, "Finetune/Loss (Raw)": 0.8995749950408936, "Finetune/Step": 2595, "Finetune/Step Time": 5.508668130263686} +{"Finetune/Learning Rate": 1.0501273490785789e-05, "Finetune/Loss": 0.8757898807525635, "Finetune/Loss (Raw)": 0.8690852522850037, "Finetune/Step": 2596, "Finetune/Step Time": 5.501180104911327} +{"Finetune/Learning Rate": 1.049505161491888e-05, "Finetune/Loss": 0.875433623790741, "Finetune/Loss (Raw)": 0.9634400010108948, "Finetune/Step": 2597, "Finetune/Step Time": 5.504034414887428} +{"Finetune/Learning Rate": 1.0488829546932106e-05, "Finetune/Loss": 0.8746826648712158, "Finetune/Loss (Raw)": 0.7498235106468201, "Finetune/Step": 2598, "Finetune/Step Time": 5.5212977062910795} +{"Finetune/Learning Rate": 1.0482607289240128e-05, "Finetune/Loss": 0.874417245388031, "Finetune/Loss (Raw)": 0.9640092253684998, "Finetune/Step": 2599, "Finetune/Step Time": 5.525512423366308} +{"Finetune/Learning Rate": 1.0476384844257686e-05, "Finetune/Loss": 0.8739609122276306, "Finetune/Loss (Raw)": 0.8928278684616089, "Finetune/Step": 2600, "Finetune/Step Time": 5.529881179332733} +{"Finetune/Learning Rate": 1.0470162214399582e-05, "Finetune/Loss": 0.8737927079200745, "Finetune/Loss (Raw)": 0.926169753074646, "Finetune/Step": 2601, "Finetune/Step Time": 5.536799432709813} +{"Finetune/Learning Rate": 1.0463939402080706e-05, "Finetune/Loss": 0.874398410320282, "Finetune/Loss (Raw)": 1.0264612436294556, "Finetune/Step": 2602, "Finetune/Step Time": 5.545529384166002} +{"Finetune/Learning Rate": 1.0457716409716002e-05, "Finetune/Loss": 0.8752161264419556, "Finetune/Loss (Raw)": 0.9942215085029602, "Finetune/Step": 2603, "Finetune/Step Time": 5.544748133048415} +{"Finetune/Learning Rate": 1.0451493239720496e-05, "Finetune/Loss": 0.8748496174812317, "Finetune/Loss (Raw)": 0.8925889730453491, "Finetune/Step": 2604, "Finetune/Step Time": 5.550542125478387} +{"Finetune/Learning Rate": 1.0445269894509283e-05, "Finetune/Loss": 0.8737285733222961, "Finetune/Loss (Raw)": 0.7897146940231323, "Finetune/Step": 2605, "Finetune/Step Time": 5.543493730947375} +{"Finetune/Learning Rate": 1.043904637649751e-05, "Finetune/Loss": 0.8730872273445129, "Finetune/Loss (Raw)": 0.8428833484649658, "Finetune/Step": 2606, "Finetune/Step Time": 5.572507344186306} +{"Finetune/Learning Rate": 1.0432822688100414e-05, "Finetune/Loss": 0.874043345451355, "Finetune/Loss (Raw)": 0.9619783163070679, "Finetune/Step": 2607, "Finetune/Step Time": 5.581754829734564} +{"Finetune/Learning Rate": 1.0426598831733283e-05, "Finetune/Loss": 0.8748903274536133, "Finetune/Loss (Raw)": 0.9597339034080505, "Finetune/Step": 2608, "Finetune/Step Time": 5.579681573435664} +{"Finetune/Learning Rate": 1.0420374809811472e-05, "Finetune/Loss": 0.8792181015014648, "Finetune/Loss (Raw)": 0.9311656355857849, "Finetune/Step": 2609, "Finetune/Step Time": 5.551423188298941} +{"Finetune/Learning Rate": 1.0414150624750408e-05, "Finetune/Loss": 0.8774683475494385, "Finetune/Loss (Raw)": 0.8783258199691772, "Finetune/Step": 2610, "Finetune/Step Time": 5.563323151320219} +{"Finetune/Learning Rate": 1.0407926278965568e-05, "Finetune/Loss": 0.8758018016815186, "Finetune/Loss (Raw)": 0.7154487371444702, "Finetune/Step": 2611, "Finetune/Step Time": 5.569808043539524} +{"Finetune/Learning Rate": 1.0401701774872508e-05, "Finetune/Loss": 0.8757769465446472, "Finetune/Loss (Raw)": 0.8899838924407959, "Finetune/Step": 2612, "Finetune/Step Time": 5.557689379900694} +{"Finetune/Learning Rate": 1.0395477114886827e-05, "Finetune/Loss": 0.8763790726661682, "Finetune/Loss (Raw)": 1.0206815004348755, "Finetune/Step": 2613, "Finetune/Step Time": 5.5563301760703325} +{"Finetune/Learning Rate": 1.0389252301424197e-05, "Finetune/Loss": 0.8765212893486023, "Finetune/Loss (Raw)": 0.8488490581512451, "Finetune/Step": 2614, "Finetune/Step Time": 5.54836605116725} +{"Finetune/Learning Rate": 1.0383027336900356e-05, "Finetune/Loss": 0.8816075325012207, "Finetune/Loss (Raw)": 0.9659366011619568, "Finetune/Step": 2615, "Finetune/Step Time": 5.527584904804826} +{"Finetune/Learning Rate": 1.0376802223731077e-05, "Finetune/Loss": 0.8809165954589844, "Finetune/Loss (Raw)": 0.9532852172851562, "Finetune/Step": 2616, "Finetune/Step Time": 5.530855592340231} +{"Finetune/Learning Rate": 1.0370576964332217e-05, "Finetune/Loss": 0.8763762712478638, "Finetune/Loss (Raw)": 0.39625051617622375, "Finetune/Step": 2617, "Finetune/Step Time": 5.537889629602432} +{"Finetune/Learning Rate": 1.0364351561119667e-05, "Finetune/Loss": 0.8801729679107666, "Finetune/Loss (Raw)": 0.8057422637939453, "Finetune/Step": 2618, "Finetune/Step Time": 5.512707287445664} +{"Finetune/Learning Rate": 1.0358126016509392e-05, "Finetune/Loss": 0.8806429505348206, "Finetune/Loss (Raw)": 0.9665582180023193, "Finetune/Step": 2619, "Finetune/Step Time": 5.506488798186183} +{"Finetune/Learning Rate": 1.0351900332917401e-05, "Finetune/Loss": 0.8808891177177429, "Finetune/Loss (Raw)": 0.8348453640937805, "Finetune/Step": 2620, "Finetune/Step Time": 5.502230130136013} +{"Finetune/Learning Rate": 1.034567451275976e-05, "Finetune/Loss": 0.8801581263542175, "Finetune/Loss (Raw)": 0.8912393450737, "Finetune/Step": 2621, "Finetune/Step Time": 5.4990216214209795} +{"Finetune/Learning Rate": 1.0339448558452594e-05, "Finetune/Loss": 0.8795185089111328, "Finetune/Loss (Raw)": 0.9456889629364014, "Finetune/Step": 2622, "Finetune/Step Time": 5.494252094998956} +{"Finetune/Learning Rate": 1.0333222472412064e-05, "Finetune/Loss": 0.8792368769645691, "Finetune/Loss (Raw)": 0.9085463285446167, "Finetune/Step": 2623, "Finetune/Step Time": 5.514956435188651} +{"Finetune/Learning Rate": 1.03269962570544e-05, "Finetune/Loss": 0.8790385723114014, "Finetune/Loss (Raw)": 0.9094862937927246, "Finetune/Step": 2624, "Finetune/Step Time": 5.5037139765918255} +{"Finetune/Learning Rate": 1.0320769914795874e-05, "Finetune/Loss": 0.8787996768951416, "Finetune/Loss (Raw)": 0.777762770652771, "Finetune/Step": 2625, "Finetune/Step Time": 5.517386568710208} +{"Finetune/Learning Rate": 1.0314543448052806e-05, "Finetune/Loss": 0.8732757568359375, "Finetune/Loss (Raw)": 0.3113381862640381, "Finetune/Step": 2626, "Finetune/Step Time": 5.54788575693965} +{"Finetune/Learning Rate": 1.0308316859241567e-05, "Finetune/Loss": 0.875021755695343, "Finetune/Loss (Raw)": 0.8923289179801941, "Finetune/Step": 2627, "Finetune/Step Time": 5.530167395249009} +{"Finetune/Learning Rate": 1.0302090150778572e-05, "Finetune/Loss": 0.8753448724746704, "Finetune/Loss (Raw)": 0.9888149499893188, "Finetune/Step": 2628, "Finetune/Step Time": 5.5297169126570225} +{"Finetune/Learning Rate": 1.029586332508029e-05, "Finetune/Loss": 0.8729500770568848, "Finetune/Loss (Raw)": 0.6380152702331543, "Finetune/Step": 2629, "Finetune/Step Time": 5.545086532831192} +{"Finetune/Learning Rate": 1.0289636384563226e-05, "Finetune/Loss": 0.87186199426651, "Finetune/Loss (Raw)": 0.8103891611099243, "Finetune/Step": 2630, "Finetune/Step Time": 5.545189877972007} +{"Finetune/Learning Rate": 1.0283409331643936e-05, "Finetune/Loss": 0.8729972243309021, "Finetune/Loss (Raw)": 1.0435593128204346, "Finetune/Step": 2631, "Finetune/Step Time": 5.542380692437291} +{"Finetune/Learning Rate": 1.0277182168739019e-05, "Finetune/Loss": 0.872279167175293, "Finetune/Loss (Raw)": 0.8052005171775818, "Finetune/Step": 2632, "Finetune/Step Time": 5.537846179679036} +{"Finetune/Learning Rate": 1.0270954898265113e-05, "Finetune/Loss": 0.8726054430007935, "Finetune/Loss (Raw)": 0.952886164188385, "Finetune/Step": 2633, "Finetune/Step Time": 5.539421729743481} +{"Finetune/Learning Rate": 1.0264727522638899e-05, "Finetune/Loss": 0.8734238743782043, "Finetune/Loss (Raw)": 0.9712355732917786, "Finetune/Step": 2634, "Finetune/Step Time": 5.539976185187697} +{"Finetune/Learning Rate": 1.0258500044277106e-05, "Finetune/Loss": 0.87281334400177, "Finetune/Loss (Raw)": 0.9311445355415344, "Finetune/Step": 2635, "Finetune/Step Time": 5.534289909526706} +{"Finetune/Learning Rate": 1.025227246559649e-05, "Finetune/Loss": 0.8741052746772766, "Finetune/Loss (Raw)": 1.0157800912857056, "Finetune/Step": 2636, "Finetune/Step Time": 5.533776260912418} +{"Finetune/Learning Rate": 1.0246044789013855e-05, "Finetune/Loss": 0.8740418553352356, "Finetune/Loss (Raw)": 0.8371984362602234, "Finetune/Step": 2637, "Finetune/Step Time": 5.5385742001235485} +{"Finetune/Learning Rate": 1.023981701694604e-05, "Finetune/Loss": 0.8734767436981201, "Finetune/Loss (Raw)": 0.9806454181671143, "Finetune/Step": 2638, "Finetune/Step Time": 5.558709107339382} +{"Finetune/Learning Rate": 1.0233589151809927e-05, "Finetune/Loss": 0.8723064064979553, "Finetune/Loss (Raw)": 0.8364865779876709, "Finetune/Step": 2639, "Finetune/Step Time": 5.549939537420869} +{"Finetune/Learning Rate": 1.0227361196022421e-05, "Finetune/Loss": 0.8686655759811401, "Finetune/Loss (Raw)": 0.34640419483184814, "Finetune/Step": 2640, "Finetune/Step Time": 5.572279650717974} +{"Finetune/Learning Rate": 1.0221133152000474e-05, "Finetune/Loss": 0.8691291809082031, "Finetune/Loss (Raw)": 0.9139369130134583, "Finetune/Step": 2641, "Finetune/Step Time": 5.563300045207143} +{"Finetune/Learning Rate": 1.021490502216107e-05, "Finetune/Loss": 0.8681600689888, "Finetune/Loss (Raw)": 0.9016861915588379, "Finetune/Step": 2642, "Finetune/Step Time": 5.5527431927621365} +{"Finetune/Learning Rate": 1.020867680892122e-05, "Finetune/Loss": 0.8678338527679443, "Finetune/Loss (Raw)": 0.7388561367988586, "Finetune/Step": 2643, "Finetune/Step Time": 5.5500757191330194} +{"Finetune/Learning Rate": 1.0202448514697975e-05, "Finetune/Loss": 0.8665093779563904, "Finetune/Loss (Raw)": 0.7744221091270447, "Finetune/Step": 2644, "Finetune/Step Time": 5.5779964830726385} +{"Finetune/Learning Rate": 1.0196220141908417e-05, "Finetune/Loss": 0.8666455745697021, "Finetune/Loss (Raw)": 0.951276957988739, "Finetune/Step": 2645, "Finetune/Step Time": 5.581240855157375} +{"Finetune/Learning Rate": 1.018999169296965e-05, "Finetune/Loss": 0.8672906160354614, "Finetune/Loss (Raw)": 0.9587926864624023, "Finetune/Step": 2646, "Finetune/Step Time": 5.567712942138314} +{"Finetune/Learning Rate": 1.0183763170298818e-05, "Finetune/Loss": 0.8668122291564941, "Finetune/Loss (Raw)": 0.8232117295265198, "Finetune/Step": 2647, "Finetune/Step Time": 5.573715545237064} +{"Finetune/Learning Rate": 1.0177534576313084e-05, "Finetune/Loss": 0.866539716720581, "Finetune/Loss (Raw)": 0.96565181016922, "Finetune/Step": 2648, "Finetune/Step Time": 5.584784148260951} +{"Finetune/Learning Rate": 1.0171305913429648e-05, "Finetune/Loss": 0.8700529336929321, "Finetune/Loss (Raw)": 0.8529467582702637, "Finetune/Step": 2649, "Finetune/Step Time": 5.571648944169283} +{"Finetune/Learning Rate": 1.0165077184065732e-05, "Finetune/Loss": 0.8688770532608032, "Finetune/Loss (Raw)": 0.8110805749893188, "Finetune/Step": 2650, "Finetune/Step Time": 5.554698262363672} +{"Finetune/Learning Rate": 1.0158848390638588e-05, "Finetune/Loss": 0.8699839115142822, "Finetune/Loss (Raw)": 0.8565171957015991, "Finetune/Step": 2651, "Finetune/Step Time": 5.5502262860536575} +{"Finetune/Learning Rate": 1.015261953556548e-05, "Finetune/Loss": 0.8710988759994507, "Finetune/Loss (Raw)": 1.0253673791885376, "Finetune/Step": 2652, "Finetune/Step Time": 5.556887488812208} +{"Finetune/Learning Rate": 1.0146390621263709e-05, "Finetune/Loss": 0.8713217377662659, "Finetune/Loss (Raw)": 0.8929613828659058, "Finetune/Step": 2653, "Finetune/Step Time": 5.568523744121194} +{"Finetune/Learning Rate": 1.0140161650150598e-05, "Finetune/Loss": 0.8665350675582886, "Finetune/Loss (Raw)": 0.33990639448165894, "Finetune/Step": 2654, "Finetune/Step Time": 5.592345301061869} +{"Finetune/Learning Rate": 1.0133932624643487e-05, "Finetune/Loss": 0.8660445809364319, "Finetune/Loss (Raw)": 0.9406404495239258, "Finetune/Step": 2655, "Finetune/Step Time": 5.590630481019616} +{"Finetune/Learning Rate": 1.012770354715974e-05, "Finetune/Loss": 0.8652514219284058, "Finetune/Loss (Raw)": 0.9023832082748413, "Finetune/Step": 2656, "Finetune/Step Time": 5.577145664021373} +{"Finetune/Learning Rate": 1.012147442011674e-05, "Finetune/Loss": 0.8688167333602905, "Finetune/Loss (Raw)": 0.7292577028274536, "Finetune/Step": 2657, "Finetune/Step Time": 5.5614677127450705} +{"Finetune/Learning Rate": 1.0115245245931883e-05, "Finetune/Loss": 0.8670190572738647, "Finetune/Loss (Raw)": 0.7054915428161621, "Finetune/Step": 2658, "Finetune/Step Time": 5.580345572903752} +{"Finetune/Learning Rate": 1.0109016027022602e-05, "Finetune/Loss": 0.8683791160583496, "Finetune/Loss (Raw)": 1.0096129179000854, "Finetune/Step": 2659, "Finetune/Step Time": 5.576826713979244} +{"Finetune/Learning Rate": 1.0102786765806329e-05, "Finetune/Loss": 0.8680248856544495, "Finetune/Loss (Raw)": 0.9417883157730103, "Finetune/Step": 2660, "Finetune/Step Time": 5.5607626382261515} +{"Finetune/Learning Rate": 1.0096557464700518e-05, "Finetune/Loss": 0.8697939515113831, "Finetune/Loss (Raw)": 1.1155723333358765, "Finetune/Step": 2661, "Finetune/Step Time": 5.564480684697628} +{"Finetune/Learning Rate": 1.0090328126122642e-05, "Finetune/Loss": 0.8747866153717041, "Finetune/Loss (Raw)": 1.0167452096939087, "Finetune/Step": 2662, "Finetune/Step Time": 5.536616073921323} +{"Finetune/Learning Rate": 1.0084098752490179e-05, "Finetune/Loss": 0.8744897842407227, "Finetune/Loss (Raw)": 0.9855512380599976, "Finetune/Step": 2663, "Finetune/Step Time": 5.5442885383963585} +{"Finetune/Learning Rate": 1.0077869346220635e-05, "Finetune/Loss": 0.8721637725830078, "Finetune/Loss (Raw)": 0.4367820620536804, "Finetune/Step": 2664, "Finetune/Step Time": 5.557379795238376} +{"Finetune/Learning Rate": 1.0071639909731519e-05, "Finetune/Loss": 0.8711961507797241, "Finetune/Loss (Raw)": 0.7849329113960266, "Finetune/Step": 2665, "Finetune/Step Time": 5.550434177741408} +{"Finetune/Learning Rate": 1.0065410445440352e-05, "Finetune/Loss": 0.8710123300552368, "Finetune/Loss (Raw)": 0.9760812520980835, "Finetune/Step": 2666, "Finetune/Step Time": 5.552064770832658} +{"Finetune/Learning Rate": 1.0059180955764666e-05, "Finetune/Loss": 0.8703358173370361, "Finetune/Loss (Raw)": 0.7989272475242615, "Finetune/Step": 2667, "Finetune/Step Time": 5.546883242204785} +{"Finetune/Learning Rate": 1.005295144312201e-05, "Finetune/Loss": 0.8691059947013855, "Finetune/Loss (Raw)": 0.8586447834968567, "Finetune/Step": 2668, "Finetune/Step Time": 5.542986607179046} +{"Finetune/Learning Rate": 1.0046721909929928e-05, "Finetune/Loss": 0.8702878952026367, "Finetune/Loss (Raw)": 1.0072253942489624, "Finetune/Step": 2669, "Finetune/Step Time": 5.521526286378503} +{"Finetune/Learning Rate": 1.0040492358605989e-05, "Finetune/Loss": 0.8703116178512573, "Finetune/Loss (Raw)": 0.9249898791313171, "Finetune/Step": 2670, "Finetune/Step Time": 5.522105781361461} +{"Finetune/Learning Rate": 1.0034262791567755e-05, "Finetune/Loss": 0.8691264986991882, "Finetune/Loss (Raw)": 0.7834799289703369, "Finetune/Step": 2671, "Finetune/Step Time": 5.522922167554498} +{"Finetune/Learning Rate": 1.0028033211232802e-05, "Finetune/Loss": 0.8688146471977234, "Finetune/Loss (Raw)": 0.6913413405418396, "Finetune/Step": 2672, "Finetune/Step Time": 5.522088760510087} +{"Finetune/Learning Rate": 1.0021803620018706e-05, "Finetune/Loss": 0.8687001466751099, "Finetune/Loss (Raw)": 0.865721583366394, "Finetune/Step": 2673, "Finetune/Step Time": 5.522629739716649} +{"Finetune/Learning Rate": 1.0015574020343048e-05, "Finetune/Loss": 0.8730902671813965, "Finetune/Loss (Raw)": 0.8414753675460815, "Finetune/Step": 2674, "Finetune/Step Time": 5.508076608181} +{"Finetune/Learning Rate": 1.0009344414623418e-05, "Finetune/Loss": 0.8729751110076904, "Finetune/Loss (Raw)": 0.8637328147888184, "Finetune/Step": 2675, "Finetune/Step Time": 5.508026674389839} +{"Finetune/Learning Rate": 1.0003114805277405e-05, "Finetune/Loss": 0.8714233636856079, "Finetune/Loss (Raw)": 0.8407548069953918, "Finetune/Step": 2676, "Finetune/Step Time": 5.511783581227064} +{"Finetune/Learning Rate": 9.9968851947226e-06, "Finetune/Loss": 0.8766489028930664, "Finetune/Loss (Raw)": 0.9939592480659485, "Finetune/Step": 2677, "Finetune/Step Time": 5.479176461696625} +{"Finetune/Learning Rate": 9.990655585376583e-06, "Finetune/Loss": 0.8761658072471619, "Finetune/Loss (Raw)": 0.8778960704803467, "Finetune/Step": 2678, "Finetune/Step Time": 5.478793069720268} +{"Finetune/Learning Rate": 9.984425979656955e-06, "Finetune/Loss": 0.8766701817512512, "Finetune/Loss (Raw)": 1.0283727645874023, "Finetune/Step": 2679, "Finetune/Step Time": 5.476433675736189} +{"Finetune/Learning Rate": 9.978196379981297e-06, "Finetune/Loss": 0.8753489851951599, "Finetune/Loss (Raw)": 0.9223686456680298, "Finetune/Step": 2680, "Finetune/Step Time": 5.48470688983798} +{"Finetune/Learning Rate": 9.971966788767198e-06, "Finetune/Loss": 0.8711811304092407, "Finetune/Loss (Raw)": 0.3412574529647827, "Finetune/Step": 2681, "Finetune/Step Time": 5.502791274338961} +{"Finetune/Learning Rate": 9.96573720843225e-06, "Finetune/Loss": 0.8707383871078491, "Finetune/Loss (Raw)": 0.9009864926338196, "Finetune/Step": 2682, "Finetune/Step Time": 5.478423699736595} +{"Finetune/Learning Rate": 9.959507641394015e-06, "Finetune/Loss": 0.8752791881561279, "Finetune/Loss (Raw)": 0.9127398133277893, "Finetune/Step": 2683, "Finetune/Step Time": 5.445835145190358} +{"Finetune/Learning Rate": 9.953278090070075e-06, "Finetune/Loss": 0.875141441822052, "Finetune/Loss (Raw)": 0.8879451751708984, "Finetune/Step": 2684, "Finetune/Step Time": 5.446024434641004} +{"Finetune/Learning Rate": 9.947048556877992e-06, "Finetune/Loss": 0.8762153387069702, "Finetune/Loss (Raw)": 0.9036425352096558, "Finetune/Step": 2685, "Finetune/Step Time": 5.449577260762453} +{"Finetune/Learning Rate": 9.940819044235335e-06, "Finetune/Loss": 0.8756197094917297, "Finetune/Loss (Raw)": 0.8012721538543701, "Finetune/Step": 2686, "Finetune/Step Time": 5.451969116926193} +{"Finetune/Learning Rate": 9.934589554559653e-06, "Finetune/Loss": 0.8755304217338562, "Finetune/Loss (Raw)": 0.901260256767273, "Finetune/Step": 2687, "Finetune/Step Time": 5.453487699851394} +{"Finetune/Learning Rate": 9.928360090268484e-06, "Finetune/Loss": 0.8758519887924194, "Finetune/Loss (Raw)": 0.9344094395637512, "Finetune/Step": 2688, "Finetune/Step Time": 5.434239931404591} +{"Finetune/Learning Rate": 9.922130653779368e-06, "Finetune/Loss": 0.8768647909164429, "Finetune/Loss (Raw)": 1.0307786464691162, "Finetune/Step": 2689, "Finetune/Step Time": 5.43151025287807} +{"Finetune/Learning Rate": 9.915901247509821e-06, "Finetune/Loss": 0.8717539310455322, "Finetune/Loss (Raw)": 0.3688947856426239, "Finetune/Step": 2690, "Finetune/Step Time": 5.463232295587659} +{"Finetune/Learning Rate": 9.909671873877363e-06, "Finetune/Loss": 0.8748946189880371, "Finetune/Loss (Raw)": 1.0564327239990234, "Finetune/Step": 2691, "Finetune/Step Time": 5.455839019268751} +{"Finetune/Learning Rate": 9.903442535299487e-06, "Finetune/Loss": 0.8743077516555786, "Finetune/Loss (Raw)": 0.8576619625091553, "Finetune/Step": 2692, "Finetune/Step Time": 5.455039534717798} +{"Finetune/Learning Rate": 9.897213234193673e-06, "Finetune/Loss": 0.8741105794906616, "Finetune/Loss (Raw)": 0.960128128528595, "Finetune/Step": 2693, "Finetune/Step Time": 5.45277414470911} +{"Finetune/Learning Rate": 9.890983972977401e-06, "Finetune/Loss": 0.8738417625427246, "Finetune/Loss (Raw)": 0.902396023273468, "Finetune/Step": 2694, "Finetune/Step Time": 5.454228814691305} +{"Finetune/Learning Rate": 9.884754754068116e-06, "Finetune/Loss": 0.874161958694458, "Finetune/Loss (Raw)": 0.8509224653244019, "Finetune/Step": 2695, "Finetune/Step Time": 5.453424893319607} +{"Finetune/Learning Rate": 9.878525579883264e-06, "Finetune/Loss": 0.8754661679267883, "Finetune/Loss (Raw)": 0.9725663661956787, "Finetune/Step": 2696, "Finetune/Step Time": 5.465367596596479} +{"Finetune/Learning Rate": 9.872296452840266e-06, "Finetune/Loss": 0.8734285831451416, "Finetune/Loss (Raw)": 0.8707033395767212, "Finetune/Step": 2697, "Finetune/Step Time": 5.467898279428482} +{"Finetune/Learning Rate": 9.866067375356516e-06, "Finetune/Loss": 0.8726307153701782, "Finetune/Loss (Raw)": 0.8251434564590454, "Finetune/Step": 2698, "Finetune/Step Time": 5.462972333654761} +{"Finetune/Learning Rate": 9.859838349849404e-06, "Finetune/Loss": 0.8722450733184814, "Finetune/Loss (Raw)": 1.0123281478881836, "Finetune/Step": 2699, "Finetune/Step Time": 5.477276528254151} +{"Finetune/Learning Rate": 9.853609378736293e-06, "Finetune/Loss": 0.8720800280570984, "Finetune/Loss (Raw)": 0.8649400472640991, "Finetune/Step": 2700, "Finetune/Step Time": 5.476756036281586} +{"Finetune/Learning Rate": 9.847380464434525e-06, "Finetune/Loss": 0.871880292892456, "Finetune/Loss (Raw)": 0.8833054900169373, "Finetune/Step": 2701, "Finetune/Step Time": 5.499512488022447} +{"Finetune/Learning Rate": 9.841151609361419e-06, "Finetune/Loss": 0.8722331523895264, "Finetune/Loss (Raw)": 0.9323148131370544, "Finetune/Step": 2702, "Finetune/Step Time": 5.487682197242975} +{"Finetune/Learning Rate": 9.83492281593427e-06, "Finetune/Loss": 0.8721930384635925, "Finetune/Loss (Raw)": 0.8306419253349304, "Finetune/Step": 2703, "Finetune/Step Time": 5.505024770274758} +{"Finetune/Learning Rate": 9.828694086570353e-06, "Finetune/Loss": 0.8710397481918335, "Finetune/Loss (Raw)": 0.7996765971183777, "Finetune/Step": 2704, "Finetune/Step Time": 5.517102623358369} +{"Finetune/Learning Rate": 9.822465423686917e-06, "Finetune/Loss": 0.8702011108398438, "Finetune/Loss (Raw)": 0.8767507076263428, "Finetune/Step": 2705, "Finetune/Step Time": 5.5177121087908745} +{"Finetune/Learning Rate": 9.816236829701187e-06, "Finetune/Loss": 0.8698035478591919, "Finetune/Loss (Raw)": 0.9812754392623901, "Finetune/Step": 2706, "Finetune/Step Time": 5.524908199906349} +{"Finetune/Learning Rate": 9.810008307030356e-06, "Finetune/Loss": 0.8696628212928772, "Finetune/Loss (Raw)": 1.0037598609924316, "Finetune/Step": 2707, "Finetune/Step Time": 5.521457571536303} +{"Finetune/Learning Rate": 9.803779858091586e-06, "Finetune/Loss": 0.8720176219940186, "Finetune/Loss (Raw)": 1.0154062509536743, "Finetune/Step": 2708, "Finetune/Step Time": 5.518421879038215} +{"Finetune/Learning Rate": 9.797551485302027e-06, "Finetune/Loss": 0.8705347180366516, "Finetune/Loss (Raw)": 0.8677572011947632, "Finetune/Step": 2709, "Finetune/Step Time": 5.522273346781731} +{"Finetune/Learning Rate": 9.791323191078781e-06, "Finetune/Loss": 0.8665510416030884, "Finetune/Loss (Raw)": 0.37850403785705566, "Finetune/Step": 2710, "Finetune/Step Time": 5.554942777380347} +{"Finetune/Learning Rate": 9.785094977838935e-06, "Finetune/Loss": 0.8676683902740479, "Finetune/Loss (Raw)": 0.9676348567008972, "Finetune/Step": 2711, "Finetune/Step Time": 5.5451976135373116} +{"Finetune/Learning Rate": 9.77886684799953e-06, "Finetune/Loss": 0.8668116331100464, "Finetune/Loss (Raw)": 0.9352282285690308, "Finetune/Step": 2712, "Finetune/Step Time": 5.563994947820902} +{"Finetune/Learning Rate": 9.77263880397758e-06, "Finetune/Loss": 0.8673171997070312, "Finetune/Loss (Raw)": 0.8822681903839111, "Finetune/Step": 2713, "Finetune/Step Time": 5.570787625387311} +{"Finetune/Learning Rate": 9.766410848190076e-06, "Finetune/Loss": 0.8671258687973022, "Finetune/Loss (Raw)": 0.9551554322242737, "Finetune/Step": 2714, "Finetune/Step Time": 5.570385193452239} +{"Finetune/Learning Rate": 9.76018298305396e-06, "Finetune/Loss": 0.866199791431427, "Finetune/Loss (Raw)": 0.8314210176467896, "Finetune/Step": 2715, "Finetune/Step Time": 5.563268290832639} +{"Finetune/Learning Rate": 9.75395521098615e-06, "Finetune/Loss": 0.8661774396896362, "Finetune/Loss (Raw)": 0.8612625002861023, "Finetune/Step": 2716, "Finetune/Step Time": 5.5609413757920265} +{"Finetune/Learning Rate": 9.747727534403515e-06, "Finetune/Loss": 0.865444540977478, "Finetune/Loss (Raw)": 0.9059811234474182, "Finetune/Step": 2717, "Finetune/Step Time": 5.5427000895142555} +{"Finetune/Learning Rate": 9.741499955722897e-06, "Finetune/Loss": 0.8649675846099854, "Finetune/Loss (Raw)": 0.8678736090660095, "Finetune/Step": 2718, "Finetune/Step Time": 5.542974619194865} +{"Finetune/Learning Rate": 9.735272477361103e-06, "Finetune/Loss": 0.8659600615501404, "Finetune/Loss (Raw)": 0.9399932622909546, "Finetune/Step": 2719, "Finetune/Step Time": 5.5057805217802525} +{"Finetune/Learning Rate": 9.729045101734889e-06, "Finetune/Loss": 0.8662251234054565, "Finetune/Loss (Raw)": 0.9499868154525757, "Finetune/Step": 2720, "Finetune/Step Time": 5.501452816650271} +{"Finetune/Learning Rate": 9.722817831260985e-06, "Finetune/Loss": 0.8664252758026123, "Finetune/Loss (Raw)": 0.863323450088501, "Finetune/Step": 2721, "Finetune/Step Time": 5.496630370616913} +{"Finetune/Learning Rate": 9.716590668356067e-06, "Finetune/Loss": 0.8668503761291504, "Finetune/Loss (Raw)": 0.8868405222892761, "Finetune/Step": 2722, "Finetune/Step Time": 5.487101783975959} +{"Finetune/Learning Rate": 9.710363615436777e-06, "Finetune/Loss": 0.8664889931678772, "Finetune/Loss (Raw)": 0.8533183336257935, "Finetune/Step": 2723, "Finetune/Step Time": 5.487567009404302} +{"Finetune/Learning Rate": 9.704136674919712e-06, "Finetune/Loss": 0.8669477105140686, "Finetune/Loss (Raw)": 0.9277968406677246, "Finetune/Step": 2724, "Finetune/Step Time": 5.485236847773194} +{"Finetune/Learning Rate": 9.697909849221428e-06, "Finetune/Loss": 0.8666424751281738, "Finetune/Loss (Raw)": 0.9243707656860352, "Finetune/Step": 2725, "Finetune/Step Time": 5.479130202904344} +{"Finetune/Learning Rate": 9.691683140758439e-06, "Finetune/Loss": 0.8679760098457336, "Finetune/Loss (Raw)": 0.920522153377533, "Finetune/Step": 2726, "Finetune/Step Time": 5.470972090959549} +{"Finetune/Learning Rate": 9.685456551947199e-06, "Finetune/Loss": 0.8673693537712097, "Finetune/Loss (Raw)": 0.8863515257835388, "Finetune/Step": 2727, "Finetune/Step Time": 5.473805669695139} +{"Finetune/Learning Rate": 9.67923008520413e-06, "Finetune/Loss": 0.8666250705718994, "Finetune/Loss (Raw)": 0.7975591421127319, "Finetune/Step": 2728, "Finetune/Step Time": 5.472306748852134} +{"Finetune/Learning Rate": 9.673003742945602e-06, "Finetune/Loss": 0.8626755475997925, "Finetune/Loss (Raw)": 0.42063724994659424, "Finetune/Step": 2729, "Finetune/Step Time": 5.498534105718136} +{"Finetune/Learning Rate": 9.666777527587936e-06, "Finetune/Loss": 0.8607957363128662, "Finetune/Loss (Raw)": 0.7858355641365051, "Finetune/Step": 2730, "Finetune/Step Time": 5.489232299849391} +{"Finetune/Learning Rate": 9.660551441547411e-06, "Finetune/Loss": 0.8601309061050415, "Finetune/Loss (Raw)": 0.909126341342926, "Finetune/Step": 2731, "Finetune/Step Time": 5.49188913218677} +{"Finetune/Learning Rate": 9.654325487240243e-06, "Finetune/Loss": 0.8605161905288696, "Finetune/Loss (Raw)": 0.9418978691101074, "Finetune/Step": 2732, "Finetune/Step Time": 5.4872275441884995} +{"Finetune/Learning Rate": 9.6480996670826e-06, "Finetune/Loss": 0.8613086938858032, "Finetune/Loss (Raw)": 0.8911671042442322, "Finetune/Step": 2733, "Finetune/Step Time": 5.487605208531022} +{"Finetune/Learning Rate": 9.64187398349061e-06, "Finetune/Loss": 0.8622071146965027, "Finetune/Loss (Raw)": 0.9578810334205627, "Finetune/Step": 2734, "Finetune/Step Time": 5.463903922587633} +{"Finetune/Learning Rate": 9.635648438880334e-06, "Finetune/Loss": 0.8622921109199524, "Finetune/Loss (Raw)": 0.9728549718856812, "Finetune/Step": 2735, "Finetune/Step Time": 5.440931756049395} +{"Finetune/Learning Rate": 9.629423035667788e-06, "Finetune/Loss": 0.8583677411079407, "Finetune/Loss (Raw)": 0.457415908575058, "Finetune/Step": 2736, "Finetune/Step Time": 5.464804170653224} +{"Finetune/Learning Rate": 9.623197776268926e-06, "Finetune/Loss": 0.8559590578079224, "Finetune/Loss (Raw)": 0.6228507161140442, "Finetune/Step": 2737, "Finetune/Step Time": 5.468706490471959} +{"Finetune/Learning Rate": 9.616972663099648e-06, "Finetune/Loss": 0.8563021421432495, "Finetune/Loss (Raw)": 0.9222384691238403, "Finetune/Step": 2738, "Finetune/Step Time": 5.457915220409632} +{"Finetune/Learning Rate": 9.610747698575801e-06, "Finetune/Loss": 0.858054518699646, "Finetune/Loss (Raw)": 0.9397605657577515, "Finetune/Step": 2739, "Finetune/Step Time": 5.449514247477055} +{"Finetune/Learning Rate": 9.60452288511318e-06, "Finetune/Loss": 0.8577028512954712, "Finetune/Loss (Raw)": 0.8449581265449524, "Finetune/Step": 2740, "Finetune/Step Time": 5.458673777058721} +{"Finetune/Learning Rate": 9.598298225127499e-06, "Finetune/Loss": 0.8575625419616699, "Finetune/Loss (Raw)": 1.002726435661316, "Finetune/Step": 2741, "Finetune/Step Time": 5.459642987698317} +{"Finetune/Learning Rate": 9.592073721034435e-06, "Finetune/Loss": 0.8580374717712402, "Finetune/Loss (Raw)": 0.9096441268920898, "Finetune/Step": 2742, "Finetune/Step Time": 5.464632971212268} +{"Finetune/Learning Rate": 9.585849375249595e-06, "Finetune/Loss": 0.8580482602119446, "Finetune/Loss (Raw)": 0.9673161506652832, "Finetune/Step": 2743, "Finetune/Step Time": 5.456844255328178} +{"Finetune/Learning Rate": 9.57962519018853e-06, "Finetune/Loss": 0.8579097986221313, "Finetune/Loss (Raw)": 0.935563325881958, "Finetune/Step": 2744, "Finetune/Step Time": 5.453151674941182} +{"Finetune/Learning Rate": 9.573401168266722e-06, "Finetune/Loss": 0.862524688243866, "Finetune/Loss (Raw)": 0.9869564771652222, "Finetune/Step": 2745, "Finetune/Step Time": 5.422079320997} +{"Finetune/Learning Rate": 9.567177311899589e-06, "Finetune/Loss": 0.863368034362793, "Finetune/Loss (Raw)": 0.9136874675750732, "Finetune/Step": 2746, "Finetune/Step Time": 5.429083667695522} +{"Finetune/Learning Rate": 9.560953623502493e-06, "Finetune/Loss": 0.8627709746360779, "Finetune/Loss (Raw)": 0.8901405334472656, "Finetune/Step": 2747, "Finetune/Step Time": 5.445571837946773} +{"Finetune/Learning Rate": 9.554730105490722e-06, "Finetune/Loss": 0.864237368106842, "Finetune/Loss (Raw)": 1.0225403308868408, "Finetune/Step": 2748, "Finetune/Step Time": 5.4537106435745955} +{"Finetune/Learning Rate": 9.548506760279506e-06, "Finetune/Loss": 0.8646949529647827, "Finetune/Loss (Raw)": 0.9498187899589539, "Finetune/Step": 2749, "Finetune/Step Time": 5.450242165476084} +{"Finetune/Learning Rate": 9.542283590284003e-06, "Finetune/Loss": 0.8640700578689575, "Finetune/Loss (Raw)": 0.8656896352767944, "Finetune/Step": 2750, "Finetune/Step Time": 5.454093946143985} +{"Finetune/Learning Rate": 9.5360605979193e-06, "Finetune/Loss": 0.8643279671669006, "Finetune/Loss (Raw)": 0.9415603876113892, "Finetune/Step": 2751, "Finetune/Step Time": 5.443071104586124} +{"Finetune/Learning Rate": 9.52983778560042e-06, "Finetune/Loss": 0.8646795749664307, "Finetune/Loss (Raw)": 0.9544918537139893, "Finetune/Step": 2752, "Finetune/Step Time": 5.442613931372762} +{"Finetune/Learning Rate": 9.523615155742317e-06, "Finetune/Loss": 0.8663490414619446, "Finetune/Loss (Raw)": 0.9914590716362, "Finetune/Step": 2753, "Finetune/Step Time": 5.428952630609274} +{"Finetune/Learning Rate": 9.517392710759872e-06, "Finetune/Loss": 0.871307373046875, "Finetune/Loss (Raw)": 0.945997953414917, "Finetune/Step": 2754, "Finetune/Step Time": 5.395680200308561} +{"Finetune/Learning Rate": 9.511170453067899e-06, "Finetune/Loss": 0.8671214580535889, "Finetune/Loss (Raw)": 0.3565426766872406, "Finetune/Step": 2755, "Finetune/Step Time": 5.430771557614207} +{"Finetune/Learning Rate": 9.504948385081121e-06, "Finetune/Loss": 0.8669517636299133, "Finetune/Loss (Raw)": 0.9670883417129517, "Finetune/Step": 2756, "Finetune/Step Time": 5.446966912597418} +{"Finetune/Learning Rate": 9.498726509214216e-06, "Finetune/Loss": 0.8681061267852783, "Finetune/Loss (Raw)": 0.7857740521430969, "Finetune/Step": 2757, "Finetune/Step Time": 5.46353517845273} +{"Finetune/Learning Rate": 9.492504827881761e-06, "Finetune/Loss": 0.8691962957382202, "Finetune/Loss (Raw)": 0.9499322772026062, "Finetune/Step": 2758, "Finetune/Step Time": 5.4635377284139395} +{"Finetune/Learning Rate": 9.486283343498277e-06, "Finetune/Loss": 0.8675734400749207, "Finetune/Loss (Raw)": 0.8358345031738281, "Finetune/Step": 2759, "Finetune/Step Time": 5.464240945875645} +{"Finetune/Learning Rate": 9.480062058478201e-06, "Finetune/Loss": 0.867064356803894, "Finetune/Loss (Raw)": 0.7400386333465576, "Finetune/Step": 2760, "Finetune/Step Time": 5.458289036527276} +{"Finetune/Learning Rate": 9.473840975235883e-06, "Finetune/Loss": 0.86289381980896, "Finetune/Loss (Raw)": 0.4190537929534912, "Finetune/Step": 2761, "Finetune/Step Time": 5.490046316757798} +{"Finetune/Learning Rate": 9.467620096185616e-06, "Finetune/Loss": 0.8618978261947632, "Finetune/Loss (Raw)": 0.8437452912330627, "Finetune/Step": 2762, "Finetune/Step Time": 5.499079683795571} +{"Finetune/Learning Rate": 9.46139942374159e-06, "Finetune/Loss": 0.8615753650665283, "Finetune/Loss (Raw)": 0.8898739814758301, "Finetune/Step": 2763, "Finetune/Step Time": 5.502387139946222} +{"Finetune/Learning Rate": 9.455178960317936e-06, "Finetune/Loss": 0.8605461716651917, "Finetune/Loss (Raw)": 0.8840419054031372, "Finetune/Step": 2764, "Finetune/Step Time": 5.502968981862068} +{"Finetune/Learning Rate": 9.448958708328694e-06, "Finetune/Loss": 0.8567541837692261, "Finetune/Loss (Raw)": 0.3518276512622833, "Finetune/Step": 2765, "Finetune/Step Time": 5.528140453621745} +{"Finetune/Learning Rate": 9.442738670187814e-06, "Finetune/Loss": 0.8558845520019531, "Finetune/Loss (Raw)": 0.8693252801895142, "Finetune/Step": 2766, "Finetune/Step Time": 5.513569183647633} +{"Finetune/Learning Rate": 9.436518848309178e-06, "Finetune/Loss": 0.8571947813034058, "Finetune/Loss (Raw)": 1.0042030811309814, "Finetune/Step": 2767, "Finetune/Step Time": 5.5265275333076715} +{"Finetune/Learning Rate": 9.430299245106574e-06, "Finetune/Loss": 0.8619772791862488, "Finetune/Loss (Raw)": 0.9585592150688171, "Finetune/Step": 2768, "Finetune/Step Time": 5.496960686519742} +{"Finetune/Learning Rate": 9.424079862993712e-06, "Finetune/Loss": 0.8621436357498169, "Finetune/Loss (Raw)": 0.9352310299873352, "Finetune/Step": 2769, "Finetune/Step Time": 5.505540985614061} +{"Finetune/Learning Rate": 9.417860704384214e-06, "Finetune/Loss": 0.8604187369346619, "Finetune/Loss (Raw)": 0.6809027194976807, "Finetune/Step": 2770, "Finetune/Step Time": 5.52229218930006} +{"Finetune/Learning Rate": 9.411641771691608e-06, "Finetune/Loss": 0.8617391586303711, "Finetune/Loss (Raw)": 0.9078627228736877, "Finetune/Step": 2771, "Finetune/Step Time": 5.5150740798562765} +{"Finetune/Learning Rate": 9.405423067329344e-06, "Finetune/Loss": 0.8629024028778076, "Finetune/Loss (Raw)": 0.9233253002166748, "Finetune/Step": 2772, "Finetune/Step Time": 5.484080670401454} +{"Finetune/Learning Rate": 9.399204593710777e-06, "Finetune/Loss": 0.8629460334777832, "Finetune/Loss (Raw)": 0.9568629860877991, "Finetune/Step": 2773, "Finetune/Step Time": 5.476804109290242} +{"Finetune/Learning Rate": 9.392986353249183e-06, "Finetune/Loss": 0.8580414652824402, "Finetune/Loss (Raw)": 0.33100655674934387, "Finetune/Step": 2774, "Finetune/Step Time": 5.507564386352897} +{"Finetune/Learning Rate": 9.386768348357735e-06, "Finetune/Loss": 0.8588529825210571, "Finetune/Loss (Raw)": 0.9270807504653931, "Finetune/Step": 2775, "Finetune/Step Time": 5.497193496674299} +{"Finetune/Learning Rate": 9.380550581449512e-06, "Finetune/Loss": 0.8576551675796509, "Finetune/Loss (Raw)": 0.8123382925987244, "Finetune/Step": 2776, "Finetune/Step Time": 5.489874970167875} +{"Finetune/Learning Rate": 9.37433305493752e-06, "Finetune/Loss": 0.8576695322990417, "Finetune/Loss (Raw)": 0.8547841310501099, "Finetune/Step": 2777, "Finetune/Step Time": 5.4719343315809965} +{"Finetune/Learning Rate": 9.368115771234651e-06, "Finetune/Loss": 0.8583506941795349, "Finetune/Loss (Raw)": 0.8982671499252319, "Finetune/Step": 2778, "Finetune/Step Time": 5.472630919888616} +{"Finetune/Learning Rate": 9.361898732753715e-06, "Finetune/Loss": 0.8582711815834045, "Finetune/Loss (Raw)": 0.8463379740715027, "Finetune/Step": 2779, "Finetune/Step Time": 5.478363502770662} +{"Finetune/Learning Rate": 9.355681941907428e-06, "Finetune/Loss": 0.8572038412094116, "Finetune/Loss (Raw)": 0.8887419104576111, "Finetune/Step": 2780, "Finetune/Step Time": 5.477166863158345} +{"Finetune/Learning Rate": 9.349465401108393e-06, "Finetune/Loss": 0.8580610752105713, "Finetune/Loss (Raw)": 1.0026973485946655, "Finetune/Step": 2781, "Finetune/Step Time": 5.461578458547592} +{"Finetune/Learning Rate": 9.34324911276914e-06, "Finetune/Loss": 0.862196147441864, "Finetune/Loss (Raw)": 0.8691944479942322, "Finetune/Step": 2782, "Finetune/Step Time": 5.438201371580362} +{"Finetune/Learning Rate": 9.337033079302082e-06, "Finetune/Loss": 0.8631119728088379, "Finetune/Loss (Raw)": 1.057860016822815, "Finetune/Step": 2783, "Finetune/Step Time": 5.442053502425551} +{"Finetune/Learning Rate": 9.330817303119545e-06, "Finetune/Loss": 0.8616706728935242, "Finetune/Loss (Raw)": 0.7179003953933716, "Finetune/Step": 2784, "Finetune/Step Time": 5.454217152670026} +{"Finetune/Learning Rate": 9.324601786633747e-06, "Finetune/Loss": 0.8628355860710144, "Finetune/Loss (Raw)": 0.8783640265464783, "Finetune/Step": 2785, "Finetune/Step Time": 5.448871962726116} +{"Finetune/Learning Rate": 9.318386532256807e-06, "Finetune/Loss": 0.8648824095726013, "Finetune/Loss (Raw)": 0.9674884676933289, "Finetune/Step": 2786, "Finetune/Step Time": 5.426030898466706} +{"Finetune/Learning Rate": 9.312171542400749e-06, "Finetune/Loss": 0.8635001182556152, "Finetune/Loss (Raw)": 0.8326829075813293, "Finetune/Step": 2787, "Finetune/Step Time": 5.426574196666479} +{"Finetune/Learning Rate": 9.305956819477485e-06, "Finetune/Loss": 0.8634848594665527, "Finetune/Loss (Raw)": 0.9398289918899536, "Finetune/Step": 2788, "Finetune/Step Time": 5.436458896845579} +{"Finetune/Learning Rate": 9.299742365898831e-06, "Finetune/Loss": 0.861538827419281, "Finetune/Loss (Raw)": 0.8664818406105042, "Finetune/Step": 2789, "Finetune/Step Time": 5.4496538527309895} +{"Finetune/Learning Rate": 9.293528184076495e-06, "Finetune/Loss": 0.8608250617980957, "Finetune/Loss (Raw)": 0.9253811240196228, "Finetune/Step": 2790, "Finetune/Step Time": 5.44678289629519} +{"Finetune/Learning Rate": 9.287314276422074e-06, "Finetune/Loss": 0.8590071201324463, "Finetune/Loss (Raw)": 0.7528618574142456, "Finetune/Step": 2791, "Finetune/Step Time": 5.45680682733655} +{"Finetune/Learning Rate": 9.28110064534707e-06, "Finetune/Loss": 0.8633167743682861, "Finetune/Loss (Raw)": 0.9884175658226013, "Finetune/Step": 2792, "Finetune/Step Time": 5.440093209967017} +{"Finetune/Learning Rate": 9.274887293262869e-06, "Finetune/Loss": 0.8649621605873108, "Finetune/Loss (Raw)": 0.9955406785011292, "Finetune/Step": 2793, "Finetune/Step Time": 5.432761024683714} +{"Finetune/Learning Rate": 9.268674222580757e-06, "Finetune/Loss": 0.8638516664505005, "Finetune/Loss (Raw)": 0.8339377045631409, "Finetune/Step": 2794, "Finetune/Step Time": 5.435644943267107} +{"Finetune/Learning Rate": 9.262461435711898e-06, "Finetune/Loss": 0.8648185133934021, "Finetune/Loss (Raw)": 0.9226809144020081, "Finetune/Step": 2795, "Finetune/Step Time": 5.446632727980614} +{"Finetune/Learning Rate": 9.256248935067356e-06, "Finetune/Loss": 0.8648009300231934, "Finetune/Loss (Raw)": 0.8563904166221619, "Finetune/Step": 2796, "Finetune/Step Time": 5.463352799415588} +{"Finetune/Learning Rate": 9.250036723058083e-06, "Finetune/Loss": 0.863004207611084, "Finetune/Loss (Raw)": 0.7772553563117981, "Finetune/Step": 2797, "Finetune/Step Time": 5.48102417960763} +{"Finetune/Learning Rate": 9.243824802094914e-06, "Finetune/Loss": 0.8587549924850464, "Finetune/Loss (Raw)": 0.3810895085334778, "Finetune/Step": 2798, "Finetune/Step Time": 5.516558004543185} +{"Finetune/Learning Rate": 9.237613174588583e-06, "Finetune/Loss": 0.8584641218185425, "Finetune/Loss (Raw)": 0.7462413907051086, "Finetune/Step": 2799, "Finetune/Step Time": 5.519123295322061} +{"Finetune/Learning Rate": 9.231401842949688e-06, "Finetune/Loss": 0.8588775396347046, "Finetune/Loss (Raw)": 0.744256317615509, "Finetune/Step": 2800, "Finetune/Step Time": 5.5133919809013605} +{"Finetune/Learning Rate": 9.225190809588731e-06, "Finetune/Loss": 0.8598331212997437, "Finetune/Loss (Raw)": 0.9880446195602417, "Finetune/Step": 2801, "Finetune/Step Time": 5.516465991735458} +{"Finetune/Learning Rate": 9.218980076916099e-06, "Finetune/Loss": 0.860933780670166, "Finetune/Loss (Raw)": 0.982353150844574, "Finetune/Step": 2802, "Finetune/Step Time": 5.510797617956996} +{"Finetune/Learning Rate": 9.212769647342046e-06, "Finetune/Loss": 0.8613830804824829, "Finetune/Loss (Raw)": 0.9212458729743958, "Finetune/Step": 2803, "Finetune/Step Time": 5.5096622090786695} +{"Finetune/Learning Rate": 9.206559523276731e-06, "Finetune/Loss": 0.8620315790176392, "Finetune/Loss (Raw)": 0.9237645864486694, "Finetune/Step": 2804, "Finetune/Step Time": 5.5039627850055695} +{"Finetune/Learning Rate": 9.200349707130168e-06, "Finetune/Loss": 0.8623087406158447, "Finetune/Loss (Raw)": 1.0294373035430908, "Finetune/Step": 2805, "Finetune/Step Time": 5.502647338435054} +{"Finetune/Learning Rate": 9.194140201312274e-06, "Finetune/Loss": 0.862047553062439, "Finetune/Loss (Raw)": 0.8444538712501526, "Finetune/Step": 2806, "Finetune/Step Time": 5.5092442855238914} +{"Finetune/Learning Rate": 9.187931008232832e-06, "Finetune/Loss": 0.8606435060501099, "Finetune/Loss (Raw)": 0.8486624956130981, "Finetune/Step": 2807, "Finetune/Step Time": 5.513045048341155} +{"Finetune/Learning Rate": 9.181722130301512e-06, "Finetune/Loss": 0.8594604730606079, "Finetune/Loss (Raw)": 0.7709395289421082, "Finetune/Step": 2808, "Finetune/Step Time": 5.517879720777273} +{"Finetune/Learning Rate": 9.175513569927865e-06, "Finetune/Loss": 0.8595348596572876, "Finetune/Loss (Raw)": 0.3507820665836334, "Finetune/Step": 2809, "Finetune/Step Time": 5.518301444128156} +{"Finetune/Learning Rate": 9.169305329521296e-06, "Finetune/Loss": 0.8584607839584351, "Finetune/Loss (Raw)": 0.7635048627853394, "Finetune/Step": 2810, "Finetune/Step Time": 5.516615070402622} +{"Finetune/Learning Rate": 9.163097411491115e-06, "Finetune/Loss": 0.8585394620895386, "Finetune/Loss (Raw)": 0.9228110909461975, "Finetune/Step": 2811, "Finetune/Step Time": 5.525702940300107} +{"Finetune/Learning Rate": 9.156889818246485e-06, "Finetune/Loss": 0.8585515022277832, "Finetune/Loss (Raw)": 0.8894789814949036, "Finetune/Step": 2812, "Finetune/Step Time": 5.524963458999991} +{"Finetune/Learning Rate": 9.150682552196461e-06, "Finetune/Loss": 0.8593035340309143, "Finetune/Loss (Raw)": 0.9999094605445862, "Finetune/Step": 2813, "Finetune/Step Time": 5.52350995875895} +{"Finetune/Learning Rate": 9.144475615749957e-06, "Finetune/Loss": 0.8557924628257751, "Finetune/Loss (Raw)": 0.3518502414226532, "Finetune/Step": 2814, "Finetune/Step Time": 5.55372853949666} +{"Finetune/Learning Rate": 9.138269011315761e-06, "Finetune/Loss": 0.8567677736282349, "Finetune/Loss (Raw)": 1.0260957479476929, "Finetune/Step": 2815, "Finetune/Step Time": 5.555574199184775} +{"Finetune/Learning Rate": 9.13206274130254e-06, "Finetune/Loss": 0.8567665815353394, "Finetune/Loss (Raw)": 0.9342612028121948, "Finetune/Step": 2816, "Finetune/Step Time": 5.553872969001532} +{"Finetune/Learning Rate": 9.125856808118822e-06, "Finetune/Loss": 0.8554717302322388, "Finetune/Loss (Raw)": 0.8650408387184143, "Finetune/Step": 2817, "Finetune/Step Time": 5.560938134789467} +{"Finetune/Learning Rate": 9.119651214173012e-06, "Finetune/Loss": 0.8591946363449097, "Finetune/Loss (Raw)": 0.8454263210296631, "Finetune/Step": 2818, "Finetune/Step Time": 5.5345160365104675} +{"Finetune/Learning Rate": 9.11344596187338e-06, "Finetune/Loss": 0.8590549230575562, "Finetune/Loss (Raw)": 1.0385476350784302, "Finetune/Step": 2819, "Finetune/Step Time": 5.537895468994975} +{"Finetune/Learning Rate": 9.107241053628058e-06, "Finetune/Loss": 0.8603087663650513, "Finetune/Loss (Raw)": 1.0181529521942139, "Finetune/Step": 2820, "Finetune/Step Time": 5.537660151720047} +{"Finetune/Learning Rate": 9.101036491845056e-06, "Finetune/Loss": 0.8584473729133606, "Finetune/Loss (Raw)": 0.7218695878982544, "Finetune/Step": 2821, "Finetune/Step Time": 5.550646673887968} +{"Finetune/Learning Rate": 9.094832278932238e-06, "Finetune/Loss": 0.8585556745529175, "Finetune/Loss (Raw)": 0.9162580370903015, "Finetune/Step": 2822, "Finetune/Step Time": 5.546107841655612} +{"Finetune/Learning Rate": 9.088628417297346e-06, "Finetune/Loss": 0.8550521731376648, "Finetune/Loss (Raw)": 0.4024750292301178, "Finetune/Step": 2823, "Finetune/Step Time": 5.571321167051792} +{"Finetune/Learning Rate": 9.082424909347969e-06, "Finetune/Loss": 0.8538050651550293, "Finetune/Loss (Raw)": 0.8129385113716125, "Finetune/Step": 2824, "Finetune/Step Time": 5.557252077385783} +{"Finetune/Learning Rate": 9.076221757491568e-06, "Finetune/Loss": 0.8533066511154175, "Finetune/Loss (Raw)": 0.8069103956222534, "Finetune/Step": 2825, "Finetune/Step Time": 5.55586714297533} +{"Finetune/Learning Rate": 9.070018964135471e-06, "Finetune/Loss": 0.8529996871948242, "Finetune/Loss (Raw)": 0.7858483791351318, "Finetune/Step": 2826, "Finetune/Step Time": 5.558366557583213} +{"Finetune/Learning Rate": 9.063816531686855e-06, "Finetune/Loss": 0.8524911999702454, "Finetune/Loss (Raw)": 0.9472388029098511, "Finetune/Step": 2827, "Finetune/Step Time": 5.540871825069189} +{"Finetune/Learning Rate": 9.057614462552769e-06, "Finetune/Loss": 0.853950023651123, "Finetune/Loss (Raw)": 1.0516740083694458, "Finetune/Step": 2828, "Finetune/Step Time": 5.540454996749759} +{"Finetune/Learning Rate": 9.05141275914011e-06, "Finetune/Loss": 0.8543708324432373, "Finetune/Loss (Raw)": 0.937169075012207, "Finetune/Step": 2829, "Finetune/Step Time": 5.523579450324178} +{"Finetune/Learning Rate": 9.045211423855637e-06, "Finetune/Loss": 0.8546510934829712, "Finetune/Loss (Raw)": 0.9681785702705383, "Finetune/Step": 2830, "Finetune/Step Time": 5.515695529058576} +{"Finetune/Learning Rate": 9.039010459105974e-06, "Finetune/Loss": 0.8564432859420776, "Finetune/Loss (Raw)": 1.0600481033325195, "Finetune/Step": 2831, "Finetune/Step Time": 5.499019647017121} +{"Finetune/Learning Rate": 9.032809867297586e-06, "Finetune/Loss": 0.858047366142273, "Finetune/Loss (Raw)": 1.0050002336502075, "Finetune/Step": 2832, "Finetune/Step Time": 5.494361910969019} +{"Finetune/Learning Rate": 9.02660965083681e-06, "Finetune/Loss": 0.8581345081329346, "Finetune/Loss (Raw)": 0.8879000544548035, "Finetune/Step": 2833, "Finetune/Step Time": 5.500730488449335} +{"Finetune/Learning Rate": 9.020409812129824e-06, "Finetune/Loss": 0.8532440662384033, "Finetune/Loss (Raw)": 0.3553023338317871, "Finetune/Step": 2834, "Finetune/Step Time": 5.524882802739739} +{"Finetune/Learning Rate": 9.014210353582662e-06, "Finetune/Loss": 0.8526728749275208, "Finetune/Loss (Raw)": 0.930645227432251, "Finetune/Step": 2835, "Finetune/Step Time": 5.527607237920165} +{"Finetune/Learning Rate": 9.008011277601217e-06, "Finetune/Loss": 0.8505977392196655, "Finetune/Loss (Raw)": 0.7497850656509399, "Finetune/Step": 2836, "Finetune/Step Time": 5.519832946360111} +{"Finetune/Learning Rate": 9.001812586591228e-06, "Finetune/Loss": 0.8513367176055908, "Finetune/Loss (Raw)": 0.9623510241508484, "Finetune/Step": 2837, "Finetune/Step Time": 5.508616648614407} +{"Finetune/Learning Rate": 8.995614282958289e-06, "Finetune/Loss": 0.8549365401268005, "Finetune/Loss (Raw)": 0.8392849564552307, "Finetune/Step": 2838, "Finetune/Step Time": 5.472409643232822} +{"Finetune/Learning Rate": 8.989416369107837e-06, "Finetune/Loss": 0.8539857864379883, "Finetune/Loss (Raw)": 0.8459386825561523, "Finetune/Step": 2839, "Finetune/Step Time": 5.478038070723414} +{"Finetune/Learning Rate": 8.983218847445157e-06, "Finetune/Loss": 0.8540443181991577, "Finetune/Loss (Raw)": 0.9427237510681152, "Finetune/Step": 2840, "Finetune/Step Time": 5.461760502308607} +{"Finetune/Learning Rate": 8.977021720375395e-06, "Finetune/Loss": 0.855548620223999, "Finetune/Loss (Raw)": 1.0748130083084106, "Finetune/Step": 2841, "Finetune/Step Time": 5.456087814643979} +{"Finetune/Learning Rate": 8.970824990303532e-06, "Finetune/Loss": 0.8551987409591675, "Finetune/Loss (Raw)": 0.9103739261627197, "Finetune/Step": 2842, "Finetune/Step Time": 5.45825762860477} +{"Finetune/Learning Rate": 8.9646286596344e-06, "Finetune/Loss": 0.8546009063720703, "Finetune/Loss (Raw)": 0.7548933029174805, "Finetune/Step": 2843, "Finetune/Step Time": 5.463532557711005} +{"Finetune/Learning Rate": 8.958432730772667e-06, "Finetune/Loss": 0.8549371957778931, "Finetune/Loss (Raw)": 0.9043083190917969, "Finetune/Step": 2844, "Finetune/Step Time": 5.464265210554004} +{"Finetune/Learning Rate": 8.952237206122855e-06, "Finetune/Loss": 0.8553912043571472, "Finetune/Loss (Raw)": 0.9640991687774658, "Finetune/Step": 2845, "Finetune/Step Time": 5.475320562720299} +{"Finetune/Learning Rate": 8.946042088089331e-06, "Finetune/Loss": 0.8553239107131958, "Finetune/Loss (Raw)": 0.8592541217803955, "Finetune/Step": 2846, "Finetune/Step Time": 5.473299652338028} +{"Finetune/Learning Rate": 8.939847379076293e-06, "Finetune/Loss": 0.8533790707588196, "Finetune/Loss (Raw)": 0.6910598874092102, "Finetune/Step": 2847, "Finetune/Step Time": 5.485220650210977} +{"Finetune/Learning Rate": 8.933653081487794e-06, "Finetune/Loss": 0.8535950183868408, "Finetune/Loss (Raw)": 0.9776262640953064, "Finetune/Step": 2848, "Finetune/Step Time": 5.49187932908535} +{"Finetune/Learning Rate": 8.927459197727712e-06, "Finetune/Loss": 0.8536174297332764, "Finetune/Loss (Raw)": 0.8661863207817078, "Finetune/Step": 2849, "Finetune/Step Time": 5.48284712806344} +{"Finetune/Learning Rate": 8.921265730199775e-06, "Finetune/Loss": 0.853554368019104, "Finetune/Loss (Raw)": 0.8787750005722046, "Finetune/Step": 2850, "Finetune/Step Time": 5.477971643209457} +{"Finetune/Learning Rate": 8.91507268130755e-06, "Finetune/Loss": 0.8542431592941284, "Finetune/Loss (Raw)": 0.9414790272712708, "Finetune/Step": 2851, "Finetune/Step Time": 5.4692133739590645} +{"Finetune/Learning Rate": 8.908880053454432e-06, "Finetune/Loss": 0.8547627925872803, "Finetune/Loss (Raw)": 0.9943177103996277, "Finetune/Step": 2852, "Finetune/Step Time": 5.46961534768343} +{"Finetune/Learning Rate": 8.90268784904367e-06, "Finetune/Loss": 0.8544126152992249, "Finetune/Loss (Raw)": 0.8795421719551086, "Finetune/Step": 2853, "Finetune/Step Time": 5.46971195936203} +{"Finetune/Learning Rate": 8.896496070478326e-06, "Finetune/Loss": 0.8546502590179443, "Finetune/Loss (Raw)": 0.9509372115135193, "Finetune/Step": 2854, "Finetune/Step Time": 5.467630136758089} +{"Finetune/Learning Rate": 8.890304720161311e-06, "Finetune/Loss": 0.855161190032959, "Finetune/Loss (Raw)": 0.9517609477043152, "Finetune/Step": 2855, "Finetune/Step Time": 5.472606848925352} +{"Finetune/Learning Rate": 8.884113800495372e-06, "Finetune/Loss": 0.8564666509628296, "Finetune/Loss (Raw)": 0.9646463394165039, "Finetune/Step": 2856, "Finetune/Step Time": 5.491464748978615} +{"Finetune/Learning Rate": 8.87792331388308e-06, "Finetune/Loss": 0.8608434796333313, "Finetune/Loss (Raw)": 0.9808728694915771, "Finetune/Step": 2857, "Finetune/Step Time": 5.45784373767674} +{"Finetune/Learning Rate": 8.871733262726846e-06, "Finetune/Loss": 0.8611148595809937, "Finetune/Loss (Raw)": 0.8205743432044983, "Finetune/Step": 2858, "Finetune/Step Time": 5.451367497444153} +{"Finetune/Learning Rate": 8.865543649428903e-06, "Finetune/Loss": 0.8611021637916565, "Finetune/Loss (Raw)": 0.9075026512145996, "Finetune/Step": 2859, "Finetune/Step Time": 5.4524860717356205} +{"Finetune/Learning Rate": 8.859354476391319e-06, "Finetune/Loss": 0.8600020408630371, "Finetune/Loss (Raw)": 0.8010828495025635, "Finetune/Step": 2860, "Finetune/Step Time": 5.4628134444355965} +{"Finetune/Learning Rate": 8.853165746015997e-06, "Finetune/Loss": 0.8611117005348206, "Finetune/Loss (Raw)": 1.0332036018371582, "Finetune/Step": 2861, "Finetune/Step Time": 5.462540505453944} +{"Finetune/Learning Rate": 8.846977460704656e-06, "Finetune/Loss": 0.860058069229126, "Finetune/Loss (Raw)": 0.8230188488960266, "Finetune/Step": 2862, "Finetune/Step Time": 5.479068830609322} +{"Finetune/Learning Rate": 8.840789622858856e-06, "Finetune/Loss": 0.8597451448440552, "Finetune/Loss (Raw)": 0.9327962398529053, "Finetune/Step": 2863, "Finetune/Step Time": 5.4764198530465364} +{"Finetune/Learning Rate": 8.834602234879968e-06, "Finetune/Loss": 0.8628369569778442, "Finetune/Loss (Raw)": 0.8531681895256042, "Finetune/Step": 2864, "Finetune/Step Time": 5.449588391929865} +{"Finetune/Learning Rate": 8.8284152991692e-06, "Finetune/Loss": 0.8658180236816406, "Finetune/Loss (Raw)": 1.0044227838516235, "Finetune/Step": 2865, "Finetune/Step Time": 5.436733925715089} +{"Finetune/Learning Rate": 8.822228818127584e-06, "Finetune/Loss": 0.8616855144500732, "Finetune/Loss (Raw)": 0.39328914880752563, "Finetune/Step": 2866, "Finetune/Step Time": 5.468968745321035} +{"Finetune/Learning Rate": 8.81604279415597e-06, "Finetune/Loss": 0.8614671230316162, "Finetune/Loss (Raw)": 0.9118067026138306, "Finetune/Step": 2867, "Finetune/Step Time": 5.466411452740431} +{"Finetune/Learning Rate": 8.809857229655029e-06, "Finetune/Loss": 0.8629182577133179, "Finetune/Loss (Raw)": 1.0306956768035889, "Finetune/Step": 2868, "Finetune/Step Time": 5.461031749844551} +{"Finetune/Learning Rate": 8.803672127025264e-06, "Finetune/Loss": 0.8607710599899292, "Finetune/Loss (Raw)": 0.7278860211372375, "Finetune/Step": 2869, "Finetune/Step Time": 5.492212044075131} +{"Finetune/Learning Rate": 8.797487488666986e-06, "Finetune/Loss": 0.8566137552261353, "Finetune/Loss (Raw)": 0.37750744819641113, "Finetune/Step": 2870, "Finetune/Step Time": 5.522228952497244} +{"Finetune/Learning Rate": 8.791303316980342e-06, "Finetune/Loss": 0.8570023775100708, "Finetune/Loss (Raw)": 1.017066478729248, "Finetune/Step": 2871, "Finetune/Step Time": 5.520208997651935} +{"Finetune/Learning Rate": 8.785119614365282e-06, "Finetune/Loss": 0.8574190139770508, "Finetune/Loss (Raw)": 0.9888899922370911, "Finetune/Step": 2872, "Finetune/Step Time": 5.520455006510019} +{"Finetune/Learning Rate": 8.77893638322158e-06, "Finetune/Loss": 0.8566670417785645, "Finetune/Loss (Raw)": 0.8907002806663513, "Finetune/Step": 2873, "Finetune/Step Time": 5.522209834307432} +{"Finetune/Learning Rate": 8.772753625948828e-06, "Finetune/Loss": 0.8523455858230591, "Finetune/Loss (Raw)": 0.36054256558418274, "Finetune/Step": 2874, "Finetune/Step Time": 5.539766859263182} +{"Finetune/Learning Rate": 8.766571344946433e-06, "Finetune/Loss": 0.8532847762107849, "Finetune/Loss (Raw)": 1.0103585720062256, "Finetune/Step": 2875, "Finetune/Step Time": 5.525209592655301} +{"Finetune/Learning Rate": 8.760389542613619e-06, "Finetune/Loss": 0.852200984954834, "Finetune/Loss (Raw)": 0.8838087320327759, "Finetune/Step": 2876, "Finetune/Step Time": 5.533653922379017} +{"Finetune/Learning Rate": 8.754208221349427e-06, "Finetune/Loss": 0.8513909578323364, "Finetune/Loss (Raw)": 0.846136748790741, "Finetune/Step": 2877, "Finetune/Step Time": 5.536709176376462} +{"Finetune/Learning Rate": 8.748027383552696e-06, "Finetune/Loss": 0.8520479798316956, "Finetune/Loss (Raw)": 0.9497979283332825, "Finetune/Step": 2878, "Finetune/Step Time": 5.541691105812788} +{"Finetune/Learning Rate": 8.741847031622096e-06, "Finetune/Loss": 0.8508592844009399, "Finetune/Loss (Raw)": 0.7894073128700256, "Finetune/Step": 2879, "Finetune/Step Time": 5.543132433667779} +{"Finetune/Learning Rate": 8.7356671679561e-06, "Finetune/Loss": 0.8511014580726624, "Finetune/Loss (Raw)": 0.985480010509491, "Finetune/Step": 2880, "Finetune/Step Time": 5.534988567233086} +{"Finetune/Learning Rate": 8.729487794952992e-06, "Finetune/Loss": 0.8510304093360901, "Finetune/Loss (Raw)": 0.9823683500289917, "Finetune/Step": 2881, "Finetune/Step Time": 5.539543453603983} +{"Finetune/Learning Rate": 8.72330891501087e-06, "Finetune/Loss": 0.8514255881309509, "Finetune/Loss (Raw)": 0.996584415435791, "Finetune/Step": 2882, "Finetune/Step Time": 5.540854411199689} +{"Finetune/Learning Rate": 8.717130530527626e-06, "Finetune/Loss": 0.8565270900726318, "Finetune/Loss (Raw)": 1.009535789489746, "Finetune/Step": 2883, "Finetune/Step Time": 5.512110052630305} +{"Finetune/Learning Rate": 8.710952643900983e-06, "Finetune/Loss": 0.8556997776031494, "Finetune/Loss (Raw)": 0.8611949682235718, "Finetune/Step": 2884, "Finetune/Step Time": 5.51935537904501} +{"Finetune/Learning Rate": 8.704775257528446e-06, "Finetune/Loss": 0.8569218516349792, "Finetune/Loss (Raw)": 0.9421927332878113, "Finetune/Step": 2885, "Finetune/Step Time": 5.487025868147612} +{"Finetune/Learning Rate": 8.69859837380735e-06, "Finetune/Loss": 0.8549923896789551, "Finetune/Loss (Raw)": 0.7029632925987244, "Finetune/Step": 2886, "Finetune/Step Time": 5.510302986949682} +{"Finetune/Learning Rate": 8.692421995134819e-06, "Finetune/Loss": 0.854656994342804, "Finetune/Loss (Raw)": 0.7929014563560486, "Finetune/Step": 2887, "Finetune/Step Time": 5.5150781981647015} +{"Finetune/Learning Rate": 8.686246123907778e-06, "Finetune/Loss": 0.8571850061416626, "Finetune/Loss (Raw)": 1.063620924949646, "Finetune/Step": 2888, "Finetune/Step Time": 5.522786444053054} +{"Finetune/Learning Rate": 8.680070762522969e-06, "Finetune/Loss": 0.8564914464950562, "Finetune/Loss (Raw)": 0.33027979731559753, "Finetune/Step": 2889, "Finetune/Step Time": 5.521421439945698} +{"Finetune/Learning Rate": 8.673895913376928e-06, "Finetune/Loss": 0.8562443256378174, "Finetune/Loss (Raw)": 0.8121196031570435, "Finetune/Step": 2890, "Finetune/Step Time": 5.5102354269474745} +{"Finetune/Learning Rate": 8.667721578865993e-06, "Finetune/Loss": 0.8563438653945923, "Finetune/Loss (Raw)": 0.9026102423667908, "Finetune/Step": 2891, "Finetune/Step Time": 5.509114084765315} +{"Finetune/Learning Rate": 8.661547761386307e-06, "Finetune/Loss": 0.8580031991004944, "Finetune/Loss (Raw)": 1.0964395999908447, "Finetune/Step": 2892, "Finetune/Step Time": 5.518198046833277} +{"Finetune/Learning Rate": 8.655374463333797e-06, "Finetune/Loss": 0.8621829748153687, "Finetune/Loss (Raw)": 0.8868337273597717, "Finetune/Step": 2893, "Finetune/Step Time": 5.484624741598964} +{"Finetune/Learning Rate": 8.649201687104209e-06, "Finetune/Loss": 0.8633597493171692, "Finetune/Loss (Raw)": 1.0199579000473022, "Finetune/Step": 2894, "Finetune/Step Time": 5.483907265588641} +{"Finetune/Learning Rate": 8.643029435093072e-06, "Finetune/Loss": 0.8624783158302307, "Finetune/Loss (Raw)": 0.8913809657096863, "Finetune/Step": 2895, "Finetune/Step Time": 5.4706145487725735} +{"Finetune/Learning Rate": 8.63685770969572e-06, "Finetune/Loss": 0.862382173538208, "Finetune/Loss (Raw)": 0.9462440609931946, "Finetune/Step": 2896, "Finetune/Step Time": 5.472901813685894} +{"Finetune/Learning Rate": 8.630686513307282e-06, "Finetune/Loss": 0.8627758026123047, "Finetune/Loss (Raw)": 0.985622763633728, "Finetune/Step": 2897, "Finetune/Step Time": 5.467731868848205} +{"Finetune/Learning Rate": 8.62451584832267e-06, "Finetune/Loss": 0.8646190166473389, "Finetune/Loss (Raw)": 0.9168381094932556, "Finetune/Step": 2898, "Finetune/Step Time": 5.450680658221245} +{"Finetune/Learning Rate": 8.618345717136604e-06, "Finetune/Loss": 0.8643105626106262, "Finetune/Loss (Raw)": 0.8683736324310303, "Finetune/Step": 2899, "Finetune/Step Time": 5.451383356004953} +{"Finetune/Learning Rate": 8.612176122143589e-06, "Finetune/Loss": 0.8632456660270691, "Finetune/Loss (Raw)": 0.7870214581489563, "Finetune/Step": 2900, "Finetune/Step Time": 5.455035323277116} +{"Finetune/Learning Rate": 8.606007065737928e-06, "Finetune/Loss": 0.8619918823242188, "Finetune/Loss (Raw)": 0.7963774800300598, "Finetune/Step": 2901, "Finetune/Step Time": 5.464507583528757} +{"Finetune/Learning Rate": 8.599838550313714e-06, "Finetune/Loss": 0.8667393922805786, "Finetune/Loss (Raw)": 0.9386894702911377, "Finetune/Step": 2902, "Finetune/Step Time": 5.45598392188549} +{"Finetune/Learning Rate": 8.593670578264814e-06, "Finetune/Loss": 0.8670371174812317, "Finetune/Loss (Raw)": 0.9651874303817749, "Finetune/Step": 2903, "Finetune/Step Time": 5.459254270419478} +{"Finetune/Learning Rate": 8.587503151984911e-06, "Finetune/Loss": 0.8674280643463135, "Finetune/Loss (Raw)": 0.8623766899108887, "Finetune/Step": 2904, "Finetune/Step Time": 5.457127036526799} +{"Finetune/Learning Rate": 8.581336273867455e-06, "Finetune/Loss": 0.8677928447723389, "Finetune/Loss (Raw)": 0.9014748334884644, "Finetune/Step": 2905, "Finetune/Step Time": 5.458745373412967} +{"Finetune/Learning Rate": 8.575169946305699e-06, "Finetune/Loss": 0.8680744171142578, "Finetune/Loss (Raw)": 0.9343074560165405, "Finetune/Step": 2906, "Finetune/Step Time": 5.459702838212252} +{"Finetune/Learning Rate": 8.569004171692667e-06, "Finetune/Loss": 0.8681955337524414, "Finetune/Loss (Raw)": 0.8618460297584534, "Finetune/Step": 2907, "Finetune/Step Time": 5.452152026817203} +{"Finetune/Learning Rate": 8.562838952421177e-06, "Finetune/Loss": 0.8686707019805908, "Finetune/Loss (Raw)": 0.949569821357727, "Finetune/Step": 2908, "Finetune/Step Time": 5.460556838661432} +{"Finetune/Learning Rate": 8.556674290883833e-06, "Finetune/Loss": 0.8693177700042725, "Finetune/Loss (Raw)": 1.085519790649414, "Finetune/Step": 2909, "Finetune/Step Time": 5.455077463760972} +{"Finetune/Learning Rate": 8.550510189473016e-06, "Finetune/Loss": 0.8702657222747803, "Finetune/Loss (Raw)": 0.9905276298522949, "Finetune/Step": 2910, "Finetune/Step Time": 5.450617840513587} +{"Finetune/Learning Rate": 8.544346650580899e-06, "Finetune/Loss": 0.8702404499053955, "Finetune/Loss (Raw)": 1.054627776145935, "Finetune/Step": 2911, "Finetune/Step Time": 5.436549335718155} +{"Finetune/Learning Rate": 8.538183676599426e-06, "Finetune/Loss": 0.8720525503158569, "Finetune/Loss (Raw)": 0.9498483538627625, "Finetune/Step": 2912, "Finetune/Step Time": 5.428562615066767} +{"Finetune/Learning Rate": 8.532021269920326e-06, "Finetune/Loss": 0.8725489377975464, "Finetune/Loss (Raw)": 0.9419023990631104, "Finetune/Step": 2913, "Finetune/Step Time": 5.41539291664958} +{"Finetune/Learning Rate": 8.525859432935114e-06, "Finetune/Loss": 0.8705156445503235, "Finetune/Loss (Raw)": 0.7072259187698364, "Finetune/Step": 2914, "Finetune/Step Time": 5.4422381445765495} +{"Finetune/Learning Rate": 8.519698168035074e-06, "Finetune/Loss": 0.8698729276657104, "Finetune/Loss (Raw)": 0.7504181861877441, "Finetune/Step": 2915, "Finetune/Step Time": 5.4481833558529615} +{"Finetune/Learning Rate": 8.513537477611278e-06, "Finetune/Loss": 0.8697775602340698, "Finetune/Loss (Raw)": 0.9276186227798462, "Finetune/Step": 2916, "Finetune/Step Time": 5.434972306713462} +{"Finetune/Learning Rate": 8.507377364054566e-06, "Finetune/Loss": 0.8706552982330322, "Finetune/Loss (Raw)": 0.97883540391922, "Finetune/Step": 2917, "Finetune/Step Time": 5.424847334623337} +{"Finetune/Learning Rate": 8.501217829755554e-06, "Finetune/Loss": 0.8701232075691223, "Finetune/Loss (Raw)": 0.8572729825973511, "Finetune/Step": 2918, "Finetune/Step Time": 5.430249175056815} +{"Finetune/Learning Rate": 8.495058877104642e-06, "Finetune/Loss": 0.8713574409484863, "Finetune/Loss (Raw)": 0.9108448624610901, "Finetune/Step": 2919, "Finetune/Step Time": 5.408316785469651} +{"Finetune/Learning Rate": 8.488900508491999e-06, "Finetune/Loss": 0.8663402795791626, "Finetune/Loss (Raw)": 0.3462243974208832, "Finetune/Step": 2920, "Finetune/Step Time": 5.425365595147014} +{"Finetune/Learning Rate": 8.48274272630757e-06, "Finetune/Loss": 0.8655964136123657, "Finetune/Loss (Raw)": 0.9003236293792725, "Finetune/Step": 2921, "Finetune/Step Time": 5.442739624530077} +{"Finetune/Learning Rate": 8.476585532941064e-06, "Finetune/Loss": 0.8657854199409485, "Finetune/Loss (Raw)": 0.8581261038780212, "Finetune/Step": 2922, "Finetune/Step Time": 5.435728022828698} +{"Finetune/Learning Rate": 8.470428930781969e-06, "Finetune/Loss": 0.8655864596366882, "Finetune/Loss (Raw)": 0.8972163200378418, "Finetune/Step": 2923, "Finetune/Step Time": 5.462379548698664} +{"Finetune/Learning Rate": 8.464272922219547e-06, "Finetune/Loss": 0.8667619228363037, "Finetune/Loss (Raw)": 1.0068449974060059, "Finetune/Step": 2924, "Finetune/Step Time": 5.44551557302475} +{"Finetune/Learning Rate": 8.458117509642817e-06, "Finetune/Loss": 0.8686567544937134, "Finetune/Loss (Raw)": 1.0197945833206177, "Finetune/Step": 2925, "Finetune/Step Time": 5.429706510156393} +{"Finetune/Learning Rate": 8.451962695440586e-06, "Finetune/Loss": 0.8726727962493896, "Finetune/Loss (Raw)": 0.8951491713523865, "Finetune/Step": 2926, "Finetune/Step Time": 5.420681679621339} +{"Finetune/Learning Rate": 8.445808482001405e-06, "Finetune/Loss": 0.8723268508911133, "Finetune/Loss (Raw)": 0.7019569873809814, "Finetune/Step": 2927, "Finetune/Step Time": 5.4303589183837175} +{"Finetune/Learning Rate": 8.439654871713608e-06, "Finetune/Loss": 0.8735928535461426, "Finetune/Loss (Raw)": 0.9063013195991516, "Finetune/Step": 2928, "Finetune/Step Time": 5.444443287327886} +{"Finetune/Learning Rate": 8.433501866965294e-06, "Finetune/Loss": 0.8740250468254089, "Finetune/Loss (Raw)": 1.0433710813522339, "Finetune/Step": 2929, "Finetune/Step Time": 5.4403914120048285} +{"Finetune/Learning Rate": 8.42734947014432e-06, "Finetune/Loss": 0.8739780187606812, "Finetune/Loss (Raw)": 0.9763307571411133, "Finetune/Step": 2930, "Finetune/Step Time": 5.432476093992591} +{"Finetune/Learning Rate": 8.42119768363832e-06, "Finetune/Loss": 0.8741836547851562, "Finetune/Loss (Raw)": 0.947558581829071, "Finetune/Step": 2931, "Finetune/Step Time": 5.435147136449814} +{"Finetune/Learning Rate": 8.415046509834668e-06, "Finetune/Loss": 0.8743242025375366, "Finetune/Loss (Raw)": 0.9417650103569031, "Finetune/Step": 2932, "Finetune/Step Time": 5.438996143639088} +{"Finetune/Learning Rate": 8.408895951120526e-06, "Finetune/Loss": 0.8734434247016907, "Finetune/Loss (Raw)": 0.9166961908340454, "Finetune/Step": 2933, "Finetune/Step Time": 5.450315689668059} +{"Finetune/Learning Rate": 8.402746009882798e-06, "Finetune/Loss": 0.8740800619125366, "Finetune/Loss (Raw)": 0.9259479641914368, "Finetune/Step": 2934, "Finetune/Step Time": 5.443826265633106} +{"Finetune/Learning Rate": 8.39659668850816e-06, "Finetune/Loss": 0.8745574355125427, "Finetune/Loss (Raw)": 0.9097629189491272, "Finetune/Step": 2935, "Finetune/Step Time": 5.442807834595442} +{"Finetune/Learning Rate": 8.39044798938305e-06, "Finetune/Loss": 0.8765496015548706, "Finetune/Loss (Raw)": 1.0259385108947754, "Finetune/Step": 2936, "Finetune/Step Time": 5.448679676279426} +{"Finetune/Learning Rate": 8.384299914893645e-06, "Finetune/Loss": 0.8820894956588745, "Finetune/Loss (Raw)": 1.0598840713500977, "Finetune/Step": 2937, "Finetune/Step Time": 5.418512191623449} +{"Finetune/Learning Rate": 8.378152467425902e-06, "Finetune/Loss": 0.883490800857544, "Finetune/Loss (Raw)": 0.9428731799125671, "Finetune/Step": 2938, "Finetune/Step Time": 5.415014585480094} +{"Finetune/Learning Rate": 8.37200564936552e-06, "Finetune/Loss": 0.8833606243133545, "Finetune/Loss (Raw)": 0.9061434268951416, "Finetune/Step": 2939, "Finetune/Step Time": 5.413439534604549} +{"Finetune/Learning Rate": 8.365859463097963e-06, "Finetune/Loss": 0.8820830583572388, "Finetune/Loss (Raw)": 0.7259525656700134, "Finetune/Step": 2940, "Finetune/Step Time": 5.4156135227531195} +{"Finetune/Learning Rate": 8.359713911008447e-06, "Finetune/Loss": 0.8796185255050659, "Finetune/Loss (Raw)": 0.6844491362571716, "Finetune/Step": 2941, "Finetune/Step Time": 5.433679983019829} +{"Finetune/Learning Rate": 8.353568995481934e-06, "Finetune/Loss": 0.8849657773971558, "Finetune/Loss (Raw)": 1.0363057851791382, "Finetune/Step": 2942, "Finetune/Step Time": 5.406849330291152} +{"Finetune/Learning Rate": 8.347424718903152e-06, "Finetune/Loss": 0.8843238949775696, "Finetune/Loss (Raw)": 0.9439334869384766, "Finetune/Step": 2943, "Finetune/Step Time": 5.4115976318717} +{"Finetune/Learning Rate": 8.341281083656571e-06, "Finetune/Loss": 0.8842147588729858, "Finetune/Loss (Raw)": 0.9202896952629089, "Finetune/Step": 2944, "Finetune/Step Time": 5.419194012880325} +{"Finetune/Learning Rate": 8.33513809212642e-06, "Finetune/Loss": 0.883972704410553, "Finetune/Loss (Raw)": 0.8340569734573364, "Finetune/Step": 2945, "Finetune/Step Time": 5.427980482578278} +{"Finetune/Learning Rate": 8.328995746696674e-06, "Finetune/Loss": 0.8845807313919067, "Finetune/Loss (Raw)": 0.9232544302940369, "Finetune/Step": 2946, "Finetune/Step Time": 5.41940395347774} +{"Finetune/Learning Rate": 8.32285404975105e-06, "Finetune/Loss": 0.8833664059638977, "Finetune/Loss (Raw)": 0.8831088542938232, "Finetune/Step": 2947, "Finetune/Step Time": 5.441471930593252} +{"Finetune/Learning Rate": 8.316713003673028e-06, "Finetune/Loss": 0.883060872554779, "Finetune/Loss (Raw)": 0.979051947593689, "Finetune/Step": 2948, "Finetune/Step Time": 5.451914584264159} +{"Finetune/Learning Rate": 8.310572610845824e-06, "Finetune/Loss": 0.8848193883895874, "Finetune/Loss (Raw)": 0.9469521641731262, "Finetune/Step": 2949, "Finetune/Step Time": 5.441720193251967} +{"Finetune/Learning Rate": 8.304432873652412e-06, "Finetune/Loss": 0.8847785592079163, "Finetune/Loss (Raw)": 0.9110350608825684, "Finetune/Step": 2950, "Finetune/Step Time": 5.4418886210769415} +{"Finetune/Learning Rate": 8.298293794475495e-06, "Finetune/Loss": 0.884006142616272, "Finetune/Loss (Raw)": 0.3036104738712311, "Finetune/Step": 2951, "Finetune/Step Time": 5.442256832495332} +{"Finetune/Learning Rate": 8.29215537569753e-06, "Finetune/Loss": 0.8851495385169983, "Finetune/Loss (Raw)": 0.9592880606651306, "Finetune/Step": 2952, "Finetune/Step Time": 5.439843785017729} +{"Finetune/Learning Rate": 8.286017619700725e-06, "Finetune/Loss": 0.8866310119628906, "Finetune/Loss (Raw)": 0.9965357184410095, "Finetune/Step": 2953, "Finetune/Step Time": 5.439121626317501} +{"Finetune/Learning Rate": 8.279880528867017e-06, "Finetune/Loss": 0.8867310285568237, "Finetune/Loss (Raw)": 0.7986606955528259, "Finetune/Step": 2954, "Finetune/Step Time": 5.443802807480097} +{"Finetune/Learning Rate": 8.273744105578097e-06, "Finetune/Loss": 0.8876033425331116, "Finetune/Loss (Raw)": 1.0588867664337158, "Finetune/Step": 2955, "Finetune/Step Time": 5.4509326964616776} +{"Finetune/Learning Rate": 8.267608352215385e-06, "Finetune/Loss": 0.8867455720901489, "Finetune/Loss (Raw)": 0.9418818950653076, "Finetune/Step": 2956, "Finetune/Step Time": 5.448332879692316} +{"Finetune/Learning Rate": 8.261473271160046e-06, "Finetune/Loss": 0.8864868879318237, "Finetune/Loss (Raw)": 0.9040520191192627, "Finetune/Step": 2957, "Finetune/Step Time": 5.436723152175546} +{"Finetune/Learning Rate": 8.255338864792993e-06, "Finetune/Loss": 0.886067807674408, "Finetune/Loss (Raw)": 0.9145414233207703, "Finetune/Step": 2958, "Finetune/Step Time": 5.439654439687729} +{"Finetune/Learning Rate": 8.249205135494862e-06, "Finetune/Loss": 0.8802671432495117, "Finetune/Loss (Raw)": 0.31756168603897095, "Finetune/Step": 2959, "Finetune/Step Time": 5.467799711972475} +{"Finetune/Learning Rate": 8.24307208564604e-06, "Finetune/Loss": 0.879525899887085, "Finetune/Loss (Raw)": 0.9101235866546631, "Finetune/Step": 2960, "Finetune/Step Time": 5.465096866711974} +{"Finetune/Learning Rate": 8.236939717626639e-06, "Finetune/Loss": 0.879204511642456, "Finetune/Loss (Raw)": 0.846761167049408, "Finetune/Step": 2961, "Finetune/Step Time": 5.451504530385137} +{"Finetune/Learning Rate": 8.23080803381651e-06, "Finetune/Loss": 0.8848233222961426, "Finetune/Loss (Raw)": 1.0745176076889038, "Finetune/Step": 2962, "Finetune/Step Time": 5.426740476861596} +{"Finetune/Learning Rate": 8.224677036595245e-06, "Finetune/Loss": 0.8852403163909912, "Finetune/Loss (Raw)": 0.9840115308761597, "Finetune/Step": 2963, "Finetune/Step Time": 5.437260519713163} +{"Finetune/Learning Rate": 8.218546728342161e-06, "Finetune/Loss": 0.8871681094169617, "Finetune/Loss (Raw)": 0.9965453147888184, "Finetune/Step": 2964, "Finetune/Step Time": 5.440072193741798} +{"Finetune/Learning Rate": 8.212417111436315e-06, "Finetune/Loss": 0.8871784806251526, "Finetune/Loss (Raw)": 0.9636722207069397, "Finetune/Step": 2965, "Finetune/Step Time": 5.436285579577088} +{"Finetune/Learning Rate": 8.206288188256486e-06, "Finetune/Loss": 0.8870658874511719, "Finetune/Loss (Raw)": 0.8248753547668457, "Finetune/Step": 2966, "Finetune/Step Time": 5.444315105676651} +{"Finetune/Learning Rate": 8.20015996118119e-06, "Finetune/Loss": 0.8866662979125977, "Finetune/Loss (Raw)": 0.7947932481765747, "Finetune/Step": 2967, "Finetune/Step Time": 5.455739304423332} +{"Finetune/Learning Rate": 8.194032432588677e-06, "Finetune/Loss": 0.8873432874679565, "Finetune/Loss (Raw)": 1.029378056526184, "Finetune/Step": 2968, "Finetune/Step Time": 5.464063199236989} +{"Finetune/Learning Rate": 8.187905604856916e-06, "Finetune/Loss": 0.8857177495956421, "Finetune/Loss (Raw)": 0.8667494654655457, "Finetune/Step": 2969, "Finetune/Step Time": 5.460494404658675} +{"Finetune/Learning Rate": 8.181779480363614e-06, "Finetune/Loss": 0.885780394077301, "Finetune/Loss (Raw)": 0.9183900952339172, "Finetune/Step": 2970, "Finetune/Step Time": 5.472389370203018} +{"Finetune/Learning Rate": 8.175654061486197e-06, "Finetune/Loss": 0.8869156837463379, "Finetune/Loss (Raw)": 0.900205135345459, "Finetune/Step": 2971, "Finetune/Step Time": 5.479417115449905} +{"Finetune/Learning Rate": 8.16952935060182e-06, "Finetune/Loss": 0.8870861530303955, "Finetune/Loss (Raw)": 0.9261284470558167, "Finetune/Step": 2972, "Finetune/Step Time": 5.469428976997733} +{"Finetune/Learning Rate": 8.163405350087366e-06, "Finetune/Loss": 0.8857164978981018, "Finetune/Loss (Raw)": 0.7887813448905945, "Finetune/Step": 2973, "Finetune/Step Time": 5.472871016710997} +{"Finetune/Learning Rate": 8.157282062319437e-06, "Finetune/Loss": 0.8863112926483154, "Finetune/Loss (Raw)": 0.9353970289230347, "Finetune/Step": 2974, "Finetune/Step Time": 5.474035423249006} +{"Finetune/Learning Rate": 8.15115948967437e-06, "Finetune/Loss": 0.883971095085144, "Finetune/Loss (Raw)": 0.391512006521225, "Finetune/Step": 2975, "Finetune/Step Time": 5.491560412570834} +{"Finetune/Learning Rate": 8.145037634528205e-06, "Finetune/Loss": 0.8828293085098267, "Finetune/Loss (Raw)": 0.8314764499664307, "Finetune/Step": 2976, "Finetune/Step Time": 5.491096997633576} +{"Finetune/Learning Rate": 8.138916499256718e-06, "Finetune/Loss": 0.883345901966095, "Finetune/Loss (Raw)": 0.9323105812072754, "Finetune/Step": 2977, "Finetune/Step Time": 5.4885036163032055} +{"Finetune/Learning Rate": 8.132796086235405e-06, "Finetune/Loss": 0.8842451572418213, "Finetune/Loss (Raw)": 0.9938814640045166, "Finetune/Step": 2978, "Finetune/Step Time": 5.489269020035863} +{"Finetune/Learning Rate": 8.126676397839475e-06, "Finetune/Loss": 0.8839311599731445, "Finetune/Loss (Raw)": 0.901280403137207, "Finetune/Step": 2979, "Finetune/Step Time": 5.499266738072038} +{"Finetune/Learning Rate": 8.120557436443866e-06, "Finetune/Loss": 0.8827701807022095, "Finetune/Loss (Raw)": 0.8457128405570984, "Finetune/Step": 2980, "Finetune/Step Time": 5.502400813624263} +{"Finetune/Learning Rate": 8.11443920442322e-06, "Finetune/Loss": 0.8820376992225647, "Finetune/Loss (Raw)": 0.7857889533042908, "Finetune/Step": 2981, "Finetune/Step Time": 5.529159927740693} +{"Finetune/Learning Rate": 8.108321704151903e-06, "Finetune/Loss": 0.8810815811157227, "Finetune/Loss (Raw)": 0.8285607099533081, "Finetune/Step": 2982, "Finetune/Step Time": 5.522771146148443} +{"Finetune/Learning Rate": 8.102204938004004e-06, "Finetune/Loss": 0.881557822227478, "Finetune/Loss (Raw)": 1.0127156972885132, "Finetune/Step": 2983, "Finetune/Step Time": 5.512809516862035} +{"Finetune/Learning Rate": 8.096088908353316e-06, "Finetune/Loss": 0.8819528818130493, "Finetune/Loss (Raw)": 1.0152109861373901, "Finetune/Step": 2984, "Finetune/Step Time": 5.507587101310492} +{"Finetune/Learning Rate": 8.089973617573355e-06, "Finetune/Loss": 0.8811631202697754, "Finetune/Loss (Raw)": 0.8797858953475952, "Finetune/Step": 2985, "Finetune/Step Time": 5.512444349005818} +{"Finetune/Learning Rate": 8.08385906803734e-06, "Finetune/Loss": 0.8773061633110046, "Finetune/Loss (Raw)": 0.3268818259239197, "Finetune/Step": 2986, "Finetune/Step Time": 5.545071806758642} +{"Finetune/Learning Rate": 8.07774526211821e-06, "Finetune/Loss": 0.8767479658126831, "Finetune/Loss (Raw)": 0.836059033870697, "Finetune/Step": 2987, "Finetune/Step Time": 5.558249024674296} +{"Finetune/Learning Rate": 8.071632202188618e-06, "Finetune/Loss": 0.8779110908508301, "Finetune/Loss (Raw)": 0.9499558806419373, "Finetune/Step": 2988, "Finetune/Step Time": 5.55506869405508} +{"Finetune/Learning Rate": 8.065519890620922e-06, "Finetune/Loss": 0.8762716054916382, "Finetune/Loss (Raw)": 0.823348879814148, "Finetune/Step": 2989, "Finetune/Step Time": 5.559971485286951} +{"Finetune/Learning Rate": 8.059408329787184e-06, "Finetune/Loss": 0.8777755498886108, "Finetune/Loss (Raw)": 1.0155234336853027, "Finetune/Step": 2990, "Finetune/Step Time": 5.535709992051125} +{"Finetune/Learning Rate": 8.053297522059189e-06, "Finetune/Loss": 0.877959668636322, "Finetune/Loss (Raw)": 0.9563637375831604, "Finetune/Step": 2991, "Finetune/Step Time": 5.5404243022203445} +{"Finetune/Learning Rate": 8.047187469808418e-06, "Finetune/Loss": 0.8774910569190979, "Finetune/Loss (Raw)": 0.793190062046051, "Finetune/Step": 2992, "Finetune/Step Time": 5.55121561139822} +{"Finetune/Learning Rate": 8.041078175406064e-06, "Finetune/Loss": 0.8774892687797546, "Finetune/Loss (Raw)": 1.004191517829895, "Finetune/Step": 2993, "Finetune/Step Time": 5.54715047031641} +{"Finetune/Learning Rate": 8.03496964122303e-06, "Finetune/Loss": 0.8824577331542969, "Finetune/Loss (Raw)": 1.0292543172836304, "Finetune/Step": 2994, "Finetune/Step Time": 5.507717430591583} +{"Finetune/Learning Rate": 8.02886186962991e-06, "Finetune/Loss": 0.8824560642242432, "Finetune/Loss (Raw)": 0.9115886092185974, "Finetune/Step": 2995, "Finetune/Step Time": 5.511227523908019} +{"Finetune/Learning Rate": 8.022754862997016e-06, "Finetune/Loss": 0.8813071250915527, "Finetune/Loss (Raw)": 0.8836350440979004, "Finetune/Step": 2996, "Finetune/Step Time": 5.526196772232652} +{"Finetune/Learning Rate": 8.016648623694354e-06, "Finetune/Loss": 0.8821517825126648, "Finetune/Loss (Raw)": 0.8360059261322021, "Finetune/Step": 2997, "Finetune/Step Time": 5.50093249976635} +{"Finetune/Learning Rate": 8.010543154091644e-06, "Finetune/Loss": 0.8860212564468384, "Finetune/Loss (Raw)": 0.8727973699569702, "Finetune/Step": 2998, "Finetune/Step Time": 5.473475616425276} +{"Finetune/Learning Rate": 8.004438456558297e-06, "Finetune/Loss": 0.8865123391151428, "Finetune/Loss (Raw)": 1.0799214839935303, "Finetune/Step": 2999, "Finetune/Step Time": 5.485354432836175} +{"Finetune/Learning Rate": 7.99833453346342e-06, "Finetune/Loss": 0.885777473449707, "Finetune/Loss (Raw)": 0.8948269486427307, "Finetune/Step": 3000, "Finetune/Step Time": 5.489907309412956} +{"Finetune/Learning Rate": 7.992231387175832e-06, "Finetune/Loss": 0.8856186866760254, "Finetune/Loss (Raw)": 0.8703830242156982, "Finetune/Step": 3001, "Finetune/Step Time": 5.493121601641178} +{"Finetune/Learning Rate": 7.986129020064044e-06, "Finetune/Loss": 0.8901004195213318, "Finetune/Loss (Raw)": 0.9341963529586792, "Finetune/Step": 3002, "Finetune/Step Time": 5.464993042871356} +{"Finetune/Learning Rate": 7.980027434496267e-06, "Finetune/Loss": 0.8891235589981079, "Finetune/Loss (Raw)": 0.8853222131729126, "Finetune/Step": 3003, "Finetune/Step Time": 5.465967770665884} +{"Finetune/Learning Rate": 7.97392663284041e-06, "Finetune/Loss": 0.8895401358604431, "Finetune/Loss (Raw)": 0.9371334314346313, "Finetune/Step": 3004, "Finetune/Step Time": 5.452996810898185} +{"Finetune/Learning Rate": 7.967826617464065e-06, "Finetune/Loss": 0.8886256217956543, "Finetune/Loss (Raw)": 0.7290769815444946, "Finetune/Step": 3005, "Finetune/Step Time": 5.47100499458611} +{"Finetune/Learning Rate": 7.961727390734538e-06, "Finetune/Loss": 0.8885349035263062, "Finetune/Loss (Raw)": 0.9381884932518005, "Finetune/Step": 3006, "Finetune/Step Time": 5.475247165188193} +{"Finetune/Learning Rate": 7.955628955018813e-06, "Finetune/Loss": 0.8883720636367798, "Finetune/Loss (Raw)": 0.7685548663139343, "Finetune/Step": 3007, "Finetune/Step Time": 5.466058133170009} +{"Finetune/Learning Rate": 7.949531312683582e-06, "Finetune/Loss": 0.8880841732025146, "Finetune/Loss (Raw)": 0.9486411213874817, "Finetune/Step": 3008, "Finetune/Step Time": 5.467307427898049} +{"Finetune/Learning Rate": 7.943434466095216e-06, "Finetune/Loss": 0.8882070779800415, "Finetune/Loss (Raw)": 0.998092532157898, "Finetune/Step": 3009, "Finetune/Step Time": 5.460173657163978} +{"Finetune/Learning Rate": 7.937338417619781e-06, "Finetune/Loss": 0.8872263431549072, "Finetune/Loss (Raw)": 0.8710542321205139, "Finetune/Step": 3010, "Finetune/Step Time": 5.4606949128210545} +{"Finetune/Learning Rate": 7.931243169623037e-06, "Finetune/Loss": 0.8873317241668701, "Finetune/Loss (Raw)": 1.0230293273925781, "Finetune/Step": 3011, "Finetune/Step Time": 5.45793117582798} +{"Finetune/Learning Rate": 7.925148724470428e-06, "Finetune/Loss": 0.8879817128181458, "Finetune/Loss (Raw)": 0.9443846344947815, "Finetune/Step": 3012, "Finetune/Step Time": 5.429346790537238} +{"Finetune/Learning Rate": 7.919055084527094e-06, "Finetune/Loss": 0.8875877857208252, "Finetune/Loss (Raw)": 0.8917723894119263, "Finetune/Step": 3013, "Finetune/Step Time": 5.42588078789413} +{"Finetune/Learning Rate": 7.912962252157854e-06, "Finetune/Loss": 0.888764500617981, "Finetune/Loss (Raw)": 0.8535827994346619, "Finetune/Step": 3014, "Finetune/Step Time": 5.409251330420375} +{"Finetune/Learning Rate": 7.906870229727216e-06, "Finetune/Loss": 0.8885709047317505, "Finetune/Loss (Raw)": 0.7681217193603516, "Finetune/Step": 3015, "Finetune/Step Time": 5.418474204838276} +{"Finetune/Learning Rate": 7.900779019599378e-06, "Finetune/Loss": 0.8874155879020691, "Finetune/Loss (Raw)": 0.9157450795173645, "Finetune/Step": 3016, "Finetune/Step Time": 5.422171315178275} +{"Finetune/Learning Rate": 7.894688624138217e-06, "Finetune/Loss": 0.8909226655960083, "Finetune/Loss (Raw)": 0.7791786789894104, "Finetune/Step": 3017, "Finetune/Step Time": 5.396750623360276} +{"Finetune/Learning Rate": 7.8885990457073e-06, "Finetune/Loss": 0.8912543654441833, "Finetune/Loss (Raw)": 0.8545814156532288, "Finetune/Step": 3018, "Finetune/Step Time": 5.39237923361361} +{"Finetune/Learning Rate": 7.882510286669875e-06, "Finetune/Loss": 0.8915102481842041, "Finetune/Loss (Raw)": 0.9353609681129456, "Finetune/Step": 3019, "Finetune/Step Time": 5.394023265689611} +{"Finetune/Learning Rate": 7.876422349388861e-06, "Finetune/Loss": 0.8897961974143982, "Finetune/Loss (Raw)": 0.8770455121994019, "Finetune/Step": 3020, "Finetune/Step Time": 5.375808842480183} +{"Finetune/Learning Rate": 7.870335236226878e-06, "Finetune/Loss": 0.8898302316665649, "Finetune/Loss (Raw)": 0.8911885023117065, "Finetune/Step": 3021, "Finetune/Step Time": 5.3796742763370275} +{"Finetune/Learning Rate": 7.864248949546209e-06, "Finetune/Loss": 0.8894470930099487, "Finetune/Loss (Raw)": 0.9709067344665527, "Finetune/Step": 3022, "Finetune/Step Time": 5.379686668515205} +{"Finetune/Learning Rate": 7.858163491708827e-06, "Finetune/Loss": 0.888259768486023, "Finetune/Loss (Raw)": 0.739406406879425, "Finetune/Step": 3023, "Finetune/Step Time": 5.38259126804769} +{"Finetune/Learning Rate": 7.85207886507638e-06, "Finetune/Loss": 0.888108491897583, "Finetune/Loss (Raw)": 0.9268832206726074, "Finetune/Step": 3024, "Finetune/Step Time": 5.38771466165781} +{"Finetune/Learning Rate": 7.845995072010188e-06, "Finetune/Loss": 0.887700617313385, "Finetune/Loss (Raw)": 0.9334150552749634, "Finetune/Step": 3025, "Finetune/Step Time": 5.386810772120953} +{"Finetune/Learning Rate": 7.839912114871256e-06, "Finetune/Loss": 0.8878411650657654, "Finetune/Loss (Raw)": 0.9348310232162476, "Finetune/Step": 3026, "Finetune/Step Time": 5.3912500608712435} +{"Finetune/Learning Rate": 7.83382999602026e-06, "Finetune/Loss": 0.8887912034988403, "Finetune/Loss (Raw)": 0.9899767637252808, "Finetune/Step": 3027, "Finetune/Step Time": 5.389244895428419} +{"Finetune/Learning Rate": 7.827748717817552e-06, "Finetune/Loss": 0.8896682262420654, "Finetune/Loss (Raw)": 0.8992780447006226, "Finetune/Step": 3028, "Finetune/Step Time": 5.392218913882971} +{"Finetune/Learning Rate": 7.821668282623159e-06, "Finetune/Loss": 0.8916583061218262, "Finetune/Loss (Raw)": 1.051108479499817, "Finetune/Step": 3029, "Finetune/Step Time": 5.383083004504442} +{"Finetune/Learning Rate": 7.815588692796771e-06, "Finetune/Loss": 0.8912788033485413, "Finetune/Loss (Raw)": 0.8901179432868958, "Finetune/Step": 3030, "Finetune/Step Time": 5.3745667319744825} +{"Finetune/Learning Rate": 7.809509950697769e-06, "Finetune/Loss": 0.8909022808074951, "Finetune/Loss (Raw)": 0.9169933795928955, "Finetune/Step": 3031, "Finetune/Step Time": 5.371402412652969} +{"Finetune/Learning Rate": 7.803432058685187e-06, "Finetune/Loss": 0.891873836517334, "Finetune/Loss (Raw)": 0.9867404103279114, "Finetune/Step": 3032, "Finetune/Step Time": 5.375942409038544} +{"Finetune/Learning Rate": 7.797355019117743e-06, "Finetune/Loss": 0.8919513821601868, "Finetune/Loss (Raw)": 0.9113951325416565, "Finetune/Step": 3033, "Finetune/Step Time": 5.3729403261095285} +{"Finetune/Learning Rate": 7.79127883435381e-06, "Finetune/Loss": 0.8929010629653931, "Finetune/Loss (Raw)": 1.0558650493621826, "Finetune/Step": 3034, "Finetune/Step Time": 5.3690048065036535} +{"Finetune/Learning Rate": 7.78520350675144e-06, "Finetune/Loss": 0.8931946754455566, "Finetune/Loss (Raw)": 0.8994253277778625, "Finetune/Step": 3035, "Finetune/Step Time": 5.370187470689416} +{"Finetune/Learning Rate": 7.779129038668347e-06, "Finetune/Loss": 0.8927230834960938, "Finetune/Loss (Raw)": 0.8892140984535217, "Finetune/Step": 3036, "Finetune/Step Time": 5.382876757532358} +{"Finetune/Learning Rate": 7.773055432461916e-06, "Finetune/Loss": 0.8922513127326965, "Finetune/Loss (Raw)": 1.0251294374465942, "Finetune/Step": 3037, "Finetune/Step Time": 5.385225484147668} +{"Finetune/Learning Rate": 7.766982690489198e-06, "Finetune/Loss": 0.8922321796417236, "Finetune/Loss (Raw)": 0.9880717396736145, "Finetune/Step": 3038, "Finetune/Step Time": 5.384631764143705} +{"Finetune/Learning Rate": 7.7609108151069e-06, "Finetune/Loss": 0.8905940055847168, "Finetune/Loss (Raw)": 0.8449428677558899, "Finetune/Step": 3039, "Finetune/Step Time": 5.384864902123809} +{"Finetune/Learning Rate": 7.754839808671397e-06, "Finetune/Loss": 0.8904930949211121, "Finetune/Loss (Raw)": 0.9369354248046875, "Finetune/Step": 3040, "Finetune/Step Time": 5.377860859036446} +{"Finetune/Learning Rate": 7.748769673538734e-06, "Finetune/Loss": 0.8892003297805786, "Finetune/Loss (Raw)": 0.7764250636100769, "Finetune/Step": 3041, "Finetune/Step Time": 5.382968366146088} +{"Finetune/Learning Rate": 7.742700412064608e-06, "Finetune/Loss": 0.8909586668014526, "Finetune/Loss (Raw)": 0.932303249835968, "Finetune/Step": 3042, "Finetune/Step Time": 5.3584271389991045} +{"Finetune/Learning Rate": 7.736632026604386e-06, "Finetune/Loss": 0.8875046968460083, "Finetune/Loss (Raw)": 0.3083059787750244, "Finetune/Step": 3043, "Finetune/Step Time": 5.379165103659034} +{"Finetune/Learning Rate": 7.730564519513082e-06, "Finetune/Loss": 0.8859047889709473, "Finetune/Loss (Raw)": 0.7228240966796875, "Finetune/Step": 3044, "Finetune/Step Time": 5.403306586667895} +{"Finetune/Learning Rate": 7.724497893145379e-06, "Finetune/Loss": 0.8846192955970764, "Finetune/Loss (Raw)": 0.8142997622489929, "Finetune/Step": 3045, "Finetune/Step Time": 5.403709381818771} +{"Finetune/Learning Rate": 7.718432149855622e-06, "Finetune/Loss": 0.8860392570495605, "Finetune/Loss (Raw)": 1.0390301942825317, "Finetune/Step": 3046, "Finetune/Step Time": 5.397842112928629} +{"Finetune/Learning Rate": 7.7123672919978e-06, "Finetune/Loss": 0.8856251239776611, "Finetune/Loss (Raw)": 0.8578229546546936, "Finetune/Step": 3047, "Finetune/Step Time": 5.408840863034129} +{"Finetune/Learning Rate": 7.706303321925577e-06, "Finetune/Loss": 0.8911656737327576, "Finetune/Loss (Raw)": 1.055423617362976, "Finetune/Step": 3048, "Finetune/Step Time": 5.377825865522027} +{"Finetune/Learning Rate": 7.700240241992247e-06, "Finetune/Loss": 0.8911043405532837, "Finetune/Loss (Raw)": 0.8924762606620789, "Finetune/Step": 3049, "Finetune/Step Time": 5.360645228996873} +{"Finetune/Learning Rate": 7.694178054550782e-06, "Finetune/Loss": 0.8922375440597534, "Finetune/Loss (Raw)": 1.0031720399856567, "Finetune/Step": 3050, "Finetune/Step Time": 5.352101869881153} +{"Finetune/Learning Rate": 7.688116761953797e-06, "Finetune/Loss": 0.8933228850364685, "Finetune/Loss (Raw)": 1.0361418724060059, "Finetune/Step": 3051, "Finetune/Step Time": 5.325450925156474} +{"Finetune/Learning Rate": 7.682056366553559e-06, "Finetune/Loss": 0.892361581325531, "Finetune/Loss (Raw)": 0.883797287940979, "Finetune/Step": 3052, "Finetune/Step Time": 5.329764250665903} +{"Finetune/Learning Rate": 7.675996870701999e-06, "Finetune/Loss": 0.8920403718948364, "Finetune/Loss (Raw)": 0.9786800146102905, "Finetune/Step": 3053, "Finetune/Step Time": 5.333022061735392} +{"Finetune/Learning Rate": 7.669938276750673e-06, "Finetune/Loss": 0.8929896354675293, "Finetune/Loss (Raw)": 1.0166516304016113, "Finetune/Step": 3054, "Finetune/Step Time": 5.307359088212252} +{"Finetune/Learning Rate": 7.663880587050812e-06, "Finetune/Loss": 0.8947341442108154, "Finetune/Loss (Raw)": 0.9252544045448303, "Finetune/Step": 3055, "Finetune/Step Time": 5.293548904359341} +{"Finetune/Learning Rate": 7.657823803953288e-06, "Finetune/Loss": 0.8953511714935303, "Finetune/Loss (Raw)": 0.9852856397628784, "Finetune/Step": 3056, "Finetune/Step Time": 5.267170615494251} +{"Finetune/Learning Rate": 7.651767929808615e-06, "Finetune/Loss": 0.893454909324646, "Finetune/Loss (Raw)": 0.8006493449211121, "Finetune/Step": 3057, "Finetune/Step Time": 5.27940671518445} +{"Finetune/Learning Rate": 7.645712966966973e-06, "Finetune/Loss": 0.8935014009475708, "Finetune/Loss (Raw)": 0.982282280921936, "Finetune/Step": 3058, "Finetune/Step Time": 5.276577292010188} +{"Finetune/Learning Rate": 7.639658917778156e-06, "Finetune/Loss": 0.8934910297393799, "Finetune/Loss (Raw)": 0.9462308287620544, "Finetune/Step": 3059, "Finetune/Step Time": 5.278804557397962} +{"Finetune/Learning Rate": 7.633605784591633e-06, "Finetune/Loss": 0.8937650918960571, "Finetune/Loss (Raw)": 0.9768362641334534, "Finetune/Step": 3060, "Finetune/Step Time": 5.283228926360607} +{"Finetune/Learning Rate": 7.627553569756505e-06, "Finetune/Loss": 0.8940808773040771, "Finetune/Loss (Raw)": 0.957120418548584, "Finetune/Step": 3061, "Finetune/Step Time": 5.279857071116567} +{"Finetune/Learning Rate": 7.621502275621518e-06, "Finetune/Loss": 0.893613338470459, "Finetune/Loss (Raw)": 0.8661001920700073, "Finetune/Step": 3062, "Finetune/Step Time": 5.308317044749856} +{"Finetune/Learning Rate": 7.615451904535066e-06, "Finetune/Loss": 0.8932930827140808, "Finetune/Loss (Raw)": 0.8687734603881836, "Finetune/Step": 3063, "Finetune/Step Time": 5.300192831084132} +{"Finetune/Learning Rate": 7.609402458845171e-06, "Finetune/Loss": 0.892410397529602, "Finetune/Loss (Raw)": 0.9129514694213867, "Finetune/Step": 3064, "Finetune/Step Time": 5.281865477561951} +{"Finetune/Learning Rate": 7.60335394089951e-06, "Finetune/Loss": 0.890508234500885, "Finetune/Loss (Raw)": 0.8164089322090149, "Finetune/Step": 3065, "Finetune/Step Time": 5.29902377910912} +{"Finetune/Learning Rate": 7.597306353045393e-06, "Finetune/Loss": 0.8885918855667114, "Finetune/Loss (Raw)": 0.6975812315940857, "Finetune/Step": 3066, "Finetune/Step Time": 5.332183388993144} +{"Finetune/Learning Rate": 7.5912596976297735e-06, "Finetune/Loss": 0.8884826898574829, "Finetune/Loss (Raw)": 0.8921699523925781, "Finetune/Step": 3067, "Finetune/Step Time": 5.327326010912657} +{"Finetune/Learning Rate": 7.58521397699924e-06, "Finetune/Loss": 0.8898046016693115, "Finetune/Loss (Raw)": 0.8951522707939148, "Finetune/Step": 3068, "Finetune/Step Time": 5.338920200243592} +{"Finetune/Learning Rate": 7.579169193500014e-06, "Finetune/Loss": 0.8906325697898865, "Finetune/Loss (Raw)": 0.790428102016449, "Finetune/Step": 3069, "Finetune/Step Time": 5.330284535884857} +{"Finetune/Learning Rate": 7.573125349477964e-06, "Finetune/Loss": 0.889804482460022, "Finetune/Loss (Raw)": 0.9303179979324341, "Finetune/Step": 3070, "Finetune/Step Time": 5.328040614724159} +{"Finetune/Learning Rate": 7.567082447278582e-06, "Finetune/Loss": 0.8903490900993347, "Finetune/Loss (Raw)": 1.013640284538269, "Finetune/Step": 3071, "Finetune/Step Time": 5.3263482339680195} +{"Finetune/Learning Rate": 7.561040489247008e-06, "Finetune/Loss": 0.8899333477020264, "Finetune/Loss (Raw)": 0.8670724034309387, "Finetune/Step": 3072, "Finetune/Step Time": 5.327175930142403} +{"Finetune/Learning Rate": 7.554999477728001e-06, "Finetune/Loss": 0.8904274106025696, "Finetune/Loss (Raw)": 0.8973020315170288, "Finetune/Step": 3073, "Finetune/Step Time": 5.317544292658567} +{"Finetune/Learning Rate": 7.548959415065961e-06, "Finetune/Loss": 0.8899393081665039, "Finetune/Loss (Raw)": 0.8607720136642456, "Finetune/Step": 3074, "Finetune/Step Time": 5.3200447503477335} +{"Finetune/Learning Rate": 7.54292030360492e-06, "Finetune/Loss": 0.8853344917297363, "Finetune/Loss (Raw)": 0.293696790933609, "Finetune/Step": 3075, "Finetune/Step Time": 5.335115592926741} +{"Finetune/Learning Rate": 7.5368821456885375e-06, "Finetune/Loss": 0.8836574554443359, "Finetune/Loss (Raw)": 0.7643935680389404, "Finetune/Step": 3076, "Finetune/Step Time": 5.320908671244979} +{"Finetune/Learning Rate": 7.530844943660111e-06, "Finetune/Loss": 0.8836487531661987, "Finetune/Loss (Raw)": 0.945828378200531, "Finetune/Step": 3077, "Finetune/Step Time": 5.321243774145842} +{"Finetune/Learning Rate": 7.524808699862552e-06, "Finetune/Loss": 0.8828846216201782, "Finetune/Loss (Raw)": 0.8132314085960388, "Finetune/Step": 3078, "Finetune/Step Time": 5.317514738067985} +{"Finetune/Learning Rate": 7.518773416638412e-06, "Finetune/Loss": 0.8865600824356079, "Finetune/Loss (Raw)": 0.7740750908851624, "Finetune/Step": 3079, "Finetune/Step Time": 5.309221738949418} +{"Finetune/Learning Rate": 7.512739096329868e-06, "Finetune/Loss": 0.885002851486206, "Finetune/Loss (Raw)": 0.7599513530731201, "Finetune/Step": 3080, "Finetune/Step Time": 5.313487196341157} +{"Finetune/Learning Rate": 7.50670574127872e-06, "Finetune/Loss": 0.8835974931716919, "Finetune/Loss (Raw)": 0.8166493773460388, "Finetune/Step": 3081, "Finetune/Step Time": 5.31658492051065} +{"Finetune/Learning Rate": 7.500673353826399e-06, "Finetune/Loss": 0.8841912746429443, "Finetune/Loss (Raw)": 0.8746715188026428, "Finetune/Step": 3082, "Finetune/Step Time": 5.311631826683879} +{"Finetune/Learning Rate": 7.4946419363139535e-06, "Finetune/Loss": 0.8833600282669067, "Finetune/Loss (Raw)": 0.9524844884872437, "Finetune/Step": 3083, "Finetune/Step Time": 5.3171186447143555} +{"Finetune/Learning Rate": 7.488611491082056e-06, "Finetune/Loss": 0.8824021816253662, "Finetune/Loss (Raw)": 0.8192766904830933, "Finetune/Step": 3084, "Finetune/Step Time": 5.327812355011702} +{"Finetune/Learning Rate": 7.482582020471012e-06, "Finetune/Loss": 0.883769154548645, "Finetune/Loss (Raw)": 1.079031229019165, "Finetune/Step": 3085, "Finetune/Step Time": 5.325094038620591} +{"Finetune/Learning Rate": 7.476553526820733e-06, "Finetune/Loss": 0.8842467069625854, "Finetune/Loss (Raw)": 0.975662112236023, "Finetune/Step": 3086, "Finetune/Step Time": 5.324558414518833} +{"Finetune/Learning Rate": 7.4705260124707705e-06, "Finetune/Loss": 0.8888503313064575, "Finetune/Loss (Raw)": 0.9068294167518616, "Finetune/Step": 3087, "Finetune/Step Time": 5.298231294378638} +{"Finetune/Learning Rate": 7.464499479760275e-06, "Finetune/Loss": 0.8873245120048523, "Finetune/Loss (Raw)": 0.7148141860961914, "Finetune/Step": 3088, "Finetune/Step Time": 5.2972002029418945} +{"Finetune/Learning Rate": 7.458473931028029e-06, "Finetune/Loss": 0.888077437877655, "Finetune/Loss (Raw)": 0.9431383609771729, "Finetune/Step": 3089, "Finetune/Step Time": 5.300996081903577} +{"Finetune/Learning Rate": 7.452449368612433e-06, "Finetune/Loss": 0.886242151260376, "Finetune/Loss (Raw)": 0.8396018743515015, "Finetune/Step": 3090, "Finetune/Step Time": 5.294551653787494} +{"Finetune/Learning Rate": 7.446425794851498e-06, "Finetune/Loss": 0.885050892829895, "Finetune/Loss (Raw)": 0.8315279483795166, "Finetune/Step": 3091, "Finetune/Step Time": 5.298554861918092} +{"Finetune/Learning Rate": 7.440403212082863e-06, "Finetune/Loss": 0.8796124458312988, "Finetune/Loss (Raw)": 0.3004298806190491, "Finetune/Step": 3092, "Finetune/Step Time": 5.326947810128331} +{"Finetune/Learning Rate": 7.434381622643766e-06, "Finetune/Loss": 0.8789777159690857, "Finetune/Loss (Raw)": 0.882421612739563, "Finetune/Step": 3093, "Finetune/Step Time": 5.333444723859429} +{"Finetune/Learning Rate": 7.42836102887107e-06, "Finetune/Loss": 0.8796836137771606, "Finetune/Loss (Raw)": 0.9152280688285828, "Finetune/Step": 3094, "Finetune/Step Time": 5.32807500846684} +{"Finetune/Learning Rate": 7.422341433101254e-06, "Finetune/Loss": 0.8804466724395752, "Finetune/Loss (Raw)": 0.8924693465232849, "Finetune/Step": 3095, "Finetune/Step Time": 5.309983279556036} +{"Finetune/Learning Rate": 7.4163228376704e-06, "Finetune/Loss": 0.8795275688171387, "Finetune/Loss (Raw)": 0.9117257595062256, "Finetune/Step": 3096, "Finetune/Step Time": 5.304691484197974} +{"Finetune/Learning Rate": 7.410305244914216e-06, "Finetune/Loss": 0.8751367330551147, "Finetune/Loss (Raw)": 0.3047236502170563, "Finetune/Step": 3097, "Finetune/Step Time": 5.337873982265592} +{"Finetune/Learning Rate": 7.404288657168006e-06, "Finetune/Loss": 0.8736460208892822, "Finetune/Loss (Raw)": 0.7275820970535278, "Finetune/Step": 3098, "Finetune/Step Time": 5.3255266956985} +{"Finetune/Learning Rate": 7.398273076766687e-06, "Finetune/Loss": 0.8739874958992004, "Finetune/Loss (Raw)": 0.9439150094985962, "Finetune/Step": 3099, "Finetune/Step Time": 5.328887766227126} +{"Finetune/Learning Rate": 7.392258506044796e-06, "Finetune/Loss": 0.8752160668373108, "Finetune/Loss (Raw)": 1.083380937576294, "Finetune/Step": 3100, "Finetune/Step Time": 5.33084181137383} +{"Finetune/Learning Rate": 7.386244947336463e-06, "Finetune/Loss": 0.874192476272583, "Finetune/Loss (Raw)": 0.6577625274658203, "Finetune/Step": 3101, "Finetune/Step Time": 5.3327454049140215} +{"Finetune/Learning Rate": 7.380232402975443e-06, "Finetune/Loss": 0.8745619058609009, "Finetune/Loss (Raw)": 0.9826875329017639, "Finetune/Step": 3102, "Finetune/Step Time": 5.339894888922572} +{"Finetune/Learning Rate": 7.3742208752950776e-06, "Finetune/Loss": 0.8795329332351685, "Finetune/Loss (Raw)": 1.0278010368347168, "Finetune/Step": 3103, "Finetune/Step Time": 5.321451673284173} +{"Finetune/Learning Rate": 7.3682103666283245e-06, "Finetune/Loss": 0.8802616596221924, "Finetune/Loss (Raw)": 0.9247567653656006, "Finetune/Step": 3104, "Finetune/Step Time": 5.3166081961244345} +{"Finetune/Learning Rate": 7.362200879307748e-06, "Finetune/Loss": 0.8800913691520691, "Finetune/Loss (Raw)": 0.9105114936828613, "Finetune/Step": 3105, "Finetune/Step Time": 5.30704958178103} +{"Finetune/Learning Rate": 7.3561924156655105e-06, "Finetune/Loss": 0.8790334463119507, "Finetune/Loss (Raw)": 0.8584665656089783, "Finetune/Step": 3106, "Finetune/Step Time": 5.3070260770618916} +{"Finetune/Learning Rate": 7.350184978033386e-06, "Finetune/Loss": 0.8799329400062561, "Finetune/Loss (Raw)": 1.0164167881011963, "Finetune/Step": 3107, "Finetune/Step Time": 5.300619950518012} +{"Finetune/Learning Rate": 7.344178568742734e-06, "Finetune/Loss": 0.8796987533569336, "Finetune/Loss (Raw)": 0.815731406211853, "Finetune/Step": 3108, "Finetune/Step Time": 5.292060177773237} +{"Finetune/Learning Rate": 7.338173190124526e-06, "Finetune/Loss": 0.8805572390556335, "Finetune/Loss (Raw)": 0.8956789970397949, "Finetune/Step": 3109, "Finetune/Step Time": 5.26777028478682} +{"Finetune/Learning Rate": 7.3321688445093395e-06, "Finetune/Loss": 0.8807103633880615, "Finetune/Loss (Raw)": 0.848167359828949, "Finetune/Step": 3110, "Finetune/Step Time": 5.259323578327894} +{"Finetune/Learning Rate": 7.326165534227341e-06, "Finetune/Loss": 0.8796435594558716, "Finetune/Loss (Raw)": 0.8761611580848694, "Finetune/Step": 3111, "Finetune/Step Time": 5.274067025631666} +{"Finetune/Learning Rate": 7.320163261608293e-06, "Finetune/Loss": 0.8793796300888062, "Finetune/Loss (Raw)": 0.9814300537109375, "Finetune/Step": 3112, "Finetune/Step Time": 5.258335877209902} +{"Finetune/Learning Rate": 7.314162028981564e-06, "Finetune/Loss": 0.8793294429779053, "Finetune/Loss (Raw)": 0.8733550310134888, "Finetune/Step": 3113, "Finetune/Step Time": 5.266269041225314} +{"Finetune/Learning Rate": 7.308161838676113e-06, "Finetune/Loss": 0.8841437101364136, "Finetune/Loss (Raw)": 0.9431143999099731, "Finetune/Step": 3114, "Finetune/Step Time": 5.241055620834231} +{"Finetune/Learning Rate": 7.302162693020503e-06, "Finetune/Loss": 0.8843442797660828, "Finetune/Loss (Raw)": 0.861733615398407, "Finetune/Step": 3115, "Finetune/Step Time": 5.223515275865793} +{"Finetune/Learning Rate": 7.2961645943428804e-06, "Finetune/Loss": 0.8826515674591064, "Finetune/Loss (Raw)": 0.733281135559082, "Finetune/Step": 3116, "Finetune/Step Time": 5.2507471814751625} +{"Finetune/Learning Rate": 7.290167544970989e-06, "Finetune/Loss": 0.8832614421844482, "Finetune/Loss (Raw)": 0.90141761302948, "Finetune/Step": 3117, "Finetune/Step Time": 5.245450293645263} +{"Finetune/Learning Rate": 7.284171547232168e-06, "Finetune/Loss": 0.8790448904037476, "Finetune/Loss (Raw)": 0.47579970955848694, "Finetune/Step": 3118, "Finetune/Step Time": 5.278283564373851} +{"Finetune/Learning Rate": 7.278176603453347e-06, "Finetune/Loss": 0.8783913850784302, "Finetune/Loss (Raw)": 0.8727220892906189, "Finetune/Step": 3119, "Finetune/Step Time": 5.278087966144085} +{"Finetune/Learning Rate": 7.272182715961049e-06, "Finetune/Loss": 0.8793370723724365, "Finetune/Loss (Raw)": 0.9142327904701233, "Finetune/Step": 3120, "Finetune/Step Time": 5.272571213543415} +{"Finetune/Learning Rate": 7.266189887081385e-06, "Finetune/Loss": 0.8773310780525208, "Finetune/Loss (Raw)": 0.7474268078804016, "Finetune/Step": 3121, "Finetune/Step Time": 5.28370157815516} +{"Finetune/Learning Rate": 7.2601981191400484e-06, "Finetune/Loss": 0.8763333559036255, "Finetune/Loss (Raw)": 0.9015491604804993, "Finetune/Step": 3122, "Finetune/Step Time": 5.285441352054477} +{"Finetune/Learning Rate": 7.254207414462333e-06, "Finetune/Loss": 0.8757905960083008, "Finetune/Loss (Raw)": 0.8421118855476379, "Finetune/Step": 3123, "Finetune/Step Time": 5.280088426545262} +{"Finetune/Learning Rate": 7.248217775373112e-06, "Finetune/Loss": 0.8766306638717651, "Finetune/Loss (Raw)": 0.9911617040634155, "Finetune/Step": 3124, "Finetune/Step Time": 5.264142265543342} +{"Finetune/Learning Rate": 7.242229204196849e-06, "Finetune/Loss": 0.8774869441986084, "Finetune/Loss (Raw)": 0.9456048011779785, "Finetune/Step": 3125, "Finetune/Step Time": 5.258749375119805} +{"Finetune/Learning Rate": 7.236241703257594e-06, "Finetune/Loss": 0.8782366514205933, "Finetune/Loss (Raw)": 0.968760073184967, "Finetune/Step": 3126, "Finetune/Step Time": 5.249246578663588} +{"Finetune/Learning Rate": 7.230255274878972e-06, "Finetune/Loss": 0.8776718378067017, "Finetune/Loss (Raw)": 1.0076286792755127, "Finetune/Step": 3127, "Finetune/Step Time": 5.245663546025753} +{"Finetune/Learning Rate": 7.224269921384206e-06, "Finetune/Loss": 0.8761870861053467, "Finetune/Loss (Raw)": 0.7047746181488037, "Finetune/Step": 3128, "Finetune/Step Time": 5.281787823885679} +{"Finetune/Learning Rate": 7.218285645096088e-06, "Finetune/Loss": 0.876049816608429, "Finetune/Loss (Raw)": 0.8528172373771667, "Finetune/Step": 3129, "Finetune/Step Time": 5.284021902829409} +{"Finetune/Learning Rate": 7.2123024483370056e-06, "Finetune/Loss": 0.8751745223999023, "Finetune/Loss (Raw)": 0.8221622109413147, "Finetune/Step": 3130, "Finetune/Step Time": 5.298790875822306} +{"Finetune/Learning Rate": 7.20632033342892e-06, "Finetune/Loss": 0.8750013113021851, "Finetune/Loss (Raw)": 0.8631519675254822, "Finetune/Step": 3131, "Finetune/Step Time": 5.300271201878786} +{"Finetune/Learning Rate": 7.200339302693367e-06, "Finetune/Loss": 0.8709270358085632, "Finetune/Loss (Raw)": 0.4156252443790436, "Finetune/Step": 3132, "Finetune/Step Time": 5.331559842452407} +{"Finetune/Learning Rate": 7.194359358451475e-06, "Finetune/Loss": 0.8724663257598877, "Finetune/Loss (Raw)": 0.9261051416397095, "Finetune/Step": 3133, "Finetune/Step Time": 5.311816960573196} +{"Finetune/Learning Rate": 7.188380503023936e-06, "Finetune/Loss": 0.8719491362571716, "Finetune/Loss (Raw)": 0.8719838261604309, "Finetune/Step": 3134, "Finetune/Step Time": 5.315420761704445} +{"Finetune/Learning Rate": 7.182402738731036e-06, "Finetune/Loss": 0.8732467293739319, "Finetune/Loss (Raw)": 0.9346465468406677, "Finetune/Step": 3135, "Finetune/Step Time": 5.323183612897992} +{"Finetune/Learning Rate": 7.176426067892623e-06, "Finetune/Loss": 0.873385488986969, "Finetune/Loss (Raw)": 0.9664083123207092, "Finetune/Step": 3136, "Finetune/Step Time": 5.324622450396419} +{"Finetune/Learning Rate": 7.170450492828126e-06, "Finetune/Loss": 0.8720822334289551, "Finetune/Loss (Raw)": 0.8312695026397705, "Finetune/Step": 3137, "Finetune/Step Time": 5.340920360758901} +{"Finetune/Learning Rate": 7.164476015856552e-06, "Finetune/Loss": 0.8712939023971558, "Finetune/Loss (Raw)": 0.7701534032821655, "Finetune/Step": 3138, "Finetune/Step Time": 5.348706101998687} +{"Finetune/Learning Rate": 7.158502639296475e-06, "Finetune/Loss": 0.8702075481414795, "Finetune/Loss (Raw)": 0.8839708566665649, "Finetune/Step": 3139, "Finetune/Step Time": 5.355042636394501} +{"Finetune/Learning Rate": 7.152530365466051e-06, "Finetune/Loss": 0.8697028160095215, "Finetune/Loss (Raw)": 0.8797811269760132, "Finetune/Step": 3140, "Finetune/Step Time": 5.365960659459233} +{"Finetune/Learning Rate": 7.146559196683001e-06, "Finetune/Loss": 0.8692426681518555, "Finetune/Loss (Raw)": 0.8328820466995239, "Finetune/Step": 3141, "Finetune/Step Time": 5.3681584894657135} +{"Finetune/Learning Rate": 7.140589135264614e-06, "Finetune/Loss": 0.8692237138748169, "Finetune/Loss (Raw)": 0.8511474132537842, "Finetune/Step": 3142, "Finetune/Step Time": 5.3728436678647995} +{"Finetune/Learning Rate": 7.13462018352776e-06, "Finetune/Loss": 0.8686761856079102, "Finetune/Loss (Raw)": 0.6980427503585815, "Finetune/Step": 3143, "Finetune/Step Time": 5.371951919049025} +{"Finetune/Learning Rate": 7.12865234378887e-06, "Finetune/Loss": 0.8692731857299805, "Finetune/Loss (Raw)": 0.9921568632125854, "Finetune/Step": 3144, "Finetune/Step Time": 5.363615637645125} +{"Finetune/Learning Rate": 7.122685618363948e-06, "Finetune/Loss": 0.8708810806274414, "Finetune/Loss (Raw)": 0.9849885702133179, "Finetune/Step": 3145, "Finetune/Step Time": 5.358452823013067} +{"Finetune/Learning Rate": 7.116720009568565e-06, "Finetune/Loss": 0.8715648651123047, "Finetune/Loss (Raw)": 0.9421048164367676, "Finetune/Step": 3146, "Finetune/Step Time": 5.3589308969676495} +{"Finetune/Learning Rate": 7.110755519717851e-06, "Finetune/Loss": 0.871942937374115, "Finetune/Loss (Raw)": 0.9837589263916016, "Finetune/Step": 3147, "Finetune/Step Time": 5.366369936615229} +{"Finetune/Learning Rate": 7.104792151126515e-06, "Finetune/Loss": 0.8733092546463013, "Finetune/Loss (Raw)": 1.051931619644165, "Finetune/Step": 3148, "Finetune/Step Time": 5.387033548206091} +{"Finetune/Learning Rate": 7.098829906108819e-06, "Finetune/Loss": 0.869637131690979, "Finetune/Loss (Raw)": 0.4211580455303192, "Finetune/Step": 3149, "Finetune/Step Time": 5.41548746265471} +{"Finetune/Learning Rate": 7.092868786978596e-06, "Finetune/Loss": 0.8690086603164673, "Finetune/Loss (Raw)": 0.8904587626457214, "Finetune/Step": 3150, "Finetune/Step Time": 5.411971516907215} +{"Finetune/Learning Rate": 7.086908796049243e-06, "Finetune/Loss": 0.8702480792999268, "Finetune/Loss (Raw)": 0.8980543613433838, "Finetune/Step": 3151, "Finetune/Step Time": 5.413494128733873} +{"Finetune/Learning Rate": 7.08094993563371e-06, "Finetune/Loss": 0.8694531321525574, "Finetune/Loss (Raw)": 0.8251344561576843, "Finetune/Step": 3152, "Finetune/Step Time": 5.40812529809773} +{"Finetune/Learning Rate": 7.074992208044518e-06, "Finetune/Loss": 0.8693721294403076, "Finetune/Loss (Raw)": 0.923041820526123, "Finetune/Step": 3153, "Finetune/Step Time": 5.40817341953516} +{"Finetune/Learning Rate": 7.069035615593743e-06, "Finetune/Loss": 0.8690480589866638, "Finetune/Loss (Raw)": 0.893352746963501, "Finetune/Step": 3154, "Finetune/Step Time": 5.400113252922893} +{"Finetune/Learning Rate": 7.063080160593026e-06, "Finetune/Loss": 0.868867039680481, "Finetune/Loss (Raw)": 0.9668107628822327, "Finetune/Step": 3155, "Finetune/Step Time": 5.394172575324774} +{"Finetune/Learning Rate": 7.0571258453535585e-06, "Finetune/Loss": 0.868217945098877, "Finetune/Loss (Raw)": 0.8161864876747131, "Finetune/Step": 3156, "Finetune/Step Time": 5.389055484905839} +{"Finetune/Learning Rate": 7.051172672186093e-06, "Finetune/Loss": 0.8679319024085999, "Finetune/Loss (Raw)": 1.0144991874694824, "Finetune/Step": 3157, "Finetune/Step Time": 5.3946763798594475} +{"Finetune/Learning Rate": 7.045220643400946e-06, "Finetune/Loss": 0.8631982803344727, "Finetune/Loss (Raw)": 0.28421086072921753, "Finetune/Step": 3158, "Finetune/Step Time": 5.4131198059767485} +{"Finetune/Learning Rate": 7.039269761307977e-06, "Finetune/Loss": 0.8627547025680542, "Finetune/Loss (Raw)": 0.8602195382118225, "Finetune/Step": 3159, "Finetune/Step Time": 5.41667715460062} +{"Finetune/Learning Rate": 7.0333200282166146e-06, "Finetune/Loss": 0.8618861436843872, "Finetune/Loss (Raw)": 0.8755630850791931, "Finetune/Step": 3160, "Finetune/Step Time": 5.406794305890799} +{"Finetune/Learning Rate": 7.027371446435829e-06, "Finetune/Loss": 0.8618259429931641, "Finetune/Loss (Raw)": 0.9036893248558044, "Finetune/Step": 3161, "Finetune/Step Time": 5.426180221140385} +{"Finetune/Learning Rate": 7.0214240182741455e-06, "Finetune/Loss": 0.8567840456962585, "Finetune/Loss (Raw)": 0.4105037450790405, "Finetune/Step": 3162, "Finetune/Step Time": 5.457433747127652} +{"Finetune/Learning Rate": 7.015477746039652e-06, "Finetune/Loss": 0.8579109907150269, "Finetune/Loss (Raw)": 1.043670654296875, "Finetune/Step": 3163, "Finetune/Step Time": 5.468637198209763} +{"Finetune/Learning Rate": 7.009532632039975e-06, "Finetune/Loss": 0.8584474325180054, "Finetune/Loss (Raw)": 0.9578768610954285, "Finetune/Step": 3164, "Finetune/Step Time": 5.444351844489574} +{"Finetune/Learning Rate": 7.003588678582303e-06, "Finetune/Loss": 0.8558695912361145, "Finetune/Loss (Raw)": 0.6951698064804077, "Finetune/Step": 3165, "Finetune/Step Time": 5.446652578189969} +{"Finetune/Learning Rate": 6.9976458879733635e-06, "Finetune/Loss": 0.854612410068512, "Finetune/Loss (Raw)": 0.8271549344062805, "Finetune/Step": 3166, "Finetune/Step Time": 5.446721240878105} +{"Finetune/Learning Rate": 6.991704262519436e-06, "Finetune/Loss": 0.8552135229110718, "Finetune/Loss (Raw)": 0.9218838214874268, "Finetune/Step": 3167, "Finetune/Step Time": 5.444219965487719} +{"Finetune/Learning Rate": 6.985763804526354e-06, "Finetune/Loss": 0.8558381199836731, "Finetune/Loss (Raw)": 1.0168864727020264, "Finetune/Step": 3168, "Finetune/Step Time": 5.450132634490728} +{"Finetune/Learning Rate": 6.979824516299491e-06, "Finetune/Loss": 0.8522272109985352, "Finetune/Loss (Raw)": 0.3142314851284027, "Finetune/Step": 3169, "Finetune/Step Time": 5.477858493104577} +{"Finetune/Learning Rate": 6.973886400143772e-06, "Finetune/Loss": 0.8510804772377014, "Finetune/Loss (Raw)": 0.7855173945426941, "Finetune/Step": 3170, "Finetune/Step Time": 5.484096389263868} +{"Finetune/Learning Rate": 6.9679494583636595e-06, "Finetune/Loss": 0.8550742268562317, "Finetune/Loss (Raw)": 0.8195046186447144, "Finetune/Step": 3171, "Finetune/Step Time": 5.455586304888129} +{"Finetune/Learning Rate": 6.9620136932631655e-06, "Finetune/Loss": 0.8563829660415649, "Finetune/Loss (Raw)": 0.8903401494026184, "Finetune/Step": 3172, "Finetune/Step Time": 5.432577472180128} +{"Finetune/Learning Rate": 6.956079107145845e-06, "Finetune/Loss": 0.8562837839126587, "Finetune/Loss (Raw)": 0.8016052842140198, "Finetune/Step": 3173, "Finetune/Step Time": 5.444015245884657} +{"Finetune/Learning Rate": 6.950145702314795e-06, "Finetune/Loss": 0.8563182950019836, "Finetune/Loss (Raw)": 1.0434496402740479, "Finetune/Step": 3174, "Finetune/Step Time": 5.449048265814781} +{"Finetune/Learning Rate": 6.9442134810726615e-06, "Finetune/Loss": 0.8565715551376343, "Finetune/Loss (Raw)": 0.8902376890182495, "Finetune/Step": 3175, "Finetune/Step Time": 5.440279956907034} +{"Finetune/Learning Rate": 6.938282445721609e-06, "Finetune/Loss": 0.8564449548721313, "Finetune/Loss (Raw)": 1.039223551750183, "Finetune/Step": 3176, "Finetune/Step Time": 5.442355699837208} +{"Finetune/Learning Rate": 6.932352598563365e-06, "Finetune/Loss": 0.8561505079269409, "Finetune/Loss (Raw)": 0.854786217212677, "Finetune/Step": 3177, "Finetune/Step Time": 5.446318270638585} +{"Finetune/Learning Rate": 6.92642394189919e-06, "Finetune/Loss": 0.8550835847854614, "Finetune/Loss (Raw)": 0.8665989637374878, "Finetune/Step": 3178, "Finetune/Step Time": 5.45742342248559} +{"Finetune/Learning Rate": 6.920496478029873e-06, "Finetune/Loss": 0.8554010391235352, "Finetune/Loss (Raw)": 1.0767829418182373, "Finetune/Step": 3179, "Finetune/Step Time": 5.463748212903738} +{"Finetune/Learning Rate": 6.914570209255758e-06, "Finetune/Loss": 0.854949951171875, "Finetune/Loss (Raw)": 0.8260490894317627, "Finetune/Step": 3180, "Finetune/Step Time": 5.459510928019881} +{"Finetune/Learning Rate": 6.908645137876699e-06, "Finetune/Loss": 0.8547974228858948, "Finetune/Loss (Raw)": 0.9591631889343262, "Finetune/Step": 3181, "Finetune/Step Time": 5.454859333112836} +{"Finetune/Learning Rate": 6.902721266192112e-06, "Finetune/Loss": 0.8540114164352417, "Finetune/Loss (Raw)": 0.9160395264625549, "Finetune/Step": 3182, "Finetune/Step Time": 5.457432735711336} +{"Finetune/Learning Rate": 6.896798596500929e-06, "Finetune/Loss": 0.8535125851631165, "Finetune/Loss (Raw)": 0.8614063858985901, "Finetune/Step": 3183, "Finetune/Step Time": 5.457002151757479} +{"Finetune/Learning Rate": 6.8908771311016276e-06, "Finetune/Loss": 0.8529354333877563, "Finetune/Loss (Raw)": 0.9114101529121399, "Finetune/Step": 3184, "Finetune/Step Time": 5.467883771285415} +{"Finetune/Learning Rate": 6.884956872292216e-06, "Finetune/Loss": 0.8539236783981323, "Finetune/Loss (Raw)": 0.9271501898765564, "Finetune/Step": 3185, "Finetune/Step Time": 5.457129430025816} +{"Finetune/Learning Rate": 6.87903782237022e-06, "Finetune/Loss": 0.8529290556907654, "Finetune/Loss (Raw)": 0.8549664616584778, "Finetune/Step": 3186, "Finetune/Step Time": 5.4666836224496365} +{"Finetune/Learning Rate": 6.8731199836327155e-06, "Finetune/Loss": 0.8529672026634216, "Finetune/Loss (Raw)": 0.9511075019836426, "Finetune/Step": 3187, "Finetune/Step Time": 5.459513694047928} +{"Finetune/Learning Rate": 6.867203358376295e-06, "Finetune/Loss": 0.8521307706832886, "Finetune/Loss (Raw)": 0.8697769045829773, "Finetune/Step": 3188, "Finetune/Step Time": 5.467982847243547} +{"Finetune/Learning Rate": 6.861287948897091e-06, "Finetune/Loss": 0.8513064384460449, "Finetune/Loss (Raw)": 0.8515971899032593, "Finetune/Step": 3189, "Finetune/Step Time": 5.459617104381323} +{"Finetune/Learning Rate": 6.855373757490761e-06, "Finetune/Loss": 0.851661205291748, "Finetune/Loss (Raw)": 0.9115241765975952, "Finetune/Step": 3190, "Finetune/Step Time": 5.434517718851566} +{"Finetune/Learning Rate": 6.849460786452476e-06, "Finetune/Loss": 0.8524589538574219, "Finetune/Loss (Raw)": 0.9708733558654785, "Finetune/Step": 3191, "Finetune/Step Time": 5.460808381438255} +{"Finetune/Learning Rate": 6.843549038076951e-06, "Finetune/Loss": 0.851948618888855, "Finetune/Loss (Raw)": 0.8476328253746033, "Finetune/Step": 3192, "Finetune/Step Time": 5.476618446409702} +{"Finetune/Learning Rate": 6.83763851465842e-06, "Finetune/Loss": 0.8523368835449219, "Finetune/Loss (Raw)": 0.8661125898361206, "Finetune/Step": 3193, "Finetune/Step Time": 5.461101604625583} +{"Finetune/Learning Rate": 6.831729218490645e-06, "Finetune/Loss": 0.8529951572418213, "Finetune/Loss (Raw)": 0.7818353176116943, "Finetune/Step": 3194, "Finetune/Step Time": 5.433628857135773} +{"Finetune/Learning Rate": 6.825821151866903e-06, "Finetune/Loss": 0.8536158204078674, "Finetune/Loss (Raw)": 0.9716164469718933, "Finetune/Step": 3195, "Finetune/Step Time": 5.445369206368923} +{"Finetune/Learning Rate": 6.819914317079999e-06, "Finetune/Loss": 0.8538349866867065, "Finetune/Loss (Raw)": 0.9232043623924255, "Finetune/Step": 3196, "Finetune/Step Time": 5.440422823652625} +{"Finetune/Learning Rate": 6.814008716422265e-06, "Finetune/Loss": 0.8553953766822815, "Finetune/Loss (Raw)": 0.990159809589386, "Finetune/Step": 3197, "Finetune/Step Time": 5.432341694831848} +{"Finetune/Learning Rate": 6.808104352185544e-06, "Finetune/Loss": 0.854275107383728, "Finetune/Loss (Raw)": 0.7869255542755127, "Finetune/Step": 3198, "Finetune/Step Time": 5.440749010071158} +{"Finetune/Learning Rate": 6.8022012266612116e-06, "Finetune/Loss": 0.853472113609314, "Finetune/Loss (Raw)": 0.9108507633209229, "Finetune/Step": 3199, "Finetune/Step Time": 5.440259607508779} +{"Finetune/Learning Rate": 6.79629934214015e-06, "Finetune/Loss": 0.8550975918769836, "Finetune/Loss (Raw)": 1.0751354694366455, "Finetune/Step": 3200, "Finetune/Step Time": 5.430461464449763} +{"Finetune/Learning Rate": 6.790398700912764e-06, "Finetune/Loss": 0.8543331027030945, "Finetune/Loss (Raw)": 0.799452006816864, "Finetune/Step": 3201, "Finetune/Step Time": 5.432332519441843} +{"Finetune/Learning Rate": 6.784499305268983e-06, "Finetune/Loss": 0.8542966842651367, "Finetune/Loss (Raw)": 0.8561132550239563, "Finetune/Step": 3202, "Finetune/Step Time": 5.428954577073455} +{"Finetune/Learning Rate": 6.778601157498241e-06, "Finetune/Loss": 0.858646810054779, "Finetune/Loss (Raw)": 0.8505013585090637, "Finetune/Step": 3203, "Finetune/Step Time": 5.391637304797769} +{"Finetune/Learning Rate": 6.772704259889503e-06, "Finetune/Loss": 0.8592306971549988, "Finetune/Loss (Raw)": 0.8391361236572266, "Finetune/Step": 3204, "Finetune/Step Time": 5.398844456300139} +{"Finetune/Learning Rate": 6.76680861473123e-06, "Finetune/Loss": 0.858273983001709, "Finetune/Loss (Raw)": 0.8233693242073059, "Finetune/Step": 3205, "Finetune/Step Time": 5.394876988604665} +{"Finetune/Learning Rate": 6.7609142243114115e-06, "Finetune/Loss": 0.8580737113952637, "Finetune/Loss (Raw)": 0.7875921130180359, "Finetune/Step": 3206, "Finetune/Step Time": 5.39611286111176} +{"Finetune/Learning Rate": 6.755021090917545e-06, "Finetune/Loss": 0.8592034578323364, "Finetune/Loss (Raw)": 0.9186907410621643, "Finetune/Step": 3207, "Finetune/Step Time": 5.395713174715638} +{"Finetune/Learning Rate": 6.7491292168366386e-06, "Finetune/Loss": 0.8607393503189087, "Finetune/Loss (Raw)": 0.956542432308197, "Finetune/Step": 3208, "Finetune/Step Time": 5.3894886411726475} +{"Finetune/Learning Rate": 6.743238604355219e-06, "Finetune/Loss": 0.861089289188385, "Finetune/Loss (Raw)": 0.8614428639411926, "Finetune/Step": 3209, "Finetune/Step Time": 5.392198242247105} +{"Finetune/Learning Rate": 6.737349255759312e-06, "Finetune/Loss": 0.860477864742279, "Finetune/Loss (Raw)": 0.796410858631134, "Finetune/Step": 3210, "Finetune/Step Time": 5.391008395701647} +{"Finetune/Learning Rate": 6.731461173334459e-06, "Finetune/Loss": 0.8603302836418152, "Finetune/Loss (Raw)": 0.9335931539535522, "Finetune/Step": 3211, "Finetune/Step Time": 5.3847178891301155} +{"Finetune/Learning Rate": 6.725574359365714e-06, "Finetune/Loss": 0.8570186495780945, "Finetune/Loss (Raw)": 0.39537879824638367, "Finetune/Step": 3212, "Finetune/Step Time": 5.4081043023616076} +{"Finetune/Learning Rate": 6.7196888161376305e-06, "Finetune/Loss": 0.8560391664505005, "Finetune/Loss (Raw)": 0.9536609649658203, "Finetune/Step": 3213, "Finetune/Step Time": 5.405620930716395} +{"Finetune/Learning Rate": 6.71380454593428e-06, "Finetune/Loss": 0.8557546138763428, "Finetune/Loss (Raw)": 0.9392411112785339, "Finetune/Step": 3214, "Finetune/Step Time": 5.417913731187582} +{"Finetune/Learning Rate": 6.707921551039224e-06, "Finetune/Loss": 0.8549039363861084, "Finetune/Loss (Raw)": 0.7979482412338257, "Finetune/Step": 3215, "Finetune/Step Time": 5.426638953387737} +{"Finetune/Learning Rate": 6.70203983373554e-06, "Finetune/Loss": 0.8550500869750977, "Finetune/Loss (Raw)": 0.7335149645805359, "Finetune/Step": 3216, "Finetune/Step Time": 5.447216898202896} +{"Finetune/Learning Rate": 6.69615939630581e-06, "Finetune/Loss": 0.8549200296401978, "Finetune/Loss (Raw)": 0.9264927506446838, "Finetune/Step": 3217, "Finetune/Step Time": 5.438958205282688} +{"Finetune/Learning Rate": 6.690280241032117e-06, "Finetune/Loss": 0.8555773496627808, "Finetune/Loss (Raw)": 0.9237431883811951, "Finetune/Step": 3218, "Finetune/Step Time": 5.436835661530495} +{"Finetune/Learning Rate": 6.684402370196048e-06, "Finetune/Loss": 0.8551195859909058, "Finetune/Loss (Raw)": 0.7729309797286987, "Finetune/Step": 3219, "Finetune/Step Time": 5.420943297445774} +{"Finetune/Learning Rate": 6.678525786078684e-06, "Finetune/Loss": 0.8611288070678711, "Finetune/Loss (Raw)": 1.069608211517334, "Finetune/Step": 3220, "Finetune/Step Time": 5.388654636219144} +{"Finetune/Learning Rate": 6.672650490960617e-06, "Finetune/Loss": 0.8607327938079834, "Finetune/Loss (Raw)": 0.8317285180091858, "Finetune/Step": 3221, "Finetune/Step Time": 5.392473965883255} +{"Finetune/Learning Rate": 6.666776487121934e-06, "Finetune/Loss": 0.8561851978302002, "Finetune/Loss (Raw)": 0.333138644695282, "Finetune/Step": 3222, "Finetune/Step Time": 5.421950161457062} +{"Finetune/Learning Rate": 6.660903776842218e-06, "Finetune/Loss": 0.8558494448661804, "Finetune/Loss (Raw)": 0.8494938015937805, "Finetune/Step": 3223, "Finetune/Step Time": 5.426713632419705} +{"Finetune/Learning Rate": 6.655032362400562e-06, "Finetune/Loss": 0.8559197783470154, "Finetune/Loss (Raw)": 0.9207318425178528, "Finetune/Step": 3224, "Finetune/Step Time": 5.428020486608148} +{"Finetune/Learning Rate": 6.649162246075538e-06, "Finetune/Loss": 0.8603928089141846, "Finetune/Loss (Raw)": 0.8772730231285095, "Finetune/Step": 3225, "Finetune/Step Time": 5.39984430000186} +{"Finetune/Learning Rate": 6.6432934301452235e-06, "Finetune/Loss": 0.8617517948150635, "Finetune/Loss (Raw)": 0.9015316963195801, "Finetune/Step": 3226, "Finetune/Step Time": 5.39908567070961} +{"Finetune/Learning Rate": 6.637425916887198e-06, "Finetune/Loss": 0.8609323501586914, "Finetune/Loss (Raw)": 0.8390215635299683, "Finetune/Step": 3227, "Finetune/Step Time": 5.399719823151827} +{"Finetune/Learning Rate": 6.631559708578524e-06, "Finetune/Loss": 0.8592679500579834, "Finetune/Loss (Raw)": 0.870335042476654, "Finetune/Step": 3228, "Finetune/Step Time": 5.3974851835519075} +{"Finetune/Learning Rate": 6.625694807495766e-06, "Finetune/Loss": 0.8617278933525085, "Finetune/Loss (Raw)": 0.9726388454437256, "Finetune/Step": 3229, "Finetune/Step Time": 5.3879127986729145} +{"Finetune/Learning Rate": 6.619831215914974e-06, "Finetune/Loss": 0.8606603741645813, "Finetune/Loss (Raw)": 0.8460451364517212, "Finetune/Step": 3230, "Finetune/Step Time": 5.390479288995266} +{"Finetune/Learning Rate": 6.6139689361116945e-06, "Finetune/Loss": 0.8595528602600098, "Finetune/Loss (Raw)": 0.8860396146774292, "Finetune/Step": 3231, "Finetune/Step Time": 5.403291571885347} +{"Finetune/Learning Rate": 6.608107970360964e-06, "Finetune/Loss": 0.860301673412323, "Finetune/Loss (Raw)": 1.020599603652954, "Finetune/Step": 3232, "Finetune/Step Time": 5.399711227044463} +{"Finetune/Learning Rate": 6.602248320937308e-06, "Finetune/Loss": 0.8618935346603394, "Finetune/Loss (Raw)": 1.114280104637146, "Finetune/Step": 3233, "Finetune/Step Time": 5.403041407465935} +{"Finetune/Learning Rate": 6.596389990114748e-06, "Finetune/Loss": 0.861296534538269, "Finetune/Loss (Raw)": 0.7820479869842529, "Finetune/Step": 3234, "Finetune/Step Time": 5.4251494482159615} +{"Finetune/Learning Rate": 6.5905329801667796e-06, "Finetune/Loss": 0.861413836479187, "Finetune/Loss (Raw)": 1.0314263105392456, "Finetune/Step": 3235, "Finetune/Step Time": 5.429428676143289} +{"Finetune/Learning Rate": 6.584677293366396e-06, "Finetune/Loss": 0.8608886003494263, "Finetune/Loss (Raw)": 0.7485060095787048, "Finetune/Step": 3236, "Finetune/Step Time": 5.436246832832694} +{"Finetune/Learning Rate": 6.578822931986078e-06, "Finetune/Loss": 0.8615149259567261, "Finetune/Loss (Raw)": 0.975847065448761, "Finetune/Step": 3237, "Finetune/Step Time": 5.434116788208485} +{"Finetune/Learning Rate": 6.57296989829779e-06, "Finetune/Loss": 0.8580102920532227, "Finetune/Loss (Raw)": 0.39957162737846375, "Finetune/Step": 3238, "Finetune/Step Time": 5.466765150427818} +{"Finetune/Learning Rate": 6.567118194572972e-06, "Finetune/Loss": 0.8592106699943542, "Finetune/Loss (Raw)": 1.0298099517822266, "Finetune/Step": 3239, "Finetune/Step Time": 5.455714410170913} +{"Finetune/Learning Rate": 6.5612678230825646e-06, "Finetune/Loss": 0.8600351214408875, "Finetune/Loss (Raw)": 1.0869587659835815, "Finetune/Step": 3240, "Finetune/Step Time": 5.457560043781996} +{"Finetune/Learning Rate": 6.555418786096976e-06, "Finetune/Loss": 0.8607741594314575, "Finetune/Loss (Raw)": 0.9679514169692993, "Finetune/Step": 3241, "Finetune/Step Time": 5.452069349586964} +{"Finetune/Learning Rate": 6.549571085886111e-06, "Finetune/Loss": 0.8610557913780212, "Finetune/Loss (Raw)": 0.9791653752326965, "Finetune/Step": 3242, "Finetune/Step Time": 5.448734352365136} +{"Finetune/Learning Rate": 6.5437247247193445e-06, "Finetune/Loss": 0.8617768883705139, "Finetune/Loss (Raw)": 0.9540327787399292, "Finetune/Step": 3243, "Finetune/Step Time": 5.447577117010951} +{"Finetune/Learning Rate": 6.537879704865531e-06, "Finetune/Loss": 0.8636004328727722, "Finetune/Loss (Raw)": 0.9666948318481445, "Finetune/Step": 3244, "Finetune/Step Time": 5.412040838971734} +{"Finetune/Learning Rate": 6.532036028593012e-06, "Finetune/Loss": 0.8644075393676758, "Finetune/Loss (Raw)": 1.0047286748886108, "Finetune/Step": 3245, "Finetune/Step Time": 5.417461432516575} +{"Finetune/Learning Rate": 6.526193698169603e-06, "Finetune/Loss": 0.8663837909698486, "Finetune/Loss (Raw)": 0.7287558317184448, "Finetune/Step": 3246, "Finetune/Step Time": 5.389376563951373} +{"Finetune/Learning Rate": 6.520352715862601e-06, "Finetune/Loss": 0.8662341237068176, "Finetune/Loss (Raw)": 0.8535719513893127, "Finetune/Step": 3247, "Finetune/Step Time": 5.388853115960956} +{"Finetune/Learning Rate": 6.514513083938779e-06, "Finetune/Loss": 0.8668903112411499, "Finetune/Loss (Raw)": 0.9982210397720337, "Finetune/Step": 3248, "Finetune/Step Time": 5.386844074353576} +{"Finetune/Learning Rate": 6.508674804664374e-06, "Finetune/Loss": 0.8688623309135437, "Finetune/Loss (Raw)": 0.9998449087142944, "Finetune/Step": 3249, "Finetune/Step Time": 5.376034090295434} +{"Finetune/Learning Rate": 6.50283788030512e-06, "Finetune/Loss": 0.8684720993041992, "Finetune/Loss (Raw)": 0.8515969514846802, "Finetune/Step": 3250, "Finetune/Step Time": 5.3949841894209385} +{"Finetune/Learning Rate": 6.497002313126204e-06, "Finetune/Loss": 0.8686230778694153, "Finetune/Loss (Raw)": 0.8614437580108643, "Finetune/Step": 3251, "Finetune/Step Time": 5.422389375045896} +{"Finetune/Learning Rate": 6.491168105392303e-06, "Finetune/Loss": 0.8689658641815186, "Finetune/Loss (Raw)": 1.0350329875946045, "Finetune/Step": 3252, "Finetune/Step Time": 5.427377017214894} +{"Finetune/Learning Rate": 6.485335259367558e-06, "Finetune/Loss": 0.8692754507064819, "Finetune/Loss (Raw)": 0.985236406326294, "Finetune/Step": 3253, "Finetune/Step Time": 5.429982300847769} +{"Finetune/Learning Rate": 6.4795037773155765e-06, "Finetune/Loss": 0.8691579699516296, "Finetune/Loss (Raw)": 0.9537184834480286, "Finetune/Step": 3254, "Finetune/Step Time": 5.429606776684523} +{"Finetune/Learning Rate": 6.473673661499451e-06, "Finetune/Loss": 0.8685113787651062, "Finetune/Loss (Raw)": 0.9248663187026978, "Finetune/Step": 3255, "Finetune/Step Time": 5.440322557464242} +{"Finetune/Learning Rate": 6.467844914181727e-06, "Finetune/Loss": 0.8708535432815552, "Finetune/Loss (Raw)": 1.004570722579956, "Finetune/Step": 3256, "Finetune/Step Time": 5.406105345115066} +{"Finetune/Learning Rate": 6.462017537624437e-06, "Finetune/Loss": 0.8703081011772156, "Finetune/Loss (Raw)": 0.783008337020874, "Finetune/Step": 3257, "Finetune/Step Time": 5.410340627655387} +{"Finetune/Learning Rate": 6.4561915340890665e-06, "Finetune/Loss": 0.8722682595252991, "Finetune/Loss (Raw)": 1.0730546712875366, "Finetune/Step": 3258, "Finetune/Step Time": 5.398516893386841} +{"Finetune/Learning Rate": 6.450366905836573e-06, "Finetune/Loss": 0.8722202777862549, "Finetune/Loss (Raw)": 0.8570144176483154, "Finetune/Step": 3259, "Finetune/Step Time": 5.4079612120985985} +{"Finetune/Learning Rate": 6.444543655127383e-06, "Finetune/Loss": 0.8764950037002563, "Finetune/Loss (Raw)": 0.962785005569458, "Finetune/Step": 3260, "Finetune/Step Time": 5.393879815936089} +{"Finetune/Learning Rate": 6.438721784221385e-06, "Finetune/Loss": 0.8756523132324219, "Finetune/Loss (Raw)": 0.8182398080825806, "Finetune/Step": 3261, "Finetune/Step Time": 5.400206346064806} +{"Finetune/Learning Rate": 6.432901295377936e-06, "Finetune/Loss": 0.8765273094177246, "Finetune/Loss (Raw)": 0.9839869737625122, "Finetune/Step": 3262, "Finetune/Step Time": 5.3864867109805346} +{"Finetune/Learning Rate": 6.427082190855855e-06, "Finetune/Loss": 0.8770788908004761, "Finetune/Loss (Raw)": 1.005254864692688, "Finetune/Step": 3263, "Finetune/Step Time": 5.384897978976369} +{"Finetune/Learning Rate": 6.4212644729134155e-06, "Finetune/Loss": 0.8762763738632202, "Finetune/Loss (Raw)": 0.8636758923530579, "Finetune/Step": 3264, "Finetune/Step Time": 5.391237208619714} +{"Finetune/Learning Rate": 6.415448143808367e-06, "Finetune/Loss": 0.8781825304031372, "Finetune/Loss (Raw)": 1.0752633810043335, "Finetune/Step": 3265, "Finetune/Step Time": 5.374541774392128} +{"Finetune/Learning Rate": 6.409633205797911e-06, "Finetune/Loss": 0.8791323304176331, "Finetune/Loss (Raw)": 0.8917256593704224, "Finetune/Step": 3266, "Finetune/Step Time": 5.368963867425919} +{"Finetune/Learning Rate": 6.403819661138712e-06, "Finetune/Loss": 0.8795164227485657, "Finetune/Loss (Raw)": 0.9331364035606384, "Finetune/Step": 3267, "Finetune/Step Time": 5.377771256491542} +{"Finetune/Learning Rate": 6.398007512086897e-06, "Finetune/Loss": 0.8802329301834106, "Finetune/Loss (Raw)": 0.9714888334274292, "Finetune/Step": 3268, "Finetune/Step Time": 5.37187528796494} +{"Finetune/Learning Rate": 6.3921967608980375e-06, "Finetune/Loss": 0.8812878131866455, "Finetune/Loss (Raw)": 0.9679110050201416, "Finetune/Step": 3269, "Finetune/Step Time": 5.374144610017538} +{"Finetune/Learning Rate": 6.386387409827182e-06, "Finetune/Loss": 0.881253719329834, "Finetune/Loss (Raw)": 0.8467822074890137, "Finetune/Step": 3270, "Finetune/Step Time": 5.363771127536893} +{"Finetune/Learning Rate": 6.38057946112882e-06, "Finetune/Loss": 0.8826457262039185, "Finetune/Loss (Raw)": 0.876218855381012, "Finetune/Step": 3271, "Finetune/Step Time": 5.348419737070799} +{"Finetune/Learning Rate": 6.374772917056908e-06, "Finetune/Loss": 0.8823102712631226, "Finetune/Loss (Raw)": 0.949221670627594, "Finetune/Step": 3272, "Finetune/Step Time": 5.377689465880394} +{"Finetune/Learning Rate": 6.368967779864854e-06, "Finetune/Loss": 0.8809157609939575, "Finetune/Loss (Raw)": 0.8064988255500793, "Finetune/Step": 3273, "Finetune/Step Time": 5.3835807498544455} +{"Finetune/Learning Rate": 6.3631640518055085e-06, "Finetune/Loss": 0.8760817050933838, "Finetune/Loss (Raw)": 0.32333946228027344, "Finetune/Step": 3274, "Finetune/Step Time": 5.414996465668082} +{"Finetune/Learning Rate": 6.3573617351311945e-06, "Finetune/Loss": 0.8745331764221191, "Finetune/Loss (Raw)": 0.7855464220046997, "Finetune/Step": 3275, "Finetune/Step Time": 5.41236799955368} +{"Finetune/Learning Rate": 6.351560832093673e-06, "Finetune/Loss": 0.8733501434326172, "Finetune/Loss (Raw)": 0.9005060791969299, "Finetune/Step": 3276, "Finetune/Step Time": 5.397429803386331} +{"Finetune/Learning Rate": 6.3457613449441654e-06, "Finetune/Loss": 0.8774176239967346, "Finetune/Loss (Raw)": 0.941792368888855, "Finetune/Step": 3277, "Finetune/Step Time": 5.364960860460997} +{"Finetune/Learning Rate": 6.3399632759333364e-06, "Finetune/Loss": 0.8771675825119019, "Finetune/Loss (Raw)": 0.858449399471283, "Finetune/Step": 3278, "Finetune/Step Time": 5.368872886523604} +{"Finetune/Learning Rate": 6.3341666273113e-06, "Finetune/Loss": 0.8776448965072632, "Finetune/Loss (Raw)": 0.9591615200042725, "Finetune/Step": 3279, "Finetune/Step Time": 5.3767267391085625} +{"Finetune/Learning Rate": 6.328371401327629e-06, "Finetune/Loss": 0.8789270520210266, "Finetune/Loss (Raw)": 0.9892453551292419, "Finetune/Step": 3280, "Finetune/Step Time": 5.375873610377312} +{"Finetune/Learning Rate": 6.322577600231332e-06, "Finetune/Loss": 0.8797262907028198, "Finetune/Loss (Raw)": 1.025349497795105, "Finetune/Step": 3281, "Finetune/Step Time": 5.377976866438985} +{"Finetune/Learning Rate": 6.316785226270878e-06, "Finetune/Loss": 0.8795205354690552, "Finetune/Loss (Raw)": 0.8670064806938171, "Finetune/Step": 3282, "Finetune/Step Time": 5.379337806254625} +{"Finetune/Learning Rate": 6.310994281694167e-06, "Finetune/Loss": 0.8748559951782227, "Finetune/Loss (Raw)": 0.36974862217903137, "Finetune/Step": 3283, "Finetune/Step Time": 5.410291967913508} +{"Finetune/Learning Rate": 6.305204768748551e-06, "Finetune/Loss": 0.8752550482749939, "Finetune/Loss (Raw)": 0.8672677278518677, "Finetune/Step": 3284, "Finetune/Step Time": 5.413346711546183} +{"Finetune/Learning Rate": 6.299416689680831e-06, "Finetune/Loss": 0.8749434947967529, "Finetune/Loss (Raw)": 0.974622905254364, "Finetune/Step": 3285, "Finetune/Step Time": 5.407018354162574} +{"Finetune/Learning Rate": 6.293630046737245e-06, "Finetune/Loss": 0.8795086145401001, "Finetune/Loss (Raw)": 0.8685420751571655, "Finetune/Step": 3286, "Finetune/Step Time": 5.38142547570169} +{"Finetune/Learning Rate": 6.28784484216348e-06, "Finetune/Loss": 0.879950761795044, "Finetune/Loss (Raw)": 0.9168136715888977, "Finetune/Step": 3287, "Finetune/Step Time": 5.388535005971789} +{"Finetune/Learning Rate": 6.282061078204657e-06, "Finetune/Loss": 0.8807789087295532, "Finetune/Loss (Raw)": 0.9815732836723328, "Finetune/Step": 3288, "Finetune/Step Time": 5.39422208070755} +{"Finetune/Learning Rate": 6.276278757105341e-06, "Finetune/Loss": 0.8800525665283203, "Finetune/Loss (Raw)": 0.8107149600982666, "Finetune/Step": 3289, "Finetune/Step Time": 5.375410821288824} +{"Finetune/Learning Rate": 6.270497881109541e-06, "Finetune/Loss": 0.8841909170150757, "Finetune/Loss (Raw)": 0.9402146935462952, "Finetune/Step": 3290, "Finetune/Step Time": 5.344860874116421} +{"Finetune/Learning Rate": 6.264718452460699e-06, "Finetune/Loss": 0.882131814956665, "Finetune/Loss (Raw)": 0.7801005840301514, "Finetune/Step": 3291, "Finetune/Step Time": 5.340291615575552} +{"Finetune/Learning Rate": 6.258940473401705e-06, "Finetune/Loss": 0.8812764883041382, "Finetune/Loss (Raw)": 0.8483919501304626, "Finetune/Step": 3292, "Finetune/Step Time": 5.341358972713351} +{"Finetune/Learning Rate": 6.253163946174872e-06, "Finetune/Loss": 0.8830920457839966, "Finetune/Loss (Raw)": 0.9275727868080139, "Finetune/Step": 3293, "Finetune/Step Time": 5.342770595103502} +{"Finetune/Learning Rate": 6.247388873021959e-06, "Finetune/Loss": 0.8831765651702881, "Finetune/Loss (Raw)": 0.8379712104797363, "Finetune/Step": 3294, "Finetune/Step Time": 5.369688495993614} +{"Finetune/Learning Rate": 6.2416152561841635e-06, "Finetune/Loss": 0.8824825286865234, "Finetune/Loss (Raw)": 0.8330500721931458, "Finetune/Step": 3295, "Finetune/Step Time": 5.372499855235219} +{"Finetune/Learning Rate": 6.2358430979021075e-06, "Finetune/Loss": 0.8817905783653259, "Finetune/Loss (Raw)": 0.9283088445663452, "Finetune/Step": 3296, "Finetune/Step Time": 5.373348152264953} +{"Finetune/Learning Rate": 6.230072400415863e-06, "Finetune/Loss": 0.8858106732368469, "Finetune/Loss (Raw)": 0.828806459903717, "Finetune/Step": 3297, "Finetune/Step Time": 5.362861607223749} +{"Finetune/Learning Rate": 6.224303165964913e-06, "Finetune/Loss": 0.8874099254608154, "Finetune/Loss (Raw)": 0.9902178645133972, "Finetune/Step": 3298, "Finetune/Step Time": 5.355560138821602} +{"Finetune/Learning Rate": 6.21853539678819e-06, "Finetune/Loss": 0.8840122222900391, "Finetune/Loss (Raw)": 0.3845996558666229, "Finetune/Step": 3299, "Finetune/Step Time": 5.385011218488216} +{"Finetune/Learning Rate": 6.212769095124055e-06, "Finetune/Loss": 0.884257435798645, "Finetune/Loss (Raw)": 0.9217302799224854, "Finetune/Step": 3300, "Finetune/Step Time": 5.387020941823721} +{"Finetune/Learning Rate": 6.207004263210292e-06, "Finetune/Loss": 0.8838299512863159, "Finetune/Loss (Raw)": 0.7468878626823425, "Finetune/Step": 3301, "Finetune/Step Time": 5.375851277261972} +{"Finetune/Learning Rate": 6.2012409032841294e-06, "Finetune/Loss": 0.8832388520240784, "Finetune/Loss (Raw)": 0.967788577079773, "Finetune/Step": 3302, "Finetune/Step Time": 5.374197445809841} +{"Finetune/Learning Rate": 6.1954790175822e-06, "Finetune/Loss": 0.8830342292785645, "Finetune/Loss (Raw)": 0.8640460968017578, "Finetune/Step": 3303, "Finetune/Step Time": 5.377573044970632} +{"Finetune/Learning Rate": 6.189718608340587e-06, "Finetune/Loss": 0.8814845085144043, "Finetune/Loss (Raw)": 0.8408653140068054, "Finetune/Step": 3304, "Finetune/Step Time": 5.387106020003557} +{"Finetune/Learning Rate": 6.183959677794795e-06, "Finetune/Loss": 0.8825446367263794, "Finetune/Loss (Raw)": 0.9904676675796509, "Finetune/Step": 3305, "Finetune/Step Time": 5.382970539852977} +{"Finetune/Learning Rate": 6.1782022281797485e-06, "Finetune/Loss": 0.8823482990264893, "Finetune/Loss (Raw)": 0.8414738774299622, "Finetune/Step": 3306, "Finetune/Step Time": 5.36959626711905} +{"Finetune/Learning Rate": 6.17244626172981e-06, "Finetune/Loss": 0.8819963932037354, "Finetune/Loss (Raw)": 1.031734585762024, "Finetune/Step": 3307, "Finetune/Step Time": 5.359298167750239} +{"Finetune/Learning Rate": 6.166691780678743e-06, "Finetune/Loss": 0.8832600712776184, "Finetune/Loss (Raw)": 0.9878045916557312, "Finetune/Step": 3308, "Finetune/Step Time": 5.365569487214088} +{"Finetune/Learning Rate": 6.1609387872597595e-06, "Finetune/Loss": 0.8834798336029053, "Finetune/Loss (Raw)": 0.9872899055480957, "Finetune/Step": 3309, "Finetune/Step Time": 5.376130120828748} +{"Finetune/Learning Rate": 6.155187283705479e-06, "Finetune/Loss": 0.8827186822891235, "Finetune/Loss (Raw)": 0.8186209201812744, "Finetune/Step": 3310, "Finetune/Step Time": 5.373209526762366} +{"Finetune/Learning Rate": 6.14943727224795e-06, "Finetune/Loss": 0.882270336151123, "Finetune/Loss (Raw)": 0.8040186166763306, "Finetune/Step": 3311, "Finetune/Step Time": 5.381595803424716} +{"Finetune/Learning Rate": 6.1436887551186466e-06, "Finetune/Loss": 0.8817970156669617, "Finetune/Loss (Raw)": 0.8508191704750061, "Finetune/Step": 3312, "Finetune/Step Time": 5.375491475686431} +{"Finetune/Learning Rate": 6.137941734548443e-06, "Finetune/Loss": 0.8779279589653015, "Finetune/Loss (Raw)": 0.431905061006546, "Finetune/Step": 3313, "Finetune/Step Time": 5.400713814422488} +{"Finetune/Learning Rate": 6.132196212767653e-06, "Finetune/Loss": 0.8775134682655334, "Finetune/Loss (Raw)": 0.8019208312034607, "Finetune/Step": 3314, "Finetune/Step Time": 5.39157866127789} +{"Finetune/Learning Rate": 6.126452192005999e-06, "Finetune/Loss": 0.8787702322006226, "Finetune/Loss (Raw)": 1.111972689628601, "Finetune/Step": 3315, "Finetune/Step Time": 5.391273019835353} +{"Finetune/Learning Rate": 6.120709674492627e-06, "Finetune/Loss": 0.8784523010253906, "Finetune/Loss (Raw)": 0.8290842771530151, "Finetune/Step": 3316, "Finetune/Step Time": 5.382664762437344} +{"Finetune/Learning Rate": 6.114968662456093e-06, "Finetune/Loss": 0.8788571357727051, "Finetune/Loss (Raw)": 0.9034122228622437, "Finetune/Step": 3317, "Finetune/Step Time": 5.383687283843756} +{"Finetune/Learning Rate": 6.10922915812437e-06, "Finetune/Loss": 0.8743515610694885, "Finetune/Loss (Raw)": 0.3348138630390167, "Finetune/Step": 3318, "Finetune/Step Time": 5.415706591680646} +{"Finetune/Learning Rate": 6.103491163724853e-06, "Finetune/Loss": 0.8743962049484253, "Finetune/Loss (Raw)": 0.9765877723693848, "Finetune/Step": 3319, "Finetune/Step Time": 5.40105476975441} +{"Finetune/Learning Rate": 6.09775468148434e-06, "Finetune/Loss": 0.8745455741882324, "Finetune/Loss (Raw)": 0.8667519092559814, "Finetune/Step": 3320, "Finetune/Step Time": 5.382373934611678} +{"Finetune/Learning Rate": 6.092019713629053e-06, "Finetune/Loss": 0.8742281198501587, "Finetune/Loss (Raw)": 0.8254751563072205, "Finetune/Step": 3321, "Finetune/Step Time": 5.38202129304409} +{"Finetune/Learning Rate": 6.086286262384617e-06, "Finetune/Loss": 0.8747864961624146, "Finetune/Loss (Raw)": 0.8533086776733398, "Finetune/Step": 3322, "Finetune/Step Time": 5.37504000775516} +{"Finetune/Learning Rate": 6.080554329976071e-06, "Finetune/Loss": 0.8751025795936584, "Finetune/Loss (Raw)": 1.0120714902877808, "Finetune/Step": 3323, "Finetune/Step Time": 5.376750303432345} +{"Finetune/Learning Rate": 6.074823918627872e-06, "Finetune/Loss": 0.8744550943374634, "Finetune/Loss (Raw)": 0.8403324484825134, "Finetune/Step": 3324, "Finetune/Step Time": 5.384576795622706} +{"Finetune/Learning Rate": 6.069095030563877e-06, "Finetune/Loss": 0.8730935454368591, "Finetune/Loss (Raw)": 0.8158787488937378, "Finetune/Step": 3325, "Finetune/Step Time": 5.384166607633233} +{"Finetune/Learning Rate": 6.063367668007356e-06, "Finetune/Loss": 0.8743537068367004, "Finetune/Loss (Raw)": 0.9482235312461853, "Finetune/Step": 3326, "Finetune/Step Time": 5.380812078714371} +{"Finetune/Learning Rate": 6.057641833180987e-06, "Finetune/Loss": 0.8730247020721436, "Finetune/Loss (Raw)": 0.7407350540161133, "Finetune/Step": 3327, "Finetune/Step Time": 5.387872789055109} +{"Finetune/Learning Rate": 6.0519175283068534e-06, "Finetune/Loss": 0.8726193308830261, "Finetune/Loss (Raw)": 1.0232526063919067, "Finetune/Step": 3328, "Finetune/Step Time": 5.392066413536668} +{"Finetune/Learning Rate": 6.046194755606448e-06, "Finetune/Loss": 0.8743585348129272, "Finetune/Loss (Raw)": 1.0220699310302734, "Finetune/Step": 3329, "Finetune/Step Time": 5.400208942592144} +{"Finetune/Learning Rate": 6.040473517300665e-06, "Finetune/Loss": 0.873921275138855, "Finetune/Loss (Raw)": 0.8001417517662048, "Finetune/Step": 3330, "Finetune/Step Time": 5.418848920613527} +{"Finetune/Learning Rate": 6.0347538156098104e-06, "Finetune/Loss": 0.8745055794715881, "Finetune/Loss (Raw)": 0.9252945184707642, "Finetune/Step": 3331, "Finetune/Step Time": 5.437552401795983} +{"Finetune/Learning Rate": 6.0290356527535836e-06, "Finetune/Loss": 0.8749197721481323, "Finetune/Loss (Raw)": 0.8921557664871216, "Finetune/Step": 3332, "Finetune/Step Time": 5.431014932692051} +{"Finetune/Learning Rate": 6.023319030951091e-06, "Finetune/Loss": 0.8758357763290405, "Finetune/Loss (Raw)": 0.9406145811080933, "Finetune/Step": 3333, "Finetune/Step Time": 5.431042382493615} +{"Finetune/Learning Rate": 6.017603952420846e-06, "Finetune/Loss": 0.8766224384307861, "Finetune/Loss (Raw)": 0.8882886171340942, "Finetune/Step": 3334, "Finetune/Step Time": 5.43162533454597} +{"Finetune/Learning Rate": 6.011890419380756e-06, "Finetune/Loss": 0.8760288953781128, "Finetune/Loss (Raw)": 0.8427139520645142, "Finetune/Step": 3335, "Finetune/Step Time": 5.40999237075448} +{"Finetune/Learning Rate": 6.0061784340481366e-06, "Finetune/Loss": 0.8753401041030884, "Finetune/Loss (Raw)": 0.8683717846870422, "Finetune/Step": 3336, "Finetune/Step Time": 5.409025091677904} +{"Finetune/Learning Rate": 6.000467998639692e-06, "Finetune/Loss": 0.8747302293777466, "Finetune/Loss (Raw)": 0.7833817005157471, "Finetune/Step": 3337, "Finetune/Step Time": 5.412487769499421} +{"Finetune/Learning Rate": 5.994759115371531e-06, "Finetune/Loss": 0.8754901885986328, "Finetune/Loss (Raw)": 0.8936907649040222, "Finetune/Step": 3338, "Finetune/Step Time": 5.415670583024621} +{"Finetune/Learning Rate": 5.989051786459163e-06, "Finetune/Loss": 0.8756880760192871, "Finetune/Loss (Raw)": 0.9589224457740784, "Finetune/Step": 3339, "Finetune/Step Time": 5.408315835520625} +{"Finetune/Learning Rate": 5.98334601411749e-06, "Finetune/Loss": 0.8796772956848145, "Finetune/Loss (Raw)": 0.9059935808181763, "Finetune/Step": 3340, "Finetune/Step Time": 5.379501640796661} +{"Finetune/Learning Rate": 5.977641800560811e-06, "Finetune/Loss": 0.8803159594535828, "Finetune/Loss (Raw)": 1.0354151725769043, "Finetune/Step": 3341, "Finetune/Step Time": 5.386059708893299} +{"Finetune/Learning Rate": 5.971939148002819e-06, "Finetune/Loss": 0.8808253407478333, "Finetune/Loss (Raw)": 1.0044350624084473, "Finetune/Step": 3342, "Finetune/Step Time": 5.366897858679295} +{"Finetune/Learning Rate": 5.966238058656601e-06, "Finetune/Loss": 0.8818954229354858, "Finetune/Loss (Raw)": 0.9349227547645569, "Finetune/Step": 3343, "Finetune/Step Time": 5.354311278089881} +{"Finetune/Learning Rate": 5.960538534734641e-06, "Finetune/Loss": 0.8818133473396301, "Finetune/Loss (Raw)": 0.7230097651481628, "Finetune/Step": 3344, "Finetune/Step Time": 5.334322230890393} +{"Finetune/Learning Rate": 5.9548405784488095e-06, "Finetune/Loss": 0.8773261308670044, "Finetune/Loss (Raw)": 0.35213151574134827, "Finetune/Step": 3345, "Finetune/Step Time": 5.369943713769317} +{"Finetune/Learning Rate": 5.949144192010378e-06, "Finetune/Loss": 0.8781046271324158, "Finetune/Loss (Raw)": 1.0233858823776245, "Finetune/Step": 3346, "Finetune/Step Time": 5.368148962035775} +{"Finetune/Learning Rate": 5.94344937763e-06, "Finetune/Loss": 0.8780806660652161, "Finetune/Loss (Raw)": 0.7698625922203064, "Finetune/Step": 3347, "Finetune/Step Time": 5.369335891678929} +{"Finetune/Learning Rate": 5.9377561375177164e-06, "Finetune/Loss": 0.877121090888977, "Finetune/Loss (Raw)": 0.9467912316322327, "Finetune/Step": 3348, "Finetune/Step Time": 5.374035123735666} +{"Finetune/Learning Rate": 5.9320644738829706e-06, "Finetune/Loss": 0.8780117034912109, "Finetune/Loss (Raw)": 0.9457276463508606, "Finetune/Step": 3349, "Finetune/Step Time": 5.371024690568447} +{"Finetune/Learning Rate": 5.926374388934579e-06, "Finetune/Loss": 0.8837524652481079, "Finetune/Loss (Raw)": 1.067946434020996, "Finetune/Step": 3350, "Finetune/Step Time": 5.346815790981054} +{"Finetune/Learning Rate": 5.920685884880762e-06, "Finetune/Loss": 0.8837186694145203, "Finetune/Loss (Raw)": 0.845169186592102, "Finetune/Step": 3351, "Finetune/Step Time": 5.354497097432613} +{"Finetune/Learning Rate": 5.914998963929109e-06, "Finetune/Loss": 0.8838360905647278, "Finetune/Loss (Raw)": 0.9357630014419556, "Finetune/Step": 3352, "Finetune/Step Time": 5.358222233131528} +{"Finetune/Learning Rate": 5.9093136282866014e-06, "Finetune/Loss": 0.8845775127410889, "Finetune/Loss (Raw)": 0.972177267074585, "Finetune/Step": 3353, "Finetune/Step Time": 5.374585935845971} +{"Finetune/Learning Rate": 5.903629880159614e-06, "Finetune/Loss": 0.8848428726196289, "Finetune/Loss (Raw)": 0.9354894757270813, "Finetune/Step": 3354, "Finetune/Step Time": 5.372425597161055} +{"Finetune/Learning Rate": 5.897947721753894e-06, "Finetune/Loss": 0.8859087228775024, "Finetune/Loss (Raw)": 0.9754541516304016, "Finetune/Step": 3355, "Finetune/Step Time": 5.363853057846427} +{"Finetune/Learning Rate": 5.89226715527458e-06, "Finetune/Loss": 0.8864356875419617, "Finetune/Loss (Raw)": 0.9377927184104919, "Finetune/Step": 3356, "Finetune/Step Time": 5.366302043199539} +{"Finetune/Learning Rate": 5.886588182926185e-06, "Finetune/Loss": 0.8856362104415894, "Finetune/Loss (Raw)": 0.8702998757362366, "Finetune/Step": 3357, "Finetune/Step Time": 5.361064611002803} +{"Finetune/Learning Rate": 5.880910806912607e-06, "Finetune/Loss": 0.8853856325149536, "Finetune/Loss (Raw)": 0.8139748573303223, "Finetune/Step": 3358, "Finetune/Step Time": 5.354730011895299} +{"Finetune/Learning Rate": 5.8752350294371274e-06, "Finetune/Loss": 0.8867090344429016, "Finetune/Loss (Raw)": 1.0554362535476685, "Finetune/Step": 3359, "Finetune/Step Time": 5.3285043481737375} +{"Finetune/Learning Rate": 5.8695608527024075e-06, "Finetune/Loss": 0.88658207654953, "Finetune/Loss (Raw)": 1.0043469667434692, "Finetune/Step": 3360, "Finetune/Step Time": 5.32941104285419} +{"Finetune/Learning Rate": 5.863888278910475e-06, "Finetune/Loss": 0.8853113055229187, "Finetune/Loss (Raw)": 0.9516174793243408, "Finetune/Step": 3361, "Finetune/Step Time": 5.347748219966888} +{"Finetune/Learning Rate": 5.8582173102627526e-06, "Finetune/Loss": 0.8856860399246216, "Finetune/Loss (Raw)": 0.830010712146759, "Finetune/Step": 3362, "Finetune/Step Time": 5.3301025331020355} +{"Finetune/Learning Rate": 5.852547948960029e-06, "Finetune/Loss": 0.8849382400512695, "Finetune/Loss (Raw)": 0.9357089996337891, "Finetune/Step": 3363, "Finetune/Step Time": 5.3305229134857655} +{"Finetune/Learning Rate": 5.8468801972024755e-06, "Finetune/Loss": 0.8866592049598694, "Finetune/Loss (Raw)": 0.9687952995300293, "Finetune/Step": 3364, "Finetune/Step Time": 5.322676979005337} +{"Finetune/Learning Rate": 5.841214057189638e-06, "Finetune/Loss": 0.8858634233474731, "Finetune/Loss (Raw)": 0.8739904761314392, "Finetune/Step": 3365, "Finetune/Step Time": 5.336914444342256} +{"Finetune/Learning Rate": 5.8355495311204266e-06, "Finetune/Loss": 0.8891456127166748, "Finetune/Loss (Raw)": 0.8196858763694763, "Finetune/Step": 3366, "Finetune/Step Time": 5.317501278594136} +{"Finetune/Learning Rate": 5.829886621193141e-06, "Finetune/Loss": 0.8874962329864502, "Finetune/Loss (Raw)": 0.8186858296394348, "Finetune/Step": 3367, "Finetune/Step Time": 5.315261531621218} +{"Finetune/Learning Rate": 5.824225329605442e-06, "Finetune/Loss": 0.8824183940887451, "Finetune/Loss (Raw)": 0.43699756264686584, "Finetune/Step": 3368, "Finetune/Step Time": 5.344579583033919} +{"Finetune/Learning Rate": 5.818565658554364e-06, "Finetune/Loss": 0.882560670375824, "Finetune/Loss (Raw)": 0.9861680269241333, "Finetune/Step": 3369, "Finetune/Step Time": 5.337642427533865} +{"Finetune/Learning Rate": 5.8129076102363266e-06, "Finetune/Loss": 0.8823604583740234, "Finetune/Loss (Raw)": 0.9535351991653442, "Finetune/Step": 3370, "Finetune/Step Time": 5.335354195907712} +{"Finetune/Learning Rate": 5.807251186847095e-06, "Finetune/Loss": 0.8803128600120544, "Finetune/Loss (Raw)": 0.6919416785240173, "Finetune/Step": 3371, "Finetune/Step Time": 5.36232696287334} +{"Finetune/Learning Rate": 5.801596390581817e-06, "Finetune/Loss": 0.879596471786499, "Finetune/Loss (Raw)": 0.8749989867210388, "Finetune/Step": 3372, "Finetune/Step Time": 5.373431408777833} +{"Finetune/Learning Rate": 5.795943223635018e-06, "Finetune/Loss": 0.8791776299476624, "Finetune/Loss (Raw)": 0.9511131644248962, "Finetune/Step": 3373, "Finetune/Step Time": 5.378957161679864} +{"Finetune/Learning Rate": 5.7902916882005755e-06, "Finetune/Loss": 0.8805469274520874, "Finetune/Loss (Raw)": 0.9040263891220093, "Finetune/Step": 3374, "Finetune/Step Time": 5.381719009950757} +{"Finetune/Learning Rate": 5.784641786471742e-06, "Finetune/Loss": 0.8827110528945923, "Finetune/Loss (Raw)": 1.1305793523788452, "Finetune/Step": 3375, "Finetune/Step Time": 5.379674311727285} +{"Finetune/Learning Rate": 5.778993520641132e-06, "Finetune/Loss": 0.8828238248825073, "Finetune/Loss (Raw)": 1.0126605033874512, "Finetune/Step": 3376, "Finetune/Step Time": 5.372243674471974} +{"Finetune/Learning Rate": 5.7733468929007294e-06, "Finetune/Loss": 0.8813560605049133, "Finetune/Loss (Raw)": 0.8119661808013916, "Finetune/Step": 3377, "Finetune/Step Time": 5.383260648697615} +{"Finetune/Learning Rate": 5.7677019054418805e-06, "Finetune/Loss": 0.8824158906936646, "Finetune/Loss (Raw)": 0.9872551560401917, "Finetune/Step": 3378, "Finetune/Step Time": 5.379011886194348} +{"Finetune/Learning Rate": 5.762058560455287e-06, "Finetune/Loss": 0.8812282085418701, "Finetune/Loss (Raw)": 0.7094173431396484, "Finetune/Step": 3379, "Finetune/Step Time": 5.353380085900426} +{"Finetune/Learning Rate": 5.756416860131036e-06, "Finetune/Loss": 0.8793118000030518, "Finetune/Loss (Raw)": 0.7897382974624634, "Finetune/Step": 3380, "Finetune/Step Time": 5.351752633228898} +{"Finetune/Learning Rate": 5.750776806658549e-06, "Finetune/Loss": 0.8785241842269897, "Finetune/Loss (Raw)": 0.8844229578971863, "Finetune/Step": 3381, "Finetune/Step Time": 5.346358954906464} +{"Finetune/Learning Rate": 5.745138402226621e-06, "Finetune/Loss": 0.879210352897644, "Finetune/Loss (Raw)": 1.041540265083313, "Finetune/Step": 3382, "Finetune/Step Time": 5.348376022651792} +{"Finetune/Learning Rate": 5.739501649023413e-06, "Finetune/Loss": 0.8786588907241821, "Finetune/Loss (Raw)": 0.8542892336845398, "Finetune/Step": 3383, "Finetune/Step Time": 5.331062028184533} +{"Finetune/Learning Rate": 5.7338665492364365e-06, "Finetune/Loss": 0.8781534433364868, "Finetune/Loss (Raw)": 0.9398703575134277, "Finetune/Step": 3384, "Finetune/Step Time": 5.326837899163365} +{"Finetune/Learning Rate": 5.728233105052563e-06, "Finetune/Loss": 0.878177285194397, "Finetune/Loss (Raw)": 0.7860640287399292, "Finetune/Step": 3385, "Finetune/Step Time": 5.315917011350393} +{"Finetune/Learning Rate": 5.722601318658021e-06, "Finetune/Loss": 0.8766821622848511, "Finetune/Loss (Raw)": 0.8816699385643005, "Finetune/Step": 3386, "Finetune/Step Time": 5.313624000176787} +{"Finetune/Learning Rate": 5.7169711922383984e-06, "Finetune/Loss": 0.876263439655304, "Finetune/Loss (Raw)": 0.8034265041351318, "Finetune/Step": 3387, "Finetune/Step Time": 5.296279653906822} +{"Finetune/Learning Rate": 5.711342727978637e-06, "Finetune/Loss": 0.8751455545425415, "Finetune/Loss (Raw)": 0.8196948170661926, "Finetune/Step": 3388, "Finetune/Step Time": 5.278521403670311} +{"Finetune/Learning Rate": 5.7057159280630316e-06, "Finetune/Loss": 0.8760375380516052, "Finetune/Loss (Raw)": 0.9324129223823547, "Finetune/Step": 3389, "Finetune/Step Time": 5.277516866102815} +{"Finetune/Learning Rate": 5.700090794675241e-06, "Finetune/Loss": 0.8758252859115601, "Finetune/Loss (Raw)": 0.9568222165107727, "Finetune/Step": 3390, "Finetune/Step Time": 5.282558852806687} +{"Finetune/Learning Rate": 5.694467329998264e-06, "Finetune/Loss": 0.8762903213500977, "Finetune/Loss (Raw)": 1.0647711753845215, "Finetune/Step": 3391, "Finetune/Step Time": 5.27348536811769} +{"Finetune/Learning Rate": 5.68884553621445e-06, "Finetune/Loss": 0.8762558698654175, "Finetune/Loss (Raw)": 0.8592697978019714, "Finetune/Step": 3392, "Finetune/Step Time": 5.273154217749834} +{"Finetune/Learning Rate": 5.683225415505521e-06, "Finetune/Loss": 0.875636100769043, "Finetune/Loss (Raw)": 0.9959274530410767, "Finetune/Step": 3393, "Finetune/Step Time": 5.275523141026497} +{"Finetune/Learning Rate": 5.67760697005253e-06, "Finetune/Loss": 0.8757709860801697, "Finetune/Loss (Raw)": 0.9089937210083008, "Finetune/Step": 3394, "Finetune/Step Time": 5.295740718021989} +{"Finetune/Learning Rate": 5.671990202035886e-06, "Finetune/Loss": 0.876194179058075, "Finetune/Loss (Raw)": 0.9873074293136597, "Finetune/Step": 3395, "Finetune/Step Time": 5.279249457642436} +{"Finetune/Learning Rate": 5.666375113635347e-06, "Finetune/Loss": 0.8745320439338684, "Finetune/Loss (Raw)": 0.7587355971336365, "Finetune/Step": 3396, "Finetune/Step Time": 5.27915820479393} +{"Finetune/Learning Rate": 5.660761707030022e-06, "Finetune/Loss": 0.8739327192306519, "Finetune/Loss (Raw)": 0.891196608543396, "Finetune/Step": 3397, "Finetune/Step Time": 5.282569844275713} +{"Finetune/Learning Rate": 5.655149984398359e-06, "Finetune/Loss": 0.8742771148681641, "Finetune/Loss (Raw)": 0.8908594250679016, "Finetune/Step": 3398, "Finetune/Step Time": 5.286721671000123} +{"Finetune/Learning Rate": 5.649539947918159e-06, "Finetune/Loss": 0.8738824725151062, "Finetune/Loss (Raw)": 0.8257126212120056, "Finetune/Step": 3399, "Finetune/Step Time": 5.295594647526741} +{"Finetune/Learning Rate": 5.643931599766577e-06, "Finetune/Loss": 0.8730737566947937, "Finetune/Loss (Raw)": 0.8457075357437134, "Finetune/Step": 3400, "Finetune/Step Time": 5.2687159683555365} +{"Finetune/Learning Rate": 5.638324942120091e-06, "Finetune/Loss": 0.8731459975242615, "Finetune/Loss (Raw)": 0.8157414793968201, "Finetune/Step": 3401, "Finetune/Step Time": 5.2605824656784534} +{"Finetune/Learning Rate": 5.632719977154537e-06, "Finetune/Loss": 0.8775503039360046, "Finetune/Loss (Raw)": 0.8870903253555298, "Finetune/Step": 3402, "Finetune/Step Time": 5.22926464304328} +{"Finetune/Learning Rate": 5.6271167070451016e-06, "Finetune/Loss": 0.8779128789901733, "Finetune/Loss (Raw)": 0.8319600820541382, "Finetune/Step": 3403, "Finetune/Step Time": 5.224962102249265} +{"Finetune/Learning Rate": 5.621515133966301e-06, "Finetune/Loss": 0.8779733180999756, "Finetune/Loss (Raw)": 0.9082401394844055, "Finetune/Step": 3404, "Finetune/Step Time": 5.2213126718997955} +{"Finetune/Learning Rate": 5.615915260091987e-06, "Finetune/Loss": 0.8764570355415344, "Finetune/Loss (Raw)": 0.7477081418037415, "Finetune/Step": 3405, "Finetune/Step Time": 5.292297473177314} +{"Finetune/Learning Rate": 5.610317087595372e-06, "Finetune/Loss": 0.8769718408584595, "Finetune/Loss (Raw)": 0.9243507981300354, "Finetune/Step": 3406, "Finetune/Step Time": 5.290248131379485} +{"Finetune/Learning Rate": 5.604720618648993e-06, "Finetune/Loss": 0.8765109181404114, "Finetune/Loss (Raw)": 0.9001597166061401, "Finetune/Step": 3407, "Finetune/Step Time": 5.277245061472058} +{"Finetune/Learning Rate": 5.599125855424734e-06, "Finetune/Loss": 0.876164436340332, "Finetune/Loss (Raw)": 0.944899320602417, "Finetune/Step": 3408, "Finetune/Step Time": 5.277785563841462} +{"Finetune/Learning Rate": 5.593532800093808e-06, "Finetune/Loss": 0.8750662803649902, "Finetune/Loss (Raw)": 0.8847745656967163, "Finetune/Step": 3409, "Finetune/Step Time": 5.275727452710271} +{"Finetune/Learning Rate": 5.5879414548267754e-06, "Finetune/Loss": 0.8747484683990479, "Finetune/Loss (Raw)": 0.8263260126113892, "Finetune/Step": 3410, "Finetune/Step Time": 5.279342053458095} +{"Finetune/Learning Rate": 5.582351821793525e-06, "Finetune/Loss": 0.8797453045845032, "Finetune/Loss (Raw)": 1.00934636592865, "Finetune/Step": 3411, "Finetune/Step Time": 5.248271260410547} +{"Finetune/Learning Rate": 5.576763903163281e-06, "Finetune/Loss": 0.8804045915603638, "Finetune/Loss (Raw)": 0.9516640305519104, "Finetune/Step": 3412, "Finetune/Step Time": 5.243670081719756} +{"Finetune/Learning Rate": 5.571177701104614e-06, "Finetune/Loss": 0.8807914853096008, "Finetune/Loss (Raw)": 1.0241386890411377, "Finetune/Step": 3413, "Finetune/Step Time": 5.251340055838227} +{"Finetune/Learning Rate": 5.56559321778542e-06, "Finetune/Loss": 0.8821362257003784, "Finetune/Loss (Raw)": 1.0406724214553833, "Finetune/Step": 3414, "Finetune/Step Time": 5.249645566567779} +{"Finetune/Learning Rate": 5.560010455372915e-06, "Finetune/Loss": 0.8808059692382812, "Finetune/Loss (Raw)": 0.7465390563011169, "Finetune/Step": 3415, "Finetune/Step Time": 5.247085142880678} +{"Finetune/Learning Rate": 5.5544294160336735e-06, "Finetune/Loss": 0.8764630556106567, "Finetune/Loss (Raw)": 0.42568421363830566, "Finetune/Step": 3416, "Finetune/Step Time": 5.275163754820824} +{"Finetune/Learning Rate": 5.5488501019335824e-06, "Finetune/Loss": 0.877189040184021, "Finetune/Loss (Raw)": 0.9036365151405334, "Finetune/Step": 3417, "Finetune/Step Time": 5.281363353133202} +{"Finetune/Learning Rate": 5.5432725152378675e-06, "Finetune/Loss": 0.8768431544303894, "Finetune/Loss (Raw)": 0.8959422707557678, "Finetune/Step": 3418, "Finetune/Step Time": 5.2930122427642345} +{"Finetune/Learning Rate": 5.537696658111077e-06, "Finetune/Loss": 0.8778864145278931, "Finetune/Loss (Raw)": 0.913632869720459, "Finetune/Step": 3419, "Finetune/Step Time": 5.285261910408735} +{"Finetune/Learning Rate": 5.532122532717095e-06, "Finetune/Loss": 0.8763293623924255, "Finetune/Loss (Raw)": 0.6490911841392517, "Finetune/Step": 3420, "Finetune/Step Time": 5.315606143325567} +{"Finetune/Learning Rate": 5.52655014121913e-06, "Finetune/Loss": 0.8762552738189697, "Finetune/Loss (Raw)": 0.9180924296379089, "Finetune/Step": 3421, "Finetune/Step Time": 5.312423575669527} +{"Finetune/Learning Rate": 5.5209794857797135e-06, "Finetune/Loss": 0.876516580581665, "Finetune/Loss (Raw)": 0.8714137077331543, "Finetune/Step": 3422, "Finetune/Step Time": 5.305287415161729} +{"Finetune/Learning Rate": 5.51541056856072e-06, "Finetune/Loss": 0.8764058351516724, "Finetune/Loss (Raw)": 0.818877637386322, "Finetune/Step": 3423, "Finetune/Step Time": 5.303906712681055} +{"Finetune/Learning Rate": 5.509843391723332e-06, "Finetune/Loss": 0.8718825578689575, "Finetune/Loss (Raw)": 0.3493388891220093, "Finetune/Step": 3424, "Finetune/Step Time": 5.328974064439535} +{"Finetune/Learning Rate": 5.504277957428053e-06, "Finetune/Loss": 0.8723392486572266, "Finetune/Loss (Raw)": 0.8872565031051636, "Finetune/Step": 3425, "Finetune/Step Time": 5.315689442679286} +{"Finetune/Learning Rate": 5.498714267834731e-06, "Finetune/Loss": 0.8703906536102295, "Finetune/Loss (Raw)": 0.7407923936843872, "Finetune/Step": 3426, "Finetune/Step Time": 5.3177880719304085} +{"Finetune/Learning Rate": 5.493152325102523e-06, "Finetune/Loss": 0.8750002980232239, "Finetune/Loss (Raw)": 0.9746344089508057, "Finetune/Step": 3427, "Finetune/Step Time": 5.290235850960016} +{"Finetune/Learning Rate": 5.487592131389907e-06, "Finetune/Loss": 0.8747768402099609, "Finetune/Loss (Raw)": 0.8931353688240051, "Finetune/Step": 3428, "Finetune/Step Time": 5.299011372029781} +{"Finetune/Learning Rate": 5.482033688854689e-06, "Finetune/Loss": 0.8761927485466003, "Finetune/Loss (Raw)": 0.9281160831451416, "Finetune/Step": 3429, "Finetune/Step Time": 5.2954747918993235} +{"Finetune/Learning Rate": 5.47647699965399e-06, "Finetune/Loss": 0.8752591609954834, "Finetune/Loss (Raw)": 0.8482913374900818, "Finetune/Step": 3430, "Finetune/Step Time": 5.289639225229621} +{"Finetune/Learning Rate": 5.470922065944253e-06, "Finetune/Loss": 0.8762120008468628, "Finetune/Loss (Raw)": 0.9860043525695801, "Finetune/Step": 3431, "Finetune/Step Time": 5.288118088617921} +{"Finetune/Learning Rate": 5.465368889881235e-06, "Finetune/Loss": 0.875745415687561, "Finetune/Loss (Raw)": 0.7811543345451355, "Finetune/Step": 3432, "Finetune/Step Time": 5.290085529908538} +{"Finetune/Learning Rate": 5.459817473620024e-06, "Finetune/Loss": 0.8747881650924683, "Finetune/Loss (Raw)": 0.8679344058036804, "Finetune/Step": 3433, "Finetune/Step Time": 5.307877913117409} +{"Finetune/Learning Rate": 5.454267819315016e-06, "Finetune/Loss": 0.8749663233757019, "Finetune/Loss (Raw)": 0.8642825484275818, "Finetune/Step": 3434, "Finetune/Step Time": 5.332550130784512} +{"Finetune/Learning Rate": 5.448719929119916e-06, "Finetune/Loss": 0.8736618757247925, "Finetune/Loss (Raw)": 0.8647644519805908, "Finetune/Step": 3435, "Finetune/Step Time": 5.351435700431466} +{"Finetune/Learning Rate": 5.44317380518775e-06, "Finetune/Loss": 0.872083306312561, "Finetune/Loss (Raw)": 0.7857378125190735, "Finetune/Step": 3436, "Finetune/Step Time": 5.344749171286821} +{"Finetune/Learning Rate": 5.43762944967087e-06, "Finetune/Loss": 0.871381402015686, "Finetune/Loss (Raw)": 0.8974502086639404, "Finetune/Step": 3437, "Finetune/Step Time": 5.333513718098402} +{"Finetune/Learning Rate": 5.432086864720929e-06, "Finetune/Loss": 0.8714706301689148, "Finetune/Loss (Raw)": 0.8300409913063049, "Finetune/Step": 3438, "Finetune/Step Time": 5.331945531070232} +{"Finetune/Learning Rate": 5.426546052488892e-06, "Finetune/Loss": 0.8728616237640381, "Finetune/Loss (Raw)": 0.9820734858512878, "Finetune/Step": 3439, "Finetune/Step Time": 5.337054818868637} +{"Finetune/Learning Rate": 5.421007015125046e-06, "Finetune/Loss": 0.8729649782180786, "Finetune/Loss (Raw)": 0.8640464544296265, "Finetune/Step": 3440, "Finetune/Step Time": 5.338892661035061} +{"Finetune/Learning Rate": 5.415469754778979e-06, "Finetune/Loss": 0.8752807974815369, "Finetune/Loss (Raw)": 0.7283244132995605, "Finetune/Step": 3441, "Finetune/Step Time": 5.330114221200347} +{"Finetune/Learning Rate": 5.409934273599594e-06, "Finetune/Loss": 0.8751043081283569, "Finetune/Loss (Raw)": 0.7793294191360474, "Finetune/Step": 3442, "Finetune/Step Time": 5.337164705619216} +{"Finetune/Learning Rate": 5.404400573735105e-06, "Finetune/Loss": 0.8728815317153931, "Finetune/Loss (Raw)": 0.8274590969085693, "Finetune/Step": 3443, "Finetune/Step Time": 5.347337802872062} +{"Finetune/Learning Rate": 5.3988686573330316e-06, "Finetune/Loss": 0.8736175298690796, "Finetune/Loss (Raw)": 0.923294186592102, "Finetune/Step": 3444, "Finetune/Step Time": 5.341189630329609} +{"Finetune/Learning Rate": 5.393338526540202e-06, "Finetune/Loss": 0.8721531629562378, "Finetune/Loss (Raw)": 0.7159671783447266, "Finetune/Step": 3445, "Finetune/Step Time": 5.3591135293245316} +{"Finetune/Learning Rate": 5.387810183502749e-06, "Finetune/Loss": 0.8762322068214417, "Finetune/Loss (Raw)": 0.8569353222846985, "Finetune/Step": 3446, "Finetune/Step Time": 5.331619963049889} +{"Finetune/Learning Rate": 5.382283630366123e-06, "Finetune/Loss": 0.8764238357543945, "Finetune/Loss (Raw)": 1.0011205673217773, "Finetune/Step": 3447, "Finetune/Step Time": 5.321293391287327} +{"Finetune/Learning Rate": 5.37675886927507e-06, "Finetune/Loss": 0.876602053642273, "Finetune/Loss (Raw)": 0.8895617127418518, "Finetune/Step": 3448, "Finetune/Step Time": 5.327144714072347} +{"Finetune/Learning Rate": 5.371235902373632e-06, "Finetune/Loss": 0.8778736591339111, "Finetune/Loss (Raw)": 0.9882370829582214, "Finetune/Step": 3449, "Finetune/Step Time": 5.322492061182857} +{"Finetune/Learning Rate": 5.365714731805174e-06, "Finetune/Loss": 0.8783202171325684, "Finetune/Loss (Raw)": 0.9104698300361633, "Finetune/Step": 3450, "Finetune/Step Time": 5.342652011662722} +{"Finetune/Learning Rate": 5.360195359712353e-06, "Finetune/Loss": 0.8769916296005249, "Finetune/Loss (Raw)": 0.8420141339302063, "Finetune/Step": 3451, "Finetune/Step Time": 5.325470816344023} +{"Finetune/Learning Rate": 5.354677788237126e-06, "Finetune/Loss": 0.8780490159988403, "Finetune/Loss (Raw)": 0.9756826162338257, "Finetune/Step": 3452, "Finetune/Step Time": 5.315246302634478} +{"Finetune/Learning Rate": 5.349162019520759e-06, "Finetune/Loss": 0.878288745880127, "Finetune/Loss (Raw)": 0.8465567827224731, "Finetune/Step": 3453, "Finetune/Step Time": 5.319066520780325} +{"Finetune/Learning Rate": 5.343648055703812e-06, "Finetune/Loss": 0.877452552318573, "Finetune/Loss (Raw)": 0.8411868214607239, "Finetune/Step": 3454, "Finetune/Step Time": 5.3221981301903725} +{"Finetune/Learning Rate": 5.338135898926147e-06, "Finetune/Loss": 0.8743340969085693, "Finetune/Loss (Raw)": 0.3415811061859131, "Finetune/Step": 3455, "Finetune/Step Time": 5.342916790395975} +{"Finetune/Learning Rate": 5.33262555132692e-06, "Finetune/Loss": 0.8737454414367676, "Finetune/Loss (Raw)": 0.947903037071228, "Finetune/Step": 3456, "Finetune/Step Time": 5.338919498026371} +{"Finetune/Learning Rate": 5.327117015044599e-06, "Finetune/Loss": 0.8726969361305237, "Finetune/Loss (Raw)": 0.8878542184829712, "Finetune/Step": 3457, "Finetune/Step Time": 5.3291415348649025} +{"Finetune/Learning Rate": 5.321610292216937e-06, "Finetune/Loss": 0.8735435009002686, "Finetune/Loss (Raw)": 0.9085007309913635, "Finetune/Step": 3458, "Finetune/Step Time": 5.33703051880002} +{"Finetune/Learning Rate": 5.316105384980974e-06, "Finetune/Loss": 0.8729994297027588, "Finetune/Loss (Raw)": 0.8556600213050842, "Finetune/Step": 3459, "Finetune/Step Time": 5.322028433904052} +{"Finetune/Learning Rate": 5.3106022954730704e-06, "Finetune/Loss": 0.8736880421638489, "Finetune/Loss (Raw)": 0.9802968502044678, "Finetune/Step": 3460, "Finetune/Step Time": 5.323938004672527} +{"Finetune/Learning Rate": 5.3051010258288624e-06, "Finetune/Loss": 0.8689570426940918, "Finetune/Loss (Raw)": 0.3350529372692108, "Finetune/Step": 3461, "Finetune/Step Time": 5.359900940209627} +{"Finetune/Learning Rate": 5.299601578183289e-06, "Finetune/Loss": 0.869358479976654, "Finetune/Loss (Raw)": 0.9396689534187317, "Finetune/Step": 3462, "Finetune/Step Time": 5.355721855536103} +{"Finetune/Learning Rate": 5.294103954670573e-06, "Finetune/Loss": 0.8695628643035889, "Finetune/Loss (Raw)": 0.8688789010047913, "Finetune/Step": 3463, "Finetune/Step Time": 5.363288914784789} +{"Finetune/Learning Rate": 5.288608157424238e-06, "Finetune/Loss": 0.8700755834579468, "Finetune/Loss (Raw)": 0.9339909553527832, "Finetune/Step": 3464, "Finetune/Step Time": 5.36437726393342} +{"Finetune/Learning Rate": 5.283114188577094e-06, "Finetune/Loss": 0.870472252368927, "Finetune/Loss (Raw)": 0.8341608047485352, "Finetune/Step": 3465, "Finetune/Step Time": 5.359497021883726} +{"Finetune/Learning Rate": 5.277622050261242e-06, "Finetune/Loss": 0.8699712157249451, "Finetune/Loss (Raw)": 0.8295562267303467, "Finetune/Step": 3466, "Finetune/Step Time": 5.357956564053893} +{"Finetune/Learning Rate": 5.272131744608078e-06, "Finetune/Loss": 0.8682581782341003, "Finetune/Loss (Raw)": 0.7396564483642578, "Finetune/Step": 3467, "Finetune/Step Time": 5.35656083188951} +{"Finetune/Learning Rate": 5.2666432737482844e-06, "Finetune/Loss": 0.8669053316116333, "Finetune/Loss (Raw)": 0.7328267693519592, "Finetune/Step": 3468, "Finetune/Step Time": 5.36519967764616} +{"Finetune/Learning Rate": 5.261156639811817e-06, "Finetune/Loss": 0.8660380840301514, "Finetune/Loss (Raw)": 0.9244091510772705, "Finetune/Step": 3469, "Finetune/Step Time": 5.373766887933016} +{"Finetune/Learning Rate": 5.255671844927944e-06, "Finetune/Loss": 0.864783525466919, "Finetune/Loss (Raw)": 0.8438486456871033, "Finetune/Step": 3470, "Finetune/Step Time": 5.380334636196494} +{"Finetune/Learning Rate": 5.250188891225203e-06, "Finetune/Loss": 0.8648355007171631, "Finetune/Loss (Raw)": 0.9415781497955322, "Finetune/Step": 3471, "Finetune/Step Time": 5.386596200987697} +{"Finetune/Learning Rate": 5.24470778083142e-06, "Finetune/Loss": 0.8654974699020386, "Finetune/Loss (Raw)": 0.8077382445335388, "Finetune/Step": 3472, "Finetune/Step Time": 5.387087393552065} +{"Finetune/Learning Rate": 5.2392285158737085e-06, "Finetune/Loss": 0.8708227872848511, "Finetune/Loss (Raw)": 1.0337789058685303, "Finetune/Step": 3473, "Finetune/Step Time": 5.375763967633247} +{"Finetune/Learning Rate": 5.233751098478465e-06, "Finetune/Loss": 0.8690614700317383, "Finetune/Loss (Raw)": 0.7979398965835571, "Finetune/Step": 3474, "Finetune/Step Time": 5.3812870141118765} +{"Finetune/Learning Rate": 5.228275530771366e-06, "Finetune/Loss": 0.8699932098388672, "Finetune/Loss (Raw)": 0.8891239762306213, "Finetune/Step": 3475, "Finetune/Step Time": 5.387391831725836} +{"Finetune/Learning Rate": 5.22280181487737e-06, "Finetune/Loss": 0.8684472441673279, "Finetune/Loss (Raw)": 0.7489024996757507, "Finetune/Step": 3476, "Finetune/Step Time": 5.392197348177433} +{"Finetune/Learning Rate": 5.217329952920727e-06, "Finetune/Loss": 0.8674970269203186, "Finetune/Loss (Raw)": 0.8241035342216492, "Finetune/Step": 3477, "Finetune/Step Time": 5.410332191735506} +{"Finetune/Learning Rate": 5.2118599470249575e-06, "Finetune/Loss": 0.8669869899749756, "Finetune/Loss (Raw)": 1.002658724784851, "Finetune/Step": 3478, "Finetune/Step Time": 5.4029245264828205} +{"Finetune/Learning Rate": 5.206391799312855e-06, "Finetune/Loss": 0.8686769008636475, "Finetune/Loss (Raw)": 1.0614819526672363, "Finetune/Step": 3479, "Finetune/Step Time": 5.389724684879184} +{"Finetune/Learning Rate": 5.200925511906513e-06, "Finetune/Loss": 0.8685716986656189, "Finetune/Loss (Raw)": 0.9222886562347412, "Finetune/Step": 3480, "Finetune/Step Time": 5.38174806535244} +{"Finetune/Learning Rate": 5.195461086927284e-06, "Finetune/Loss": 0.868475615978241, "Finetune/Loss (Raw)": 0.959880530834198, "Finetune/Step": 3481, "Finetune/Step Time": 5.358472481369972} +{"Finetune/Learning Rate": 5.189998526495808e-06, "Finetune/Loss": 0.8682286739349365, "Finetune/Loss (Raw)": 0.9038795232772827, "Finetune/Step": 3482, "Finetune/Step Time": 5.358281049877405} +{"Finetune/Learning Rate": 5.184537832731996e-06, "Finetune/Loss": 0.8675930500030518, "Finetune/Loss (Raw)": 0.8940977454185486, "Finetune/Step": 3483, "Finetune/Step Time": 5.397071804851294} +{"Finetune/Learning Rate": 5.1790790077550366e-06, "Finetune/Loss": 0.8674755096435547, "Finetune/Loss (Raw)": 0.922747790813446, "Finetune/Step": 3484, "Finetune/Step Time": 5.413381772115827} +{"Finetune/Learning Rate": 5.173622053683395e-06, "Finetune/Loss": 0.8684754371643066, "Finetune/Loss (Raw)": 0.9982881546020508, "Finetune/Step": 3485, "Finetune/Step Time": 5.410395288839936} +{"Finetune/Learning Rate": 5.168166972634803e-06, "Finetune/Loss": 0.8694876432418823, "Finetune/Loss (Raw)": 0.9435416460037231, "Finetune/Step": 3486, "Finetune/Step Time": 5.404591124504805} +{"Finetune/Learning Rate": 5.1627137667262815e-06, "Finetune/Loss": 0.8672081232070923, "Finetune/Loss (Raw)": 0.7636594176292419, "Finetune/Step": 3487, "Finetune/Step Time": 5.408558962866664} +{"Finetune/Learning Rate": 5.1572624380741044e-06, "Finetune/Loss": 0.8653319478034973, "Finetune/Loss (Raw)": 0.7641896605491638, "Finetune/Step": 3488, "Finetune/Step Time": 5.407967112958431} +{"Finetune/Learning Rate": 5.151812988793825e-06, "Finetune/Loss": 0.865654468536377, "Finetune/Loss (Raw)": 0.9928979873657227, "Finetune/Step": 3489, "Finetune/Step Time": 5.3899152763187885} +{"Finetune/Learning Rate": 5.146365421000275e-06, "Finetune/Loss": 0.8653942346572876, "Finetune/Loss (Raw)": 0.7967087030410767, "Finetune/Step": 3490, "Finetune/Step Time": 5.388154834508896} +{"Finetune/Learning Rate": 5.140919736807546e-06, "Finetune/Loss": 0.8658062815666199, "Finetune/Loss (Raw)": 0.9884505271911621, "Finetune/Step": 3491, "Finetune/Step Time": 5.38380010984838} +{"Finetune/Learning Rate": 5.135475938329002e-06, "Finetune/Loss": 0.8654305934906006, "Finetune/Loss (Raw)": 0.920701801776886, "Finetune/Step": 3492, "Finetune/Step Time": 5.3931916654109955} +{"Finetune/Learning Rate": 5.1300340276772756e-06, "Finetune/Loss": 0.8641809225082397, "Finetune/Loss (Raw)": 0.7140433192253113, "Finetune/Step": 3493, "Finetune/Step Time": 5.385333847254515} +{"Finetune/Learning Rate": 5.124594006964263e-06, "Finetune/Loss": 0.8655688166618347, "Finetune/Loss (Raw)": 0.9973289966583252, "Finetune/Step": 3494, "Finetune/Step Time": 5.372483888641} +{"Finetune/Learning Rate": 5.1191558783011315e-06, "Finetune/Loss": 0.8664801120758057, "Finetune/Loss (Raw)": 0.9353290796279907, "Finetune/Step": 3495, "Finetune/Step Time": 5.37226128578186} +{"Finetune/Learning Rate": 5.113719643798312e-06, "Finetune/Loss": 0.8705908060073853, "Finetune/Loss (Raw)": 0.9631661176681519, "Finetune/Step": 3496, "Finetune/Step Time": 5.349252697080374} +{"Finetune/Learning Rate": 5.108285305565508e-06, "Finetune/Loss": 0.868971586227417, "Finetune/Loss (Raw)": 0.7789080739021301, "Finetune/Step": 3497, "Finetune/Step Time": 5.358284233137965} +{"Finetune/Learning Rate": 5.1028528657116715e-06, "Finetune/Loss": 0.8686591982841492, "Finetune/Loss (Raw)": 0.9135500192642212, "Finetune/Step": 3498, "Finetune/Step Time": 5.361972264945507} +{"Finetune/Learning Rate": 5.097422326345024e-06, "Finetune/Loss": 0.8708091974258423, "Finetune/Loss (Raw)": 0.967147946357727, "Finetune/Step": 3499, "Finetune/Step Time": 5.34192849881947} +{"Finetune/Learning Rate": 5.091993689573062e-06, "Finetune/Loss": 0.8713997602462769, "Finetune/Loss (Raw)": 0.9505864381790161, "Finetune/Step": 3500, "Finetune/Step Time": 5.332571808248758} +{"Finetune/Learning Rate": 5.086566957502527e-06, "Finetune/Loss": 0.8713435530662537, "Finetune/Loss (Raw)": 0.9439182281494141, "Finetune/Step": 3501, "Finetune/Step Time": 5.3183380626142025} +{"Finetune/Learning Rate": 5.081142132239429e-06, "Finetune/Loss": 0.8725764751434326, "Finetune/Loss (Raw)": 1.0618404150009155, "Finetune/Step": 3502, "Finetune/Step Time": 5.3190587144345045} +{"Finetune/Learning Rate": 5.075719215889039e-06, "Finetune/Loss": 0.8712778091430664, "Finetune/Loss (Raw)": 0.9643475413322449, "Finetune/Step": 3503, "Finetune/Step Time": 5.319238696247339} +{"Finetune/Learning Rate": 5.070298210555881e-06, "Finetune/Loss": 0.8700037002563477, "Finetune/Loss (Raw)": 0.8495714068412781, "Finetune/Step": 3504, "Finetune/Step Time": 5.320562576875091} +{"Finetune/Learning Rate": 5.064879118343744e-06, "Finetune/Loss": 0.8711293935775757, "Finetune/Loss (Raw)": 0.9560562372207642, "Finetune/Step": 3505, "Finetune/Step Time": 5.316540678963065} +{"Finetune/Learning Rate": 5.059461941355665e-06, "Finetune/Loss": 0.8699443340301514, "Finetune/Loss (Raw)": 0.8355700373649597, "Finetune/Step": 3506, "Finetune/Step Time": 5.313160503283143} +{"Finetune/Learning Rate": 5.05404668169396e-06, "Finetune/Loss": 0.8696903586387634, "Finetune/Loss (Raw)": 0.6769121289253235, "Finetune/Step": 3507, "Finetune/Step Time": 5.328221736475825} +{"Finetune/Learning Rate": 5.048633341460174e-06, "Finetune/Loss": 0.8704382181167603, "Finetune/Loss (Raw)": 0.8854675889015198, "Finetune/Step": 3508, "Finetune/Step Time": 5.322991164401174} +{"Finetune/Learning Rate": 5.043221922755115e-06, "Finetune/Loss": 0.8711776733398438, "Finetune/Loss (Raw)": 0.979070246219635, "Finetune/Step": 3509, "Finetune/Step Time": 5.336556909605861} +{"Finetune/Learning Rate": 5.03781242767886e-06, "Finetune/Loss": 0.8693157434463501, "Finetune/Loss (Raw)": 0.8032057881355286, "Finetune/Step": 3510, "Finetune/Step Time": 5.335044918581843} +{"Finetune/Learning Rate": 5.032404858330721e-06, "Finetune/Loss": 0.8695375919342041, "Finetune/Loss (Raw)": 0.8826867938041687, "Finetune/Step": 3511, "Finetune/Step Time": 5.342097735032439} +{"Finetune/Learning Rate": 5.026999216809275e-06, "Finetune/Loss": 0.8692922592163086, "Finetune/Loss (Raw)": 0.9084761142730713, "Finetune/Step": 3512, "Finetune/Step Time": 5.34013026766479} +{"Finetune/Learning Rate": 5.021595505212342e-06, "Finetune/Loss": 0.8705331087112427, "Finetune/Loss (Raw)": 0.9448891282081604, "Finetune/Step": 3513, "Finetune/Step Time": 5.33999353274703} +{"Finetune/Learning Rate": 5.016193725637e-06, "Finetune/Loss": 0.8699674010276794, "Finetune/Loss (Raw)": 0.8092532753944397, "Finetune/Step": 3514, "Finetune/Step Time": 5.338856492191553} +{"Finetune/Learning Rate": 5.0107938801795695e-06, "Finetune/Loss": 0.8703991770744324, "Finetune/Loss (Raw)": 0.858695924282074, "Finetune/Step": 3515, "Finetune/Step Time": 5.345506232231855} +{"Finetune/Learning Rate": 5.005395970935627e-06, "Finetune/Loss": 0.8708934783935547, "Finetune/Loss (Raw)": 0.88297438621521, "Finetune/Step": 3516, "Finetune/Step Time": 5.346873469650745} +{"Finetune/Learning Rate": 5.000000000000003e-06, "Finetune/Loss": 0.8710287809371948, "Finetune/Loss (Raw)": 0.9497314691543579, "Finetune/Step": 3517, "Finetune/Step Time": 5.350232116878033} +{"Finetune/Learning Rate": 4.994605969466758e-06, "Finetune/Loss": 0.8704562187194824, "Finetune/Loss (Raw)": 0.8835306763648987, "Finetune/Step": 3518, "Finetune/Step Time": 5.3798019122332335} +{"Finetune/Learning Rate": 4.989213881429211e-06, "Finetune/Loss": 0.8684568405151367, "Finetune/Loss (Raw)": 0.8088489770889282, "Finetune/Step": 3519, "Finetune/Step Time": 5.403565134853125} +{"Finetune/Learning Rate": 4.983823737979933e-06, "Finetune/Loss": 0.868963897228241, "Finetune/Loss (Raw)": 0.9241698384284973, "Finetune/Step": 3520, "Finetune/Step Time": 5.399639017879963} +{"Finetune/Learning Rate": 4.97843554121073e-06, "Finetune/Loss": 0.8691672086715698, "Finetune/Loss (Raw)": 1.0219569206237793, "Finetune/Step": 3521, "Finetune/Step Time": 5.394281620159745} +{"Finetune/Learning Rate": 4.973049293212655e-06, "Finetune/Loss": 0.8682408332824707, "Finetune/Loss (Raw)": 0.7904089093208313, "Finetune/Step": 3522, "Finetune/Step Time": 5.389845238998532} +{"Finetune/Learning Rate": 4.967664996076007e-06, "Finetune/Loss": 0.8667628169059753, "Finetune/Loss (Raw)": 0.7981293797492981, "Finetune/Step": 3523, "Finetune/Step Time": 5.414918942376971} +{"Finetune/Learning Rate": 4.962282651890325e-06, "Finetune/Loss": 0.8669836521148682, "Finetune/Loss (Raw)": 0.7870001196861267, "Finetune/Step": 3524, "Finetune/Step Time": 5.429519673809409} +{"Finetune/Learning Rate": 4.9569022627443944e-06, "Finetune/Loss": 0.8682250380516052, "Finetune/Loss (Raw)": 1.050094485282898, "Finetune/Step": 3525, "Finetune/Step Time": 5.433195853605866} +{"Finetune/Learning Rate": 4.951523830726236e-06, "Finetune/Loss": 0.868035078048706, "Finetune/Loss (Raw)": 0.8665397763252258, "Finetune/Step": 3526, "Finetune/Step Time": 5.435865191742778} +{"Finetune/Learning Rate": 4.946147357923116e-06, "Finetune/Loss": 0.8690193891525269, "Finetune/Loss (Raw)": 0.9517083168029785, "Finetune/Step": 3527, "Finetune/Step Time": 5.423899706453085} +{"Finetune/Learning Rate": 4.940772846421538e-06, "Finetune/Loss": 0.8694149255752563, "Finetune/Loss (Raw)": 0.8963363170623779, "Finetune/Step": 3528, "Finetune/Step Time": 5.42327188141644} +{"Finetune/Learning Rate": 4.935400298307241e-06, "Finetune/Loss": 0.869423508644104, "Finetune/Loss (Raw)": 0.8168476819992065, "Finetune/Step": 3529, "Finetune/Step Time": 5.433364329859614} +{"Finetune/Learning Rate": 4.930029715665215e-06, "Finetune/Loss": 0.8696523904800415, "Finetune/Loss (Raw)": 0.9163761138916016, "Finetune/Step": 3530, "Finetune/Step Time": 5.440357457846403} +{"Finetune/Learning Rate": 4.9246611005796776e-06, "Finetune/Loss": 0.8707948327064514, "Finetune/Loss (Raw)": 0.9781962037086487, "Finetune/Step": 3531, "Finetune/Step Time": 5.440298777073622} +{"Finetune/Learning Rate": 4.919294455134071e-06, "Finetune/Loss": 0.8700839281082153, "Finetune/Loss (Raw)": 0.8172416687011719, "Finetune/Step": 3532, "Finetune/Step Time": 5.452848888933659} +{"Finetune/Learning Rate": 4.913929781411099e-06, "Finetune/Loss": 0.8667550086975098, "Finetune/Loss (Raw)": 0.321610689163208, "Finetune/Step": 3533, "Finetune/Step Time": 5.414867110550404} +{"Finetune/Learning Rate": 4.90856708149268e-06, "Finetune/Loss": 0.8667759895324707, "Finetune/Loss (Raw)": 0.927034854888916, "Finetune/Step": 3534, "Finetune/Step Time": 5.421308254823089} +{"Finetune/Learning Rate": 4.903206357459976e-06, "Finetune/Loss": 0.8623876571655273, "Finetune/Loss (Raw)": 0.3384536802768707, "Finetune/Step": 3535, "Finetune/Step Time": 5.451973345130682} +{"Finetune/Learning Rate": 4.897847611393378e-06, "Finetune/Loss": 0.8615716695785522, "Finetune/Loss (Raw)": 0.8404520750045776, "Finetune/Step": 3536, "Finetune/Step Time": 5.449064014479518} +{"Finetune/Learning Rate": 4.892490845372511e-06, "Finetune/Loss": 0.8626503944396973, "Finetune/Loss (Raw)": 1.0228482484817505, "Finetune/Step": 3537, "Finetune/Step Time": 5.4560943972319365} +{"Finetune/Learning Rate": 4.8871360614762295e-06, "Finetune/Loss": 0.8640204071998596, "Finetune/Loss (Raw)": 1.0016882419586182, "Finetune/Step": 3538, "Finetune/Step Time": 5.458368455991149} +{"Finetune/Learning Rate": 4.881783261782621e-06, "Finetune/Loss": 0.8635433912277222, "Finetune/Loss (Raw)": 0.9482967853546143, "Finetune/Step": 3539, "Finetune/Step Time": 5.462108705192804} +{"Finetune/Learning Rate": 4.876432448369006e-06, "Finetune/Loss": 0.8630196452140808, "Finetune/Loss (Raw)": 0.8846227526664734, "Finetune/Step": 3540, "Finetune/Step Time": 5.466618051752448} +{"Finetune/Learning Rate": 4.871083623311933e-06, "Finetune/Loss": 0.8619592189788818, "Finetune/Loss (Raw)": 0.8883985280990601, "Finetune/Step": 3541, "Finetune/Step Time": 5.457494614645839} +{"Finetune/Learning Rate": 4.865736788687164e-06, "Finetune/Loss": 0.8606134653091431, "Finetune/Loss (Raw)": 0.8684192299842834, "Finetune/Step": 3542, "Finetune/Step Time": 5.467249974608421} +{"Finetune/Learning Rate": 4.860391946569714e-06, "Finetune/Loss": 0.8616559505462646, "Finetune/Loss (Raw)": 0.8799799680709839, "Finetune/Step": 3543, "Finetune/Step Time": 5.479973206296563} +{"Finetune/Learning Rate": 4.855049099033806e-06, "Finetune/Loss": 0.8655624389648438, "Finetune/Loss (Raw)": 0.9257168769836426, "Finetune/Step": 3544, "Finetune/Step Time": 5.448433600366116} +{"Finetune/Learning Rate": 4.849708248152897e-06, "Finetune/Loss": 0.8617908954620361, "Finetune/Loss (Raw)": 0.42087095975875854, "Finetune/Step": 3545, "Finetune/Step Time": 5.4745175037533045} +{"Finetune/Learning Rate": 4.8443693959996655e-06, "Finetune/Loss": 0.8612608909606934, "Finetune/Loss (Raw)": 0.8281082510948181, "Finetune/Step": 3546, "Finetune/Step Time": 5.470898317173123} +{"Finetune/Learning Rate": 4.839032544646016e-06, "Finetune/Loss": 0.862030565738678, "Finetune/Loss (Raw)": 1.0121488571166992, "Finetune/Step": 3547, "Finetune/Step Time": 5.468131344765425} +{"Finetune/Learning Rate": 4.833697696163077e-06, "Finetune/Loss": 0.864296555519104, "Finetune/Loss (Raw)": 0.939139723777771, "Finetune/Step": 3548, "Finetune/Step Time": 5.45040000602603} +{"Finetune/Learning Rate": 4.828364852621195e-06, "Finetune/Loss": 0.8649394512176514, "Finetune/Loss (Raw)": 1.0003786087036133, "Finetune/Step": 3549, "Finetune/Step Time": 5.449291663244367} +{"Finetune/Learning Rate": 4.823034016089949e-06, "Finetune/Loss": 0.8608081936836243, "Finetune/Loss (Raw)": 0.3426172137260437, "Finetune/Step": 3550, "Finetune/Step Time": 5.459518602117896} +{"Finetune/Learning Rate": 4.8177051886381345e-06, "Finetune/Loss": 0.8605324029922485, "Finetune/Loss (Raw)": 0.7835777997970581, "Finetune/Step": 3551, "Finetune/Step Time": 5.477276226505637} +{"Finetune/Learning Rate": 4.812378372333751e-06, "Finetune/Loss": 0.8649539947509766, "Finetune/Loss (Raw)": 0.9153029322624207, "Finetune/Step": 3552, "Finetune/Step Time": 5.443666730076075} +{"Finetune/Learning Rate": 4.807053569244047e-06, "Finetune/Loss": 0.863813579082489, "Finetune/Loss (Raw)": 0.7412789463996887, "Finetune/Step": 3553, "Finetune/Step Time": 5.451814660802484} +{"Finetune/Learning Rate": 4.801730781435469e-06, "Finetune/Loss": 0.8655909299850464, "Finetune/Loss (Raw)": 0.9682962894439697, "Finetune/Step": 3554, "Finetune/Step Time": 5.454437803477049} +{"Finetune/Learning Rate": 4.796410010973686e-06, "Finetune/Loss": 0.8663194179534912, "Finetune/Loss (Raw)": 1.0678844451904297, "Finetune/Step": 3555, "Finetune/Step Time": 5.452208500355482} +{"Finetune/Learning Rate": 4.791091259923587e-06, "Finetune/Loss": 0.8620758056640625, "Finetune/Loss (Raw)": 0.34994620084762573, "Finetune/Step": 3556, "Finetune/Step Time": 5.479893019422889} +{"Finetune/Learning Rate": 4.785774530349275e-06, "Finetune/Loss": 0.8629683256149292, "Finetune/Loss (Raw)": 1.0423671007156372, "Finetune/Step": 3557, "Finetune/Step Time": 5.4777659475803375} +{"Finetune/Learning Rate": 4.7804598243140664e-06, "Finetune/Loss": 0.8633131980895996, "Finetune/Loss (Raw)": 0.8924349546432495, "Finetune/Step": 3558, "Finetune/Step Time": 5.47957793623209} +{"Finetune/Learning Rate": 4.775147143880496e-06, "Finetune/Loss": 0.863178551197052, "Finetune/Loss (Raw)": 0.968767523765564, "Finetune/Step": 3559, "Finetune/Step Time": 5.478338299319148} +{"Finetune/Learning Rate": 4.769836491110314e-06, "Finetune/Loss": 0.8649693131446838, "Finetune/Loss (Raw)": 1.010369062423706, "Finetune/Step": 3560, "Finetune/Step Time": 5.462261110544205} +{"Finetune/Learning Rate": 4.764527868064484e-06, "Finetune/Loss": 0.8663158416748047, "Finetune/Loss (Raw)": 1.0402870178222656, "Finetune/Step": 3561, "Finetune/Step Time": 5.445798143744469} +{"Finetune/Learning Rate": 4.759221276803172e-06, "Finetune/Loss": 0.8652958869934082, "Finetune/Loss (Raw)": 0.7337236404418945, "Finetune/Step": 3562, "Finetune/Step Time": 5.420874122530222} +{"Finetune/Learning Rate": 4.75391671938576e-06, "Finetune/Loss": 0.8645734786987305, "Finetune/Loss (Raw)": 0.7722997069358826, "Finetune/Step": 3563, "Finetune/Step Time": 5.40133873000741} +{"Finetune/Learning Rate": 4.748614197870852e-06, "Finetune/Loss": 0.864869236946106, "Finetune/Loss (Raw)": 0.8235957026481628, "Finetune/Step": 3564, "Finetune/Step Time": 5.404499106109142} +{"Finetune/Learning Rate": 4.743313714316249e-06, "Finetune/Loss": 0.8644764423370361, "Finetune/Loss (Raw)": 0.8471728563308716, "Finetune/Step": 3565, "Finetune/Step Time": 5.407337624579668} +{"Finetune/Learning Rate": 4.738015270778965e-06, "Finetune/Loss": 0.86592698097229, "Finetune/Loss (Raw)": 1.0157063007354736, "Finetune/Step": 3566, "Finetune/Step Time": 5.407019782811403} +{"Finetune/Learning Rate": 4.7327188693152235e-06, "Finetune/Loss": 0.865229606628418, "Finetune/Loss (Raw)": 0.8928163647651672, "Finetune/Step": 3567, "Finetune/Step Time": 5.415562231093645} +{"Finetune/Learning Rate": 4.727424511980453e-06, "Finetune/Loss": 0.8658801317214966, "Finetune/Loss (Raw)": 0.947307288646698, "Finetune/Step": 3568, "Finetune/Step Time": 5.4206338301301} +{"Finetune/Learning Rate": 4.722132200829291e-06, "Finetune/Loss": 0.8669074773788452, "Finetune/Loss (Raw)": 0.8598324656486511, "Finetune/Step": 3569, "Finetune/Step Time": 5.407954182475805} +{"Finetune/Learning Rate": 4.716841937915582e-06, "Finetune/Loss": 0.8674037456512451, "Finetune/Loss (Raw)": 0.8428500294685364, "Finetune/Step": 3570, "Finetune/Step Time": 5.402710389345884} +{"Finetune/Learning Rate": 4.71155372529237e-06, "Finetune/Loss": 0.8682098984718323, "Finetune/Loss (Raw)": 0.9306448698043823, "Finetune/Step": 3571, "Finetune/Step Time": 5.394654288887978} +{"Finetune/Learning Rate": 4.706267565011909e-06, "Finetune/Loss": 0.8683696389198303, "Finetune/Loss (Raw)": 0.9437406659126282, "Finetune/Step": 3572, "Finetune/Step Time": 5.396123584359884} +{"Finetune/Learning Rate": 4.700983459125652e-06, "Finetune/Loss": 0.8701203465461731, "Finetune/Loss (Raw)": 0.9400601983070374, "Finetune/Step": 3573, "Finetune/Step Time": 5.37819485552609} +{"Finetune/Learning Rate": 4.695701409684264e-06, "Finetune/Loss": 0.871102511882782, "Finetune/Loss (Raw)": 0.9826518893241882, "Finetune/Step": 3574, "Finetune/Step Time": 5.376333540305495} +{"Finetune/Learning Rate": 4.690421418737604e-06, "Finetune/Loss": 0.8700765371322632, "Finetune/Loss (Raw)": 0.8697980642318726, "Finetune/Step": 3575, "Finetune/Step Time": 5.377201981842518} +{"Finetune/Learning Rate": 4.685143488334723e-06, "Finetune/Loss": 0.8708083629608154, "Finetune/Loss (Raw)": 0.9832347631454468, "Finetune/Step": 3576, "Finetune/Step Time": 5.375233102589846} +{"Finetune/Learning Rate": 4.679867620523895e-06, "Finetune/Loss": 0.870011568069458, "Finetune/Loss (Raw)": 0.8862435221672058, "Finetune/Step": 3577, "Finetune/Step Time": 5.393999516963959} +{"Finetune/Learning Rate": 4.674593817352575e-06, "Finetune/Loss": 0.8705059885978699, "Finetune/Loss (Raw)": 0.9737535715103149, "Finetune/Step": 3578, "Finetune/Step Time": 5.378699786961079} +{"Finetune/Learning Rate": 4.669322080867427e-06, "Finetune/Loss": 0.8714429140090942, "Finetune/Loss (Raw)": 0.9619439244270325, "Finetune/Step": 3579, "Finetune/Step Time": 5.379722835496068} +{"Finetune/Learning Rate": 4.664052413114305e-06, "Finetune/Loss": 0.8711408376693726, "Finetune/Loss (Raw)": 0.9370127320289612, "Finetune/Step": 3580, "Finetune/Step Time": 5.381969725713134} +{"Finetune/Learning Rate": 4.658784816138266e-06, "Finetune/Loss": 0.87218177318573, "Finetune/Loss (Raw)": 0.9798016548156738, "Finetune/Step": 3581, "Finetune/Step Time": 5.390883579850197} +{"Finetune/Learning Rate": 4.653519291983561e-06, "Finetune/Loss": 0.8722217679023743, "Finetune/Loss (Raw)": 0.8463008999824524, "Finetune/Step": 3582, "Finetune/Step Time": 5.393146740272641} +{"Finetune/Learning Rate": 4.648255842693635e-06, "Finetune/Loss": 0.8759821653366089, "Finetune/Loss (Raw)": 0.8229092955589294, "Finetune/Step": 3583, "Finetune/Step Time": 5.3626400642097} +{"Finetune/Learning Rate": 4.642994470311136e-06, "Finetune/Loss": 0.87592613697052, "Finetune/Loss (Raw)": 0.9407392740249634, "Finetune/Step": 3584, "Finetune/Step Time": 5.3648489229381084} +{"Finetune/Learning Rate": 4.6377351768779e-06, "Finetune/Loss": 0.8770538568496704, "Finetune/Loss (Raw)": 1.0322014093399048, "Finetune/Step": 3585, "Finetune/Step Time": 5.364775829017162} +{"Finetune/Learning Rate": 4.632477964434944e-06, "Finetune/Loss": 0.8759770393371582, "Finetune/Loss (Raw)": 0.7706691026687622, "Finetune/Step": 3586, "Finetune/Step Time": 5.36318838223815} +{"Finetune/Learning Rate": 4.6272228350225026e-06, "Finetune/Loss": 0.8757753372192383, "Finetune/Loss (Raw)": 0.829846978187561, "Finetune/Step": 3587, "Finetune/Step Time": 5.356417680159211} +{"Finetune/Learning Rate": 4.621969790679984e-06, "Finetune/Loss": 0.8736286163330078, "Finetune/Loss (Raw)": 0.70550936460495, "Finetune/Step": 3588, "Finetune/Step Time": 5.357282567769289} +{"Finetune/Learning Rate": 4.6167188334459936e-06, "Finetune/Loss": 0.877833366394043, "Finetune/Loss (Raw)": 0.8732638359069824, "Finetune/Step": 3589, "Finetune/Step Time": 5.3353828229010105} +{"Finetune/Learning Rate": 4.611469965358323e-06, "Finetune/Loss": 0.8779497146606445, "Finetune/Loss (Raw)": 0.9545602798461914, "Finetune/Step": 3590, "Finetune/Step Time": 5.339325154200196} +{"Finetune/Learning Rate": 4.606223188453956e-06, "Finetune/Loss": 0.8774178624153137, "Finetune/Loss (Raw)": 0.8007998466491699, "Finetune/Step": 3591, "Finetune/Step Time": 5.332660969346762} +{"Finetune/Learning Rate": 4.6009785047690666e-06, "Finetune/Loss": 0.8774128556251526, "Finetune/Loss (Raw)": 0.933349609375, "Finetune/Step": 3592, "Finetune/Step Time": 5.337780095636845} +{"Finetune/Learning Rate": 4.595735916339008e-06, "Finetune/Loss": 0.8788844347000122, "Finetune/Loss (Raw)": 1.0225236415863037, "Finetune/Step": 3593, "Finetune/Step Time": 5.334684593603015} +{"Finetune/Learning Rate": 4.590495425198335e-06, "Finetune/Loss": 0.8772190809249878, "Finetune/Loss (Raw)": 0.6163902878761292, "Finetune/Step": 3594, "Finetune/Step Time": 5.346552599221468} +{"Finetune/Learning Rate": 4.585257033380779e-06, "Finetune/Loss": 0.8784763216972351, "Finetune/Loss (Raw)": 0.9005836248397827, "Finetune/Step": 3595, "Finetune/Step Time": 5.357994629070163} +{"Finetune/Learning Rate": 4.580020742919247e-06, "Finetune/Loss": 0.880804717540741, "Finetune/Loss (Raw)": 1.030860185623169, "Finetune/Step": 3596, "Finetune/Step Time": 5.343528684228659} +{"Finetune/Learning Rate": 4.5747865558458524e-06, "Finetune/Loss": 0.8755598068237305, "Finetune/Loss (Raw)": 0.2530622184276581, "Finetune/Step": 3597, "Finetune/Step Time": 5.363243360072374} +{"Finetune/Learning Rate": 4.569554474191878e-06, "Finetune/Loss": 0.8754538297653198, "Finetune/Loss (Raw)": 0.8302804231643677, "Finetune/Step": 3598, "Finetune/Step Time": 5.377346139401197} +{"Finetune/Learning Rate": 4.56432449998779e-06, "Finetune/Loss": 0.8744591474533081, "Finetune/Loss (Raw)": 0.8142545819282532, "Finetune/Step": 3599, "Finetune/Step Time": 5.368879416957498} +{"Finetune/Learning Rate": 4.559096635263242e-06, "Finetune/Loss": 0.8715563416481018, "Finetune/Loss (Raw)": 0.4361848533153534, "Finetune/Step": 3600, "Finetune/Step Time": 5.3918984569609165} +{"Finetune/Learning Rate": 4.553870882047064e-06, "Finetune/Loss": 0.8707955479621887, "Finetune/Loss (Raw)": 0.936398983001709, "Finetune/Step": 3601, "Finetune/Step Time": 5.373981237411499} +{"Finetune/Learning Rate": 4.548647242367271e-06, "Finetune/Loss": 0.8712035417556763, "Finetune/Loss (Raw)": 0.8501680493354797, "Finetune/Step": 3602, "Finetune/Step Time": 5.3690741918981075} +{"Finetune/Learning Rate": 4.543425718251051e-06, "Finetune/Loss": 0.8709379434585571, "Finetune/Loss (Raw)": 0.8551223278045654, "Finetune/Step": 3603, "Finetune/Step Time": 5.367700751870871} +{"Finetune/Learning Rate": 4.538206311724782e-06, "Finetune/Loss": 0.872222900390625, "Finetune/Loss (Raw)": 0.9133755564689636, "Finetune/Step": 3604, "Finetune/Step Time": 5.3645076639950275} +{"Finetune/Learning Rate": 4.532989024814015e-06, "Finetune/Loss": 0.8719198703765869, "Finetune/Loss (Raw)": 0.7853102684020996, "Finetune/Step": 3605, "Finetune/Step Time": 5.350684981793165} +{"Finetune/Learning Rate": 4.527773859543466e-06, "Finetune/Loss": 0.8709341287612915, "Finetune/Loss (Raw)": 0.8764875531196594, "Finetune/Step": 3606, "Finetune/Step Time": 5.363205393776298} +{"Finetune/Learning Rate": 4.52256081793705e-06, "Finetune/Loss": 0.869815468788147, "Finetune/Loss (Raw)": 0.9182896018028259, "Finetune/Step": 3607, "Finetune/Step Time": 5.366437830030918} +{"Finetune/Learning Rate": 4.517349902017841e-06, "Finetune/Loss": 0.8695347309112549, "Finetune/Loss (Raw)": 0.886353611946106, "Finetune/Step": 3608, "Finetune/Step Time": 5.394026651978493} +{"Finetune/Learning Rate": 4.512141113808097e-06, "Finetune/Loss": 0.8691636919975281, "Finetune/Loss (Raw)": 0.9123938679695129, "Finetune/Step": 3609, "Finetune/Step Time": 5.39513254724443} +{"Finetune/Learning Rate": 4.5069344553292436e-06, "Finetune/Loss": 0.868314802646637, "Finetune/Loss (Raw)": 0.7952215075492859, "Finetune/Step": 3610, "Finetune/Step Time": 5.394659759476781} +{"Finetune/Learning Rate": 4.501729928601887e-06, "Finetune/Loss": 0.8639909029006958, "Finetune/Loss (Raw)": 0.34063827991485596, "Finetune/Step": 3611, "Finetune/Step Time": 5.383263656869531} +{"Finetune/Learning Rate": 4.4965275356458e-06, "Finetune/Loss": 0.8635388612747192, "Finetune/Loss (Raw)": 0.8648830056190491, "Finetune/Step": 3612, "Finetune/Step Time": 5.380263864994049} +{"Finetune/Learning Rate": 4.491327278479924e-06, "Finetune/Loss": 0.8619405031204224, "Finetune/Loss (Raw)": 0.7937073111534119, "Finetune/Step": 3613, "Finetune/Step Time": 5.381238587200642} +{"Finetune/Learning Rate": 4.4861291591223934e-06, "Finetune/Loss": 0.8613677024841309, "Finetune/Loss (Raw)": 0.8702123761177063, "Finetune/Step": 3614, "Finetune/Step Time": 5.385223243385553} +{"Finetune/Learning Rate": 4.4809331795904835e-06, "Finetune/Loss": 0.8625520467758179, "Finetune/Loss (Raw)": 0.9152610301971436, "Finetune/Step": 3615, "Finetune/Step Time": 5.384311817586422} +{"Finetune/Learning Rate": 4.47573934190065e-06, "Finetune/Loss": 0.8626549243927002, "Finetune/Loss (Raw)": 0.7773544192314148, "Finetune/Step": 3616, "Finetune/Step Time": 5.388632325455546} +{"Finetune/Learning Rate": 4.470547648068532e-06, "Finetune/Loss": 0.8616518974304199, "Finetune/Loss (Raw)": 0.8645108938217163, "Finetune/Step": 3617, "Finetune/Step Time": 5.39728026650846} +{"Finetune/Learning Rate": 4.465358100108916e-06, "Finetune/Loss": 0.8619818091392517, "Finetune/Loss (Raw)": 0.8389413356781006, "Finetune/Step": 3618, "Finetune/Step Time": 5.4048984330147505} +{"Finetune/Learning Rate": 4.460170700035769e-06, "Finetune/Loss": 0.8609002232551575, "Finetune/Loss (Raw)": 0.8500025868415833, "Finetune/Step": 3619, "Finetune/Step Time": 5.402720630168915} +{"Finetune/Learning Rate": 4.454985449862215e-06, "Finetune/Loss": 0.8602876663208008, "Finetune/Loss (Raw)": 0.8422985076904297, "Finetune/Step": 3620, "Finetune/Step Time": 5.3928393349051476} +{"Finetune/Learning Rate": 4.449802351600551e-06, "Finetune/Loss": 0.8610994815826416, "Finetune/Loss (Raw)": 0.8179543018341064, "Finetune/Step": 3621, "Finetune/Step Time": 5.386534228920937} +{"Finetune/Learning Rate": 4.444621407262237e-06, "Finetune/Loss": 0.8603823184967041, "Finetune/Loss (Raw)": 0.9055303931236267, "Finetune/Step": 3622, "Finetune/Step Time": 5.38539963401854} +{"Finetune/Learning Rate": 4.439442618857891e-06, "Finetune/Loss": 0.8602563142776489, "Finetune/Loss (Raw)": 0.9192001819610596, "Finetune/Step": 3623, "Finetune/Step Time": 5.386106027290225} +{"Finetune/Learning Rate": 4.434265988397312e-06, "Finetune/Loss": 0.8604162931442261, "Finetune/Loss (Raw)": 0.9836485385894775, "Finetune/Step": 3624, "Finetune/Step Time": 5.385582692921162} +{"Finetune/Learning Rate": 4.4290915178894355e-06, "Finetune/Loss": 0.8618280291557312, "Finetune/Loss (Raw)": 0.9596048593521118, "Finetune/Step": 3625, "Finetune/Step Time": 5.376540409401059} +{"Finetune/Learning Rate": 4.423919209342376e-06, "Finetune/Loss": 0.860959529876709, "Finetune/Loss (Raw)": 0.8023853898048401, "Finetune/Step": 3626, "Finetune/Step Time": 5.373221281915903} +{"Finetune/Learning Rate": 4.41874906476341e-06, "Finetune/Loss": 0.8605331182479858, "Finetune/Loss (Raw)": 0.9125586152076721, "Finetune/Step": 3627, "Finetune/Step Time": 5.388340367004275} +{"Finetune/Learning Rate": 4.413581086158969e-06, "Finetune/Loss": 0.8601000308990479, "Finetune/Loss (Raw)": 0.8951589465141296, "Finetune/Step": 3628, "Finetune/Step Time": 5.393804645165801} +{"Finetune/Learning Rate": 4.408415275534641e-06, "Finetune/Loss": 0.8592283725738525, "Finetune/Loss (Raw)": 0.8323454260826111, "Finetune/Step": 3629, "Finetune/Step Time": 5.4060530085116625} +{"Finetune/Learning Rate": 4.4032516348951795e-06, "Finetune/Loss": 0.8576239347457886, "Finetune/Loss (Raw)": 0.8564726114273071, "Finetune/Step": 3630, "Finetune/Step Time": 5.400811357423663} +{"Finetune/Learning Rate": 4.3980901662444905e-06, "Finetune/Loss": 0.855705201625824, "Finetune/Loss (Raw)": 0.718752920627594, "Finetune/Step": 3631, "Finetune/Step Time": 5.414718655869365} +{"Finetune/Learning Rate": 4.39293087158564e-06, "Finetune/Loss": 0.8567918539047241, "Finetune/Loss (Raw)": 0.9886614084243774, "Finetune/Step": 3632, "Finetune/Step Time": 5.417303519323468} +{"Finetune/Learning Rate": 4.387773752920845e-06, "Finetune/Loss": 0.8571100234985352, "Finetune/Loss (Raw)": 0.996772289276123, "Finetune/Step": 3633, "Finetune/Step Time": 5.41167557798326} +{"Finetune/Learning Rate": 4.382618812251495e-06, "Finetune/Loss": 0.8574479818344116, "Finetune/Loss (Raw)": 0.8788339495658875, "Finetune/Step": 3634, "Finetune/Step Time": 5.404286060482264} +{"Finetune/Learning Rate": 4.377466051578111e-06, "Finetune/Loss": 0.8582170009613037, "Finetune/Loss (Raw)": 0.775351345539093, "Finetune/Step": 3635, "Finetune/Step Time": 5.388263467699289} +{"Finetune/Learning Rate": 4.372315472900377e-06, "Finetune/Loss": 0.8584029674530029, "Finetune/Loss (Raw)": 0.9092725515365601, "Finetune/Step": 3636, "Finetune/Step Time": 5.387561811134219} +{"Finetune/Learning Rate": 4.367167078217141e-06, "Finetune/Loss": 0.8583501577377319, "Finetune/Loss (Raw)": 0.9723029732704163, "Finetune/Step": 3637, "Finetune/Step Time": 5.37322136759758} +{"Finetune/Learning Rate": 4.36202086952639e-06, "Finetune/Loss": 0.8594616651535034, "Finetune/Loss (Raw)": 0.9454861283302307, "Finetune/Step": 3638, "Finetune/Step Time": 5.371412685140967} +{"Finetune/Learning Rate": 4.356876848825266e-06, "Finetune/Loss": 0.859510064125061, "Finetune/Loss (Raw)": 0.888880729675293, "Finetune/Step": 3639, "Finetune/Step Time": 5.360418409109116} +{"Finetune/Learning Rate": 4.351735018110066e-06, "Finetune/Loss": 0.8548400402069092, "Finetune/Loss (Raw)": 0.31071895360946655, "Finetune/Step": 3640, "Finetune/Step Time": 5.391725920140743} +{"Finetune/Learning Rate": 4.346595379376232e-06, "Finetune/Loss": 0.8537911176681519, "Finetune/Loss (Raw)": 0.810617208480835, "Finetune/Step": 3641, "Finetune/Step Time": 5.394772224128246} +{"Finetune/Learning Rate": 4.341457934618357e-06, "Finetune/Loss": 0.8549648523330688, "Finetune/Loss (Raw)": 0.9594937562942505, "Finetune/Step": 3642, "Finetune/Step Time": 5.398139392957091} +{"Finetune/Learning Rate": 4.336322685830181e-06, "Finetune/Loss": 0.8554819822311401, "Finetune/Loss (Raw)": 0.924892008304596, "Finetune/Step": 3643, "Finetune/Step Time": 5.397012945264578} +{"Finetune/Learning Rate": 4.331189635004604e-06, "Finetune/Loss": 0.8551431894302368, "Finetune/Loss (Raw)": 0.8396133184432983, "Finetune/Step": 3644, "Finetune/Step Time": 5.3939069006592035} +{"Finetune/Learning Rate": 4.326058784133652e-06, "Finetune/Loss": 0.854171872138977, "Finetune/Loss (Raw)": 0.8253987431526184, "Finetune/Step": 3645, "Finetune/Step Time": 5.387791162356734} +{"Finetune/Learning Rate": 4.3209301352085075e-06, "Finetune/Loss": 0.8549402952194214, "Finetune/Loss (Raw)": 0.981885552406311, "Finetune/Step": 3646, "Finetune/Step Time": 5.367642490193248} +{"Finetune/Learning Rate": 4.315803690219507e-06, "Finetune/Loss": 0.8558366298675537, "Finetune/Loss (Raw)": 0.923576295375824, "Finetune/Step": 3647, "Finetune/Step Time": 5.346047211438417} +{"Finetune/Learning Rate": 4.310679451156122e-06, "Finetune/Loss": 0.8560879230499268, "Finetune/Loss (Raw)": 0.9563356637954712, "Finetune/Step": 3648, "Finetune/Step Time": 5.347826676443219} +{"Finetune/Learning Rate": 4.305557420006962e-06, "Finetune/Loss": 0.8555817604064941, "Finetune/Loss (Raw)": 0.957171618938446, "Finetune/Step": 3649, "Finetune/Step Time": 5.352095533162355} +{"Finetune/Learning Rate": 4.300437598759794e-06, "Finetune/Loss": 0.8574082851409912, "Finetune/Loss (Raw)": 1.0242060422897339, "Finetune/Step": 3650, "Finetune/Step Time": 5.337926080450416} +{"Finetune/Learning Rate": 4.295319989401522e-06, "Finetune/Loss": 0.8570213317871094, "Finetune/Loss (Raw)": 0.7485957741737366, "Finetune/Step": 3651, "Finetune/Step Time": 5.332623843103647} +{"Finetune/Learning Rate": 4.2902045939181855e-06, "Finetune/Loss": 0.8582820296287537, "Finetune/Loss (Raw)": 0.9483697414398193, "Finetune/Step": 3652, "Finetune/Step Time": 5.325196053832769} +{"Finetune/Learning Rate": 4.285091414294973e-06, "Finetune/Loss": 0.8575719594955444, "Finetune/Loss (Raw)": 0.9592075347900391, "Finetune/Step": 3653, "Finetune/Step Time": 5.316946979612112} +{"Finetune/Learning Rate": 4.2799804525162085e-06, "Finetune/Loss": 0.8578219413757324, "Finetune/Loss (Raw)": 0.8985394835472107, "Finetune/Step": 3654, "Finetune/Step Time": 5.336855670437217} +{"Finetune/Learning Rate": 4.274871710565356e-06, "Finetune/Loss": 0.8586393594741821, "Finetune/Loss (Raw)": 1.0563346147537231, "Finetune/Step": 3655, "Finetune/Step Time": 5.341533752158284} +{"Finetune/Learning Rate": 4.269765190425015e-06, "Finetune/Loss": 0.8577818870544434, "Finetune/Loss (Raw)": 0.7865787744522095, "Finetune/Step": 3656, "Finetune/Step Time": 5.345790360122919} +{"Finetune/Learning Rate": 4.264660894076934e-06, "Finetune/Loss": 0.8582162857055664, "Finetune/Loss (Raw)": 0.8724532723426819, "Finetune/Step": 3657, "Finetune/Step Time": 5.340593863278627} +{"Finetune/Learning Rate": 4.25955882350199e-06, "Finetune/Loss": 0.8580446243286133, "Finetune/Loss (Raw)": 0.8944004774093628, "Finetune/Step": 3658, "Finetune/Step Time": 5.350735398009419} +{"Finetune/Learning Rate": 4.254458980680188e-06, "Finetune/Loss": 0.8575231432914734, "Finetune/Loss (Raw)": 0.9114500284194946, "Finetune/Step": 3659, "Finetune/Step Time": 5.353599088266492} +{"Finetune/Learning Rate": 4.2493613675906865e-06, "Finetune/Loss": 0.8589853048324585, "Finetune/Loss (Raw)": 1.0043933391571045, "Finetune/Step": 3660, "Finetune/Step Time": 5.340674851089716} +{"Finetune/Learning Rate": 4.244265986211766e-06, "Finetune/Loss": 0.8633143901824951, "Finetune/Loss (Raw)": 0.8757388591766357, "Finetune/Step": 3661, "Finetune/Step Time": 5.308623718097806} +{"Finetune/Learning Rate": 4.239172838520846e-06, "Finetune/Loss": 0.8637153506278992, "Finetune/Loss (Raw)": 0.9783539772033691, "Finetune/Step": 3662, "Finetune/Step Time": 5.299643114209175} +{"Finetune/Learning Rate": 4.2340819264944755e-06, "Finetune/Loss": 0.8693815469741821, "Finetune/Loss (Raw)": 1.0637273788452148, "Finetune/Step": 3663, "Finetune/Step Time": 5.266757857054472} +{"Finetune/Learning Rate": 4.22899325210834e-06, "Finetune/Loss": 0.8692923784255981, "Finetune/Loss (Raw)": 0.8290409445762634, "Finetune/Step": 3664, "Finetune/Step Time": 5.279818955808878} +{"Finetune/Learning Rate": 4.223906817337253e-06, "Finetune/Loss": 0.8684515953063965, "Finetune/Loss (Raw)": 0.9152357578277588, "Finetune/Step": 3665, "Finetune/Step Time": 5.271790489554405} +{"Finetune/Learning Rate": 4.218822624155159e-06, "Finetune/Loss": 0.8678947687149048, "Finetune/Loss (Raw)": 0.9304055571556091, "Finetune/Step": 3666, "Finetune/Step Time": 5.266795571893454} +{"Finetune/Learning Rate": 4.213740674535141e-06, "Finetune/Loss": 0.8638018369674683, "Finetune/Loss (Raw)": 0.42440417408943176, "Finetune/Step": 3667, "Finetune/Step Time": 5.294732293114066} +{"Finetune/Learning Rate": 4.2086609704494015e-06, "Finetune/Loss": 0.8633701205253601, "Finetune/Loss (Raw)": 0.8293619751930237, "Finetune/Step": 3668, "Finetune/Step Time": 5.284329921007156} +{"Finetune/Learning Rate": 4.203583513869267e-06, "Finetune/Loss": 0.8631241917610168, "Finetune/Loss (Raw)": 0.8569241166114807, "Finetune/Step": 3669, "Finetune/Step Time": 5.2879948522895575} +{"Finetune/Learning Rate": 4.198508306765209e-06, "Finetune/Loss": 0.8634618520736694, "Finetune/Loss (Raw)": 0.9116363525390625, "Finetune/Step": 3670, "Finetune/Step Time": 5.273065511137247} +{"Finetune/Learning Rate": 4.193435351106811e-06, "Finetune/Loss": 0.8642774820327759, "Finetune/Loss (Raw)": 0.984381914138794, "Finetune/Step": 3671, "Finetune/Step Time": 5.2544818334281445} +{"Finetune/Learning Rate": 4.188364648862791e-06, "Finetune/Loss": 0.8635983467102051, "Finetune/Loss (Raw)": 0.8387788534164429, "Finetune/Step": 3672, "Finetune/Step Time": 5.262339036911726} +{"Finetune/Learning Rate": 4.183296202000988e-06, "Finetune/Loss": 0.8655796051025391, "Finetune/Loss (Raw)": 0.6744837760925293, "Finetune/Step": 3673, "Finetune/Step Time": 5.235533434897661} +{"Finetune/Learning Rate": 4.178230012488368e-06, "Finetune/Loss": 0.8657249212265015, "Finetune/Loss (Raw)": 0.8466982841491699, "Finetune/Step": 3674, "Finetune/Step Time": 5.226805604994297} +{"Finetune/Learning Rate": 4.173166082291019e-06, "Finetune/Loss": 0.8607590198516846, "Finetune/Loss (Raw)": 0.3765134811401367, "Finetune/Step": 3675, "Finetune/Step Time": 5.262180600315332} +{"Finetune/Learning Rate": 4.16810441337415e-06, "Finetune/Loss": 0.8608632683753967, "Finetune/Loss (Raw)": 0.9524858593940735, "Finetune/Step": 3676, "Finetune/Step Time": 5.2649644538760185} +{"Finetune/Learning Rate": 4.163045007702105e-06, "Finetune/Loss": 0.8606156706809998, "Finetune/Loss (Raw)": 0.9686877727508545, "Finetune/Step": 3677, "Finetune/Step Time": 5.264417048543692} +{"Finetune/Learning Rate": 4.157987867238338e-06, "Finetune/Loss": 0.863025963306427, "Finetune/Loss (Raw)": 0.6511342525482178, "Finetune/Step": 3678, "Finetune/Step Time": 5.234353126958013} +{"Finetune/Learning Rate": 4.152932993945418e-06, "Finetune/Loss": 0.8647592663764954, "Finetune/Loss (Raw)": 1.0054399967193604, "Finetune/Step": 3679, "Finetune/Step Time": 5.222678506746888} +{"Finetune/Learning Rate": 4.147880389785051e-06, "Finetune/Loss": 0.8647538423538208, "Finetune/Loss (Raw)": 0.914609968662262, "Finetune/Step": 3680, "Finetune/Step Time": 5.228380447253585} +{"Finetune/Learning Rate": 4.142830056718052e-06, "Finetune/Loss": 0.8665910959243774, "Finetune/Loss (Raw)": 0.9764535427093506, "Finetune/Step": 3681, "Finetune/Step Time": 5.209323678165674} +{"Finetune/Learning Rate": 4.137781996704356e-06, "Finetune/Loss": 0.8675884008407593, "Finetune/Loss (Raw)": 1.095946192741394, "Finetune/Step": 3682, "Finetune/Step Time": 5.211957493796945} +{"Finetune/Learning Rate": 4.1327362117030174e-06, "Finetune/Loss": 0.8662528991699219, "Finetune/Loss (Raw)": 0.896942138671875, "Finetune/Step": 3683, "Finetune/Step Time": 5.212676702067256} +{"Finetune/Learning Rate": 4.127692703672207e-06, "Finetune/Loss": 0.868739128112793, "Finetune/Loss (Raw)": 0.6681820154190063, "Finetune/Step": 3684, "Finetune/Step Time": 5.194487329572439} +{"Finetune/Learning Rate": 4.1226514745692115e-06, "Finetune/Loss": 0.8673704266548157, "Finetune/Loss (Raw)": 0.8671772480010986, "Finetune/Step": 3685, "Finetune/Step Time": 5.191279295831919} +{"Finetune/Learning Rate": 4.117612526350429e-06, "Finetune/Loss": 0.8675817847251892, "Finetune/Loss (Raw)": 0.919482409954071, "Finetune/Step": 3686, "Finetune/Step Time": 5.190746080130339} +{"Finetune/Learning Rate": 4.112575860971384e-06, "Finetune/Loss": 0.8669412732124329, "Finetune/Loss (Raw)": 0.886782705783844, "Finetune/Step": 3687, "Finetune/Step Time": 5.1891011986881495} +{"Finetune/Learning Rate": 4.107541480386708e-06, "Finetune/Loss": 0.8655096292495728, "Finetune/Loss (Raw)": 0.827121376991272, "Finetune/Step": 3688, "Finetune/Step Time": 5.1974969655275345} +{"Finetune/Learning Rate": 4.102509386550141e-06, "Finetune/Loss": 0.8651169538497925, "Finetune/Loss (Raw)": 0.9900292754173279, "Finetune/Step": 3689, "Finetune/Step Time": 5.1971335131675005} +{"Finetune/Learning Rate": 4.0974795814145384e-06, "Finetune/Loss": 0.8667073845863342, "Finetune/Loss (Raw)": 0.9372904300689697, "Finetune/Step": 3690, "Finetune/Step Time": 5.200416572391987} +{"Finetune/Learning Rate": 4.092452066931877e-06, "Finetune/Loss": 0.8630985617637634, "Finetune/Loss (Raw)": 0.3103717565536499, "Finetune/Step": 3691, "Finetune/Step Time": 5.230368092656136} +{"Finetune/Learning Rate": 4.087426845053236e-06, "Finetune/Loss": 0.8645039796829224, "Finetune/Loss (Raw)": 1.0034844875335693, "Finetune/Step": 3692, "Finetune/Step Time": 5.223601046949625} +{"Finetune/Learning Rate": 4.082403917728796e-06, "Finetune/Loss": 0.8649855256080627, "Finetune/Loss (Raw)": 0.9088155627250671, "Finetune/Step": 3693, "Finetune/Step Time": 5.222882917150855} +{"Finetune/Learning Rate": 4.077383286907869e-06, "Finetune/Loss": 0.8632067441940308, "Finetune/Loss (Raw)": 0.7880213856697083, "Finetune/Step": 3694, "Finetune/Step Time": 5.246664572507143} +{"Finetune/Learning Rate": 4.072364954538858e-06, "Finetune/Loss": 0.8649709820747375, "Finetune/Loss (Raw)": 1.1186379194259644, "Finetune/Step": 3695, "Finetune/Step Time": 5.220470456406474} +{"Finetune/Learning Rate": 4.067348922569281e-06, "Finetune/Loss": 0.8643757700920105, "Finetune/Loss (Raw)": 0.8711215853691101, "Finetune/Step": 3696, "Finetune/Step Time": 5.218929935246706} +{"Finetune/Learning Rate": 4.062335192945762e-06, "Finetune/Loss": 0.8633005619049072, "Finetune/Loss (Raw)": 0.7222113609313965, "Finetune/Step": 3697, "Finetune/Step Time": 5.235490631312132} +{"Finetune/Learning Rate": 4.057323767614032e-06, "Finetune/Loss": 0.863940954208374, "Finetune/Loss (Raw)": 0.9248161911964417, "Finetune/Step": 3698, "Finetune/Step Time": 5.2420224491506815} +{"Finetune/Learning Rate": 4.052314648518925e-06, "Finetune/Loss": 0.8638980388641357, "Finetune/Loss (Raw)": 0.9251548647880554, "Finetune/Step": 3699, "Finetune/Step Time": 5.239159278571606} +{"Finetune/Learning Rate": 4.047307837604383e-06, "Finetune/Loss": 0.8636858463287354, "Finetune/Loss (Raw)": 0.9165760278701782, "Finetune/Step": 3700, "Finetune/Step Time": 5.240317191928625} +{"Finetune/Learning Rate": 4.0423033368134545e-06, "Finetune/Loss": 0.8619905114173889, "Finetune/Loss (Raw)": 0.7230582237243652, "Finetune/Step": 3701, "Finetune/Step Time": 5.2478139996528625} +{"Finetune/Learning Rate": 4.037301148088292e-06, "Finetune/Loss": 0.8616558313369751, "Finetune/Loss (Raw)": 0.9398131966590881, "Finetune/Step": 3702, "Finetune/Step Time": 5.2368522342294455} +{"Finetune/Learning Rate": 4.032301273370132e-06, "Finetune/Loss": 0.8623360395431519, "Finetune/Loss (Raw)": 0.9568678140640259, "Finetune/Step": 3703, "Finetune/Step Time": 5.234643943607807} +{"Finetune/Learning Rate": 4.027303714599345e-06, "Finetune/Loss": 0.861846923828125, "Finetune/Loss (Raw)": 0.9206221699714661, "Finetune/Step": 3704, "Finetune/Step Time": 5.239742528647184} +{"Finetune/Learning Rate": 4.022308473715379e-06, "Finetune/Loss": 0.8606780767440796, "Finetune/Loss (Raw)": 0.7366345524787903, "Finetune/Step": 3705, "Finetune/Step Time": 5.22126692160964} +{"Finetune/Learning Rate": 4.017315552656788e-06, "Finetune/Loss": 0.8598493933677673, "Finetune/Loss (Raw)": 0.8676786422729492, "Finetune/Step": 3706, "Finetune/Step Time": 5.2233788557350636} +{"Finetune/Learning Rate": 4.01232495336123e-06, "Finetune/Loss": 0.8589931130409241, "Finetune/Loss (Raw)": 0.8523417711257935, "Finetune/Step": 3707, "Finetune/Step Time": 5.24933410063386} +{"Finetune/Learning Rate": 4.0073366777654574e-06, "Finetune/Loss": 0.8578603267669678, "Finetune/Loss (Raw)": 0.7920132279396057, "Finetune/Step": 3708, "Finetune/Step Time": 5.260615389794111} +{"Finetune/Learning Rate": 4.002350727805324e-06, "Finetune/Loss": 0.8581990003585815, "Finetune/Loss (Raw)": 1.0231598615646362, "Finetune/Step": 3709, "Finetune/Step Time": 5.251459447667003} +{"Finetune/Learning Rate": 3.997367105415775e-06, "Finetune/Loss": 0.8593758940696716, "Finetune/Loss (Raw)": 0.9969351291656494, "Finetune/Step": 3710, "Finetune/Step Time": 5.238278303295374} +{"Finetune/Learning Rate": 3.992385812530864e-06, "Finetune/Loss": 0.8591458797454834, "Finetune/Loss (Raw)": 0.7934682369232178, "Finetune/Step": 3711, "Finetune/Step Time": 5.249684391543269} +{"Finetune/Learning Rate": 3.987406851083735e-06, "Finetune/Loss": 0.8597372174263, "Finetune/Loss (Raw)": 1.016435980796814, "Finetune/Step": 3712, "Finetune/Step Time": 5.248880198225379} +{"Finetune/Learning Rate": 3.982430223006614e-06, "Finetune/Loss": 0.8539032936096191, "Finetune/Loss (Raw)": 0.2854587137699127, "Finetune/Step": 3713, "Finetune/Step Time": 5.2732531893998384} +{"Finetune/Learning Rate": 3.977455930230842e-06, "Finetune/Loss": 0.8550045490264893, "Finetune/Loss (Raw)": 0.9116273522377014, "Finetune/Step": 3714, "Finetune/Step Time": 5.260248554870486} +{"Finetune/Learning Rate": 3.972483974686845e-06, "Finetune/Loss": 0.8559707999229431, "Finetune/Loss (Raw)": 0.9535258412361145, "Finetune/Step": 3715, "Finetune/Step Time": 5.264588115736842} +{"Finetune/Learning Rate": 3.967514358304139e-06, "Finetune/Loss": 0.8567777872085571, "Finetune/Loss (Raw)": 0.8088099956512451, "Finetune/Step": 3716, "Finetune/Step Time": 5.293580546975136} +{"Finetune/Learning Rate": 3.962547083011338e-06, "Finetune/Loss": 0.8561139106750488, "Finetune/Loss (Raw)": 0.7882859706878662, "Finetune/Step": 3717, "Finetune/Step Time": 5.307425167411566} +{"Finetune/Learning Rate": 3.957582150736141e-06, "Finetune/Loss": 0.8548009395599365, "Finetune/Loss (Raw)": 0.7864941954612732, "Finetune/Step": 3718, "Finetune/Step Time": 5.311691930517554} +{"Finetune/Learning Rate": 3.952619563405343e-06, "Finetune/Loss": 0.8538662791252136, "Finetune/Loss (Raw)": 0.6811620593070984, "Finetune/Step": 3719, "Finetune/Step Time": 5.327432960271835} +{"Finetune/Learning Rate": 3.9476593229448245e-06, "Finetune/Loss": 0.8544051051139832, "Finetune/Loss (Raw)": 1.0023244619369507, "Finetune/Step": 3720, "Finetune/Step Time": 5.331431742757559} +{"Finetune/Learning Rate": 3.942701431279564e-06, "Finetune/Loss": 0.8540829420089722, "Finetune/Loss (Raw)": 0.9812851548194885, "Finetune/Step": 3721, "Finetune/Step Time": 5.330822303891182} +{"Finetune/Learning Rate": 3.937745890333623e-06, "Finetune/Loss": 0.8564695715904236, "Finetune/Loss (Raw)": 0.9218762516975403, "Finetune/Step": 3722, "Finetune/Step Time": 5.327792678028345} +{"Finetune/Learning Rate": 3.932792702030139e-06, "Finetune/Loss": 0.8570690751075745, "Finetune/Loss (Raw)": 0.9773252606391907, "Finetune/Step": 3723, "Finetune/Step Time": 5.325754772871733} +{"Finetune/Learning Rate": 3.92784186829136e-06, "Finetune/Loss": 0.8554320931434631, "Finetune/Loss (Raw)": 0.8213256001472473, "Finetune/Step": 3724, "Finetune/Step Time": 5.3263000305742025} +{"Finetune/Learning Rate": 3.9228933910386014e-06, "Finetune/Loss": 0.8610273003578186, "Finetune/Loss (Raw)": 0.9692500233650208, "Finetune/Step": 3725, "Finetune/Step Time": 5.2984109707176685} +{"Finetune/Learning Rate": 3.917947272192273e-06, "Finetune/Loss": 0.8575339317321777, "Finetune/Loss (Raw)": 0.3831264078617096, "Finetune/Step": 3726, "Finetune/Step Time": 5.311296569183469} +{"Finetune/Learning Rate": 3.913003513671866e-06, "Finetune/Loss": 0.857674241065979, "Finetune/Loss (Raw)": 0.8322115540504456, "Finetune/Step": 3727, "Finetune/Step Time": 5.315462155267596} +{"Finetune/Learning Rate": 3.908062117395956e-06, "Finetune/Loss": 0.8597115874290466, "Finetune/Loss (Raw)": 0.6969712972640991, "Finetune/Step": 3728, "Finetune/Step Time": 5.292498275637627} +{"Finetune/Learning Rate": 3.903123085282204e-06, "Finetune/Loss": 0.8596364855766296, "Finetune/Loss (Raw)": 0.9267826676368713, "Finetune/Step": 3729, "Finetune/Step Time": 5.301535850390792} +{"Finetune/Learning Rate": 3.898186419247347e-06, "Finetune/Loss": 0.8602254390716553, "Finetune/Loss (Raw)": 0.9255479574203491, "Finetune/Step": 3730, "Finetune/Step Time": 5.330550255253911} +{"Finetune/Learning Rate": 3.893252121207221e-06, "Finetune/Loss": 0.8608618378639221, "Finetune/Loss (Raw)": 0.9365866780281067, "Finetune/Step": 3731, "Finetune/Step Time": 5.330274010077119} +{"Finetune/Learning Rate": 3.888320193076719e-06, "Finetune/Loss": 0.8608152866363525, "Finetune/Loss (Raw)": 0.9074128270149231, "Finetune/Step": 3732, "Finetune/Step Time": 5.328680461272597} +{"Finetune/Learning Rate": 3.8833906367698245e-06, "Finetune/Loss": 0.8613264560699463, "Finetune/Loss (Raw)": 0.8507466912269592, "Finetune/Step": 3733, "Finetune/Step Time": 5.336166566237807} +{"Finetune/Learning Rate": 3.878463454199613e-06, "Finetune/Loss": 0.8620612621307373, "Finetune/Loss (Raw)": 0.9705418944358826, "Finetune/Step": 3734, "Finetune/Step Time": 5.328720849007368} +{"Finetune/Learning Rate": 3.87353864727822e-06, "Finetune/Loss": 0.8604389429092407, "Finetune/Loss (Raw)": 0.7106321454048157, "Finetune/Step": 3735, "Finetune/Step Time": 5.329962894320488} +{"Finetune/Learning Rate": 3.8686162179168695e-06, "Finetune/Loss": 0.8620301485061646, "Finetune/Loss (Raw)": 1.090028166770935, "Finetune/Step": 3736, "Finetune/Step Time": 5.2999240178614855} +{"Finetune/Learning Rate": 3.863696168025859e-06, "Finetune/Loss": 0.8620541095733643, "Finetune/Loss (Raw)": 0.9154636859893799, "Finetune/Step": 3737, "Finetune/Step Time": 5.318002941086888} +{"Finetune/Learning Rate": 3.858778499514565e-06, "Finetune/Loss": 0.8623610138893127, "Finetune/Loss (Raw)": 0.8344998359680176, "Finetune/Step": 3738, "Finetune/Step Time": 5.327067293226719} +{"Finetune/Learning Rate": 3.853863214291439e-06, "Finetune/Loss": 0.8662673830986023, "Finetune/Loss (Raw)": 0.8406586647033691, "Finetune/Step": 3739, "Finetune/Step Time": 5.295771475881338} +{"Finetune/Learning Rate": 3.848950314264001e-06, "Finetune/Loss": 0.866448163986206, "Finetune/Loss (Raw)": 0.8880181908607483, "Finetune/Step": 3740, "Finetune/Step Time": 5.279317429289222} +{"Finetune/Learning Rate": 3.8440398013388655e-06, "Finetune/Loss": 0.8678107857704163, "Finetune/Loss (Raw)": 0.968128502368927, "Finetune/Step": 3741, "Finetune/Step Time": 5.279088092967868} +{"Finetune/Learning Rate": 3.839131677421694e-06, "Finetune/Loss": 0.8678536415100098, "Finetune/Loss (Raw)": 0.8756905198097229, "Finetune/Step": 3742, "Finetune/Step Time": 5.2750508934259415} +{"Finetune/Learning Rate": 3.834225944417235e-06, "Finetune/Loss": 0.8679366707801819, "Finetune/Loss (Raw)": 0.9258915781974792, "Finetune/Step": 3743, "Finetune/Step Time": 5.307736003771424} +{"Finetune/Learning Rate": 3.829322604229313e-06, "Finetune/Loss": 0.86467444896698, "Finetune/Loss (Raw)": 0.3597939610481262, "Finetune/Step": 3744, "Finetune/Step Time": 5.334744825959206} +{"Finetune/Learning Rate": 3.824421658760815e-06, "Finetune/Loss": 0.8604388236999512, "Finetune/Loss (Raw)": 0.32234641909599304, "Finetune/Step": 3745, "Finetune/Step Time": 5.353543344885111} +{"Finetune/Learning Rate": 3.819523109913706e-06, "Finetune/Loss": 0.8597761392593384, "Finetune/Loss (Raw)": 0.7541136741638184, "Finetune/Step": 3746, "Finetune/Step Time": 5.344019232317805} +{"Finetune/Learning Rate": 3.814626959589012e-06, "Finetune/Loss": 0.859054684638977, "Finetune/Loss (Raw)": 0.7576643228530884, "Finetune/Step": 3747, "Finetune/Step Time": 5.3639686945825815} +{"Finetune/Learning Rate": 3.8097332096868368e-06, "Finetune/Loss": 0.859840989112854, "Finetune/Loss (Raw)": 0.9429378509521484, "Finetune/Step": 3748, "Finetune/Step Time": 5.374671267345548} +{"Finetune/Learning Rate": 3.8048418621063475e-06, "Finetune/Loss": 0.8596215844154358, "Finetune/Loss (Raw)": 0.7898743748664856, "Finetune/Step": 3749, "Finetune/Step Time": 5.381482953205705} +{"Finetune/Learning Rate": 3.7999529187457775e-06, "Finetune/Loss": 0.8599656224250793, "Finetune/Loss (Raw)": 0.949561357498169, "Finetune/Step": 3750, "Finetune/Step Time": 5.390844291076064} +{"Finetune/Learning Rate": 3.7950663815024424e-06, "Finetune/Loss": 0.8607429265975952, "Finetune/Loss (Raw)": 1.018698811531067, "Finetune/Step": 3751, "Finetune/Step Time": 5.397849952802062} +{"Finetune/Learning Rate": 3.790182252272698e-06, "Finetune/Loss": 0.8596987724304199, "Finetune/Loss (Raw)": 0.8500043749809265, "Finetune/Step": 3752, "Finetune/Step Time": 5.395256964489818} +{"Finetune/Learning Rate": 3.785300532951983e-06, "Finetune/Loss": 0.8590191602706909, "Finetune/Loss (Raw)": 0.8726168870925903, "Finetune/Step": 3753, "Finetune/Step Time": 5.410154931247234} +{"Finetune/Learning Rate": 3.780421225434803e-06, "Finetune/Loss": 0.859840989112854, "Finetune/Loss (Raw)": 0.907569944858551, "Finetune/Step": 3754, "Finetune/Step Time": 5.412715593352914} +{"Finetune/Learning Rate": 3.7755443316147177e-06, "Finetune/Loss": 0.860203206539154, "Finetune/Loss (Raw)": 0.9589259028434753, "Finetune/Step": 3755, "Finetune/Step Time": 5.400472916662693} +{"Finetune/Learning Rate": 3.7706698533843556e-06, "Finetune/Loss": 0.8588005304336548, "Finetune/Loss (Raw)": 0.7156139612197876, "Finetune/Step": 3756, "Finetune/Step Time": 5.401805732399225} +{"Finetune/Learning Rate": 3.765797792635406e-06, "Finetune/Loss": 0.8597426414489746, "Finetune/Loss (Raw)": 0.9529352784156799, "Finetune/Step": 3757, "Finetune/Step Time": 5.398819170892239} +{"Finetune/Learning Rate": 3.7609281512586203e-06, "Finetune/Loss": 0.8598045110702515, "Finetune/Loss (Raw)": 0.8643938899040222, "Finetune/Step": 3758, "Finetune/Step Time": 5.413761716336012} +{"Finetune/Learning Rate": 3.756060931143811e-06, "Finetune/Loss": 0.861690878868103, "Finetune/Loss (Raw)": 0.9602089524269104, "Finetune/Step": 3759, "Finetune/Step Time": 5.398415828123689} +{"Finetune/Learning Rate": 3.7511961341798487e-06, "Finetune/Loss": 0.8605663180351257, "Finetune/Loss (Raw)": 0.8447170257568359, "Finetune/Step": 3760, "Finetune/Step Time": 5.404068836942315} +{"Finetune/Learning Rate": 3.746333762254677e-06, "Finetune/Loss": 0.8585928678512573, "Finetune/Loss (Raw)": 0.7441660761833191, "Finetune/Step": 3761, "Finetune/Step Time": 5.4151603896170855} +{"Finetune/Learning Rate": 3.7414738172552744e-06, "Finetune/Loss": 0.8586642146110535, "Finetune/Loss (Raw)": 0.8879668712615967, "Finetune/Step": 3762, "Finetune/Step Time": 5.4207938481122255} +{"Finetune/Learning Rate": 3.7366163010676937e-06, "Finetune/Loss": 0.8599430322647095, "Finetune/Loss (Raw)": 0.9390425086021423, "Finetune/Step": 3763, "Finetune/Step Time": 5.422129634767771} +{"Finetune/Learning Rate": 3.7317612155770467e-06, "Finetune/Loss": 0.8592336177825928, "Finetune/Loss (Raw)": 0.8184689879417419, "Finetune/Step": 3764, "Finetune/Step Time": 5.42222984880209} +{"Finetune/Learning Rate": 3.726908562667496e-06, "Finetune/Loss": 0.8582982420921326, "Finetune/Loss (Raw)": 0.8525736927986145, "Finetune/Step": 3765, "Finetune/Step Time": 5.436406530439854} +{"Finetune/Learning Rate": 3.72205834422226e-06, "Finetune/Loss": 0.8583289980888367, "Finetune/Loss (Raw)": 0.9494182467460632, "Finetune/Step": 3766, "Finetune/Step Time": 5.439485341310501} +{"Finetune/Learning Rate": 3.717210562123613e-06, "Finetune/Loss": 0.8587464690208435, "Finetune/Loss (Raw)": 0.9423213601112366, "Finetune/Step": 3767, "Finetune/Step Time": 5.443159073591232} +{"Finetune/Learning Rate": 3.712365218252887e-06, "Finetune/Loss": 0.8622808456420898, "Finetune/Loss (Raw)": 0.763113796710968, "Finetune/Step": 3768, "Finetune/Step Time": 5.421043887734413} +{"Finetune/Learning Rate": 3.7075223144904638e-06, "Finetune/Loss": 0.8578928709030151, "Finetune/Loss (Raw)": 0.24896860122680664, "Finetune/Step": 3769, "Finetune/Step Time": 5.449851501733065} +{"Finetune/Learning Rate": 3.702681852715777e-06, "Finetune/Loss": 0.8585944175720215, "Finetune/Loss (Raw)": 1.049281358718872, "Finetune/Step": 3770, "Finetune/Step Time": 5.441081905737519} +{"Finetune/Learning Rate": 3.6978438348073253e-06, "Finetune/Loss": 0.8597621321678162, "Finetune/Loss (Raw)": 1.0743658542633057, "Finetune/Step": 3771, "Finetune/Step Time": 5.437304342165589} +{"Finetune/Learning Rate": 3.6930082626426377e-06, "Finetune/Loss": 0.8607211709022522, "Finetune/Loss (Raw)": 0.9623688459396362, "Finetune/Step": 3772, "Finetune/Step Time": 5.4433247819542885} +{"Finetune/Learning Rate": 3.6881751380983066e-06, "Finetune/Loss": 0.8569804430007935, "Finetune/Loss (Raw)": 0.3465828597545624, "Finetune/Step": 3773, "Finetune/Step Time": 5.472458533942699} +{"Finetune/Learning Rate": 3.6833444630499804e-06, "Finetune/Loss": 0.8565793037414551, "Finetune/Loss (Raw)": 0.9305376410484314, "Finetune/Step": 3774, "Finetune/Step Time": 5.479113560169935} +{"Finetune/Learning Rate": 3.6785162393723482e-06, "Finetune/Loss": 0.8569462299346924, "Finetune/Loss (Raw)": 0.9705415964126587, "Finetune/Step": 3775, "Finetune/Step Time": 5.483443742617965} +{"Finetune/Learning Rate": 3.6736904689391417e-06, "Finetune/Loss": 0.8568603992462158, "Finetune/Loss (Raw)": 0.9453545212745667, "Finetune/Step": 3776, "Finetune/Step Time": 5.486779272556305} +{"Finetune/Learning Rate": 3.6688671536231557e-06, "Finetune/Loss": 0.8515231013298035, "Finetune/Loss (Raw)": 0.27399322390556335, "Finetune/Step": 3777, "Finetune/Step Time": 5.518108580261469} +{"Finetune/Learning Rate": 3.6640462952962217e-06, "Finetune/Loss": 0.8507890701293945, "Finetune/Loss (Raw)": 0.9302476048469543, "Finetune/Step": 3778, "Finetune/Step Time": 5.525048406794667} +{"Finetune/Learning Rate": 3.6592278958292225e-06, "Finetune/Loss": 0.8515888452529907, "Finetune/Loss (Raw)": 0.8509749174118042, "Finetune/Step": 3779, "Finetune/Step Time": 5.519511599093676} +{"Finetune/Learning Rate": 3.6544119570920845e-06, "Finetune/Loss": 0.850483775138855, "Finetune/Loss (Raw)": 0.8069223761558533, "Finetune/Step": 3780, "Finetune/Step Time": 5.514982501044869} +{"Finetune/Learning Rate": 3.6495984809537775e-06, "Finetune/Loss": 0.8504548072814941, "Finetune/Loss (Raw)": 0.9554963111877441, "Finetune/Step": 3781, "Finetune/Step Time": 5.512381579726934} +{"Finetune/Learning Rate": 3.6447874692823195e-06, "Finetune/Loss": 0.849999725818634, "Finetune/Loss (Raw)": 0.8402864933013916, "Finetune/Step": 3782, "Finetune/Step Time": 5.484642282128334} +{"Finetune/Learning Rate": 3.639978923944766e-06, "Finetune/Loss": 0.8480721712112427, "Finetune/Loss (Raw)": 0.8096036911010742, "Finetune/Step": 3783, "Finetune/Step Time": 5.486810440197587} +{"Finetune/Learning Rate": 3.6351728468072265e-06, "Finetune/Loss": 0.8482972383499146, "Finetune/Loss (Raw)": 0.8153949975967407, "Finetune/Step": 3784, "Finetune/Step Time": 5.492462437599897} +{"Finetune/Learning Rate": 3.6303692397348455e-06, "Finetune/Loss": 0.8482761383056641, "Finetune/Loss (Raw)": 0.8697530627250671, "Finetune/Step": 3785, "Finetune/Step Time": 5.490403063595295} +{"Finetune/Learning Rate": 3.6255681045918e-06, "Finetune/Loss": 0.8489905595779419, "Finetune/Loss (Raw)": 0.9858418107032776, "Finetune/Step": 3786, "Finetune/Step Time": 5.485664425417781} +{"Finetune/Learning Rate": 3.6207694432413255e-06, "Finetune/Loss": 0.8491915464401245, "Finetune/Loss (Raw)": 0.9371844530105591, "Finetune/Step": 3787, "Finetune/Step Time": 5.488478420302272} +{"Finetune/Learning Rate": 3.6159732575456862e-06, "Finetune/Loss": 0.8482476472854614, "Finetune/Loss (Raw)": 0.8835707306861877, "Finetune/Step": 3788, "Finetune/Step Time": 5.4943911004811525} +{"Finetune/Learning Rate": 3.6111795493661885e-06, "Finetune/Loss": 0.8476277589797974, "Finetune/Loss (Raw)": 0.7963894009590149, "Finetune/Step": 3789, "Finetune/Step Time": 5.512938300147653} +{"Finetune/Learning Rate": 3.606388320563177e-06, "Finetune/Loss": 0.8474223017692566, "Finetune/Loss (Raw)": 0.9520511031150818, "Finetune/Step": 3790, "Finetune/Step Time": 5.519232260063291} +{"Finetune/Learning Rate": 3.6015995729960316e-06, "Finetune/Loss": 0.8459817171096802, "Finetune/Loss (Raw)": 0.8793376088142395, "Finetune/Step": 3791, "Finetune/Step Time": 5.519303053617477} +{"Finetune/Learning Rate": 3.596813308523176e-06, "Finetune/Loss": 0.8421909213066101, "Finetune/Loss (Raw)": 0.3438204824924469, "Finetune/Step": 3792, "Finetune/Step Time": 5.538157142698765} +{"Finetune/Learning Rate": 3.592029529002059e-06, "Finetune/Loss": 0.8421880602836609, "Finetune/Loss (Raw)": 0.9148677587509155, "Finetune/Step": 3793, "Finetune/Step Time": 5.556989461183548} +{"Finetune/Learning Rate": 3.5872482362891802e-06, "Finetune/Loss": 0.842165470123291, "Finetune/Loss (Raw)": 0.927517831325531, "Finetune/Step": 3794, "Finetune/Step Time": 5.556096820160747} +{"Finetune/Learning Rate": 3.5824694322400653e-06, "Finetune/Loss": 0.8447978496551514, "Finetune/Loss (Raw)": 0.7613391280174255, "Finetune/Step": 3795, "Finetune/Step Time": 5.538499852642417} +{"Finetune/Learning Rate": 3.5776931187092667e-06, "Finetune/Loss": 0.8454643487930298, "Finetune/Loss (Raw)": 0.9146766662597656, "Finetune/Step": 3796, "Finetune/Step Time": 5.544296197593212} +{"Finetune/Learning Rate": 3.5729192975503856e-06, "Finetune/Loss": 0.844643235206604, "Finetune/Loss (Raw)": 0.7518251538276672, "Finetune/Step": 3797, "Finetune/Step Time": 5.548446413129568} +{"Finetune/Learning Rate": 3.5681479706160483e-06, "Finetune/Loss": 0.844114363193512, "Finetune/Loss (Raw)": 0.843940794467926, "Finetune/Step": 3798, "Finetune/Step Time": 5.54498409293592} +{"Finetune/Learning Rate": 3.563379139757911e-06, "Finetune/Loss": 0.8428057432174683, "Finetune/Loss (Raw)": 0.8168804049491882, "Finetune/Step": 3799, "Finetune/Step Time": 5.555068861693144} +{"Finetune/Learning Rate": 3.558612806826666e-06, "Finetune/Loss": 0.8437554836273193, "Finetune/Loss (Raw)": 0.9603450894355774, "Finetune/Step": 3800, "Finetune/Step Time": 5.5507675390690565} +{"Finetune/Learning Rate": 3.553848973672033e-06, "Finetune/Loss": 0.8453546762466431, "Finetune/Loss (Raw)": 0.8791798949241638, "Finetune/Step": 3801, "Finetune/Step Time": 5.551408821716905} +{"Finetune/Learning Rate": 3.549087642142762e-06, "Finetune/Loss": 0.8463045358657837, "Finetune/Loss (Raw)": 0.9682738780975342, "Finetune/Step": 3802, "Finetune/Step Time": 5.552790032699704} +{"Finetune/Learning Rate": 3.544328814086632e-06, "Finetune/Loss": 0.8512799739837646, "Finetune/Loss (Raw)": 1.0133740901947021, "Finetune/Step": 3803, "Finetune/Step Time": 5.518048867583275} +{"Finetune/Learning Rate": 3.5395724913504546e-06, "Finetune/Loss": 0.8510913848876953, "Finetune/Loss (Raw)": 0.9283428192138672, "Finetune/Step": 3804, "Finetune/Step Time": 5.502458101138473} +{"Finetune/Learning Rate": 3.5348186757800683e-06, "Finetune/Loss": 0.8501763939857483, "Finetune/Loss (Raw)": 0.8515797853469849, "Finetune/Step": 3805, "Finetune/Step Time": 5.5196633990854025} +{"Finetune/Learning Rate": 3.530067369220326e-06, "Finetune/Loss": 0.8520556688308716, "Finetune/Loss (Raw)": 0.8916699886322021, "Finetune/Step": 3806, "Finetune/Step Time": 5.531882410869002} +{"Finetune/Learning Rate": 3.525318573515125e-06, "Finetune/Loss": 0.8507741689682007, "Finetune/Loss (Raw)": 0.8414093852043152, "Finetune/Step": 3807, "Finetune/Step Time": 5.534133052453399} +{"Finetune/Learning Rate": 3.520572290507378e-06, "Finetune/Loss": 0.8497606515884399, "Finetune/Loss (Raw)": 0.7848840951919556, "Finetune/Step": 3808, "Finetune/Step Time": 5.541295548900962} +{"Finetune/Learning Rate": 3.5158285220390256e-06, "Finetune/Loss": 0.8487924933433533, "Finetune/Loss (Raw)": 0.8525316119194031, "Finetune/Step": 3809, "Finetune/Step Time": 5.555970158427954} +{"Finetune/Learning Rate": 3.5110872699510313e-06, "Finetune/Loss": 0.8475796580314636, "Finetune/Loss (Raw)": 0.9407010078430176, "Finetune/Step": 3810, "Finetune/Step Time": 5.54759819060564} +{"Finetune/Learning Rate": 3.5063485360833816e-06, "Finetune/Loss": 0.8471198678016663, "Finetune/Loss (Raw)": 0.8380894064903259, "Finetune/Step": 3811, "Finetune/Step Time": 5.553525509312749} +{"Finetune/Learning Rate": 3.5016123222750863e-06, "Finetune/Loss": 0.8481777906417847, "Finetune/Loss (Raw)": 0.8035879135131836, "Finetune/Step": 3812, "Finetune/Step Time": 5.548020772635937} +{"Finetune/Learning Rate": 3.4968786303641764e-06, "Finetune/Loss": 0.8482275009155273, "Finetune/Loss (Raw)": 0.8735553622245789, "Finetune/Step": 3813, "Finetune/Step Time": 5.545594351366162} +{"Finetune/Learning Rate": 3.492147462187714e-06, "Finetune/Loss": 0.8476265668869019, "Finetune/Loss (Raw)": 0.8425598740577698, "Finetune/Step": 3814, "Finetune/Step Time": 5.543652780354023} +{"Finetune/Learning Rate": 3.4874188195817626e-06, "Finetune/Loss": 0.8474566340446472, "Finetune/Loss (Raw)": 0.8650283813476562, "Finetune/Step": 3815, "Finetune/Step Time": 5.5621506832540035} +{"Finetune/Learning Rate": 3.4826927043814197e-06, "Finetune/Loss": 0.8478938341140747, "Finetune/Loss (Raw)": 0.8830782771110535, "Finetune/Step": 3816, "Finetune/Step Time": 5.5574780367314816} +{"Finetune/Learning Rate": 3.4779691184207954e-06, "Finetune/Loss": 0.846506655216217, "Finetune/Loss (Raw)": 0.8124752640724182, "Finetune/Step": 3817, "Finetune/Step Time": 5.559985348954797} +{"Finetune/Learning Rate": 3.473248063533028e-06, "Finetune/Loss": 0.8455780744552612, "Finetune/Loss (Raw)": 0.8184279799461365, "Finetune/Step": 3818, "Finetune/Step Time": 5.553301753476262} +{"Finetune/Learning Rate": 3.4685295415502663e-06, "Finetune/Loss": 0.8494588136672974, "Finetune/Loss (Raw)": 0.8071086406707764, "Finetune/Step": 3819, "Finetune/Step Time": 5.524246782064438} +{"Finetune/Learning Rate": 3.4638135543036656e-06, "Finetune/Loss": 0.8471781611442566, "Finetune/Loss (Raw)": 0.7115559577941895, "Finetune/Step": 3820, "Finetune/Step Time": 5.526141984388232} +{"Finetune/Learning Rate": 3.4591001036234206e-06, "Finetune/Loss": 0.8464725613594055, "Finetune/Loss (Raw)": 0.8185042142868042, "Finetune/Step": 3821, "Finetune/Step Time": 5.531647231429815} +{"Finetune/Learning Rate": 3.4543891913387253e-06, "Finetune/Loss": 0.8464238047599792, "Finetune/Loss (Raw)": 0.7817788124084473, "Finetune/Step": 3822, "Finetune/Step Time": 5.510859128087759} +{"Finetune/Learning Rate": 3.449680819277792e-06, "Finetune/Loss": 0.8454055190086365, "Finetune/Loss (Raw)": 0.9882976412773132, "Finetune/Step": 3823, "Finetune/Step Time": 5.514037478715181} +{"Finetune/Learning Rate": 3.4449749892678486e-06, "Finetune/Loss": 0.8456142544746399, "Finetune/Loss (Raw)": 0.897840142250061, "Finetune/Step": 3824, "Finetune/Step Time": 5.5075680408626795} +{"Finetune/Learning Rate": 3.4402717031351363e-06, "Finetune/Loss": 0.8459046483039856, "Finetune/Loss (Raw)": 0.7593823671340942, "Finetune/Step": 3825, "Finetune/Step Time": 5.4858510587364435} +{"Finetune/Learning Rate": 3.4355709627049073e-06, "Finetune/Loss": 0.8450510501861572, "Finetune/Loss (Raw)": 0.8155519962310791, "Finetune/Step": 3826, "Finetune/Step Time": 5.482314920052886} +{"Finetune/Learning Rate": 3.4308727698014243e-06, "Finetune/Loss": 0.8449243307113647, "Finetune/Loss (Raw)": 0.9089410305023193, "Finetune/Step": 3827, "Finetune/Step Time": 5.515105469152331} +{"Finetune/Learning Rate": 3.4261771262479724e-06, "Finetune/Loss": 0.845245361328125, "Finetune/Loss (Raw)": 0.9576632976531982, "Finetune/Step": 3828, "Finetune/Step Time": 5.518198477104306} +{"Finetune/Learning Rate": 3.4214840338668376e-06, "Finetune/Loss": 0.8417837619781494, "Finetune/Loss (Raw)": 0.2799749970436096, "Finetune/Step": 3829, "Finetune/Step Time": 5.543516730889678} +{"Finetune/Learning Rate": 3.416793494479308e-06, "Finetune/Loss": 0.8414303064346313, "Finetune/Loss (Raw)": 0.8945766091346741, "Finetune/Step": 3830, "Finetune/Step Time": 5.552369242534041} +{"Finetune/Learning Rate": 3.412105509905701e-06, "Finetune/Loss": 0.8405135869979858, "Finetune/Loss (Raw)": 0.8395213484764099, "Finetune/Step": 3831, "Finetune/Step Time": 5.5598174668848515} +{"Finetune/Learning Rate": 3.407420081965327e-06, "Finetune/Loss": 0.8414307236671448, "Finetune/Loss (Raw)": 1.03801691532135, "Finetune/Step": 3832, "Finetune/Step Time": 5.557214638218284} +{"Finetune/Learning Rate": 3.402737212476509e-06, "Finetune/Loss": 0.843182384967804, "Finetune/Loss (Raw)": 0.9608455896377563, "Finetune/Step": 3833, "Finetune/Step Time": 5.58072735927999} +{"Finetune/Learning Rate": 3.398056903256579e-06, "Finetune/Loss": 0.8440127968788147, "Finetune/Loss (Raw)": 0.9739747643470764, "Finetune/Step": 3834, "Finetune/Step Time": 5.575020300224423} +{"Finetune/Learning Rate": 3.393379156121873e-06, "Finetune/Loss": 0.8447285294532776, "Finetune/Loss (Raw)": 0.9439578652381897, "Finetune/Step": 3835, "Finetune/Step Time": 5.555516559630632} +{"Finetune/Learning Rate": 3.3887039728877336e-06, "Finetune/Loss": 0.8449263572692871, "Finetune/Loss (Raw)": 0.8173288106918335, "Finetune/Step": 3836, "Finetune/Step Time": 5.533598493784666} +{"Finetune/Learning Rate": 3.3840313553685034e-06, "Finetune/Loss": 0.8436154127120972, "Finetune/Loss (Raw)": 0.8553573489189148, "Finetune/Step": 3837, "Finetune/Step Time": 5.533790798857808} +{"Finetune/Learning Rate": 3.379361305377542e-06, "Finetune/Loss": 0.8421963453292847, "Finetune/Loss (Raw)": 0.8152998685836792, "Finetune/Step": 3838, "Finetune/Step Time": 5.537354651838541} +{"Finetune/Learning Rate": 3.374693824727204e-06, "Finetune/Loss": 0.842950701713562, "Finetune/Loss (Raw)": 0.8900229930877686, "Finetune/Step": 3839, "Finetune/Step Time": 5.552213000133634} +{"Finetune/Learning Rate": 3.3700289152288377e-06, "Finetune/Loss": 0.8415930271148682, "Finetune/Loss (Raw)": 0.842651903629303, "Finetune/Step": 3840, "Finetune/Step Time": 5.552446955814958} +{"Finetune/Learning Rate": 3.3653665786928138e-06, "Finetune/Loss": 0.8472177982330322, "Finetune/Loss (Raw)": 1.0054339170455933, "Finetune/Step": 3841, "Finetune/Step Time": 5.5216351971030235} +{"Finetune/Learning Rate": 3.360706816928491e-06, "Finetune/Loss": 0.8464251756668091, "Finetune/Loss (Raw)": 0.8101764917373657, "Finetune/Step": 3842, "Finetune/Step Time": 5.519691405817866} +{"Finetune/Learning Rate": 3.3560496317442305e-06, "Finetune/Loss": 0.8456050157546997, "Finetune/Loss (Raw)": 0.8485416769981384, "Finetune/Step": 3843, "Finetune/Step Time": 5.5273169204592705} +{"Finetune/Learning Rate": 3.351395024947395e-06, "Finetune/Loss": 0.8459756374359131, "Finetune/Loss (Raw)": 0.8562533855438232, "Finetune/Step": 3844, "Finetune/Step Time": 5.502654375508428} +{"Finetune/Learning Rate": 3.3467429983443477e-06, "Finetune/Loss": 0.8464662432670593, "Finetune/Loss (Raw)": 0.8510758876800537, "Finetune/Step": 3845, "Finetune/Step Time": 5.476262683048844} +{"Finetune/Learning Rate": 3.34209355374045e-06, "Finetune/Loss": 0.8458026647567749, "Finetune/Loss (Raw)": 0.7015649080276489, "Finetune/Step": 3846, "Finetune/Step Time": 5.47186260856688} +{"Finetune/Learning Rate": 3.337446692940055e-06, "Finetune/Loss": 0.8482195138931274, "Finetune/Loss (Raw)": 0.9905174374580383, "Finetune/Step": 3847, "Finetune/Step Time": 5.454474380239844} +{"Finetune/Learning Rate": 3.332802417746527e-06, "Finetune/Loss": 0.8429611921310425, "Finetune/Loss (Raw)": 0.3292538523674011, "Finetune/Step": 3848, "Finetune/Step Time": 5.4750622399151325} +{"Finetune/Learning Rate": 3.3281607299622175e-06, "Finetune/Loss": 0.8434615731239319, "Finetune/Loss (Raw)": 1.045331358909607, "Finetune/Step": 3849, "Finetune/Step Time": 5.490295736119151} +{"Finetune/Learning Rate": 3.3235216313884657e-06, "Finetune/Loss": 0.8411715030670166, "Finetune/Loss (Raw)": 0.6287572383880615, "Finetune/Step": 3850, "Finetune/Step Time": 5.492884276434779} +{"Finetune/Learning Rate": 3.3188851238256246e-06, "Finetune/Loss": 0.8395301103591919, "Finetune/Loss (Raw)": 0.7672240734100342, "Finetune/Step": 3851, "Finetune/Step Time": 5.484519576653838} +{"Finetune/Learning Rate": 3.3142512090730283e-06, "Finetune/Loss": 0.8403298854827881, "Finetune/Loss (Raw)": 0.9237006306648254, "Finetune/Step": 3852, "Finetune/Step Time": 5.4912081100046635} +{"Finetune/Learning Rate": 3.309619888929011e-06, "Finetune/Loss": 0.8363027572631836, "Finetune/Loss (Raw)": 0.4537631869316101, "Finetune/Step": 3853, "Finetune/Step Time": 5.51714182831347} +{"Finetune/Learning Rate": 3.304991165190895e-06, "Finetune/Loss": 0.8403568267822266, "Finetune/Loss (Raw)": 0.9020563364028931, "Finetune/Step": 3854, "Finetune/Step Time": 5.495783684775233} +{"Finetune/Learning Rate": 3.300365039655e-06, "Finetune/Loss": 0.8389678001403809, "Finetune/Loss (Raw)": 0.6544161438941956, "Finetune/Step": 3855, "Finetune/Step Time": 5.5114295445382595} +{"Finetune/Learning Rate": 3.2957415141166327e-06, "Finetune/Loss": 0.8413658142089844, "Finetune/Loss (Raw)": 1.0039201974868774, "Finetune/Step": 3856, "Finetune/Step Time": 5.509341241791844} +{"Finetune/Learning Rate": 3.2911205903700916e-06, "Finetune/Loss": 0.836957573890686, "Finetune/Loss (Raw)": 0.3625165820121765, "Finetune/Step": 3857, "Finetune/Step Time": 5.5292554553598166} +{"Finetune/Learning Rate": 3.2865022702086756e-06, "Finetune/Loss": 0.8323559761047363, "Finetune/Loss (Raw)": 0.33655059337615967, "Finetune/Step": 3858, "Finetune/Step Time": 5.534247409552336} +{"Finetune/Learning Rate": 3.281886555424656e-06, "Finetune/Loss": 0.8333446979522705, "Finetune/Loss (Raw)": 1.0631386041641235, "Finetune/Step": 3859, "Finetune/Step Time": 5.526668310165405} +{"Finetune/Learning Rate": 3.277273447809299e-06, "Finetune/Loss": 0.8332259654998779, "Finetune/Loss (Raw)": 0.8922221660614014, "Finetune/Step": 3860, "Finetune/Step Time": 5.5196254421025515} +{"Finetune/Learning Rate": 3.2726629491528714e-06, "Finetune/Loss": 0.8332183361053467, "Finetune/Loss (Raw)": 0.8497684001922607, "Finetune/Step": 3861, "Finetune/Step Time": 5.504936782643199} +{"Finetune/Learning Rate": 3.268055061244614e-06, "Finetune/Loss": 0.8325523138046265, "Finetune/Loss (Raw)": 0.8852841258049011, "Finetune/Step": 3862, "Finetune/Step Time": 5.503251118585467} +{"Finetune/Learning Rate": 3.2634497858727553e-06, "Finetune/Loss": 0.8335191011428833, "Finetune/Loss (Raw)": 0.8343867659568787, "Finetune/Step": 3863, "Finetune/Step Time": 5.513311959803104} +{"Finetune/Learning Rate": 3.2588471248245157e-06, "Finetune/Loss": 0.8322468400001526, "Finetune/Loss (Raw)": 0.9271751642227173, "Finetune/Step": 3864, "Finetune/Step Time": 5.5160192884504795} +{"Finetune/Learning Rate": 3.254247079886097e-06, "Finetune/Loss": 0.8320388793945312, "Finetune/Loss (Raw)": 0.888841450214386, "Finetune/Step": 3865, "Finetune/Step Time": 5.53386502340436} +{"Finetune/Learning Rate": 3.2496496528426867e-06, "Finetune/Loss": 0.8335930109024048, "Finetune/Loss (Raw)": 1.0334386825561523, "Finetune/Step": 3866, "Finetune/Step Time": 5.525377556681633} +{"Finetune/Learning Rate": 3.245054845478454e-06, "Finetune/Loss": 0.8354905247688293, "Finetune/Loss (Raw)": 1.0835365056991577, "Finetune/Step": 3867, "Finetune/Step Time": 5.525816274806857} +{"Finetune/Learning Rate": 3.240462659576562e-06, "Finetune/Loss": 0.8355686664581299, "Finetune/Loss (Raw)": 0.8980227112770081, "Finetune/Step": 3868, "Finetune/Step Time": 5.529399707913399} +{"Finetune/Learning Rate": 3.23587309691914e-06, "Finetune/Loss": 0.8344280123710632, "Finetune/Loss (Raw)": 0.822121262550354, "Finetune/Step": 3869, "Finetune/Step Time": 5.530865181237459} +{"Finetune/Learning Rate": 3.231286159287308e-06, "Finetune/Loss": 0.833922803401947, "Finetune/Loss (Raw)": 0.8110257983207703, "Finetune/Step": 3870, "Finetune/Step Time": 5.555370835587382} +{"Finetune/Learning Rate": 3.2267018484611725e-06, "Finetune/Loss": 0.833604633808136, "Finetune/Loss (Raw)": 0.8851670026779175, "Finetune/Step": 3871, "Finetune/Step Time": 5.51913402415812} +{"Finetune/Learning Rate": 3.222120166219812e-06, "Finetune/Loss": 0.8349429965019226, "Finetune/Loss (Raw)": 0.5311015248298645, "Finetune/Step": 3872, "Finetune/Step Time": 5.523076318204403} +{"Finetune/Learning Rate": 3.217541114341288e-06, "Finetune/Loss": 0.8381984233856201, "Finetune/Loss (Raw)": 0.7390385270118713, "Finetune/Step": 3873, "Finetune/Step Time": 5.501300835981965} +{"Finetune/Learning Rate": 3.2129646946026407e-06, "Finetune/Loss": 0.8392132520675659, "Finetune/Loss (Raw)": 0.8840118646621704, "Finetune/Step": 3874, "Finetune/Step Time": 5.501507891342044} +{"Finetune/Learning Rate": 3.2083909087798905e-06, "Finetune/Loss": 0.8401767015457153, "Finetune/Loss (Raw)": 0.8809890151023865, "Finetune/Step": 3875, "Finetune/Step Time": 5.483771530911326} +{"Finetune/Learning Rate": 3.2038197586480336e-06, "Finetune/Loss": 0.8397822380065918, "Finetune/Loss (Raw)": 0.8924521803855896, "Finetune/Step": 3876, "Finetune/Step Time": 5.470454653725028} +{"Finetune/Learning Rate": 3.199251245981041e-06, "Finetune/Loss": 0.8404215574264526, "Finetune/Loss (Raw)": 0.8716989159584045, "Finetune/Step": 3877, "Finetune/Step Time": 5.478341551497579} +{"Finetune/Learning Rate": 3.1946853725518724e-06, "Finetune/Loss": 0.8366919159889221, "Finetune/Loss (Raw)": 0.4721655547618866, "Finetune/Step": 3878, "Finetune/Step Time": 5.503338115289807} +{"Finetune/Learning Rate": 3.1901221401324446e-06, "Finetune/Loss": 0.835381031036377, "Finetune/Loss (Raw)": 0.8509069085121155, "Finetune/Step": 3879, "Finetune/Step Time": 5.528538376092911} +{"Finetune/Learning Rate": 3.18556155049366e-06, "Finetune/Loss": 0.835519552230835, "Finetune/Loss (Raw)": 0.8677356243133545, "Finetune/Step": 3880, "Finetune/Step Time": 5.5247260723263025} +{"Finetune/Learning Rate": 3.1810036054054005e-06, "Finetune/Loss": 0.8359254002571106, "Finetune/Loss (Raw)": 0.9245665669441223, "Finetune/Step": 3881, "Finetune/Step Time": 5.517119836062193} +{"Finetune/Learning Rate": 3.1764483066365117e-06, "Finetune/Loss": 0.8362224698066711, "Finetune/Loss (Raw)": 0.9455947875976562, "Finetune/Step": 3882, "Finetune/Step Time": 5.5208825040608644} +{"Finetune/Learning Rate": 3.1718956559548154e-06, "Finetune/Loss": 0.8364158868789673, "Finetune/Loss (Raw)": 0.9836863279342651, "Finetune/Step": 3883, "Finetune/Step Time": 5.5146900080144405} +{"Finetune/Learning Rate": 3.1673456551271086e-06, "Finetune/Loss": 0.8372724056243896, "Finetune/Loss (Raw)": 0.8252465128898621, "Finetune/Step": 3884, "Finetune/Step Time": 5.513395035639405} +{"Finetune/Learning Rate": 3.162798305919157e-06, "Finetune/Loss": 0.8351936340332031, "Finetune/Loss (Raw)": 0.6868473887443542, "Finetune/Step": 3885, "Finetune/Step Time": 5.520057002082467} +{"Finetune/Learning Rate": 3.1582536100956973e-06, "Finetune/Loss": 0.8343443274497986, "Finetune/Loss (Raw)": 0.7556850910186768, "Finetune/Step": 3886, "Finetune/Step Time": 5.506654774770141} +{"Finetune/Learning Rate": 3.1537115694204345e-06, "Finetune/Loss": 0.8331567049026489, "Finetune/Loss (Raw)": 0.8081955909729004, "Finetune/Step": 3887, "Finetune/Step Time": 5.508222971111536} +{"Finetune/Learning Rate": 3.1491721856560555e-06, "Finetune/Loss": 0.8336623907089233, "Finetune/Loss (Raw)": 0.9094409942626953, "Finetune/Step": 3888, "Finetune/Step Time": 5.5005004573613405} +{"Finetune/Learning Rate": 3.144635460564197e-06, "Finetune/Loss": 0.8347066640853882, "Finetune/Loss (Raw)": 0.8778400421142578, "Finetune/Step": 3889, "Finetune/Step Time": 5.4914320800453424} +{"Finetune/Learning Rate": 3.1401013959054726e-06, "Finetune/Loss": 0.8349842429161072, "Finetune/Loss (Raw)": 0.923492431640625, "Finetune/Step": 3890, "Finetune/Step Time": 5.490749765187502} +{"Finetune/Learning Rate": 3.1355699934394724e-06, "Finetune/Loss": 0.834587812423706, "Finetune/Loss (Raw)": 0.888302743434906, "Finetune/Step": 3891, "Finetune/Step Time": 5.505506992340088} +{"Finetune/Learning Rate": 3.1310412549247403e-06, "Finetune/Loss": 0.8349208831787109, "Finetune/Loss (Raw)": 0.8611020445823669, "Finetune/Step": 3892, "Finetune/Step Time": 5.509024014696479} +{"Finetune/Learning Rate": 3.1265151821187933e-06, "Finetune/Loss": 0.831153154373169, "Finetune/Loss (Raw)": 0.3703075349330902, "Finetune/Step": 3893, "Finetune/Step Time": 5.525362448766828} +{"Finetune/Learning Rate": 3.121991776778109e-06, "Finetune/Loss": 0.830845057964325, "Finetune/Loss (Raw)": 0.909974217414856, "Finetune/Step": 3894, "Finetune/Step Time": 5.523586865514517} +{"Finetune/Learning Rate": 3.1174710406581355e-06, "Finetune/Loss": 0.8307664394378662, "Finetune/Loss (Raw)": 0.9322646260261536, "Finetune/Step": 3895, "Finetune/Step Time": 5.521942945197225} +{"Finetune/Learning Rate": 3.1129529755132815e-06, "Finetune/Loss": 0.8322234749794006, "Finetune/Loss (Raw)": 0.9496088027954102, "Finetune/Step": 3896, "Finetune/Step Time": 5.510466009378433} +{"Finetune/Learning Rate": 3.108437583096918e-06, "Finetune/Loss": 0.8372131586074829, "Finetune/Loss (Raw)": 0.8876561522483826, "Finetune/Step": 3897, "Finetune/Step Time": 5.47876001894474} +{"Finetune/Learning Rate": 3.103924865161383e-06, "Finetune/Loss": 0.8357225060462952, "Finetune/Loss (Raw)": 0.8584705591201782, "Finetune/Step": 3898, "Finetune/Step Time": 5.49529336206615} +{"Finetune/Learning Rate": 3.0994148234579734e-06, "Finetune/Loss": 0.8351742029190063, "Finetune/Loss (Raw)": 1.004181146621704, "Finetune/Step": 3899, "Finetune/Step Time": 5.498541057109833} +{"Finetune/Learning Rate": 3.0949074597369444e-06, "Finetune/Loss": 0.8344606161117554, "Finetune/Loss (Raw)": 0.871030867099762, "Finetune/Step": 3900, "Finetune/Step Time": 5.509154183790088} +{"Finetune/Learning Rate": 3.090402775747523e-06, "Finetune/Loss": 0.8390940427780151, "Finetune/Loss (Raw)": 0.9396631121635437, "Finetune/Step": 3901, "Finetune/Step Time": 5.483172420412302} +{"Finetune/Learning Rate": 3.0859007732378897e-06, "Finetune/Loss": 0.8394419550895691, "Finetune/Loss (Raw)": 0.9750715494155884, "Finetune/Step": 3902, "Finetune/Step Time": 5.462462509050965} +{"Finetune/Learning Rate": 3.0814014539551727e-06, "Finetune/Loss": 0.8393794298171997, "Finetune/Loss (Raw)": 0.9625424742698669, "Finetune/Step": 3903, "Finetune/Step Time": 5.457504080608487} +{"Finetune/Learning Rate": 3.076904819645481e-06, "Finetune/Loss": 0.8390016555786133, "Finetune/Loss (Raw)": 0.8969994187355042, "Finetune/Step": 3904, "Finetune/Step Time": 5.4609678871929646} +{"Finetune/Learning Rate": 3.072410872053867e-06, "Finetune/Loss": 0.8423333168029785, "Finetune/Loss (Raw)": 0.7004445195198059, "Finetune/Step": 3905, "Finetune/Step Time": 5.443544181063771} +{"Finetune/Learning Rate": 3.067919612924343e-06, "Finetune/Loss": 0.8424361944198608, "Finetune/Loss (Raw)": 0.9434076547622681, "Finetune/Step": 3906, "Finetune/Step Time": 5.472291365265846} +{"Finetune/Learning Rate": 3.0634310439998803e-06, "Finetune/Loss": 0.8447995185852051, "Finetune/Loss (Raw)": 1.1534842252731323, "Finetune/Step": 3907, "Finetune/Step Time": 5.457321362569928} +{"Finetune/Learning Rate": 3.0589451670224037e-06, "Finetune/Loss": 0.844931960105896, "Finetune/Loss (Raw)": 0.8238731026649475, "Finetune/Step": 3908, "Finetune/Step Time": 5.464641632512212} +{"Finetune/Learning Rate": 3.054461983732795e-06, "Finetune/Loss": 0.8439381718635559, "Finetune/Loss (Raw)": 0.8282979130744934, "Finetune/Step": 3909, "Finetune/Step Time": 5.470646660774946} +{"Finetune/Learning Rate": 3.0499814958708883e-06, "Finetune/Loss": 0.8448927402496338, "Finetune/Loss (Raw)": 0.9624671936035156, "Finetune/Step": 3910, "Finetune/Step Time": 5.471132731065154} +{"Finetune/Learning Rate": 3.045503705175478e-06, "Finetune/Loss": 0.8465752601623535, "Finetune/Loss (Raw)": 1.0249688625335693, "Finetune/Step": 3911, "Finetune/Step Time": 5.47271485440433} +{"Finetune/Learning Rate": 3.041028613384307e-06, "Finetune/Loss": 0.8465670347213745, "Finetune/Loss (Raw)": 0.8143368363380432, "Finetune/Step": 3912, "Finetune/Step Time": 5.475869519636035} +{"Finetune/Learning Rate": 3.036556222234063e-06, "Finetune/Loss": 0.8467891216278076, "Finetune/Loss (Raw)": 0.8981831073760986, "Finetune/Step": 3913, "Finetune/Step Time": 5.479232227429748} +{"Finetune/Learning Rate": 3.032086533460402e-06, "Finetune/Loss": 0.8456369638442993, "Finetune/Loss (Raw)": 0.838362991809845, "Finetune/Step": 3914, "Finetune/Step Time": 5.47094464302063} +{"Finetune/Learning Rate": 3.027619548797921e-06, "Finetune/Loss": 0.8454021215438843, "Finetune/Loss (Raw)": 0.9071356654167175, "Finetune/Step": 3915, "Finetune/Step Time": 5.468344792723656} +{"Finetune/Learning Rate": 3.023155269980169e-06, "Finetune/Loss": 0.8454945087432861, "Finetune/Loss (Raw)": 0.8953919410705566, "Finetune/Step": 3916, "Finetune/Step Time": 5.473993591964245} +{"Finetune/Learning Rate": 3.0186936987396442e-06, "Finetune/Loss": 0.8463212847709656, "Finetune/Loss (Raw)": 0.9022161960601807, "Finetune/Step": 3917, "Finetune/Step Time": 5.47493620403111} +{"Finetune/Learning Rate": 3.014234836807798e-06, "Finetune/Loss": 0.8463705778121948, "Finetune/Loss (Raw)": 0.9583591222763062, "Finetune/Step": 3918, "Finetune/Step Time": 5.468929424881935} +{"Finetune/Learning Rate": 3.0097786859150247e-06, "Finetune/Loss": 0.8461861610412598, "Finetune/Loss (Raw)": 0.8557372689247131, "Finetune/Step": 3919, "Finetune/Step Time": 5.4790670443326235} +{"Finetune/Learning Rate": 3.005325247790668e-06, "Finetune/Loss": 0.8504215478897095, "Finetune/Loss (Raw)": 0.8859384655952454, "Finetune/Step": 3920, "Finetune/Step Time": 5.483607068657875} +{"Finetune/Learning Rate": 3.0008745241630266e-06, "Finetune/Loss": 0.8485993146896362, "Finetune/Loss (Raw)": 0.6816284656524658, "Finetune/Step": 3921, "Finetune/Step Time": 5.4931924007833} +{"Finetune/Learning Rate": 2.9964265167593386e-06, "Finetune/Loss": 0.8487597703933716, "Finetune/Loss (Raw)": 0.9480580687522888, "Finetune/Step": 3922, "Finetune/Step Time": 5.501980293542147} +{"Finetune/Learning Rate": 2.9919812273057815e-06, "Finetune/Loss": 0.8484244346618652, "Finetune/Loss (Raw)": 0.7184123992919922, "Finetune/Step": 3923, "Finetune/Step Time": 5.506760060787201} +{"Finetune/Learning Rate": 2.987538657527491e-06, "Finetune/Loss": 0.8483438491821289, "Finetune/Loss (Raw)": 0.904358983039856, "Finetune/Step": 3924, "Finetune/Step Time": 5.5101578794419765} +{"Finetune/Learning Rate": 2.9830988091485426e-06, "Finetune/Loss": 0.8495800495147705, "Finetune/Loss (Raw)": 0.910067617893219, "Finetune/Step": 3925, "Finetune/Step Time": 5.5066538993269205} +{"Finetune/Learning Rate": 2.9786616838919515e-06, "Finetune/Loss": 0.8494272232055664, "Finetune/Loss (Raw)": 0.8243756890296936, "Finetune/Step": 3926, "Finetune/Step Time": 5.5114922150969505} +{"Finetune/Learning Rate": 2.9742272834796813e-06, "Finetune/Loss": 0.8498091697692871, "Finetune/Loss (Raw)": 0.865767240524292, "Finetune/Step": 3927, "Finetune/Step Time": 5.5187566839158535} +{"Finetune/Learning Rate": 2.969795609632636e-06, "Finetune/Loss": 0.8500335216522217, "Finetune/Loss (Raw)": 0.9890539050102234, "Finetune/Step": 3928, "Finetune/Step Time": 5.519555198028684} +{"Finetune/Learning Rate": 2.9653666640706614e-06, "Finetune/Loss": 0.8503645658493042, "Finetune/Loss (Raw)": 0.9215672016143799, "Finetune/Step": 3929, "Finetune/Step Time": 5.515689892694354} +{"Finetune/Learning Rate": 2.9609404485125414e-06, "Finetune/Loss": 0.8494771718978882, "Finetune/Loss (Raw)": 0.8546797037124634, "Finetune/Step": 3930, "Finetune/Step Time": 5.521065989509225} +{"Finetune/Learning Rate": 2.9565169646760108e-06, "Finetune/Loss": 0.8492681980133057, "Finetune/Loss (Raw)": 0.9866262674331665, "Finetune/Step": 3931, "Finetune/Step Time": 5.5219239220023155} +{"Finetune/Learning Rate": 2.9520962142777367e-06, "Finetune/Loss": 0.8485816717147827, "Finetune/Loss (Raw)": 0.8404679298400879, "Finetune/Step": 3932, "Finetune/Step Time": 5.536652998998761} +{"Finetune/Learning Rate": 2.947678199033317e-06, "Finetune/Loss": 0.8471144437789917, "Finetune/Loss (Raw)": 0.6637776494026184, "Finetune/Step": 3933, "Finetune/Step Time": 5.544784838333726} +{"Finetune/Learning Rate": 2.943262920657306e-06, "Finetune/Loss": 0.8462935090065002, "Finetune/Loss (Raw)": 0.7865857481956482, "Finetune/Step": 3934, "Finetune/Step Time": 5.531344149261713} +{"Finetune/Learning Rate": 2.938850380863184e-06, "Finetune/Loss": 0.846265435218811, "Finetune/Loss (Raw)": 0.8378204107284546, "Finetune/Step": 3935, "Finetune/Step Time": 5.533931601792574} +{"Finetune/Learning Rate": 2.9344405813633724e-06, "Finetune/Loss": 0.847548246383667, "Finetune/Loss (Raw)": 0.9490808844566345, "Finetune/Step": 3936, "Finetune/Step Time": 5.525222595781088} +{"Finetune/Learning Rate": 2.930033523869228e-06, "Finetune/Loss": 0.847968339920044, "Finetune/Loss (Raw)": 0.9063100814819336, "Finetune/Step": 3937, "Finetune/Step Time": 5.519814187660813} +{"Finetune/Learning Rate": 2.925629210091043e-06, "Finetune/Loss": 0.8473795652389526, "Finetune/Loss (Raw)": 0.8653308749198914, "Finetune/Step": 3938, "Finetune/Step Time": 5.519844137132168} +{"Finetune/Learning Rate": 2.9212276417380468e-06, "Finetune/Loss": 0.8481573462486267, "Finetune/Loss (Raw)": 0.9376490116119385, "Finetune/Step": 3939, "Finetune/Step Time": 5.521086364984512} +{"Finetune/Learning Rate": 2.9168288205183983e-06, "Finetune/Loss": 0.8487520813941956, "Finetune/Loss (Raw)": 0.8797119855880737, "Finetune/Step": 3940, "Finetune/Step Time": 5.508488157764077} +{"Finetune/Learning Rate": 2.9124327481392023e-06, "Finetune/Loss": 0.8494181632995605, "Finetune/Loss (Raw)": 0.9588174223899841, "Finetune/Step": 3941, "Finetune/Step Time": 5.512082379311323} +{"Finetune/Learning Rate": 2.9080394263064815e-06, "Finetune/Loss": 0.8504804372787476, "Finetune/Loss (Raw)": 0.978519082069397, "Finetune/Step": 3942, "Finetune/Step Time": 5.518840784206986} +{"Finetune/Learning Rate": 2.9036488567252006e-06, "Finetune/Loss": 0.8499254584312439, "Finetune/Loss (Raw)": 0.7939980626106262, "Finetune/Step": 3943, "Finetune/Step Time": 5.524969918653369} +{"Finetune/Learning Rate": 2.8992610410992506e-06, "Finetune/Loss": 0.8501913547515869, "Finetune/Loss (Raw)": 0.9171046614646912, "Finetune/Step": 3944, "Finetune/Step Time": 5.524702034890652} +{"Finetune/Learning Rate": 2.894875981131463e-06, "Finetune/Loss": 0.8523667454719543, "Finetune/Loss (Raw)": 1.0909340381622314, "Finetune/Step": 3945, "Finetune/Step Time": 5.541678065434098} +{"Finetune/Learning Rate": 2.890493678523595e-06, "Finetune/Loss": 0.8528790473937988, "Finetune/Loss (Raw)": 0.8840052485466003, "Finetune/Step": 3946, "Finetune/Step Time": 5.556111887097359} +{"Finetune/Learning Rate": 2.8861141349763224e-06, "Finetune/Loss": 0.8536794185638428, "Finetune/Loss (Raw)": 0.9095538854598999, "Finetune/Step": 3947, "Finetune/Step Time": 5.563364231958985} +{"Finetune/Learning Rate": 2.881737352189271e-06, "Finetune/Loss": 0.8509447574615479, "Finetune/Loss (Raw)": 0.3615165948867798, "Finetune/Step": 3948, "Finetune/Step Time": 5.600683182477951} +{"Finetune/Learning Rate": 2.8773633318609796e-06, "Finetune/Loss": 0.852429986000061, "Finetune/Loss (Raw)": 1.0086153745651245, "Finetune/Step": 3949, "Finetune/Step Time": 5.5963673312217} +{"Finetune/Learning Rate": 2.872992075688922e-06, "Finetune/Loss": 0.8539999127388, "Finetune/Loss (Raw)": 0.9827301502227783, "Finetune/Step": 3950, "Finetune/Step Time": 5.593546167016029} +{"Finetune/Learning Rate": 2.868623585369497e-06, "Finetune/Loss": 0.853858232498169, "Finetune/Loss (Raw)": 0.970164954662323, "Finetune/Step": 3951, "Finetune/Step Time": 5.597214099019766} +{"Finetune/Learning Rate": 2.864257862598029e-06, "Finetune/Loss": 0.8529865741729736, "Finetune/Loss (Raw)": 0.7862595915794373, "Finetune/Step": 3952, "Finetune/Step Time": 5.606082033365965} +{"Finetune/Learning Rate": 2.8598949090687723e-06, "Finetune/Loss": 0.8536969423294067, "Finetune/Loss (Raw)": 0.850311815738678, "Finetune/Step": 3953, "Finetune/Step Time": 5.603857761248946} +{"Finetune/Learning Rate": 2.8555347264748988e-06, "Finetune/Loss": 0.8537518382072449, "Finetune/Loss (Raw)": 0.8225793838500977, "Finetune/Step": 3954, "Finetune/Step Time": 5.6031584441661835} +{"Finetune/Learning Rate": 2.8511773165085164e-06, "Finetune/Loss": 0.8529396057128906, "Finetune/Loss (Raw)": 0.804977536201477, "Finetune/Step": 3955, "Finetune/Step Time": 5.5662086214870214} +{"Finetune/Learning Rate": 2.8468226808606525e-06, "Finetune/Loss": 0.8538426160812378, "Finetune/Loss (Raw)": 1.073243260383606, "Finetune/Step": 3956, "Finetune/Step Time": 5.5662805531173944} +{"Finetune/Learning Rate": 2.842470821221246e-06, "Finetune/Loss": 0.8586586713790894, "Finetune/Loss (Raw)": 0.8964299559593201, "Finetune/Step": 3957, "Finetune/Step Time": 5.535035464912653} +{"Finetune/Learning Rate": 2.8381217392791784e-06, "Finetune/Loss": 0.8582444190979004, "Finetune/Loss (Raw)": 0.8415544033050537, "Finetune/Step": 3958, "Finetune/Step Time": 5.531790293753147} +{"Finetune/Learning Rate": 2.833775436722238e-06, "Finetune/Loss": 0.857869029045105, "Finetune/Loss (Raw)": 0.7914708852767944, "Finetune/Step": 3959, "Finetune/Step Time": 5.544369952753186} +{"Finetune/Learning Rate": 2.829431915237144e-06, "Finetune/Loss": 0.8562356233596802, "Finetune/Loss (Raw)": 0.8289390206336975, "Finetune/Step": 3960, "Finetune/Step Time": 5.547759501263499} +{"Finetune/Learning Rate": 2.825091176509529e-06, "Finetune/Loss": 0.8558089137077332, "Finetune/Loss (Raw)": 0.9062338471412659, "Finetune/Step": 3961, "Finetune/Step Time": 5.529591949656606} +{"Finetune/Learning Rate": 2.8207532222239497e-06, "Finetune/Loss": 0.8542420864105225, "Finetune/Loss (Raw)": 0.773417592048645, "Finetune/Step": 3962, "Finetune/Step Time": 5.5370653830468655} +{"Finetune/Learning Rate": 2.816418054063883e-06, "Finetune/Loss": 0.8540587425231934, "Finetune/Loss (Raw)": 0.9204992651939392, "Finetune/Step": 3963, "Finetune/Step Time": 5.546172276139259} +{"Finetune/Learning Rate": 2.8120856737117187e-06, "Finetune/Loss": 0.8545315265655518, "Finetune/Loss (Raw)": 0.8778429627418518, "Finetune/Step": 3964, "Finetune/Step Time": 5.555083589628339} +{"Finetune/Learning Rate": 2.807756082848775e-06, "Finetune/Loss": 0.8555496335029602, "Finetune/Loss (Raw)": 0.9856688976287842, "Finetune/Step": 3965, "Finetune/Step Time": 5.567618690431118} +{"Finetune/Learning Rate": 2.803429283155281e-06, "Finetune/Loss": 0.852200984954834, "Finetune/Loss (Raw)": 0.38667839765548706, "Finetune/Step": 3966, "Finetune/Step Time": 5.596669049933553} +{"Finetune/Learning Rate": 2.7991052763103767e-06, "Finetune/Loss": 0.8517074584960938, "Finetune/Loss (Raw)": 0.8268504738807678, "Finetune/Step": 3967, "Finetune/Step Time": 5.568851836025715} +{"Finetune/Learning Rate": 2.7947840639921308e-06, "Finetune/Loss": 0.8524802923202515, "Finetune/Loss (Raw)": 0.941569983959198, "Finetune/Step": 3968, "Finetune/Step Time": 5.567161321640015} +{"Finetune/Learning Rate": 2.7904656478775196e-06, "Finetune/Loss": 0.8518523573875427, "Finetune/Loss (Raw)": 0.9250662326812744, "Finetune/Step": 3969, "Finetune/Step Time": 5.592525955289602} +{"Finetune/Learning Rate": 2.7861500296424372e-06, "Finetune/Loss": 0.8528502583503723, "Finetune/Loss (Raw)": 0.9378992915153503, "Finetune/Step": 3970, "Finetune/Step Time": 5.58780231885612} +{"Finetune/Learning Rate": 2.781837210961689e-06, "Finetune/Loss": 0.853813886642456, "Finetune/Loss (Raw)": 0.9718935489654541, "Finetune/Step": 3971, "Finetune/Step Time": 5.579392431303859} +{"Finetune/Learning Rate": 2.7775271935089964e-06, "Finetune/Loss": 0.8533740043640137, "Finetune/Loss (Raw)": 0.799938440322876, "Finetune/Step": 3972, "Finetune/Step Time": 5.576867189258337} +{"Finetune/Learning Rate": 2.7732199789569937e-06, "Finetune/Loss": 0.8532650470733643, "Finetune/Loss (Raw)": 0.837137758731842, "Finetune/Step": 3973, "Finetune/Step Time": 5.573395363986492} +{"Finetune/Learning Rate": 2.768915568977222e-06, "Finetune/Loss": 0.8541614413261414, "Finetune/Loss (Raw)": 0.8163000345230103, "Finetune/Step": 3974, "Finetune/Step Time": 5.57666252925992} +{"Finetune/Learning Rate": 2.7646139652401473e-06, "Finetune/Loss": 0.8538784384727478, "Finetune/Loss (Raw)": 0.9542937874794006, "Finetune/Step": 3975, "Finetune/Step Time": 5.574878867715597} +{"Finetune/Learning Rate": 2.7603151694151354e-06, "Finetune/Loss": 0.8580453395843506, "Finetune/Loss (Raw)": 0.8626132607460022, "Finetune/Step": 3976, "Finetune/Step Time": 5.5561767388135195} +{"Finetune/Learning Rate": 2.7560191831704584e-06, "Finetune/Loss": 0.8558483719825745, "Finetune/Loss (Raw)": 0.7641275525093079, "Finetune/Step": 3977, "Finetune/Step Time": 5.557019194588065} +{"Finetune/Learning Rate": 2.751726008173312e-06, "Finetune/Loss": 0.8560007214546204, "Finetune/Loss (Raw)": 0.6482505202293396, "Finetune/Step": 3978, "Finetune/Step Time": 5.548006741330028} +{"Finetune/Learning Rate": 2.747435646089791e-06, "Finetune/Loss": 0.8569246530532837, "Finetune/Loss (Raw)": 0.8854917287826538, "Finetune/Step": 3979, "Finetune/Step Time": 5.551518559455872} +{"Finetune/Learning Rate": 2.7431480985849023e-06, "Finetune/Loss": 0.8548282980918884, "Finetune/Loss (Raw)": 0.6553661823272705, "Finetune/Step": 3980, "Finetune/Step Time": 5.557973638176918} +{"Finetune/Learning Rate": 2.7388633673225586e-06, "Finetune/Loss": 0.8586844801902771, "Finetune/Loss (Raw)": 0.9473516941070557, "Finetune/Step": 3981, "Finetune/Step Time": 5.5287534557282925} +{"Finetune/Learning Rate": 2.734581453965579e-06, "Finetune/Loss": 0.8582266569137573, "Finetune/Loss (Raw)": 0.8434578776359558, "Finetune/Step": 3982, "Finetune/Step Time": 5.523321168497205} +{"Finetune/Learning Rate": 2.730302360175693e-06, "Finetune/Loss": 0.8596816062927246, "Finetune/Loss (Raw)": 0.8406505584716797, "Finetune/Step": 3983, "Finetune/Step Time": 5.502426465973258} +{"Finetune/Learning Rate": 2.726026087613528e-06, "Finetune/Loss": 0.8545089960098267, "Finetune/Loss (Raw)": 0.3418262004852295, "Finetune/Step": 3984, "Finetune/Step Time": 5.528166590258479} +{"Finetune/Learning Rate": 2.7217526379386327e-06, "Finetune/Loss": 0.8589059114456177, "Finetune/Loss (Raw)": 0.9253289103507996, "Finetune/Step": 3985, "Finetune/Step Time": 5.499013831838965} +{"Finetune/Learning Rate": 2.7174820128094393e-06, "Finetune/Loss": 0.864609956741333, "Finetune/Loss (Raw)": 1.0666605234146118, "Finetune/Step": 3986, "Finetune/Step Time": 5.466444116085768} +{"Finetune/Learning Rate": 2.713214213883295e-06, "Finetune/Loss": 0.8624740839004517, "Finetune/Loss (Raw)": 0.789746105670929, "Finetune/Step": 3987, "Finetune/Step Time": 5.474565155804157} +{"Finetune/Learning Rate": 2.7089492428164556e-06, "Finetune/Loss": 0.8630236387252808, "Finetune/Loss (Raw)": 0.9625653028488159, "Finetune/Step": 3988, "Finetune/Step Time": 5.477951094508171} +{"Finetune/Learning Rate": 2.7046871012640687e-06, "Finetune/Loss": 0.8639674186706543, "Finetune/Loss (Raw)": 0.9705782532691956, "Finetune/Step": 3989, "Finetune/Step Time": 5.476780204102397} +{"Finetune/Learning Rate": 2.7004277908801877e-06, "Finetune/Loss": 0.8642848134040833, "Finetune/Loss (Raw)": 0.9259055256843567, "Finetune/Step": 3990, "Finetune/Step Time": 5.486940074712038} +{"Finetune/Learning Rate": 2.6961713133177705e-06, "Finetune/Loss": 0.8654211759567261, "Finetune/Loss (Raw)": 0.9798412919044495, "Finetune/Step": 3991, "Finetune/Step Time": 5.478785391896963} +{"Finetune/Learning Rate": 2.6919176702286696e-06, "Finetune/Loss": 0.8640583753585815, "Finetune/Loss (Raw)": 0.7527365684509277, "Finetune/Step": 3992, "Finetune/Step Time": 5.501773945987225} +{"Finetune/Learning Rate": 2.687666863263644e-06, "Finetune/Loss": 0.8641986846923828, "Finetune/Loss (Raw)": 0.9068096280097961, "Finetune/Step": 3993, "Finetune/Step Time": 5.468020275235176} +{"Finetune/Learning Rate": 2.683418894072345e-06, "Finetune/Loss": 0.8622137308120728, "Finetune/Loss (Raw)": 0.7793486714363098, "Finetune/Step": 3994, "Finetune/Step Time": 5.480763006955385} +{"Finetune/Learning Rate": 2.6791737643033333e-06, "Finetune/Loss": 0.8600906729698181, "Finetune/Loss (Raw)": 0.8117945194244385, "Finetune/Step": 3995, "Finetune/Step Time": 5.480886610224843} +{"Finetune/Learning Rate": 2.6749314756040547e-06, "Finetune/Loss": 0.8593654632568359, "Finetune/Loss (Raw)": 0.8051896095275879, "Finetune/Step": 3996, "Finetune/Step Time": 5.479138869792223} +{"Finetune/Learning Rate": 2.6706920296208573e-06, "Finetune/Loss": 0.8594996929168701, "Finetune/Loss (Raw)": 0.8393034934997559, "Finetune/Step": 3997, "Finetune/Step Time": 5.477282928302884} +{"Finetune/Learning Rate": 2.666455427998994e-06, "Finetune/Loss": 0.861136257648468, "Finetune/Loss (Raw)": 1.020504117012024, "Finetune/Step": 3998, "Finetune/Step Time": 5.460465269163251} +{"Finetune/Learning Rate": 2.6622216723826035e-06, "Finetune/Loss": 0.8615442514419556, "Finetune/Loss (Raw)": 0.9373985528945923, "Finetune/Step": 3999, "Finetune/Step Time": 5.476552305743098} +{"Finetune/Learning Rate": 2.6579907644147227e-06, "Finetune/Loss": 0.8638791441917419, "Finetune/Loss (Raw)": 0.82996666431427, "Finetune/Step": 4000, "Finetune/Step Time": 5.446443321183324} +{"Finetune/Learning Rate": 2.653762705737287e-06, "Finetune/Loss": 0.8646581172943115, "Finetune/Loss (Raw)": 0.8387525081634521, "Finetune/Step": 4001, "Finetune/Step Time": 5.457794822752476} +{"Finetune/Learning Rate": 2.6495374979911215e-06, "Finetune/Loss": 0.8640235662460327, "Finetune/Loss (Raw)": 0.8027731776237488, "Finetune/Step": 4002, "Finetune/Step Time": 5.468016827479005} +{"Finetune/Learning Rate": 2.6453151428159472e-06, "Finetune/Loss": 0.8639128804206848, "Finetune/Loss (Raw)": 0.8668316006660461, "Finetune/Step": 4003, "Finetune/Step Time": 5.480413779616356} +{"Finetune/Learning Rate": 2.641095641850375e-06, "Finetune/Loss": 0.8641819357872009, "Finetune/Loss (Raw)": 0.9268897175788879, "Finetune/Step": 4004, "Finetune/Step Time": 5.4848845936357975} +{"Finetune/Learning Rate": 2.6368789967319196e-06, "Finetune/Loss": 0.8642445802688599, "Finetune/Loss (Raw)": 0.879719078540802, "Finetune/Step": 4005, "Finetune/Step Time": 5.469871370121837} +{"Finetune/Learning Rate": 2.6326652090969697e-06, "Finetune/Loss": 0.868730902671814, "Finetune/Loss (Raw)": 1.0464149713516235, "Finetune/Step": 4006, "Finetune/Step Time": 5.436300626024604} +{"Finetune/Learning Rate": 2.6284542805808144e-06, "Finetune/Loss": 0.8649059534072876, "Finetune/Loss (Raw)": 0.36131158471107483, "Finetune/Step": 4007, "Finetune/Step Time": 5.4317436423152685} +{"Finetune/Learning Rate": 2.624246212817638e-06, "Finetune/Loss": 0.8641659021377563, "Finetune/Loss (Raw)": 0.773006796836853, "Finetune/Step": 4008, "Finetune/Step Time": 5.429884349927306} +{"Finetune/Learning Rate": 2.620041007440508e-06, "Finetune/Loss": 0.8635364770889282, "Finetune/Loss (Raw)": 0.8440094590187073, "Finetune/Step": 4009, "Finetune/Step Time": 5.4253474827855825} +{"Finetune/Learning Rate": 2.6158386660813807e-06, "Finetune/Loss": 0.8623788356781006, "Finetune/Loss (Raw)": 0.797409176826477, "Finetune/Step": 4010, "Finetune/Step Time": 5.433790050446987} +{"Finetune/Learning Rate": 2.6116391903711035e-06, "Finetune/Loss": 0.8624262809753418, "Finetune/Loss (Raw)": 0.9897634387016296, "Finetune/Step": 4011, "Finetune/Step Time": 5.443908525630832} +{"Finetune/Learning Rate": 2.607442581939411e-06, "Finetune/Loss": 0.8613264560699463, "Finetune/Loss (Raw)": 0.6844693422317505, "Finetune/Step": 4012, "Finetune/Step Time": 5.455788364633918} +{"Finetune/Learning Rate": 2.603248842414925e-06, "Finetune/Loss": 0.8627508282661438, "Finetune/Loss (Raw)": 0.8691626787185669, "Finetune/Step": 4013, "Finetune/Step Time": 5.449588691815734} +{"Finetune/Learning Rate": 2.599057973425151e-06, "Finetune/Loss": 0.8632500171661377, "Finetune/Loss (Raw)": 0.8195790648460388, "Finetune/Step": 4014, "Finetune/Step Time": 5.474987380206585} +{"Finetune/Learning Rate": 2.5948699765964915e-06, "Finetune/Loss": 0.863605260848999, "Finetune/Loss (Raw)": 0.8536748886108398, "Finetune/Step": 4015, "Finetune/Step Time": 5.47573110088706} +{"Finetune/Learning Rate": 2.5906848535542196e-06, "Finetune/Loss": 0.8634886741638184, "Finetune/Loss (Raw)": 0.8945183753967285, "Finetune/Step": 4016, "Finetune/Step Time": 5.47732156701386} +{"Finetune/Learning Rate": 2.5865026059224974e-06, "Finetune/Loss": 0.8637883067131042, "Finetune/Loss (Raw)": 0.9161911010742188, "Finetune/Step": 4017, "Finetune/Step Time": 5.479348367080092} +{"Finetune/Learning Rate": 2.58232323532438e-06, "Finetune/Loss": 0.859114408493042, "Finetune/Loss (Raw)": 0.3252307176589966, "Finetune/Step": 4018, "Finetune/Step Time": 5.505556870251894} +{"Finetune/Learning Rate": 2.5781467433817973e-06, "Finetune/Loss": 0.8600276708602905, "Finetune/Loss (Raw)": 1.0051984786987305, "Finetune/Step": 4019, "Finetune/Step Time": 5.485018953680992} +{"Finetune/Learning Rate": 2.573973131715565e-06, "Finetune/Loss": 0.8606926202774048, "Finetune/Loss (Raw)": 0.9462139010429382, "Finetune/Step": 4020, "Finetune/Step Time": 5.483460418879986} +{"Finetune/Learning Rate": 2.569802401945378e-06, "Finetune/Loss": 0.8647390007972717, "Finetune/Loss (Raw)": 0.888246476650238, "Finetune/Step": 4021, "Finetune/Step Time": 5.463242443278432} +{"Finetune/Learning Rate": 2.565634555689818e-06, "Finetune/Loss": 0.8639742136001587, "Finetune/Loss (Raw)": 0.8120826482772827, "Finetune/Step": 4022, "Finetune/Step Time": 5.466403076425195} +{"Finetune/Learning Rate": 2.5614695945663415e-06, "Finetune/Loss": 0.8638185262680054, "Finetune/Loss (Raw)": 0.9123365879058838, "Finetune/Step": 4023, "Finetune/Step Time": 5.468686945736408} +{"Finetune/Learning Rate": 2.557307520191291e-06, "Finetune/Loss": 0.8639352917671204, "Finetune/Loss (Raw)": 0.9645556211471558, "Finetune/Step": 4024, "Finetune/Step Time": 5.480504922568798} +{"Finetune/Learning Rate": 2.5531483341798856e-06, "Finetune/Loss": 0.8637762069702148, "Finetune/Loss (Raw)": 0.8672878742218018, "Finetune/Step": 4025, "Finetune/Step Time": 5.480223301798105} +{"Finetune/Learning Rate": 2.548992038146224e-06, "Finetune/Loss": 0.8647003173828125, "Finetune/Loss (Raw)": 0.9767588973045349, "Finetune/Step": 4026, "Finetune/Step Time": 5.46735642850399} +{"Finetune/Learning Rate": 2.5448386337032817e-06, "Finetune/Loss": 0.8647177219390869, "Finetune/Loss (Raw)": 1.006410837173462, "Finetune/Step": 4027, "Finetune/Step Time": 5.463887335732579} +{"Finetune/Learning Rate": 2.5406881224629175e-06, "Finetune/Loss": 0.8643072843551636, "Finetune/Loss (Raw)": 0.818500280380249, "Finetune/Step": 4028, "Finetune/Step Time": 5.448614381253719} +{"Finetune/Learning Rate": 2.5365405060358663e-06, "Finetune/Loss": 0.8640035390853882, "Finetune/Loss (Raw)": 0.9007758498191833, "Finetune/Step": 4029, "Finetune/Step Time": 5.443355429917574} +{"Finetune/Learning Rate": 2.532395786031726e-06, "Finetune/Loss": 0.8594678044319153, "Finetune/Loss (Raw)": 0.3945002853870392, "Finetune/Step": 4030, "Finetune/Step Time": 5.471679572016001} +{"Finetune/Learning Rate": 2.528253964058992e-06, "Finetune/Loss": 0.8587841987609863, "Finetune/Loss (Raw)": 0.8750459551811218, "Finetune/Step": 4031, "Finetune/Step Time": 5.471457218751311} +{"Finetune/Learning Rate": 2.5241150417250216e-06, "Finetune/Loss": 0.8586764335632324, "Finetune/Loss (Raw)": 0.8831957578659058, "Finetune/Step": 4032, "Finetune/Step Time": 5.464475106447935} +{"Finetune/Learning Rate": 2.5199790206360486e-06, "Finetune/Loss": 0.8604859709739685, "Finetune/Loss (Raw)": 0.932068407535553, "Finetune/Step": 4033, "Finetune/Step Time": 5.452560365200043} +{"Finetune/Learning Rate": 2.5158459023971826e-06, "Finetune/Loss": 0.8598090410232544, "Finetune/Loss (Raw)": 0.8567652106285095, "Finetune/Step": 4034, "Finetune/Step Time": 5.412756379693747} +{"Finetune/Learning Rate": 2.511715688612405e-06, "Finetune/Loss": 0.8574618101119995, "Finetune/Loss (Raw)": 0.8530348539352417, "Finetune/Step": 4035, "Finetune/Step Time": 5.41944689117372} +{"Finetune/Learning Rate": 2.5075883808845736e-06, "Finetune/Loss": 0.8591097593307495, "Finetune/Loss (Raw)": 1.034805417060852, "Finetune/Step": 4036, "Finetune/Step Time": 5.411352826282382} +{"Finetune/Learning Rate": 2.5034639808154115e-06, "Finetune/Loss": 0.8588619232177734, "Finetune/Loss (Raw)": 0.796576738357544, "Finetune/Step": 4037, "Finetune/Step Time": 5.417716648429632} +{"Finetune/Learning Rate": 2.4993424900055228e-06, "Finetune/Loss": 0.8541120290756226, "Finetune/Loss (Raw)": 0.3544836640357971, "Finetune/Step": 4038, "Finetune/Step Time": 5.446753038093448} +{"Finetune/Learning Rate": 2.4952239100543808e-06, "Finetune/Loss": 0.8534044027328491, "Finetune/Loss (Raw)": 0.9343929290771484, "Finetune/Step": 4039, "Finetune/Step Time": 5.461661813780665} +{"Finetune/Learning Rate": 2.491108242560313e-06, "Finetune/Loss": 0.8543049097061157, "Finetune/Loss (Raw)": 0.9295958876609802, "Finetune/Step": 4040, "Finetune/Step Time": 5.45057512819767} +{"Finetune/Learning Rate": 2.486995489120542e-06, "Finetune/Loss": 0.8536378741264343, "Finetune/Loss (Raw)": 0.8128049373626709, "Finetune/Step": 4041, "Finetune/Step Time": 5.44597783498466} +{"Finetune/Learning Rate": 2.4828856513311405e-06, "Finetune/Loss": 0.8538780808448792, "Finetune/Loss (Raw)": 0.8691111207008362, "Finetune/Step": 4042, "Finetune/Step Time": 5.464033307507634} +{"Finetune/Learning Rate": 2.4787787307870605e-06, "Finetune/Loss": 0.8539223670959473, "Finetune/Loss (Raw)": 0.9128117561340332, "Finetune/Step": 4043, "Finetune/Step Time": 5.466594476252794} +{"Finetune/Learning Rate": 2.474674729082115e-06, "Finetune/Loss": 0.853860080242157, "Finetune/Loss (Raw)": 0.8874151110649109, "Finetune/Step": 4044, "Finetune/Step Time": 5.45881257019937} +{"Finetune/Learning Rate": 2.470573647808987e-06, "Finetune/Loss": 0.8490855693817139, "Finetune/Loss (Raw)": 0.2910805940628052, "Finetune/Step": 4045, "Finetune/Step Time": 5.469940882176161} +{"Finetune/Learning Rate": 2.466475488559227e-06, "Finetune/Loss": 0.848002552986145, "Finetune/Loss (Raw)": 0.8197330832481384, "Finetune/Step": 4046, "Finetune/Step Time": 5.466887542977929} +{"Finetune/Learning Rate": 2.4623802529232476e-06, "Finetune/Loss": 0.8475538492202759, "Finetune/Loss (Raw)": 0.7983049750328064, "Finetune/Step": 4047, "Finetune/Step Time": 5.470015639439225} +{"Finetune/Learning Rate": 2.458287942490335e-06, "Finetune/Loss": 0.8435515761375427, "Finetune/Loss (Raw)": 0.3736404478549957, "Finetune/Step": 4048, "Finetune/Step Time": 5.464957352727652} +{"Finetune/Learning Rate": 2.454198558848635e-06, "Finetune/Loss": 0.8441601991653442, "Finetune/Loss (Raw)": 0.7595275640487671, "Finetune/Step": 4049, "Finetune/Step Time": 5.4478403721004725} +{"Finetune/Learning Rate": 2.4501121035851494e-06, "Finetune/Loss": 0.8432824611663818, "Finetune/Loss (Raw)": 0.8357094526290894, "Finetune/Step": 4050, "Finetune/Step Time": 5.440479787066579} +{"Finetune/Learning Rate": 2.4460285782857606e-06, "Finetune/Loss": 0.8438757061958313, "Finetune/Loss (Raw)": 0.79435133934021, "Finetune/Step": 4051, "Finetune/Step Time": 5.427312538027763} +{"Finetune/Learning Rate": 2.441947984535201e-06, "Finetune/Loss": 0.843461275100708, "Finetune/Loss (Raw)": 0.8513144254684448, "Finetune/Step": 4052, "Finetune/Step Time": 5.448331633582711} +{"Finetune/Learning Rate": 2.4378703239170707e-06, "Finetune/Loss": 0.8392767906188965, "Finetune/Loss (Raw)": 0.3744470179080963, "Finetune/Step": 4053, "Finetune/Step Time": 5.476667104288936} +{"Finetune/Learning Rate": 2.4337955980138293e-06, "Finetune/Loss": 0.8396377563476562, "Finetune/Loss (Raw)": 0.8705897927284241, "Finetune/Step": 4054, "Finetune/Step Time": 5.485681185498834} +{"Finetune/Learning Rate": 2.4297238084067988e-06, "Finetune/Loss": 0.8399577140808105, "Finetune/Loss (Raw)": 0.9067223072052002, "Finetune/Step": 4055, "Finetune/Step Time": 5.472856359556317} +{"Finetune/Learning Rate": 2.425654956676161e-06, "Finetune/Loss": 0.8372019529342651, "Finetune/Loss (Raw)": 0.6363107562065125, "Finetune/Step": 4056, "Finetune/Step Time": 5.493685077875853} +{"Finetune/Learning Rate": 2.421589044400955e-06, "Finetune/Loss": 0.8376263380050659, "Finetune/Loss (Raw)": 0.9758806228637695, "Finetune/Step": 4057, "Finetune/Step Time": 5.490291923284531} +{"Finetune/Learning Rate": 2.4175260731590878e-06, "Finetune/Loss": 0.8374295234680176, "Finetune/Loss (Raw)": 0.8294947147369385, "Finetune/Step": 4058, "Finetune/Step Time": 5.482128536328673} +{"Finetune/Learning Rate": 2.413466044527318e-06, "Finetune/Loss": 0.8366532325744629, "Finetune/Loss (Raw)": 0.8872633576393127, "Finetune/Step": 4059, "Finetune/Step Time": 5.496247723698616} +{"Finetune/Learning Rate": 2.4094089600812576e-06, "Finetune/Loss": 0.8352338075637817, "Finetune/Loss (Raw)": 0.6587767004966736, "Finetune/Step": 4060, "Finetune/Step Time": 5.498607665300369} +{"Finetune/Learning Rate": 2.405354821395388e-06, "Finetune/Loss": 0.8363491296768188, "Finetune/Loss (Raw)": 0.8065378665924072, "Finetune/Step": 4061, "Finetune/Step Time": 5.491146005690098} +{"Finetune/Learning Rate": 2.4013036300430403e-06, "Finetune/Loss": 0.8381220102310181, "Finetune/Loss (Raw)": 1.0135188102722168, "Finetune/Step": 4062, "Finetune/Step Time": 5.494562327861786} +{"Finetune/Learning Rate": 2.3972553875964035e-06, "Finetune/Loss": 0.834330677986145, "Finetune/Loss (Raw)": 0.3525349199771881, "Finetune/Step": 4063, "Finetune/Step Time": 5.514437783509493} +{"Finetune/Learning Rate": 2.393210095626515e-06, "Finetune/Loss": 0.8298999071121216, "Finetune/Loss (Raw)": 0.38194066286087036, "Finetune/Step": 4064, "Finetune/Step Time": 5.5467061214149} +{"Finetune/Learning Rate": 2.38916775570328e-06, "Finetune/Loss": 0.8289505243301392, "Finetune/Loss (Raw)": 0.784785807132721, "Finetune/Step": 4065, "Finetune/Step Time": 5.551761098206043} +{"Finetune/Learning Rate": 2.3851283693954506e-06, "Finetune/Loss": 0.8289951682090759, "Finetune/Loss (Raw)": 0.8710393905639648, "Finetune/Step": 4066, "Finetune/Step Time": 5.55554554797709} +{"Finetune/Learning Rate": 2.3810919382706297e-06, "Finetune/Loss": 0.8293970227241516, "Finetune/Loss (Raw)": 0.9890874624252319, "Finetune/Step": 4067, "Finetune/Step Time": 5.554399240761995} +{"Finetune/Learning Rate": 2.3770584638952874e-06, "Finetune/Loss": 0.8297619819641113, "Finetune/Loss (Raw)": 0.9264355301856995, "Finetune/Step": 4068, "Finetune/Step Time": 5.555884249508381} +{"Finetune/Learning Rate": 2.3730279478347275e-06, "Finetune/Loss": 0.8294926881790161, "Finetune/Loss (Raw)": 0.9243386387825012, "Finetune/Step": 4069, "Finetune/Step Time": 5.5686168018728495} +{"Finetune/Learning Rate": 2.3690003916531157e-06, "Finetune/Loss": 0.8283067941665649, "Finetune/Loss (Raw)": 0.8267234563827515, "Finetune/Step": 4070, "Finetune/Step Time": 5.564331503584981} +{"Finetune/Learning Rate": 2.3649757969134667e-06, "Finetune/Loss": 0.8289510011672974, "Finetune/Loss (Raw)": 0.876465916633606, "Finetune/Step": 4071, "Finetune/Step Time": 5.570545071735978} +{"Finetune/Learning Rate": 2.3609541651776537e-06, "Finetune/Loss": 0.8278207778930664, "Finetune/Loss (Raw)": 0.7724308967590332, "Finetune/Step": 4072, "Finetune/Step Time": 5.580390682443976} +{"Finetune/Learning Rate": 2.356935498006391e-06, "Finetune/Loss": 0.826364278793335, "Finetune/Loss (Raw)": 0.904502809047699, "Finetune/Step": 4073, "Finetune/Step Time": 5.563124306499958} +{"Finetune/Learning Rate": 2.3529197969592375e-06, "Finetune/Loss": 0.8271656036376953, "Finetune/Loss (Raw)": 0.9865754842758179, "Finetune/Step": 4074, "Finetune/Step Time": 5.550328962504864} +{"Finetune/Learning Rate": 2.3489070635946175e-06, "Finetune/Loss": 0.8265236616134644, "Finetune/Loss (Raw)": 0.8273842930793762, "Finetune/Step": 4075, "Finetune/Step Time": 5.538820046931505} +{"Finetune/Learning Rate": 2.3448972994697906e-06, "Finetune/Loss": 0.828920841217041, "Finetune/Loss (Raw)": 0.6683547496795654, "Finetune/Step": 4076, "Finetune/Step Time": 5.521470159292221} +{"Finetune/Learning Rate": 2.340890506140869e-06, "Finetune/Loss": 0.8277050256729126, "Finetune/Loss (Raw)": 0.8529929518699646, "Finetune/Step": 4077, "Finetune/Step Time": 5.535915940999985} +{"Finetune/Learning Rate": 2.3368866851628102e-06, "Finetune/Loss": 0.8264039754867554, "Finetune/Loss (Raw)": 0.8161939382553101, "Finetune/Step": 4078, "Finetune/Step Time": 5.538470696657896} +{"Finetune/Learning Rate": 2.3328858380894194e-06, "Finetune/Loss": 0.8270326256752014, "Finetune/Loss (Raw)": 1.0506337881088257, "Finetune/Step": 4079, "Finetune/Step Time": 5.55493844114244} +{"Finetune/Learning Rate": 2.328887966473348e-06, "Finetune/Loss": 0.8269753456115723, "Finetune/Loss (Raw)": 0.7789307236671448, "Finetune/Step": 4080, "Finetune/Step Time": 5.553926311433315} +{"Finetune/Learning Rate": 2.324893071866087e-06, "Finetune/Loss": 0.8270636200904846, "Finetune/Loss (Raw)": 0.8616065382957458, "Finetune/Step": 4081, "Finetune/Step Time": 5.576177509501576} +{"Finetune/Learning Rate": 2.320901155817983e-06, "Finetune/Loss": 0.8269675970077515, "Finetune/Loss (Raw)": 0.8102860450744629, "Finetune/Step": 4082, "Finetune/Step Time": 5.578472003340721} +{"Finetune/Learning Rate": 2.316912219878222e-06, "Finetune/Loss": 0.8283801078796387, "Finetune/Loss (Raw)": 0.9857771396636963, "Finetune/Step": 4083, "Finetune/Step Time": 5.580862365663052} +{"Finetune/Learning Rate": 2.312926265594825e-06, "Finetune/Loss": 0.8236342668533325, "Finetune/Loss (Raw)": 0.4657793343067169, "Finetune/Step": 4084, "Finetune/Step Time": 5.606686312705278} +{"Finetune/Learning Rate": 2.308943294514667e-06, "Finetune/Loss": 0.8232638835906982, "Finetune/Loss (Raw)": 0.8490225672721863, "Finetune/Step": 4085, "Finetune/Step Time": 5.623209962621331} +{"Finetune/Learning Rate": 2.3049633081834634e-06, "Finetune/Loss": 0.823672890663147, "Finetune/Loss (Raw)": 0.8938994407653809, "Finetune/Step": 4086, "Finetune/Step Time": 5.630583738908172} +{"Finetune/Learning Rate": 2.3009863081457663e-06, "Finetune/Loss": 0.8252913951873779, "Finetune/Loss (Raw)": 0.9986515641212463, "Finetune/Step": 4087, "Finetune/Step Time": 5.617977092042565} +{"Finetune/Learning Rate": 2.2970122959449716e-06, "Finetune/Loss": 0.8262485861778259, "Finetune/Loss (Raw)": 0.9514532089233398, "Finetune/Step": 4088, "Finetune/Step Time": 5.61303286626935} +{"Finetune/Learning Rate": 2.2930412731233174e-06, "Finetune/Loss": 0.8257498741149902, "Finetune/Loss (Raw)": 0.8423998355865479, "Finetune/Step": 4089, "Finetune/Step Time": 5.6138971745967865} +{"Finetune/Learning Rate": 2.2890732412218795e-06, "Finetune/Loss": 0.8262333273887634, "Finetune/Loss (Raw)": 0.8353003263473511, "Finetune/Step": 4090, "Finetune/Step Time": 5.606804044917226} +{"Finetune/Learning Rate": 2.2851082017805704e-06, "Finetune/Loss": 0.8255817890167236, "Finetune/Loss (Raw)": 0.8371061086654663, "Finetune/Step": 4091, "Finetune/Step Time": 5.61488457210362} +{"Finetune/Learning Rate": 2.2811461563381497e-06, "Finetune/Loss": 0.8252842426300049, "Finetune/Loss (Raw)": 0.8397560715675354, "Finetune/Step": 4092, "Finetune/Step Time": 5.616372233256698} +{"Finetune/Learning Rate": 2.2771871064322117e-06, "Finetune/Loss": 0.8253107666969299, "Finetune/Loss (Raw)": 0.9890653491020203, "Finetune/Step": 4093, "Finetune/Step Time": 5.60062026232481} +{"Finetune/Learning Rate": 2.2732310535991775e-06, "Finetune/Loss": 0.8306245803833008, "Finetune/Loss (Raw)": 1.0668425559997559, "Finetune/Step": 4094, "Finetune/Step Time": 5.571093592792749} +{"Finetune/Learning Rate": 2.2692779993743207e-06, "Finetune/Loss": 0.8302137851715088, "Finetune/Loss (Raw)": 0.7742677330970764, "Finetune/Step": 4095, "Finetune/Step Time": 5.57901830971241} +{"Finetune/Learning Rate": 2.265327945291742e-06, "Finetune/Loss": 0.8308541774749756, "Finetune/Loss (Raw)": 1.0235397815704346, "Finetune/Step": 4096, "Finetune/Step Time": 5.583600671961904} +{"Finetune/Learning Rate": 2.2613808928843827e-06, "Finetune/Loss": 0.8311716318130493, "Finetune/Loss (Raw)": 0.9657034873962402, "Finetune/Step": 4097, "Finetune/Step Time": 5.568964784964919} +{"Finetune/Learning Rate": 2.2574368436840144e-06, "Finetune/Loss": 0.8292592763900757, "Finetune/Loss (Raw)": 0.6931156516075134, "Finetune/Step": 4098, "Finetune/Step Time": 5.576930711045861} +{"Finetune/Learning Rate": 2.2534957992212457e-06, "Finetune/Loss": 0.8288627862930298, "Finetune/Loss (Raw)": 0.9211360216140747, "Finetune/Step": 4099, "Finetune/Step Time": 5.580231262370944} +{"Finetune/Learning Rate": 2.2495577610255203e-06, "Finetune/Loss": 0.8300986289978027, "Finetune/Loss (Raw)": 0.9581279158592224, "Finetune/Step": 4100, "Finetune/Step Time": 5.580260461196303} +{"Finetune/Learning Rate": 2.24562273062511e-06, "Finetune/Loss": 0.8305213451385498, "Finetune/Loss (Raw)": 0.8912487626075745, "Finetune/Step": 4101, "Finetune/Step Time": 5.600827444344759} +{"Finetune/Learning Rate": 2.2416907095471296e-06, "Finetune/Loss": 0.8307979106903076, "Finetune/Loss (Raw)": 0.8516936898231506, "Finetune/Step": 4102, "Finetune/Step Time": 5.602653516456485} +{"Finetune/Learning Rate": 2.237761699317519e-06, "Finetune/Loss": 0.830314040184021, "Finetune/Loss (Raw)": 0.8923670649528503, "Finetune/Step": 4103, "Finetune/Step Time": 5.605554578825831} +{"Finetune/Learning Rate": 2.2338357014610425e-06, "Finetune/Loss": 0.8303643465042114, "Finetune/Loss (Raw)": 0.8690502047538757, "Finetune/Step": 4104, "Finetune/Step Time": 5.5929036773741245} +{"Finetune/Learning Rate": 2.2299127175013113e-06, "Finetune/Loss": 0.8305099010467529, "Finetune/Loss (Raw)": 0.782759428024292, "Finetune/Step": 4105, "Finetune/Step Time": 5.586925353854895} +{"Finetune/Learning Rate": 2.2259927489607568e-06, "Finetune/Loss": 0.8319383859634399, "Finetune/Loss (Raw)": 0.8310964703559875, "Finetune/Step": 4106, "Finetune/Step Time": 5.585287583991885} +{"Finetune/Learning Rate": 2.222075797360641e-06, "Finetune/Loss": 0.8316471576690674, "Finetune/Loss (Raw)": 0.848219633102417, "Finetune/Step": 4107, "Finetune/Step Time": 5.587207190692425} +{"Finetune/Learning Rate": 2.2181618642210577e-06, "Finetune/Loss": 0.8339248895645142, "Finetune/Loss (Raw)": 0.94691401720047, "Finetune/Step": 4108, "Finetune/Step Time": 5.581349886953831} +{"Finetune/Learning Rate": 2.2142509510609277e-06, "Finetune/Loss": 0.8329458236694336, "Finetune/Loss (Raw)": 0.8220275640487671, "Finetune/Step": 4109, "Finetune/Step Time": 5.5893627889454365} +{"Finetune/Learning Rate": 2.210343059397999e-06, "Finetune/Loss": 0.8334639668464661, "Finetune/Loss (Raw)": 0.9097814559936523, "Finetune/Step": 4110, "Finetune/Step Time": 5.596385354176164} +{"Finetune/Learning Rate": 2.206438190748846e-06, "Finetune/Loss": 0.8346796035766602, "Finetune/Loss (Raw)": 0.9962570667266846, "Finetune/Step": 4111, "Finetune/Step Time": 5.602006817236543} +{"Finetune/Learning Rate": 2.2025363466288796e-06, "Finetune/Loss": 0.8379808664321899, "Finetune/Loss (Raw)": 0.7643852233886719, "Finetune/Step": 4112, "Finetune/Step Time": 5.584184255450964} +{"Finetune/Learning Rate": 2.1986375285523233e-06, "Finetune/Loss": 0.8363442420959473, "Finetune/Loss (Raw)": 0.7158409953117371, "Finetune/Step": 4113, "Finetune/Step Time": 5.5829333402216434} +{"Finetune/Learning Rate": 2.194741738032229e-06, "Finetune/Loss": 0.8348526358604431, "Finetune/Loss (Raw)": 0.875730037689209, "Finetune/Step": 4114, "Finetune/Step Time": 5.58426021784544} +{"Finetune/Learning Rate": 2.1908489765804853e-06, "Finetune/Loss": 0.8354965448379517, "Finetune/Loss (Raw)": 0.8721655607223511, "Finetune/Step": 4115, "Finetune/Step Time": 5.580511959269643} +{"Finetune/Learning Rate": 2.1869592457077924e-06, "Finetune/Loss": 0.8329579830169678, "Finetune/Loss (Raw)": 0.6376340985298157, "Finetune/Step": 4116, "Finetune/Step Time": 5.601141810417175} +{"Finetune/Learning Rate": 2.183072546923679e-06, "Finetune/Loss": 0.8314290046691895, "Finetune/Loss (Raw)": 0.7748690247535706, "Finetune/Step": 4117, "Finetune/Step Time": 5.610371273010969} +{"Finetune/Learning Rate": 2.179188881736498e-06, "Finetune/Loss": 0.8323726654052734, "Finetune/Loss (Raw)": 1.0466909408569336, "Finetune/Step": 4118, "Finetune/Step Time": 5.5974781811237335} +{"Finetune/Learning Rate": 2.1753082516534242e-06, "Finetune/Loss": 0.8324261903762817, "Finetune/Loss (Raw)": 0.986696183681488, "Finetune/Step": 4119, "Finetune/Step Time": 5.591969983652234} +{"Finetune/Learning Rate": 2.1714306581804533e-06, "Finetune/Loss": 0.8335066437721252, "Finetune/Loss (Raw)": 0.8910374045372009, "Finetune/Step": 4120, "Finetune/Step Time": 5.581951543688774} +{"Finetune/Learning Rate": 2.1675561028224012e-06, "Finetune/Loss": 0.8333288431167603, "Finetune/Loss (Raw)": 0.8840467929840088, "Finetune/Step": 4121, "Finetune/Step Time": 5.587639052420855} +{"Finetune/Learning Rate": 2.163684587082915e-06, "Finetune/Loss": 0.8340597748756409, "Finetune/Loss (Raw)": 0.8729031085968018, "Finetune/Step": 4122, "Finetune/Step Time": 5.586577098816633} +{"Finetune/Learning Rate": 2.159816112464447e-06, "Finetune/Loss": 0.8356349468231201, "Finetune/Loss (Raw)": 1.0134202241897583, "Finetune/Step": 4123, "Finetune/Step Time": 5.598569002002478} +{"Finetune/Learning Rate": 2.1559506804682762e-06, "Finetune/Loss": 0.8344221115112305, "Finetune/Loss (Raw)": 0.6499453783035278, "Finetune/Step": 4124, "Finetune/Step Time": 5.600419741123915} +{"Finetune/Learning Rate": 2.1520882925945074e-06, "Finetune/Loss": 0.8349429965019226, "Finetune/Loss (Raw)": 0.9059774875640869, "Finetune/Step": 4125, "Finetune/Step Time": 5.60403355397284} +{"Finetune/Learning Rate": 2.1482289503420537e-06, "Finetune/Loss": 0.8338651061058044, "Finetune/Loss (Raw)": 0.8825331926345825, "Finetune/Step": 4126, "Finetune/Step Time": 5.598677286878228} +{"Finetune/Learning Rate": 2.144372655208653e-06, "Finetune/Loss": 0.8336230516433716, "Finetune/Loss (Raw)": 0.9064107537269592, "Finetune/Step": 4127, "Finetune/Step Time": 5.5784054659307} +{"Finetune/Learning Rate": 2.1405194086908555e-06, "Finetune/Loss": 0.8336066007614136, "Finetune/Loss (Raw)": 0.8278742432594299, "Finetune/Step": 4128, "Finetune/Step Time": 5.574208250269294} +{"Finetune/Learning Rate": 2.1366692122840317e-06, "Finetune/Loss": 0.8334556221961975, "Finetune/Loss (Raw)": 0.8194244503974915, "Finetune/Step": 4129, "Finetune/Step Time": 5.554423796012998} +{"Finetune/Learning Rate": 2.1328220674823686e-06, "Finetune/Loss": 0.833733081817627, "Finetune/Loss (Raw)": 0.8382861614227295, "Finetune/Step": 4130, "Finetune/Step Time": 5.544868377968669} +{"Finetune/Learning Rate": 2.1289779757788655e-06, "Finetune/Loss": 0.8338804841041565, "Finetune/Loss (Raw)": 0.8856962323188782, "Finetune/Step": 4131, "Finetune/Step Time": 5.53769938647747} +{"Finetune/Learning Rate": 2.1251369386653454e-06, "Finetune/Loss": 0.8336730003356934, "Finetune/Loss (Raw)": 0.9003303050994873, "Finetune/Step": 4132, "Finetune/Step Time": 5.539574237540364} +{"Finetune/Learning Rate": 2.121298957632434e-06, "Finetune/Loss": 0.8334674835205078, "Finetune/Loss (Raw)": 0.8534169793128967, "Finetune/Step": 4133, "Finetune/Step Time": 5.547080000862479} +{"Finetune/Learning Rate": 2.117464034169575e-06, "Finetune/Loss": 0.8320678472518921, "Finetune/Loss (Raw)": 0.8672651648521423, "Finetune/Step": 4134, "Finetune/Step Time": 5.563423799350858} +{"Finetune/Learning Rate": 2.113632169765033e-06, "Finetune/Loss": 0.8354079723358154, "Finetune/Loss (Raw)": 0.7888339161872864, "Finetune/Step": 4135, "Finetune/Step Time": 5.533457133919001} +{"Finetune/Learning Rate": 2.1098033659058792e-06, "Finetune/Loss": 0.8359929323196411, "Finetune/Loss (Raw)": 0.8478922843933105, "Finetune/Step": 4136, "Finetune/Step Time": 5.534158259630203} +{"Finetune/Learning Rate": 2.105977624077995e-06, "Finetune/Loss": 0.835599958896637, "Finetune/Loss (Raw)": 0.7937033176422119, "Finetune/Step": 4137, "Finetune/Step Time": 5.531662417575717} +{"Finetune/Learning Rate": 2.102154945766076e-06, "Finetune/Loss": 0.8368254899978638, "Finetune/Loss (Raw)": 0.9542800784111023, "Finetune/Step": 4138, "Finetune/Step Time": 5.5174135360866785} +{"Finetune/Learning Rate": 2.0983353324536316e-06, "Finetune/Loss": 0.8352465629577637, "Finetune/Loss (Raw)": 0.7876574993133545, "Finetune/Step": 4139, "Finetune/Step Time": 5.502218162640929} +{"Finetune/Learning Rate": 2.0945187856229778e-06, "Finetune/Loss": 0.8362668752670288, "Finetune/Loss (Raw)": 0.8150759339332581, "Finetune/Step": 4140, "Finetune/Step Time": 5.491625463590026} +{"Finetune/Learning Rate": 2.0907053067552384e-06, "Finetune/Loss": 0.8324806690216064, "Finetune/Loss (Raw)": 0.3845234215259552, "Finetune/Step": 4141, "Finetune/Step Time": 5.5130972899496555} +{"Finetune/Learning Rate": 2.0868948973303593e-06, "Finetune/Loss": 0.8321533203125, "Finetune/Loss (Raw)": 0.7776802182197571, "Finetune/Step": 4142, "Finetune/Step Time": 5.489636497572064} +{"Finetune/Learning Rate": 2.0830875588270773e-06, "Finetune/Loss": 0.8316574096679688, "Finetune/Loss (Raw)": 0.7901971340179443, "Finetune/Step": 4143, "Finetune/Step Time": 5.486303448677063} +{"Finetune/Learning Rate": 2.0792832927229465e-06, "Finetune/Loss": 0.8269369602203369, "Finetune/Loss (Raw)": 0.29029858112335205, "Finetune/Step": 4144, "Finetune/Step Time": 5.513144766911864} +{"Finetune/Learning Rate": 2.0754821004943337e-06, "Finetune/Loss": 0.8255366086959839, "Finetune/Loss (Raw)": 0.7369530200958252, "Finetune/Step": 4145, "Finetune/Step Time": 5.52152000553906} +{"Finetune/Learning Rate": 2.071683983616406e-06, "Finetune/Loss": 0.8301470279693604, "Finetune/Loss (Raw)": 0.9153640866279602, "Finetune/Step": 4146, "Finetune/Step Time": 5.490534609183669} +{"Finetune/Learning Rate": 2.0678889435631333e-06, "Finetune/Loss": 0.8289512395858765, "Finetune/Loss (Raw)": 0.8521352410316467, "Finetune/Step": 4147, "Finetune/Step Time": 5.4965817872434855} +{"Finetune/Learning Rate": 2.064096981807302e-06, "Finetune/Loss": 0.8285872340202332, "Finetune/Loss (Raw)": 0.8996221423149109, "Finetune/Step": 4148, "Finetune/Step Time": 5.502840479835868} +{"Finetune/Learning Rate": 2.060308099820496e-06, "Finetune/Loss": 0.8282557725906372, "Finetune/Loss (Raw)": 0.8458095192909241, "Finetune/Step": 4149, "Finetune/Step Time": 5.491392767056823} +{"Finetune/Learning Rate": 2.056522299073108e-06, "Finetune/Loss": 0.8280377388000488, "Finetune/Loss (Raw)": 0.7841838598251343, "Finetune/Step": 4150, "Finetune/Step Time": 5.51644840836525} +{"Finetune/Learning Rate": 2.052739581034332e-06, "Finetune/Loss": 0.8276195526123047, "Finetune/Loss (Raw)": 0.85880047082901, "Finetune/Step": 4151, "Finetune/Step Time": 5.521260291337967} +{"Finetune/Learning Rate": 2.0489599471721678e-06, "Finetune/Loss": 0.8268441557884216, "Finetune/Loss (Raw)": 0.8653159141540527, "Finetune/Step": 4152, "Finetune/Step Time": 5.5206233616918325} +{"Finetune/Learning Rate": 2.0451833989534165e-06, "Finetune/Loss": 0.8265986442565918, "Finetune/Loss (Raw)": 0.8358566761016846, "Finetune/Step": 4153, "Finetune/Step Time": 5.517810499295592} +{"Finetune/Learning Rate": 2.0414099378436812e-06, "Finetune/Loss": 0.8259261846542358, "Finetune/Loss (Raw)": 0.89068603515625, "Finetune/Step": 4154, "Finetune/Step Time": 5.521968174725771} +{"Finetune/Learning Rate": 2.0376395653073735e-06, "Finetune/Loss": 0.824471116065979, "Finetune/Loss (Raw)": 0.8201642036437988, "Finetune/Step": 4155, "Finetune/Step Time": 5.524400368332863} +{"Finetune/Learning Rate": 2.0338722828077007e-06, "Finetune/Loss": 0.8249372243881226, "Finetune/Loss (Raw)": 0.8781618475914001, "Finetune/Step": 4156, "Finetune/Step Time": 5.523644745349884} +{"Finetune/Learning Rate": 2.030108091806664e-06, "Finetune/Loss": 0.8255796432495117, "Finetune/Loss (Raw)": 0.9829987287521362, "Finetune/Step": 4157, "Finetune/Step Time": 5.524223085492849} +{"Finetune/Learning Rate": 2.0263469937650813e-06, "Finetune/Loss": 0.8290107250213623, "Finetune/Loss (Raw)": 0.8336880803108215, "Finetune/Step": 4158, "Finetune/Step Time": 5.503603460267186} +{"Finetune/Learning Rate": 2.0225889901425565e-06, "Finetune/Loss": 0.8294848203659058, "Finetune/Loss (Raw)": 0.9357200860977173, "Finetune/Step": 4159, "Finetune/Step Time": 5.503605602309108} +{"Finetune/Learning Rate": 2.0188340823975005e-06, "Finetune/Loss": 0.8302590847015381, "Finetune/Loss (Raw)": 0.9823090434074402, "Finetune/Step": 4160, "Finetune/Step Time": 5.49675783701241} +{"Finetune/Learning Rate": 2.0150822719871167e-06, "Finetune/Loss": 0.8301229476928711, "Finetune/Loss (Raw)": 0.9146352410316467, "Finetune/Step": 4161, "Finetune/Step Time": 5.494245113804936} +{"Finetune/Learning Rate": 2.011333560367411e-06, "Finetune/Loss": 0.8308793306350708, "Finetune/Loss (Raw)": 0.9535925984382629, "Finetune/Step": 4162, "Finetune/Step Time": 5.496109442785382} +{"Finetune/Learning Rate": 2.007587948993185e-06, "Finetune/Loss": 0.829836368560791, "Finetune/Loss (Raw)": 0.7195264101028442, "Finetune/Step": 4163, "Finetune/Step Time": 5.507006250321865} +{"Finetune/Learning Rate": 2.003845439318035e-06, "Finetune/Loss": 0.8264684081077576, "Finetune/Loss (Raw)": 0.60370934009552, "Finetune/Step": 4164, "Finetune/Step Time": 5.513746835291386} +{"Finetune/Learning Rate": 2.000106032794361e-06, "Finetune/Loss": 0.828170657157898, "Finetune/Loss (Raw)": 1.0144646167755127, "Finetune/Step": 4165, "Finetune/Step Time": 5.506686573848128} +{"Finetune/Learning Rate": 1.996369730873353e-06, "Finetune/Loss": 0.8327678442001343, "Finetune/Loss (Raw)": 0.9429197907447815, "Finetune/Step": 4166, "Finetune/Step Time": 5.481663664802909} +{"Finetune/Learning Rate": 1.99263653500499e-06, "Finetune/Loss": 0.831650972366333, "Finetune/Loss (Raw)": 0.791436493396759, "Finetune/Step": 4167, "Finetune/Step Time": 5.466461345553398} +{"Finetune/Learning Rate": 1.9889064466380593e-06, "Finetune/Loss": 0.8299124240875244, "Finetune/Loss (Raw)": 0.7070685625076294, "Finetune/Step": 4168, "Finetune/Step Time": 5.464759945869446} +{"Finetune/Learning Rate": 1.985179467220133e-06, "Finetune/Loss": 0.8264812231063843, "Finetune/Loss (Raw)": 0.37359970808029175, "Finetune/Step": 4169, "Finetune/Step Time": 5.495649144053459} +{"Finetune/Learning Rate": 1.9814555981975803e-06, "Finetune/Loss": 0.8257567882537842, "Finetune/Loss (Raw)": 0.7763857245445251, "Finetune/Step": 4170, "Finetune/Step Time": 5.481591215357184} +{"Finetune/Learning Rate": 1.9777348410155606e-06, "Finetune/Loss": 0.8257355690002441, "Finetune/Loss (Raw)": 0.9100938439369202, "Finetune/Step": 4171, "Finetune/Step Time": 5.478340795263648} +{"Finetune/Learning Rate": 1.9740171971180278e-06, "Finetune/Loss": 0.8258697390556335, "Finetune/Loss (Raw)": 0.9045929908752441, "Finetune/Step": 4172, "Finetune/Step Time": 5.4930080864578485} +{"Finetune/Learning Rate": 1.9703026679477253e-06, "Finetune/Loss": 0.8300184011459351, "Finetune/Loss (Raw)": 0.8221120834350586, "Finetune/Step": 4173, "Finetune/Step Time": 5.473040832206607} +{"Finetune/Learning Rate": 1.9665912549461898e-06, "Finetune/Loss": 0.8264087438583374, "Finetune/Loss (Raw)": 0.3576965034008026, "Finetune/Step": 4174, "Finetune/Step Time": 5.504848554730415} +{"Finetune/Learning Rate": 1.962882959553751e-06, "Finetune/Loss": 0.8263179063796997, "Finetune/Loss (Raw)": 0.7866759896278381, "Finetune/Step": 4175, "Finetune/Step Time": 5.499111622571945} +{"Finetune/Learning Rate": 1.959177783209527e-06, "Finetune/Loss": 0.8296992182731628, "Finetune/Loss (Raw)": 0.8064459562301636, "Finetune/Step": 4176, "Finetune/Step Time": 5.481176884844899} +{"Finetune/Learning Rate": 1.955475727351417e-06, "Finetune/Loss": 0.8313497304916382, "Finetune/Loss (Raw)": 0.9707987904548645, "Finetune/Step": 4177, "Finetune/Step Time": 5.492047032341361} +{"Finetune/Learning Rate": 1.9517767934161224e-06, "Finetune/Loss": 0.8320040702819824, "Finetune/Loss (Raw)": 0.9194647669792175, "Finetune/Step": 4178, "Finetune/Step Time": 5.4946622140705585} +{"Finetune/Learning Rate": 1.9480809828391266e-06, "Finetune/Loss": 0.8327062129974365, "Finetune/Loss (Raw)": 0.884224534034729, "Finetune/Step": 4179, "Finetune/Step Time": 5.507227493450046} +{"Finetune/Learning Rate": 1.9443882970547003e-06, "Finetune/Loss": 0.8336469531059265, "Finetune/Loss (Raw)": 0.9717243909835815, "Finetune/Step": 4180, "Finetune/Step Time": 5.487195642665029} +{"Finetune/Learning Rate": 1.940698737495904e-06, "Finetune/Loss": 0.8371452689170837, "Finetune/Loss (Raw)": 0.8222380876541138, "Finetune/Step": 4181, "Finetune/Step Time": 5.465600460767746} +{"Finetune/Learning Rate": 1.937012305594583e-06, "Finetune/Loss": 0.8357446193695068, "Finetune/Loss (Raw)": 0.6913005709648132, "Finetune/Step": 4182, "Finetune/Step Time": 5.466098431497812} +{"Finetune/Learning Rate": 1.93332900278137e-06, "Finetune/Loss": 0.8360621929168701, "Finetune/Loss (Raw)": 0.947371780872345, "Finetune/Step": 4183, "Finetune/Step Time": 5.460063995793462} +{"Finetune/Learning Rate": 1.9296488304856785e-06, "Finetune/Loss": 0.8381703495979309, "Finetune/Loss (Raw)": 0.9061552882194519, "Finetune/Step": 4184, "Finetune/Step Time": 5.432697826996446} +{"Finetune/Learning Rate": 1.92597179013572e-06, "Finetune/Loss": 0.8381186723709106, "Finetune/Loss (Raw)": 0.9692687392234802, "Finetune/Step": 4185, "Finetune/Step Time": 5.435728657990694} +{"Finetune/Learning Rate": 1.9222978831584793e-06, "Finetune/Loss": 0.83899986743927, "Finetune/Loss (Raw)": 0.9422902464866638, "Finetune/Step": 4186, "Finetune/Step Time": 5.44890333712101} +{"Finetune/Learning Rate": 1.9186271109797215e-06, "Finetune/Loss": 0.8383604288101196, "Finetune/Loss (Raw)": 0.805408775806427, "Finetune/Step": 4187, "Finetune/Step Time": 5.437037993222475} +{"Finetune/Learning Rate": 1.9149594750240087e-06, "Finetune/Loss": 0.8394793272018433, "Finetune/Loss (Raw)": 0.8019958734512329, "Finetune/Step": 4188, "Finetune/Step Time": 5.432857153937221} +{"Finetune/Learning Rate": 1.911294976714676e-06, "Finetune/Loss": 0.8394820690155029, "Finetune/Loss (Raw)": 0.8068947196006775, "Finetune/Step": 4189, "Finetune/Step Time": 5.420476034283638} +{"Finetune/Learning Rate": 1.9076336174738476e-06, "Finetune/Loss": 0.8369261026382446, "Finetune/Loss (Raw)": 0.6863462328910828, "Finetune/Step": 4190, "Finetune/Step Time": 5.42051918618381} +{"Finetune/Learning Rate": 1.9039753987224164e-06, "Finetune/Loss": 0.8413999676704407, "Finetune/Loss (Raw)": 0.9251932501792908, "Finetune/Step": 4191, "Finetune/Step Time": 5.393683427944779} +{"Finetune/Learning Rate": 1.9003203218800748e-06, "Finetune/Loss": 0.8463778495788574, "Finetune/Loss (Raw)": 1.01911199092865, "Finetune/Step": 4192, "Finetune/Step Time": 5.360650077462196} +{"Finetune/Learning Rate": 1.8966683883652826e-06, "Finetune/Loss": 0.8426246047019958, "Finetune/Loss (Raw)": 0.3043690025806427, "Finetune/Step": 4193, "Finetune/Step Time": 5.381891472265124} +{"Finetune/Learning Rate": 1.8930195995952827e-06, "Finetune/Loss": 0.8428584933280945, "Finetune/Loss (Raw)": 0.9009806513786316, "Finetune/Step": 4194, "Finetune/Step Time": 5.381298905238509} +{"Finetune/Learning Rate": 1.8893739569861058e-06, "Finetune/Loss": 0.8422471284866333, "Finetune/Loss (Raw)": 0.9108330607414246, "Finetune/Step": 4195, "Finetune/Step Time": 5.381854463368654} +{"Finetune/Learning Rate": 1.8857314619525469e-06, "Finetune/Loss": 0.8420209288597107, "Finetune/Loss (Raw)": 0.8974757194519043, "Finetune/Step": 4196, "Finetune/Step Time": 5.379651714116335} +{"Finetune/Learning Rate": 1.8820921159081896e-06, "Finetune/Loss": 0.8393769264221191, "Finetune/Loss (Raw)": 0.5859149098396301, "Finetune/Step": 4197, "Finetune/Step Time": 5.388965616002679} +{"Finetune/Learning Rate": 1.8784559202653907e-06, "Finetune/Loss": 0.8404839634895325, "Finetune/Loss (Raw)": 0.9684195518493652, "Finetune/Step": 4198, "Finetune/Step Time": 5.386606335639954} +{"Finetune/Learning Rate": 1.8748228764352915e-06, "Finetune/Loss": 0.8413427472114563, "Finetune/Loss (Raw)": 0.9863940477371216, "Finetune/Step": 4199, "Finetune/Step Time": 5.356039704754949} +{"Finetune/Learning Rate": 1.8711929858278066e-06, "Finetune/Loss": 0.8423718214035034, "Finetune/Loss (Raw)": 0.9041537642478943, "Finetune/Step": 4200, "Finetune/Step Time": 5.352988621219993} +{"Finetune/Learning Rate": 1.8675662498516167e-06, "Finetune/Loss": 0.8419550657272339, "Finetune/Loss (Raw)": 0.8511542081832886, "Finetune/Step": 4201, "Finetune/Step Time": 5.36299742013216} +{"Finetune/Learning Rate": 1.863942669914196e-06, "Finetune/Loss": 0.84067702293396, "Finetune/Loss (Raw)": 0.8229830861091614, "Finetune/Step": 4202, "Finetune/Step Time": 5.365914937108755} +{"Finetune/Learning Rate": 1.8603222474217819e-06, "Finetune/Loss": 0.8403055667877197, "Finetune/Loss (Raw)": 0.7798361778259277, "Finetune/Step": 4203, "Finetune/Step Time": 5.3673485070466995} +{"Finetune/Learning Rate": 1.8567049837793894e-06, "Finetune/Loss": 0.8413783311843872, "Finetune/Loss (Raw)": 0.8056725263595581, "Finetune/Step": 4204, "Finetune/Step Time": 5.347406838089228} +{"Finetune/Learning Rate": 1.8530908803908088e-06, "Finetune/Loss": 0.8417214155197144, "Finetune/Loss (Raw)": 0.896906852722168, "Finetune/Step": 4205, "Finetune/Step Time": 5.328829290345311} +{"Finetune/Learning Rate": 1.8494799386586015e-06, "Finetune/Loss": 0.8418523073196411, "Finetune/Loss (Raw)": 0.8329475522041321, "Finetune/Step": 4206, "Finetune/Step Time": 5.32695391587913} +{"Finetune/Learning Rate": 1.8458721599841056e-06, "Finetune/Loss": 0.8408668637275696, "Finetune/Loss (Raw)": 0.9245029091835022, "Finetune/Step": 4207, "Finetune/Step Time": 5.3154632188379765} +{"Finetune/Learning Rate": 1.8422675457674254e-06, "Finetune/Loss": 0.8412696123123169, "Finetune/Loss (Raw)": 0.8304739594459534, "Finetune/Step": 4208, "Finetune/Step Time": 5.3054494727402925} +{"Finetune/Learning Rate": 1.8386660974074466e-06, "Finetune/Loss": 0.8428231477737427, "Finetune/Loss (Raw)": 1.0604649782180786, "Finetune/Step": 4209, "Finetune/Step Time": 5.283283015713096} +{"Finetune/Learning Rate": 1.8350678163018221e-06, "Finetune/Loss": 0.8430667519569397, "Finetune/Loss (Raw)": 0.8414692878723145, "Finetune/Step": 4210, "Finetune/Step Time": 5.278302811086178} +{"Finetune/Learning Rate": 1.831472703846966e-06, "Finetune/Loss": 0.8426750302314758, "Finetune/Loss (Raw)": 0.9356393218040466, "Finetune/Step": 4211, "Finetune/Step Time": 5.28327215090394} +{"Finetune/Learning Rate": 1.827880761438079e-06, "Finetune/Loss": 0.8451375961303711, "Finetune/Loss (Raw)": 0.7809789180755615, "Finetune/Step": 4212, "Finetune/Step Time": 5.2569744642823935} +{"Finetune/Learning Rate": 1.8242919904691214e-06, "Finetune/Loss": 0.8440904021263123, "Finetune/Loss (Raw)": 0.7149882316589355, "Finetune/Step": 4213, "Finetune/Step Time": 5.250251246616244} +{"Finetune/Learning Rate": 1.820706392332824e-06, "Finetune/Loss": 0.8440915942192078, "Finetune/Loss (Raw)": 0.8940473794937134, "Finetune/Step": 4214, "Finetune/Step Time": 5.240539250895381} +{"Finetune/Learning Rate": 1.8171239684206898e-06, "Finetune/Loss": 0.8431660532951355, "Finetune/Loss (Raw)": 0.8801865577697754, "Finetune/Step": 4215, "Finetune/Step Time": 5.271404402330518} +{"Finetune/Learning Rate": 1.8135447201229861e-06, "Finetune/Loss": 0.8425560593605042, "Finetune/Loss (Raw)": 0.8733723163604736, "Finetune/Step": 4216, "Finetune/Step Time": 5.266772974282503} +{"Finetune/Learning Rate": 1.8099686488287483e-06, "Finetune/Loss": 0.8433505296707153, "Finetune/Loss (Raw)": 0.9440871477127075, "Finetune/Step": 4217, "Finetune/Step Time": 5.264266530051827} +{"Finetune/Learning Rate": 1.8063957559257794e-06, "Finetune/Loss": 0.8434963226318359, "Finetune/Loss (Raw)": 0.8539666533470154, "Finetune/Step": 4218, "Finetune/Step Time": 5.263499787077308} +{"Finetune/Learning Rate": 1.8028260428006538e-06, "Finetune/Loss": 0.8428629040718079, "Finetune/Loss (Raw)": 0.7560300230979919, "Finetune/Step": 4219, "Finetune/Step Time": 5.2466559167951345} +{"Finetune/Learning Rate": 1.7992595108387068e-06, "Finetune/Loss": 0.8431674242019653, "Finetune/Loss (Raw)": 0.8787379860877991, "Finetune/Step": 4220, "Finetune/Step Time": 5.235880525782704} +{"Finetune/Learning Rate": 1.795696161424032e-06, "Finetune/Loss": 0.8432794213294983, "Finetune/Loss (Raw)": 1.0033948421478271, "Finetune/Step": 4221, "Finetune/Step Time": 5.2378584537655115} +{"Finetune/Learning Rate": 1.7921359959395046e-06, "Finetune/Loss": 0.8424886465072632, "Finetune/Loss (Raw)": 0.9656228423118591, "Finetune/Step": 4222, "Finetune/Step Time": 5.241345690563321} +{"Finetune/Learning Rate": 1.7885790157667526e-06, "Finetune/Loss": 0.8440262079238892, "Finetune/Loss (Raw)": 0.9710847735404968, "Finetune/Step": 4223, "Finetune/Step Time": 5.233007811009884} +{"Finetune/Learning Rate": 1.7850252222861697e-06, "Finetune/Loss": 0.8423707485198975, "Finetune/Loss (Raw)": 0.8116288185119629, "Finetune/Step": 4224, "Finetune/Step Time": 5.238112745806575} +{"Finetune/Learning Rate": 1.7814746168769138e-06, "Finetune/Loss": 0.8410552740097046, "Finetune/Loss (Raw)": 0.7973251342773438, "Finetune/Step": 4225, "Finetune/Step Time": 5.230548558756709} +{"Finetune/Learning Rate": 1.7779272009169068e-06, "Finetune/Loss": 0.8433417081832886, "Finetune/Loss (Raw)": 0.9857804775238037, "Finetune/Step": 4226, "Finetune/Step Time": 5.224248519167304} +{"Finetune/Learning Rate": 1.77438297578283e-06, "Finetune/Loss": 0.8444872498512268, "Finetune/Loss (Raw)": 1.067769169807434, "Finetune/Step": 4227, "Finetune/Step Time": 5.223410896956921} +{"Finetune/Learning Rate": 1.7708419428501256e-06, "Finetune/Loss": 0.8436581492424011, "Finetune/Loss (Raw)": 0.8519985675811768, "Finetune/Step": 4228, "Finetune/Step Time": 5.22567705437541} +{"Finetune/Learning Rate": 1.7673041034930083e-06, "Finetune/Loss": 0.8438109755516052, "Finetune/Loss (Raw)": 0.9108123183250427, "Finetune/Step": 4229, "Finetune/Step Time": 5.208609437569976} +{"Finetune/Learning Rate": 1.7637694590844357e-06, "Finetune/Loss": 0.8392491340637207, "Finetune/Loss (Raw)": 0.2677782475948334, "Finetune/Step": 4230, "Finetune/Step Time": 5.228493029251695} +{"Finetune/Learning Rate": 1.7602380109961348e-06, "Finetune/Loss": 0.8374342322349548, "Finetune/Loss (Raw)": 0.6600630283355713, "Finetune/Step": 4231, "Finetune/Step Time": 5.278073288500309} +{"Finetune/Learning Rate": 1.7567097605985972e-06, "Finetune/Loss": 0.8376969695091248, "Finetune/Loss (Raw)": 0.9026758074760437, "Finetune/Step": 4232, "Finetune/Step Time": 5.286784121766686} +{"Finetune/Learning Rate": 1.7531847092610644e-06, "Finetune/Loss": 0.8378351926803589, "Finetune/Loss (Raw)": 0.8004493713378906, "Finetune/Step": 4233, "Finetune/Step Time": 5.292429009452462} +{"Finetune/Learning Rate": 1.7496628583515395e-06, "Finetune/Loss": 0.8399724960327148, "Finetune/Loss (Raw)": 1.1046732664108276, "Finetune/Step": 4234, "Finetune/Step Time": 5.295274578034878} +{"Finetune/Learning Rate": 1.7461442092367864e-06, "Finetune/Loss": 0.8403640389442444, "Finetune/Loss (Raw)": 0.8983370661735535, "Finetune/Step": 4235, "Finetune/Step Time": 5.294974014163017} +{"Finetune/Learning Rate": 1.7426287632823213e-06, "Finetune/Loss": 0.8386850357055664, "Finetune/Loss (Raw)": 0.7320080399513245, "Finetune/Step": 4236, "Finetune/Step Time": 5.298607863485813} +{"Finetune/Learning Rate": 1.7391165218524232e-06, "Finetune/Loss": 0.8401206731796265, "Finetune/Loss (Raw)": 1.005786418914795, "Finetune/Step": 4237, "Finetune/Step Time": 5.290809823200107} +{"Finetune/Learning Rate": 1.735607486310119e-06, "Finetune/Loss": 0.8405070304870605, "Finetune/Loss (Raw)": 0.9592288136482239, "Finetune/Step": 4238, "Finetune/Step Time": 5.279120491817594} +{"Finetune/Learning Rate": 1.7321016580172068e-06, "Finetune/Loss": 0.8398276567459106, "Finetune/Loss (Raw)": 0.9093011021614075, "Finetune/Step": 4239, "Finetune/Step Time": 5.27525763772428} +{"Finetune/Learning Rate": 1.728599038334221e-06, "Finetune/Loss": 0.8403830528259277, "Finetune/Loss (Raw)": 0.8354793787002563, "Finetune/Step": 4240, "Finetune/Step Time": 5.263427207246423} +{"Finetune/Learning Rate": 1.7250996286204602e-06, "Finetune/Loss": 0.841456413269043, "Finetune/Loss (Raw)": 0.8532330989837646, "Finetune/Step": 4241, "Finetune/Step Time": 5.265072302892804} +{"Finetune/Learning Rate": 1.7216034302339824e-06, "Finetune/Loss": 0.8419733047485352, "Finetune/Loss (Raw)": 0.9418809413909912, "Finetune/Step": 4242, "Finetune/Step Time": 5.26388144493103} +{"Finetune/Learning Rate": 1.718110444531591e-06, "Finetune/Loss": 0.8420863151550293, "Finetune/Loss (Raw)": 0.8866370916366577, "Finetune/Step": 4243, "Finetune/Step Time": 5.283947730436921} +{"Finetune/Learning Rate": 1.7146206728688465e-06, "Finetune/Loss": 0.8440950512886047, "Finetune/Loss (Raw)": 0.8947522640228271, "Finetune/Step": 4244, "Finetune/Step Time": 5.2622919995337725} +{"Finetune/Learning Rate": 1.7111341166000594e-06, "Finetune/Loss": 0.8449525237083435, "Finetune/Loss (Raw)": 0.8846275806427002, "Finetune/Step": 4245, "Finetune/Step Time": 5.264169793576002} +{"Finetune/Learning Rate": 1.7076507770782957e-06, "Finetune/Loss": 0.8434903621673584, "Finetune/Loss (Raw)": 0.8595278263092041, "Finetune/Step": 4246, "Finetune/Step Time": 5.300912532955408} +{"Finetune/Learning Rate": 1.7041706556553717e-06, "Finetune/Loss": 0.8425574898719788, "Finetune/Loss (Raw)": 0.8672966361045837, "Finetune/Step": 4247, "Finetune/Step Time": 5.307862846180797} +{"Finetune/Learning Rate": 1.7006937536818503e-06, "Finetune/Loss": 0.837855339050293, "Finetune/Loss (Raw)": 0.28916171193122864, "Finetune/Step": 4248, "Finetune/Step Time": 5.325882505625486} +{"Finetune/Learning Rate": 1.6972200725070587e-06, "Finetune/Loss": 0.8382396101951599, "Finetune/Loss (Raw)": 0.9332296848297119, "Finetune/Step": 4249, "Finetune/Step Time": 5.324003709480166} +{"Finetune/Learning Rate": 1.6937496134790554e-06, "Finetune/Loss": 0.8371080756187439, "Finetune/Loss (Raw)": 0.7280635237693787, "Finetune/Step": 4250, "Finetune/Step Time": 5.320279594510794} +{"Finetune/Learning Rate": 1.6902823779446586e-06, "Finetune/Loss": 0.8350287675857544, "Finetune/Loss (Raw)": 0.7472688555717468, "Finetune/Step": 4251, "Finetune/Step Time": 5.309211483225226} +{"Finetune/Learning Rate": 1.686818367249441e-06, "Finetune/Loss": 0.8381911516189575, "Finetune/Loss (Raw)": 1.0547384023666382, "Finetune/Step": 4252, "Finetune/Step Time": 5.311044046655297} +{"Finetune/Learning Rate": 1.6833575827377135e-06, "Finetune/Loss": 0.8362255096435547, "Finetune/Loss (Raw)": 0.6543761491775513, "Finetune/Step": 4253, "Finetune/Step Time": 5.337546182796359} +{"Finetune/Learning Rate": 1.6799000257525388e-06, "Finetune/Loss": 0.834621250629425, "Finetune/Loss (Raw)": 0.677180290222168, "Finetune/Step": 4254, "Finetune/Step Time": 5.34809734672308} +{"Finetune/Learning Rate": 1.6764456976357279e-06, "Finetune/Loss": 0.8336189985275269, "Finetune/Loss (Raw)": 0.7781183123588562, "Finetune/Step": 4255, "Finetune/Step Time": 5.3598300367593765} +{"Finetune/Learning Rate": 1.6729945997278386e-06, "Finetune/Loss": 0.8343378305435181, "Finetune/Loss (Raw)": 0.9198922514915466, "Finetune/Step": 4256, "Finetune/Step Time": 5.375460419803858} +{"Finetune/Learning Rate": 1.6695467333681748e-06, "Finetune/Loss": 0.8355531692504883, "Finetune/Loss (Raw)": 0.9749858379364014, "Finetune/Step": 4257, "Finetune/Step Time": 5.379573166370392} +{"Finetune/Learning Rate": 1.6661020998947818e-06, "Finetune/Loss": 0.8343386650085449, "Finetune/Loss (Raw)": 0.6828246116638184, "Finetune/Step": 4258, "Finetune/Step Time": 5.380958477035165} +{"Finetune/Learning Rate": 1.6626607006444628e-06, "Finetune/Loss": 0.8353317975997925, "Finetune/Loss (Raw)": 1.0128226280212402, "Finetune/Step": 4259, "Finetune/Step Time": 5.384692857041955} +{"Finetune/Learning Rate": 1.6592225369527515e-06, "Finetune/Loss": 0.8349111080169678, "Finetune/Loss (Raw)": 0.8464891910552979, "Finetune/Step": 4260, "Finetune/Step Time": 5.381118161603808} +{"Finetune/Learning Rate": 1.6557876101539305e-06, "Finetune/Loss": 0.8354952335357666, "Finetune/Loss (Raw)": 0.9281771183013916, "Finetune/Step": 4261, "Finetune/Step Time": 5.375103168189526} +{"Finetune/Learning Rate": 1.6523559215810336e-06, "Finetune/Loss": 0.8352903127670288, "Finetune/Loss (Raw)": 0.8410344123840332, "Finetune/Step": 4262, "Finetune/Step Time": 5.3654361087828875} +{"Finetune/Learning Rate": 1.64892747256583e-06, "Finetune/Loss": 0.8372148275375366, "Finetune/Loss (Raw)": 1.0351710319519043, "Finetune/Step": 4263, "Finetune/Step Time": 5.363496042788029} +{"Finetune/Learning Rate": 1.645502264438833e-06, "Finetune/Loss": 0.8367310762405396, "Finetune/Loss (Raw)": 0.785982072353363, "Finetune/Step": 4264, "Finetune/Step Time": 5.3746902365237474} +{"Finetune/Learning Rate": 1.6420802985292995e-06, "Finetune/Loss": 0.8377429842948914, "Finetune/Loss (Raw)": 0.9232206344604492, "Finetune/Step": 4265, "Finetune/Step Time": 5.382188709452748} +{"Finetune/Learning Rate": 1.638661576165227e-06, "Finetune/Loss": 0.8377246260643005, "Finetune/Loss (Raw)": 0.9519277215003967, "Finetune/Step": 4266, "Finetune/Step Time": 5.381610386073589} +{"Finetune/Learning Rate": 1.6352460986733564e-06, "Finetune/Loss": 0.8377012014389038, "Finetune/Loss (Raw)": 0.7846628427505493, "Finetune/Step": 4267, "Finetune/Step Time": 5.39553864300251} +{"Finetune/Learning Rate": 1.6318338673791655e-06, "Finetune/Loss": 0.8375971913337708, "Finetune/Loss (Raw)": 0.8017618656158447, "Finetune/Step": 4268, "Finetune/Step Time": 5.3871588204056025} +{"Finetune/Learning Rate": 1.6284248836068772e-06, "Finetune/Loss": 0.8403059244155884, "Finetune/Loss (Raw)": 0.7312422394752502, "Finetune/Step": 4269, "Finetune/Step Time": 5.391692910343409} +{"Finetune/Learning Rate": 1.6250191486794497e-06, "Finetune/Loss": 0.8407964110374451, "Finetune/Loss (Raw)": 0.8404607772827148, "Finetune/Step": 4270, "Finetune/Step Time": 5.391651313751936} +{"Finetune/Learning Rate": 1.6216166639185804e-06, "Finetune/Loss": 0.8371591567993164, "Finetune/Loss (Raw)": 0.32463011145591736, "Finetune/Step": 4271, "Finetune/Step Time": 5.424191435799003} +{"Finetune/Learning Rate": 1.618217430644713e-06, "Finetune/Loss": 0.8420395851135254, "Finetune/Loss (Raw)": 0.9149923324584961, "Finetune/Step": 4272, "Finetune/Step Time": 5.394932057708502} +{"Finetune/Learning Rate": 1.6148214501770221e-06, "Finetune/Loss": 0.8421105146408081, "Finetune/Loss (Raw)": 0.7460253834724426, "Finetune/Step": 4273, "Finetune/Step Time": 5.388560948893428} +{"Finetune/Learning Rate": 1.6114287238334148e-06, "Finetune/Loss": 0.841699481010437, "Finetune/Loss (Raw)": 0.8627577424049377, "Finetune/Step": 4274, "Finetune/Step Time": 5.389230681583285} +{"Finetune/Learning Rate": 1.6080392529305489e-06, "Finetune/Loss": 0.8414832949638367, "Finetune/Loss (Raw)": 0.8244615793228149, "Finetune/Step": 4275, "Finetune/Step Time": 5.4004935920238495} +{"Finetune/Learning Rate": 1.60465303878381e-06, "Finetune/Loss": 0.8416385650634766, "Finetune/Loss (Raw)": 0.9195011258125305, "Finetune/Step": 4276, "Finetune/Step Time": 5.3957637175917625} +{"Finetune/Learning Rate": 1.6012700827073213e-06, "Finetune/Loss": 0.841642439365387, "Finetune/Loss (Raw)": 0.8463029861450195, "Finetune/Step": 4277, "Finetune/Step Time": 5.418053226545453} +{"Finetune/Learning Rate": 1.5978903860139416e-06, "Finetune/Loss": 0.8427473306655884, "Finetune/Loss (Raw)": 0.9256113171577454, "Finetune/Step": 4278, "Finetune/Step Time": 5.397164691239595} +{"Finetune/Learning Rate": 1.5945139500152662e-06, "Finetune/Loss": 0.8423224687576294, "Finetune/Loss (Raw)": 0.8044220805168152, "Finetune/Step": 4279, "Finetune/Step Time": 5.39554232917726} +{"Finetune/Learning Rate": 1.5911407760216235e-06, "Finetune/Loss": 0.8429973125457764, "Finetune/Loss (Raw)": 0.9516909122467041, "Finetune/Step": 4280, "Finetune/Step Time": 5.388985924422741} +{"Finetune/Learning Rate": 1.5877708653420742e-06, "Finetune/Loss": 0.84431391954422, "Finetune/Loss (Raw)": 1.00438392162323, "Finetune/Step": 4281, "Finetune/Step Time": 5.3944522980600595} +{"Finetune/Learning Rate": 1.58440421928442e-06, "Finetune/Loss": 0.8447794914245605, "Finetune/Loss (Raw)": 0.9502756595611572, "Finetune/Step": 4282, "Finetune/Step Time": 5.392496660351753} +{"Finetune/Learning Rate": 1.581040839155189e-06, "Finetune/Loss": 0.8452515602111816, "Finetune/Loss (Raw)": 0.880589485168457, "Finetune/Step": 4283, "Finetune/Step Time": 5.397474050521851} +{"Finetune/Learning Rate": 1.5776807262596371e-06, "Finetune/Loss": 0.8455373644828796, "Finetune/Loss (Raw)": 0.9147452116012573, "Finetune/Step": 4284, "Finetune/Step Time": 5.395775122568011} +{"Finetune/Learning Rate": 1.574323881901767e-06, "Finetune/Loss": 0.8431327939033508, "Finetune/Loss (Raw)": 0.6752120852470398, "Finetune/Step": 4285, "Finetune/Step Time": 5.430497959256172} +{"Finetune/Learning Rate": 1.5709703073843008e-06, "Finetune/Loss": 0.8437219858169556, "Finetune/Loss (Raw)": 0.9091089367866516, "Finetune/Step": 4286, "Finetune/Step Time": 5.421546645462513} +{"Finetune/Learning Rate": 1.5676200040086941e-06, "Finetune/Loss": 0.8435457944869995, "Finetune/Loss (Raw)": 0.9131650924682617, "Finetune/Step": 4287, "Finetune/Step Time": 5.424742698669434} +{"Finetune/Learning Rate": 1.564272973075136e-06, "Finetune/Loss": 0.8437288999557495, "Finetune/Loss (Raw)": 1.005740761756897, "Finetune/Step": 4288, "Finetune/Step Time": 5.4398057125508785} +{"Finetune/Learning Rate": 1.560929215882544e-06, "Finetune/Loss": 0.842958927154541, "Finetune/Loss (Raw)": 0.8160793781280518, "Finetune/Step": 4289, "Finetune/Step Time": 5.441097034141421} +{"Finetune/Learning Rate": 1.5575887337285644e-06, "Finetune/Loss": 0.8431211709976196, "Finetune/Loss (Raw)": 0.974364161491394, "Finetune/Step": 4290, "Finetune/Step Time": 5.441504195332527} +{"Finetune/Learning Rate": 1.5542515279095704e-06, "Finetune/Loss": 0.844746470451355, "Finetune/Loss (Raw)": 0.9275678992271423, "Finetune/Step": 4291, "Finetune/Step Time": 5.426161019131541} +{"Finetune/Learning Rate": 1.5509175997206705e-06, "Finetune/Loss": 0.8472950458526611, "Finetune/Loss (Raw)": 0.929918110370636, "Finetune/Step": 4292, "Finetune/Step Time": 5.421673284843564} +{"Finetune/Learning Rate": 1.5475869504556996e-06, "Finetune/Loss": 0.8465150594711304, "Finetune/Loss (Raw)": 0.9146403074264526, "Finetune/Step": 4293, "Finetune/Step Time": 5.429523531347513} +{"Finetune/Learning Rate": 1.5442595814072081e-06, "Finetune/Loss": 0.8459954261779785, "Finetune/Loss (Raw)": 0.8763967752456665, "Finetune/Step": 4294, "Finetune/Step Time": 5.445288699120283} +{"Finetune/Learning Rate": 1.540935493866491e-06, "Finetune/Loss": 0.8421692252159119, "Finetune/Loss (Raw)": 0.3016863763332367, "Finetune/Step": 4295, "Finetune/Step Time": 5.473202537745237} +{"Finetune/Learning Rate": 1.53761468912356e-06, "Finetune/Loss": 0.8443574905395508, "Finetune/Loss (Raw)": 0.9871630668640137, "Finetune/Step": 4296, "Finetune/Step Time": 5.470185009762645} +{"Finetune/Learning Rate": 1.5342971684671537e-06, "Finetune/Loss": 0.848490297794342, "Finetune/Loss (Raw)": 0.9026027321815491, "Finetune/Step": 4297, "Finetune/Step Time": 5.441850379109383} +{"Finetune/Learning Rate": 1.530982933184737e-06, "Finetune/Loss": 0.8479835391044617, "Finetune/Loss (Raw)": 0.7115200757980347, "Finetune/Step": 4298, "Finetune/Step Time": 5.436842696741223} +{"Finetune/Learning Rate": 1.527671984562501e-06, "Finetune/Loss": 0.8481809496879578, "Finetune/Loss (Raw)": 0.9353591203689575, "Finetune/Step": 4299, "Finetune/Step Time": 5.444019285961986} +{"Finetune/Learning Rate": 1.524364323885359e-06, "Finetune/Loss": 0.8485077619552612, "Finetune/Loss (Raw)": 0.946428120136261, "Finetune/Step": 4300, "Finetune/Step Time": 5.434231661260128} +{"Finetune/Learning Rate": 1.5210599524369484e-06, "Finetune/Loss": 0.84492427110672, "Finetune/Loss (Raw)": 0.36342552304267883, "Finetune/Step": 4301, "Finetune/Step Time": 5.454185929149389} +{"Finetune/Learning Rate": 1.517758871499635e-06, "Finetune/Loss": 0.8497161865234375, "Finetune/Loss (Raw)": 0.9710593223571777, "Finetune/Step": 4302, "Finetune/Step Time": 5.427617086097598} +{"Finetune/Learning Rate": 1.5144610823545035e-06, "Finetune/Loss": 0.8513486981391907, "Finetune/Loss (Raw)": 0.9956393241882324, "Finetune/Step": 4303, "Finetune/Step Time": 5.422388190403581} +{"Finetune/Learning Rate": 1.511166586281355e-06, "Finetune/Loss": 0.8477252125740051, "Finetune/Loss (Raw)": 0.3426378071308136, "Finetune/Step": 4304, "Finetune/Step Time": 5.440472336485982} +{"Finetune/Learning Rate": 1.507875384558727e-06, "Finetune/Loss": 0.8462059497833252, "Finetune/Loss (Raw)": 0.7763407230377197, "Finetune/Step": 4305, "Finetune/Step Time": 5.431520160287619} +{"Finetune/Learning Rate": 1.504587478463867e-06, "Finetune/Loss": 0.8455387353897095, "Finetune/Loss (Raw)": 0.834052562713623, "Finetune/Step": 4306, "Finetune/Step Time": 5.437455154955387} +{"Finetune/Learning Rate": 1.5013028692727483e-06, "Finetune/Loss": 0.8439247608184814, "Finetune/Loss (Raw)": 0.6776453852653503, "Finetune/Step": 4307, "Finetune/Step Time": 5.430932464078069} +{"Finetune/Learning Rate": 1.4980215582600621e-06, "Finetune/Loss": 0.8433371782302856, "Finetune/Loss (Raw)": 0.8965112566947937, "Finetune/Step": 4308, "Finetune/Step Time": 5.43181299418211} +{"Finetune/Learning Rate": 1.4947435466992222e-06, "Finetune/Loss": 0.8436547517776489, "Finetune/Loss (Raw)": 0.8628827333450317, "Finetune/Step": 4309, "Finetune/Step Time": 5.430742213502526} +{"Finetune/Learning Rate": 1.4914688358623608e-06, "Finetune/Loss": 0.8410006761550903, "Finetune/Loss (Raw)": 0.35158273577690125, "Finetune/Step": 4310, "Finetune/Step Time": 5.457060003653169} +{"Finetune/Learning Rate": 1.488197427020327e-06, "Finetune/Loss": 0.8404563665390015, "Finetune/Loss (Raw)": 0.8776897192001343, "Finetune/Step": 4311, "Finetune/Step Time": 5.462502930313349} +{"Finetune/Learning Rate": 1.4849293214426975e-06, "Finetune/Loss": 0.8410512208938599, "Finetune/Loss (Raw)": 0.9823077321052551, "Finetune/Step": 4312, "Finetune/Step Time": 5.4702980648726225} +{"Finetune/Learning Rate": 1.481664520397753e-06, "Finetune/Loss": 0.8411689400672913, "Finetune/Loss (Raw)": 0.9843297600746155, "Finetune/Step": 4313, "Finetune/Step Time": 5.481050167232752} +{"Finetune/Learning Rate": 1.4784030251524994e-06, "Finetune/Loss": 0.8410977721214294, "Finetune/Loss (Raw)": 0.9331862330436707, "Finetune/Step": 4314, "Finetune/Step Time": 5.471218017861247} +{"Finetune/Learning Rate": 1.4751448369726652e-06, "Finetune/Loss": 0.8414240479469299, "Finetune/Loss (Raw)": 0.8471649885177612, "Finetune/Step": 4315, "Finetune/Step Time": 5.468076007440686} +{"Finetune/Learning Rate": 1.4718899571226842e-06, "Finetune/Loss": 0.8415724039077759, "Finetune/Loss (Raw)": 0.8209913372993469, "Finetune/Step": 4316, "Finetune/Step Time": 5.472234163433313} +{"Finetune/Learning Rate": 1.468638386865716e-06, "Finetune/Loss": 0.8420475721359253, "Finetune/Loss (Raw)": 0.8677172064781189, "Finetune/Step": 4317, "Finetune/Step Time": 5.465046675875783} +{"Finetune/Learning Rate": 1.465390127463625e-06, "Finetune/Loss": 0.842904806137085, "Finetune/Loss (Raw)": 0.7960655093193054, "Finetune/Step": 4318, "Finetune/Step Time": 5.479207295924425} +{"Finetune/Learning Rate": 1.4621451801770038e-06, "Finetune/Loss": 0.8414831757545471, "Finetune/Loss (Raw)": 0.7432260513305664, "Finetune/Step": 4319, "Finetune/Step Time": 5.485051652416587} +{"Finetune/Learning Rate": 1.4589035462651502e-06, "Finetune/Loss": 0.8400188684463501, "Finetune/Loss (Raw)": 0.831685483455658, "Finetune/Step": 4320, "Finetune/Step Time": 5.4885160233825445} +{"Finetune/Learning Rate": 1.4556652269860772e-06, "Finetune/Loss": 0.843187689781189, "Finetune/Loss (Raw)": 0.7099794745445251, "Finetune/Step": 4321, "Finetune/Step Time": 5.471118573099375} +{"Finetune/Learning Rate": 1.4524302235965203e-06, "Finetune/Loss": 0.8440379500389099, "Finetune/Loss (Raw)": 1.009812831878662, "Finetune/Step": 4322, "Finetune/Step Time": 5.4743435103446245} +{"Finetune/Learning Rate": 1.4491985373519146e-06, "Finetune/Loss": 0.8425008058547974, "Finetune/Loss (Raw)": 0.7140771150588989, "Finetune/Step": 4323, "Finetune/Step Time": 5.471759906038642} +{"Finetune/Learning Rate": 1.4459701695064155e-06, "Finetune/Loss": 0.8418764472007751, "Finetune/Loss (Raw)": 0.8175594806671143, "Finetune/Step": 4324, "Finetune/Step Time": 5.47095226123929} +{"Finetune/Learning Rate": 1.4427451213128873e-06, "Finetune/Loss": 0.8425223231315613, "Finetune/Loss (Raw)": 0.6685819625854492, "Finetune/Step": 4325, "Finetune/Step Time": 5.460597816854715} +{"Finetune/Learning Rate": 1.4395233940229147e-06, "Finetune/Loss": 0.8408154249191284, "Finetune/Loss (Raw)": 0.7499383687973022, "Finetune/Step": 4326, "Finetune/Step Time": 5.46896124817431} +{"Finetune/Learning Rate": 1.4363049888867853e-06, "Finetune/Loss": 0.8387038707733154, "Finetune/Loss (Raw)": 0.7161083221435547, "Finetune/Step": 4327, "Finetune/Step Time": 5.48599367775023} +{"Finetune/Learning Rate": 1.4330899071534942e-06, "Finetune/Loss": 0.8383781313896179, "Finetune/Loss (Raw)": 0.8624706268310547, "Finetune/Step": 4328, "Finetune/Step Time": 5.479317996650934} +{"Finetune/Learning Rate": 1.4298781500707558e-06, "Finetune/Loss": 0.8384116888046265, "Finetune/Loss (Raw)": 0.8554445505142212, "Finetune/Step": 4329, "Finetune/Step Time": 5.469092955812812} +{"Finetune/Learning Rate": 1.426669718884991e-06, "Finetune/Loss": 0.8394775390625, "Finetune/Loss (Raw)": 0.9594185948371887, "Finetune/Step": 4330, "Finetune/Step Time": 5.468805268406868} +{"Finetune/Learning Rate": 1.4234646148413277e-06, "Finetune/Loss": 0.8408300280570984, "Finetune/Loss (Raw)": 0.9529467225074768, "Finetune/Step": 4331, "Finetune/Step Time": 5.468254905194044} +{"Finetune/Learning Rate": 1.420262839183605e-06, "Finetune/Loss": 0.8415371179580688, "Finetune/Loss (Raw)": 0.8961834907531738, "Finetune/Step": 4332, "Finetune/Step Time": 5.471302006393671} +{"Finetune/Learning Rate": 1.4170643931543693e-06, "Finetune/Loss": 0.8420676589012146, "Finetune/Loss (Raw)": 0.9648138880729675, "Finetune/Step": 4333, "Finetune/Step Time": 5.472945231944323} +{"Finetune/Learning Rate": 1.4138692779948749e-06, "Finetune/Loss": 0.842924952507019, "Finetune/Loss (Raw)": 0.9426880478858948, "Finetune/Step": 4334, "Finetune/Step Time": 5.471492953598499} +{"Finetune/Learning Rate": 1.4106774949450808e-06, "Finetune/Loss": 0.842846691608429, "Finetune/Loss (Raw)": 0.9144814610481262, "Finetune/Step": 4335, "Finetune/Step Time": 5.4665188528597355} +{"Finetune/Learning Rate": 1.4074890452436607e-06, "Finetune/Loss": 0.8442103862762451, "Finetune/Loss (Raw)": 1.0050243139266968, "Finetune/Step": 4336, "Finetune/Step Time": 5.482828779146075} +{"Finetune/Learning Rate": 1.4043039301279904e-06, "Finetune/Loss": 0.841142475605011, "Finetune/Loss (Raw)": 0.6677772998809814, "Finetune/Step": 4337, "Finetune/Step Time": 5.505458064377308} +{"Finetune/Learning Rate": 1.4011221508341433e-06, "Finetune/Loss": 0.8413347601890564, "Finetune/Loss (Raw)": 0.8660829067230225, "Finetune/Step": 4338, "Finetune/Step Time": 5.505219928920269} +{"Finetune/Learning Rate": 1.3979437085969127e-06, "Finetune/Loss": 0.8402384519577026, "Finetune/Loss (Raw)": 0.7952996492385864, "Finetune/Step": 4339, "Finetune/Step Time": 5.505602015182376} +{"Finetune/Learning Rate": 1.3947686046497878e-06, "Finetune/Loss": 0.8409463167190552, "Finetune/Loss (Raw)": 0.8715884685516357, "Finetune/Step": 4340, "Finetune/Step Time": 5.51452524214983} +{"Finetune/Learning Rate": 1.3915968402249647e-06, "Finetune/Loss": 0.8429123163223267, "Finetune/Loss (Raw)": 0.9666401743888855, "Finetune/Step": 4341, "Finetune/Step Time": 5.506094392389059} +{"Finetune/Learning Rate": 1.3884284165533423e-06, "Finetune/Loss": 0.8424088954925537, "Finetune/Loss (Raw)": 0.8296163082122803, "Finetune/Step": 4342, "Finetune/Step Time": 5.5519731398671865} +{"Finetune/Learning Rate": 1.3852633348645262e-06, "Finetune/Loss": 0.8418387174606323, "Finetune/Loss (Raw)": 0.8071978688240051, "Finetune/Step": 4343, "Finetune/Step Time": 5.515825619921088} +{"Finetune/Learning Rate": 1.38210159638682e-06, "Finetune/Loss": 0.8420844078063965, "Finetune/Loss (Raw)": 0.9048258662223816, "Finetune/Step": 4344, "Finetune/Step Time": 5.5245464742183685} +{"Finetune/Learning Rate": 1.3789432023472315e-06, "Finetune/Loss": 0.8414404392242432, "Finetune/Loss (Raw)": 0.8616513609886169, "Finetune/Step": 4345, "Finetune/Step Time": 5.523668559268117} +{"Finetune/Learning Rate": 1.3757881539714768e-06, "Finetune/Loss": 0.841628909111023, "Finetune/Loss (Raw)": 0.8780979514122009, "Finetune/Step": 4346, "Finetune/Step Time": 5.527095202356577} +{"Finetune/Learning Rate": 1.3726364524839687e-06, "Finetune/Loss": 0.8432140350341797, "Finetune/Loss (Raw)": 0.9589201807975769, "Finetune/Step": 4347, "Finetune/Step Time": 5.525049148127437} +{"Finetune/Learning Rate": 1.369488099107814e-06, "Finetune/Loss": 0.8435802459716797, "Finetune/Loss (Raw)": 0.9256130456924438, "Finetune/Step": 4348, "Finetune/Step Time": 5.53106608428061} +{"Finetune/Learning Rate": 1.3663430950648326e-06, "Finetune/Loss": 0.8434087038040161, "Finetune/Loss (Raw)": 0.9814441204071045, "Finetune/Step": 4349, "Finetune/Step Time": 5.527760922908783} +{"Finetune/Learning Rate": 1.363201441575539e-06, "Finetune/Loss": 0.8409156203269958, "Finetune/Loss (Raw)": 0.6465044021606445, "Finetune/Step": 4350, "Finetune/Step Time": 5.53996168076992} +{"Finetune/Learning Rate": 1.360063139859147e-06, "Finetune/Loss": 0.84036785364151, "Finetune/Loss (Raw)": 0.9009703397750854, "Finetune/Step": 4351, "Finetune/Step Time": 5.539363728836179} +{"Finetune/Learning Rate": 1.3569281911335686e-06, "Finetune/Loss": 0.8406974673271179, "Finetune/Loss (Raw)": 0.853819727897644, "Finetune/Step": 4352, "Finetune/Step Time": 5.530960248783231} +{"Finetune/Learning Rate": 1.3537965966154177e-06, "Finetune/Loss": 0.8415945172309875, "Finetune/Loss (Raw)": 0.9121460318565369, "Finetune/Step": 4353, "Finetune/Step Time": 5.529017997905612} +{"Finetune/Learning Rate": 1.3506683575200053e-06, "Finetune/Loss": 0.8397786021232605, "Finetune/Loss (Raw)": 0.7533478736877441, "Finetune/Step": 4354, "Finetune/Step Time": 5.5277128126472235} +{"Finetune/Learning Rate": 1.347543475061337e-06, "Finetune/Loss": 0.8384108543395996, "Finetune/Loss (Raw)": 0.8926979303359985, "Finetune/Step": 4355, "Finetune/Step Time": 5.5236960630863905} +{"Finetune/Learning Rate": 1.344421950452126e-06, "Finetune/Loss": 0.8387465476989746, "Finetune/Loss (Raw)": 0.8949575424194336, "Finetune/Step": 4356, "Finetune/Step Time": 5.521756896749139} +{"Finetune/Learning Rate": 1.341303784903767e-06, "Finetune/Loss": 0.8388316631317139, "Finetune/Loss (Raw)": 0.9217168092727661, "Finetune/Step": 4357, "Finetune/Step Time": 5.521872729063034} +{"Finetune/Learning Rate": 1.3381889796263602e-06, "Finetune/Loss": 0.8439153432846069, "Finetune/Loss (Raw)": 0.9184826016426086, "Finetune/Step": 4358, "Finetune/Step Time": 5.500874910503626} +{"Finetune/Learning Rate": 1.335077535828706e-06, "Finetune/Loss": 0.8457017540931702, "Finetune/Loss (Raw)": 0.8887263536453247, "Finetune/Step": 4359, "Finetune/Step Time": 5.4506889041513205} +{"Finetune/Learning Rate": 1.3319694547182916e-06, "Finetune/Loss": 0.8450223207473755, "Finetune/Loss (Raw)": 0.8157070279121399, "Finetune/Step": 4360, "Finetune/Step Time": 5.44291440397501} +{"Finetune/Learning Rate": 1.328864737501302e-06, "Finetune/Loss": 0.8471231460571289, "Finetune/Loss (Raw)": 1.0693485736846924, "Finetune/Step": 4361, "Finetune/Step Time": 5.446243258193135} +{"Finetune/Learning Rate": 1.325763385382618e-06, "Finetune/Loss": 0.844855785369873, "Finetune/Loss (Raw)": 0.814460813999176, "Finetune/Step": 4362, "Finetune/Step Time": 5.4483700934797525} +{"Finetune/Learning Rate": 1.3226653995658144e-06, "Finetune/Loss": 0.8443814516067505, "Finetune/Loss (Raw)": 0.8376216292381287, "Finetune/Step": 4363, "Finetune/Step Time": 5.450623361393809} +{"Finetune/Learning Rate": 1.319570781253159e-06, "Finetune/Loss": 0.8448043465614319, "Finetune/Loss (Raw)": 0.7861353158950806, "Finetune/Step": 4364, "Finetune/Step Time": 5.4622372183948755} +{"Finetune/Learning Rate": 1.3164795316456091e-06, "Finetune/Loss": 0.8430517911911011, "Finetune/Loss (Raw)": 0.7814650535583496, "Finetune/Step": 4365, "Finetune/Step Time": 5.477091316133738} +{"Finetune/Learning Rate": 1.313391651942827e-06, "Finetune/Loss": 0.8426921367645264, "Finetune/Loss (Raw)": 0.9131908416748047, "Finetune/Step": 4366, "Finetune/Step Time": 5.481839410960674} +{"Finetune/Learning Rate": 1.3103071433431501e-06, "Finetune/Loss": 0.8418073654174805, "Finetune/Loss (Raw)": 0.7960525155067444, "Finetune/Step": 4367, "Finetune/Step Time": 5.48787328414619} +{"Finetune/Learning Rate": 1.307226007043617e-06, "Finetune/Loss": 0.8425090312957764, "Finetune/Loss (Raw)": 0.9252897500991821, "Finetune/Step": 4368, "Finetune/Step Time": 5.484249766916037} +{"Finetune/Learning Rate": 1.30414824423996e-06, "Finetune/Loss": 0.8422542214393616, "Finetune/Loss (Raw)": 0.8206161260604858, "Finetune/Step": 4369, "Finetune/Step Time": 5.490999041125178} +{"Finetune/Learning Rate": 1.301073856126598e-06, "Finetune/Loss": 0.8424806594848633, "Finetune/Loss (Raw)": 0.9708613157272339, "Finetune/Step": 4370, "Finetune/Step Time": 5.493445428088307} +{"Finetune/Learning Rate": 1.2980028438966396e-06, "Finetune/Loss": 0.8433263301849365, "Finetune/Loss (Raw)": 0.9948907494544983, "Finetune/Step": 4371, "Finetune/Step Time": 5.471510065719485} +{"Finetune/Learning Rate": 1.2949352087418864e-06, "Finetune/Loss": 0.8382693529129028, "Finetune/Loss (Raw)": 0.247453972697258, "Finetune/Step": 4372, "Finetune/Step Time": 5.503432400524616} +{"Finetune/Learning Rate": 1.291870951852826e-06, "Finetune/Loss": 0.837312638759613, "Finetune/Loss (Raw)": 0.7621710300445557, "Finetune/Step": 4373, "Finetune/Step Time": 5.5001680832356215} +{"Finetune/Learning Rate": 1.288810074418637e-06, "Finetune/Loss": 0.8371177911758423, "Finetune/Loss (Raw)": 0.8345832824707031, "Finetune/Step": 4374, "Finetune/Step Time": 5.465492757037282} +{"Finetune/Learning Rate": 1.2857525776271852e-06, "Finetune/Loss": 0.8366161584854126, "Finetune/Loss (Raw)": 0.8030851483345032, "Finetune/Step": 4375, "Finetune/Step Time": 5.467635655775666} +{"Finetune/Learning Rate": 1.2826984626650297e-06, "Finetune/Loss": 0.8416547775268555, "Finetune/Loss (Raw)": 0.9341065287590027, "Finetune/Step": 4376, "Finetune/Step Time": 5.456229068338871} +{"Finetune/Learning Rate": 1.2796477307174082e-06, "Finetune/Loss": 0.841184139251709, "Finetune/Loss (Raw)": 0.8729884028434753, "Finetune/Step": 4377, "Finetune/Step Time": 5.452384332194924} +{"Finetune/Learning Rate": 1.2766003829682504e-06, "Finetune/Loss": 0.8423289060592651, "Finetune/Loss (Raw)": 0.8746005892753601, "Finetune/Step": 4378, "Finetune/Step Time": 5.462219130247831} +{"Finetune/Learning Rate": 1.273556420600175e-06, "Finetune/Loss": 0.8424679040908813, "Finetune/Loss (Raw)": 0.7650571465492249, "Finetune/Step": 4379, "Finetune/Step Time": 5.473409624770284} +{"Finetune/Learning Rate": 1.2705158447944843e-06, "Finetune/Loss": 0.8418985605239868, "Finetune/Loss (Raw)": 0.9818641543388367, "Finetune/Step": 4380, "Finetune/Step Time": 5.4762190990149975} +{"Finetune/Learning Rate": 1.2674786567311648e-06, "Finetune/Loss": 0.8438667058944702, "Finetune/Loss (Raw)": 0.9063016772270203, "Finetune/Step": 4381, "Finetune/Step Time": 5.462761083617806} +{"Finetune/Learning Rate": 1.2644448575888913e-06, "Finetune/Loss": 0.8458741903305054, "Finetune/Loss (Raw)": 0.9341316819190979, "Finetune/Step": 4382, "Finetune/Step Time": 5.45663059130311} +{"Finetune/Learning Rate": 1.2614144485450208e-06, "Finetune/Loss": 0.8455378413200378, "Finetune/Loss (Raw)": 0.7350670695304871, "Finetune/Step": 4383, "Finetune/Step Time": 5.465493584051728} +{"Finetune/Learning Rate": 1.2583874307755961e-06, "Finetune/Loss": 0.8451923727989197, "Finetune/Loss (Raw)": 0.8756701946258545, "Finetune/Step": 4384, "Finetune/Step Time": 5.451621761545539} +{"Finetune/Learning Rate": 1.2553638054553418e-06, "Finetune/Loss": 0.8438919186592102, "Finetune/Loss (Raw)": 0.8085330128669739, "Finetune/Step": 4385, "Finetune/Step Time": 5.444920927286148} +{"Finetune/Learning Rate": 1.252343573757674e-06, "Finetune/Loss": 0.8452824354171753, "Finetune/Loss (Raw)": 0.8608132004737854, "Finetune/Step": 4386, "Finetune/Step Time": 5.4721702840179205} +{"Finetune/Learning Rate": 1.2493267368546802e-06, "Finetune/Loss": 0.8456096053123474, "Finetune/Loss (Raw)": 1.0546945333480835, "Finetune/Step": 4387, "Finetune/Step Time": 5.46008899435401} +{"Finetune/Learning Rate": 1.2463132959171343e-06, "Finetune/Loss": 0.8460339903831482, "Finetune/Loss (Raw)": 0.9008082747459412, "Finetune/Step": 4388, "Finetune/Step Time": 5.464711386710405} +{"Finetune/Learning Rate": 1.2433032521145006e-06, "Finetune/Loss": 0.8459558486938477, "Finetune/Loss (Raw)": 0.9181801080703735, "Finetune/Step": 4389, "Finetune/Step Time": 5.463558504357934} +{"Finetune/Learning Rate": 1.2402966066149135e-06, "Finetune/Loss": 0.8458839654922485, "Finetune/Loss (Raw)": 0.8318308591842651, "Finetune/Step": 4390, "Finetune/Step Time": 5.4625472370535135} +{"Finetune/Learning Rate": 1.2372933605851944e-06, "Finetune/Loss": 0.8444058895111084, "Finetune/Loss (Raw)": 0.845974326133728, "Finetune/Step": 4391, "Finetune/Step Time": 5.47004272043705} +{"Finetune/Learning Rate": 1.2342935151908453e-06, "Finetune/Loss": 0.8410226106643677, "Finetune/Loss (Raw)": 0.35292527079582214, "Finetune/Step": 4392, "Finetune/Step Time": 5.493611069396138} +{"Finetune/Learning Rate": 1.231297071596046e-06, "Finetune/Loss": 0.8396281599998474, "Finetune/Loss (Raw)": 0.7447289824485779, "Finetune/Step": 4393, "Finetune/Step Time": 5.501126889139414} +{"Finetune/Learning Rate": 1.2283040309636596e-06, "Finetune/Loss": 0.8386505842208862, "Finetune/Loss (Raw)": 0.8267983794212341, "Finetune/Step": 4394, "Finetune/Step Time": 5.5108866430819035} +{"Finetune/Learning Rate": 1.2253143944552258e-06, "Finetune/Loss": 0.8391686677932739, "Finetune/Loss (Raw)": 0.850973904132843, "Finetune/Step": 4395, "Finetune/Step Time": 5.500977333635092} +{"Finetune/Learning Rate": 1.2223281632309636e-06, "Finetune/Loss": 0.840430498123169, "Finetune/Loss (Raw)": 0.9632819890975952, "Finetune/Step": 4396, "Finetune/Step Time": 5.509233647957444} +{"Finetune/Learning Rate": 1.2193453384497722e-06, "Finetune/Loss": 0.8379814028739929, "Finetune/Loss (Raw)": 0.41776126623153687, "Finetune/Step": 4397, "Finetune/Step Time": 5.506573397666216} +{"Finetune/Learning Rate": 1.2163659212692246e-06, "Finetune/Loss": 0.8383985757827759, "Finetune/Loss (Raw)": 0.8938599824905396, "Finetune/Step": 4398, "Finetune/Step Time": 5.508934900164604} +{"Finetune/Learning Rate": 1.2133899128455796e-06, "Finetune/Loss": 0.8421643972396851, "Finetune/Loss (Raw)": 0.8066509962081909, "Finetune/Step": 4399, "Finetune/Step Time": 5.479756100103259} +{"Finetune/Learning Rate": 1.2104173143337672e-06, "Finetune/Loss": 0.842010498046875, "Finetune/Loss (Raw)": 0.8952961564064026, "Finetune/Step": 4400, "Finetune/Step Time": 5.487389642745256} +{"Finetune/Learning Rate": 1.2074481268873894e-06, "Finetune/Loss": 0.8387832641601562, "Finetune/Loss (Raw)": 0.3329385221004486, "Finetune/Step": 4401, "Finetune/Step Time": 5.513796074315906} +{"Finetune/Learning Rate": 1.2044823516587345e-06, "Finetune/Loss": 0.838411271572113, "Finetune/Loss (Raw)": 0.8151401877403259, "Finetune/Step": 4402, "Finetune/Step Time": 5.531858054921031} +{"Finetune/Learning Rate": 1.2015199897987616e-06, "Finetune/Loss": 0.8392016291618347, "Finetune/Loss (Raw)": 0.9256222248077393, "Finetune/Step": 4403, "Finetune/Step Time": 5.516173891723156} +{"Finetune/Learning Rate": 1.198561042457105e-06, "Finetune/Loss": 0.8399612903594971, "Finetune/Loss (Raw)": 1.0167391300201416, "Finetune/Step": 4404, "Finetune/Step Time": 5.51714894361794} +{"Finetune/Learning Rate": 1.195605510782073e-06, "Finetune/Loss": 0.8397650122642517, "Finetune/Loss (Raw)": 0.821186900138855, "Finetune/Step": 4405, "Finetune/Step Time": 5.512574847787619} +{"Finetune/Learning Rate": 1.192653395920652e-06, "Finetune/Loss": 0.8384599685668945, "Finetune/Loss (Raw)": 0.7585563063621521, "Finetune/Step": 4406, "Finetune/Step Time": 5.504929695278406} +{"Finetune/Learning Rate": 1.189704699018499e-06, "Finetune/Loss": 0.8382687568664551, "Finetune/Loss (Raw)": 0.7799568176269531, "Finetune/Step": 4407, "Finetune/Step Time": 5.502728998661041} +{"Finetune/Learning Rate": 1.186759421219943e-06, "Finetune/Loss": 0.8387982845306396, "Finetune/Loss (Raw)": 1.0194603204727173, "Finetune/Step": 4408, "Finetune/Step Time": 5.501769484952092} +{"Finetune/Learning Rate": 1.1838175636679938e-06, "Finetune/Loss": 0.8373755216598511, "Finetune/Loss (Raw)": 0.8222715258598328, "Finetune/Step": 4409, "Finetune/Step Time": 5.501101357862353} +{"Finetune/Learning Rate": 1.1808791275043297e-06, "Finetune/Loss": 0.8368929624557495, "Finetune/Loss (Raw)": 0.8885114789009094, "Finetune/Step": 4410, "Finetune/Step Time": 5.498027540743351} +{"Finetune/Learning Rate": 1.1779441138692926e-06, "Finetune/Loss": 0.8366247415542603, "Finetune/Loss (Raw)": 0.846254825592041, "Finetune/Step": 4411, "Finetune/Step Time": 5.491250799968839} +{"Finetune/Learning Rate": 1.1750125239019118e-06, "Finetune/Loss": 0.8358827829360962, "Finetune/Loss (Raw)": 0.8197730183601379, "Finetune/Step": 4412, "Finetune/Step Time": 5.493383968248963} +{"Finetune/Learning Rate": 1.1720843587398756e-06, "Finetune/Loss": 0.8391112685203552, "Finetune/Loss (Raw)": 1.0884664058685303, "Finetune/Step": 4413, "Finetune/Step Time": 5.456078872084618} +{"Finetune/Learning Rate": 1.1691596195195498e-06, "Finetune/Loss": 0.8383299112319946, "Finetune/Loss (Raw)": 0.809090256690979, "Finetune/Step": 4414, "Finetune/Step Time": 5.460959766060114} +{"Finetune/Learning Rate": 1.1662383073759686e-06, "Finetune/Loss": 0.8385802507400513, "Finetune/Loss (Raw)": 0.9452022314071655, "Finetune/Step": 4415, "Finetune/Step Time": 5.466148192062974} +{"Finetune/Learning Rate": 1.1633204234428352e-06, "Finetune/Loss": 0.8368127346038818, "Finetune/Loss (Raw)": 0.779509961605072, "Finetune/Step": 4416, "Finetune/Step Time": 5.46674426086247} +{"Finetune/Learning Rate": 1.1604059688525238e-06, "Finetune/Loss": 0.8379533290863037, "Finetune/Loss (Raw)": 0.9620710611343384, "Finetune/Step": 4417, "Finetune/Step Time": 5.467482585459948} +{"Finetune/Learning Rate": 1.157494944736075e-06, "Finetune/Loss": 0.8370157480239868, "Finetune/Loss (Raw)": 0.854353129863739, "Finetune/Step": 4418, "Finetune/Step Time": 5.4679843578487635} +{"Finetune/Learning Rate": 1.1545873522232055e-06, "Finetune/Loss": 0.8370566368103027, "Finetune/Loss (Raw)": 0.9327973127365112, "Finetune/Step": 4419, "Finetune/Step Time": 5.472812745720148} +{"Finetune/Learning Rate": 1.1516831924422945e-06, "Finetune/Loss": 0.8357657790184021, "Finetune/Loss (Raw)": 0.764695405960083, "Finetune/Step": 4420, "Finetune/Step Time": 5.48491844534874} +{"Finetune/Learning Rate": 1.1487824665203828e-06, "Finetune/Loss": 0.8351670503616333, "Finetune/Loss (Raw)": 0.8380072116851807, "Finetune/Step": 4421, "Finetune/Step Time": 5.482560167089105} +{"Finetune/Learning Rate": 1.1458851755831934e-06, "Finetune/Loss": 0.8353388905525208, "Finetune/Loss (Raw)": 0.8983853459358215, "Finetune/Step": 4422, "Finetune/Step Time": 5.46485929004848} +{"Finetune/Learning Rate": 1.1429913207551046e-06, "Finetune/Loss": 0.8391367197036743, "Finetune/Loss (Raw)": 0.7878084182739258, "Finetune/Step": 4423, "Finetune/Step Time": 5.439238585531712} +{"Finetune/Learning Rate": 1.1401009031591658e-06, "Finetune/Loss": 0.8340110182762146, "Finetune/Loss (Raw)": 0.33107346296310425, "Finetune/Step": 4424, "Finetune/Step Time": 5.474128408357501} +{"Finetune/Learning Rate": 1.137213923917092e-06, "Finetune/Loss": 0.8333858251571655, "Finetune/Loss (Raw)": 0.8225799202919006, "Finetune/Step": 4425, "Finetune/Step Time": 5.4906283132731915} +{"Finetune/Learning Rate": 1.1343303841492626e-06, "Finetune/Loss": 0.8349744081497192, "Finetune/Loss (Raw)": 0.9148540496826172, "Finetune/Step": 4426, "Finetune/Step Time": 5.486736986786127} +{"Finetune/Learning Rate": 1.1314502849747244e-06, "Finetune/Loss": 0.8338021636009216, "Finetune/Loss (Raw)": 0.7853134274482727, "Finetune/Step": 4427, "Finetune/Step Time": 5.495056016370654} +{"Finetune/Learning Rate": 1.1285736275111836e-06, "Finetune/Loss": 0.833447277545929, "Finetune/Loss (Raw)": 0.9010085463523865, "Finetune/Step": 4428, "Finetune/Step Time": 5.494837518781424} +{"Finetune/Learning Rate": 1.1257004128750205e-06, "Finetune/Loss": 0.8374385833740234, "Finetune/Loss (Raw)": 0.8743133544921875, "Finetune/Step": 4429, "Finetune/Step Time": 5.46502498537302} +{"Finetune/Learning Rate": 1.122830642181273e-06, "Finetune/Loss": 0.8352853655815125, "Finetune/Loss (Raw)": 0.6954445838928223, "Finetune/Step": 4430, "Finetune/Step Time": 5.505499556660652} +{"Finetune/Learning Rate": 1.119964316543637e-06, "Finetune/Loss": 0.8343346118927002, "Finetune/Loss (Raw)": 0.8739386200904846, "Finetune/Step": 4431, "Finetune/Step Time": 5.518243519589305} +{"Finetune/Learning Rate": 1.1171014370744837e-06, "Finetune/Loss": 0.8385683298110962, "Finetune/Loss (Raw)": 0.8845586776733398, "Finetune/Step": 4432, "Finetune/Step Time": 5.493190975859761} +{"Finetune/Learning Rate": 1.114242004884839e-06, "Finetune/Loss": 0.8381232023239136, "Finetune/Loss (Raw)": 0.7193595767021179, "Finetune/Step": 4433, "Finetune/Step Time": 5.5028789937496185} +{"Finetune/Learning Rate": 1.111386021084394e-06, "Finetune/Loss": 0.839148998260498, "Finetune/Loss (Raw)": 0.9653609395027161, "Finetune/Step": 4434, "Finetune/Step Time": 5.497071474790573} +{"Finetune/Learning Rate": 1.1085334867814946e-06, "Finetune/Loss": 0.8406772613525391, "Finetune/Loss (Raw)": 0.8732579350471497, "Finetune/Step": 4435, "Finetune/Step Time": 5.4874472338706255} +{"Finetune/Learning Rate": 1.1056844030831603e-06, "Finetune/Loss": 0.8396445512771606, "Finetune/Loss (Raw)": 0.764325737953186, "Finetune/Step": 4436, "Finetune/Step Time": 5.49897444434464} +{"Finetune/Learning Rate": 1.1028387710950616e-06, "Finetune/Loss": 0.8393399715423584, "Finetune/Loss (Raw)": 0.8239024877548218, "Finetune/Step": 4437, "Finetune/Step Time": 5.499225560575724} +{"Finetune/Learning Rate": 1.0999965919215327e-06, "Finetune/Loss": 0.8434666395187378, "Finetune/Loss (Raw)": 0.8797897696495056, "Finetune/Step": 4438, "Finetune/Step Time": 5.463814195245504} +{"Finetune/Learning Rate": 1.0971578666655713e-06, "Finetune/Loss": 0.8447000980377197, "Finetune/Loss (Raw)": 1.035564661026001, "Finetune/Step": 4439, "Finetune/Step Time": 5.461098246276379} +{"Finetune/Learning Rate": 1.0943225964288272e-06, "Finetune/Loss": 0.84492027759552, "Finetune/Loss (Raw)": 1.0105011463165283, "Finetune/Step": 4440, "Finetune/Step Time": 5.45895510353148} +{"Finetune/Learning Rate": 1.091490782311614e-06, "Finetune/Loss": 0.8448918461799622, "Finetune/Loss (Raw)": 0.9806910753250122, "Finetune/Step": 4441, "Finetune/Step Time": 5.457614243030548} +{"Finetune/Learning Rate": 1.088662425412903e-06, "Finetune/Loss": 0.8449958562850952, "Finetune/Loss (Raw)": 0.946494460105896, "Finetune/Step": 4442, "Finetune/Step Time": 5.472578635439277} +{"Finetune/Learning Rate": 1.0858375268303267e-06, "Finetune/Loss": 0.844940721988678, "Finetune/Loss (Raw)": 0.8401066660881042, "Finetune/Step": 4443, "Finetune/Step Time": 5.472745966166258} +{"Finetune/Learning Rate": 1.0830160876601737e-06, "Finetune/Loss": 0.8448817729949951, "Finetune/Loss (Raw)": 0.8134544491767883, "Finetune/Step": 4444, "Finetune/Step Time": 5.457081401720643} +{"Finetune/Learning Rate": 1.0801981089973844e-06, "Finetune/Loss": 0.8438202142715454, "Finetune/Loss (Raw)": 0.7318372130393982, "Finetune/Step": 4445, "Finetune/Step Time": 5.464826546609402} +{"Finetune/Learning Rate": 1.0773835919355669e-06, "Finetune/Loss": 0.8452117443084717, "Finetune/Loss (Raw)": 0.9741837978363037, "Finetune/Step": 4446, "Finetune/Step Time": 5.4458441231399775} +{"Finetune/Learning Rate": 1.0745725375669768e-06, "Finetune/Loss": 0.8460044860839844, "Finetune/Loss (Raw)": 0.8446922302246094, "Finetune/Step": 4447, "Finetune/Step Time": 5.4358661491423845} +{"Finetune/Learning Rate": 1.0717649469825298e-06, "Finetune/Loss": 0.84632408618927, "Finetune/Loss (Raw)": 0.8725920915603638, "Finetune/Step": 4448, "Finetune/Step Time": 5.43466491997242} +{"Finetune/Learning Rate": 1.0689608212718005e-06, "Finetune/Loss": 0.8486729860305786, "Finetune/Loss (Raw)": 1.0106364488601685, "Finetune/Step": 4449, "Finetune/Step Time": 5.42097176797688} +{"Finetune/Learning Rate": 1.0661601615230122e-06, "Finetune/Loss": 0.846843957901001, "Finetune/Loss (Raw)": 0.7757073640823364, "Finetune/Step": 4450, "Finetune/Step Time": 5.414806418120861} +{"Finetune/Learning Rate": 1.0633629688230452e-06, "Finetune/Loss": 0.8473092317581177, "Finetune/Loss (Raw)": 0.773628294467926, "Finetune/Step": 4451, "Finetune/Step Time": 5.425182605162263} +{"Finetune/Learning Rate": 1.0605692442574356e-06, "Finetune/Loss": 0.8436353802680969, "Finetune/Loss (Raw)": 0.3473013639450073, "Finetune/Step": 4452, "Finetune/Step Time": 5.455170249566436} +{"Finetune/Learning Rate": 1.0577789889103763e-06, "Finetune/Loss": 0.8443222045898438, "Finetune/Loss (Raw)": 0.756497323513031, "Finetune/Step": 4453, "Finetune/Step Time": 5.451810667291284} +{"Finetune/Learning Rate": 1.054992203864712e-06, "Finetune/Loss": 0.8452804088592529, "Finetune/Loss (Raw)": 0.8725883364677429, "Finetune/Step": 4454, "Finetune/Step Time": 5.450251588597894} +{"Finetune/Learning Rate": 1.0522088902019324e-06, "Finetune/Loss": 0.8454970717430115, "Finetune/Loss (Raw)": 0.743837296962738, "Finetune/Step": 4455, "Finetune/Step Time": 5.447239171713591} +{"Finetune/Learning Rate": 1.049429049002193e-06, "Finetune/Loss": 0.8451421856880188, "Finetune/Loss (Raw)": 0.8170503377914429, "Finetune/Step": 4456, "Finetune/Step Time": 5.448731552809477} +{"Finetune/Learning Rate": 1.0466526813442957e-06, "Finetune/Loss": 0.8446526527404785, "Finetune/Loss (Raw)": 0.7927775382995605, "Finetune/Step": 4457, "Finetune/Step Time": 5.45969920232892} +{"Finetune/Learning Rate": 1.0438797883056927e-06, "Finetune/Loss": 0.8437376022338867, "Finetune/Loss (Raw)": 0.8423015475273132, "Finetune/Step": 4458, "Finetune/Step Time": 5.473229601979256} +{"Finetune/Learning Rate": 1.0411103709624903e-06, "Finetune/Loss": 0.8428807258605957, "Finetune/Loss (Raw)": 0.8432587385177612, "Finetune/Step": 4459, "Finetune/Step Time": 5.479099960997701} +{"Finetune/Learning Rate": 1.0383444303894453e-06, "Finetune/Loss": 0.8433714509010315, "Finetune/Loss (Raw)": 0.9589994549751282, "Finetune/Step": 4460, "Finetune/Step Time": 5.47937441803515} +{"Finetune/Learning Rate": 1.0355819676599655e-06, "Finetune/Loss": 0.84206223487854, "Finetune/Loss (Raw)": 0.7972407937049866, "Finetune/Step": 4461, "Finetune/Step Time": 5.478496605530381} +{"Finetune/Learning Rate": 1.0328229838461069e-06, "Finetune/Loss": 0.8418874740600586, "Finetune/Loss (Raw)": 0.9203126430511475, "Finetune/Step": 4462, "Finetune/Step Time": 5.491035830229521} +{"Finetune/Learning Rate": 1.0300674800185795e-06, "Finetune/Loss": 0.8399052619934082, "Finetune/Loss (Raw)": 0.6607632637023926, "Finetune/Step": 4463, "Finetune/Step Time": 5.500385716557503} +{"Finetune/Learning Rate": 1.0273154572467436e-06, "Finetune/Loss": 0.8387563228607178, "Finetune/Loss (Raw)": 0.8579594492912292, "Finetune/Step": 4464, "Finetune/Step Time": 5.503165129572153} +{"Finetune/Learning Rate": 1.0245669165985973e-06, "Finetune/Loss": 0.8398027420043945, "Finetune/Loss (Raw)": 0.8017168045043945, "Finetune/Step": 4465, "Finetune/Step Time": 5.480068910866976} +{"Finetune/Learning Rate": 1.0218218591408013e-06, "Finetune/Loss": 0.839571475982666, "Finetune/Loss (Raw)": 0.8364766240119934, "Finetune/Step": 4466, "Finetune/Step Time": 5.479058852419257} +{"Finetune/Learning Rate": 1.0190802859386585e-06, "Finetune/Loss": 0.8404808044433594, "Finetune/Loss (Raw)": 0.9116942286491394, "Finetune/Step": 4467, "Finetune/Step Time": 5.473512578755617} +{"Finetune/Learning Rate": 1.0163421980561184e-06, "Finetune/Loss": 0.840356707572937, "Finetune/Loss (Raw)": 0.855697751045227, "Finetune/Step": 4468, "Finetune/Step Time": 5.465384906157851} +{"Finetune/Learning Rate": 1.0136075965557812e-06, "Finetune/Loss": 0.8394412994384766, "Finetune/Loss (Raw)": 0.8494791388511658, "Finetune/Step": 4469, "Finetune/Step Time": 5.465304480865598} +{"Finetune/Learning Rate": 1.0108764824988903e-06, "Finetune/Loss": 0.8395688533782959, "Finetune/Loss (Raw)": 0.8459324836730957, "Finetune/Step": 4470, "Finetune/Step Time": 5.421654276549816} +{"Finetune/Learning Rate": 1.008148856945339e-06, "Finetune/Loss": 0.839479923248291, "Finetune/Loss (Raw)": 0.7958203554153442, "Finetune/Step": 4471, "Finetune/Step Time": 5.429350143298507} +{"Finetune/Learning Rate": 1.0054247209536628e-06, "Finetune/Loss": 0.8392593264579773, "Finetune/Loss (Raw)": 0.8765939474105835, "Finetune/Step": 4472, "Finetune/Step Time": 5.420635491609573} +{"Finetune/Learning Rate": 1.0027040755810513e-06, "Finetune/Loss": 0.8406952619552612, "Finetune/Loss (Raw)": 1.0454535484313965, "Finetune/Step": 4473, "Finetune/Step Time": 5.418774738907814} +{"Finetune/Learning Rate": 9.999869218833313e-07, "Finetune/Loss": 0.8409348130226135, "Finetune/Loss (Raw)": 0.9087525606155396, "Finetune/Step": 4474, "Finetune/Step Time": 5.416857939213514} +{"Finetune/Learning Rate": 9.972732609149738e-07, "Finetune/Loss": 0.8395344018936157, "Finetune/Loss (Raw)": 0.779663622379303, "Finetune/Step": 4475, "Finetune/Step Time": 5.432402398437262} +{"Finetune/Learning Rate": 9.945630937291006e-07, "Finetune/Loss": 0.8395569920539856, "Finetune/Loss (Raw)": 0.9285113215446472, "Finetune/Step": 4476, "Finetune/Step Time": 5.433537013828754} +{"Finetune/Learning Rate": 9.918564213774762e-07, "Finetune/Loss": 0.8381272554397583, "Finetune/Loss (Raw)": 0.7984411120414734, "Finetune/Step": 4477, "Finetune/Step Time": 5.434171475470066} +{"Finetune/Learning Rate": 9.891532449105045e-07, "Finetune/Loss": 0.8404542207717896, "Finetune/Loss (Raw)": 0.9443581700325012, "Finetune/Step": 4478, "Finetune/Step Time": 5.416165243834257} +{"Finetune/Learning Rate": 9.864535653772367e-07, "Finetune/Loss": 0.8403013944625854, "Finetune/Loss (Raw)": 0.8813977837562561, "Finetune/Step": 4479, "Finetune/Step Time": 5.4255196377635} +{"Finetune/Learning Rate": 9.837573838253655e-07, "Finetune/Loss": 0.8401828408241272, "Finetune/Loss (Raw)": 0.8386463522911072, "Finetune/Step": 4480, "Finetune/Step Time": 5.430378891527653} +{"Finetune/Learning Rate": 9.810647013012265e-07, "Finetune/Loss": 0.8396815061569214, "Finetune/Loss (Raw)": 0.8479745984077454, "Finetune/Step": 4481, "Finetune/Step Time": 5.432074476033449} +{"Finetune/Learning Rate": 9.783755188497944e-07, "Finetune/Loss": 0.8401614427566528, "Finetune/Loss (Raw)": 0.8147833347320557, "Finetune/Step": 4482, "Finetune/Step Time": 5.446472121402621} +{"Finetune/Learning Rate": 9.75689837514696e-07, "Finetune/Loss": 0.8399196863174438, "Finetune/Loss (Raw)": 0.8617509603500366, "Finetune/Step": 4483, "Finetune/Step Time": 5.449501624330878} +{"Finetune/Learning Rate": 9.730076583381841e-07, "Finetune/Loss": 0.8400319814682007, "Finetune/Loss (Raw)": 0.9093282222747803, "Finetune/Step": 4484, "Finetune/Step Time": 5.464061509817839} +{"Finetune/Learning Rate": 9.703289823611606e-07, "Finetune/Loss": 0.8394696116447449, "Finetune/Loss (Raw)": 0.8497401475906372, "Finetune/Step": 4485, "Finetune/Step Time": 5.4647339172661304} +{"Finetune/Learning Rate": 9.676538106231725e-07, "Finetune/Loss": 0.8390935659408569, "Finetune/Loss (Raw)": 0.8703424334526062, "Finetune/Step": 4486, "Finetune/Step Time": 5.462252881377935} +{"Finetune/Learning Rate": 9.649821441623986e-07, "Finetune/Loss": 0.8346852660179138, "Finetune/Loss (Raw)": 0.3244676887989044, "Finetune/Step": 4487, "Finetune/Step Time": 5.492810297757387} +{"Finetune/Learning Rate": 9.623139840156604e-07, "Finetune/Loss": 0.835763692855835, "Finetune/Loss (Raw)": 0.9537503719329834, "Finetune/Step": 4488, "Finetune/Step Time": 5.498942008242011} +{"Finetune/Learning Rate": 9.596493312184197e-07, "Finetune/Loss": 0.8327305912971497, "Finetune/Loss (Raw)": 0.6811119318008423, "Finetune/Step": 4489, "Finetune/Step Time": 5.488550653681159} +{"Finetune/Learning Rate": 9.569881868047747e-07, "Finetune/Loss": 0.8321932554244995, "Finetune/Loss (Raw)": 0.7456783652305603, "Finetune/Step": 4490, "Finetune/Step Time": 5.4930274579674006} +{"Finetune/Learning Rate": 9.543305518074652e-07, "Finetune/Loss": 0.8318225741386414, "Finetune/Loss (Raw)": 0.7901736497879028, "Finetune/Step": 4491, "Finetune/Step Time": 5.486758783459663} +{"Finetune/Learning Rate": 9.516764272578638e-07, "Finetune/Loss": 0.8328183889389038, "Finetune/Loss (Raw)": 0.9135994911193848, "Finetune/Step": 4492, "Finetune/Step Time": 5.468816315755248} +{"Finetune/Learning Rate": 9.49025814185992e-07, "Finetune/Loss": 0.833122730255127, "Finetune/Loss (Raw)": 0.8204167485237122, "Finetune/Step": 4493, "Finetune/Step Time": 5.464397815987468} +{"Finetune/Learning Rate": 9.463787136204938e-07, "Finetune/Loss": 0.8315086960792542, "Finetune/Loss (Raw)": 0.7065995335578918, "Finetune/Step": 4494, "Finetune/Step Time": 5.464096650481224} +{"Finetune/Learning Rate": 9.437351265886585e-07, "Finetune/Loss": 0.8323289155960083, "Finetune/Loss (Raw)": 0.9010366201400757, "Finetune/Step": 4495, "Finetune/Step Time": 5.463945511728525} +{"Finetune/Learning Rate": 9.410950541164143e-07, "Finetune/Loss": 0.8311500549316406, "Finetune/Loss (Raw)": 0.7743966579437256, "Finetune/Step": 4496, "Finetune/Step Time": 5.471067385748029} +{"Finetune/Learning Rate": 9.384584972283184e-07, "Finetune/Loss": 0.8320916891098022, "Finetune/Loss (Raw)": 0.9411491751670837, "Finetune/Step": 4497, "Finetune/Step Time": 5.479093646630645} +{"Finetune/Learning Rate": 9.358254569475689e-07, "Finetune/Loss": 0.8305100202560425, "Finetune/Loss (Raw)": 0.7684042453765869, "Finetune/Step": 4498, "Finetune/Step Time": 5.48379729129374} +{"Finetune/Learning Rate": 9.331959342959951e-07, "Finetune/Loss": 0.8301345705986023, "Finetune/Loss (Raw)": 0.9468329548835754, "Finetune/Step": 4499, "Finetune/Step Time": 5.488965159282088} +{"Finetune/Learning Rate": 9.305699302940662e-07, "Finetune/Loss": 0.8349722027778625, "Finetune/Loss (Raw)": 0.8666740655899048, "Finetune/Step": 4500, "Finetune/Step Time": 5.465067947283387} +{"Finetune/Learning Rate": 9.279474459608806e-07, "Finetune/Loss": 0.8364384174346924, "Finetune/Loss (Raw)": 0.9498531222343445, "Finetune/Step": 4501, "Finetune/Step Time": 5.464158449321985} +{"Finetune/Learning Rate": 9.253284823141728e-07, "Finetune/Loss": 0.8355625867843628, "Finetune/Loss (Raw)": 0.7224653959274292, "Finetune/Step": 4502, "Finetune/Step Time": 5.468948569148779} +{"Finetune/Learning Rate": 9.227130403703178e-07, "Finetune/Loss": 0.8366716504096985, "Finetune/Loss (Raw)": 0.9450503587722778, "Finetune/Step": 4503, "Finetune/Step Time": 5.459137437865138} +{"Finetune/Learning Rate": 9.201011211443101e-07, "Finetune/Loss": 0.8359851241111755, "Finetune/Loss (Raw)": 0.8462305068969727, "Finetune/Step": 4504, "Finetune/Step Time": 5.443829415366054} +{"Finetune/Learning Rate": 9.174927256497846e-07, "Finetune/Loss": 0.8355861902236938, "Finetune/Loss (Raw)": 0.8219304084777832, "Finetune/Step": 4505, "Finetune/Step Time": 5.443333983421326} +{"Finetune/Learning Rate": 9.148878548990137e-07, "Finetune/Loss": 0.8366330862045288, "Finetune/Loss (Raw)": 1.0085943937301636, "Finetune/Step": 4506, "Finetune/Step Time": 5.422298459336162} +{"Finetune/Learning Rate": 9.122865099028944e-07, "Finetune/Loss": 0.8332148790359497, "Finetune/Loss (Raw)": 0.3275358974933624, "Finetune/Step": 4507, "Finetune/Step Time": 5.439387856051326} +{"Finetune/Learning Rate": 9.09688691670958e-07, "Finetune/Loss": 0.8323591947555542, "Finetune/Loss (Raw)": 0.8723275661468506, "Finetune/Step": 4508, "Finetune/Step Time": 5.438353147357702} +{"Finetune/Learning Rate": 9.070944012113669e-07, "Finetune/Loss": 0.8314323425292969, "Finetune/Loss (Raw)": 0.787666916847229, "Finetune/Step": 4509, "Finetune/Step Time": 5.4306576400995255} +{"Finetune/Learning Rate": 9.045036395309148e-07, "Finetune/Loss": 0.8316718935966492, "Finetune/Loss (Raw)": 0.964794397354126, "Finetune/Step": 4510, "Finetune/Step Time": 5.4269809648394585} +{"Finetune/Learning Rate": 9.019164076350262e-07, "Finetune/Loss": 0.8338217735290527, "Finetune/Loss (Raw)": 1.010250210762024, "Finetune/Step": 4511, "Finetune/Step Time": 5.413966614753008} +{"Finetune/Learning Rate": 8.993327065277524e-07, "Finetune/Loss": 0.8297861218452454, "Finetune/Loss (Raw)": 0.35910704731941223, "Finetune/Step": 4512, "Finetune/Step Time": 5.439926391467452} +{"Finetune/Learning Rate": 8.967525372117825e-07, "Finetune/Loss": 0.8300162553787231, "Finetune/Loss (Raw)": 0.8379958271980286, "Finetune/Step": 4513, "Finetune/Step Time": 5.449995571747422} +{"Finetune/Learning Rate": 8.941759006884265e-07, "Finetune/Loss": 0.8292597532272339, "Finetune/Loss (Raw)": 0.7639728784561157, "Finetune/Step": 4514, "Finetune/Step Time": 5.426933249458671} +{"Finetune/Learning Rate": 8.916027979576247e-07, "Finetune/Loss": 0.8275772929191589, "Finetune/Loss (Raw)": 0.8393425941467285, "Finetune/Step": 4515, "Finetune/Step Time": 5.4301191587001085} +{"Finetune/Learning Rate": 8.890332300179516e-07, "Finetune/Loss": 0.8265615701675415, "Finetune/Loss (Raw)": 0.7707962989807129, "Finetune/Step": 4516, "Finetune/Step Time": 5.42469665594399} +{"Finetune/Learning Rate": 8.864671978666084e-07, "Finetune/Loss": 0.8252521753311157, "Finetune/Loss (Raw)": 0.7505745887756348, "Finetune/Step": 4517, "Finetune/Step Time": 5.426252728328109} +{"Finetune/Learning Rate": 8.839047024994152e-07, "Finetune/Loss": 0.826637864112854, "Finetune/Loss (Raw)": 1.009203314781189, "Finetune/Step": 4518, "Finetune/Step Time": 5.421629099175334} +{"Finetune/Learning Rate": 8.813457449108309e-07, "Finetune/Loss": 0.8273794651031494, "Finetune/Loss (Raw)": 0.9409036040306091, "Finetune/Step": 4519, "Finetune/Step Time": 5.415492190048099} +{"Finetune/Learning Rate": 8.787903260939368e-07, "Finetune/Loss": 0.8318924903869629, "Finetune/Loss (Raw)": 0.9305897951126099, "Finetune/Step": 4520, "Finetune/Step Time": 5.387894654646516} +{"Finetune/Learning Rate": 8.762384470404417e-07, "Finetune/Loss": 0.8338125944137573, "Finetune/Loss (Raw)": 0.9904999136924744, "Finetune/Step": 4521, "Finetune/Step Time": 5.37579395249486} +{"Finetune/Learning Rate": 8.736901087406779e-07, "Finetune/Loss": 0.833929717540741, "Finetune/Loss (Raw)": 0.8417890667915344, "Finetune/Step": 4522, "Finetune/Step Time": 5.372607726603746} +{"Finetune/Learning Rate": 8.711453121836066e-07, "Finetune/Loss": 0.8341096043586731, "Finetune/Loss (Raw)": 0.8740012645721436, "Finetune/Step": 4523, "Finetune/Step Time": 5.376482078805566} +{"Finetune/Learning Rate": 8.686040583568134e-07, "Finetune/Loss": 0.8332543969154358, "Finetune/Loss (Raw)": 0.8538135290145874, "Finetune/Step": 4524, "Finetune/Step Time": 5.370538393035531} +{"Finetune/Learning Rate": 8.660663482465093e-07, "Finetune/Loss": 0.8364772200584412, "Finetune/Loss (Raw)": 0.830288290977478, "Finetune/Step": 4525, "Finetune/Step Time": 5.341984689235687} +{"Finetune/Learning Rate": 8.63532182837531e-07, "Finetune/Loss": 0.8347188234329224, "Finetune/Loss (Raw)": 0.6687874794006348, "Finetune/Step": 4526, "Finetune/Step Time": 5.33798510581255} +{"Finetune/Learning Rate": 8.610015631133395e-07, "Finetune/Loss": 0.8326370716094971, "Finetune/Loss (Raw)": 0.5401750802993774, "Finetune/Step": 4527, "Finetune/Step Time": 5.356625776737928} +{"Finetune/Learning Rate": 8.584744900560149e-07, "Finetune/Loss": 0.8322618007659912, "Finetune/Loss (Raw)": 0.8472668528556824, "Finetune/Step": 4528, "Finetune/Step Time": 5.347232868894935} +{"Finetune/Learning Rate": 8.559509646462693e-07, "Finetune/Loss": 0.8368251323699951, "Finetune/Loss (Raw)": 0.917041003704071, "Finetune/Step": 4529, "Finetune/Step Time": 5.319693868979812} +{"Finetune/Learning Rate": 8.534309878634317e-07, "Finetune/Loss": 0.8379708528518677, "Finetune/Loss (Raw)": 0.9618011713027954, "Finetune/Step": 4530, "Finetune/Step Time": 5.302888914942741} +{"Finetune/Learning Rate": 8.509145606854552e-07, "Finetune/Loss": 0.8385624885559082, "Finetune/Loss (Raw)": 1.0013492107391357, "Finetune/Step": 4531, "Finetune/Step Time": 5.310056950896978} +{"Finetune/Learning Rate": 8.484016840889176e-07, "Finetune/Loss": 0.8357983827590942, "Finetune/Loss (Raw)": 0.6629300117492676, "Finetune/Step": 4532, "Finetune/Step Time": 5.3198161367326975} +{"Finetune/Learning Rate": 8.458923590490165e-07, "Finetune/Loss": 0.8363287448883057, "Finetune/Loss (Raw)": 0.8890697360038757, "Finetune/Step": 4533, "Finetune/Step Time": 5.30685062892735} +{"Finetune/Learning Rate": 8.433865865395718e-07, "Finetune/Loss": 0.8330227732658386, "Finetune/Loss (Raw)": 0.33539995551109314, "Finetune/Step": 4534, "Finetune/Step Time": 5.338459607213736} +{"Finetune/Learning Rate": 8.408843675330225e-07, "Finetune/Loss": 0.8328271508216858, "Finetune/Loss (Raw)": 0.7549121379852295, "Finetune/Step": 4535, "Finetune/Step Time": 5.3466205801814795} +{"Finetune/Learning Rate": 8.383857030004361e-07, "Finetune/Loss": 0.8316476345062256, "Finetune/Loss (Raw)": 0.8684912919998169, "Finetune/Step": 4536, "Finetune/Step Time": 5.356932764872909} +{"Finetune/Learning Rate": 8.358905939114948e-07, "Finetune/Loss": 0.8320265412330627, "Finetune/Loss (Raw)": 0.8707628846168518, "Finetune/Step": 4537, "Finetune/Step Time": 5.3547445591539145} +{"Finetune/Learning Rate": 8.333990412344961e-07, "Finetune/Loss": 0.8321020603179932, "Finetune/Loss (Raw)": 0.8981781601905823, "Finetune/Step": 4538, "Finetune/Step Time": 5.359607053920627} +{"Finetune/Learning Rate": 8.30911045936369e-07, "Finetune/Loss": 0.83152174949646, "Finetune/Loss (Raw)": 0.7719731330871582, "Finetune/Step": 4539, "Finetune/Step Time": 5.366768095642328} +{"Finetune/Learning Rate": 8.284266089826531e-07, "Finetune/Loss": 0.8275380730628967, "Finetune/Loss (Raw)": 0.3098656237125397, "Finetune/Step": 4540, "Finetune/Step Time": 5.398536842316389} +{"Finetune/Learning Rate": 8.259457313375097e-07, "Finetune/Loss": 0.8250188827514648, "Finetune/Loss (Raw)": 0.7660147547721863, "Finetune/Step": 4541, "Finetune/Step Time": 5.420736480504274} +{"Finetune/Learning Rate": 8.234684139637205e-07, "Finetune/Loss": 0.8245600461959839, "Finetune/Loss (Raw)": 0.7503587007522583, "Finetune/Step": 4542, "Finetune/Step Time": 5.416720815002918} +{"Finetune/Learning Rate": 8.209946578226836e-07, "Finetune/Loss": 0.823782742023468, "Finetune/Loss (Raw)": 0.8457006812095642, "Finetune/Step": 4543, "Finetune/Step Time": 5.410607537254691} +{"Finetune/Learning Rate": 8.185244638744139e-07, "Finetune/Loss": 0.8248209953308105, "Finetune/Loss (Raw)": 0.9124128222465515, "Finetune/Step": 4544, "Finetune/Step Time": 5.402222916483879} +{"Finetune/Learning Rate": 8.160578330775459e-07, "Finetune/Loss": 0.8245691657066345, "Finetune/Loss (Raw)": 0.9298310279846191, "Finetune/Step": 4545, "Finetune/Step Time": 5.4193648211658} +{"Finetune/Learning Rate": 8.135947663893329e-07, "Finetune/Loss": 0.825266420841217, "Finetune/Loss (Raw)": 0.9436054825782776, "Finetune/Step": 4546, "Finetune/Step Time": 5.41848954372108} +{"Finetune/Learning Rate": 8.111352647656423e-07, "Finetune/Loss": 0.8252202272415161, "Finetune/Loss (Raw)": 0.926880419254303, "Finetune/Step": 4547, "Finetune/Step Time": 5.41245854459703} +{"Finetune/Learning Rate": 8.086793291609562e-07, "Finetune/Loss": 0.8237744569778442, "Finetune/Loss (Raw)": 0.5796377658843994, "Finetune/Step": 4548, "Finetune/Step Time": 5.458223739638925} +{"Finetune/Learning Rate": 8.062269605283779e-07, "Finetune/Loss": 0.8229334950447083, "Finetune/Loss (Raw)": 0.7303652167320251, "Finetune/Step": 4549, "Finetune/Step Time": 5.459047622978687} +{"Finetune/Learning Rate": 8.037781598196225e-07, "Finetune/Loss": 0.8236574530601501, "Finetune/Loss (Raw)": 0.9910517930984497, "Finetune/Step": 4550, "Finetune/Step Time": 5.456363199278712} +{"Finetune/Learning Rate": 8.013329279850235e-07, "Finetune/Loss": 0.8245984315872192, "Finetune/Loss (Raw)": 0.9082473516464233, "Finetune/Step": 4551, "Finetune/Step Time": 5.453688140958548} +{"Finetune/Learning Rate": 7.988912659735249e-07, "Finetune/Loss": 0.8274499773979187, "Finetune/Loss (Raw)": 0.6960822939872742, "Finetune/Step": 4552, "Finetune/Step Time": 5.43386959284544} +{"Finetune/Learning Rate": 7.964531747326898e-07, "Finetune/Loss": 0.8282653093338013, "Finetune/Loss (Raw)": 0.9269366264343262, "Finetune/Step": 4553, "Finetune/Step Time": 5.4140613824129105} +{"Finetune/Learning Rate": 7.940186552086937e-07, "Finetune/Loss": 0.8292591571807861, "Finetune/Loss (Raw)": 1.0420671701431274, "Finetune/Step": 4554, "Finetune/Step Time": 5.412278134375811} +{"Finetune/Learning Rate": 7.915877083463242e-07, "Finetune/Loss": 0.8299605250358582, "Finetune/Loss (Raw)": 0.875085711479187, "Finetune/Step": 4555, "Finetune/Step Time": 5.413612756878138} +{"Finetune/Learning Rate": 7.891603350889888e-07, "Finetune/Loss": 0.8312578201293945, "Finetune/Loss (Raw)": 1.0670596361160278, "Finetune/Step": 4556, "Finetune/Step Time": 5.410578399896622} +{"Finetune/Learning Rate": 7.867365363787005e-07, "Finetune/Loss": 0.8305206298828125, "Finetune/Loss (Raw)": 0.7799524068832397, "Finetune/Step": 4557, "Finetune/Step Time": 5.413746131584048} +{"Finetune/Learning Rate": 7.843163131560871e-07, "Finetune/Loss": 0.8302145004272461, "Finetune/Loss (Raw)": 0.6562675833702087, "Finetune/Step": 4558, "Finetune/Step Time": 5.386581141501665} +{"Finetune/Learning Rate": 7.818996663603917e-07, "Finetune/Loss": 0.8309800028800964, "Finetune/Loss (Raw)": 0.9719207882881165, "Finetune/Step": 4559, "Finetune/Step Time": 5.381819762289524} +{"Finetune/Learning Rate": 7.794865969294685e-07, "Finetune/Loss": 0.8306471109390259, "Finetune/Loss (Raw)": 0.8419480323791504, "Finetune/Step": 4560, "Finetune/Step Time": 5.3777032401412725} +{"Finetune/Learning Rate": 7.770771057997828e-07, "Finetune/Loss": 0.8320978879928589, "Finetune/Loss (Raw)": 0.9050611853599548, "Finetune/Step": 4561, "Finetune/Step Time": 5.361221820116043} +{"Finetune/Learning Rate": 7.746711939064066e-07, "Finetune/Loss": 0.8319178819656372, "Finetune/Loss (Raw)": 0.9423236846923828, "Finetune/Step": 4562, "Finetune/Step Time": 5.362221645191312} +{"Finetune/Learning Rate": 7.722688621830333e-07, "Finetune/Loss": 0.8274796009063721, "Finetune/Loss (Raw)": 0.3051488697528839, "Finetune/Step": 4563, "Finetune/Step Time": 5.391621207818389} +{"Finetune/Learning Rate": 7.698701115619578e-07, "Finetune/Loss": 0.8282443284988403, "Finetune/Loss (Raw)": 0.8622145056724548, "Finetune/Step": 4564, "Finetune/Step Time": 5.380465652793646} +{"Finetune/Learning Rate": 7.674749429740868e-07, "Finetune/Loss": 0.8279843330383301, "Finetune/Loss (Raw)": 0.7906190752983093, "Finetune/Step": 4565, "Finetune/Step Time": 5.37529543787241} +{"Finetune/Learning Rate": 7.650833573489436e-07, "Finetune/Loss": 0.8276354074478149, "Finetune/Loss (Raw)": 0.8351283073425293, "Finetune/Step": 4566, "Finetune/Step Time": 5.3780333790928125} +{"Finetune/Learning Rate": 7.626953556146521e-07, "Finetune/Loss": 0.8271248936653137, "Finetune/Loss (Raw)": 0.9702231287956238, "Finetune/Step": 4567, "Finetune/Step Time": 5.376024218276143} +{"Finetune/Learning Rate": 7.603109386979501e-07, "Finetune/Loss": 0.826446533203125, "Finetune/Loss (Raw)": 0.9236754775047302, "Finetune/Step": 4568, "Finetune/Step Time": 5.373683946207166} +{"Finetune/Learning Rate": 7.579301075241808e-07, "Finetune/Loss": 0.8266817331314087, "Finetune/Loss (Raw)": 1.0107972621917725, "Finetune/Step": 4569, "Finetune/Step Time": 5.372085018083453} +{"Finetune/Learning Rate": 7.555528630173014e-07, "Finetune/Loss": 0.8265783786773682, "Finetune/Loss (Raw)": 0.9332596659660339, "Finetune/Step": 4570, "Finetune/Step Time": 5.358209386467934} +{"Finetune/Learning Rate": 7.531792060998766e-07, "Finetune/Loss": 0.8265437483787537, "Finetune/Loss (Raw)": 0.8356744647026062, "Finetune/Step": 4571, "Finetune/Step Time": 5.376099087297916} +{"Finetune/Learning Rate": 7.508091376930682e-07, "Finetune/Loss": 0.8268448114395142, "Finetune/Loss (Raw)": 0.851992666721344, "Finetune/Step": 4572, "Finetune/Step Time": 5.38135102391243} +{"Finetune/Learning Rate": 7.484426587166615e-07, "Finetune/Loss": 0.8274655938148499, "Finetune/Loss (Raw)": 0.811292290687561, "Finetune/Step": 4573, "Finetune/Step Time": 5.391697393730283} +{"Finetune/Learning Rate": 7.460797700890365e-07, "Finetune/Loss": 0.8268665075302124, "Finetune/Loss (Raw)": 0.8975057601928711, "Finetune/Step": 4574, "Finetune/Step Time": 5.3906982727348804} +{"Finetune/Learning Rate": 7.437204727271852e-07, "Finetune/Loss": 0.82663893699646, "Finetune/Loss (Raw)": 0.8155609369277954, "Finetune/Step": 4575, "Finetune/Step Time": 5.404992438852787} +{"Finetune/Learning Rate": 7.41364767546705e-07, "Finetune/Loss": 0.8260196447372437, "Finetune/Loss (Raw)": 0.7933198809623718, "Finetune/Step": 4576, "Finetune/Step Time": 5.418142566457391} +{"Finetune/Learning Rate": 7.390126554617982e-07, "Finetune/Loss": 0.825901985168457, "Finetune/Loss (Raw)": 0.9955775141716003, "Finetune/Step": 4577, "Finetune/Step Time": 5.426238330081105} +{"Finetune/Learning Rate": 7.366641373852745e-07, "Finetune/Loss": 0.8273317813873291, "Finetune/Loss (Raw)": 0.9587185978889465, "Finetune/Step": 4578, "Finetune/Step Time": 5.429741336032748} +{"Finetune/Learning Rate": 7.343192142285471e-07, "Finetune/Loss": 0.8245189189910889, "Finetune/Loss (Raw)": 0.41358956694602966, "Finetune/Step": 4579, "Finetune/Step Time": 5.449587173759937} +{"Finetune/Learning Rate": 7.319778869016358e-07, "Finetune/Loss": 0.8238847255706787, "Finetune/Loss (Raw)": 0.2661250829696655, "Finetune/Step": 4580, "Finetune/Step Time": 5.450264390558004} +{"Finetune/Learning Rate": 7.296401563131661e-07, "Finetune/Loss": 0.8246138095855713, "Finetune/Loss (Raw)": 0.8498136401176453, "Finetune/Step": 4581, "Finetune/Step Time": 5.454260403290391} +{"Finetune/Learning Rate": 7.273060233703599e-07, "Finetune/Loss": 0.8248553276062012, "Finetune/Loss (Raw)": 0.903504490852356, "Finetune/Step": 4582, "Finetune/Step Time": 5.457521364092827} +{"Finetune/Learning Rate": 7.249754889790539e-07, "Finetune/Loss": 0.8266090154647827, "Finetune/Loss (Raw)": 0.9683097004890442, "Finetune/Step": 4583, "Finetune/Step Time": 5.4440216440707445} +{"Finetune/Learning Rate": 7.2264855404368e-07, "Finetune/Loss": 0.826348066329956, "Finetune/Loss (Raw)": 0.7836500406265259, "Finetune/Step": 4584, "Finetune/Step Time": 5.450335146859288} +{"Finetune/Learning Rate": 7.203252194672783e-07, "Finetune/Loss": 0.8259614706039429, "Finetune/Loss (Raw)": 0.7432986497879028, "Finetune/Step": 4585, "Finetune/Step Time": 5.438618021085858} +{"Finetune/Learning Rate": 7.180054861514885e-07, "Finetune/Loss": 0.8252806663513184, "Finetune/Loss (Raw)": 0.7551460862159729, "Finetune/Step": 4586, "Finetune/Step Time": 5.4302103985100985} +{"Finetune/Learning Rate": 7.156893549965538e-07, "Finetune/Loss": 0.8257927894592285, "Finetune/Loss (Raw)": 0.9088143706321716, "Finetune/Step": 4587, "Finetune/Step Time": 5.426265442743897} +{"Finetune/Learning Rate": 7.133768269013186e-07, "Finetune/Loss": 0.8263767957687378, "Finetune/Loss (Raw)": 1.0337541103363037, "Finetune/Step": 4588, "Finetune/Step Time": 5.449892232194543} +{"Finetune/Learning Rate": 7.110679027632295e-07, "Finetune/Loss": 0.8267228603363037, "Finetune/Loss (Raw)": 0.8415345549583435, "Finetune/Step": 4589, "Finetune/Step Time": 5.449876639991999} +{"Finetune/Learning Rate": 7.087625834783385e-07, "Finetune/Loss": 0.8259156942367554, "Finetune/Loss (Raw)": 0.8170002102851868, "Finetune/Step": 4590, "Finetune/Step Time": 5.439986852928996} +{"Finetune/Learning Rate": 7.064608699412944e-07, "Finetune/Loss": 0.8278888463973999, "Finetune/Loss (Raw)": 0.9133301377296448, "Finetune/Step": 4591, "Finetune/Step Time": 5.42411139793694} +{"Finetune/Learning Rate": 7.041627630453418e-07, "Finetune/Loss": 0.8234718441963196, "Finetune/Loss (Raw)": 0.2925805449485779, "Finetune/Step": 4592, "Finetune/Step Time": 5.43648243509233} +{"Finetune/Learning Rate": 7.018682636823371e-07, "Finetune/Loss": 0.8243761658668518, "Finetune/Loss (Raw)": 0.9174685478210449, "Finetune/Step": 4593, "Finetune/Step Time": 5.440823711454868} +{"Finetune/Learning Rate": 6.99577372742728e-07, "Finetune/Loss": 0.8210633397102356, "Finetune/Loss (Raw)": 0.41243305802345276, "Finetune/Step": 4594, "Finetune/Step Time": 5.47115452773869} +{"Finetune/Learning Rate": 6.972900911155656e-07, "Finetune/Loss": 0.821179986000061, "Finetune/Loss (Raw)": 0.9266236424446106, "Finetune/Step": 4595, "Finetune/Step Time": 5.473001390695572} +{"Finetune/Learning Rate": 6.950064196884987e-07, "Finetune/Loss": 0.8218609094619751, "Finetune/Loss (Raw)": 0.9428548216819763, "Finetune/Step": 4596, "Finetune/Step Time": 5.475307131186128} +{"Finetune/Learning Rate": 6.927263593477751e-07, "Finetune/Loss": 0.8211486339569092, "Finetune/Loss (Raw)": 0.7583096027374268, "Finetune/Step": 4597, "Finetune/Step Time": 5.477472702041268} +{"Finetune/Learning Rate": 6.904499109782426e-07, "Finetune/Loss": 0.8213252425193787, "Finetune/Loss (Raw)": 0.8685422539710999, "Finetune/Step": 4598, "Finetune/Step Time": 5.475077632814646} +{"Finetune/Learning Rate": 6.881770754633444e-07, "Finetune/Loss": 0.8214025497436523, "Finetune/Loss (Raw)": 0.8057193160057068, "Finetune/Step": 4599, "Finetune/Step Time": 5.469086639583111} +{"Finetune/Learning Rate": 6.859078536851283e-07, "Finetune/Loss": 0.8210841417312622, "Finetune/Loss (Raw)": 0.8358293175697327, "Finetune/Step": 4600, "Finetune/Step Time": 5.47786377184093} +{"Finetune/Learning Rate": 6.836422465242298e-07, "Finetune/Loss": 0.818569540977478, "Finetune/Loss (Raw)": 0.7235845327377319, "Finetune/Step": 4601, "Finetune/Step Time": 5.482909644022584} +{"Finetune/Learning Rate": 6.81380254859888e-07, "Finetune/Loss": 0.8195661306381226, "Finetune/Loss (Raw)": 1.0363150835037231, "Finetune/Step": 4602, "Finetune/Step Time": 5.48473952151835} +{"Finetune/Learning Rate": 6.791218795699406e-07, "Finetune/Loss": 0.8205864429473877, "Finetune/Loss (Raw)": 0.9102721810340881, "Finetune/Step": 4603, "Finetune/Step Time": 5.477362914010882} +{"Finetune/Learning Rate": 6.768671215308165e-07, "Finetune/Loss": 0.819908857345581, "Finetune/Loss (Raw)": 0.841773509979248, "Finetune/Step": 4604, "Finetune/Step Time": 5.471325011923909} +{"Finetune/Learning Rate": 6.746159816175435e-07, "Finetune/Loss": 0.8162503242492676, "Finetune/Loss (Raw)": 0.33015120029449463, "Finetune/Step": 4605, "Finetune/Step Time": 5.501200743019581} +{"Finetune/Learning Rate": 6.723684607037451e-07, "Finetune/Loss": 0.8158169984817505, "Finetune/Loss (Raw)": 0.8888904452323914, "Finetune/Step": 4606, "Finetune/Step Time": 5.508410319685936} +{"Finetune/Learning Rate": 6.701245596616402e-07, "Finetune/Loss": 0.8153557181358337, "Finetune/Loss (Raw)": 0.8223593235015869, "Finetune/Step": 4607, "Finetune/Step Time": 5.500657329335809} +{"Finetune/Learning Rate": 6.678842793620433e-07, "Finetune/Loss": 0.8154412508010864, "Finetune/Loss (Raw)": 0.8495920300483704, "Finetune/Step": 4608, "Finetune/Step Time": 5.496569260954857} +{"Finetune/Learning Rate": 6.656476206743623e-07, "Finetune/Loss": 0.8169784545898438, "Finetune/Loss (Raw)": 1.0447359085083008, "Finetune/Step": 4609, "Finetune/Step Time": 5.4949867855757475} +{"Finetune/Learning Rate": 6.634145844666051e-07, "Finetune/Loss": 0.817052960395813, "Finetune/Loss (Raw)": 0.8243145942687988, "Finetune/Step": 4610, "Finetune/Step Time": 5.47729392722249} +{"Finetune/Learning Rate": 6.611851716053641e-07, "Finetune/Loss": 0.8125579357147217, "Finetune/Loss (Raw)": 0.28638964891433716, "Finetune/Step": 4611, "Finetune/Step Time": 5.507822412997484} +{"Finetune/Learning Rate": 6.589593829558316e-07, "Finetune/Loss": 0.8123594522476196, "Finetune/Loss (Raw)": 0.8839273452758789, "Finetune/Step": 4612, "Finetune/Step Time": 5.489341916516423} +{"Finetune/Learning Rate": 6.567372193817967e-07, "Finetune/Loss": 0.8121259808540344, "Finetune/Loss (Raw)": 0.8198530077934265, "Finetune/Step": 4613, "Finetune/Step Time": 5.5140887927263975} +{"Finetune/Learning Rate": 6.545186817456339e-07, "Finetune/Loss": 0.811313271522522, "Finetune/Loss (Raw)": 0.7663212418556213, "Finetune/Step": 4614, "Finetune/Step Time": 5.520087255164981} +{"Finetune/Learning Rate": 6.523037709083158e-07, "Finetune/Loss": 0.8163167238235474, "Finetune/Loss (Raw)": 0.9649054408073425, "Finetune/Step": 4615, "Finetune/Step Time": 5.488872962072492} +{"Finetune/Learning Rate": 6.500924877294057e-07, "Finetune/Loss": 0.8153281211853027, "Finetune/Loss (Raw)": 0.8272088766098022, "Finetune/Step": 4616, "Finetune/Step Time": 5.481313263997436} +{"Finetune/Learning Rate": 6.478848330670607e-07, "Finetune/Loss": 0.8160281181335449, "Finetune/Loss (Raw)": 0.7707116603851318, "Finetune/Step": 4617, "Finetune/Step Time": 5.473283460363746} +{"Finetune/Learning Rate": 6.456808077780263e-07, "Finetune/Loss": 0.8170778751373291, "Finetune/Loss (Raw)": 0.880044162273407, "Finetune/Step": 4618, "Finetune/Step Time": 5.464877635240555} +{"Finetune/Learning Rate": 6.434804127176409e-07, "Finetune/Loss": 0.818854808807373, "Finetune/Loss (Raw)": 1.0176215171813965, "Finetune/Step": 4619, "Finetune/Step Time": 5.467056343331933} +{"Finetune/Learning Rate": 6.4128364873984e-07, "Finetune/Loss": 0.8182061910629272, "Finetune/Loss (Raw)": 0.8305755853652954, "Finetune/Step": 4620, "Finetune/Step Time": 5.471388824284077} +{"Finetune/Learning Rate": 6.390905166971406e-07, "Finetune/Loss": 0.8195946216583252, "Finetune/Loss (Raw)": 0.9981454014778137, "Finetune/Step": 4621, "Finetune/Step Time": 5.461083820089698} +{"Finetune/Learning Rate": 6.369010174406531e-07, "Finetune/Loss": 0.8209846019744873, "Finetune/Loss (Raw)": 0.8845159411430359, "Finetune/Step": 4622, "Finetune/Step Time": 5.457819908857346} +{"Finetune/Learning Rate": 6.347151518200834e-07, "Finetune/Loss": 0.8215683698654175, "Finetune/Loss (Raw)": 0.9757586121559143, "Finetune/Step": 4623, "Finetune/Step Time": 5.4552516136318445} +{"Finetune/Learning Rate": 6.325329206837217e-07, "Finetune/Loss": 0.8221331238746643, "Finetune/Loss (Raw)": 0.8466790914535522, "Finetune/Step": 4624, "Finetune/Step Time": 5.448523309081793} +{"Finetune/Learning Rate": 6.303543248784483e-07, "Finetune/Loss": 0.8217875361442566, "Finetune/Loss (Raw)": 0.8969144821166992, "Finetune/Step": 4625, "Finetune/Step Time": 5.443811455741525} +{"Finetune/Learning Rate": 6.281793652497359e-07, "Finetune/Loss": 0.8184893727302551, "Finetune/Loss (Raw)": 0.34624090790748596, "Finetune/Step": 4626, "Finetune/Step Time": 5.469584185630083} +{"Finetune/Learning Rate": 6.260080426416415e-07, "Finetune/Loss": 0.8180242776870728, "Finetune/Loss (Raw)": 0.8872990012168884, "Finetune/Step": 4627, "Finetune/Step Time": 5.4795465506613255} +{"Finetune/Learning Rate": 6.238403578968144e-07, "Finetune/Loss": 0.8174866437911987, "Finetune/Loss (Raw)": 0.7978578209877014, "Finetune/Step": 4628, "Finetune/Step Time": 5.4830583818256855} +{"Finetune/Learning Rate": 6.216763118564883e-07, "Finetune/Loss": 0.8167502880096436, "Finetune/Loss (Raw)": 0.8555999398231506, "Finetune/Step": 4629, "Finetune/Step Time": 5.480759397149086} +{"Finetune/Learning Rate": 6.195159053604926e-07, "Finetune/Loss": 0.8173297643661499, "Finetune/Loss (Raw)": 0.7966445088386536, "Finetune/Step": 4630, "Finetune/Step Time": 5.488643378019333} +{"Finetune/Learning Rate": 6.173591392472334e-07, "Finetune/Loss": 0.8155484795570374, "Finetune/Loss (Raw)": 0.7170372009277344, "Finetune/Step": 4631, "Finetune/Step Time": 5.50767607241869} +{"Finetune/Learning Rate": 6.152060143537109e-07, "Finetune/Loss": 0.811031699180603, "Finetune/Loss (Raw)": 0.2680806815624237, "Finetune/Step": 4632, "Finetune/Step Time": 5.533092584460974} +{"Finetune/Learning Rate": 6.130565315155124e-07, "Finetune/Loss": 0.8110988736152649, "Finetune/Loss (Raw)": 0.8305352330207825, "Finetune/Step": 4633, "Finetune/Step Time": 5.545029204338789} +{"Finetune/Learning Rate": 6.109106915668106e-07, "Finetune/Loss": 0.8103153109550476, "Finetune/Loss (Raw)": 0.9082974195480347, "Finetune/Step": 4634, "Finetune/Step Time": 5.55792829580605} +{"Finetune/Learning Rate": 6.087684953403616e-07, "Finetune/Loss": 0.8131885528564453, "Finetune/Loss (Raw)": 0.6953038573265076, "Finetune/Step": 4635, "Finetune/Step Time": 5.546516180038452} +{"Finetune/Learning Rate": 6.066299436675117e-07, "Finetune/Loss": 0.8130413293838501, "Finetune/Loss (Raw)": 0.8534866571426392, "Finetune/Step": 4636, "Finetune/Step Time": 5.543244360014796} +{"Finetune/Learning Rate": 6.04495037378191e-07, "Finetune/Loss": 0.8139138221740723, "Finetune/Loss (Raw)": 0.8993526697158813, "Finetune/Step": 4637, "Finetune/Step Time": 5.537633342668414} +{"Finetune/Learning Rate": 6.023637773009139e-07, "Finetune/Loss": 0.8149002194404602, "Finetune/Loss (Raw)": 1.0910462141036987, "Finetune/Step": 4638, "Finetune/Step Time": 5.541249517351389} +{"Finetune/Learning Rate": 6.002361642627807e-07, "Finetune/Loss": 0.8125818967819214, "Finetune/Loss (Raw)": 0.713504433631897, "Finetune/Step": 4639, "Finetune/Step Time": 5.566391950473189} +{"Finetune/Learning Rate": 5.981121990894789e-07, "Finetune/Loss": 0.8173496723175049, "Finetune/Loss (Raw)": 0.9693869352340698, "Finetune/Step": 4640, "Finetune/Step Time": 5.541896548122168} +{"Finetune/Learning Rate": 5.959918826052757e-07, "Finetune/Loss": 0.8175969123840332, "Finetune/Loss (Raw)": 0.8696432709693909, "Finetune/Step": 4641, "Finetune/Step Time": 5.540896702557802} +{"Finetune/Learning Rate": 5.938752156330241e-07, "Finetune/Loss": 0.8184446096420288, "Finetune/Loss (Raw)": 0.872471034526825, "Finetune/Step": 4642, "Finetune/Step Time": 5.536317102611065} +{"Finetune/Learning Rate": 5.917621989941624e-07, "Finetune/Loss": 0.8192384839057922, "Finetune/Loss (Raw)": 0.9409617185592651, "Finetune/Step": 4643, "Finetune/Step Time": 5.545088458806276} +{"Finetune/Learning Rate": 5.896528335087137e-07, "Finetune/Loss": 0.8161206245422363, "Finetune/Loss (Raw)": 0.3717074990272522, "Finetune/Step": 4644, "Finetune/Step Time": 5.577251901850104} +{"Finetune/Learning Rate": 5.875471199952754e-07, "Finetune/Loss": 0.8180717825889587, "Finetune/Loss (Raw)": 1.0003302097320557, "Finetune/Step": 4645, "Finetune/Step Time": 5.575159922242165} +{"Finetune/Learning Rate": 5.854450592710382e-07, "Finetune/Loss": 0.8161535263061523, "Finetune/Loss (Raw)": 0.763669490814209, "Finetune/Step": 4646, "Finetune/Step Time": 5.593188516795635} +{"Finetune/Learning Rate": 5.833466521517695e-07, "Finetune/Loss": 0.815818727016449, "Finetune/Loss (Raw)": 0.8980374932289124, "Finetune/Step": 4647, "Finetune/Step Time": 5.597317216917872} +{"Finetune/Learning Rate": 5.812518994518213e-07, "Finetune/Loss": 0.8157562017440796, "Finetune/Loss (Raw)": 0.9225932955741882, "Finetune/Step": 4648, "Finetune/Step Time": 5.597071530297399} +{"Finetune/Learning Rate": 5.791608019841244e-07, "Finetune/Loss": 0.8154624700546265, "Finetune/Loss (Raw)": 0.9529040455818176, "Finetune/Step": 4649, "Finetune/Step Time": 5.593325847759843} +{"Finetune/Learning Rate": 5.770733605601953e-07, "Finetune/Loss": 0.8150906562805176, "Finetune/Loss (Raw)": 0.7941998243331909, "Finetune/Step": 4650, "Finetune/Step Time": 5.601341946050525} +{"Finetune/Learning Rate": 5.74989575990127e-07, "Finetune/Loss": 0.8151007294654846, "Finetune/Loss (Raw)": 0.875283420085907, "Finetune/Step": 4651, "Finetune/Step Time": 5.594647327437997} +{"Finetune/Learning Rate": 5.729094490825948e-07, "Finetune/Loss": 0.8150249719619751, "Finetune/Loss (Raw)": 0.8441179394721985, "Finetune/Step": 4652, "Finetune/Step Time": 5.5929495599120855} +{"Finetune/Learning Rate": 5.708329806448587e-07, "Finetune/Loss": 0.8108768463134766, "Finetune/Loss (Raw)": 0.299331933259964, "Finetune/Step": 4653, "Finetune/Step Time": 5.621035886928439} +{"Finetune/Learning Rate": 5.687601714827551e-07, "Finetune/Loss": 0.8125057220458984, "Finetune/Loss (Raw)": 0.8772796988487244, "Finetune/Step": 4654, "Finetune/Step Time": 5.625570015981793} +{"Finetune/Learning Rate": 5.666910224006971e-07, "Finetune/Loss": 0.8138377666473389, "Finetune/Loss (Raw)": 0.7106735110282898, "Finetune/Step": 4655, "Finetune/Step Time": 5.609702551737428} +{"Finetune/Learning Rate": 5.646255342016848e-07, "Finetune/Loss": 0.8106067776679993, "Finetune/Loss (Raw)": 0.43370920419692993, "Finetune/Step": 4656, "Finetune/Step Time": 5.643190314993262} +{"Finetune/Learning Rate": 5.625637076872914e-07, "Finetune/Loss": 0.8100237846374512, "Finetune/Loss (Raw)": 0.8424133658409119, "Finetune/Step": 4657, "Finetune/Step Time": 5.64011456258595} +{"Finetune/Learning Rate": 5.605055436576746e-07, "Finetune/Loss": 0.8098679184913635, "Finetune/Loss (Raw)": 0.9418473839759827, "Finetune/Step": 4658, "Finetune/Step Time": 5.6361619010567665} +{"Finetune/Learning Rate": 5.584510429115641e-07, "Finetune/Loss": 0.8092502355575562, "Finetune/Loss (Raw)": 0.9222835898399353, "Finetune/Step": 4659, "Finetune/Step Time": 5.633163254708052} +{"Finetune/Learning Rate": 5.564002062462737e-07, "Finetune/Loss": 0.811445415019989, "Finetune/Loss (Raw)": 0.9439176321029663, "Finetune/Step": 4660, "Finetune/Step Time": 5.640219584107399} +{"Finetune/Learning Rate": 5.54353034457693e-07, "Finetune/Loss": 0.8107454776763916, "Finetune/Loss (Raw)": 0.7994787096977234, "Finetune/Step": 4661, "Finetune/Step Time": 5.641839276999235} +{"Finetune/Learning Rate": 5.523095283402868e-07, "Finetune/Loss": 0.8151493072509766, "Finetune/Loss (Raw)": 0.8990869522094727, "Finetune/Step": 4662, "Finetune/Step Time": 5.616042861714959} +{"Finetune/Learning Rate": 5.502696886871039e-07, "Finetune/Loss": 0.8164750337600708, "Finetune/Loss (Raw)": 0.9246026873588562, "Finetune/Step": 4663, "Finetune/Step Time": 5.604854626581073} +{"Finetune/Learning Rate": 5.482335162897657e-07, "Finetune/Loss": 0.8164499998092651, "Finetune/Loss (Raw)": 0.865289568901062, "Finetune/Step": 4664, "Finetune/Step Time": 5.592638734728098} +{"Finetune/Learning Rate": 5.462010119384665e-07, "Finetune/Loss": 0.8158879280090332, "Finetune/Loss (Raw)": 0.7988191246986389, "Finetune/Step": 4665, "Finetune/Step Time": 5.592495137825608} +{"Finetune/Learning Rate": 5.441721764219853e-07, "Finetune/Loss": 0.8145905137062073, "Finetune/Loss (Raw)": 0.7321075201034546, "Finetune/Step": 4666, "Finetune/Step Time": 5.606569346040487} +{"Finetune/Learning Rate": 5.42147010527675e-07, "Finetune/Loss": 0.8161656856536865, "Finetune/Loss (Raw)": 0.9735957384109497, "Finetune/Step": 4667, "Finetune/Step Time": 5.593747925013304} +{"Finetune/Learning Rate": 5.401255150414598e-07, "Finetune/Loss": 0.8203634023666382, "Finetune/Loss (Raw)": 0.8471797108650208, "Finetune/Step": 4668, "Finetune/Step Time": 5.572972176596522} +{"Finetune/Learning Rate": 5.381076907478433e-07, "Finetune/Loss": 0.8171058893203735, "Finetune/Loss (Raw)": 0.34904929995536804, "Finetune/Step": 4669, "Finetune/Step Time": 5.584025731310248} +{"Finetune/Learning Rate": 5.360935384299049e-07, "Finetune/Loss": 0.8182750940322876, "Finetune/Loss (Raw)": 0.9000189304351807, "Finetune/Step": 4670, "Finetune/Step Time": 5.5826751962304115} +{"Finetune/Learning Rate": 5.34083058869297e-07, "Finetune/Loss": 0.8180093765258789, "Finetune/Loss (Raw)": 0.8116850256919861, "Finetune/Step": 4671, "Finetune/Step Time": 5.57817729562521} +{"Finetune/Learning Rate": 5.320762528462453e-07, "Finetune/Loss": 0.8177746534347534, "Finetune/Loss (Raw)": 0.8823752403259277, "Finetune/Step": 4672, "Finetune/Step Time": 5.592337338253856} +{"Finetune/Learning Rate": 5.30073121139556e-07, "Finetune/Loss": 0.8170846104621887, "Finetune/Loss (Raw)": 0.8414969444274902, "Finetune/Step": 4673, "Finetune/Step Time": 5.576260035857558} +{"Finetune/Learning Rate": 5.28073664526606e-07, "Finetune/Loss": 0.8171426057815552, "Finetune/Loss (Raw)": 0.9510263800621033, "Finetune/Step": 4674, "Finetune/Step Time": 5.600276513025165} +{"Finetune/Learning Rate": 5.260778837833391e-07, "Finetune/Loss": 0.8168447613716125, "Finetune/Loss (Raw)": 0.8887606859207153, "Finetune/Step": 4675, "Finetune/Step Time": 5.601887272670865} +{"Finetune/Learning Rate": 5.240857796842846e-07, "Finetune/Loss": 0.8188995122909546, "Finetune/Loss (Raw)": 0.8426517844200134, "Finetune/Step": 4676, "Finetune/Step Time": 5.539688153192401} +{"Finetune/Learning Rate": 5.220973530025375e-07, "Finetune/Loss": 0.8203368186950684, "Finetune/Loss (Raw)": 0.9143330454826355, "Finetune/Step": 4677, "Finetune/Step Time": 5.529492469504476} +{"Finetune/Learning Rate": 5.201126045097671e-07, "Finetune/Loss": 0.8198562860488892, "Finetune/Loss (Raw)": 0.9295468926429749, "Finetune/Step": 4678, "Finetune/Step Time": 5.560233484953642} +{"Finetune/Learning Rate": 5.181315349762162e-07, "Finetune/Loss": 0.8185677528381348, "Finetune/Loss (Raw)": 0.7433133125305176, "Finetune/Step": 4679, "Finetune/Step Time": 5.559558877721429} +{"Finetune/Learning Rate": 5.161541451706975e-07, "Finetune/Loss": 0.8210621476173401, "Finetune/Loss (Raw)": 1.015364646911621, "Finetune/Step": 4680, "Finetune/Step Time": 5.551070369780064} +{"Finetune/Learning Rate": 5.141804358605995e-07, "Finetune/Loss": 0.8206785917282104, "Finetune/Loss (Raw)": 0.8778377771377563, "Finetune/Step": 4681, "Finetune/Step Time": 5.567918371409178} +{"Finetune/Learning Rate": 5.122104078118773e-07, "Finetune/Loss": 0.8195717334747314, "Finetune/Loss (Raw)": 0.900395929813385, "Finetune/Step": 4682, "Finetune/Step Time": 5.579460525885224} +{"Finetune/Learning Rate": 5.102440617890658e-07, "Finetune/Loss": 0.8199239373207092, "Finetune/Loss (Raw)": 0.920163631439209, "Finetune/Step": 4683, "Finetune/Step Time": 5.563835959881544} +{"Finetune/Learning Rate": 5.082813985552604e-07, "Finetune/Loss": 0.8189958333969116, "Finetune/Loss (Raw)": 0.9482662677764893, "Finetune/Step": 4684, "Finetune/Step Time": 5.562309358268976} +{"Finetune/Learning Rate": 5.063224188721327e-07, "Finetune/Loss": 0.8200997710227966, "Finetune/Loss (Raw)": 0.9212543964385986, "Finetune/Step": 4685, "Finetune/Step Time": 5.563989823684096} +{"Finetune/Learning Rate": 5.043671234999271e-07, "Finetune/Loss": 0.8207624554634094, "Finetune/Loss (Raw)": 0.7410909533500671, "Finetune/Step": 4686, "Finetune/Step Time": 5.562582390382886} +{"Finetune/Learning Rate": 5.02415513197454e-07, "Finetune/Loss": 0.820027232170105, "Finetune/Loss (Raw)": 0.8778034448623657, "Finetune/Step": 4687, "Finetune/Step Time": 5.5543402582407} +{"Finetune/Learning Rate": 5.004675887220978e-07, "Finetune/Loss": 0.8220577239990234, "Finetune/Loss (Raw)": 1.1018650531768799, "Finetune/Step": 4688, "Finetune/Step Time": 5.55164429731667} +{"Finetune/Learning Rate": 4.985233508298037e-07, "Finetune/Loss": 0.8227506875991821, "Finetune/Loss (Raw)": 0.9937503933906555, "Finetune/Step": 4689, "Finetune/Step Time": 5.543585961684585} +{"Finetune/Learning Rate": 4.965828002750994e-07, "Finetune/Loss": 0.8216083645820618, "Finetune/Loss (Raw)": 0.7961094975471497, "Finetune/Step": 4690, "Finetune/Step Time": 5.55716279707849} +{"Finetune/Learning Rate": 4.946459378110701e-07, "Finetune/Loss": 0.8268473744392395, "Finetune/Loss (Raw)": 0.9757435917854309, "Finetune/Step": 4691, "Finetune/Step Time": 5.533045474439859} +{"Finetune/Learning Rate": 4.927127641893759e-07, "Finetune/Loss": 0.8281004428863525, "Finetune/Loss (Raw)": 1.0226036310195923, "Finetune/Step": 4692, "Finetune/Step Time": 5.528300192207098} +{"Finetune/Learning Rate": 4.907832801602452e-07, "Finetune/Loss": 0.8290232419967651, "Finetune/Loss (Raw)": 0.9087415337562561, "Finetune/Step": 4693, "Finetune/Step Time": 5.539995286613703} +{"Finetune/Learning Rate": 4.888574864724715e-07, "Finetune/Loss": 0.8302445411682129, "Finetune/Loss (Raw)": 0.9914515018463135, "Finetune/Step": 4694, "Finetune/Step Time": 5.539782680571079} +{"Finetune/Learning Rate": 4.869353838734169e-07, "Finetune/Loss": 0.8304061889648438, "Finetune/Loss (Raw)": 0.9909100532531738, "Finetune/Step": 4695, "Finetune/Step Time": 5.5440408661961555} +{"Finetune/Learning Rate": 4.850169731090127e-07, "Finetune/Loss": 0.8293991684913635, "Finetune/Loss (Raw)": 0.7947800755500793, "Finetune/Step": 4696, "Finetune/Step Time": 5.555602218955755} +{"Finetune/Learning Rate": 4.831022549237574e-07, "Finetune/Loss": 0.828392744064331, "Finetune/Loss (Raw)": 0.8819695711135864, "Finetune/Step": 4697, "Finetune/Step Time": 5.544900996610522} +{"Finetune/Learning Rate": 4.811912300607169e-07, "Finetune/Loss": 0.8283836245536804, "Finetune/Loss (Raw)": 0.9321019649505615, "Finetune/Step": 4698, "Finetune/Step Time": 5.54886064492166} +{"Finetune/Learning Rate": 4.792838992615179e-07, "Finetune/Loss": 0.8294431567192078, "Finetune/Loss (Raw)": 0.9712961316108704, "Finetune/Step": 4699, "Finetune/Step Time": 5.544514687731862} +{"Finetune/Learning Rate": 4.773802632663627e-07, "Finetune/Loss": 0.8297224640846252, "Finetune/Loss (Raw)": 0.8877415060997009, "Finetune/Step": 4700, "Finetune/Step Time": 5.540151169523597} +{"Finetune/Learning Rate": 4.7548032281401367e-07, "Finetune/Loss": 0.8305333256721497, "Finetune/Loss (Raw)": 0.915081262588501, "Finetune/Step": 4701, "Finetune/Step Time": 5.5286706779152155} +{"Finetune/Learning Rate": 4.735840786418011e-07, "Finetune/Loss": 0.8300990462303162, "Finetune/Loss (Raw)": 0.8419210314750671, "Finetune/Step": 4702, "Finetune/Step Time": 5.536211742088199} +{"Finetune/Learning Rate": 4.716915314856196e-07, "Finetune/Loss": 0.831282377243042, "Finetune/Loss (Raw)": 0.9670289158821106, "Finetune/Step": 4703, "Finetune/Step Time": 5.51865485496819} +{"Finetune/Learning Rate": 4.698026820799306e-07, "Finetune/Loss": 0.8309100866317749, "Finetune/Loss (Raw)": 0.7456601858139038, "Finetune/Step": 4704, "Finetune/Step Time": 5.50708344578743} +{"Finetune/Learning Rate": 4.6791753115775884e-07, "Finetune/Loss": 0.8306776285171509, "Finetune/Loss (Raw)": 0.965825080871582, "Finetune/Step": 4705, "Finetune/Step Time": 5.502680163830519} +{"Finetune/Learning Rate": 4.660360794506946e-07, "Finetune/Loss": 0.8296360373497009, "Finetune/Loss (Raw)": 0.8253934979438782, "Finetune/Step": 4706, "Finetune/Step Time": 5.496622832491994} +{"Finetune/Learning Rate": 4.641583276888939e-07, "Finetune/Loss": 0.8322924375534058, "Finetune/Loss (Raw)": 0.7536067962646484, "Finetune/Step": 4707, "Finetune/Step Time": 5.467590149492025} +{"Finetune/Learning Rate": 4.622842766010749e-07, "Finetune/Loss": 0.8362300395965576, "Finetune/Loss (Raw)": 0.7701376676559448, "Finetune/Step": 4708, "Finetune/Step Time": 5.43695118278265} +{"Finetune/Learning Rate": 4.604139269145191e-07, "Finetune/Loss": 0.8353633880615234, "Finetune/Loss (Raw)": 0.7388927340507507, "Finetune/Step": 4709, "Finetune/Step Time": 5.441419685259461} +{"Finetune/Learning Rate": 4.585472793550738e-07, "Finetune/Loss": 0.8361432552337646, "Finetune/Loss (Raw)": 1.0033234357833862, "Finetune/Step": 4710, "Finetune/Step Time": 5.442730937153101} +{"Finetune/Learning Rate": 4.5668433464714835e-07, "Finetune/Loss": 0.8352499008178711, "Finetune/Loss (Raw)": 0.853961169719696, "Finetune/Step": 4711, "Finetune/Step Time": 5.443045096471906} +{"Finetune/Learning Rate": 4.548250935137144e-07, "Finetune/Loss": 0.8354285359382629, "Finetune/Loss (Raw)": 0.8065089583396912, "Finetune/Step": 4712, "Finetune/Step Time": 5.4502814542502165} +{"Finetune/Learning Rate": 4.5296955667630926e-07, "Finetune/Loss": 0.8370844125747681, "Finetune/Loss (Raw)": 0.955248236656189, "Finetune/Step": 4713, "Finetune/Step Time": 5.452472917735577} +{"Finetune/Learning Rate": 4.51117724855028e-07, "Finetune/Loss": 0.8376725912094116, "Finetune/Loss (Raw)": 0.8304356336593628, "Finetune/Step": 4714, "Finetune/Step Time": 5.45563293248415} +{"Finetune/Learning Rate": 4.492695987685314e-07, "Finetune/Loss": 0.8377217650413513, "Finetune/Loss (Raw)": 0.9151157736778259, "Finetune/Step": 4715, "Finetune/Step Time": 5.459359899163246} +{"Finetune/Learning Rate": 4.474251791340412e-07, "Finetune/Loss": 0.8363487720489502, "Finetune/Loss (Raw)": 0.8580158948898315, "Finetune/Step": 4716, "Finetune/Step Time": 5.435091741383076} +{"Finetune/Learning Rate": 4.4558446666734147e-07, "Finetune/Loss": 0.8367922902107239, "Finetune/Loss (Raw)": 0.898293673992157, "Finetune/Step": 4717, "Finetune/Step Time": 5.432503622025251} +{"Finetune/Learning Rate": 4.437474620827764e-07, "Finetune/Loss": 0.8355013132095337, "Finetune/Loss (Raw)": 0.651764988899231, "Finetune/Step": 4718, "Finetune/Step Time": 5.446077276021242} +{"Finetune/Learning Rate": 4.419141660932491e-07, "Finetune/Loss": 0.8343215584754944, "Finetune/Loss (Raw)": 0.76231449842453, "Finetune/Step": 4719, "Finetune/Step Time": 5.4456198289990425} +{"Finetune/Learning Rate": 4.4008457941022933e-07, "Finetune/Loss": 0.8367582559585571, "Finetune/Loss (Raw)": 0.6044756174087524, "Finetune/Step": 4720, "Finetune/Step Time": 5.450426459312439} +{"Finetune/Learning Rate": 4.382587027437435e-07, "Finetune/Loss": 0.835902214050293, "Finetune/Loss (Raw)": 0.8078972101211548, "Finetune/Step": 4721, "Finetune/Step Time": 5.455117084085941} +{"Finetune/Learning Rate": 4.364365368023771e-07, "Finetune/Loss": 0.8392871618270874, "Finetune/Loss (Raw)": 0.8457018733024597, "Finetune/Step": 4722, "Finetune/Step Time": 5.425570510327816} +{"Finetune/Learning Rate": 4.3461808229327773e-07, "Finetune/Loss": 0.8389135003089905, "Finetune/Loss (Raw)": 0.8788012862205505, "Finetune/Step": 4723, "Finetune/Step Time": 5.424592666327953} +{"Finetune/Learning Rate": 4.3280333992215206e-07, "Finetune/Loss": 0.8380743265151978, "Finetune/Loss (Raw)": 0.8354371190071106, "Finetune/Step": 4724, "Finetune/Step Time": 5.446180844679475} +{"Finetune/Learning Rate": 4.3099231039326785e-07, "Finetune/Loss": 0.8389037847518921, "Finetune/Loss (Raw)": 0.8644848465919495, "Finetune/Step": 4725, "Finetune/Step Time": 5.448452403768897} +{"Finetune/Learning Rate": 4.291849944094462e-07, "Finetune/Loss": 0.837982714176178, "Finetune/Loss (Raw)": 0.750644326210022, "Finetune/Step": 4726, "Finetune/Step Time": 5.450844418257475} +{"Finetune/Learning Rate": 4.273813926720771e-07, "Finetune/Loss": 0.8381932377815247, "Finetune/Loss (Raw)": 0.8326652646064758, "Finetune/Step": 4727, "Finetune/Step Time": 5.447429681196809} +{"Finetune/Learning Rate": 4.2558150588109747e-07, "Finetune/Loss": 0.838643491268158, "Finetune/Loss (Raw)": 0.8934572339057922, "Finetune/Step": 4728, "Finetune/Step Time": 5.44523636251688} +{"Finetune/Learning Rate": 4.237853347350107e-07, "Finetune/Loss": 0.8412555456161499, "Finetune/Loss (Raw)": 1.0579285621643066, "Finetune/Step": 4729, "Finetune/Step Time": 5.441791547462344} +{"Finetune/Learning Rate": 4.219928799308759e-07, "Finetune/Loss": 0.8400555849075317, "Finetune/Loss (Raw)": 0.8827248215675354, "Finetune/Step": 4730, "Finetune/Step Time": 5.442463969811797} +{"Finetune/Learning Rate": 4.2020414216431106e-07, "Finetune/Loss": 0.8390322923660278, "Finetune/Loss (Raw)": 0.7792887687683105, "Finetune/Step": 4731, "Finetune/Step Time": 5.441168008372188} +{"Finetune/Learning Rate": 4.184191221294875e-07, "Finetune/Loss": 0.8390921354293823, "Finetune/Loss (Raw)": 0.8494400978088379, "Finetune/Step": 4732, "Finetune/Step Time": 5.441077619791031} +{"Finetune/Learning Rate": 4.16637820519139e-07, "Finetune/Loss": 0.8436803817749023, "Finetune/Loss (Raw)": 0.91744464635849, "Finetune/Step": 4733, "Finetune/Step Time": 5.405211508274078} +{"Finetune/Learning Rate": 4.1486023802455253e-07, "Finetune/Loss": 0.8429242372512817, "Finetune/Loss (Raw)": 0.7920982241630554, "Finetune/Step": 4734, "Finetune/Step Time": 5.412124076858163} +{"Finetune/Learning Rate": 4.1308637533557294e-07, "Finetune/Loss": 0.8439755439758301, "Finetune/Loss (Raw)": 0.9569246768951416, "Finetune/Step": 4735, "Finetune/Step Time": 5.411995999515057} +{"Finetune/Learning Rate": 4.1131623314060175e-07, "Finetune/Loss": 0.8429273366928101, "Finetune/Loss (Raw)": 0.7154290676116943, "Finetune/Step": 4736, "Finetune/Step Time": 5.4216534197330475} +{"Finetune/Learning Rate": 4.095498121265995e-07, "Finetune/Loss": 0.8409616947174072, "Finetune/Loss (Raw)": 0.7931311130523682, "Finetune/Step": 4737, "Finetune/Step Time": 5.436344692483544} +{"Finetune/Learning Rate": 4.077871129790767e-07, "Finetune/Loss": 0.8422228097915649, "Finetune/Loss (Raw)": 0.9857392907142639, "Finetune/Step": 4738, "Finetune/Step Time": 5.4455807618796825} +{"Finetune/Learning Rate": 4.060281363821017e-07, "Finetune/Loss": 0.846137523651123, "Finetune/Loss (Raw)": 0.7874743938446045, "Finetune/Step": 4739, "Finetune/Step Time": 5.412827497348189} +{"Finetune/Learning Rate": 4.042728830183018e-07, "Finetune/Loss": 0.8456783294677734, "Finetune/Loss (Raw)": 0.8251450657844543, "Finetune/Step": 4740, "Finetune/Step Time": 5.416637329384685} +{"Finetune/Learning Rate": 4.025213535688544e-07, "Finetune/Loss": 0.8446739912033081, "Finetune/Loss (Raw)": 0.6912980079650879, "Finetune/Step": 4741, "Finetune/Step Time": 5.398566629737616} +{"Finetune/Learning Rate": 4.007735487134956e-07, "Finetune/Loss": 0.844782829284668, "Finetune/Loss (Raw)": 0.780253529548645, "Finetune/Step": 4742, "Finetune/Step Time": 5.409456552937627} +{"Finetune/Learning Rate": 3.99029469130513e-07, "Finetune/Loss": 0.8429534435272217, "Finetune/Loss (Raw)": 0.7307435870170593, "Finetune/Step": 4743, "Finetune/Step Time": 5.42511211335659} +{"Finetune/Learning Rate": 3.972891154967506e-07, "Finetune/Loss": 0.8443575501441956, "Finetune/Loss (Raw)": 1.0069308280944824, "Finetune/Step": 4744, "Finetune/Step Time": 5.42610527575016} +{"Finetune/Learning Rate": 3.95552488487605e-07, "Finetune/Loss": 0.8443889617919922, "Finetune/Loss (Raw)": 0.7747315168380737, "Finetune/Step": 4745, "Finetune/Step Time": 5.445336891338229} +{"Finetune/Learning Rate": 3.9381958877702684e-07, "Finetune/Loss": 0.8436836004257202, "Finetune/Loss (Raw)": 0.7897567749023438, "Finetune/Step": 4746, "Finetune/Step Time": 5.469497071579099} +{"Finetune/Learning Rate": 3.920904170375239e-07, "Finetune/Loss": 0.8437719345092773, "Finetune/Loss (Raw)": 1.0289287567138672, "Finetune/Step": 4747, "Finetune/Step Time": 5.468874309211969} +{"Finetune/Learning Rate": 3.903649739401494e-07, "Finetune/Loss": 0.844628095626831, "Finetune/Loss (Raw)": 0.9401748180389404, "Finetune/Step": 4748, "Finetune/Step Time": 5.464117657393217} +{"Finetune/Learning Rate": 3.8864326015451536e-07, "Finetune/Loss": 0.843413770198822, "Finetune/Loss (Raw)": 0.8427072167396545, "Finetune/Step": 4749, "Finetune/Step Time": 5.460608329623938} +{"Finetune/Learning Rate": 3.8692527634878853e-07, "Finetune/Loss": 0.8432778120040894, "Finetune/Loss (Raw)": 0.8671127557754517, "Finetune/Step": 4750, "Finetune/Step Time": 5.4632184356451035} +{"Finetune/Learning Rate": 3.8521102318968106e-07, "Finetune/Loss": 0.8425935506820679, "Finetune/Loss (Raw)": 0.8881663084030151, "Finetune/Step": 4751, "Finetune/Step Time": 5.4792639296501875} +{"Finetune/Learning Rate": 3.835005013424631e-07, "Finetune/Loss": 0.8430123329162598, "Finetune/Loss (Raw)": 0.9002922177314758, "Finetune/Step": 4752, "Finetune/Step Time": 5.482812602072954} +{"Finetune/Learning Rate": 3.8179371147095354e-07, "Finetune/Loss": 0.8420233726501465, "Finetune/Loss (Raw)": 0.7703198790550232, "Finetune/Step": 4753, "Finetune/Step Time": 5.474793802946806} +{"Finetune/Learning Rate": 3.80090654237526e-07, "Finetune/Loss": 0.8464929461479187, "Finetune/Loss (Raw)": 0.9183520078659058, "Finetune/Step": 4754, "Finetune/Step Time": 5.449204154312611} +{"Finetune/Learning Rate": 3.7839133030310174e-07, "Finetune/Loss": 0.8468412756919861, "Finetune/Loss (Raw)": 0.9318831562995911, "Finetune/Step": 4755, "Finetune/Step Time": 5.443673349916935} +{"Finetune/Learning Rate": 3.766957403271554e-07, "Finetune/Loss": 0.8474982976913452, "Finetune/Loss (Raw)": 0.8819620013237, "Finetune/Step": 4756, "Finetune/Step Time": 5.436476605013013} +{"Finetune/Learning Rate": 3.750038849677162e-07, "Finetune/Loss": 0.8476109504699707, "Finetune/Loss (Raw)": 0.8700144290924072, "Finetune/Step": 4757, "Finetune/Step Time": 5.435936952009797} +{"Finetune/Learning Rate": 3.7331576488135456e-07, "Finetune/Loss": 0.8480034470558167, "Finetune/Loss (Raw)": 0.8468825221061707, "Finetune/Step": 4758, "Finetune/Step Time": 5.425985729321837} +{"Finetune/Learning Rate": 3.716313807231986e-07, "Finetune/Loss": 0.8489252328872681, "Finetune/Loss (Raw)": 0.8350206613540649, "Finetune/Step": 4759, "Finetune/Step Time": 5.411695517599583} +{"Finetune/Learning Rate": 3.699507331469276e-07, "Finetune/Loss": 0.8536167144775391, "Finetune/Loss (Raw)": 0.8685970902442932, "Finetune/Step": 4760, "Finetune/Step Time": 5.383559808135033} +{"Finetune/Learning Rate": 3.682738228047655e-07, "Finetune/Loss": 0.8530893325805664, "Finetune/Loss (Raw)": 0.7630312442779541, "Finetune/Step": 4761, "Finetune/Step Time": 5.3920107036828995} +{"Finetune/Learning Rate": 3.666006503474895e-07, "Finetune/Loss": 0.8521959781646729, "Finetune/Loss (Raw)": 0.7939454913139343, "Finetune/Step": 4762, "Finetune/Step Time": 5.384851796552539} +{"Finetune/Learning Rate": 3.6493121642442454e-07, "Finetune/Loss": 0.853631854057312, "Finetune/Loss (Raw)": 0.8790974617004395, "Finetune/Step": 4763, "Finetune/Step Time": 5.375054396688938} +{"Finetune/Learning Rate": 3.6326552168344574e-07, "Finetune/Loss": 0.8535784482955933, "Finetune/Loss (Raw)": 0.8466493487358093, "Finetune/Step": 4764, "Finetune/Step Time": 5.3789145946502686} +{"Finetune/Learning Rate": 3.616035667709761e-07, "Finetune/Loss": 0.8526151180267334, "Finetune/Loss (Raw)": 0.7760478258132935, "Finetune/Step": 4765, "Finetune/Step Time": 5.378405066207051} +{"Finetune/Learning Rate": 3.5994535233198844e-07, "Finetune/Loss": 0.852134108543396, "Finetune/Loss (Raw)": 1.0294815301895142, "Finetune/Step": 4766, "Finetune/Step Time": 5.380212474614382} +{"Finetune/Learning Rate": 3.582908790100037e-07, "Finetune/Loss": 0.8534969687461853, "Finetune/Loss (Raw)": 0.8879448771476746, "Finetune/Step": 4767, "Finetune/Step Time": 5.3556382935494184} +{"Finetune/Learning Rate": 3.566401474470904e-07, "Finetune/Loss": 0.8522710204124451, "Finetune/Loss (Raw)": 0.8124693632125854, "Finetune/Step": 4768, "Finetune/Step Time": 5.359752299264073} +{"Finetune/Learning Rate": 3.5499315828386393e-07, "Finetune/Loss": 0.8518949747085571, "Finetune/Loss (Raw)": 0.8215017914772034, "Finetune/Step": 4769, "Finetune/Step Time": 5.353249138221145} +{"Finetune/Learning Rate": 3.533499121594919e-07, "Finetune/Loss": 0.8500497341156006, "Finetune/Loss (Raw)": 0.6362895965576172, "Finetune/Step": 4770, "Finetune/Step Time": 5.357235504314303} +{"Finetune/Learning Rate": 3.5171040971168414e-07, "Finetune/Loss": 0.8504647016525269, "Finetune/Loss (Raw)": 0.9940754771232605, "Finetune/Step": 4771, "Finetune/Step Time": 5.348846333101392} +{"Finetune/Learning Rate": 3.500746515766973e-07, "Finetune/Loss": 0.8535848259925842, "Finetune/Loss (Raw)": 0.7710812091827393, "Finetune/Step": 4772, "Finetune/Step Time": 5.3305725157260895} +{"Finetune/Learning Rate": 3.4844263838934023e-07, "Finetune/Loss": 0.8506344556808472, "Finetune/Loss (Raw)": 0.6226798892021179, "Finetune/Step": 4773, "Finetune/Step Time": 5.336394147947431} +{"Finetune/Learning Rate": 3.468143707829641e-07, "Finetune/Loss": 0.8515799045562744, "Finetune/Loss (Raw)": 0.8846924901008606, "Finetune/Step": 4774, "Finetune/Step Time": 5.32157076895237} +{"Finetune/Learning Rate": 3.45189849389469e-07, "Finetune/Loss": 0.8521186113357544, "Finetune/Loss (Raw)": 0.966988742351532, "Finetune/Step": 4775, "Finetune/Step Time": 5.319042442366481} +{"Finetune/Learning Rate": 3.4356907483929835e-07, "Finetune/Loss": 0.8511759042739868, "Finetune/Loss (Raw)": 0.8019241690635681, "Finetune/Step": 4776, "Finetune/Step Time": 5.336370874196291} +{"Finetune/Learning Rate": 3.419520477614435e-07, "Finetune/Loss": 0.8506916761398315, "Finetune/Loss (Raw)": 0.8909275531768799, "Finetune/Step": 4777, "Finetune/Step Time": 5.337474400177598} +{"Finetune/Learning Rate": 3.403387687834425e-07, "Finetune/Loss": 0.8467366099357605, "Finetune/Loss (Raw)": 0.28795623779296875, "Finetune/Step": 4778, "Finetune/Step Time": 5.358105791732669} +{"Finetune/Learning Rate": 3.3872923853137453e-07, "Finetune/Loss": 0.8478312492370605, "Finetune/Loss (Raw)": 1.0153900384902954, "Finetune/Step": 4779, "Finetune/Step Time": 5.359210509806871} +{"Finetune/Learning Rate": 3.371234576298699e-07, "Finetune/Loss": 0.8477638363838196, "Finetune/Loss (Raw)": 0.8354878425598145, "Finetune/Step": 4780, "Finetune/Step Time": 5.364157764241099} +{"Finetune/Learning Rate": 3.355214267021012e-07, "Finetune/Loss": 0.8517065048217773, "Finetune/Loss (Raw)": 0.8039939403533936, "Finetune/Step": 4781, "Finetune/Step Time": 5.341542452573776} +{"Finetune/Learning Rate": 3.3392314636978117e-07, "Finetune/Loss": 0.8513182401657104, "Finetune/Loss (Raw)": 0.8275830149650574, "Finetune/Step": 4782, "Finetune/Step Time": 5.343508770689368} +{"Finetune/Learning Rate": 3.323286172531748e-07, "Finetune/Loss": 0.8486204743385315, "Finetune/Loss (Raw)": 0.3653619885444641, "Finetune/Step": 4783, "Finetune/Step Time": 5.369709528982639} +{"Finetune/Learning Rate": 3.30737839971087e-07, "Finetune/Loss": 0.8521273136138916, "Finetune/Loss (Raw)": 0.8825861215591431, "Finetune/Step": 4784, "Finetune/Step Time": 5.33704848587513} +{"Finetune/Learning Rate": 3.2915081514086624e-07, "Finetune/Loss": 0.8536877632141113, "Finetune/Loss (Raw)": 1.0421481132507324, "Finetune/Step": 4785, "Finetune/Step Time": 5.339092332869768} +{"Finetune/Learning Rate": 3.2756754337840757e-07, "Finetune/Loss": 0.8533375859260559, "Finetune/Loss (Raw)": 0.897026002407074, "Finetune/Step": 4786, "Finetune/Step Time": 5.341442238539457} +{"Finetune/Learning Rate": 3.2598802529814734e-07, "Finetune/Loss": 0.8527743220329285, "Finetune/Loss (Raw)": 0.8501870632171631, "Finetune/Step": 4787, "Finetune/Step Time": 5.336683074012399} +{"Finetune/Learning Rate": 3.2441226151306403e-07, "Finetune/Loss": 0.8518603444099426, "Finetune/Loss (Raw)": 0.826921284198761, "Finetune/Step": 4788, "Finetune/Step Time": 5.345647877082229} +{"Finetune/Learning Rate": 3.228402526346819e-07, "Finetune/Loss": 0.8511989116668701, "Finetune/Loss (Raw)": 0.7148244976997375, "Finetune/Step": 4789, "Finetune/Step Time": 5.345501255244017} +{"Finetune/Learning Rate": 3.212719992730684e-07, "Finetune/Loss": 0.8516091108322144, "Finetune/Loss (Raw)": 0.9515889883041382, "Finetune/Step": 4790, "Finetune/Step Time": 5.345160501077771} +{"Finetune/Learning Rate": 3.197075020368312e-07, "Finetune/Loss": 0.8467370867729187, "Finetune/Loss (Raw)": 0.30098599195480347, "Finetune/Step": 4791, "Finetune/Step Time": 5.377052087336779} +{"Finetune/Learning Rate": 3.1814676153311907e-07, "Finetune/Loss": 0.8479348421096802, "Finetune/Loss (Raw)": 1.0186032056808472, "Finetune/Step": 4792, "Finetune/Step Time": 5.380777223035693} +{"Finetune/Learning Rate": 3.165897783676275e-07, "Finetune/Loss": 0.8488714694976807, "Finetune/Loss (Raw)": 0.9187015295028687, "Finetune/Step": 4793, "Finetune/Step Time": 5.384987335652113} +{"Finetune/Learning Rate": 3.1503655314458983e-07, "Finetune/Loss": 0.8495355248451233, "Finetune/Loss (Raw)": 0.8171116709709167, "Finetune/Step": 4794, "Finetune/Step Time": 5.366357542574406} +{"Finetune/Learning Rate": 3.134870864667827e-07, "Finetune/Loss": 0.8481353521347046, "Finetune/Loss (Raw)": 0.7943724989891052, "Finetune/Step": 4795, "Finetune/Step Time": 5.398271273821592} +{"Finetune/Learning Rate": 3.1194137893552523e-07, "Finetune/Loss": 0.8488842248916626, "Finetune/Loss (Raw)": 0.9430331587791443, "Finetune/Step": 4796, "Finetune/Step Time": 5.3919187020510435} +{"Finetune/Learning Rate": 3.1039943115067417e-07, "Finetune/Loss": 0.853294312953949, "Finetune/Loss (Raw)": 0.9135447144508362, "Finetune/Step": 4797, "Finetune/Step Time": 5.361672680824995} +{"Finetune/Learning Rate": 3.088612437106309e-07, "Finetune/Loss": 0.8517029285430908, "Finetune/Loss (Raw)": 0.6963174939155579, "Finetune/Step": 4798, "Finetune/Step Time": 5.378309436142445} +{"Finetune/Learning Rate": 3.0732681721233446e-07, "Finetune/Loss": 0.8522199392318726, "Finetune/Loss (Raw)": 0.8778699040412903, "Finetune/Step": 4799, "Finetune/Step Time": 5.3829850517213345} +{"Finetune/Learning Rate": 3.057961522512676e-07, "Finetune/Loss": 0.8535853028297424, "Finetune/Loss (Raw)": 1.0571376085281372, "Finetune/Step": 4800, "Finetune/Step Time": 5.371814334765077} +{"Finetune/Learning Rate": 3.0426924942145166e-07, "Finetune/Loss": 0.8542764186859131, "Finetune/Loss (Raw)": 0.9299609661102295, "Finetune/Step": 4801, "Finetune/Step Time": 5.364485070109367} +{"Finetune/Learning Rate": 3.027461093154449e-07, "Finetune/Loss": 0.8537417650222778, "Finetune/Loss (Raw)": 0.8825916051864624, "Finetune/Step": 4802, "Finetune/Step Time": 5.3379650842398405} +{"Finetune/Learning Rate": 3.012267325243512e-07, "Finetune/Loss": 0.8539770841598511, "Finetune/Loss (Raw)": 0.9188798069953918, "Finetune/Step": 4803, "Finetune/Step Time": 5.342812957242131} +{"Finetune/Learning Rate": 2.9971111963780994e-07, "Finetune/Loss": 0.8544708490371704, "Finetune/Loss (Raw)": 0.9058461785316467, "Finetune/Step": 4804, "Finetune/Step Time": 5.345287540927529} +{"Finetune/Learning Rate": 2.981992712440007e-07, "Finetune/Loss": 0.8547431230545044, "Finetune/Loss (Raw)": 0.9491861462593079, "Finetune/Step": 4805, "Finetune/Step Time": 5.351530991494656} +{"Finetune/Learning Rate": 2.96691187929643e-07, "Finetune/Loss": 0.8545531034469604, "Finetune/Loss (Raw)": 0.9052265882492065, "Finetune/Step": 4806, "Finetune/Step Time": 5.327598003670573} +{"Finetune/Learning Rate": 2.9518687027999427e-07, "Finetune/Loss": 0.8553611040115356, "Finetune/Loss (Raw)": 0.8467430472373962, "Finetune/Step": 4807, "Finetune/Step Time": 5.328992595896125} +{"Finetune/Learning Rate": 2.936863188788497e-07, "Finetune/Loss": 0.854951024055481, "Finetune/Loss (Raw)": 0.9628693461418152, "Finetune/Step": 4808, "Finetune/Step Time": 5.32533810660243} +{"Finetune/Learning Rate": 2.921895343085446e-07, "Finetune/Loss": 0.8533250093460083, "Finetune/Loss (Raw)": 0.6697092056274414, "Finetune/Step": 4809, "Finetune/Step Time": 5.314357463270426} +{"Finetune/Learning Rate": 2.9069651714995426e-07, "Finetune/Loss": 0.8528210520744324, "Finetune/Loss (Raw)": 0.8358856439590454, "Finetune/Step": 4810, "Finetune/Step Time": 5.304920237511396} +{"Finetune/Learning Rate": 2.8920726798248645e-07, "Finetune/Loss": 0.8526155948638916, "Finetune/Loss (Raw)": 0.8938650488853455, "Finetune/Step": 4811, "Finetune/Step Time": 5.3205023147165775} +{"Finetune/Learning Rate": 2.8772178738408985e-07, "Finetune/Loss": 0.8520559668540955, "Finetune/Loss (Raw)": 0.8766321539878845, "Finetune/Step": 4812, "Finetune/Step Time": 5.3199079018086195} +{"Finetune/Learning Rate": 2.862400759312522e-07, "Finetune/Loss": 0.8485755920410156, "Finetune/Loss (Raw)": 0.4757751226425171, "Finetune/Step": 4813, "Finetune/Step Time": 5.346609627828002} +{"Finetune/Learning Rate": 2.847621341989959e-07, "Finetune/Loss": 0.8495966196060181, "Finetune/Loss (Raw)": 0.8717796802520752, "Finetune/Step": 4814, "Finetune/Step Time": 5.336660588160157} +{"Finetune/Learning Rate": 2.832879627608831e-07, "Finetune/Loss": 0.8501005172729492, "Finetune/Loss (Raw)": 0.9422984719276428, "Finetune/Step": 4815, "Finetune/Step Time": 5.334427630528808} +{"Finetune/Learning Rate": 2.8181756218900626e-07, "Finetune/Loss": 0.8478701114654541, "Finetune/Loss (Raw)": 0.8163795471191406, "Finetune/Step": 4816, "Finetune/Step Time": 5.350659312680364} +{"Finetune/Learning Rate": 2.803509330540033e-07, "Finetune/Loss": 0.8470959067344666, "Finetune/Loss (Raw)": 0.8946479558944702, "Finetune/Step": 4817, "Finetune/Step Time": 5.355031108483672} +{"Finetune/Learning Rate": 2.788880759250423e-07, "Finetune/Loss": 0.848432183265686, "Finetune/Loss (Raw)": 0.9671508073806763, "Finetune/Step": 4818, "Finetune/Step Time": 5.345252554863691} +{"Finetune/Learning Rate": 2.7742899136983026e-07, "Finetune/Loss": 0.8482551574707031, "Finetune/Loss (Raw)": 0.9530888795852661, "Finetune/Step": 4819, "Finetune/Step Time": 5.339147476479411} +{"Finetune/Learning Rate": 2.759736799546109e-07, "Finetune/Loss": 0.8473783731460571, "Finetune/Loss (Raw)": 0.9103720188140869, "Finetune/Step": 4820, "Finetune/Step Time": 5.34187819249928} +{"Finetune/Learning Rate": 2.745221422441602e-07, "Finetune/Loss": 0.847666323184967, "Finetune/Loss (Raw)": 0.9455968737602234, "Finetune/Step": 4821, "Finetune/Step Time": 5.329299718141556} +{"Finetune/Learning Rate": 2.73074378801792e-07, "Finetune/Loss": 0.8468601703643799, "Finetune/Loss (Raw)": 0.8882656693458557, "Finetune/Step": 4822, "Finetune/Step Time": 5.331625474616885} +{"Finetune/Learning Rate": 2.7163039018935357e-07, "Finetune/Loss": 0.8464932441711426, "Finetune/Loss (Raw)": 0.943939745426178, "Finetune/Step": 4823, "Finetune/Step Time": 5.330834221094847} +{"Finetune/Learning Rate": 2.701901769672322e-07, "Finetune/Loss": 0.8425862789154053, "Finetune/Loss (Raw)": 0.2946957051753998, "Finetune/Step": 4824, "Finetune/Step Time": 5.346365647390485} +{"Finetune/Learning Rate": 2.687537396943451e-07, "Finetune/Loss": 0.8424614071846008, "Finetune/Loss (Raw)": 0.865985095500946, "Finetune/Step": 4825, "Finetune/Step Time": 5.350499715656042} +{"Finetune/Learning Rate": 2.673210789281433e-07, "Finetune/Loss": 0.842292845249176, "Finetune/Loss (Raw)": 0.9105250239372253, "Finetune/Step": 4826, "Finetune/Step Time": 5.349694026634097} +{"Finetune/Learning Rate": 2.658921952246163e-07, "Finetune/Loss": 0.8420565128326416, "Finetune/Loss (Raw)": 0.9410450458526611, "Finetune/Step": 4827, "Finetune/Step Time": 5.339044062420726} +{"Finetune/Learning Rate": 2.6446708913828725e-07, "Finetune/Loss": 0.8425511121749878, "Finetune/Loss (Raw)": 0.9510459899902344, "Finetune/Step": 4828, "Finetune/Step Time": 5.345940008759499} +{"Finetune/Learning Rate": 2.6304576122221035e-07, "Finetune/Loss": 0.8420097231864929, "Finetune/Loss (Raw)": 0.8457868099212646, "Finetune/Step": 4829, "Finetune/Step Time": 5.3520695343613625} +{"Finetune/Learning Rate": 2.616282120279745e-07, "Finetune/Loss": 0.842379093170166, "Finetune/Loss (Raw)": 0.8891952633857727, "Finetune/Step": 4830, "Finetune/Step Time": 5.349777914583683} +{"Finetune/Learning Rate": 2.602144421057051e-07, "Finetune/Loss": 0.8375911712646484, "Finetune/Loss (Raw)": 0.3541790246963501, "Finetune/Step": 4831, "Finetune/Step Time": 5.389623699709773} +{"Finetune/Learning Rate": 2.5880445200405667e-07, "Finetune/Loss": 0.8377201557159424, "Finetune/Loss (Raw)": 0.7621728181838989, "Finetune/Step": 4832, "Finetune/Step Time": 5.418277332559228} +{"Finetune/Learning Rate": 2.573982422702193e-07, "Finetune/Loss": 0.8367897272109985, "Finetune/Loss (Raw)": 0.8467223048210144, "Finetune/Step": 4833, "Finetune/Step Time": 5.420824384316802} +{"Finetune/Learning Rate": 2.559958134499163e-07, "Finetune/Loss": 0.8381154537200928, "Finetune/Loss (Raw)": 0.9950881600379944, "Finetune/Step": 4834, "Finetune/Step Time": 5.425054110586643} +{"Finetune/Learning Rate": 2.545971660874036e-07, "Finetune/Loss": 0.8395631909370422, "Finetune/Loss (Raw)": 0.9389187693595886, "Finetune/Step": 4835, "Finetune/Step Time": 5.416069066151977} +{"Finetune/Learning Rate": 2.532023007254658e-07, "Finetune/Loss": 0.8410986661911011, "Finetune/Loss (Raw)": 0.966674268245697, "Finetune/Step": 4836, "Finetune/Step Time": 5.426258344203234} +{"Finetune/Learning Rate": 2.518112179054266e-07, "Finetune/Loss": 0.842082142829895, "Finetune/Loss (Raw)": 0.8647807240486145, "Finetune/Step": 4837, "Finetune/Step Time": 5.408054722473025} +{"Finetune/Learning Rate": 2.504239181671353e-07, "Finetune/Loss": 0.8418488502502441, "Finetune/Loss (Raw)": 0.973468005657196, "Finetune/Step": 4838, "Finetune/Step Time": 5.39826106838882} +{"Finetune/Learning Rate": 2.490404020489767e-07, "Finetune/Loss": 0.841632068157196, "Finetune/Loss (Raw)": 0.8262089490890503, "Finetune/Step": 4839, "Finetune/Step Time": 5.399483131244779} +{"Finetune/Learning Rate": 2.476606700878681e-07, "Finetune/Loss": 0.8423563838005066, "Finetune/Loss (Raw)": 0.8992285132408142, "Finetune/Step": 4840, "Finetune/Step Time": 5.389807842671871} +{"Finetune/Learning Rate": 2.4628472281925443e-07, "Finetune/Loss": 0.8412273526191711, "Finetune/Loss (Raw)": 0.8107266426086426, "Finetune/Step": 4841, "Finetune/Step Time": 5.395864687860012} +{"Finetune/Learning Rate": 2.449125607771141e-07, "Finetune/Loss": 0.8414540886878967, "Finetune/Loss (Raw)": 0.8594611287117004, "Finetune/Step": 4842, "Finetune/Step Time": 5.405908420681953} +{"Finetune/Learning Rate": 2.435441844939557e-07, "Finetune/Loss": 0.8424777388572693, "Finetune/Loss (Raw)": 1.0461419820785522, "Finetune/Step": 4843, "Finetune/Step Time": 5.4018180556595325} +{"Finetune/Learning Rate": 2.4217959450082207e-07, "Finetune/Loss": 0.8422224521636963, "Finetune/Loss (Raw)": 0.8253406286239624, "Finetune/Step": 4844, "Finetune/Step Time": 5.42207127250731} +{"Finetune/Learning Rate": 2.408187913272841e-07, "Finetune/Loss": 0.8430900573730469, "Finetune/Loss (Raw)": 1.0093408823013306, "Finetune/Step": 4845, "Finetune/Step Time": 5.423929434269667} +{"Finetune/Learning Rate": 2.3946177550143835e-07, "Finetune/Loss": 0.8454419374465942, "Finetune/Loss (Raw)": 0.9528098106384277, "Finetune/Step": 4846, "Finetune/Step Time": 5.435301512479782} +{"Finetune/Learning Rate": 2.3810854754992008e-07, "Finetune/Loss": 0.8419986963272095, "Finetune/Loss (Raw)": 0.321584552526474, "Finetune/Step": 4847, "Finetune/Step Time": 5.467452837154269} +{"Finetune/Learning Rate": 2.3675910799789036e-07, "Finetune/Loss": 0.8432704210281372, "Finetune/Loss (Raw)": 0.7672549486160278, "Finetune/Step": 4848, "Finetune/Step Time": 5.430197199806571} +{"Finetune/Learning Rate": 2.354134573690392e-07, "Finetune/Loss": 0.842896044254303, "Finetune/Loss (Raw)": 0.7599686980247498, "Finetune/Step": 4849, "Finetune/Step Time": 5.446108739823103} +{"Finetune/Learning Rate": 2.3407159618558772e-07, "Finetune/Loss": 0.8437573313713074, "Finetune/Loss (Raw)": 0.955951988697052, "Finetune/Step": 4850, "Finetune/Step Time": 5.4549690298736095} +{"Finetune/Learning Rate": 2.3273352496828605e-07, "Finetune/Loss": 0.844031572341919, "Finetune/Loss (Raw)": 0.9139047861099243, "Finetune/Step": 4851, "Finetune/Step Time": 5.4654680620878935} +{"Finetune/Learning Rate": 2.3139924423641325e-07, "Finetune/Loss": 0.844009518623352, "Finetune/Loss (Raw)": 0.8326082825660706, "Finetune/Step": 4852, "Finetune/Step Time": 5.457968672737479} +{"Finetune/Learning Rate": 2.3006875450777733e-07, "Finetune/Loss": 0.8451362252235413, "Finetune/Loss (Raw)": 1.008705735206604, "Finetune/Step": 4853, "Finetune/Step Time": 5.4646579790860415} +{"Finetune/Learning Rate": 2.287420562987175e-07, "Finetune/Loss": 0.8440704345703125, "Finetune/Loss (Raw)": 0.6142253279685974, "Finetune/Step": 4854, "Finetune/Step Time": 5.467596268281341} +{"Finetune/Learning Rate": 2.2741915012409744e-07, "Finetune/Loss": 0.8446921110153198, "Finetune/Loss (Raw)": 0.9122368097305298, "Finetune/Step": 4855, "Finetune/Step Time": 5.468751631677151} +{"Finetune/Learning Rate": 2.2610003649731093e-07, "Finetune/Loss": 0.8447081446647644, "Finetune/Loss (Raw)": 0.8955143094062805, "Finetune/Step": 4856, "Finetune/Step Time": 5.4630114659667015} +{"Finetune/Learning Rate": 2.2478471593028073e-07, "Finetune/Loss": 0.8439729809761047, "Finetune/Loss (Raw)": 0.9638249278068542, "Finetune/Step": 4857, "Finetune/Step Time": 5.461771197617054} +{"Finetune/Learning Rate": 2.234731889334585e-07, "Finetune/Loss": 0.8448411822319031, "Finetune/Loss (Raw)": 0.9938569068908691, "Finetune/Step": 4858, "Finetune/Step Time": 5.471665345132351} +{"Finetune/Learning Rate": 2.221654560158204e-07, "Finetune/Loss": 0.8457130193710327, "Finetune/Loss (Raw)": 0.8908836841583252, "Finetune/Step": 4859, "Finetune/Step Time": 5.457681033760309} +{"Finetune/Learning Rate": 2.2086151768487273e-07, "Finetune/Loss": 0.8452486395835876, "Finetune/Loss (Raw)": 0.7899966835975647, "Finetune/Step": 4860, "Finetune/Step Time": 5.456405322998762} +{"Finetune/Learning Rate": 2.195613744466485e-07, "Finetune/Loss": 0.8447871208190918, "Finetune/Loss (Raw)": 0.8583745360374451, "Finetune/Step": 4861, "Finetune/Step Time": 5.471217976883054} +{"Finetune/Learning Rate": 2.182650268057085e-07, "Finetune/Loss": 0.8461146354675293, "Finetune/Loss (Raw)": 0.9620192050933838, "Finetune/Step": 4862, "Finetune/Step Time": 5.456108074635267} +{"Finetune/Learning Rate": 2.1697247526513698e-07, "Finetune/Loss": 0.8463823795318604, "Finetune/Loss (Raw)": 0.9911894202232361, "Finetune/Step": 4863, "Finetune/Step Time": 5.4606780130416155} +{"Finetune/Learning Rate": 2.1568372032655384e-07, "Finetune/Loss": 0.8480542898178101, "Finetune/Loss (Raw)": 0.9294400811195374, "Finetune/Step": 4864, "Finetune/Step Time": 5.466097678989172} +{"Finetune/Learning Rate": 2.143987624900945e-07, "Finetune/Loss": 0.8489867448806763, "Finetune/Loss (Raw)": 0.9124851226806641, "Finetune/Step": 4865, "Finetune/Step Time": 5.455411868169904} +{"Finetune/Learning Rate": 2.1311760225442678e-07, "Finetune/Loss": 0.8473519086837769, "Finetune/Loss (Raw)": 0.7764751315116882, "Finetune/Step": 4866, "Finetune/Step Time": 5.447479851543903} +{"Finetune/Learning Rate": 2.118402401167452e-07, "Finetune/Loss": 0.8439431190490723, "Finetune/Loss (Raw)": 0.3511510491371155, "Finetune/Step": 4867, "Finetune/Step Time": 5.478834491223097} +{"Finetune/Learning Rate": 2.1056667657276876e-07, "Finetune/Loss": 0.8434359431266785, "Finetune/Loss (Raw)": 0.7602230906486511, "Finetune/Step": 4868, "Finetune/Step Time": 5.477858439087868} +{"Finetune/Learning Rate": 2.0929691211674319e-07, "Finetune/Loss": 0.8441185355186462, "Finetune/Loss (Raw)": 0.7786783576011658, "Finetune/Step": 4869, "Finetune/Step Time": 5.48254176042974} +{"Finetune/Learning Rate": 2.0803094724143879e-07, "Finetune/Loss": 0.8441593647003174, "Finetune/Loss (Raw)": 0.7854706048965454, "Finetune/Step": 4870, "Finetune/Step Time": 5.467869242653251} +{"Finetune/Learning Rate": 2.0676878243815145e-07, "Finetune/Loss": 0.8457911014556885, "Finetune/Loss (Raw)": 0.9396144151687622, "Finetune/Step": 4871, "Finetune/Step Time": 5.467730985954404} +{"Finetune/Learning Rate": 2.055104181967027e-07, "Finetune/Loss": 0.8444800972938538, "Finetune/Loss (Raw)": 0.8391185998916626, "Finetune/Step": 4872, "Finetune/Step Time": 5.467777451500297} +{"Finetune/Learning Rate": 2.0425585500543854e-07, "Finetune/Loss": 0.8447253108024597, "Finetune/Loss (Raw)": 0.8061204552650452, "Finetune/Step": 4873, "Finetune/Step Time": 5.464823005720973} +{"Finetune/Learning Rate": 2.0300509335123286e-07, "Finetune/Loss": 0.843774676322937, "Finetune/Loss (Raw)": 0.6680783629417419, "Finetune/Step": 4874, "Finetune/Step Time": 5.448568291962147} +{"Finetune/Learning Rate": 2.0175813371948072e-07, "Finetune/Loss": 0.842220664024353, "Finetune/Loss (Raw)": 0.8300160765647888, "Finetune/Step": 4875, "Finetune/Step Time": 5.457482572644949} +{"Finetune/Learning Rate": 2.0051497659409947e-07, "Finetune/Loss": 0.8421876430511475, "Finetune/Loss (Raw)": 0.9359437227249146, "Finetune/Step": 4876, "Finetune/Step Time": 5.4507096242159605} +{"Finetune/Learning Rate": 1.9927562245753874e-07, "Finetune/Loss": 0.8432486057281494, "Finetune/Loss (Raw)": 0.9785098433494568, "Finetune/Step": 4877, "Finetune/Step Time": 5.452472593635321} +{"Finetune/Learning Rate": 1.9804007179076602e-07, "Finetune/Loss": 0.8430485129356384, "Finetune/Loss (Raw)": 0.8414989113807678, "Finetune/Step": 4878, "Finetune/Step Time": 5.454111879691482} +{"Finetune/Learning Rate": 1.968083250732744e-07, "Finetune/Loss": 0.8432987928390503, "Finetune/Loss (Raw)": 0.9202069640159607, "Finetune/Step": 4879, "Finetune/Step Time": 5.434825394302607} +{"Finetune/Learning Rate": 1.9558038278308044e-07, "Finetune/Loss": 0.8436893224716187, "Finetune/Loss (Raw)": 0.9502703547477722, "Finetune/Step": 4880, "Finetune/Step Time": 5.430961893871427} +{"Finetune/Learning Rate": 1.9435624539672515e-07, "Finetune/Loss": 0.8459558486938477, "Finetune/Loss (Raw)": 1.0604503154754639, "Finetune/Step": 4881, "Finetune/Step Time": 5.424979627132416} +{"Finetune/Learning Rate": 1.9313591338927185e-07, "Finetune/Loss": 0.8448220491409302, "Finetune/Loss (Raw)": 0.7732192277908325, "Finetune/Step": 4882, "Finetune/Step Time": 5.427149526774883} +{"Finetune/Learning Rate": 1.9191938723430614e-07, "Finetune/Loss": 0.8434669971466064, "Finetune/Loss (Raw)": 0.7584388852119446, "Finetune/Step": 4883, "Finetune/Step Time": 5.418297966942191} +{"Finetune/Learning Rate": 1.9070666740394151e-07, "Finetune/Loss": 0.8443031907081604, "Finetune/Loss (Raw)": 0.9889892339706421, "Finetune/Step": 4884, "Finetune/Step Time": 5.420593094080687} +{"Finetune/Learning Rate": 1.8949775436880924e-07, "Finetune/Loss": 0.8441283106803894, "Finetune/Loss (Raw)": 0.8476366996765137, "Finetune/Step": 4885, "Finetune/Step Time": 5.423766991123557} +{"Finetune/Learning Rate": 1.8829264859806296e-07, "Finetune/Loss": 0.8451026678085327, "Finetune/Loss (Raw)": 0.9716004133224487, "Finetune/Step": 4886, "Finetune/Step Time": 5.430856812745333} +{"Finetune/Learning Rate": 1.8709135055938298e-07, "Finetune/Loss": 0.84552001953125, "Finetune/Loss (Raw)": 0.8884425163269043, "Finetune/Step": 4887, "Finetune/Step Time": 5.437164539471269} +{"Finetune/Learning Rate": 1.8589386071896975e-07, "Finetune/Loss": 0.8460896611213684, "Finetune/Loss (Raw)": 0.9415021538734436, "Finetune/Step": 4888, "Finetune/Step Time": 5.447112096473575} +{"Finetune/Learning Rate": 1.8470017954154262e-07, "Finetune/Loss": 0.8469873666763306, "Finetune/Loss (Raw)": 0.8779363036155701, "Finetune/Step": 4889, "Finetune/Step Time": 5.43055328913033} +{"Finetune/Learning Rate": 1.8351030749034992e-07, "Finetune/Loss": 0.8475115299224854, "Finetune/Loss (Raw)": 0.8610472083091736, "Finetune/Step": 4890, "Finetune/Step Time": 5.449922828003764} +{"Finetune/Learning Rate": 1.8232424502715672e-07, "Finetune/Loss": 0.8467695713043213, "Finetune/Loss (Raw)": 0.7841209769248962, "Finetune/Step": 4891, "Finetune/Step Time": 5.447011042386293} +{"Finetune/Learning Rate": 1.811419926122493e-07, "Finetune/Loss": 0.8470671772956848, "Finetune/Loss (Raw)": 0.8847424983978271, "Finetune/Step": 4892, "Finetune/Step Time": 5.442794909700751} +{"Finetune/Learning Rate": 1.7996355070443727e-07, "Finetune/Loss": 0.8488271832466125, "Finetune/Loss (Raw)": 1.0013339519500732, "Finetune/Step": 4893, "Finetune/Step Time": 5.442965777590871} +{"Finetune/Learning Rate": 1.787889197610515e-07, "Finetune/Loss": 0.8472389578819275, "Finetune/Loss (Raw)": 0.826183021068573, "Finetune/Step": 4894, "Finetune/Step Time": 5.454834895208478} +{"Finetune/Learning Rate": 1.776181002379429e-07, "Finetune/Loss": 0.8464136123657227, "Finetune/Loss (Raw)": 0.7823095321655273, "Finetune/Step": 4895, "Finetune/Step Time": 5.451099714264274} +{"Finetune/Learning Rate": 1.7645109258948245e-07, "Finetune/Loss": 0.8473676443099976, "Finetune/Loss (Raw)": 0.9345850944519043, "Finetune/Step": 4896, "Finetune/Step Time": 5.44426591694355} +{"Finetune/Learning Rate": 1.7528789726856566e-07, "Finetune/Loss": 0.8463970422744751, "Finetune/Loss (Raw)": 0.6972559690475464, "Finetune/Step": 4897, "Finetune/Step Time": 5.449919618666172} +{"Finetune/Learning Rate": 1.7412851472660585e-07, "Finetune/Loss": 0.8474426865577698, "Finetune/Loss (Raw)": 0.7701343894004822, "Finetune/Step": 4898, "Finetune/Step Time": 5.457070222124457} +{"Finetune/Learning Rate": 1.7297294541353317e-07, "Finetune/Loss": 0.8472869396209717, "Finetune/Loss (Raw)": 0.9741371870040894, "Finetune/Step": 4899, "Finetune/Step Time": 5.459862234070897} +{"Finetune/Learning Rate": 1.7182118977780548e-07, "Finetune/Loss": 0.847964346408844, "Finetune/Loss (Raw)": 0.8577929139137268, "Finetune/Step": 4900, "Finetune/Step Time": 5.446898529306054} +{"Finetune/Learning Rate": 1.706732482663942e-07, "Finetune/Loss": 0.8495204448699951, "Finetune/Loss (Raw)": 0.8218610286712646, "Finetune/Step": 4901, "Finetune/Step Time": 5.439581260085106} +{"Finetune/Learning Rate": 1.6952912132479404e-07, "Finetune/Loss": 0.8490461707115173, "Finetune/Loss (Raw)": 0.823986291885376, "Finetune/Step": 4902, "Finetune/Step Time": 5.439537862315774} +{"Finetune/Learning Rate": 1.6838880939701875e-07, "Finetune/Loss": 0.8480541706085205, "Finetune/Loss (Raw)": 0.8400113582611084, "Finetune/Step": 4903, "Finetune/Step Time": 5.438987141475081} +{"Finetune/Learning Rate": 1.6725231292559985e-07, "Finetune/Loss": 0.8484398126602173, "Finetune/Loss (Raw)": 0.8512856364250183, "Finetune/Step": 4904, "Finetune/Step Time": 5.42430217564106} +{"Finetune/Learning Rate": 1.661196323515901e-07, "Finetune/Loss": 0.848849356174469, "Finetune/Loss (Raw)": 0.9433472752571106, "Finetune/Step": 4905, "Finetune/Step Time": 5.428094236180186} +{"Finetune/Learning Rate": 1.6499076811456015e-07, "Finetune/Loss": 0.8532079458236694, "Finetune/Loss (Raw)": 0.8458626866340637, "Finetune/Step": 4906, "Finetune/Step Time": 5.394579291343689} +{"Finetune/Learning Rate": 1.6386572065260065e-07, "Finetune/Loss": 0.8512929677963257, "Finetune/Loss (Raw)": 0.7702613472938538, "Finetune/Step": 4907, "Finetune/Step Time": 5.402461990714073} +{"Finetune/Learning Rate": 1.6274449040232233e-07, "Finetune/Loss": 0.8516963124275208, "Finetune/Loss (Raw)": 0.8871170282363892, "Finetune/Step": 4908, "Finetune/Step Time": 5.414618622511625} +{"Finetune/Learning Rate": 1.6162707779884933e-07, "Finetune/Loss": 0.8520721197128296, "Finetune/Loss (Raw)": 0.8521013855934143, "Finetune/Step": 4909, "Finetune/Step Time": 5.4109936989843845} +{"Finetune/Learning Rate": 1.6051348327583038e-07, "Finetune/Loss": 0.852635383605957, "Finetune/Loss (Raw)": 0.8996773958206177, "Finetune/Step": 4910, "Finetune/Step Time": 5.400339180603623} +{"Finetune/Learning Rate": 1.5940370726542864e-07, "Finetune/Loss": 0.8564020395278931, "Finetune/Loss (Raw)": 0.8475024104118347, "Finetune/Step": 4911, "Finetune/Step Time": 5.368733409792185} +{"Finetune/Learning Rate": 1.5829775019832628e-07, "Finetune/Loss": 0.8566164970397949, "Finetune/Loss (Raw)": 0.9100354909896851, "Finetune/Step": 4912, "Finetune/Step Time": 5.376022046431899} +{"Finetune/Learning Rate": 1.571956125037244e-07, "Finetune/Loss": 0.8557663559913635, "Finetune/Loss (Raw)": 0.9333259463310242, "Finetune/Step": 4913, "Finetune/Step Time": 5.384391261264682} +{"Finetune/Learning Rate": 1.5609729460934087e-07, "Finetune/Loss": 0.8519260883331299, "Finetune/Loss (Raw)": 0.4054705798625946, "Finetune/Step": 4914, "Finetune/Step Time": 5.412673654034734} +{"Finetune/Learning Rate": 1.550027969414114e-07, "Finetune/Loss": 0.8511197566986084, "Finetune/Loss (Raw)": 0.7469794750213623, "Finetune/Step": 4915, "Finetune/Step Time": 5.421134889125824} +{"Finetune/Learning Rate": 1.5391211992468736e-07, "Finetune/Loss": 0.8513429164886475, "Finetune/Loss (Raw)": 0.855485737323761, "Finetune/Step": 4916, "Finetune/Step Time": 5.395190116018057} +{"Finetune/Learning Rate": 1.5282526398244236e-07, "Finetune/Loss": 0.8489552736282349, "Finetune/Loss (Raw)": 0.40920397639274597, "Finetune/Step": 4917, "Finetune/Step Time": 5.422550279647112} +{"Finetune/Learning Rate": 1.5174222953646344e-07, "Finetune/Loss": 0.8484508991241455, "Finetune/Loss (Raw)": 0.8870258331298828, "Finetune/Step": 4918, "Finetune/Step Time": 5.415992142632604} +{"Finetune/Learning Rate": 1.5066301700705334e-07, "Finetune/Loss": 0.8528192043304443, "Finetune/Loss (Raw)": 0.8601279854774475, "Finetune/Step": 4919, "Finetune/Step Time": 5.386838873848319} +{"Finetune/Learning Rate": 1.4958762681303362e-07, "Finetune/Loss": 0.8519728779792786, "Finetune/Loss (Raw)": 0.9102813601493835, "Finetune/Step": 4920, "Finetune/Step Time": 5.382989721372724} +{"Finetune/Learning Rate": 1.4851605937174384e-07, "Finetune/Loss": 0.8519267439842224, "Finetune/Loss (Raw)": 0.9127921462059021, "Finetune/Step": 4921, "Finetune/Step Time": 5.390289921313524} +{"Finetune/Learning Rate": 1.474483150990369e-07, "Finetune/Loss": 0.8514457941055298, "Finetune/Loss (Raw)": 0.755555272102356, "Finetune/Step": 4922, "Finetune/Step Time": 5.391538513824344} +{"Finetune/Learning Rate": 1.4638439440928354e-07, "Finetune/Loss": 0.8515788316726685, "Finetune/Loss (Raw)": 0.8113934397697449, "Finetune/Step": 4923, "Finetune/Step Time": 5.3658414874225855} +{"Finetune/Learning Rate": 1.4532429771537237e-07, "Finetune/Loss": 0.8510028719902039, "Finetune/Loss (Raw)": 0.8693099617958069, "Finetune/Step": 4924, "Finetune/Step Time": 5.3671964276582} +{"Finetune/Learning Rate": 1.4426802542870433e-07, "Finetune/Loss": 0.8503397107124329, "Finetune/Loss (Raw)": 0.8286608457565308, "Finetune/Step": 4925, "Finetune/Step Time": 5.366335719823837} +{"Finetune/Learning Rate": 1.4321557795919705e-07, "Finetune/Loss": 0.8512381315231323, "Finetune/Loss (Raw)": 0.8113168478012085, "Finetune/Step": 4926, "Finetune/Step Time": 5.350542673841119} +{"Finetune/Learning Rate": 1.4216695571528828e-07, "Finetune/Loss": 0.8507306575775146, "Finetune/Loss (Raw)": 0.8129194974899292, "Finetune/Step": 4927, "Finetune/Step Time": 5.354262996464968} +{"Finetune/Learning Rate": 1.411221591039269e-07, "Finetune/Loss": 0.8475908041000366, "Finetune/Loss (Raw)": 0.6552326083183289, "Finetune/Step": 4928, "Finetune/Step Time": 5.371972307562828} +{"Finetune/Learning Rate": 1.4008118853057529e-07, "Finetune/Loss": 0.8468447327613831, "Finetune/Loss (Raw)": 0.8344610929489136, "Finetune/Step": 4929, "Finetune/Step Time": 5.376142363995314} +{"Finetune/Learning Rate": 1.3904404439921582e-07, "Finetune/Loss": 0.8468388319015503, "Finetune/Loss (Raw)": 0.8818342685699463, "Finetune/Step": 4930, "Finetune/Step Time": 5.375704228878021} +{"Finetune/Learning Rate": 1.3801072711234541e-07, "Finetune/Loss": 0.8470462560653687, "Finetune/Loss (Raw)": 0.945435643196106, "Finetune/Step": 4931, "Finetune/Step Time": 5.37385767698288} +{"Finetune/Learning Rate": 1.369812370709711e-07, "Finetune/Loss": 0.8470628261566162, "Finetune/Loss (Raw)": 0.9079685211181641, "Finetune/Step": 4932, "Finetune/Step Time": 5.371464500203729} +{"Finetune/Learning Rate": 1.3595557467461994e-07, "Finetune/Loss": 0.8458943963050842, "Finetune/Loss (Raw)": 0.7996225953102112, "Finetune/Step": 4933, "Finetune/Step Time": 5.381929786875844} +{"Finetune/Learning Rate": 1.3493374032133134e-07, "Finetune/Loss": 0.8461061120033264, "Finetune/Loss (Raw)": 0.9323247671127319, "Finetune/Step": 4934, "Finetune/Step Time": 5.376584919169545} +{"Finetune/Learning Rate": 1.3391573440765915e-07, "Finetune/Loss": 0.8468449115753174, "Finetune/Loss (Raw)": 0.9413146376609802, "Finetune/Step": 4935, "Finetune/Step Time": 5.375859746709466} +{"Finetune/Learning Rate": 1.329015573286696e-07, "Finetune/Loss": 0.8455109596252441, "Finetune/Loss (Raw)": 0.7921218872070312, "Finetune/Step": 4936, "Finetune/Step Time": 5.386201974004507} +{"Finetune/Learning Rate": 1.3189120947794897e-07, "Finetune/Loss": 0.8481942415237427, "Finetune/Loss (Raw)": 1.01316237449646, "Finetune/Step": 4937, "Finetune/Step Time": 5.379569206386805} +{"Finetune/Learning Rate": 1.308846912475914e-07, "Finetune/Loss": 0.8487980365753174, "Finetune/Loss (Raw)": 0.9131796360015869, "Finetune/Step": 4938, "Finetune/Step Time": 5.396309083327651} +{"Finetune/Learning Rate": 1.2988200302820552e-07, "Finetune/Loss": 0.8444697856903076, "Finetune/Loss (Raw)": 0.33984896540641785, "Finetune/Step": 4939, "Finetune/Step Time": 5.4058621898293495} +{"Finetune/Learning Rate": 1.2888314520891676e-07, "Finetune/Loss": 0.8430242538452148, "Finetune/Loss (Raw)": 0.6915971040725708, "Finetune/Step": 4940, "Finetune/Step Time": 5.405672013759613} +{"Finetune/Learning Rate": 1.2788811817736168e-07, "Finetune/Loss": 0.8451724052429199, "Finetune/Loss (Raw)": 0.7507360577583313, "Finetune/Step": 4941, "Finetune/Step Time": 5.375389978289604} +{"Finetune/Learning Rate": 1.2689692231969364e-07, "Finetune/Loss": 0.8447594046592712, "Finetune/Loss (Raw)": 0.8189219832420349, "Finetune/Step": 4942, "Finetune/Step Time": 5.381831651553512} +{"Finetune/Learning Rate": 1.2590955802057047e-07, "Finetune/Loss": 0.8435481786727905, "Finetune/Loss (Raw)": 0.7872533202171326, "Finetune/Step": 4943, "Finetune/Step Time": 5.392916847020388} +{"Finetune/Learning Rate": 1.2492602566317457e-07, "Finetune/Loss": 0.8443554639816284, "Finetune/Loss (Raw)": 0.9197226166725159, "Finetune/Step": 4944, "Finetune/Step Time": 5.379093391820788} +{"Finetune/Learning Rate": 1.2394632562919283e-07, "Finetune/Loss": 0.844769299030304, "Finetune/Loss (Raw)": 0.9476171135902405, "Finetune/Step": 4945, "Finetune/Step Time": 5.378798043355346} +{"Finetune/Learning Rate": 1.2297045829882892e-07, "Finetune/Loss": 0.8439596891403198, "Finetune/Loss (Raw)": 0.8635236620903015, "Finetune/Step": 4946, "Finetune/Step Time": 5.379516011103988} +{"Finetune/Learning Rate": 1.2199842405079766e-07, "Finetune/Loss": 0.8437501192092896, "Finetune/Loss (Raw)": 0.9262556433677673, "Finetune/Step": 4947, "Finetune/Step Time": 5.381500940769911} +{"Finetune/Learning Rate": 1.2103022326232505e-07, "Finetune/Loss": 0.8431024551391602, "Finetune/Loss (Raw)": 0.827478289604187, "Finetune/Step": 4948, "Finetune/Step Time": 5.378597270697355} +{"Finetune/Learning Rate": 1.2006585630915391e-07, "Finetune/Loss": 0.8420261144638062, "Finetune/Loss (Raw)": 0.8078228235244751, "Finetune/Step": 4949, "Finetune/Step Time": 5.376534670591354} +{"Finetune/Learning Rate": 1.1910532356553261e-07, "Finetune/Loss": 0.8414541482925415, "Finetune/Loss (Raw)": 0.8150590658187866, "Finetune/Step": 4950, "Finetune/Step Time": 5.383458584547043} +{"Finetune/Learning Rate": 1.1814862540422744e-07, "Finetune/Loss": 0.8411982655525208, "Finetune/Loss (Raw)": 0.9111803770065308, "Finetune/Step": 4951, "Finetune/Step Time": 5.3816984836012125} +{"Finetune/Learning Rate": 1.1719576219651585e-07, "Finetune/Loss": 0.8460683226585388, "Finetune/Loss (Raw)": 0.918063759803772, "Finetune/Step": 4952, "Finetune/Step Time": 5.36074479855597} +{"Finetune/Learning Rate": 1.1624673431218092e-07, "Finetune/Loss": 0.8457607626914978, "Finetune/Loss (Raw)": 0.8266146779060364, "Finetune/Step": 4953, "Finetune/Step Time": 5.3621127381920815} +{"Finetune/Learning Rate": 1.1530154211952583e-07, "Finetune/Loss": 0.8458617329597473, "Finetune/Loss (Raw)": 0.9234510064125061, "Finetune/Step": 4954, "Finetune/Step Time": 5.413120169192553} +{"Finetune/Learning Rate": 1.1436018598535937e-07, "Finetune/Loss": 0.8452973365783691, "Finetune/Loss (Raw)": 0.8688070774078369, "Finetune/Step": 4955, "Finetune/Step Time": 5.416170317679644} +{"Finetune/Learning Rate": 1.1342266627500375e-07, "Finetune/Loss": 0.845196545124054, "Finetune/Loss (Raw)": 0.9381430149078369, "Finetune/Step": 4956, "Finetune/Step Time": 5.403967199847102} +{"Finetune/Learning Rate": 1.1248898335229241e-07, "Finetune/Loss": 0.8462035655975342, "Finetune/Loss (Raw)": 0.9746882319450378, "Finetune/Step": 4957, "Finetune/Step Time": 5.398814344778657} +{"Finetune/Learning Rate": 1.1155913757956881e-07, "Finetune/Loss": 0.8456077575683594, "Finetune/Loss (Raw)": 0.8129336833953857, "Finetune/Step": 4958, "Finetune/Step Time": 5.41064783371985} +{"Finetune/Learning Rate": 1.1063312931768877e-07, "Finetune/Loss": 0.846284031867981, "Finetune/Loss (Raw)": 0.4407346248626709, "Finetune/Step": 4959, "Finetune/Step Time": 5.411009738221765} +{"Finetune/Learning Rate": 1.0971095892601702e-07, "Finetune/Loss": 0.8468049764633179, "Finetune/Loss (Raw)": 0.8288559913635254, "Finetune/Step": 4960, "Finetune/Step Time": 5.378879863768816} +{"Finetune/Learning Rate": 1.0879262676243174e-07, "Finetune/Loss": 0.8465644717216492, "Finetune/Loss (Raw)": 0.8159416317939758, "Finetune/Step": 4961, "Finetune/Step Time": 5.377058727666736} +{"Finetune/Learning Rate": 1.0787813318331786e-07, "Finetune/Loss": 0.8459833860397339, "Finetune/Loss (Raw)": 0.9207056164741516, "Finetune/Step": 4962, "Finetune/Step Time": 5.375954560935497} +{"Finetune/Learning Rate": 1.0696747854357259e-07, "Finetune/Loss": 0.8452917337417603, "Finetune/Loss (Raw)": 0.8503878116607666, "Finetune/Step": 4963, "Finetune/Step Time": 5.380342086777091} +{"Finetune/Learning Rate": 1.0606066319660435e-07, "Finetune/Loss": 0.8451402187347412, "Finetune/Loss (Raw)": 0.947278618812561, "Finetune/Step": 4964, "Finetune/Step Time": 5.372084315866232} +{"Finetune/Learning Rate": 1.0515768749432942e-07, "Finetune/Loss": 0.8460050821304321, "Finetune/Loss (Raw)": 0.975489616394043, "Finetune/Step": 4965, "Finetune/Step Time": 5.372652670368552} +{"Finetune/Learning Rate": 1.0425855178717636e-07, "Finetune/Loss": 0.8446618318557739, "Finetune/Loss (Raw)": 0.8015276789665222, "Finetune/Step": 4966, "Finetune/Step Time": 5.375993965193629} +{"Finetune/Learning Rate": 1.0336325642408052e-07, "Finetune/Loss": 0.8454862236976624, "Finetune/Loss (Raw)": 0.9317284822463989, "Finetune/Step": 4967, "Finetune/Step Time": 5.384938515722752} +{"Finetune/Learning Rate": 1.024718017524895e-07, "Finetune/Loss": 0.8454230427742004, "Finetune/Loss (Raw)": 0.8911399841308594, "Finetune/Step": 4968, "Finetune/Step Time": 5.380769636482} +{"Finetune/Learning Rate": 1.015841881183588e-07, "Finetune/Loss": 0.8459644913673401, "Finetune/Loss (Raw)": 0.8800330758094788, "Finetune/Step": 4969, "Finetune/Step Time": 5.384159602224827} +{"Finetune/Learning Rate": 1.0070041586615398e-07, "Finetune/Loss": 0.8465721607208252, "Finetune/Loss (Raw)": 0.9372366070747375, "Finetune/Step": 4970, "Finetune/Step Time": 5.371598031371832} +{"Finetune/Learning Rate": 9.98204853388518e-08, "Finetune/Loss": 0.8463037610054016, "Finetune/Loss (Raw)": 1.0117909908294678, "Finetune/Step": 4971, "Finetune/Step Time": 5.383445341140032} +{"Finetune/Learning Rate": 9.894439687793466e-08, "Finetune/Loss": 0.8466036319732666, "Finetune/Loss (Raw)": 0.8637257814407349, "Finetune/Step": 4972, "Finetune/Step Time": 5.377098981291056} +{"Finetune/Learning Rate": 9.807215082339395e-08, "Finetune/Loss": 0.8465229272842407, "Finetune/Loss (Raw)": 0.9990151524543762, "Finetune/Step": 4973, "Finetune/Step Time": 5.377723891288042} +{"Finetune/Learning Rate": 9.720374751373329e-08, "Finetune/Loss": 0.841454267501831, "Finetune/Loss (Raw)": 0.30402153730392456, "Finetune/Step": 4974, "Finetune/Step Time": 5.381266100332141} +{"Finetune/Learning Rate": 9.633918728596203e-08, "Finetune/Loss": 0.8452877998352051, "Finetune/Loss (Raw)": 0.8122789263725281, "Finetune/Step": 4975, "Finetune/Step Time": 5.3540786150842905} +{"Finetune/Learning Rate": 9.547847047559955e-08, "Finetune/Loss": 0.8470787405967712, "Finetune/Loss (Raw)": 0.9964917898178101, "Finetune/Step": 4976, "Finetune/Step Time": 5.357176234945655} +{"Finetune/Learning Rate": 9.462159741667309e-08, "Finetune/Loss": 0.8470131158828735, "Finetune/Loss (Raw)": 0.7515701055526733, "Finetune/Step": 4977, "Finetune/Step Time": 5.356549868360162} +{"Finetune/Learning Rate": 9.376856844171889e-08, "Finetune/Loss": 0.8458626866340637, "Finetune/Loss (Raw)": 0.8086937069892883, "Finetune/Step": 4978, "Finetune/Step Time": 5.352167213335633} +{"Finetune/Learning Rate": 9.291938388178102e-08, "Finetune/Loss": 0.8451635837554932, "Finetune/Loss (Raw)": 0.8244270086288452, "Finetune/Step": 4979, "Finetune/Step Time": 5.349127883091569} +{"Finetune/Learning Rate": 9.207404406640919e-08, "Finetune/Loss": 0.8441845178604126, "Finetune/Loss (Raw)": 0.7072843313217163, "Finetune/Step": 4980, "Finetune/Step Time": 5.342397227883339} +{"Finetune/Learning Rate": 9.123254932366655e-08, "Finetune/Loss": 0.8423223495483398, "Finetune/Loss (Raw)": 0.770346999168396, "Finetune/Step": 4981, "Finetune/Step Time": 5.333084283396602} +{"Finetune/Learning Rate": 9.039489998011853e-08, "Finetune/Loss": 0.8450672626495361, "Finetune/Loss (Raw)": 0.9655717015266418, "Finetune/Step": 4982, "Finetune/Step Time": 5.328439809381962} +{"Finetune/Learning Rate": 8.956109636084065e-08, "Finetune/Loss": 0.8411582708358765, "Finetune/Loss (Raw)": 0.41188815236091614, "Finetune/Step": 4983, "Finetune/Step Time": 5.357498526573181} +{"Finetune/Learning Rate": 8.873113878941631e-08, "Finetune/Loss": 0.8403602838516235, "Finetune/Loss (Raw)": 0.7933700084686279, "Finetune/Step": 4984, "Finetune/Step Time": 5.368502086028457} +{"Finetune/Learning Rate": 8.790502758793562e-08, "Finetune/Loss": 0.8408122658729553, "Finetune/Loss (Raw)": 1.0216830968856812, "Finetune/Step": 4985, "Finetune/Step Time": 5.370834473520517} +{"Finetune/Learning Rate": 8.70827630769977e-08, "Finetune/Loss": 0.8385605216026306, "Finetune/Loss (Raw)": 0.7056317925453186, "Finetune/Step": 4986, "Finetune/Step Time": 5.357455218210816} +{"Finetune/Learning Rate": 8.626434557570507e-08, "Finetune/Loss": 0.8388022184371948, "Finetune/Loss (Raw)": 0.9218198657035828, "Finetune/Step": 4987, "Finetune/Step Time": 5.364816369488835} +{"Finetune/Learning Rate": 8.544977540167032e-08, "Finetune/Loss": 0.8400297164916992, "Finetune/Loss (Raw)": 0.9471207857131958, "Finetune/Step": 4988, "Finetune/Step Time": 5.373298300430179} +{"Finetune/Learning Rate": 8.463905287101282e-08, "Finetune/Loss": 0.8405644297599792, "Finetune/Loss (Raw)": 0.9268123507499695, "Finetune/Step": 4989, "Finetune/Step Time": 5.3649061024188995} +{"Finetune/Learning Rate": 8.383217829835755e-08, "Finetune/Loss": 0.839812159538269, "Finetune/Loss (Raw)": 0.8657273650169373, "Finetune/Step": 4990, "Finetune/Step Time": 5.368341218680143} +{"Finetune/Learning Rate": 8.302915199683737e-08, "Finetune/Loss": 0.8386014699935913, "Finetune/Loss (Raw)": 0.8362239599227905, "Finetune/Step": 4991, "Finetune/Step Time": 5.362612966448069} +{"Finetune/Learning Rate": 8.222997427809076e-08, "Finetune/Loss": 0.8390980958938599, "Finetune/Loss (Raw)": 0.9930124878883362, "Finetune/Step": 4992, "Finetune/Step Time": 5.3625917211174965} +{"Finetune/Learning Rate": 8.143464545226298e-08, "Finetune/Loss": 0.8366687297821045, "Finetune/Loss (Raw)": 0.6015225052833557, "Finetune/Step": 4993, "Finetune/Step Time": 5.376112870872021} +{"Finetune/Learning Rate": 8.064316582800491e-08, "Finetune/Loss": 0.8371491432189941, "Finetune/Loss (Raw)": 0.8379712104797363, "Finetune/Step": 4994, "Finetune/Step Time": 5.383046140894294} +{"Finetune/Learning Rate": 7.98555357124764e-08, "Finetune/Loss": 0.8392717838287354, "Finetune/Loss (Raw)": 0.6228470802307129, "Finetune/Step": 4995, "Finetune/Step Time": 5.355261305347085} +{"Finetune/Learning Rate": 7.907175541133849e-08, "Finetune/Loss": 0.8412431478500366, "Finetune/Loss (Raw)": 1.0125551223754883, "Finetune/Step": 4996, "Finetune/Step Time": 5.357048312202096} +{"Finetune/Learning Rate": 7.829182522876344e-08, "Finetune/Loss": 0.8424480557441711, "Finetune/Loss (Raw)": 0.9329046010971069, "Finetune/Step": 4997, "Finetune/Step Time": 5.341474046930671} +{"Finetune/Learning Rate": 7.75157454674258e-08, "Finetune/Loss": 0.8448781371116638, "Finetune/Loss (Raw)": 1.096523404121399, "Finetune/Step": 4998, "Finetune/Step Time": 5.33915650844574} +{"Finetune/Learning Rate": 7.674351642850797e-08, "Finetune/Loss": 0.8452889323234558, "Finetune/Loss (Raw)": 0.9921948909759521, "Finetune/Step": 4999, "Finetune/Step Time": 5.324381208047271} +{"Finetune/Learning Rate": 7.597513841169469e-08, "Finetune/Loss": 0.8414870500564575, "Finetune/Loss (Raw)": 0.3524813950061798, "Finetune/Step": 5000, "Finetune/Step Time": 5.362054940313101} +{"Finetune/Learning Rate": 7.521061171518184e-08, "Finetune/Loss": 0.8425092697143555, "Finetune/Loss (Raw)": 0.936966598033905, "Finetune/Step": 5001, "Finetune/Step Time": 5.344163242727518} +{"Finetune/Learning Rate": 7.444993663566325e-08, "Finetune/Loss": 0.842647910118103, "Finetune/Loss (Raw)": 0.6858213543891907, "Finetune/Step": 5002, "Finetune/Step Time": 5.346050003543496} +{"Finetune/Learning Rate": 7.369311346834496e-08, "Finetune/Loss": 0.8424416184425354, "Finetune/Loss (Raw)": 0.8036112785339355, "Finetune/Step": 5003, "Finetune/Step Time": 5.334347220137715} +{"Finetune/Learning Rate": 7.294014250693538e-08, "Finetune/Loss": 0.8409788012504578, "Finetune/Loss (Raw)": 0.7487056255340576, "Finetune/Step": 5004, "Finetune/Step Time": 5.3479982782155275} +{"Finetune/Learning Rate": 7.21910240436463e-08, "Finetune/Loss": 0.8400700688362122, "Finetune/Loss (Raw)": 0.8621825575828552, "Finetune/Step": 5005, "Finetune/Step Time": 5.3498259745538235} +{"Finetune/Learning Rate": 7.144575836919632e-08, "Finetune/Loss": 0.8404463529586792, "Finetune/Loss (Raw)": 0.8896692991256714, "Finetune/Step": 5006, "Finetune/Step Time": 5.341382620856166} +{"Finetune/Learning Rate": 7.07043457728096e-08, "Finetune/Loss": 0.8394907712936401, "Finetune/Loss (Raw)": 0.7978885769844055, "Finetune/Step": 5007, "Finetune/Step Time": 5.35126843675971} +{"Finetune/Learning Rate": 6.996678654221267e-08, "Finetune/Loss": 0.8382541537284851, "Finetune/Loss (Raw)": 0.7919841408729553, "Finetune/Step": 5008, "Finetune/Step Time": 5.3581237606704235} +{"Finetune/Learning Rate": 6.923308096363879e-08, "Finetune/Loss": 0.837852954864502, "Finetune/Loss (Raw)": 1.0091017484664917, "Finetune/Step": 5009, "Finetune/Step Time": 5.3610598389059305} +{"Finetune/Learning Rate": 6.850322932182352e-08, "Finetune/Loss": 0.8392022848129272, "Finetune/Loss (Raw)": 0.9459304809570312, "Finetune/Step": 5010, "Finetune/Step Time": 5.352239392697811} +{"Finetune/Learning Rate": 6.777723190000918e-08, "Finetune/Loss": 0.840038537979126, "Finetune/Loss (Raw)": 0.8654744029045105, "Finetune/Step": 5011, "Finetune/Step Time": 5.352981332689524} +{"Finetune/Learning Rate": 6.705508897994151e-08, "Finetune/Loss": 0.8388817310333252, "Finetune/Loss (Raw)": 0.8409299254417419, "Finetune/Step": 5012, "Finetune/Step Time": 5.351479072123766} +{"Finetune/Learning Rate": 6.633680084186856e-08, "Finetune/Loss": 0.8380098342895508, "Finetune/Loss (Raw)": 0.7360270619392395, "Finetune/Step": 5013, "Finetune/Step Time": 5.350183289498091} +{"Finetune/Learning Rate": 6.562236776454512e-08, "Finetune/Loss": 0.8366159200668335, "Finetune/Loss (Raw)": 0.7931758165359497, "Finetune/Step": 5014, "Finetune/Step Time": 5.339713659137487} +{"Finetune/Learning Rate": 6.491179002522829e-08, "Finetune/Loss": 0.8357548713684082, "Finetune/Loss (Raw)": 0.7782313227653503, "Finetune/Step": 5015, "Finetune/Step Time": 5.328694149851799} +{"Finetune/Learning Rate": 6.420506789967973e-08, "Finetune/Loss": 0.8351998925209045, "Finetune/Loss (Raw)": 0.8704614043235779, "Finetune/Step": 5016, "Finetune/Step Time": 5.343897078186274} +{"Finetune/Learning Rate": 6.350220166216337e-08, "Finetune/Loss": 0.8345632553100586, "Finetune/Loss (Raw)": 0.7964483499526978, "Finetune/Step": 5017, "Finetune/Step Time": 5.343920229002833} +{"Finetune/Learning Rate": 6.280319158544989e-08, "Finetune/Loss": 0.8336840867996216, "Finetune/Loss (Raw)": 0.748511552810669, "Finetune/Step": 5018, "Finetune/Step Time": 5.3320866022259} +{"Finetune/Learning Rate": 6.21080379408101e-08, "Finetune/Loss": 0.8337024450302124, "Finetune/Loss (Raw)": 0.7864760160446167, "Finetune/Step": 5019, "Finetune/Step Time": 5.325232554227114} +{"Finetune/Learning Rate": 6.14167409980193e-08, "Finetune/Loss": 0.8294971585273743, "Finetune/Loss (Raw)": 0.34646591544151306, "Finetune/Step": 5020, "Finetune/Step Time": 5.354237539693713} +{"Finetune/Learning Rate": 6.072930102535734e-08, "Finetune/Loss": 0.8289417624473572, "Finetune/Loss (Raw)": 0.930238664150238, "Finetune/Step": 5021, "Finetune/Step Time": 5.352964656427503} +{"Finetune/Learning Rate": 6.004571828960525e-08, "Finetune/Loss": 0.8293818235397339, "Finetune/Loss (Raw)": 0.8825158476829529, "Finetune/Step": 5022, "Finetune/Step Time": 5.329571569338441} +{"Finetune/Learning Rate": 5.936599305604862e-08, "Finetune/Loss": 0.8301821351051331, "Finetune/Loss (Raw)": 0.8847405910491943, "Finetune/Step": 5023, "Finetune/Step Time": 5.336088925600052} +{"Finetune/Learning Rate": 5.869012558847642e-08, "Finetune/Loss": 0.8293402791023254, "Finetune/Loss (Raw)": 0.8268266320228577, "Finetune/Step": 5024, "Finetune/Step Time": 5.3449991550296545} +{"Finetune/Learning Rate": 5.801811614917885e-08, "Finetune/Loss": 0.8304105401039124, "Finetune/Loss (Raw)": 0.8342569470405579, "Finetune/Step": 5025, "Finetune/Step Time": 5.349664343520999} +{"Finetune/Learning Rate": 5.73499649989484e-08, "Finetune/Loss": 0.8316657543182373, "Finetune/Loss (Raw)": 0.9308015704154968, "Finetune/Step": 5026, "Finetune/Step Time": 5.341308223083615} +{"Finetune/Learning Rate": 5.6685672397083226e-08, "Finetune/Loss": 0.831296980381012, "Finetune/Loss (Raw)": 0.9269329905509949, "Finetune/Step": 5027, "Finetune/Step Time": 5.338802726939321} +{"Finetune/Learning Rate": 5.602523860138154e-08, "Finetune/Loss": 0.8321186304092407, "Finetune/Loss (Raw)": 0.9629685878753662, "Finetune/Step": 5028, "Finetune/Step Time": 5.3412312883883715} +{"Finetune/Learning Rate": 5.536866386814499e-08, "Finetune/Loss": 0.8326266407966614, "Finetune/Loss (Raw)": 0.886879026889801, "Finetune/Step": 5029, "Finetune/Step Time": 5.343090558424592} +{"Finetune/Learning Rate": 5.471594845217865e-08, "Finetune/Loss": 0.8326141834259033, "Finetune/Loss (Raw)": 0.8224002718925476, "Finetune/Step": 5030, "Finetune/Step Time": 5.351358827203512} +{"Finetune/Learning Rate": 5.4067092606785445e-08, "Finetune/Loss": 0.8330767154693604, "Finetune/Loss (Raw)": 0.8992071151733398, "Finetune/Step": 5031, "Finetune/Step Time": 5.3532012142241} +{"Finetune/Learning Rate": 5.342209658377728e-08, "Finetune/Loss": 0.8338584303855896, "Finetune/Loss (Raw)": 0.9513471722602844, "Finetune/Step": 5032, "Finetune/Step Time": 5.345707742497325} +{"Finetune/Learning Rate": 5.278096063346172e-08, "Finetune/Loss": 0.8331539630889893, "Finetune/Loss (Raw)": 0.8531703352928162, "Finetune/Step": 5033, "Finetune/Step Time": 5.355786809697747} +{"Finetune/Learning Rate": 5.214368500465305e-08, "Finetune/Loss": 0.8336886167526245, "Finetune/Loss (Raw)": 0.9143030643463135, "Finetune/Step": 5034, "Finetune/Step Time": 5.35316669754684} +{"Finetune/Learning Rate": 5.151026994466457e-08, "Finetune/Loss": 0.8338779807090759, "Finetune/Loss (Raw)": 0.7945026159286499, "Finetune/Step": 5035, "Finetune/Step Time": 5.350509295240045} +{"Finetune/Learning Rate": 5.088071569931185e-08, "Finetune/Loss": 0.8333941102027893, "Finetune/Loss (Raw)": 0.8251816630363464, "Finetune/Step": 5036, "Finetune/Step Time": 5.330014877021313} +{"Finetune/Learning Rate": 5.025502251291392e-08, "Finetune/Loss": 0.8343473076820374, "Finetune/Loss (Raw)": 0.9741111397743225, "Finetune/Step": 5037, "Finetune/Step Time": 5.3250862918794155} +{"Finetune/Learning Rate": 4.963319062828875e-08, "Finetune/Loss": 0.8349230885505676, "Finetune/Loss (Raw)": 0.9733723402023315, "Finetune/Step": 5038, "Finetune/Step Time": 5.333201382309198} +{"Finetune/Learning Rate": 4.901522028675776e-08, "Finetune/Loss": 0.8367631435394287, "Finetune/Loss (Raw)": 1.0830310583114624, "Finetune/Step": 5039, "Finetune/Step Time": 5.338493101298809} +{"Finetune/Learning Rate": 4.8401111728143545e-08, "Finetune/Loss": 0.8353754878044128, "Finetune/Loss (Raw)": 0.7324130535125732, "Finetune/Step": 5040, "Finetune/Step Time": 5.342776261270046} +{"Finetune/Learning Rate": 4.779086519076992e-08, "Finetune/Loss": 0.8358354568481445, "Finetune/Loss (Raw)": 0.9922071099281311, "Finetune/Step": 5041, "Finetune/Step Time": 5.331124285236001} +{"Finetune/Learning Rate": 4.7184480911459665e-08, "Finetune/Loss": 0.8401844501495361, "Finetune/Loss (Raw)": 0.9621426463127136, "Finetune/Step": 5042, "Finetune/Step Time": 5.313472243025899} +{"Finetune/Learning Rate": 4.65819591255412e-08, "Finetune/Loss": 0.8411692380905151, "Finetune/Loss (Raw)": 0.8730292320251465, "Finetune/Step": 5043, "Finetune/Step Time": 5.309967763721943} +{"Finetune/Learning Rate": 4.5983300066839706e-08, "Finetune/Loss": 0.8411470055580139, "Finetune/Loss (Raw)": 0.8526446223258972, "Finetune/Step": 5044, "Finetune/Step Time": 5.312902571633458} +{"Finetune/Learning Rate": 4.538850396768379e-08, "Finetune/Loss": 0.8402050733566284, "Finetune/Loss (Raw)": 0.28863632678985596, "Finetune/Step": 5045, "Finetune/Step Time": 5.311528638005257} +{"Finetune/Learning Rate": 4.479757105890104e-08, "Finetune/Loss": 0.8358827829360962, "Finetune/Loss (Raw)": 0.3337751626968384, "Finetune/Step": 5046, "Finetune/Step Time": 5.347518123686314} +{"Finetune/Learning Rate": 4.4210501569823536e-08, "Finetune/Loss": 0.8361715078353882, "Finetune/Loss (Raw)": 0.8970838189125061, "Finetune/Step": 5047, "Finetune/Step Time": 5.346576334908605} +{"Finetune/Learning Rate": 4.362729572827906e-08, "Finetune/Loss": 0.8366042375564575, "Finetune/Loss (Raw)": 0.9656617641448975, "Finetune/Step": 5048, "Finetune/Step Time": 5.347594778984785} +{"Finetune/Learning Rate": 4.304795376059989e-08, "Finetune/Loss": 0.8361451625823975, "Finetune/Loss (Raw)": 0.8540343642234802, "Finetune/Step": 5049, "Finetune/Step Time": 5.345331460237503} +{"Finetune/Learning Rate": 4.2472475891617295e-08, "Finetune/Loss": 0.8368240594863892, "Finetune/Loss (Raw)": 0.8424513936042786, "Finetune/Step": 5050, "Finetune/Step Time": 5.3562032263726} +{"Finetune/Learning Rate": 4.1900862344661506e-08, "Finetune/Loss": 0.8365591764450073, "Finetune/Loss (Raw)": 0.7774912118911743, "Finetune/Step": 5051, "Finetune/Step Time": 5.370263289660215} +{"Finetune/Learning Rate": 4.133311334156509e-08, "Finetune/Loss": 0.8326748013496399, "Finetune/Loss (Raw)": 0.3721104562282562, "Finetune/Step": 5052, "Finetune/Step Time": 5.395932896062732} +{"Finetune/Learning Rate": 4.076922910266179e-08, "Finetune/Loss": 0.8325037956237793, "Finetune/Loss (Raw)": 0.8067801594734192, "Finetune/Step": 5053, "Finetune/Step Time": 5.4046237748116255} +{"Finetune/Learning Rate": 4.020920984678323e-08, "Finetune/Loss": 0.8340831398963928, "Finetune/Loss (Raw)": 1.0134608745574951, "Finetune/Step": 5054, "Finetune/Step Time": 5.410619951784611} +{"Finetune/Learning Rate": 3.965305579126111e-08, "Finetune/Loss": 0.8342301249504089, "Finetune/Loss (Raw)": 0.8317433595657349, "Finetune/Step": 5055, "Finetune/Step Time": 5.405341871082783} +{"Finetune/Learning Rate": 3.9100767151928345e-08, "Finetune/Loss": 0.8360480070114136, "Finetune/Loss (Raw)": 0.8879163861274719, "Finetune/Step": 5056, "Finetune/Step Time": 5.379539351910353} +{"Finetune/Learning Rate": 3.855234414311904e-08, "Finetune/Loss": 0.836410403251648, "Finetune/Loss (Raw)": 0.8808476328849792, "Finetune/Step": 5057, "Finetune/Step Time": 5.383441397920251} +{"Finetune/Learning Rate": 3.800778697766294e-08, "Finetune/Loss": 0.8364479541778564, "Finetune/Loss (Raw)": 0.8866412043571472, "Finetune/Step": 5058, "Finetune/Step Time": 5.387077387422323} +{"Finetune/Learning Rate": 3.746709586689323e-08, "Finetune/Loss": 0.8365485072135925, "Finetune/Loss (Raw)": 0.9583074450492859, "Finetune/Step": 5059, "Finetune/Step Time": 5.384724646806717} +{"Finetune/Learning Rate": 3.693027102064206e-08, "Finetune/Loss": 0.8370637893676758, "Finetune/Loss (Raw)": 0.973924994468689, "Finetune/Step": 5060, "Finetune/Step Time": 5.384672863408923} +{"Finetune/Learning Rate": 3.639731264724056e-08, "Finetune/Loss": 0.8379198312759399, "Finetune/Loss (Raw)": 0.9091976284980774, "Finetune/Step": 5061, "Finetune/Step Time": 5.367331510409713} +{"Finetune/Learning Rate": 3.586822095351772e-08, "Finetune/Loss": 0.83687824010849, "Finetune/Loss (Raw)": 0.79899662733078, "Finetune/Step": 5062, "Finetune/Step Time": 5.372481156140566} +{"Finetune/Learning Rate": 3.5342996144805966e-08, "Finetune/Loss": 0.8369214534759521, "Finetune/Loss (Raw)": 0.9468535780906677, "Finetune/Step": 5063, "Finetune/Step Time": 5.369536362588406} +{"Finetune/Learning Rate": 3.482163842493447e-08, "Finetune/Loss": 0.836709201335907, "Finetune/Loss (Raw)": 0.7649440765380859, "Finetune/Step": 5064, "Finetune/Step Time": 5.3650902677327394} +{"Finetune/Learning Rate": 3.4304147996230275e-08, "Finetune/Loss": 0.8363267183303833, "Finetune/Loss (Raw)": 0.9642073512077332, "Finetune/Step": 5065, "Finetune/Step Time": 5.371613562107086} +{"Finetune/Learning Rate": 3.379052505952274e-08, "Finetune/Loss": 0.8365175724029541, "Finetune/Loss (Raw)": 0.9376081824302673, "Finetune/Step": 5066, "Finetune/Step Time": 5.35760785266757} +{"Finetune/Learning Rate": 3.328076981413908e-08, "Finetune/Loss": 0.8402996063232422, "Finetune/Loss (Raw)": 0.823948860168457, "Finetune/Step": 5067, "Finetune/Step Time": 5.33576182462275} +{"Finetune/Learning Rate": 3.2774882457904386e-08, "Finetune/Loss": 0.8420186638832092, "Finetune/Loss (Raw)": 0.9116348028182983, "Finetune/Step": 5068, "Finetune/Step Time": 5.3291466273367405} +{"Finetune/Learning Rate": 3.227286318714384e-08, "Finetune/Loss": 0.8422813415527344, "Finetune/Loss (Raw)": 0.7843590378761292, "Finetune/Step": 5069, "Finetune/Step Time": 5.331262189894915} +{"Finetune/Learning Rate": 3.177471219668271e-08, "Finetune/Loss": 0.8422106504440308, "Finetune/Loss (Raw)": 0.8098772168159485, "Finetune/Step": 5070, "Finetune/Step Time": 5.328798398375511} +{"Finetune/Learning Rate": 3.1280429679841905e-08, "Finetune/Loss": 0.8425595760345459, "Finetune/Loss (Raw)": 0.8319185376167297, "Finetune/Step": 5071, "Finetune/Step Time": 5.335726855322719} +{"Finetune/Learning Rate": 3.079001582844354e-08, "Finetune/Loss": 0.8437764644622803, "Finetune/Loss (Raw)": 1.0754821300506592, "Finetune/Step": 5072, "Finetune/Step Time": 5.339058792218566} +{"Finetune/Learning Rate": 3.0303470832806494e-08, "Finetune/Loss": 0.8434150218963623, "Finetune/Loss (Raw)": 0.901351809501648, "Finetune/Step": 5073, "Finetune/Step Time": 5.342993127182126} +{"Finetune/Learning Rate": 2.9820794881751934e-08, "Finetune/Loss": 0.844021201133728, "Finetune/Loss (Raw)": 0.9411103129386902, "Finetune/Step": 5074, "Finetune/Step Time": 5.338923797011375} +{"Finetune/Learning Rate": 2.9341988162595593e-08, "Finetune/Loss": 0.8396397829055786, "Finetune/Loss (Raw)": 0.3654438257217407, "Finetune/Step": 5075, "Finetune/Step Time": 5.367607936263084} +{"Finetune/Learning Rate": 2.886705086115216e-08, "Finetune/Loss": 0.8404980301856995, "Finetune/Loss (Raw)": 0.9373287558555603, "Finetune/Step": 5076, "Finetune/Step Time": 5.365808818489313} +{"Finetune/Learning Rate": 2.8395983161736418e-08, "Finetune/Loss": 0.8408730030059814, "Finetune/Loss (Raw)": 0.8558162450790405, "Finetune/Step": 5077, "Finetune/Step Time": 5.387752719223499} +{"Finetune/Learning Rate": 2.7928785247161027e-08, "Finetune/Loss": 0.841274619102478, "Finetune/Loss (Raw)": 0.8664714694023132, "Finetune/Step": 5078, "Finetune/Step Time": 5.371392913162708} +{"Finetune/Learning Rate": 2.7465457298736508e-08, "Finetune/Loss": 0.8420867919921875, "Finetune/Loss (Raw)": 1.0151365995407104, "Finetune/Step": 5079, "Finetune/Step Time": 5.371700368821621} +{"Finetune/Learning Rate": 2.700599949626903e-08, "Finetune/Loss": 0.8424261212348938, "Finetune/Loss (Raw)": 0.9614948630332947, "Finetune/Step": 5080, "Finetune/Step Time": 5.36381458863616} +{"Finetune/Learning Rate": 2.6550412018067072e-08, "Finetune/Loss": 0.8415839672088623, "Finetune/Loss (Raw)": 0.7188147306442261, "Finetune/Step": 5081, "Finetune/Step Time": 5.360185123980045} +{"Finetune/Learning Rate": 2.6098695040935874e-08, "Finetune/Loss": 0.8370468616485596, "Finetune/Loss (Raw)": 0.3427061438560486, "Finetune/Step": 5082, "Finetune/Step Time": 5.335557244718075} +{"Finetune/Learning Rate": 2.5650848740177425e-08, "Finetune/Loss": 0.8369871377944946, "Finetune/Loss (Raw)": 0.8611624240875244, "Finetune/Step": 5083, "Finetune/Step Time": 5.33973322995007} +{"Finetune/Learning Rate": 2.5206873289590484e-08, "Finetune/Loss": 0.8355816006660461, "Finetune/Loss (Raw)": 0.7582345604896545, "Finetune/Step": 5084, "Finetune/Step Time": 5.350229332223535} +{"Finetune/Learning Rate": 2.4766768861475e-08, "Finetune/Loss": 0.834783673286438, "Finetune/Loss (Raw)": 0.8725503087043762, "Finetune/Step": 5085, "Finetune/Step Time": 5.348184013739228} +{"Finetune/Learning Rate": 2.4330535626627684e-08, "Finetune/Loss": 0.8361800909042358, "Finetune/Loss (Raw)": 0.9916819930076599, "Finetune/Step": 5086, "Finetune/Step Time": 5.335079478099942} +{"Finetune/Learning Rate": 2.389817375433978e-08, "Finetune/Loss": 0.8398478031158447, "Finetune/Loss (Raw)": 0.9101974368095398, "Finetune/Step": 5087, "Finetune/Step Time": 5.321349276229739} +{"Finetune/Learning Rate": 2.3469683412403745e-08, "Finetune/Loss": 0.8411888480186462, "Finetune/Loss (Raw)": 1.0005115270614624, "Finetune/Step": 5088, "Finetune/Step Time": 5.321275141090155} +{"Finetune/Learning Rate": 2.304506476710877e-08, "Finetune/Loss": 0.8421155214309692, "Finetune/Loss (Raw)": 0.9345610737800598, "Finetune/Step": 5089, "Finetune/Step Time": 5.326113471761346} +{"Finetune/Learning Rate": 2.262431798323972e-08, "Finetune/Loss": 0.8412299156188965, "Finetune/Loss (Raw)": 0.8073470592498779, "Finetune/Step": 5090, "Finetune/Step Time": 5.327743198722601} +{"Finetune/Learning Rate": 2.2207443224081528e-08, "Finetune/Loss": 0.8420105576515198, "Finetune/Loss (Raw)": 0.9503048658370972, "Finetune/Step": 5091, "Finetune/Step Time": 5.335857376456261} +{"Finetune/Learning Rate": 2.1794440651414782e-08, "Finetune/Loss": 0.8371468186378479, "Finetune/Loss (Raw)": 0.3247257173061371, "Finetune/Step": 5092, "Finetune/Step Time": 5.362819451838732} +{"Finetune/Learning Rate": 2.1385310425517945e-08, "Finetune/Loss": 0.8365572690963745, "Finetune/Loss (Raw)": 0.9000257849693298, "Finetune/Step": 5093, "Finetune/Step Time": 5.369554087519646} +{"Finetune/Learning Rate": 2.0980052705165123e-08, "Finetune/Loss": 0.8366401195526123, "Finetune/Loss (Raw)": 0.8121325969696045, "Finetune/Step": 5094, "Finetune/Step Time": 5.379133243113756} +{"Finetune/Learning Rate": 2.05786676476305e-08, "Finetune/Loss": 0.8348639607429504, "Finetune/Loss (Raw)": 0.7043721079826355, "Finetune/Step": 5095, "Finetune/Step Time": 5.375321554020047} +{"Finetune/Learning Rate": 2.0181155408682816e-08, "Finetune/Loss": 0.8340723514556885, "Finetune/Loss (Raw)": 0.7898172736167908, "Finetune/Step": 5096, "Finetune/Step Time": 5.407756373286247} +{"Finetune/Learning Rate": 1.978751614258978e-08, "Finetune/Loss": 0.832190752029419, "Finetune/Loss (Raw)": 0.6391897797584534, "Finetune/Step": 5097, "Finetune/Step Time": 5.398476488888264} +{"Finetune/Learning Rate": 1.9397750002113634e-08, "Finetune/Loss": 0.8305405378341675, "Finetune/Loss (Raw)": 0.7260132431983948, "Finetune/Step": 5098, "Finetune/Step Time": 5.394734021276236} +{"Finetune/Learning Rate": 1.901185713851672e-08, "Finetune/Loss": 0.828769326210022, "Finetune/Loss (Raw)": 0.7850769758224487, "Finetune/Step": 5099, "Finetune/Step Time": 5.387625258415937} +{"Finetune/Learning Rate": 1.8629837701554797e-08, "Finetune/Loss": 0.8280585408210754, "Finetune/Loss (Raw)": 0.772740364074707, "Finetune/Step": 5100, "Finetune/Step Time": 5.373623657971621} +{"Finetune/Learning Rate": 1.825169183948372e-08, "Finetune/Loss": 0.8264336585998535, "Finetune/Loss (Raw)": 0.7910308241844177, "Finetune/Step": 5101, "Finetune/Step Time": 5.380571926012635} +{"Finetune/Learning Rate": 1.7877419699053876e-08, "Finetune/Loss": 0.8309561014175415, "Finetune/Loss (Raw)": 0.8828950524330139, "Finetune/Step": 5102, "Finetune/Step Time": 5.35243652574718} +{"Finetune/Learning Rate": 1.7507021425512417e-08, "Finetune/Loss": 0.8320999145507812, "Finetune/Loss (Raw)": 0.9586859941482544, "Finetune/Step": 5103, "Finetune/Step Time": 5.347682390362024} +{"Finetune/Learning Rate": 1.7140497162605464e-08, "Finetune/Loss": 0.8293026685714722, "Finetune/Loss (Raw)": 0.6384456753730774, "Finetune/Step": 5104, "Finetune/Step Time": 5.350088378414512} +{"Finetune/Learning Rate": 1.677784705257146e-08, "Finetune/Loss": 0.830350399017334, "Finetune/Loss (Raw)": 0.8856832981109619, "Finetune/Step": 5105, "Finetune/Step Time": 5.328087920323014} +{"Finetune/Learning Rate": 1.6419071236148943e-08, "Finetune/Loss": 0.8317623138427734, "Finetune/Loss (Raw)": 0.9894198775291443, "Finetune/Step": 5106, "Finetune/Step Time": 5.328952187672257} +{"Finetune/Learning Rate": 1.6064169852572088e-08, "Finetune/Loss": 0.8328696489334106, "Finetune/Loss (Raw)": 0.9661546349525452, "Finetune/Step": 5107, "Finetune/Step Time": 5.328118864446878} +{"Finetune/Learning Rate": 1.5713143039571833e-08, "Finetune/Loss": 0.8339699506759644, "Finetune/Loss (Raw)": 0.8481278419494629, "Finetune/Step": 5108, "Finetune/Step Time": 5.311835801228881} +{"Finetune/Learning Rate": 1.5365990933373653e-08, "Finetune/Loss": 0.835495114326477, "Finetune/Loss (Raw)": 0.9655728340148926, "Finetune/Step": 5109, "Finetune/Step Time": 5.314772276207805} +{"Finetune/Learning Rate": 1.5022713668699785e-08, "Finetune/Loss": 0.8348414301872253, "Finetune/Loss (Raw)": 0.881895899772644, "Finetune/Step": 5110, "Finetune/Step Time": 5.326618194580078} +{"Finetune/Learning Rate": 1.4683311378771437e-08, "Finetune/Loss": 0.8375242352485657, "Finetune/Loss (Raw)": 0.7552824020385742, "Finetune/Step": 5111, "Finetune/Step Time": 5.3218075558543205} +{"Finetune/Learning Rate": 1.434778419530325e-08, "Finetune/Loss": 0.8340435028076172, "Finetune/Loss (Raw)": 0.3478361964225769, "Finetune/Step": 5112, "Finetune/Step Time": 5.34210910089314} +{"Finetune/Learning Rate": 1.401613224850551e-08, "Finetune/Loss": 0.8283310532569885, "Finetune/Loss (Raw)": 0.2904934883117676, "Finetune/Step": 5113, "Finetune/Step Time": 5.377238441258669} +{"Finetune/Learning Rate": 1.3688355667087483e-08, "Finetune/Loss": 0.8295007944107056, "Finetune/Loss (Raw)": 0.8553538918495178, "Finetune/Step": 5114, "Finetune/Step Time": 5.378124570474029} +{"Finetune/Learning Rate": 1.3364454578252972e-08, "Finetune/Loss": 0.8283505439758301, "Finetune/Loss (Raw)": 0.7745990753173828, "Finetune/Step": 5115, "Finetune/Step Time": 5.375823734328151} +{"Finetune/Learning Rate": 1.3044429107700319e-08, "Finetune/Loss": 0.8229654431343079, "Finetune/Loss (Raw)": 0.25782057642936707, "Finetune/Step": 5116, "Finetune/Step Time": 5.400378385558724} +{"Finetune/Learning Rate": 1.272827937962573e-08, "Finetune/Loss": 0.8239010572433472, "Finetune/Loss (Raw)": 1.0465710163116455, "Finetune/Step": 5117, "Finetune/Step Time": 5.401366274803877} +{"Finetune/Learning Rate": 1.2416005516722174e-08, "Finetune/Loss": 0.8249057531356812, "Finetune/Loss (Raw)": 0.9943264126777649, "Finetune/Step": 5118, "Finetune/Step Time": 5.403889222070575} +{"Finetune/Learning Rate": 1.2107607640174934e-08, "Finetune/Loss": 0.8263804912567139, "Finetune/Loss (Raw)": 1.0249913930892944, "Finetune/Step": 5119, "Finetune/Step Time": 5.404382519423962} +{"Finetune/Learning Rate": 1.1803085869667164e-08, "Finetune/Loss": 0.8213104009628296, "Finetune/Loss (Raw)": 0.3440427780151367, "Finetune/Step": 5120, "Finetune/Step Time": 5.419470457360148} +{"Finetune/Learning Rate": 1.1502440323379882e-08, "Finetune/Loss": 0.8229674696922302, "Finetune/Loss (Raw)": 0.8136278986930847, "Finetune/Step": 5121, "Finetune/Step Time": 5.405298130586743} +{"Finetune/Learning Rate": 1.1205671117985318e-08, "Finetune/Loss": 0.8225260972976685, "Finetune/Loss (Raw)": 0.7814759612083435, "Finetune/Step": 5122, "Finetune/Step Time": 5.3967962097376585} +{"Finetune/Learning Rate": 1.0912778368655786e-08, "Finetune/Loss": 0.8247029781341553, "Finetune/Loss (Raw)": 0.9014857411384583, "Finetune/Step": 5123, "Finetune/Step Time": 5.398919252678752} +{"Finetune/Learning Rate": 1.0623762189057029e-08, "Finetune/Loss": 0.824122965335846, "Finetune/Loss (Raw)": 0.938309907913208, "Finetune/Step": 5124, "Finetune/Step Time": 5.399885738268495} +{"Finetune/Learning Rate": 1.0338622691349332e-08, "Finetune/Loss": 0.824592113494873, "Finetune/Loss (Raw)": 0.9929673075675964, "Finetune/Step": 5125, "Finetune/Step Time": 5.404987787827849} +{"Finetune/Learning Rate": 1.005735998619084e-08, "Finetune/Loss": 0.8190493583679199, "Finetune/Loss (Raw)": 0.3870375454425812, "Finetune/Step": 5126, "Finetune/Step Time": 5.434761818498373} +{"Finetune/Learning Rate": 9.779974182733131e-09, "Finetune/Loss": 0.8180514574050903, "Finetune/Loss (Raw)": 0.8644698262214661, "Finetune/Step": 5127, "Finetune/Step Time": 5.447814593091607} +{"Finetune/Learning Rate": 9.506465388624541e-09, "Finetune/Loss": 0.8220521211624146, "Finetune/Loss (Raw)": 0.8645597100257874, "Finetune/Step": 5128, "Finetune/Step Time": 5.414639972150326} +{"Finetune/Learning Rate": 9.236833710010163e-09, "Finetune/Loss": 0.8170139789581299, "Finetune/Loss (Raw)": 0.29208630323410034, "Finetune/Step": 5129, "Finetune/Step Time": 5.446819618344307} +{"Finetune/Learning Rate": 8.971079251526293e-09, "Finetune/Loss": 0.8182090520858765, "Finetune/Loss (Raw)": 0.838793158531189, "Finetune/Step": 5130, "Finetune/Step Time": 5.440925749018788} +{"Finetune/Learning Rate": 8.709202116308214e-09, "Finetune/Loss": 0.8176422715187073, "Finetune/Loss (Raw)": 0.7310693264007568, "Finetune/Step": 5131, "Finetune/Step Time": 5.44704619422555} +{"Finetune/Learning Rate": 8.451202405985736e-09, "Finetune/Loss": 0.8177028894424438, "Finetune/Loss (Raw)": 0.7564573287963867, "Finetune/Step": 5132, "Finetune/Step Time": 5.441123092547059} +{"Finetune/Learning Rate": 8.197080220683217e-09, "Finetune/Loss": 0.8177988529205322, "Finetune/Loss (Raw)": 0.8744676113128662, "Finetune/Step": 5133, "Finetune/Step Time": 5.441539894789457} +{"Finetune/Learning Rate": 7.946835659020657e-09, "Finetune/Loss": 0.817792534828186, "Finetune/Loss (Raw)": 0.8888663649559021, "Finetune/Step": 5134, "Finetune/Step Time": 5.445222904905677} +{"Finetune/Learning Rate": 7.700468818111483e-09, "Finetune/Loss": 0.817272424697876, "Finetune/Loss (Raw)": 0.7313144207000732, "Finetune/Step": 5135, "Finetune/Step Time": 5.439401023089886} +{"Finetune/Learning Rate": 7.457979793568105e-09, "Finetune/Loss": 0.8192040920257568, "Finetune/Loss (Raw)": 1.0392378568649292, "Finetune/Step": 5136, "Finetune/Step Time": 5.44919590651989} +{"Finetune/Learning Rate": 7.219368679495242e-09, "Finetune/Loss": 0.8177417516708374, "Finetune/Loss (Raw)": 0.8219151496887207, "Finetune/Step": 5137, "Finetune/Step Time": 5.45207617431879} +{"Finetune/Learning Rate": 6.984635568492159e-09, "Finetune/Loss": 0.8168438673019409, "Finetune/Loss (Raw)": 0.8310005068778992, "Finetune/Step": 5138, "Finetune/Step Time": 5.466157633811235} +{"Finetune/Learning Rate": 6.753780551654876e-09, "Finetune/Loss": 0.8127458095550537, "Finetune/Loss (Raw)": 0.34093213081359863, "Finetune/Step": 5139, "Finetune/Step Time": 5.495834367349744} +{"Finetune/Learning Rate": 6.526803718572838e-09, "Finetune/Loss": 0.8133726716041565, "Finetune/Loss (Raw)": 0.9211588501930237, "Finetune/Step": 5140, "Finetune/Step Time": 5.490481697022915} +{"Finetune/Learning Rate": 6.3037051573333615e-09, "Finetune/Loss": 0.815209150314331, "Finetune/Loss (Raw)": 0.9711079001426697, "Finetune/Step": 5141, "Finetune/Step Time": 5.488038521260023} +{"Finetune/Learning Rate": 6.0844849545149685e-09, "Finetune/Loss": 0.815474271774292, "Finetune/Loss (Raw)": 0.8270998597145081, "Finetune/Step": 5142, "Finetune/Step Time": 5.491965875029564} +{"Finetune/Learning Rate": 5.869143195192939e-09, "Finetune/Loss": 0.8167875409126282, "Finetune/Loss (Raw)": 0.9463348388671875, "Finetune/Step": 5143, "Finetune/Step Time": 5.49487154558301} +{"Finetune/Learning Rate": 5.657679962938201e-09, "Finetune/Loss": 0.8151870965957642, "Finetune/Loss (Raw)": 0.6656088829040527, "Finetune/Step": 5144, "Finetune/Step Time": 5.500590331852436} +{"Finetune/Learning Rate": 5.4500953398139985e-09, "Finetune/Loss": 0.8153930306434631, "Finetune/Loss (Raw)": 0.8228000998497009, "Finetune/Step": 5145, "Finetune/Step Time": 5.501622619107366} +{"Finetune/Learning Rate": 5.246389406381447e-09, "Finetune/Loss": 0.8173990845680237, "Finetune/Loss (Raw)": 1.0052908658981323, "Finetune/Step": 5146, "Finetune/Step Time": 5.496608218178153} +{"Finetune/Learning Rate": 5.046562241693975e-09, "Finetune/Loss": 0.8179781436920166, "Finetune/Loss (Raw)": 0.8605967164039612, "Finetune/Step": 5147, "Finetune/Step Time": 5.4985729791224} +{"Finetune/Learning Rate": 4.850613923300662e-09, "Finetune/Loss": 0.8205714225769043, "Finetune/Loss (Raw)": 0.6783994436264038, "Finetune/Step": 5148, "Finetune/Step Time": 5.4738861955702305} +{"Finetune/Learning Rate": 4.658544527244013e-09, "Finetune/Loss": 0.8197425603866577, "Finetune/Loss (Raw)": 0.8241509199142456, "Finetune/Step": 5149, "Finetune/Step Time": 5.480738898739219} +{"Finetune/Learning Rate": 4.470354128065513e-09, "Finetune/Loss": 0.8193023204803467, "Finetune/Loss (Raw)": 0.8261622786521912, "Finetune/Step": 5150, "Finetune/Step Time": 5.487636191770434} +{"Finetune/Learning Rate": 4.286042798794521e-09, "Finetune/Loss": 0.8186512589454651, "Finetune/Loss (Raw)": 0.8014076948165894, "Finetune/Step": 5151, "Finetune/Step Time": 5.4894436057657} +{"Finetune/Learning Rate": 4.105610610961597e-09, "Finetune/Loss": 0.8182985782623291, "Finetune/Loss (Raw)": 0.7816818356513977, "Finetune/Step": 5152, "Finetune/Step Time": 5.486108303070068} +{"Finetune/Learning Rate": 3.929057634587397e-09, "Finetune/Loss": 0.8150643706321716, "Finetune/Loss (Raw)": 0.42027750611305237, "Finetune/Step": 5153, "Finetune/Step Time": 5.505057053640485} +{"Finetune/Learning Rate": 3.756383938189334e-09, "Finetune/Loss": 0.8106898069381714, "Finetune/Loss (Raw)": 0.37085503339767456, "Finetune/Step": 5154, "Finetune/Step Time": 5.530718736350536} +{"Finetune/Learning Rate": 3.587589588778251e-09, "Finetune/Loss": 0.8093981146812439, "Finetune/Loss (Raw)": 0.7616024613380432, "Finetune/Step": 5155, "Finetune/Step Time": 5.531381642445922} +{"Finetune/Learning Rate": 3.4226746518606356e-09, "Finetune/Loss": 0.8093824982643127, "Finetune/Loss (Raw)": 0.9609635472297668, "Finetune/Step": 5156, "Finetune/Step Time": 5.5438726507127285} +{"Finetune/Learning Rate": 3.2616391914364056e-09, "Finetune/Loss": 0.8092101216316223, "Finetune/Loss (Raw)": 0.8648160696029663, "Finetune/Step": 5157, "Finetune/Step Time": 5.543207321316004} +{"Finetune/Learning Rate": 3.1044832699989035e-09, "Finetune/Loss": 0.8094245195388794, "Finetune/Loss (Raw)": 0.8498439192771912, "Finetune/Step": 5158, "Finetune/Step Time": 5.534808740019798} +{"Finetune/Learning Rate": 2.9512069485393426e-09, "Finetune/Loss": 0.8098278045654297, "Finetune/Loss (Raw)": 0.9508233070373535, "Finetune/Step": 5159, "Finetune/Step Time": 5.542427025735378} +{"Finetune/Learning Rate": 2.801810286540141e-09, "Finetune/Loss": 0.8102564215660095, "Finetune/Loss (Raw)": 1.0062191486358643, "Finetune/Step": 5160, "Finetune/Step Time": 5.541578123345971} +{"Finetune/Learning Rate": 2.656293341980476e-09, "Finetune/Loss": 0.8108314871788025, "Finetune/Loss (Raw)": 0.9267701506614685, "Finetune/Step": 5161, "Finetune/Step Time": 5.542535414919257} +{"Finetune/Learning Rate": 2.5146561713307315e-09, "Finetune/Loss": 0.8098926544189453, "Finetune/Loss (Raw)": 0.7941397428512573, "Finetune/Step": 5162, "Finetune/Step Time": 5.552296631038189} +{"Finetune/Learning Rate": 2.3768988295580497e-09, "Finetune/Loss": 0.8108048439025879, "Finetune/Loss (Raw)": 0.911263108253479, "Finetune/Step": 5163, "Finetune/Step Time": 5.55394016392529} +{"Finetune/Learning Rate": 2.2430213701252203e-09, "Finetune/Loss": 0.8119528889656067, "Finetune/Loss (Raw)": 0.9721254706382751, "Finetune/Step": 5164, "Finetune/Step Time": 5.568029947578907} +{"Finetune/Learning Rate": 2.1130238449851292e-09, "Finetune/Loss": 0.809941291809082, "Finetune/Loss (Raw)": 0.716628909111023, "Finetune/Step": 5165, "Finetune/Step Time": 5.578012024983764} +{"Finetune/Learning Rate": 1.986906304588532e-09, "Finetune/Loss": 0.8094832897186279, "Finetune/Loss (Raw)": 0.9147460460662842, "Finetune/Step": 5166, "Finetune/Step Time": 5.585125645622611} +{"Finetune/Learning Rate": 1.8646687978784992e-09, "Finetune/Loss": 0.8072113990783691, "Finetune/Loss (Raw)": 0.7922278046607971, "Finetune/Step": 5167, "Finetune/Step Time": 5.578048963099718} +{"Finetune/Learning Rate": 1.7463113722937518e-09, "Finetune/Loss": 0.8087888956069946, "Finetune/Loss (Raw)": 0.9343368411064148, "Finetune/Step": 5168, "Finetune/Step Time": 5.568573269993067} +{"Finetune/Learning Rate": 1.631834073765326e-09, "Finetune/Loss": 0.8082860708236694, "Finetune/Loss (Raw)": 0.9278413653373718, "Finetune/Step": 5169, "Finetune/Step Time": 5.574971400201321} +{"Finetune/Learning Rate": 1.5212369467210164e-09, "Finetune/Loss": 0.8064301013946533, "Finetune/Loss (Raw)": 0.724584698677063, "Finetune/Step": 5170, "Finetune/Step Time": 5.571856863796711} +{"Finetune/Learning Rate": 1.4145200340809352e-09, "Finetune/Loss": 0.8074241280555725, "Finetune/Loss (Raw)": 1.0002593994140625, "Finetune/Step": 5171, "Finetune/Step Time": 5.567451499402523} +{"Finetune/Learning Rate": 1.3116833772597315e-09, "Finetune/Loss": 0.8074725866317749, "Finetune/Loss (Raw)": 0.8588520884513855, "Finetune/Step": 5172, "Finetune/Step Time": 5.564297636970878} +{"Finetune/Learning Rate": 1.2127270161654824e-09, "Finetune/Loss": 0.8114336729049683, "Finetune/Loss (Raw)": 0.7956429719924927, "Finetune/Step": 5173, "Finetune/Step Time": 5.5330796875059605} +{"Finetune/Learning Rate": 1.1176509892030229e-09, "Finetune/Loss": 0.8173459768295288, "Finetune/Loss (Raw)": 1.0905593633651733, "Finetune/Step": 5174, "Finetune/Step Time": 5.497586823999882} +{"Finetune/Learning Rate": 1.0264553332683946e-09, "Finetune/Loss": 0.8172172904014587, "Finetune/Loss (Raw)": 0.8806093335151672, "Finetune/Step": 5175, "Finetune/Step Time": 5.504402609542012} +{"Finetune/Learning Rate": 9.39140083752177e-10, "Finetune/Loss": 0.8122178912162781, "Finetune/Loss (Raw)": 0.3257467746734619, "Finetune/Step": 5176, "Finetune/Step Time": 5.539121961221099} +{"Finetune/Learning Rate": 8.557052745405969e-10, "Finetune/Loss": 0.8132811188697815, "Finetune/Loss (Raw)": 0.9901182055473328, "Finetune/Step": 5177, "Finetune/Step Time": 5.52705180644989} +{"Finetune/Learning Rate": 7.761509380133092e-10, "Finetune/Loss": 0.812637209892273, "Finetune/Loss (Raw)": 0.7600358128547668, "Finetune/Step": 5178, "Finetune/Step Time": 5.511937107890844} +{"Finetune/Learning Rate": 7.004771050433956e-10, "Finetune/Loss": 0.8127577900886536, "Finetune/Loss (Raw)": 0.7929243445396423, "Finetune/Step": 5179, "Finetune/Step Time": 5.508046789094806} +{"Finetune/Learning Rate": 6.286838049984755e-10, "Finetune/Loss": 0.8159753680229187, "Finetune/Loss (Raw)": 0.7839578986167908, "Finetune/Step": 5180, "Finetune/Step Time": 5.485664827749133} +{"Finetune/Learning Rate": 5.607710657407061e-10, "Finetune/Loss": 0.8167165517807007, "Finetune/Loss (Raw)": 0.9016562104225159, "Finetune/Step": 5181, "Finetune/Step Time": 5.480955969542265} +{"Finetune/Learning Rate": 4.967389136245615e-10, "Finetune/Loss": 0.8144099116325378, "Finetune/Loss (Raw)": 0.7182123064994812, "Finetune/Step": 5182, "Finetune/Step Time": 5.484252009540796} +{"Finetune/Learning Rate": 4.3658737350016354e-10, "Finetune/Loss": 0.814934253692627, "Finetune/Loss (Raw)": 0.8988580703735352, "Finetune/Step": 5183, "Finetune/Step Time": 5.4887406174093485} +{"Finetune/Learning Rate": 3.8031646871106166e-10, "Finetune/Loss": 0.8146680593490601, "Finetune/Loss (Raw)": 0.8538431525230408, "Finetune/Step": 5184, "Finetune/Step Time": 5.499092919752002} +{"Finetune/Learning Rate": 3.279262210953427e-10, "Finetune/Loss": 0.8155325651168823, "Finetune/Loss (Raw)": 0.9914944171905518, "Finetune/Step": 5185, "Finetune/Step Time": 5.515826044604182} +{"Finetune/Learning Rate": 2.794166509845209e-10, "Finetune/Loss": 0.8115699291229248, "Finetune/Loss (Raw)": 0.37943196296691895, "Finetune/Step": 5186, "Finetune/Step Time": 5.5441896710544825} +{"Finetune/Learning Rate": 2.3478777720353785e-10, "Finetune/Loss": 0.8099503517150879, "Finetune/Loss (Raw)": 0.7509982585906982, "Finetune/Step": 5187, "Finetune/Step Time": 5.562200354412198} +{"Finetune/Learning Rate": 1.940396170718728e-10, "Finetune/Loss": 0.8086986541748047, "Finetune/Loss (Raw)": 0.8137142062187195, "Finetune/Step": 5188, "Finetune/Step Time": 5.563565203920007} +{"Finetune/Learning Rate": 1.5717218640465272e-10, "Finetune/Loss": 0.8075987100601196, "Finetune/Loss (Raw)": 0.7684016227722168, "Finetune/Step": 5189, "Finetune/Step Time": 5.576619379222393} +{"Finetune/Learning Rate": 1.2418549950821145e-10, "Finetune/Loss": 0.8071743845939636, "Finetune/Loss (Raw)": 0.7446853518486023, "Finetune/Step": 5190, "Finetune/Step Time": 5.574938662350178} +{"Finetune/Learning Rate": 9.507956918453076e-11, "Finetune/Loss": 0.8054454326629639, "Finetune/Loss (Raw)": 0.7255394458770752, "Finetune/Step": 5191, "Finetune/Step Time": 5.587520934641361} +{"Finetune/Learning Rate": 6.985440672790945e-11, "Finetune/Loss": 0.8060706853866577, "Finetune/Loss (Raw)": 0.844984769821167, "Finetune/Step": 5192, "Finetune/Step Time": 5.583210403099656} +{"Finetune/Learning Rate": 4.851002192940435e-11, "Finetune/Loss": 0.8042833805084229, "Finetune/Loss (Raw)": 0.7354269623756409, "Finetune/Step": 5193, "Finetune/Step Time": 5.594780722633004} +{"Finetune/Learning Rate": 3.1046423071279254e-11, "Finetune/Loss": 0.7995514869689941, "Finetune/Loss (Raw)": 0.3319265842437744, "Finetune/Step": 5194, "Finetune/Step Time": 5.624965002760291} +{"Finetune/Learning Rate": 1.7463616930335493e-11, "Finetune/Loss": 0.7997618317604065, "Finetune/Loss (Raw)": 0.8508701324462891, "Finetune/Step": 5195, "Finetune/Step Time": 5.6175906509160995} +{"Finetune/Learning Rate": 7.761608780132434e-12, "Finetune/Loss": 0.7989013195037842, "Finetune/Loss (Raw)": 0.8014904856681824, "Finetune/Step": 5196, "Finetune/Step Time": 5.623703815042973} +{"Finetune/Learning Rate": 1.9404023832159115e-12, "Finetune/Loss": 0.799597978591919, "Finetune/Loss (Raw)": 0.8735318779945374, "Finetune/Step": 5197, "Finetune/Step Time": 5.615746298804879} +{"Finetune/Learning Rate": 0.0, "Finetune/Loss": 0.8000658750534058, "Finetune/Loss (Raw)": 0.8697790503501892, "Finetune/Step": 5198, "Finetune/Step Time": 5.611753150820732} diff --git a/qformer2_256/run-metrics.jsonl b/qformer2_256/run-metrics.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..f422218e34e7508980afb138942ae5efc9610f03 --- /dev/null +++ b/qformer2_256/run-metrics.jsonl @@ -0,0 +1 @@ +{"hparams": {"dataset": {"align_stage_components": ["download/llava-laion-cc-sbu-558k/chat.json", "download/llava-laion-cc-sbu-558k"], "dataset_id": "llava-v15", "dataset_root_dir": "data", "finetune_stage_components": ["download/llava-v1.5-instruct/llava_v1_5_mix665k.json", "download/llava-v1.5-instruct"], "type": "llava-v15"}, "hf_token": ".hf_token", "max_length": 4096, "model": {"align_epochs": 1, "align_global_batch_size": 256, "align_learning_rate": 0.001, "align_lr_scheduler_type": "linear-warmup+cosine-decay", "align_max_grad_norm": 1.0, "align_max_steps": null, "align_per_device_batch_size": 16, "align_train_strategy": "fsdp-shard-grad-op", "align_warmup_ratio": 0.03, "align_weight_decay": 0.0, "arch_specifier": "qformer2_256", "enable_gradient_checkpointing": true, "enable_mixed_precision_training": true, "finetune_epochs": 1, "finetune_global_batch_size": 128, "finetune_learning_rate": 2e-05, "finetune_lr_scheduler_type": "linear-warmup+cosine-decay", "finetune_max_grad_norm": 1.0, "finetune_max_steps": null, "finetune_per_device_batch_size": 16, "finetune_train_strategy": "fsdp-full-shard", "finetune_warmup_ratio": 0.03, "finetune_weight_decay": 0.1, "image_resize_strategy": "letterbox", "llm_backbone_id": "vicuna-v15-7b", "llm_max_length": 2048, "model_id": "one-stage+7b_qformer2_256", "reduce_in_full_precision": false, "type": "one-stage+7b", "vision_backbone_id": "clip-vit-l-336px"}, "pretrained_checkpoint": null, "run_id": "qformer2_256", "run_root_dir": "runs", "seed": 7, "stage": "finetune", "trackers": ["jsonl", "wandb"], "wandb_entity": "lilei_stones", "wandb_project": "hf-vlms"}, "run_id": "qformer2_256"} diff --git a/qformer2_256/wandb/debug-internal.log b/qformer2_256/wandb/debug-internal.log new file mode 120000 index 0000000000000000000000000000000000000000..a897fd5f39ae487fcc55c02fec63c14e6576ee50 --- /dev/null +++ b/qformer2_256/wandb/debug-internal.log @@ -0,0 +1 @@ +run-20240312_142903-yrs4wcl6/logs/debug-internal.log \ No newline at end of file diff --git a/qformer2_256/wandb/debug.log b/qformer2_256/wandb/debug.log new file mode 120000 index 0000000000000000000000000000000000000000..4a815c1864bb6053f2f3bb1ab3ef9ff0fc716dda --- /dev/null +++ b/qformer2_256/wandb/debug.log @@ -0,0 +1 @@ +run-20240312_142903-yrs4wcl6/logs/debug.log \ No newline at end of file diff --git a/qformer2_256/wandb/latest-run b/qformer2_256/wandb/latest-run new file mode 120000 index 0000000000000000000000000000000000000000..5c8abb9a9d33703f052782a105299512ee47bb44 --- /dev/null +++ b/qformer2_256/wandb/latest-run @@ -0,0 +1 @@ +run-20240312_142903-yrs4wcl6 \ No newline at end of file diff --git a/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/conda-environment.yaml b/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/conda-environment.yaml new file mode 100644 index 0000000000000000000000000000000000000000..799a6f3a49522423b58a3ae238b95584ecfe64c1 --- /dev/null +++ b/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/conda-environment.yaml @@ -0,0 +1,194 @@ +name: pvlm +channels: + - pytorch + - nvidia + - defaults +dependencies: + - _libgcc_mutex=0.1=main + - _openmp_mutex=5.1=1_gnu + - blas=1.0=mkl + - bzip2=1.0.8=h7b6447c_0 + - ca-certificates=2023.12.12=h06a4308_0 + - certifi=2024.2.2=py310h06a4308_0 + - charset-normalizer=2.0.4=pyhd3eb1b0_0 + - cuda-cudart=11.8.89=0 + - cuda-cupti=11.8.87=0 + - cuda-libraries=11.8.0=0 + - cuda-nvrtc=11.8.89=0 + - cuda-nvtx=11.8.86=0 + - cuda-runtime=11.8.0=0 + - ffmpeg=4.3=hf484d3e_0 + - filelock=3.13.1=py310h06a4308_0 + - freetype=2.12.1=h4a9f257_0 + - gmp=6.2.1=h295c915_3 + - gmpy2=2.1.2=py310heeb90bb_0 + - gnutls=3.6.15=he1e5248_0 + - idna=3.4=py310h06a4308_0 + - intel-openmp=2023.1.0=hdb19cb5_46306 + - jinja2=3.1.3=py310h06a4308_0 + - jpeg=9e=h5eee18b_1 + - lame=3.100=h7b6447c_0 + - lcms2=2.12=h3be6417_0 + - ld_impl_linux-64=2.38=h1181459_1 + - lerc=3.0=h295c915_0 + - libcublas=11.11.3.6=0 + - libcufft=10.9.0.58=0 + - libcufile=1.8.1.2=0 + - libcurand=10.3.4.107=0 + - libcusolver=11.4.1.48=0 + - libcusparse=11.7.5.86=0 + - libdeflate=1.17=h5eee18b_1 + - libffi=3.4.4=h6a678d5_0 + - libgcc-ng=11.2.0=h1234567_1 + - libgomp=11.2.0=h1234567_1 + - libiconv=1.16=h7f8727e_2 + - libidn2=2.3.4=h5eee18b_0 + - libjpeg-turbo=2.0.0=h9bf148f_0 + - libnpp=11.8.0.86=0 + - libnvjpeg=11.9.0.86=0 + - libpng=1.6.39=h5eee18b_0 + - libstdcxx-ng=11.2.0=h1234567_1 + - libtasn1=4.19.0=h5eee18b_0 + - libtiff=4.5.1=h6a678d5_0 + - libunistring=0.9.10=h27cfd23_0 + - libuuid=1.41.5=h5eee18b_0 + - libwebp-base=1.3.2=h5eee18b_0 + - llvm-openmp=14.0.6=h9e868ea_0 + - lz4-c=1.9.4=h6a678d5_0 + - markupsafe=2.1.3=py310h5eee18b_0 + - mkl=2023.1.0=h213fc3f_46344 + - mkl-service=2.4.0=py310h5eee18b_1 + - mkl_fft=1.3.8=py310h5eee18b_0 + - mkl_random=1.2.4=py310hdb19cb5_0 + - mpc=1.1.0=h10f8cd9_1 + - mpfr=4.0.2=hb69a4c5_1 + - mpmath=1.3.0=py310h06a4308_0 + - ncurses=6.4=h6a678d5_0 + - nettle=3.7.3=hbbd107a_1 + - networkx=3.1=py310h06a4308_0 + - numpy=1.26.4=py310h5f9d8c6_0 + - numpy-base=1.26.4=py310hb5e798b_0 + - openh264=2.1.1=h4ff587b_0 + - openjpeg=2.4.0=h3ad879b_0 + - openssl=3.0.13=h7f8727e_0 + - pillow=10.2.0=py310h5eee18b_0 + - pip=23.3.1=py310h06a4308_0 + - python=3.10.13=h955ad1f_0 + - pytorch=2.2.1=py3.10_cuda11.8_cudnn8.7.0_0 + - pytorch-cuda=11.8=h7e8668a_5 + - pytorch-mutex=1.0=cuda + - pyyaml=6.0.1=py310h5eee18b_0 + - readline=8.2=h5eee18b_0 + - requests=2.31.0=py310h06a4308_1 + - setuptools=68.2.2=py310h06a4308_0 + - sqlite=3.41.2=h5eee18b_0 + - sympy=1.12=py310h06a4308_0 + - tbb=2021.8.0=hdb19cb5_0 + - tk=8.6.12=h1ccaba5_0 + - torchaudio=2.2.1=py310_cu118 + - torchtriton=2.2.0=py310 + - torchvision=0.17.1=py310_cu118 + - typing_extensions=4.9.0=py310h06a4308_1 + - urllib3=2.1.0=py310h06a4308_0 + - wheel=0.41.2=py310h06a4308_0 + - xz=5.4.5=h5eee18b_0 + - yaml=0.2.5=h7b6447c_0 + - zlib=1.2.13=h5eee18b_0 + - zstd=1.5.5=hc292b87_0 + - pip: + - accelerate==0.27.2 + - aiohttp==3.9.3 + - aiosignal==1.3.1 + - annotated-types==0.6.0 + - appdirs==1.4.4 + - async-timeout==4.0.3 + - attrs==23.2.0 + - beautifulsoup4==4.12.3 + - blis==0.7.11 + - catalogue==2.0.10 + - click==8.1.7 + - cloudpathlib==0.16.0 + - confection==0.1.4 + - contourpy==1.2.0 + - cycler==0.12.1 + - cymem==2.0.8 + - datasets==2.18.0 + - deepspeed==0.13.4 + - dill==0.3.8 + - docker-pycreds==0.4.0 + - draccus==0.7.1 + - einops==0.7.0 + - flash-attn==2.5.5 + - fonttools==4.49.0 + - frozenlist==1.4.1 + - fsspec==2024.2.0 + - gdown==5.1.0 + - gitdb==4.0.11 + - gitpython==3.1.42 + - hjson==3.1.0 + - huggingface-hub==0.20.3 + - install==1.3.5 + - jsonlines==4.0.0 + - kiwisolver==1.4.5 + - langcodes==3.3.0 + - markdown-it-py==3.0.0 + - matplotlib==3.8.3 + - mdurl==0.1.2 + - mergedeep==1.3.4 + - multidict==6.0.5 + - multiprocess==0.70.16 + - murmurhash==1.0.10 + - mypy-extensions==1.0.0 + - ninja==1.11.1.1 + - packaging==23.2 + - pandas==2.2.1 + - preshed==3.0.9 + - prismatic==0.0.1 + - protobuf==4.25.3 + - psutil==5.9.8 + - py-cpuinfo==9.0.0 + - pyarrow==15.0.0 + - pyarrow-hotfix==0.6 + - pycocoevalcap==1.2 + - pycocotools==2.0.7 + - pydantic==2.6.3 + - pydantic-core==2.16.3 + - pygments==2.17.2 + - pynvml==11.5.0 + - pyparsing==3.1.1 + - pysocks==1.7.1 + - pythainlp==5.0.1 + - python-dateutil==2.9.0.post0 + - pytz==2024.1 + - pyyaml-include==1.3.2 + - regex==2023.12.25 + - rich==13.7.0 + - safetensors==0.4.2 + - sentencepiece==0.2.0 + - sentry-sdk==1.40.5 + - setproctitle==1.3.3 + - six==1.16.0 + - smart-open==6.4.0 + - smmap==5.0.1 + - soupsieve==2.5 + - spacy==3.7.4 + - spacy-legacy==3.0.12 + - spacy-loggers==1.0.5 + - srsly==2.4.8 + - sudachidict-core==20240109 + - sudachipy==0.6.8 + - thinc==8.2.3 + - timm==0.9.16 + - tokenizers==0.15.2 + - toml==0.10.2 + - tqdm==4.66.2 + - transformers==4.38.1 + - typer==0.9.0 + - typing-inspect==0.9.0 + - tzdata==2024.1 + - wandb==0.16.3 + - wasabi==1.1.2 + - weasel==0.3.4 + - xxhash==3.4.1 + - yarl==1.9.4 +prefix: /home/lilei/anaconda3/envs/pvlm diff --git a/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/config.yaml b/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..50830b801a03a220b17eeb5d5876473d41017879 --- /dev/null +++ b/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/config.yaml @@ -0,0 +1,119 @@ +wandb_version: 1 + +model: + desc: null + value: + type: one-stage+7b + model_id: one-stage+7b_qformer2_256 + arch_specifier: qformer2_256 + vision_backbone_id: clip-vit-l-336px + llm_backbone_id: vicuna-v15-7b + image_resize_strategy: letterbox + llm_max_length: 2048 + align_epochs: 1 + align_max_steps: null + align_global_batch_size: 256 + align_per_device_batch_size: 16 + align_learning_rate: 0.001 + align_weight_decay: 0.0 + align_max_grad_norm: 1.0 + align_lr_scheduler_type: linear-warmup+cosine-decay + align_warmup_ratio: 0.03 + align_train_strategy: fsdp-shard-grad-op + finetune_epochs: 1 + finetune_max_steps: null + finetune_global_batch_size: 128 + finetune_per_device_batch_size: 16 + finetune_learning_rate: 2.0e-05 + finetune_weight_decay: 0.1 + finetune_max_grad_norm: 1.0 + finetune_lr_scheduler_type: linear-warmup+cosine-decay + finetune_warmup_ratio: 0.03 + finetune_train_strategy: fsdp-full-shard + enable_gradient_checkpointing: true + enable_mixed_precision_training: true + reduce_in_full_precision: false +dataset: + desc: null + value: + type: llava-v15 + dataset_id: llava-v15 + align_stage_components: + - download/llava-laion-cc-sbu-558k/chat.json + - download/llava-laion-cc-sbu-558k + finetune_stage_components: + - download/llava-v1.5-instruct/llava_v1_5_mix665k.json + - download/llava-v1.5-instruct + dataset_root_dir: data +stage: + desc: null + value: finetune +pretrained_checkpoint: + desc: null + value: null +run_id: + desc: null + value: qformer2_256 +run_root_dir: + desc: null + value: runs +seed: + desc: null + value: 7 +hf_token: + desc: null + value: .hf_token +trackers: + desc: null + value: + - jsonl + - wandb +wandb_project: + desc: null + value: hf-vlms +wandb_entity: + desc: null + value: lilei_stones +max_length: + desc: null + value: 4096 +_wandb: + desc: null + value: + python_version: 3.10.13 + cli_version: 0.16.3 + framework: huggingface + huggingface_version: 4.38.1 + is_jupyter_run: false + is_kaggle_kernel: false + start_time: 1710253743.636872 + t: + 1: + - 1 + - 11 + - 41 + - 49 + - 51 + - 55 + - 63 + - 71 + 2: + - 1 + - 11 + - 41 + - 49 + - 51 + - 55 + - 63 + - 71 + 3: + - 2 + - 13 + - 16 + - 23 + 4: 3.10.13 + 5: 0.16.3 + 6: 4.38.1 + 8: + - 5 + 13: linux-x86_64 diff --git a/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log b/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..54853aa64d2c52e8ca68897f7dc574007ed2f423 --- /dev/null +++ b/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log @@ -0,0 +1,5203 @@ + +03/12 [14:29:13] INFO  | >> [*] Starting Training Loop ]8;id=72103;file:///home/lilei/prismatic-vlms/scripts/pretrain.py\pretrain.py]8;;\:]8;id=880770;file:///home/lilei/prismatic-vlms/scripts/pretrain.py#226\226]8;;\ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +=>> [Global Step] 005198 =>> LR :: 0.000000 -- Loss :: 0.8001: 100%|█████████████████████████████████████████████| 5198/5198 [7:52:38<00:00, 5.07s/it] +03/12 [22:26:21] INFO  | >> [*] Done with Training =>> Finalizing Metrics ]8;id=696414;file:///home/lilei/prismatic-vlms/scripts/pretrain.py\pretrain.py]8;;\:]8;id=68157;file:///home/lilei/prismatic-vlms/scripts/pretrain.py#230\230]8;;\ \ No newline at end of file diff --git a/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/requirements.txt b/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..52215be56573f749927d1265d23e43a5fe36d5b1 --- /dev/null +++ b/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/requirements.txt @@ -0,0 +1,122 @@ +accelerate==0.27.2 +aiohttp==3.9.3 +aiosignal==1.3.1 +annotated-types==0.6.0 +appdirs==1.4.4 +async-timeout==4.0.3 +attrs==23.2.0 +beautifulsoup4==4.12.3 +blis==0.7.11 +catalogue==2.0.10 +certifi==2024.2.2 +charset-normalizer==2.0.4 +click==8.1.7 +cloudpathlib==0.16.0 +confection==0.1.4 +contourpy==1.2.0 +cycler==0.12.1 +cymem==2.0.8 +datasets==2.18.0 +deepspeed==0.13.4 +dill==0.3.8 +docker-pycreds==0.4.0 +draccus==0.7.1 +einops==0.7.0 +filelock==3.13.1 +flash-attn==2.5.5 +fonttools==4.49.0 +frozenlist==1.4.1 +fsspec==2024.2.0 +gdown==5.1.0 +gitdb==4.0.11 +gitpython==3.1.42 +gmpy2==2.1.2 +hjson==3.1.0 +huggingface-hub==0.20.3 +idna==3.4 +install==1.3.5 +jinja2==3.1.3 +jsonlines==4.0.0 +kiwisolver==1.4.5 +langcodes==3.3.0 +markdown-it-py==3.0.0 +markupsafe==2.1.3 +matplotlib==3.8.3 +mdurl==0.1.2 +mergedeep==1.3.4 +mkl-fft==1.3.8 +mkl-random==1.2.4 +mkl-service==2.4.0 +mpmath==1.3.0 +multidict==6.0.5 +multiprocess==0.70.16 +murmurhash==1.0.10 +mypy-extensions==1.0.0 +networkx==3.1 +ninja==1.11.1.1 +numpy==1.26.4 +ocifs==1.3.1 +packaging==23.2 +pandas==2.2.1 +pillow==10.2.0 +pip==23.3.1 +preshed==3.0.9 +prismatic==0.0.1 +protobuf==4.25.3 +psutil==5.9.8 +py-cpuinfo==9.0.0 +pyarrow-hotfix==0.6 +pyarrow==15.0.0 +pycocoevalcap==1.2 +pycocotools==2.0.7 +pydantic-core==2.16.3 +pydantic==2.6.3 +pygments==2.17.2 +pynvml==11.5.0 +pyparsing==3.1.1 +pysocks==1.7.1 +pythainlp==5.0.1 +python-dateutil==2.9.0.post0 +pytz==2024.1 +pyyaml-include==1.3.2 +pyyaml==6.0.1 +regex==2023.12.25 +requests==2.31.0 +rich==13.7.0 +safetensors==0.4.2 +sentencepiece==0.2.0 +sentry-sdk==1.40.5 +setproctitle==1.3.3 +setuptools==68.2.2 +six==1.16.0 +smart-open==6.4.0 +smmap==5.0.1 +soupsieve==2.5 +spacy-legacy==3.0.12 +spacy-loggers==1.0.5 +spacy==3.7.4 +srsly==2.4.8 +sudachidict-core==20240109 +sudachipy==0.6.8 +sympy==1.12 +thinc==8.2.3 +timm==0.9.16 +tokenizers==0.15.2 +toml==0.10.2 +torch==2.2.1 +torchaudio==2.2.1 +torchvision==0.17.1 +tqdm==4.66.2 +transformers==4.38.1 +triton==2.2.0 +typer==0.9.0 +typing-extensions==4.9.0 +typing-inspect==0.9.0 +tzdata==2024.1 +urllib3==2.1.0 +wandb==0.16.3 +wasabi==1.1.2 +weasel==0.3.4 +wheel==0.41.2 +xxhash==3.4.1 +yarl==1.9.4 \ No newline at end of file diff --git a/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-metadata.json b/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..e26a9c4bf8ad468466b5c5d79d81b77472102356 --- /dev/null +++ b/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-metadata.json @@ -0,0 +1,1360 @@ +{ + "os": "Linux-5.15.0-1027-oracle-x86_64-with-glibc2.31", + "python": "3.10.13", + "heartbeatAt": "2024-03-12T14:29:08.013324", + "startedAt": "2024-03-12T14:29:03.589151", + "docker": null, + "cuda": null, + "args": [ + "--model.type", + "one-stage+7b", + "--model.model_id", + "one-stage+7b_qformer2_256", + "--run_id", + "qformer2_256", + "--model.arch_specifier", + "qformer2_256" + ], + "state": "running", + "program": "/home/lilei/prismatic-vlms/scripts/pretrain.py", + "codePathLocal": "scripts/pretrain.py", + "codePath": "scripts/pretrain.py", + "git": { + "remote": "https://github.com/TRI-ML/prismatic-vlms.git", + "commit": "b1c33c284c6e76d908c29e5545b00f5b8b4c601f" + }, + "email": "tobiaslee@qq.com", + "root": "/home/lilei/prismatic-vlms", + "host": "compute-permanent-node-824", + "username": "lilei", + "executable": "/home/lilei/anaconda3/envs/pvlm/bin/python", + "cpu_count": 128, + "cpu_count_logical": 255, + "cpu_freq": { + "current": 2578.3530274509803, + "min": 1500.0, + "max": 2550.0 + }, + "cpu_freq_per_core": [ + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 3249.171, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 3249.526, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 3250.052, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 3248.918, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 3249.534, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 3248.538, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 3249.553, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 3249.537, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 3249.359, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 3072.53, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 3249.242, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 3249.221, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + } + ], + "disk": { + "/": { + "total": 242.1086883544922, + "used": 50.49263000488281 + } + }, + "gpu": "NVIDIA A100-SXM4-80GB", + "gpu_count": 8, + "gpu_devices": [ + { + "name": "NVIDIA A100-SXM4-80GB", + "memory_total": 85899345920 + }, + { + "name": "NVIDIA A100-SXM4-80GB", + "memory_total": 85899345920 + }, + { + "name": "NVIDIA A100-SXM4-80GB", + "memory_total": 85899345920 + }, + { + "name": "NVIDIA A100-SXM4-80GB", + "memory_total": 85899345920 + }, + { + "name": "NVIDIA A100-SXM4-80GB", + "memory_total": 85899345920 + }, + { + "name": "NVIDIA A100-SXM4-80GB", + "memory_total": 85899345920 + }, + { + "name": "NVIDIA A100-SXM4-80GB", + "memory_total": 85899345920 + }, + { + "name": "NVIDIA A100-SXM4-80GB", + "memory_total": 85899345920 + } + ], + "memory": { + "total": 2015.6942749023438 + } +} diff --git a/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json b/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json new file mode 100644 index 0000000000000000000000000000000000000000..c977d5b98fac955057d111ae3a1555ef8b091064 --- /dev/null +++ b/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json @@ -0,0 +1 @@ +{"Finetune/Step": 5198, "Finetune/Loss": 0.8000658750534058, "Finetune/Loss (Raw)": 0.8697790503501892, "Finetune/Learning Rate": 0.0, "Finetune/Step Time": 5.611753150820732, "_timestamp": 1710282114.6367517, "_runtime": 28370.999879598618, "_step": 5198, "_wandb": {"runtime": 28633}} \ No newline at end of file diff --git a/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/logs/debug-internal.log b/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..532c0cba79fc41e822328c06c7760d970a5c6f00 --- /dev/null +++ b/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/logs/debug-internal.log @@ -0,0 +1,43317 @@ +2024-03-12 14:29:03,635 INFO StreamThr :2491270 [internal.py:wandb_internal():86] W&B internal server running at pid: 2491270, started at: 2024-03-12 14:29:03.634346 +2024-03-12 14:29:03,636 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status +2024-03-12 14:29:03,649 INFO WriterThread:2491270 [datastore.py:open_for_write():87] open: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/run-yrs4wcl6.wandb +2024-03-12 14:29:03,651 DEBUG SenderThread:2491270 [sender.py:send():382] send: header +2024-03-12 14:29:03,676 DEBUG SenderThread:2491270 [sender.py:send():382] send: run +2024-03-12 14:29:07,863 INFO SenderThread:2491270 [dir_watcher.py:__init__():211] watching files in: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files +2024-03-12 14:29:07,863 INFO SenderThread:2491270 [sender.py:_start_run_threads():1136] run started: yrs4wcl6 with start time 1710253743.636872 +2024-03-12 14:29:07,875 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: check_version +2024-03-12 14:29:07,875 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: check_version +2024-03-12 14:29:07,930 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: run_start +2024-03-12 14:29:07,941 DEBUG HandlerThread:2491270 [system_info.py:__init__():27] System info init +2024-03-12 14:29:07,941 DEBUG HandlerThread:2491270 [system_info.py:__init__():42] System info init done +2024-03-12 14:29:07,941 INFO HandlerThread:2491270 [system_monitor.py:start():194] Starting system monitor +2024-03-12 14:29:07,941 INFO SystemMonitor:2491270 [system_monitor.py:_start():158] Starting system asset monitoring threads +2024-03-12 14:29:07,942 INFO HandlerThread:2491270 [system_monitor.py:probe():214] Collecting system info +2024-03-12 14:29:07,943 INFO SystemMonitor:2491270 [interfaces.py:start():190] Started cpu monitoring +2024-03-12 14:29:07,943 INFO SystemMonitor:2491270 [interfaces.py:start():190] Started disk monitoring +2024-03-12 14:29:07,943 INFO SystemMonitor:2491270 [interfaces.py:start():190] Started gpu monitoring +2024-03-12 14:29:07,944 INFO SystemMonitor:2491270 [interfaces.py:start():190] Started memory monitoring +2024-03-12 14:29:07,944 INFO SystemMonitor:2491270 [interfaces.py:start():190] Started network monitoring +2024-03-12 14:29:08,013 DEBUG HandlerThread:2491270 [system_info.py:probe():151] Probing system +2024-03-12 14:29:08,017 DEBUG HandlerThread:2491270 [system_info.py:_probe_git():136] Probing git +2024-03-12 14:29:08,037 DEBUG HandlerThread:2491270 [system_info.py:_probe_git():144] Probing git done +2024-03-12 14:29:08,037 DEBUG HandlerThread:2491270 [system_info.py:probe():199] Probing system done +2024-03-12 14:29:08,037 DEBUG HandlerThread:2491270 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-1027-oracle-x86_64-with-glibc2.31', 'python': '3.10.13', 'heartbeatAt': '2024-03-12T14:29:08.013324', 'startedAt': '2024-03-12T14:29:03.589151', 'docker': None, 'cuda': None, 'args': ('--model.type', 'one-stage+7b', '--model.model_id', 'one-stage+7b_qformer2_256', '--run_id', 'qformer2_256', '--model.arch_specifier', 'qformer2_256'), 'state': 'running', 'program': '/home/lilei/prismatic-vlms/scripts/pretrain.py', 'codePathLocal': 'scripts/pretrain.py', 'codePath': 'scripts/pretrain.py', 'git': {'remote': 'https://github.com/TRI-ML/prismatic-vlms.git', 'commit': 'b1c33c284c6e76d908c29e5545b00f5b8b4c601f'}, 'email': 'tobiaslee@qq.com', 'root': '/home/lilei/prismatic-vlms', 'host': 'compute-permanent-node-824', 'username': 'lilei', 'executable': '/home/lilei/anaconda3/envs/pvlm/bin/python', 'cpu_count': 128, 'cpu_count_logical': 255, 'cpu_freq': {'current': 2578.3530274509803, 'min': 1500.0, 'max': 2550.0}, 'cpu_freq_per_core': [{'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 3249.171, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 3249.526, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 3250.052, 'min': 1500.0, 'max': 2550.0}, {'current': 3248.918, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 3249.534, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 3248.538, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 3249.553, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 3249.537, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 3249.359, 'min': 1500.0, 'max': 2550.0}, {'current': 3072.53, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 3249.242, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 3249.221, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}], 'disk': {'/': {'total': 242.1086883544922, 'used': 50.49263000488281}}, 'gpu': 'NVIDIA A100-SXM4-80GB', 'gpu_count': 8, 'gpu_devices': [{'name': 'NVIDIA A100-SXM4-80GB', 'memory_total': 85899345920}, {'name': 'NVIDIA A100-SXM4-80GB', 'memory_total': 85899345920}, {'name': 'NVIDIA A100-SXM4-80GB', 'memory_total': 85899345920}, {'name': 'NVIDIA A100-SXM4-80GB', 'memory_total': 85899345920}, {'name': 'NVIDIA A100-SXM4-80GB', 'memory_total': 85899345920}, {'name': 'NVIDIA A100-SXM4-80GB', 'memory_total': 85899345920}, {'name': 'NVIDIA A100-SXM4-80GB', 'memory_total': 85899345920}, {'name': 'NVIDIA A100-SXM4-80GB', 'memory_total': 85899345920}], 'memory': {'total': 2015.6942749023438}} +2024-03-12 14:29:08,037 INFO HandlerThread:2491270 [system_monitor.py:probe():224] Finished collecting system info +2024-03-12 14:29:08,037 INFO HandlerThread:2491270 [system_monitor.py:probe():227] Publishing system info +2024-03-12 14:29:08,038 DEBUG HandlerThread:2491270 [system_info.py:_save_conda():208] Saving list of conda packages installed into the current environment +2024-03-12 14:29:08,866 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_created():271] file/dir created: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/conda-environment.yaml +2024-03-12 14:29:13,095 DEBUG HandlerThread:2491270 [system_info.py:_save_conda():220] Saving conda packages done +2024-03-12 14:29:13,103 INFO HandlerThread:2491270 [system_monitor.py:probe():229] Finished publishing system info +2024-03-12 14:29:13,108 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:29:13,108 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 14:29:13,109 DEBUG SenderThread:2491270 [sender.py:send():382] send: files +2024-03-12 14:29:13,109 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-metadata.json with policy now +2024-03-12 14:29:13,119 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: python_packages +2024-03-12 14:29:13,119 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: python_packages +2024-03-12 14:29:13,120 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:29:13,121 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:29:13,130 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:29:13,426 DEBUG SenderThread:2491270 [sender.py:send():382] send: telemetry +2024-03-12 14:29:13,869 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/conda-environment.yaml +2024-03-12 14:29:13,870 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_created():271] file/dir created: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-metadata.json +2024-03-12 14:29:13,870 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_created():271] file/dir created: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/requirements.txt +2024-03-12 14:29:13,870 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_created():271] file/dir created: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:29:14,066 INFO wandb-upload_0:2491270 [upload_job.py:push():131] Uploaded file /tmp/tmpa2oem8qgwandb/ijnknfbx-wandb-metadata.json +2024-03-12 14:29:14,428 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:29:15,873 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:29:18,877 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:29:19,512 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:29:24,512 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:29:28,120 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:29:28,120 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:29:28,120 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:29:30,403 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:29:35,407 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:29:35,890 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/config.yaml +2024-03-12 14:29:39,369 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:29:40,894 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:29:41,414 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:29:43,120 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:29:43,120 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:29:43,120 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:29:47,105 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:29:47,106 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:29:47,106 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:29:47,107 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:29:47,111 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:29:47,899 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_created():271] file/dir created: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:29:48,903 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:29:51,328 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:29:51,329 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:29:51,329 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:29:51,334 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:29:51,905 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:29:52,335 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:29:52,908 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:29:55,727 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:29:55,728 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:29:55,728 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:29:55,733 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:29:55,910 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:29:56,912 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:29:57,734 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:29:58,120 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:29:58,120 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:29:58,122 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:30:01,154 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:30:01,154 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:30:01,155 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:30:01,160 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:30:01,915 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:30:02,918 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:30:03,198 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:30:06,058 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:30:06,059 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:30:06,059 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:30:06,064 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:30:06,923 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:30:06,923 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:30:07,944 DEBUG SystemMonitor:2491270 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2024-03-12 14:30:07,947 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 14:30:08,948 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:30:10,970 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:30:10,970 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:30:10,971 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:30:10,975 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:30:11,927 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:30:12,930 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:30:13,120 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:30:13,120 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:30:13,120 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:30:14,388 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:30:17,115 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:30:17,116 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:30:17,116 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:30:17,121 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:30:17,933 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:30:18,936 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:30:20,158 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:30:22,148 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:30:22,149 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:30:22,149 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:30:22,154 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:30:22,938 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:30:24,941 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:30:26,155 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:30:28,120 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:30:28,120 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:30:28,120 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:30:29,592 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:30:29,593 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:30:29,593 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:30:29,598 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:30:29,944 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:30:30,947 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:30:31,634 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:30:34,244 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:30:34,245 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:30:34,245 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:30:34,250 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:30:34,949 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:30:36,951 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:30:37,251 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:30:37,949 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 14:30:39,050 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:30:39,051 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:30:39,051 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:30:39,055 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:30:39,953 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:30:40,956 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:30:43,056 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:30:43,120 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:30:43,120 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:30:43,120 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:30:43,852 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:30:43,853 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:30:43,853 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:30:43,858 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:30:43,957 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:30:44,960 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:30:48,248 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:30:48,249 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:30:48,250 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:30:48,250 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:30:48,254 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:30:48,962 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:30:50,965 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:30:53,255 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:30:54,115 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:30:54,116 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:30:54,116 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:30:54,121 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:30:54,968 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:30:56,971 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:30:58,120 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:30:58,120 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:30:58,121 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:30:58,316 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:31:00,123 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:31:00,124 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:31:00,124 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:31:00,129 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:31:00,975 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:31:02,978 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:31:04,167 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:31:05,101 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:31:05,102 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:31:05,103 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:31:05,107 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:31:05,981 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:31:06,983 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:31:07,952 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 14:31:09,953 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:31:10,495 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:31:10,495 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:31:10,496 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:31:10,501 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:31:10,985 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:31:12,989 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:31:13,120 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:31:13,120 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:31:13,121 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:31:14,968 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:31:14,968 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:31:14,969 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:31:14,969 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:31:14,974 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:31:14,990 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:31:16,993 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:31:19,201 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:31:19,201 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:31:19,202 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:31:19,206 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:31:19,995 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:31:20,207 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:31:20,998 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:31:25,207 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:31:25,254 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:31:25,255 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:31:25,255 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:31:25,260 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:31:26,001 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:31:27,003 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:31:28,120 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:31:28,121 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:31:28,121 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:31:30,161 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:31:30,162 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:31:30,162 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:31:30,166 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:31:31,006 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:31:31,206 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:31:33,009 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:31:36,207 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:31:37,954 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 14:31:38,412 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:31:38,412 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:31:38,413 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:31:38,418 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:31:39,013 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:31:41,016 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:31:41,419 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:31:42,684 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:31:42,685 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:31:42,685 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:31:42,690 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:31:43,018 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:31:43,120 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:31:43,121 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:31:43,121 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:31:45,021 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:31:47,407 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:31:47,621 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:31:47,621 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:31:47,622 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:31:47,626 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:31:48,023 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:31:49,025 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:31:52,667 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:31:56,842 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:31:56,843 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:31:56,843 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:31:56,848 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:31:57,030 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:31:57,849 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:31:58,120 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:31:58,120 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:31:58,121 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:31:59,033 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:32:00,860 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:32:00,861 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:32:00,861 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:32:00,866 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:32:01,035 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:32:02,902 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:32:03,039 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:32:05,155 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:32:05,155 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:32:05,156 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:32:05,161 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:32:06,041 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:32:07,044 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:32:07,964 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 14:32:07,966 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:32:10,939 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:32:10,940 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:32:10,940 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:32:10,946 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:32:11,046 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:32:13,049 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:32:13,120 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:32:13,121 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:32:13,121 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:32:13,360 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:32:16,819 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:32:16,820 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:32:16,820 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:32:16,825 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:32:17,052 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:32:18,862 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:32:19,055 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:32:21,899 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:32:21,900 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:32:21,900 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:32:21,905 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:32:22,057 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:32:23,059 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:32:23,905 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:32:28,120 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:32:28,121 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:32:28,121 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:32:28,734 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:32:28,735 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:32:28,735 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:32:28,740 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:32:29,063 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:32:29,778 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:32:31,066 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:32:34,078 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:32:34,079 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:32:34,079 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:32:34,084 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:32:35,070 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:32:35,071 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:32:35,085 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:32:37,958 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 14:32:38,802 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:32:38,803 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:32:38,803 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:32:38,809 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:32:39,073 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:32:40,809 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:32:41,077 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:32:43,120 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:32:43,121 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:32:43,121 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:32:43,853 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:32:43,854 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:32:43,854 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:32:43,859 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:32:44,079 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:32:45,081 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:32:45,898 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:32:50,187 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:32:50,188 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:32:50,188 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:32:50,194 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:32:51,085 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:32:51,085 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:32:51,194 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:32:56,195 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:32:58,032 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:32:58,033 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:32:58,033 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:32:58,038 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:32:58,089 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:32:58,121 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:32:58,121 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:32:58,121 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:32:59,092 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:33:01,382 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:33:02,337 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:33:02,338 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:33:02,338 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:33:02,343 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:33:03,096 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:33:05,099 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:33:06,383 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:33:06,653 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:33:06,654 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:33:06,655 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:33:06,659 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:33:07,100 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:33:07,966 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 14:33:09,104 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:33:11,968 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:33:12,823 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:33:12,824 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:33:12,824 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:33:12,830 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:33:13,106 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:33:13,121 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:33:13,121 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:33:13,121 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:33:15,109 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:33:16,631 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:33:16,632 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:33:16,632 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:33:16,637 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:33:17,111 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:33:17,674 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:33:19,114 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:33:20,924 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:33:20,924 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:33:20,925 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:33:20,929 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:33:21,115 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:33:22,930 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:33:23,118 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:33:26,682 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:33:26,683 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:33:26,683 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:33:26,688 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:33:27,121 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:33:28,121 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:33:28,121 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:33:28,121 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:33:28,383 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:33:29,126 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:33:32,153 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:33:32,154 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:33:32,154 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:33:32,159 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:33:33,130 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:33:33,130 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:33:34,198 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:33:37,963 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 14:33:39,964 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:33:40,597 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:33:40,597 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:33:40,598 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:33:40,602 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:33:41,136 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:33:43,121 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:33:43,121 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:33:43,121 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:33:43,139 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:33:45,413 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:33:50,413 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:33:50,691 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:33:50,692 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:33:50,692 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:33:50,697 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:33:51,145 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:33:53,148 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:33:55,734 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:33:56,225 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:33:56,226 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:33:56,226 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:33:56,231 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:33:57,152 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:33:57,153 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:33:58,121 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:33:58,121 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:33:58,122 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:34:01,393 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:34:02,540 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:34:02,541 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:34:02,541 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:34:02,546 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:34:03,158 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:34:05,161 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:34:06,582 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:34:07,320 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:34:07,320 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:34:07,321 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:34:07,325 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:34:07,965 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 14:34:08,163 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:34:09,166 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:34:11,966 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:34:12,294 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:34:12,295 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:34:12,295 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:34:12,300 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:34:13,121 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:34:13,121 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:34:13,122 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:34:13,170 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:34:13,171 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:34:16,903 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:34:16,904 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:34:16,904 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:34:16,908 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:34:17,173 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:34:17,946 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:34:19,176 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:34:22,404 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:34:22,405 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:34:22,405 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:34:22,410 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:34:23,179 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:34:23,411 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:34:25,182 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:34:28,121 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:34:28,121 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:34:28,122 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:34:29,014 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:34:29,014 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:34:29,014 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:34:29,015 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:34:29,019 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:34:29,184 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:34:31,187 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:34:34,059 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:34:34,619 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:34:34,620 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:34:34,620 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:34:34,625 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:34:35,189 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:34:37,193 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:34:37,972 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 14:34:39,974 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:34:40,645 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:34:40,646 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:34:40,646 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:34:40,651 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:34:41,196 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:34:43,121 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:34:43,121 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:34:43,123 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:34:43,199 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:34:45,362 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:34:47,126 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:34:47,127 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:34:47,127 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:34:47,132 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:34:47,201 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:34:49,204 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:34:51,170 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:34:51,791 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:34:51,792 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:34:51,793 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:34:51,797 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:34:52,206 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:34:53,208 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:34:56,798 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:34:58,121 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:34:58,121 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:34:58,122 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:34:59,259 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:34:59,259 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:34:59,260 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:34:59,264 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:35:00,214 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:35:01,216 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:35:02,302 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:35:05,630 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:35:05,630 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:35:05,631 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:35:05,635 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:35:06,221 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:35:07,223 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:35:07,636 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:35:07,969 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 14:35:10,105 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:35:10,105 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:35:10,106 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:35:10,110 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:35:10,225 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:35:11,227 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:35:13,111 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:35:13,121 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:35:13,121 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:35:13,122 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:35:15,141 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:35:15,141 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:35:15,142 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:35:15,147 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:35:15,230 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:35:17,233 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:35:18,186 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:35:19,500 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:35:19,501 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:35:19,501 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:35:19,506 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:35:20,235 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:35:21,238 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:35:23,507 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:35:24,105 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:35:24,105 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:35:24,106 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:35:24,110 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:35:24,240 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:35:25,242 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:35:28,121 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:35:28,122 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:35:28,122 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:35:29,320 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:35:32,196 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:35:32,197 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:35:32,197 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:35:32,202 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:35:32,246 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:35:33,249 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:35:35,238 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:35:37,702 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:35:37,703 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:35:37,703 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:35:37,708 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:35:37,977 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 14:35:38,253 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:35:39,256 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:35:40,978 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:35:42,658 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:35:42,658 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:35:42,659 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:35:42,664 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:35:43,122 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:35:43,122 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:35:43,122 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:35:43,259 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:35:45,262 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:35:46,377 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:35:46,926 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:35:46,927 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:35:46,927 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:35:46,932 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:35:47,263 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:35:49,266 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:35:51,970 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:35:52,636 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:35:52,636 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:35:52,636 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:35:52,641 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:35:53,269 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:35:55,272 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:35:56,889 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:35:56,890 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:35:56,890 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:35:56,895 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:35:57,273 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:35:57,895 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:35:58,122 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:35:58,122 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:35:58,122 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:35:59,276 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:36:01,369 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:36:01,370 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:36:01,370 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:36:01,375 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:36:02,280 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:36:03,282 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:36:03,414 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:36:05,958 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:36:05,958 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:36:05,959 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:36:05,963 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:36:06,284 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:36:07,287 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:36:07,973 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 14:36:08,975 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:36:10,090 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:36:10,090 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:36:10,091 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:36:10,095 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:36:10,288 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:36:11,291 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:36:13,122 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:36:13,122 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:36:13,122 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:36:14,361 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:36:14,566 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:36:14,566 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:36:14,567 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:36:14,571 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:36:15,294 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:36:17,297 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:36:19,124 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:36:19,124 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:36:19,125 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:36:19,130 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:36:19,298 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:36:20,130 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:36:21,301 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:36:24,435 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:36:24,436 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:36:24,436 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:36:24,441 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:36:25,303 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:36:25,304 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:36:25,442 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:36:28,122 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:36:28,122 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:36:28,122 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:36:29,730 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:36:29,731 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:36:29,731 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:36:29,736 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:36:30,306 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:36:30,774 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:36:31,309 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:36:33,819 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:36:33,819 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:36:33,820 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:36:33,825 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:36:34,310 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:36:35,313 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:36:35,825 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:36:37,976 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 14:36:37,996 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:36:37,997 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:36:37,997 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:36:38,002 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:36:38,315 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:36:39,318 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:36:41,003 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:36:42,495 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:36:42,496 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:36:42,496 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:36:42,501 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:36:43,122 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:36:43,122 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:36:43,122 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:36:43,323 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:36:43,323 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:36:46,330 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:36:48,646 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:36:48,647 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:36:48,647 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:36:48,652 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:36:49,326 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:36:51,329 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:36:51,690 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:36:54,151 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:36:54,152 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:36:54,152 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:36:54,157 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:36:54,331 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:36:55,334 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:36:57,158 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:36:58,122 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:36:58,122 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:36:58,122 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:36:58,574 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:36:58,575 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:36:58,575 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:36:58,580 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:36:59,336 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:37:01,341 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:37:02,621 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:37:04,930 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:37:04,931 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:37:04,931 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:37:04,936 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:37:05,343 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:37:07,347 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:37:07,937 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:37:07,978 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 14:37:10,804 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:37:10,805 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:37:10,805 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:37:10,810 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:37:11,349 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:37:13,122 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:37:13,122 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:37:13,123 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:37:13,352 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:37:13,391 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:37:15,320 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:37:15,321 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:37:15,321 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:37:15,326 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:37:15,354 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:37:17,357 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:37:19,363 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:37:24,363 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:37:24,431 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:37:24,431 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:37:24,432 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:37:24,436 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:37:25,364 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:37:25,364 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:37:28,122 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:37:28,122 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:37:28,123 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:37:28,917 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:37:28,918 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:37:28,918 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:37:28,923 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:37:29,367 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:37:29,962 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:37:31,370 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:37:34,963 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:37:35,297 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:37:35,297 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:37:35,298 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:37:35,303 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:37:35,372 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:37:37,376 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:37:37,980 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 14:37:39,817 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:37:39,818 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:37:39,819 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:37:39,824 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:37:40,378 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:37:40,824 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:37:41,380 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:37:43,122 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:37:43,122 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:37:43,123 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:37:46,227 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:37:46,228 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:37:46,228 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:37:46,228 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:37:46,233 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:37:46,384 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:37:47,386 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:37:51,271 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:37:52,176 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:37:52,177 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:37:52,178 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:37:52,183 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:37:52,389 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:37:53,392 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:37:56,851 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:37:56,852 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:37:56,853 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:37:56,853 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:37:56,858 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:37:57,394 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:37:58,122 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:37:58,123 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:37:58,123 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:37:59,397 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:38:01,882 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:38:01,882 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:38:01,883 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:38:01,883 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:38:01,888 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:38:02,401 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:38:03,403 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:38:05,968 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:38:05,969 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:38:05,969 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:38:05,975 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:38:06,406 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:38:06,975 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:38:07,408 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:38:07,988 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 14:38:10,519 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:38:10,520 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:38:10,520 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:38:10,525 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:38:11,413 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:38:11,413 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:38:12,525 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:38:13,122 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:38:13,123 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:38:13,123 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:38:16,444 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:38:16,444 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:38:16,445 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:38:16,450 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:38:17,419 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:38:17,419 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:38:18,486 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:38:21,695 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:38:21,696 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:38:21,696 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:38:21,701 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:38:22,422 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:38:23,425 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:38:23,701 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:38:26,186 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:38:26,186 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:38:26,187 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:38:26,191 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:38:26,426 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:38:27,429 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:38:28,122 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:38:28,123 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:38:28,123 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:38:29,386 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:38:34,386 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:38:34,430 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:38:34,430 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:38:34,430 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:38:34,435 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:38:35,436 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:38:35,436 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:38:37,984 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 14:38:39,439 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:38:39,439 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:38:39,440 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:38:39,441 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:38:39,446 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:38:40,443 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:38:41,445 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:38:43,122 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:38:43,123 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:38:43,123 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:38:45,236 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:38:45,237 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:38:45,237 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:38:45,237 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:38:45,242 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:38:45,447 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:38:47,451 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:38:49,224 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:38:49,225 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:38:49,225 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:38:49,230 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:38:49,452 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:38:51,230 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:38:51,455 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:38:56,231 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:38:57,639 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:38:57,640 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:38:57,640 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:38:57,644 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:38:58,123 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:38:58,123 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:38:58,123 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:38:58,459 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:38:59,461 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:39:01,362 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:39:01,946 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:39:01,947 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:39:01,947 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:39:01,952 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:39:02,465 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:39:03,468 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:39:05,857 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:39:05,858 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:39:05,858 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:39:05,863 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:39:06,470 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:39:06,864 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:39:07,473 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:39:07,992 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 14:39:10,228 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:39:10,229 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:39:10,229 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:39:10,234 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:39:10,475 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:39:11,478 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:39:12,235 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:39:13,123 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:39:13,123 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:39:13,123 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:39:14,688 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:39:14,688 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:39:14,689 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:39:14,694 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:39:15,481 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:39:17,482 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:39:17,730 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:39:19,380 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:39:19,381 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:39:19,381 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:39:19,386 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:39:19,483 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:39:21,487 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:39:23,387 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:39:23,788 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:39:23,788 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:39:23,789 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:39:23,794 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:39:24,489 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:39:25,493 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:39:28,062 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:39:28,063 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:39:28,063 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:39:28,068 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:39:28,123 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:39:28,123 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:39:28,123 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:39:28,415 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:39:28,495 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:39:29,498 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:39:33,416 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:39:34,264 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:39:34,265 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:39:34,265 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:39:34,270 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:39:34,501 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:39:35,504 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:39:37,989 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 14:39:38,990 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:39:40,817 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:39:40,817 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:39:40,818 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:39:40,823 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:39:41,509 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:39:43,123 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:39:43,123 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:39:43,124 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:39:43,512 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:39:44,425 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:39:46,740 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:39:46,740 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:39:46,741 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:39:46,746 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:39:47,515 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:39:49,518 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:39:49,782 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:39:54,783 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:39:55,309 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:39:55,310 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:39:55,310 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:39:55,315 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:39:55,522 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:39:57,526 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:39:58,123 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:39:58,123 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:39:58,123 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:40:00,424 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:40:00,496 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:40:00,497 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:40:00,497 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:40:00,503 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:40:00,529 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:40:01,532 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:40:05,539 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:40:07,352 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:40:07,352 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:40:07,353 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:40:07,358 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:40:07,537 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:40:07,991 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 14:40:09,542 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:40:10,993 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:40:12,420 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:40:12,420 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:40:12,421 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:40:12,427 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:40:12,544 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:40:13,123 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:40:13,123 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:40:13,124 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:40:13,546 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:40:16,412 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:40:16,850 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:40:16,851 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:40:16,851 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:40:16,857 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:40:17,549 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:40:19,552 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:40:21,426 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:40:21,427 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:40:21,428 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:40:21,428 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:40:21,433 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:40:21,553 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:40:23,557 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:40:26,434 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:40:26,556 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:40:26,557 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:40:26,557 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:40:26,562 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:40:27,561 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:40:27,561 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:40:28,123 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:40:28,123 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:40:28,124 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:40:32,408 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:40:33,194 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:40:33,194 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:40:33,194 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:40:33,206 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:40:33,565 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:40:35,573 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:40:37,610 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:40:37,619 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:40:37,620 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:40:37,620 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:40:37,625 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:40:37,994 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 14:40:38,575 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:40:39,583 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:40:42,091 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:40:42,092 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:40:42,092 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:40:42,103 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:40:42,587 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:40:43,104 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:40:43,123 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:40:43,123 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:40:43,124 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:40:43,593 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:40:46,896 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:40:46,897 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:40:46,897 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:40:46,906 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:40:47,596 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:40:48,938 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:40:49,602 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:40:51,641 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:40:51,642 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:40:51,642 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:40:51,653 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:40:52,604 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:40:53,608 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:40:54,653 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:40:58,123 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:40:58,124 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:40:58,124 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:40:58,665 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:40:58,666 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:40:58,666 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:40:58,677 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:40:59,616 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:40:59,617 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:40:59,710 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:41:03,010 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:41:03,011 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:41:03,011 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:41:03,019 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:41:03,622 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:41:05,019 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:41:05,629 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:41:07,702 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:41:07,702 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:41:07,703 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:41:07,710 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:41:07,996 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 14:41:08,631 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:41:09,635 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:41:10,998 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:41:12,212 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:41:12,213 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:41:12,213 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:41:12,223 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:41:12,638 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:41:13,123 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:41:13,124 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:41:13,124 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:41:13,645 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:41:16,413 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:41:17,781 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:41:17,782 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:41:17,782 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:41:17,789 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:41:18,650 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:41:19,656 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:41:21,827 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:41:22,015 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:41:22,016 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:41:22,017 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:41:22,026 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:41:22,658 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:41:23,665 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:41:26,457 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:41:26,458 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:41:26,458 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:41:26,471 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:41:26,667 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:41:27,471 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:41:27,676 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:41:28,123 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:41:28,124 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:41:28,124 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:41:33,062 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:41:33,063 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:41:33,063 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:41:33,063 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:41:33,071 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:41:33,680 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:41:35,688 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:41:37,504 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:41:37,505 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:41:37,505 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:41:37,514 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:41:37,692 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:41:38,009 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 14:41:39,010 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:41:39,698 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:41:42,587 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:41:42,588 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:41:42,588 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:41:42,595 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:41:42,700 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:41:43,124 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:41:43,124 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:41:43,124 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:41:43,706 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:41:44,403 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:41:46,884 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:41:46,884 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:41:46,884 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:41:46,897 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:41:47,709 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:41:49,711 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:41:49,926 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:41:52,171 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:41:52,171 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:41:52,172 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:41:52,181 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:41:52,713 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:41:53,719 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:41:55,181 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:41:56,016 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:41:56,017 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:41:56,017 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:41:56,024 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:41:56,721 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:41:57,728 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:41:58,124 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:41:58,124 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:41:58,124 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:42:00,362 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:42:00,863 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:42:00,864 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:42:00,864 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:42:00,872 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:42:01,733 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:42:03,737 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:42:04,919 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:42:04,920 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:42:04,920 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:42:04,928 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:42:05,739 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:42:05,928 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:42:07,746 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:42:08,000 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 14:42:08,858 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:42:08,859 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:42:08,859 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:42:08,865 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:42:09,750 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:42:09,750 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:42:11,865 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:42:13,124 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:42:13,124 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:42:13,124 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:42:17,400 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:42:18,448 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:42:18,449 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:42:18,449 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:42:18,459 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:42:18,760 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:42:19,762 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:42:22,491 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:42:26,189 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:42:26,190 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:42:26,190 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:42:26,203 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:42:26,767 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:42:27,773 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:42:28,124 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:42:28,124 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:42:28,124 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:42:28,356 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:42:33,357 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:42:34,638 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:42:34,639 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:42:34,639 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:42:34,646 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:42:34,779 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:42:35,783 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:42:38,002 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 14:42:39,004 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:42:43,047 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:42:43,048 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:42:43,049 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:42:43,062 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:42:43,124 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:42:43,124 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:42:43,124 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:42:43,790 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:42:44,355 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:42:45,797 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:42:47,192 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:42:47,193 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:42:47,193 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:42:47,205 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:42:47,798 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:42:49,800 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:42:50,235 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:42:52,455 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:42:52,463 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:42:52,463 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:42:52,469 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:42:52,802 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:42:53,805 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:42:55,470 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:42:57,021 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:42:57,022 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:42:57,022 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:42:57,032 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:42:57,810 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:42:58,124 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:42:58,124 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:42:58,125 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:42:59,815 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:43:01,426 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:43:01,704 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:43:01,705 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:43:01,705 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:43:01,714 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:43:01,819 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:43:03,826 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:43:06,747 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:43:07,127 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:43:07,128 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:43:07,128 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:43:07,135 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:43:07,830 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:43:08,005 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 14:43:09,839 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:43:12,007 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:43:12,178 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:43:12,179 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:43:12,180 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:43:12,189 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:43:12,841 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:43:13,124 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:43:13,124 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:43:13,125 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:43:13,847 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:43:17,115 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:43:17,115 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:43:17,116 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:43:17,116 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:43:17,128 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:43:17,852 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:43:19,857 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:43:22,159 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:43:23,245 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:43:23,246 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:43:23,246 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:43:23,252 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:43:23,860 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:43:25,868 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:43:27,253 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:43:27,887 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:43:27,888 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:43:27,888 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:43:27,896 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:43:28,124 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:43:28,124 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:43:28,125 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:43:28,871 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:43:29,877 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:43:32,412 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:43:32,990 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:43:32,990 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:43:32,991 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:43:32,998 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:43:33,881 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:43:35,888 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:43:38,007 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 14:43:38,008 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:43:41,492 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:43:41,493 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:43:41,494 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:43:41,505 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:43:41,895 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:43:43,124 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:43:43,125 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:43:43,125 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:43:43,346 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:43:43,899 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:43:46,016 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:43:46,016 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:43:46,017 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:43:46,025 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:43:46,902 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:43:47,907 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:43:49,058 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:43:50,794 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:43:50,795 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:43:50,795 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:43:50,805 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:43:50,910 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:43:51,912 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:43:54,806 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:43:54,862 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:43:54,862 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:43:54,863 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:43:54,871 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:43:54,915 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:43:55,920 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:43:58,124 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:43:58,124 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:43:58,125 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:44:00,348 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:44:00,349 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:44:00,349 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:44:00,349 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:44:00,357 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:44:00,927 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:44:01,931 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:44:05,225 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:44:05,226 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:44:05,226 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:44:05,233 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:44:05,935 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:44:06,234 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:44:07,940 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:44:08,019 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 14:44:09,734 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:44:09,734 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:44:09,735 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:44:09,741 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:44:09,942 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:44:11,741 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:44:11,947 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:44:13,124 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:44:13,125 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:44:13,125 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:44:14,853 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:44:14,854 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:44:14,854 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:44:14,862 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:44:14,950 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:44:15,954 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:44:16,898 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:44:20,922 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:44:20,923 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:44:20,924 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:44:20,931 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:44:20,958 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:44:21,932 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:44:21,963 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:44:25,810 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:44:25,811 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:44:25,811 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:44:25,818 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:44:25,966 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:44:27,819 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:44:27,973 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:44:28,124 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:44:28,125 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:44:28,125 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:44:30,369 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:44:30,370 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:44:30,370 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:44:30,377 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:44:30,975 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:44:31,980 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:44:33,414 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:44:34,885 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:44:34,886 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:44:34,886 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:44:34,896 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:44:34,983 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:44:35,986 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:44:38,011 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 14:44:39,012 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:44:39,593 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:44:39,594 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:44:39,595 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:44:39,600 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:44:39,990 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:44:41,995 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:44:43,124 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:44:43,125 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:44:43,125 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:44:44,058 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:44:44,059 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:44:44,059 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:44:44,059 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:44:44,065 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:44:44,998 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:44:46,002 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:44:49,103 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:44:52,191 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:44:52,192 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:44:52,192 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:44:52,201 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:44:53,009 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:44:54,013 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:44:54,201 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:44:57,694 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:44:57,694 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:44:57,695 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:44:57,704 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:44:58,017 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:44:58,125 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:44:58,125 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:44:58,125 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:44:59,376 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:45:00,024 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:45:03,497 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:45:03,498 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:45:03,498 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:45:03,504 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:45:04,029 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:45:04,542 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:45:06,033 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:45:07,888 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:45:07,889 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:45:07,889 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:45:07,895 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:45:08,020 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 14:45:08,035 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:45:10,022 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:45:10,038 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:45:13,058 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:45:13,058 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:45:13,059 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:45:13,065 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:45:13,125 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:45:13,125 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:45:13,125 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:45:14,044 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:45:14,045 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:45:15,423 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:45:17,359 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:45:17,360 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:45:17,360 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:45:17,365 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:45:18,048 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:45:20,051 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:45:21,402 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:45:25,764 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:45:25,765 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:45:25,765 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:45:25,771 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:45:26,056 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:45:26,772 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:45:28,060 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:45:28,125 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:45:28,125 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:45:28,125 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:45:31,612 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:45:31,612 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:45:31,612 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:45:31,618 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:45:32,063 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:45:32,654 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:45:34,066 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:45:36,790 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:45:36,791 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:45:36,791 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:45:36,796 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:45:37,068 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:45:37,797 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:45:38,016 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 14:45:38,071 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:45:41,963 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:45:41,963 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:45:41,964 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:45:41,968 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:45:42,074 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:45:42,969 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:45:43,125 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:45:43,125 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:45:43,125 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:45:44,078 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:45:47,766 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:45:47,767 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:45:47,767 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:45:47,772 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:45:48,080 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:45:48,810 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:45:50,083 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:45:52,578 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:45:52,579 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:45:52,579 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:45:52,584 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:45:53,085 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:45:54,086 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:45:54,584 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:45:58,037 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:45:58,038 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:45:58,038 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:45:58,044 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:45:58,088 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:45:58,125 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:45:58,126 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:45:58,126 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:45:59,648 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:46:00,091 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:46:01,841 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:46:01,841 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:46:01,842 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:46:01,847 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:46:02,094 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:46:04,098 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:46:04,886 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:46:06,609 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:46:06,609 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:46:06,610 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:46:06,615 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:46:07,100 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:46:08,018 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 14:46:08,103 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:46:10,019 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:46:10,960 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:46:10,961 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:46:10,961 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:46:10,966 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:46:11,104 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:46:12,107 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:46:13,125 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:46:13,125 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:46:13,127 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:46:15,325 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:46:17,168 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:46:17,168 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:46:17,168 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:46:17,173 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:46:18,112 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:46:20,115 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:46:21,211 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:46:21,733 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:46:21,734 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:46:21,734 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:46:21,739 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:46:22,116 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:46:24,119 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:46:26,012 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:46:26,013 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:46:26,013 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:46:26,018 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:46:26,121 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:46:27,019 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:46:28,124 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:46:28,125 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:46:28,125 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:46:28,126 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:46:32,374 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:46:32,374 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:46:32,375 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:46:32,376 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:46:32,379 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:46:33,127 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:46:34,130 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:46:37,419 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:46:37,520 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:46:37,521 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:46:37,522 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:46:37,527 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:46:38,020 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 14:46:38,132 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:46:40,136 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:46:42,757 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:46:42,758 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:46:42,758 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:46:42,759 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:46:42,763 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:46:43,125 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:46:43,126 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:46:43,126 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:46:43,138 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:46:44,140 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:46:48,254 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:46:48,254 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:46:48,255 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:46:48,255 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:46:48,259 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:46:49,143 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:46:50,146 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:46:52,934 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:46:52,935 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:46:52,935 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:46:52,940 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:46:53,148 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:46:53,940 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:46:54,150 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:46:57,771 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:46:57,772 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:46:57,772 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:46:57,777 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:46:58,125 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:46:58,126 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:46:58,126 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:46:58,153 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:46:59,407 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:47:00,156 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:47:02,870 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:47:02,870 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:47:02,871 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:47:02,876 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:47:03,160 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:47:04,163 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:47:04,914 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:47:07,986 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:47:07,988 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:47:07,988 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:47:07,993 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:47:08,022 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 14:47:08,166 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:47:10,024 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:47:10,169 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:47:13,108 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:47:13,109 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:47:13,110 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:47:13,115 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:47:13,126 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:47:13,126 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:47:13,126 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:47:13,171 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:47:14,173 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:47:15,413 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:47:17,447 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:47:17,448 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:47:17,448 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:47:17,453 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:47:18,176 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:47:20,179 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:47:20,491 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:47:22,670 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:47:22,670 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:47:22,671 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:47:22,676 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:47:23,181 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:47:24,184 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:47:25,677 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:47:27,258 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:47:27,258 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:47:27,259 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:47:27,263 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:47:28,125 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:47:28,126 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:47:28,126 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:47:28,186 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:47:30,189 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:47:31,411 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:47:32,379 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:47:32,379 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:47:32,380 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:47:32,385 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:47:33,191 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:47:34,194 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:47:36,422 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:47:37,220 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:47:37,221 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:47:37,221 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:47:37,226 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:47:38,030 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 14:47:38,199 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:47:38,200 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:47:41,904 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:47:41,905 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:47:41,905 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:47:41,906 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:47:41,910 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:47:42,202 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:47:43,125 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:47:43,126 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:47:43,126 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:47:44,206 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:47:46,351 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:47:46,352 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:47:46,352 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:47:46,358 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:47:47,207 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:47:47,394 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:47:48,210 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:47:52,395 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:47:55,391 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:47:55,392 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:47:55,392 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:47:55,398 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:47:56,215 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:47:57,399 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:47:58,126 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:47:58,126 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:47:58,127 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:47:58,218 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:48:00,810 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:48:00,811 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:48:00,811 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:48:00,816 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:48:01,221 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:48:02,224 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:48:02,854 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:48:07,855 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:48:08,027 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 14:48:08,175 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:48:08,176 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:48:08,176 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:48:08,181 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:48:08,228 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:48:10,232 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:48:12,477 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:48:12,478 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:48:12,478 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:48:12,483 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:48:13,126 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:48:13,126 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:48:13,126 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:48:13,234 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:48:13,322 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:48:14,236 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:48:17,280 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:48:17,281 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:48:17,281 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:48:17,286 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:48:18,240 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:48:19,321 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:48:20,243 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:48:24,322 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:48:25,554 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:48:25,555 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:48:25,555 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:48:25,560 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:48:26,247 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:48:28,126 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:48:28,126 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:48:28,127 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:48:28,251 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:48:29,396 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:48:30,923 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:48:30,924 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:48:30,924 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:48:30,929 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:48:31,252 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:48:32,255 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:48:34,966 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:48:36,463 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:48:36,464 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:48:36,464 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:48:36,469 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:48:37,259 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:48:38,035 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 14:48:38,261 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:48:40,036 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:48:41,410 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:48:41,411 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:48:41,411 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:48:41,416 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:48:42,264 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:48:43,127 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:48:43,127 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:48:43,127 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:48:44,267 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:48:45,380 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:48:48,168 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:48:48,169 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:48:48,169 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:48:48,174 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:48:48,270 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:48:50,273 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:48:51,209 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:48:52,668 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:48:52,669 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:48:52,669 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:48:52,674 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:48:53,276 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:48:54,279 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:48:56,675 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:48:58,126 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:48:58,127 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:48:58,127 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:48:58,829 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:48:58,830 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:48:58,830 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:48:58,835 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:48:59,282 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:49:00,284 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:49:01,873 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:49:03,083 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:49:03,084 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:49:03,084 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:49:03,089 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:49:03,288 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:49:04,289 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:49:07,090 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:49:08,031 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 14:49:08,137 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:49:08,138 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:49:08,138 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:49:08,144 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:49:08,291 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:49:10,294 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:49:12,145 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:49:12,597 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:49:12,598 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:49:12,599 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:49:12,603 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:49:13,127 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:49:13,127 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:49:13,127 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:49:13,296 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:49:14,299 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:49:16,911 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:49:16,911 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:49:16,912 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:49:16,916 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:49:17,301 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:49:17,953 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:49:18,304 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:49:22,203 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:49:22,204 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:49:22,205 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:49:22,210 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:49:22,306 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:49:23,210 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:49:24,309 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:49:27,181 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:49:27,182 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:49:27,183 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:49:27,188 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:49:27,311 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:49:28,126 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:49:28,127 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:49:28,127 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:49:28,313 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:49:28,349 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:49:31,876 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:49:31,877 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:49:31,877 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:49:31,882 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:49:32,315 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:49:33,917 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:49:34,319 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:49:38,033 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 14:49:39,035 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:49:40,303 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:49:40,304 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:49:40,304 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:49:40,310 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:49:40,323 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:49:42,326 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:49:43,127 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:49:43,127 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:49:43,127 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:49:44,411 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:49:45,534 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:49:45,534 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:49:45,535 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:49:45,540 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:49:46,329 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:49:48,332 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:49:49,578 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:49:49,628 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:49:49,629 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:49:49,630 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:49:49,634 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:49:50,333 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:49:52,336 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:49:53,936 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:49:53,937 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:49:53,937 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:49:53,942 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:49:54,337 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:49:54,943 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:49:56,341 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:49:58,127 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:49:58,127 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:49:58,127 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:49:58,200 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:49:58,332 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:49:58,332 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:49:58,337 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:49:58,342 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:50:00,337 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:50:00,343 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:50:03,236 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:50:03,237 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:50:03,237 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:50:03,242 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:50:03,346 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:50:04,349 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:50:06,243 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:50:07,518 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:50:07,518 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:50:07,519 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:50:07,523 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:50:08,036 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 14:50:08,352 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:50:10,355 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:50:12,037 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:50:12,152 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:50:12,152 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:50:12,153 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:50:12,158 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:50:12,356 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:50:13,127 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:50:13,127 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:50:13,127 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:50:14,360 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:50:16,842 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:50:16,842 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:50:16,842 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:50:16,847 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:50:17,362 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:50:17,885 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:50:18,364 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:50:22,886 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:50:25,158 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:50:25,159 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:50:25,159 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:50:25,164 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:50:25,368 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:50:26,371 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:50:28,127 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:50:28,127 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:50:28,127 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:50:28,384 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:50:31,926 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:50:31,927 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:50:31,927 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:50:31,932 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:50:32,375 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:50:33,969 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:50:34,378 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:50:38,038 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 14:50:39,039 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:50:41,573 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:50:41,574 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:50:41,574 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:50:41,579 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:50:42,383 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:50:43,127 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:50:43,127 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:50:43,127 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:50:44,371 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:50:44,385 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:50:46,295 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:50:46,296 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:50:46,296 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:50:46,301 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:50:46,386 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:50:48,389 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:50:50,337 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:50:54,992 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:50:54,992 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:50:54,993 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:50:54,998 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:50:55,393 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:50:55,998 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:50:56,395 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:50:58,127 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:50:58,127 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:50:58,128 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:50:59,740 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:50:59,740 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:50:59,741 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:50:59,745 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:51:00,398 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:51:01,781 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:51:02,402 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:51:04,661 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:51:04,662 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:51:04,662 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:51:04,667 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:51:05,405 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:51:06,407 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:51:07,667 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:51:08,043 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 14:51:09,523 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:51:09,524 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:51:09,524 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:51:09,529 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:51:10,410 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:51:12,414 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:51:13,127 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:51:13,127 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:51:13,128 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:51:13,376 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:51:13,802 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:51:13,803 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:51:13,803 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:51:13,808 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:51:14,415 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:51:16,418 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:51:18,652 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:51:18,653 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:51:18,653 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:51:18,653 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:51:18,658 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:51:19,420 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:51:20,423 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:51:23,659 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:51:25,813 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:51:25,814 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:51:25,814 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:51:25,819 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:51:26,426 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:51:28,127 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:51:28,127 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:51:28,128 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:51:28,429 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:51:29,348 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:51:30,101 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:51:30,102 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:51:30,102 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:51:30,107 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:51:30,431 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:51:32,434 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:51:34,875 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:51:34,875 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:51:34,876 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:51:34,876 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:51:34,881 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:51:35,436 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:51:36,438 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:51:38,042 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 14:51:40,044 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:51:40,424 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:51:40,425 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:51:40,425 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:51:40,430 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:51:40,441 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:51:42,445 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:51:43,127 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:51:43,127 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:51:43,128 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:51:45,408 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:51:45,855 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:51:45,856 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:51:45,856 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:51:45,861 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:51:46,448 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:51:48,451 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:51:50,897 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:51:52,042 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:51:52,042 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:51:52,043 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:51:52,047 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:51:52,453 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:51:54,456 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:51:56,048 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:51:56,171 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:51:56,172 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:51:56,172 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:51:56,177 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:51:56,457 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:51:58,127 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:51:58,127 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:51:58,128 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:51:58,461 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:52:00,597 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:52:00,598 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:52:00,598 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:52:00,603 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:52:01,464 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:52:01,641 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:52:02,466 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:52:06,642 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:52:06,883 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:52:06,884 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:52:06,885 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:52:06,889 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:52:07,470 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:52:08,044 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 14:52:08,472 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:52:11,532 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:52:11,532 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:52:11,533 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:52:11,537 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:52:12,476 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:52:12,477 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:52:12,538 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:52:13,127 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:52:13,127 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:52:13,128 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:52:15,645 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:52:15,649 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:52:15,649 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:52:15,654 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:52:16,479 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:52:17,689 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:52:18,482 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:52:20,261 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:52:20,262 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:52:20,262 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:52:20,267 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:52:20,483 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:52:22,487 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:52:23,267 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:52:28,128 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:52:28,128 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:52:28,128 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:52:28,399 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:52:28,471 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:52:28,471 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:52:28,471 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:52:28,477 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:52:28,490 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:52:30,493 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:52:33,189 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:52:33,189 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:52:33,189 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:52:33,194 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:52:33,495 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:52:34,195 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:52:34,498 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:52:37,650 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:52:37,651 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:52:37,651 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:52:37,655 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:52:38,047 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 14:52:38,500 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:52:40,048 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:52:40,504 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:52:43,128 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:52:43,128 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:52:43,128 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:52:43,206 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:52:43,330 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:52:43,330 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:52:43,335 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:52:43,506 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:52:44,508 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:52:45,336 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:52:47,694 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:52:47,695 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:52:47,695 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:52:47,700 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:52:48,511 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:52:50,514 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:52:50,701 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:52:54,662 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:52:54,662 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:52:54,663 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:52:54,668 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:52:55,517 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:52:56,519 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:52:56,668 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:52:58,128 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:52:58,128 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:52:58,128 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:52:59,137 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:52:59,137 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:52:59,137 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:52:59,142 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:52:59,521 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:53:00,524 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:53:02,181 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:53:04,955 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:53:04,956 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:53:04,956 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:53:04,961 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:53:05,529 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:53:06,531 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:53:07,962 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:53:08,049 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 14:53:09,878 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:53:09,879 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:53:09,879 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:53:09,884 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:53:10,534 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:53:12,537 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:53:13,128 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:53:13,128 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:53:13,128 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:53:13,401 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:53:16,012 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:53:16,013 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:53:16,013 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:53:16,018 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:53:16,540 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:53:18,543 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:53:19,053 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:53:20,862 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:53:20,863 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:53:20,863 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:53:20,868 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:53:21,545 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:53:22,547 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:53:24,869 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:53:26,138 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:53:26,138 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:53:26,138 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:53:26,143 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:53:26,550 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:53:28,128 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:53:28,128 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:53:28,128 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:53:28,553 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:53:30,383 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:53:30,657 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:53:30,658 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:53:30,658 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:53:30,663 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:53:31,555 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:53:32,558 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:53:35,152 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:53:35,153 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:53:35,153 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:53:35,159 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:53:35,560 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:53:36,159 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:53:36,562 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:53:38,055 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 14:53:39,941 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:53:39,942 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:53:39,942 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:53:39,947 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:53:40,565 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:53:41,948 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:53:42,569 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:53:43,128 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:53:43,128 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:53:43,128 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:53:44,975 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:53:44,976 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:53:44,976 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:53:44,981 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:53:45,571 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:53:46,573 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:53:47,017 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:53:51,488 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:53:51,489 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:53:51,489 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:53:51,494 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:53:51,576 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:53:52,495 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:53:52,579 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:53:56,422 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:53:56,423 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:53:56,423 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:53:56,428 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:53:56,581 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:53:58,128 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:53:58,128 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:53:58,128 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:53:58,327 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:53:58,585 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:54:01,302 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:54:01,302 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:54:01,302 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:54:01,307 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:54:01,588 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:54:02,591 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:54:03,345 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:54:06,385 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:54:06,386 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:54:06,386 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:54:06,391 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:54:06,593 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:54:08,053 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 14:54:08,598 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:54:09,055 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:54:11,226 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:54:11,226 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:54:11,227 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:54:11,231 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:54:11,599 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:54:12,602 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:54:13,128 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:54:13,128 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:54:13,129 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:54:14,415 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:54:19,415 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:54:19,978 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:54:19,978 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:54:19,978 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:54:19,983 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:54:20,607 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:54:22,611 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:54:24,571 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:54:24,572 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:54:24,572 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:54:24,572 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:54:24,577 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:54:24,612 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:54:26,615 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:54:28,128 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:54:28,128 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:54:28,129 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:54:29,020 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:54:29,021 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:54:29,021 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:54:29,026 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:54:29,617 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:54:30,061 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:54:30,620 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:54:34,790 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:54:34,790 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:54:34,791 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:54:34,796 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:54:35,623 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:54:35,796 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:54:36,626 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:54:38,059 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 14:54:39,713 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:54:39,713 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:54:39,713 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:54:39,719 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:54:40,629 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:54:41,719 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:54:42,633 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:54:43,128 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:54:43,128 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:54:43,129 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:54:43,910 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:54:43,910 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:54:43,910 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:54:43,915 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:54:44,634 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:54:46,637 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:54:46,953 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:54:48,446 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:54:48,447 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:54:48,447 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:54:48,452 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:54:48,639 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:54:50,642 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:54:52,453 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:54:53,251 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:54:53,251 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:54:53,252 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:54:53,256 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:54:53,644 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:54:54,647 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:54:57,515 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:54:57,515 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:54:57,516 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:54:57,516 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:54:57,520 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:54:57,649 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:54:58,128 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:54:58,129 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:54:58,129 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:54:58,651 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:55:03,394 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:55:04,290 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:55:04,290 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:55:04,290 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:55:04,295 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:55:04,657 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:55:06,660 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:55:08,058 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 14:55:09,059 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:55:10,891 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:55:10,892 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:55:10,892 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:55:10,897 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:55:11,664 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:55:12,667 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:55:13,128 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:55:13,129 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:55:13,129 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:55:14,430 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:55:15,992 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:55:15,992 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:55:15,993 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:55:15,998 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:55:16,670 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:55:18,673 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:55:20,033 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:55:22,387 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:55:22,388 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:55:22,389 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:55:22,394 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:55:22,675 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:55:24,678 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:55:25,394 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:55:27,747 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:55:27,747 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:55:27,748 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:55:27,753 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:55:28,128 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:55:28,129 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:55:28,129 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:55:28,683 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:55:28,683 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:55:30,423 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:55:33,716 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:55:33,717 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:55:33,717 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:55:33,723 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:55:34,688 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:55:34,689 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:55:35,757 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:55:38,060 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 14:55:38,078 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:55:38,078 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:55:38,079 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:55:38,084 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:55:38,691 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:55:40,694 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:55:41,084 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:55:42,671 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:55:42,671 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:55:42,672 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:55:42,677 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:55:42,696 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:55:43,128 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:55:43,129 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:55:43,130 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:55:44,700 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:55:46,410 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:55:47,066 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:55:47,067 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:55:47,067 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:55:47,072 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:55:47,702 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:55:48,704 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:55:52,110 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:55:52,407 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:55:52,408 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:55:52,408 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:55:52,413 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:55:52,707 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:55:54,710 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:55:57,414 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:55:57,756 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:55:57,756 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:55:57,757 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:55:57,762 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:55:58,129 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:55:58,129 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:55:58,130 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:55:58,714 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:55:58,714 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:56:02,429 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:56:03,064 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:56:03,065 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:56:03,065 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:56:03,070 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:56:03,719 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:56:04,722 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:56:07,913 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:56:07,914 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:56:07,914 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:56:07,914 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:56:07,919 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:56:08,062 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 14:56:08,724 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:56:10,728 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:56:13,064 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:56:13,129 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:56:13,129 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:56:13,129 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:56:15,474 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:56:15,474 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:56:15,474 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:56:15,480 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:56:15,731 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:56:16,734 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:56:18,517 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:56:23,518 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:56:24,799 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:56:24,800 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:56:24,800 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:56:24,805 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:56:25,739 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:56:26,742 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:56:28,129 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:56:28,129 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:56:28,129 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:56:29,352 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:56:33,075 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:56:33,076 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:56:33,076 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:56:33,081 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:56:33,746 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:56:34,747 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:56:35,117 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:56:38,064 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 14:56:41,066 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:56:41,470 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:56:41,471 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:56:41,471 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:56:41,476 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:56:41,752 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:56:42,754 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:56:43,129 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:56:43,129 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:56:43,129 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:56:46,406 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:56:47,698 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:56:47,698 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:56:47,699 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:56:47,703 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:56:47,758 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:56:48,760 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:56:51,742 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:56:56,010 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:56:56,011 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:56:56,011 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:56:56,016 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:56:56,765 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:56:57,016 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:56:58,129 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:56:58,129 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:56:58,131 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:56:58,768 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:57:01,366 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:57:01,367 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:57:01,367 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:57:01,372 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:57:01,772 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:57:02,409 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:57:02,772 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:57:05,484 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:57:05,485 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:57:05,485 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:57:05,490 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:57:05,775 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:57:06,777 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:57:07,490 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:57:08,070 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 14:57:10,857 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:57:10,858 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:57:10,858 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:57:10,863 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:57:11,780 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:57:12,783 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:57:12,863 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:57:13,129 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:57:13,129 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:57:13,129 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:57:18,344 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:57:18,378 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:57:18,378 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:57:18,379 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:57:18,384 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:57:18,787 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:57:20,791 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:57:23,421 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:57:23,520 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:57:23,521 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:57:23,521 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:57:23,526 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:57:23,793 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:57:24,796 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:57:28,129 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:57:28,129 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:57:28,130 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:57:29,357 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:57:30,149 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:57:30,150 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:57:30,150 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:57:30,155 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:57:30,799 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:57:32,803 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:57:35,194 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:57:35,648 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:57:35,649 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:57:35,649 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:57:35,654 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:57:35,804 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:57:36,807 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:57:38,069 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 14:57:40,314 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:57:40,315 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:57:40,315 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:57:40,315 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:57:40,320 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:57:40,810 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:57:42,813 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:57:43,129 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:57:43,130 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:57:43,130 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:57:45,008 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:57:45,009 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:57:45,009 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:57:45,014 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:57:45,815 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:57:46,049 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:57:46,817 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:57:50,423 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:57:50,423 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:57:50,424 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:57:50,428 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:57:50,820 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:57:51,429 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:57:52,823 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:57:54,445 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:57:54,446 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:57:54,446 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:57:54,451 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:57:54,824 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:57:56,452 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:57:56,825 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:57:58,129 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:57:58,130 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:57:58,130 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:57:59,753 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:57:59,754 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:57:59,754 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:57:59,759 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:57:59,827 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:58:00,830 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:58:01,797 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:58:06,798 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:58:08,071 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 14:58:08,163 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:58:08,164 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:58:08,165 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:58:08,170 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:58:08,838 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:58:10,841 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:58:12,170 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:58:13,129 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:58:13,130 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:58:13,130 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:58:13,333 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:58:13,404 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:58:13,404 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:58:13,410 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:58:13,843 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:58:14,846 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:58:17,410 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:58:17,542 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:58:17,548 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:58:17,549 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:58:17,553 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:58:17,848 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:58:18,850 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:58:22,586 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:58:23,726 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:58:23,727 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:58:23,727 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:58:23,732 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:58:23,853 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:58:24,856 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:58:27,733 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:58:28,129 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:58:28,130 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:58:28,130 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:58:28,751 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:58:28,752 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:58:28,752 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:58:28,757 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:58:28,858 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:58:30,862 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:58:32,794 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:58:37,794 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:58:37,905 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:58:37,906 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:58:37,906 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:58:37,911 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:58:38,074 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 14:58:38,869 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:58:38,869 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:58:42,806 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:58:42,807 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:58:42,807 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:58:42,808 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:58:42,812 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:58:42,872 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:58:43,130 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:58:43,130 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:58:43,130 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:58:44,875 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:58:47,267 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:58:47,268 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:58:47,268 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:58:47,273 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:58:47,877 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:58:48,309 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:58:48,880 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:58:52,371 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:58:52,372 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:58:52,372 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:58:52,377 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:58:52,882 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:58:53,378 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:58:54,885 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:58:56,983 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:58:56,984 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:58:56,984 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:58:56,989 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:58:57,887 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:58:58,130 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:58:58,130 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:58:58,130 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:58:58,389 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:58:58,890 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:59:01,473 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:59:01,474 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:59:01,474 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:59:01,479 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:59:01,893 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:59:02,895 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:59:03,517 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:59:05,894 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:59:05,894 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:59:05,895 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:59:05,900 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:59:05,900 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:59:06,900 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:59:08,076 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 14:59:09,077 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:59:11,538 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:59:11,539 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:59:11,539 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:59:11,544 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:59:11,904 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:59:12,906 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:59:13,130 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:59:13,130 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:59:13,130 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:59:14,348 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:59:18,559 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:59:18,560 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:59:18,560 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:59:18,565 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:59:18,910 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:59:19,601 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:59:20,913 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:59:24,053 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:59:24,053 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:59:24,054 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:59:24,059 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:59:24,916 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:59:25,059 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:59:26,919 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:59:28,130 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:59:28,130 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:59:28,131 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:59:29,232 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:59:29,233 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:59:29,233 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:59:29,237 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:59:29,921 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:59:30,273 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:59:30,924 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:59:35,274 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:59:35,742 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:59:35,743 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:59:35,743 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:59:35,748 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:59:35,927 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:59:36,929 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:59:38,081 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 14:59:40,932 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:59:40,933 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:59:40,933 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:59:40,934 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:59:40,938 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:59:41,934 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:59:42,936 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:59:43,130 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:59:43,130 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:59:43,130 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:59:45,497 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:59:45,498 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:59:45,498 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:59:45,503 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:59:45,938 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:59:46,541 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:59:46,941 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:59:51,317 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:59:51,318 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:59:51,318 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:59:51,322 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:59:51,944 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:59:52,323 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:59:52,947 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:59:56,003 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:59:56,004 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 14:59:56,004 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:59:56,009 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:59:56,951 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 14:59:56,951 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 14:59:58,010 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:59:58,130 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:59:58,130 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:59:58,130 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:00:00,155 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:00:00,155 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:00:00,155 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:00:00,160 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:00:00,954 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:00:02,958 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:00:03,197 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:00:05,646 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:00:05,647 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:00:05,647 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:00:05,652 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:00:05,960 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:00:06,963 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:00:08,080 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 15:00:09,081 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:00:11,445 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:00:11,446 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:00:11,446 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:00:11,451 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:00:11,966 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:00:12,969 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:00:13,130 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:00:13,130 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:00:13,131 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:00:14,402 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:00:16,776 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:00:16,777 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:00:16,777 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:00:16,782 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:00:16,972 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:00:18,975 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:00:19,817 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:00:21,893 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:00:21,894 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:00:21,894 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:00:21,899 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:00:21,977 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:00:22,980 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:00:24,899 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:00:26,327 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:00:26,328 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:00:26,328 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:00:26,333 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:00:26,982 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:00:28,130 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:00:28,130 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:00:28,131 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:00:28,985 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:00:30,338 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:00:30,586 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:00:30,586 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:00:30,586 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:00:30,591 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:00:30,986 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:00:32,990 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:00:34,901 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:00:34,901 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:00:34,902 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:00:34,906 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:00:34,991 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:00:35,907 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:00:36,994 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:00:38,085 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 15:00:39,500 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:00:39,501 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:00:39,501 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:00:39,506 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:00:39,996 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:00:40,999 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:00:41,506 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:00:43,130 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:00:43,131 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:00:43,131 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:00:45,413 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:00:45,414 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:00:45,414 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:00:45,419 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:00:46,004 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:00:47,006 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:00:47,457 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:00:50,467 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:00:50,467 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:00:50,467 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:00:50,472 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:00:51,009 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:00:52,473 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:00:53,012 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:00:57,248 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:00:57,249 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:00:57,249 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:00:57,254 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:00:58,015 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:00:58,130 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:00:58,131 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:00:58,131 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:00:58,360 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:00:59,018 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:01:03,360 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:01:03,644 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:01:03,645 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:01:03,645 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:01:03,650 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:01:04,023 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:01:05,026 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:01:08,084 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 15:01:09,085 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:01:10,467 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:01:10,468 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:01:10,468 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:01:10,473 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:01:11,030 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:01:13,033 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:01:13,131 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:01:13,131 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:01:13,131 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:01:14,341 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:01:16,874 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:01:16,874 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:01:16,875 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:01:16,880 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:01:17,036 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:01:19,039 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:01:19,917 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:01:22,642 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:01:22,643 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:01:22,643 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:01:22,648 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:01:23,042 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:01:25,045 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:01:25,648 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:01:27,730 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:01:27,731 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:01:27,731 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:01:27,736 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:01:28,047 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:01:28,131 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:01:28,131 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:01:28,131 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:01:29,049 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:01:31,351 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:01:32,828 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:01:32,829 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:01:32,829 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:01:32,834 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:01:33,052 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:01:35,055 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:01:36,870 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:01:38,086 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 15:01:38,340 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:01:38,341 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:01:38,341 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:01:38,347 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:01:39,058 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:01:41,061 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:01:42,347 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:01:43,131 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:01:43,131 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:01:43,131 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:01:43,573 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:01:43,574 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:01:43,574 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:01:43,579 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:01:44,064 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:01:45,067 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:01:47,617 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:01:50,340 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:01:50,341 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:01:50,341 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:01:50,346 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:01:51,071 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:01:53,072 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:01:53,347 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:01:58,131 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:01:58,131 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:01:58,131 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:01:58,509 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:01:58,510 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:01:58,510 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:01:58,510 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:01:58,515 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:01:59,075 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:02:01,078 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:02:03,227 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:02:03,228 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:02:03,228 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:02:03,233 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:02:04,083 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:02:04,236 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:02:05,085 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:02:07,995 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:02:07,996 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:02:07,996 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:02:08,001 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:02:08,089 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 15:02:08,091 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:02:09,094 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:02:10,090 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:02:12,331 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:02:12,332 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:02:12,333 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:02:12,338 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:02:13,097 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:02:13,131 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:02:13,131 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:02:13,131 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:02:15,100 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:02:15,321 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:02:18,135 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:02:18,135 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:02:18,136 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:02:18,141 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:02:19,105 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:02:19,105 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:02:21,177 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:02:22,465 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:02:22,465 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:02:22,466 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:02:22,471 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:02:23,108 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:02:25,111 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:02:26,472 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:02:27,019 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:02:27,020 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:02:27,021 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:02:27,025 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:02:27,112 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:02:28,131 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:02:28,131 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:02:28,131 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:02:29,115 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:02:31,826 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:02:31,827 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:02:31,827 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:02:31,827 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:02:31,832 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:02:32,117 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:02:33,120 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:02:36,869 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:02:37,257 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:02:37,258 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:02:37,258 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:02:37,263 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:02:38,091 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 15:02:38,123 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:02:39,126 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:02:42,092 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:02:42,149 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:02:42,150 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:02:42,150 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:02:42,155 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:02:43,131 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:02:43,131 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:02:43,131 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:02:43,132 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:02:43,132 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:02:46,998 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:02:46,999 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:02:46,999 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:02:47,004 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:02:47,133 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:02:48,041 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:02:49,137 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:02:51,339 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:02:51,339 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:02:51,340 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:02:51,346 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:02:52,139 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:02:53,141 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:02:53,346 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:02:56,072 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:02:56,073 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:02:56,073 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:02:56,079 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:02:56,143 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:02:57,146 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:02:58,131 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:02:58,131 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:02:58,132 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:02:59,324 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:03:01,410 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:03:01,411 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:03:01,411 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:03:01,417 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:03:02,150 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:03:03,153 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:03:04,453 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:03:07,484 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:03:07,485 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:03:07,485 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:03:07,490 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:03:08,097 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 15:03:08,156 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:03:09,159 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:03:10,098 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:03:13,131 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:03:13,131 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:03:13,132 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:03:15,380 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:03:16,556 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:03:16,556 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:03:16,556 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:03:16,561 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:03:17,164 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:03:19,168 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:03:20,597 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:03:24,920 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:03:24,921 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:03:24,921 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:03:24,927 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:03:25,171 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:03:25,927 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:03:27,174 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:03:28,131 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:03:28,131 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:03:28,132 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:03:29,169 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:03:29,170 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:03:29,170 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:03:29,175 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:03:29,176 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:03:31,177 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:03:31,213 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:03:35,310 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:03:35,310 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:03:35,311 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:03:35,316 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:03:36,180 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:03:36,316 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:03:37,183 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:03:38,095 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 15:03:40,286 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:03:40,287 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:03:40,287 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:03:40,292 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:03:41,188 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:03:41,188 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:03:42,293 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:03:43,131 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:03:43,132 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:03:43,132 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:03:45,233 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:03:45,234 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:03:45,234 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:03:45,239 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:03:46,192 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:03:47,195 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:03:48,277 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:03:49,915 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:03:49,916 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:03:49,917 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:03:49,922 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:03:50,197 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:03:51,199 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:03:53,922 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:03:54,201 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:03:54,202 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:03:54,202 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:03:54,207 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:03:55,205 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:03:55,205 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:03:58,131 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:03:58,132 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:03:58,132 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:03:59,350 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:03:59,350 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:03:59,351 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:03:59,351 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:03:59,356 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:04:00,208 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:04:01,211 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:04:04,394 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:04:08,097 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 15:04:09,917 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:04:09,918 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:04:09,918 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:04:09,918 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:04:09,923 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:04:10,219 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:04:11,222 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:04:13,131 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:04:13,132 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:04:13,132 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:04:15,372 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:04:18,520 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:04:18,521 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:04:18,521 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:04:18,527 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:04:19,228 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:04:20,561 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:04:21,231 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:04:25,562 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:04:26,714 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:04:26,715 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:04:26,715 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:04:26,721 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:04:27,235 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:04:28,132 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:04:28,132 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:04:28,132 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:04:29,239 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:04:30,811 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:04:30,811 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:04:30,812 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:04:30,812 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:04:30,817 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:04:31,240 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:04:33,244 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:04:35,854 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:04:36,977 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:04:36,977 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:04:36,978 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:04:36,983 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:04:37,247 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:04:38,099 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 15:04:39,250 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:04:41,101 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:04:41,713 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:04:41,714 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:04:41,714 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:04:41,720 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:04:42,253 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:04:43,132 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:04:43,132 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:04:43,132 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:04:43,253 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:04:45,846 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:04:45,847 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:04:45,847 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:04:45,852 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:04:46,255 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:04:46,889 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:04:47,258 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:04:50,242 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:04:50,243 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:04:50,243 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:04:50,248 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:04:50,260 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:04:51,263 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:04:52,249 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:04:54,404 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:04:54,405 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:04:54,405 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:04:54,410 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:04:55,265 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:04:57,268 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:04:57,411 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:04:58,132 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:04:58,132 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:04:58,132 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:04:58,660 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:04:58,660 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:04:58,661 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:04:58,666 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:04:59,269 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:05:01,273 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:05:02,701 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:05:04,388 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:05:04,389 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:05:04,389 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:05:04,394 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:05:05,279 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:05:05,280 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:05:08,102 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 15:05:08,103 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:05:10,773 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:05:10,774 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:05:10,774 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:05:10,779 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:05:11,283 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:05:13,132 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:05:13,132 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:05:13,132 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:05:13,285 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:05:13,435 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:05:17,999 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:05:17,999 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:05:18,000 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:05:18,005 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:05:18,288 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:05:19,041 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:05:19,297 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:05:22,443 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:05:22,443 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:05:22,444 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:05:22,455 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:05:23,299 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:05:24,455 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:05:25,309 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:05:26,999 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:05:27,000 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:05:27,000 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:05:27,008 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:05:27,310 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:05:28,132 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:05:28,132 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:05:28,132 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:05:29,315 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:05:30,385 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:05:33,716 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:05:33,717 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:05:33,717 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:05:33,724 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:05:34,319 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:05:35,324 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:05:35,757 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:05:37,980 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:05:37,981 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:05:37,981 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:05:37,988 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:05:38,109 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 15:05:38,326 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:05:39,329 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:05:41,111 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:05:42,976 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:05:42,976 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:05:42,977 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:05:42,983 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:05:43,132 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:05:43,132 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:05:43,133 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:05:43,333 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:05:45,343 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:05:46,411 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:05:47,630 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:05:47,630 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:05:47,631 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:05:47,640 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:05:48,345 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:05:49,349 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:05:51,673 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:05:52,919 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:05:52,921 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:05:52,921 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:05:52,929 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:05:53,352 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:05:55,356 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:05:56,929 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:05:58,132 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:05:58,132 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:05:58,133 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:05:59,381 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:05:59,382 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:05:59,382 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:05:59,390 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:06:00,359 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:06:01,366 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:06:02,425 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:06:04,218 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:06:04,219 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:06:04,219 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:06:04,229 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:06:04,370 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:06:05,376 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:06:08,106 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 15:06:08,107 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:06:08,770 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:06:08,771 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:06:08,771 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:06:08,779 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:06:09,379 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:06:11,386 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:06:13,132 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:06:13,132 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:06:13,133 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:06:13,383 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:06:14,931 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:06:14,932 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:06:14,932 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:06:14,941 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:06:15,389 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:06:17,394 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:06:18,974 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:06:23,393 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:06:23,394 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:06:23,394 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:06:23,401 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:06:23,401 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:06:24,401 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:06:25,406 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:06:28,132 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:06:28,132 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:06:28,133 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:06:30,401 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:06:31,611 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:06:31,612 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:06:31,612 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:06:31,623 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:06:32,411 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:06:33,415 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:06:35,654 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:06:36,472 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:06:36,473 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:06:36,473 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:06:36,484 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:06:37,421 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:06:37,421 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:06:38,111 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 15:06:40,870 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:06:40,871 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:06:40,872 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:06:40,872 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:06:40,880 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:06:41,424 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:06:43,132 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:06:43,133 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:06:43,133 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:06:43,430 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:06:45,125 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:06:45,126 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:06:45,126 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:06:45,134 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:06:45,432 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:06:46,169 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:06:47,437 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:06:50,263 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:06:50,263 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:06:50,264 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:06:50,271 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:06:50,439 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:06:51,271 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:06:51,440 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:06:56,272 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:06:58,132 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:06:58,133 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:06:58,133 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:06:59,655 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:06:59,656 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:06:59,656 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:06:59,665 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:07:00,446 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:07:01,450 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:07:01,697 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:07:03,552 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:07:03,552 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:07:03,553 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:07:03,561 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:07:04,456 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:07:05,460 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:07:07,562 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:07:08,110 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 15:07:08,256 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:07:08,257 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:07:08,257 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:07:08,265 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:07:08,462 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:07:09,467 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:07:12,921 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:07:12,922 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:07:12,922 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:07:12,922 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:07:12,930 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:07:13,132 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:07:13,133 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:07:13,134 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:07:13,469 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:07:15,475 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:07:17,565 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:07:17,566 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:07:17,566 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:07:17,575 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:07:18,479 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:07:18,609 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:07:19,482 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:07:22,308 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:07:22,309 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:07:22,309 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:07:22,317 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:07:22,484 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:07:23,487 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:07:24,318 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:07:26,673 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:07:26,674 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:07:26,674 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:07:26,680 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:07:27,490 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:07:28,132 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:07:28,133 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:07:28,133 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:07:29,353 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:07:29,494 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:07:33,263 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:07:33,264 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:07:33,264 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:07:33,271 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:07:33,497 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:07:35,305 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:07:35,500 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:07:38,112 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 15:07:41,114 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:07:41,676 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:07:41,677 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:07:41,678 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:07:41,683 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:07:42,506 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:07:43,133 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:07:43,133 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:07:43,133 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:07:43,508 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:07:45,962 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:07:45,962 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:07:45,963 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:07:45,968 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:07:46,511 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:07:47,005 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:07:47,514 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:07:52,006 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:07:52,077 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:07:52,078 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:07:52,078 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:07:52,085 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:07:52,517 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:07:53,520 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:07:56,565 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:07:56,566 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:07:56,566 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:07:56,572 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:07:57,524 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:07:57,524 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:07:57,572 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:07:58,133 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:07:58,133 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:07:58,133 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:08:01,813 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:08:01,814 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:08:01,814 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:08:01,820 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:08:02,529 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:08:02,857 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:08:03,532 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:08:07,858 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:08:08,121 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 15:08:11,137 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:08:11,138 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:08:11,139 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:08:11,144 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:08:11,539 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:08:13,133 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:08:13,133 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:08:13,133 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:08:13,388 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:08:13,540 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:08:16,039 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:08:16,040 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:08:16,040 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:08:16,046 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:08:16,543 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:08:17,546 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:08:19,081 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:08:21,549 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:08:21,550 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:08:21,550 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:08:21,556 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:08:22,549 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:08:23,551 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:08:24,557 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:08:25,819 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:08:25,820 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:08:25,820 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:08:25,825 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:08:26,553 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:08:27,556 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:08:28,133 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:08:28,133 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:08:28,133 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:08:30,367 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:08:30,813 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:08:30,813 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:08:30,814 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:08:30,819 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:08:31,559 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:08:33,562 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:08:35,858 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:08:36,996 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:08:36,997 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:08:36,997 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:08:37,002 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:08:37,565 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:08:38,117 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 15:08:39,568 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:08:41,118 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:08:42,466 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:08:42,466 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:08:42,467 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:08:42,472 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:08:42,571 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:08:43,133 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:08:43,133 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:08:43,133 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:08:43,573 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:08:46,380 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:08:47,987 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:08:47,987 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:08:47,988 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:08:47,993 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:08:48,577 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:08:49,579 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:08:52,029 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:08:52,474 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:08:52,475 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:08:52,475 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:08:52,480 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:08:52,581 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:08:53,584 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:08:57,481 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:08:57,855 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:08:57,856 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:08:57,856 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:08:57,861 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:08:58,133 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:08:58,133 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:08:58,134 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:08:58,587 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:08:59,590 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:09:02,071 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:09:02,072 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:09:02,072 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:09:02,077 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:09:02,593 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:09:03,113 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:09:03,596 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:09:06,793 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:09:06,794 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:09:06,794 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:09:06,799 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:09:07,599 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:09:08,122 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 15:09:08,124 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:09:09,602 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:09:12,949 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:09:12,950 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:09:12,950 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:09:12,956 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:09:13,133 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:09:13,133 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:09:13,134 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:09:13,425 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:09:13,605 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:09:15,608 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:09:17,463 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:09:17,464 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:09:17,464 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:09:17,469 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:09:17,610 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:09:18,505 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:09:19,613 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:09:22,920 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:09:22,921 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:09:22,921 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:09:22,927 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:09:23,616 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:09:23,928 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:09:25,619 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:09:28,133 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:09:28,133 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:09:28,134 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:09:28,360 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:09:28,394 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:09:28,395 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:09:28,399 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:09:28,621 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:09:29,400 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:09:29,624 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:09:33,378 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:09:33,378 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:09:33,379 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:09:33,384 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:09:33,626 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:09:34,421 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:09:35,629 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:09:38,122 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 15:09:40,123 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:09:41,707 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:09:41,708 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:09:41,708 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:09:41,713 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:09:42,634 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:09:43,133 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:09:43,134 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:09:43,134 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:09:43,637 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:09:45,432 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:09:47,685 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:09:47,686 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:09:47,686 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:09:47,691 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:09:48,640 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:09:49,643 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:09:50,729 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:09:52,100 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:09:52,101 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:09:52,101 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:09:52,106 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:09:52,644 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:09:53,647 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:09:56,107 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:09:57,353 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:09:57,353 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:09:57,354 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:09:57,359 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:09:57,649 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:09:58,133 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:09:58,134 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:09:58,134 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:09:59,653 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:10:01,351 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:10:02,664 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:10:02,664 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:10:02,665 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:10:02,670 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:10:03,658 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:10:03,658 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:10:06,706 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:10:07,611 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:10:07,612 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:10:07,612 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:10:07,617 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:10:07,661 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:10:08,127 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 15:10:09,665 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:10:12,128 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:10:13,133 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:10:13,134 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:10:13,134 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:10:13,251 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:10:13,340 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:10:13,340 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:10:13,346 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:10:13,667 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:10:15,670 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:10:17,346 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:10:18,281 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:10:18,281 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:10:18,282 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:10:18,287 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:10:18,673 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:10:19,675 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:10:23,090 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:10:23,091 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:10:23,091 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:10:23,092 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:10:23,097 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:10:23,678 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:10:25,681 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:10:27,984 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:10:27,985 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:10:27,985 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:10:27,990 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:10:28,133 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:10:28,134 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:10:28,134 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:10:28,357 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:10:28,683 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:10:29,686 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:10:32,218 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:10:32,219 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:10:32,219 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:10:32,224 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:10:32,688 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:10:33,688 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:10:34,261 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:10:36,287 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:10:36,288 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:10:36,289 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:10:36,294 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:10:36,690 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:10:37,693 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:10:38,126 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 15:10:40,127 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:10:42,619 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:10:42,620 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:10:42,620 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:10:42,626 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:10:42,697 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:10:43,134 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:10:43,134 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:10:43,134 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:10:43,700 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:10:45,324 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:10:46,871 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:10:46,872 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:10:46,872 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:10:46,877 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:10:47,702 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:10:49,705 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:10:50,913 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:10:54,791 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:10:54,792 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:10:54,792 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:10:54,797 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:10:55,711 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:10:55,711 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:10:56,797 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:10:58,134 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:10:58,134 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:10:58,134 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:10:59,456 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:10:59,456 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:10:59,456 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:10:59,462 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:10:59,713 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:11:01,717 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:11:02,497 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:11:04,562 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:11:04,563 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:11:04,563 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:11:04,568 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:11:04,720 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:11:05,723 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:11:07,569 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:11:08,128 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 15:11:10,068 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:11:10,069 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:11:10,069 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:11:10,074 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:11:10,726 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:11:11,729 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:11:13,075 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:11:13,134 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:11:13,134 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:11:13,134 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:11:16,556 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:11:16,557 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:11:16,557 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:11:16,562 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:11:16,732 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:11:17,735 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:11:18,597 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:11:21,600 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:11:21,601 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:11:21,601 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:11:21,606 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:11:21,737 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:11:23,607 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:11:23,741 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:11:26,159 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:11:26,160 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:11:26,160 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:11:26,166 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:11:26,743 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:11:27,745 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:11:28,134 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:11:28,134 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:11:28,134 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:11:29,360 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:11:31,308 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:11:31,309 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:11:31,309 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:11:31,315 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:11:31,748 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:11:33,751 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:11:35,349 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:11:38,134 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 15:11:39,713 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:11:39,714 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:11:39,714 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:11:39,719 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:11:39,755 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:11:40,720 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:11:41,758 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:11:43,134 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:11:43,134 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:11:43,134 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:11:44,837 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:11:44,838 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:11:44,838 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:11:44,844 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:11:45,761 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:11:45,881 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:11:47,764 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:11:50,882 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:11:50,929 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:11:50,930 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:11:50,930 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:11:50,936 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:11:51,767 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:11:53,770 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:11:55,937 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:11:56,721 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:11:56,722 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:11:56,722 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:11:56,728 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:11:56,771 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:11:57,774 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:11:58,134 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:11:58,134 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:11:58,134 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:12:01,372 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:12:03,553 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:12:03,553 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:12:03,554 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:12:03,559 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:12:03,779 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:12:05,783 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:12:06,597 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:12:08,132 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 15:12:11,958 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:12:11,958 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:12:11,959 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:12:11,959 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:12:11,964 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:12:12,788 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:12:13,134 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:12:13,134 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:12:13,134 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:12:13,790 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:12:16,361 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:12:16,362 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:12:16,362 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:12:16,367 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:12:16,793 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:12:17,405 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:12:17,794 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:12:22,406 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:12:22,525 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:12:22,526 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:12:22,526 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:12:22,531 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:12:22,797 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:12:23,799 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:12:27,051 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:12:27,052 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:12:27,052 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:12:27,058 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:12:27,802 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:12:28,058 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:12:28,134 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:12:28,134 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:12:28,134 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:12:29,805 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:12:31,355 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:12:31,355 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:12:31,355 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:12:31,360 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:12:31,806 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:12:33,397 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:12:33,809 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:12:36,782 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:12:36,783 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:12:36,784 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:12:36,789 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:12:36,811 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:12:37,814 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:12:38,137 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 15:12:39,138 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:12:41,594 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:12:41,595 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:12:41,595 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:12:41,600 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:12:41,817 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:12:43,134 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:12:43,134 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:12:43,135 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:12:43,819 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:12:44,435 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:12:46,461 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:12:46,462 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:12:46,462 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:12:46,467 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:12:46,821 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:12:47,823 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:12:49,505 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:12:53,570 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:12:53,571 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:12:53,571 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:12:53,576 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:12:53,827 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:12:54,577 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:12:55,830 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:12:57,421 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:12:57,421 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:12:57,422 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:12:57,427 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:12:57,831 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:12:58,134 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:12:58,135 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:12:58,135 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:12:59,835 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:13:00,399 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:13:01,928 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:13:01,929 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:13:01,929 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:13:01,934 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:13:02,838 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:13:03,841 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:13:05,969 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:13:06,746 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:13:06,747 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:13:06,747 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:13:06,752 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:13:06,843 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:13:07,846 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:13:08,137 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 15:13:11,138 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:13:12,342 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:13:12,342 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:13:12,343 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:13:12,348 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:13:12,849 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:13:13,135 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:13:13,135 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:13:13,136 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:13:13,852 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:13:16,407 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:13:18,538 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:13:18,538 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:13:18,538 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:13:18,543 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:13:18,856 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:13:19,858 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:13:21,581 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:13:25,476 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:13:25,477 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:13:25,477 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:13:25,482 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:13:25,862 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:13:27,483 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:13:27,865 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:13:28,135 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:13:28,135 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:13:28,136 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:13:31,227 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:13:31,228 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:13:31,228 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:13:31,234 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:13:31,868 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:13:33,269 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:13:33,871 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:13:38,139 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 15:13:38,494 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:13:38,494 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:13:38,495 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:13:38,495 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:13:38,503 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:13:38,875 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:13:39,878 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:13:43,135 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:13:43,135 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:13:43,136 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:13:44,366 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:13:44,423 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:13:44,424 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:13:44,424 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:13:44,429 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:13:44,881 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:13:45,884 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:13:49,090 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:13:49,091 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:13:49,092 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:13:49,097 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:13:49,886 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:13:50,097 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:13:51,889 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:13:53,484 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:13:53,485 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:13:53,485 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:13:53,490 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:13:53,891 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:13:55,490 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:13:55,894 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:13:58,135 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:13:58,135 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:13:58,135 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:13:58,435 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:13:58,436 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:13:58,436 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:13:58,441 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:13:58,896 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:13:59,898 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:14:01,481 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:14:05,029 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:14:05,030 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:14:05,030 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:14:05,036 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:14:05,904 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:14:07,036 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:14:07,907 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:14:08,141 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 15:14:12,143 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:14:13,016 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:14:13,017 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:14:13,017 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:14:13,022 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:14:13,135 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:14:13,135 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:14:13,136 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:14:13,911 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:14:15,914 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:14:17,383 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:14:18,305 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:14:18,306 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:14:18,306 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:14:18,311 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:14:18,917 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:14:19,919 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:14:23,350 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:14:23,414 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:14:23,415 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:14:23,415 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:14:23,420 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:14:23,922 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:14:25,925 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:14:28,135 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:14:28,135 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:14:28,136 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:14:28,417 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:14:30,200 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:14:30,201 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:14:30,201 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:14:30,206 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:14:30,928 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:14:31,931 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:14:34,241 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:14:35,103 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:14:35,104 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:14:35,104 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:14:35,110 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:14:35,933 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:14:37,937 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:14:38,144 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 15:14:40,145 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:14:41,635 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:14:41,636 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:14:41,636 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:14:41,641 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:14:41,939 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:14:43,135 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:14:43,135 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:14:43,136 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:14:43,943 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:14:45,408 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:14:46,313 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:14:46,314 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:14:46,314 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:14:46,319 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:14:46,945 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:14:47,948 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:14:50,850 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:14:50,851 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:14:50,851 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:14:50,851 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:14:50,856 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:14:50,950 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:14:51,952 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:14:55,693 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:14:55,694 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:14:55,694 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:14:55,699 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:14:55,955 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:14:56,700 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:14:57,958 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:14:58,135 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:14:58,135 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:14:58,136 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:15:01,677 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:15:01,677 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:15:01,677 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:15:01,682 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:15:01,721 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:15:01,960 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:15:03,965 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:15:06,722 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:15:07,578 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:15:07,579 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:15:07,579 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:15:07,584 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:15:07,968 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:15:08,149 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 15:15:09,971 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:15:12,151 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:15:13,135 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:15:13,136 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:15:13,136 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:15:15,649 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:15:15,650 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:15:15,650 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:15:15,655 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:15:15,975 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:15:17,693 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:15:17,979 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:15:22,589 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:15:22,590 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:15:22,590 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:15:22,595 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:15:22,982 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:15:23,596 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:15:23,985 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:15:27,547 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:15:27,547 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:15:27,548 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:15:27,553 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:15:27,987 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:15:28,136 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:15:28,136 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:15:28,136 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:15:29,346 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:15:29,990 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:15:31,886 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:15:31,887 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:15:31,887 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:15:31,892 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:15:31,992 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:15:33,995 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:15:34,929 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:15:37,675 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:15:37,676 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:15:37,676 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:15:37,682 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:15:37,998 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:15:38,148 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 15:15:40,001 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:15:40,150 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:15:42,490 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:15:42,491 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:15:42,491 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:15:42,495 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:15:43,003 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:15:43,136 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:15:43,136 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:15:43,136 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:15:44,006 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:15:45,375 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:15:46,807 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:15:46,808 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:15:46,808 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:15:46,813 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:15:47,008 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:15:48,010 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:15:50,849 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:15:52,494 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:15:52,495 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:15:52,495 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:15:52,500 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:15:53,013 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:15:54,016 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:15:56,501 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:15:57,772 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:15:57,773 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:15:57,773 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:15:57,777 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:15:58,018 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:15:58,136 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:15:58,136 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:15:58,136 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:16:00,022 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:16:02,397 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:16:02,686 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:16:02,686 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:16:02,686 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:16:02,691 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:16:03,025 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:16:04,028 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:16:07,730 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:16:07,865 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:16:07,866 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:16:07,866 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:16:07,872 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:16:08,030 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:16:08,155 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 15:16:10,034 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:16:12,494 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:16:12,495 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:16:12,496 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:16:12,501 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:16:13,036 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:16:13,136 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:16:13,136 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:16:13,136 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:16:13,382 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:16:14,039 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:16:18,383 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:16:19,677 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:16:19,678 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:16:19,678 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:16:19,683 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:16:20,043 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:16:22,046 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:16:23,722 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:16:28,136 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:16:28,136 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:16:28,136 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:16:28,138 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:16:28,420 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:16:28,420 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:16:28,425 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:16:29,050 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:16:29,425 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:16:30,053 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:16:32,687 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:16:32,688 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:16:32,688 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:16:32,693 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:16:33,055 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:16:34,057 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:16:34,693 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:16:36,948 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:16:36,949 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:16:36,949 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:16:36,954 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:16:37,059 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:16:38,062 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:16:38,153 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 15:16:40,155 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:16:42,132 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:16:42,133 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:16:42,133 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:16:42,138 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:16:43,067 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:16:43,136 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:16:43,136 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:16:43,136 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:16:44,070 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:16:45,420 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:16:48,261 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:16:48,262 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:16:48,262 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:16:48,267 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:16:49,075 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:16:50,079 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:16:51,305 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:16:53,963 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:16:53,966 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:16:53,966 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:16:53,971 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:16:54,083 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:16:56,088 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:16:56,971 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:16:58,136 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:16:58,136 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:16:58,136 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:16:59,317 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:16:59,317 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:16:59,317 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:16:59,322 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:17:00,092 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:17:02,095 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:17:02,361 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:17:04,177 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:17:04,178 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:17:04,178 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:17:04,183 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:17:05,099 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:17:06,102 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:17:08,155 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 15:17:08,157 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:17:09,413 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:17:09,414 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:17:09,414 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:17:09,419 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:17:10,106 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:17:12,110 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:17:13,136 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:17:13,136 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:17:13,137 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:17:13,402 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:17:13,494 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:17:13,494 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:17:13,495 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:17:13,499 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:17:14,113 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:17:16,118 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:17:18,062 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:17:18,063 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:17:18,063 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:17:18,068 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:17:18,120 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:17:19,069 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:17:20,124 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:17:23,961 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:17:23,962 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:17:23,962 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:17:23,967 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:17:24,127 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:17:24,968 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:17:26,133 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:17:28,136 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:17:28,136 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:17:28,137 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:17:28,195 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:17:28,413 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:17:28,413 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:17:28,418 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:17:29,134 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:17:30,138 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:17:30,419 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:17:32,211 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:17:32,212 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:17:32,212 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:17:32,217 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:17:33,141 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:17:34,145 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:17:36,218 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:17:37,376 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:17:37,376 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:17:37,377 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:17:37,381 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:17:38,149 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:17:38,157 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 15:17:40,153 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:17:41,956 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:17:41,957 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:17:41,957 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:17:41,957 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:17:41,962 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:17:42,155 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:17:43,136 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:17:43,136 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:17:43,137 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:17:44,160 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:17:46,256 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:17:46,257 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:17:46,257 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:17:46,262 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:17:47,162 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:17:47,297 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:17:48,166 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:17:50,552 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:17:50,553 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:17:50,553 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:17:50,558 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:17:51,168 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:17:52,172 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:17:52,559 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:17:56,254 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:17:56,255 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:17:56,255 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:17:56,260 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:17:57,177 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:17:58,136 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:17:58,136 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:17:58,137 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:17:58,179 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:17:58,398 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:18:01,752 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:18:01,752 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:18:01,753 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:18:01,757 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:18:02,185 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:18:03,793 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:18:04,190 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:18:07,582 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:18:07,583 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:18:07,583 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:18:07,588 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:18:08,159 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 15:18:08,194 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:18:09,161 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:18:10,199 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:18:13,136 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:18:13,137 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:18:13,137 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:18:14,432 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:18:15,301 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:18:15,302 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:18:15,302 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:18:15,307 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:18:16,207 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:18:18,211 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:18:19,916 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:18:19,917 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:18:19,917 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:18:19,917 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:18:19,922 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:18:20,213 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:18:22,217 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:18:24,923 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:18:25,430 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:18:25,431 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:18:25,431 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:18:25,436 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:18:26,221 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:18:28,136 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:18:28,137 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:18:28,137 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:18:28,226 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:18:29,694 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:18:29,695 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:18:29,695 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:18:29,700 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:18:30,227 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:18:30,737 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:18:32,232 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:18:33,984 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:18:33,984 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:18:33,985 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:18:33,990 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:18:34,233 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:18:35,990 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:18:36,236 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:18:38,165 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 15:18:38,851 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:18:38,852 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:18:38,852 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:18:38,857 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:18:39,240 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:18:40,244 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:18:41,858 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:18:43,136 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:18:43,137 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:18:43,137 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:18:43,383 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:18:43,384 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:18:43,384 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:18:43,389 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:18:44,249 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:18:46,253 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:18:47,426 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:18:48,035 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:18:48,036 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:18:48,036 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:18:48,042 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:18:48,255 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:18:50,259 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:18:52,802 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:18:52,803 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:18:52,803 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:18:52,803 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:18:52,808 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:18:53,261 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:18:54,265 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:18:57,205 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:18:57,205 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:18:57,206 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:18:57,211 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:18:57,267 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:18:58,137 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:18:58,137 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:18:58,137 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:18:58,269 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:18:58,412 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:19:01,735 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:19:01,736 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:19:01,736 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:19:01,741 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:19:02,275 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:19:03,777 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:19:04,280 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:19:07,228 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:19:07,229 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:19:07,229 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:19:07,234 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:19:07,282 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:19:08,164 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 15:19:08,285 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:19:09,165 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:19:13,137 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:19:13,137 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:19:13,137 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:19:13,533 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:19:13,534 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:19:13,534 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:19:13,540 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:19:14,293 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:19:14,577 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:19:16,298 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:19:19,034 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:19:19,034 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:19:19,035 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:19:19,040 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:19:19,300 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:19:20,040 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:19:20,302 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:19:22,870 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:19:22,871 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:19:22,872 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:19:22,876 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:19:23,305 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:19:24,309 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:19:25,877 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:19:27,491 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:19:27,492 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:19:27,492 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:19:27,497 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:19:28,137 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:19:28,137 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:19:28,138 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:19:28,313 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:19:30,318 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:19:31,401 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:19:32,189 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:19:32,190 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:19:32,190 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:19:32,195 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:19:32,319 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:19:34,324 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:19:36,849 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:19:36,850 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:19:36,851 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:19:36,851 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:19:36,855 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:19:37,326 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:19:38,166 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 15:19:38,330 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:19:41,388 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:19:41,392 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:19:41,393 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:19:41,397 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:19:42,335 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:19:42,398 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:19:43,137 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:19:43,137 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:19:43,137 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:19:44,339 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:19:48,373 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:19:48,647 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:19:48,647 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:19:48,648 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:19:48,652 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:19:49,344 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:19:50,348 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:19:53,690 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:19:54,566 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:19:54,567 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:19:54,567 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:19:54,572 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:19:55,352 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:19:56,356 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:19:58,137 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:19:58,137 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:19:58,137 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:19:59,091 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:19:59,092 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:19:59,092 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:19:59,092 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:19:59,097 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:19:59,358 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:20:00,362 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:20:04,133 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:20:07,620 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:20:07,621 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:20:07,621 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:20:07,626 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:20:08,168 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 15:20:08,371 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:20:09,169 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:20:10,376 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:20:12,287 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:20:12,287 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:20:12,288 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:20:12,293 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:20:12,378 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:20:13,137 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:20:13,137 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:20:13,138 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:20:14,359 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:20:14,383 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:20:17,107 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:20:17,107 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:20:17,107 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:20:17,112 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:20:17,385 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:20:18,387 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:20:20,149 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:20:22,506 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:20:22,506 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:20:22,507 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:20:22,512 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:20:23,393 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:20:24,397 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:20:25,512 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:20:28,137 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:20:28,137 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:20:28,138 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:20:28,288 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:20:28,365 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:20:28,366 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:20:28,371 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:20:28,401 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:20:30,406 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:20:31,371 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:20:32,610 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:20:32,610 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:20:32,611 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:20:32,615 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:20:33,408 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:20:34,412 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:20:36,653 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:20:38,170 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 15:20:38,547 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:20:38,548 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:20:38,548 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:20:38,553 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:20:39,417 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:20:40,420 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:20:42,554 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:20:43,137 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:20:43,137 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:20:43,138 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:20:44,593 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:20:44,594 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:20:44,594 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:20:44,600 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:20:45,426 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:20:46,430 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:20:47,637 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:20:49,312 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:20:49,312 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:20:49,313 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:20:49,318 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:20:49,432 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:20:50,436 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:20:53,318 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:20:54,993 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:20:54,994 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:20:54,994 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:20:54,998 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:20:55,440 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:20:56,444 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:20:58,137 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:20:58,137 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:20:58,138 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:20:58,421 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:21:03,192 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:21:03,192 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:21:03,193 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:21:03,197 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:21:03,453 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:21:04,233 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:21:04,456 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:21:08,176 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 15:21:08,970 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:21:08,971 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:21:08,971 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:21:08,976 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:21:09,460 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:21:09,976 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:21:10,464 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:21:13,137 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:21:13,138 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:21:13,138 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:21:14,656 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:21:14,657 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:21:14,658 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:21:14,664 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:21:15,471 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:21:15,701 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:21:16,474 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:21:20,702 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:21:21,226 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:21:21,227 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:21:21,227 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:21:21,232 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:21:21,478 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:21:22,483 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:21:25,548 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:21:25,549 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:21:25,549 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:21:25,554 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:21:26,487 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:21:26,555 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:21:28,137 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:21:28,138 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:21:28,138 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:21:28,491 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:21:30,780 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:21:30,780 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:21:30,781 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:21:30,785 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:21:31,493 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:21:31,821 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:21:32,497 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:21:34,995 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:21:34,996 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:21:34,996 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:21:35,001 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:21:35,499 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:21:36,503 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:21:37,002 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:21:38,175 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 15:21:42,177 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:21:42,800 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:21:42,800 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:21:42,801 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:21:42,805 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:21:43,137 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:21:43,138 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:21:43,138 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:21:43,509 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:21:44,513 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:21:47,383 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:21:49,346 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:21:49,347 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:21:49,347 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:21:49,351 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:21:49,515 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:21:50,517 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:21:52,389 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:21:53,839 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:21:53,840 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:21:53,840 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:21:53,845 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:21:54,520 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:21:56,523 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:21:57,846 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:21:58,093 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:21:58,094 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:21:58,094 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:21:58,099 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:21:58,138 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:21:58,138 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:21:58,138 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:21:58,524 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:22:00,528 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:22:03,424 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:22:04,659 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:22:04,660 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:22:04,660 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:22:04,665 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:22:05,533 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:22:06,535 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:22:08,180 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 15:22:09,181 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:22:09,845 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:22:09,846 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:22:09,846 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:22:09,851 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:22:10,538 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:22:12,541 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:22:13,138 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:22:13,138 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:22:13,138 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:22:14,336 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:22:15,253 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:22:15,254 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:22:15,254 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:22:15,259 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:22:15,543 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:22:16,546 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:22:20,297 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:22:21,057 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:22:21,058 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:22:21,058 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:22:21,064 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:22:21,550 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:22:22,553 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:22:25,938 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:22:25,939 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:22:25,939 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:22:25,939 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:22:25,944 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:22:26,556 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:22:28,138 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:22:28,138 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:22:28,138 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:22:28,559 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:22:31,292 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:22:31,292 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:22:31,293 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:22:31,293 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:22:31,298 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:22:31,561 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:22:32,564 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:22:36,218 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:22:36,218 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:22:36,219 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:22:36,223 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:22:36,566 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:22:37,224 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:22:38,179 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 15:22:38,570 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:22:43,138 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:22:43,138 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:22:43,138 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:22:43,412 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:22:44,764 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:22:44,765 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:22:44,765 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:22:44,770 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:22:45,575 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:22:46,577 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:22:48,805 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:22:49,073 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:22:49,074 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:22:49,074 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:22:49,079 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:22:49,579 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:22:50,582 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:22:54,080 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:22:56,627 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:22:56,627 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:22:56,628 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:22:56,633 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:22:57,586 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:22:58,138 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:22:58,138 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:22:58,138 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:22:58,588 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:22:59,404 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:23:01,005 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:23:01,005 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:23:01,005 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:23:01,010 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:23:01,590 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:23:02,593 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:23:05,049 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:23:05,726 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:23:05,726 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:23:05,727 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:23:05,732 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:23:06,597 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:23:08,181 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 15:23:08,600 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:23:10,183 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:23:11,066 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:23:11,067 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:23:11,067 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:23:11,072 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:23:11,603 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:23:12,606 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:23:13,138 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:23:13,138 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:23:13,138 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:23:15,384 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:23:15,541 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:23:15,542 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:23:15,542 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:23:15,548 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:23:15,608 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:23:16,613 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:23:20,585 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:23:22,479 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:23:22,480 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:23:22,480 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:23:22,486 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:23:22,618 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:23:24,624 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:23:26,487 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:23:26,992 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:23:26,993 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:23:26,993 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:23:27,001 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:23:27,626 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:23:28,138 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:23:28,138 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:23:28,139 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:23:28,633 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:23:31,512 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:23:31,513 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:23:31,513 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:23:31,513 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:23:31,524 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:23:31,635 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:23:32,639 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:23:36,146 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:23:36,147 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:23:36,147 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:23:36,153 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:23:36,642 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:23:37,154 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:23:38,183 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 15:23:38,647 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:23:41,040 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:23:41,041 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:23:41,041 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:23:41,050 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:23:41,650 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:23:42,651 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:23:43,051 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:23:43,138 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:23:43,138 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:23:43,138 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:23:48,226 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:23:48,227 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:23:48,227 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:23:48,227 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:23:48,234 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:23:48,655 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:23:50,660 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:23:52,117 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:23:52,118 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:23:52,118 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:23:52,125 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:23:52,661 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:23:54,125 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:23:54,665 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:23:57,210 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:23:57,211 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:23:57,211 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:23:57,220 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:23:57,667 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:23:58,138 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:23:58,138 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:23:58,140 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:23:58,671 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:23:59,545 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:24:04,546 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:24:06,105 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:24:06,105 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:24:06,106 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:24:06,111 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:24:06,678 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:24:08,186 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 15:24:08,681 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:24:10,187 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:24:10,612 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:24:10,613 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:24:10,613 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:24:10,618 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:24:10,683 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:24:12,687 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:24:13,138 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:24:13,139 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:24:13,139 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:24:15,381 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:24:16,054 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:24:16,054 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:24:16,054 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:24:16,060 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:24:16,689 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:24:18,695 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:24:20,914 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:24:20,915 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:24:20,915 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:24:20,915 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:24:20,922 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:24:21,697 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:24:22,700 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:24:25,923 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:24:27,406 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:24:27,407 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:24:27,407 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:24:27,414 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:24:27,703 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:24:28,138 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:24:28,139 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:24:28,139 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:24:28,706 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:24:31,447 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:24:33,448 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:24:33,449 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:24:33,449 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:24:33,454 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:24:33,710 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:24:34,713 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:24:36,489 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:24:38,192 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 15:24:41,729 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:24:41,730 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:24:41,730 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:24:41,730 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:24:41,736 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:24:42,719 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:24:43,138 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:24:43,139 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:24:43,139 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:24:44,723 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:24:46,135 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:24:46,136 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:24:46,136 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:24:46,141 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:24:46,725 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:24:47,177 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:24:48,729 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:24:51,199 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:24:51,200 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:24:51,200 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:24:51,206 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:24:51,731 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:24:52,206 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:24:52,732 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:24:56,250 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:24:56,251 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:24:56,251 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:24:56,256 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:24:56,734 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:24:57,257 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:24:58,138 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:24:58,139 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:24:58,139 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:24:58,736 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:25:00,345 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:25:00,345 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:25:00,346 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:25:00,351 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:25:00,737 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:25:02,389 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:25:02,741 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:25:05,853 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:25:05,854 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:25:05,854 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:25:05,859 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:25:06,746 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:25:07,860 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:25:08,190 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 15:25:08,749 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:25:12,044 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:25:12,045 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:25:12,045 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:25:12,050 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:25:12,752 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:25:13,051 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:25:13,138 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:25:13,139 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:25:13,140 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:25:14,755 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:25:16,114 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:25:16,114 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:25:16,114 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:25:16,119 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:25:16,756 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:25:18,157 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:25:18,760 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:25:21,561 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:25:21,562 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:25:21,562 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:25:21,567 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:25:21,762 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:25:22,764 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:25:23,568 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:25:26,226 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:25:26,226 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:25:26,226 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:25:26,232 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:25:26,767 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:25:28,139 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:25:28,139 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:25:28,139 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:25:28,768 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:25:29,397 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:25:30,176 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:25:30,177 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:25:30,177 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:25:30,182 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:25:30,770 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:25:32,773 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:25:35,217 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:25:36,046 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:25:36,047 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:25:36,047 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:25:36,053 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:25:36,776 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:25:38,195 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 15:25:38,779 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:25:41,196 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:25:43,139 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:25:43,139 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:25:43,139 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:25:43,318 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:25:43,363 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:25:43,363 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:25:43,368 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:25:43,783 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:25:44,786 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:25:46,368 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:25:47,433 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:25:47,434 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:25:47,434 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:25:47,439 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:25:47,788 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:25:48,791 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:25:51,477 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:25:55,002 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:25:55,003 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:25:55,003 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:25:55,008 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:25:55,795 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:25:56,796 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:25:57,009 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:25:58,139 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:25:58,139 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:25:58,139 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:26:00,268 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:26:00,269 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:26:00,269 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:26:00,275 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:26:00,798 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:26:02,313 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:26:02,800 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:26:05,373 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:26:05,374 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:26:05,374 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:26:05,379 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:26:05,803 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:26:06,806 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:26:07,380 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:26:08,194 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 15:26:10,132 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:26:10,133 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:26:10,133 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:26:10,139 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:26:10,809 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:26:12,810 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:26:13,139 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:26:13,139 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:26:13,139 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:26:13,353 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:26:14,826 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:26:14,827 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:26:14,827 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:26:14,832 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:26:15,812 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:26:16,815 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:26:18,870 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:26:19,081 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:26:19,082 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:26:19,082 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:26:19,087 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:26:19,817 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:26:20,820 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:26:24,088 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:26:24,211 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:26:24,212 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:26:24,212 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:26:24,217 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:26:24,822 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:26:26,825 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:26:28,139 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:26:28,139 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:26:28,139 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:26:29,401 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:26:29,705 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:26:29,706 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:26:29,706 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:26:29,711 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:26:29,827 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:26:30,830 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:26:34,253 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:26:34,254 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:26:34,255 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:26:34,261 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:26:34,832 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:26:35,261 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:26:36,835 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:26:38,196 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 15:26:39,353 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:26:39,354 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:26:39,355 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:26:39,360 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:26:39,837 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:26:40,360 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:26:40,840 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:26:43,139 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:26:43,139 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:26:43,139 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:26:43,908 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:26:43,908 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:26:43,909 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:26:43,914 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:26:44,843 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:26:45,949 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:26:46,846 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:26:49,250 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:26:49,251 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:26:49,251 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:26:49,256 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:26:49,848 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:26:50,851 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:26:51,257 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:26:53,881 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:26:53,882 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:26:53,882 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:26:53,888 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:26:54,854 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:26:56,855 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:26:56,888 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:26:58,139 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:26:58,139 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:26:58,139 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:26:58,415 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:26:58,415 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:26:58,416 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:26:58,425 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:26:58,856 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:27:00,859 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:27:02,458 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:27:02,986 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:27:02,987 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:27:02,987 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:27:02,992 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:27:03,863 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:27:04,865 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:27:07,994 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:27:08,199 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 15:27:10,540 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:27:10,541 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:27:10,541 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:27:10,546 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:27:10,870 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:27:12,873 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:27:13,139 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:27:13,139 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:27:13,140 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:27:13,367 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:27:15,544 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:27:15,544 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:27:15,545 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:27:15,550 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:27:15,875 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:27:16,877 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:27:18,585 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:27:20,277 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:27:20,278 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:27:20,278 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:27:20,283 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:27:20,880 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:27:22,884 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:27:24,284 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:27:24,697 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:27:24,698 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:27:24,699 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:27:24,703 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:27:24,885 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:27:26,888 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:27:28,139 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:27:28,139 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:27:28,140 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:27:29,243 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:27:29,244 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:27:29,244 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:27:29,249 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:27:29,285 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:27:29,890 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:27:30,892 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:27:34,286 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:27:34,805 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:27:34,806 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:27:34,807 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:27:34,812 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:27:34,895 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:27:36,898 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:27:38,201 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 15:27:40,202 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:27:43,139 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:27:43,139 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:27:43,140 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:27:43,183 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:27:43,364 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:27:43,364 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:27:43,369 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:27:43,903 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:27:44,906 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:27:45,370 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:27:49,079 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:27:49,080 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:27:49,080 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:27:49,085 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:27:49,909 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:27:50,912 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:27:51,086 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:27:54,248 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:27:54,249 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:27:54,249 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:27:54,254 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:27:54,914 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:27:56,255 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:27:56,917 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:27:58,139 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:27:58,140 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:27:58,140 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:27:58,286 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:27:58,356 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:27:58,356 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:27:58,361 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:27:58,918 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:28:00,922 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:28:01,362 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:28:04,171 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:28:04,172 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:28:04,173 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:28:04,177 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:28:04,925 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:28:06,929 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:28:07,213 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:28:08,207 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 15:28:10,277 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:28:10,278 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:28:10,278 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:28:10,283 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:28:10,932 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:28:12,284 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:28:12,935 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:28:13,139 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:28:13,140 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:28:13,140 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:28:15,773 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:28:15,774 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:28:15,774 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:28:15,779 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:28:15,936 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:28:16,939 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:28:17,817 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:28:21,178 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:28:21,179 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:28:21,179 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:28:21,184 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:28:21,942 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:28:22,945 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:28:23,185 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:28:27,755 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:28:27,756 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:28:27,757 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:28:27,762 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:28:27,948 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:28:28,139 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:28:28,140 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:28:28,140 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:28:28,343 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:28:28,949 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:28:33,343 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:28:34,734 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:28:34,735 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:28:34,735 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:28:34,740 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:28:34,952 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:28:36,956 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:28:38,205 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 15:28:39,207 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:28:39,382 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:28:39,383 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:28:39,383 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:28:39,389 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:28:39,958 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:28:40,960 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:28:43,140 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:28:43,140 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:28:43,140 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:28:44,299 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:28:44,299 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:28:44,300 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:28:44,300 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:28:44,305 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:28:44,963 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:28:46,966 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:28:49,127 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:28:49,128 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:28:49,128 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:28:49,134 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:28:49,968 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:28:50,134 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:28:50,971 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:28:54,481 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:28:54,482 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:28:54,483 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:28:54,488 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:28:54,973 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:28:55,488 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:28:56,977 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:28:58,140 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:28:58,140 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:28:58,140 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:28:58,926 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:28:58,927 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:28:58,927 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:28:58,932 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:28:58,978 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:29:00,969 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:29:00,981 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:29:05,970 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:29:06,744 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:29:06,745 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:29:06,745 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:29:06,750 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:29:06,986 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:29:08,211 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 15:29:08,990 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:29:11,212 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:29:11,554 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:29:11,554 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:29:11,554 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:29:11,559 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:29:11,992 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:29:12,994 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:29:13,140 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:29:13,140 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:29:13,140 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:29:16,405 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:29:18,673 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:29:18,674 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:29:18,674 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:29:18,679 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:29:18,998 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:29:21,001 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:29:21,717 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:29:23,658 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:29:23,659 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:29:23,660 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:29:23,668 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:29:24,003 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:29:25,006 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:29:27,669 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:29:28,140 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:29:28,140 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:29:28,141 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:29:28,288 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:29:28,440 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:29:28,440 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:29:28,445 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:29:29,009 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:29:31,012 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:29:32,429 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:29:32,430 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:29:32,431 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:29:32,436 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:29:33,013 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:29:33,436 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:29:35,017 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:29:36,892 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:29:36,893 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:29:36,893 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:29:36,898 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:29:37,021 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:29:38,210 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 15:29:39,024 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:29:39,211 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:29:42,938 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:29:42,939 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:29:42,939 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:29:42,944 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:29:43,027 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:29:43,140 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:29:43,140 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:29:43,141 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:29:44,373 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:29:45,030 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:29:48,532 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:29:48,533 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:29:48,533 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:29:48,538 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:29:49,033 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:29:49,573 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:29:51,036 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:29:53,037 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:29:53,038 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:29:53,038 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:29:53,043 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:29:54,039 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:29:55,042 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:29:55,043 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:29:58,140 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:29:58,140 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:29:58,140 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:29:59,092 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:29:59,092 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:29:59,093 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:29:59,097 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:30:00,045 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:30:00,133 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:30:01,048 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:30:04,041 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:30:04,042 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:30:04,042 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:30:04,047 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:30:04,051 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:30:05,053 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:30:06,047 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:30:08,145 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:30:08,146 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:30:08,146 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:30:08,151 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:30:08,212 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 15:30:09,056 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:30:11,060 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:30:11,213 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:30:13,060 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:30:13,061 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:30:13,061 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:30:13,066 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:30:13,140 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:30:13,140 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:30:13,141 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:30:14,064 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:30:15,066 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:30:16,427 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:30:17,529 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:30:17,530 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:30:17,530 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:30:17,535 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:30:18,068 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:30:19,069 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:30:21,573 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:30:21,822 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:30:21,823 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:30:21,823 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:30:21,828 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:30:22,071 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:30:23,074 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:30:26,376 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:30:26,377 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:30:26,377 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:30:26,382 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:30:27,076 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:30:27,382 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:30:28,140 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:30:28,140 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:30:28,141 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:30:29,080 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:30:30,925 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:30:30,926 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:30:30,926 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:30:30,931 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:30:31,081 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:30:32,969 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:30:33,084 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:30:36,711 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:30:36,715 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:30:36,715 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:30:36,720 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:30:37,087 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:30:38,214 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 15:30:38,215 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:30:39,090 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:30:42,358 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:30:42,359 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:30:42,359 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:30:42,364 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:30:43,093 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:30:43,140 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:30:43,141 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:30:43,141 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:30:43,425 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:30:45,097 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:30:48,426 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:30:51,877 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:30:51,878 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:30:51,878 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:30:51,883 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:30:52,101 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:30:53,104 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:30:53,921 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:30:57,344 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:30:57,345 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:30:57,345 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:30:57,350 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:30:58,107 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:30:58,140 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:30:58,141 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:30:58,141 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:30:59,110 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:30:59,438 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:31:02,613 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:31:02,614 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:31:02,614 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:31:02,619 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:31:03,112 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:31:04,657 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:31:05,117 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:31:07,050 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:31:07,050 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:31:07,051 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:31:07,056 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:31:07,119 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:31:08,216 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 15:31:09,122 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:31:10,218 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:31:11,536 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:31:11,537 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:31:11,537 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:31:11,542 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:31:12,124 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:31:13,127 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:31:13,140 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:31:13,141 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:31:13,141 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:31:15,384 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:31:15,655 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:31:15,656 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:31:15,656 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:31:15,661 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:31:16,129 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:31:17,131 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:31:20,513 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:31:20,514 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:31:20,514 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:31:20,514 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:31:20,519 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:31:21,134 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:31:23,137 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:31:24,966 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:31:24,967 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:31:24,967 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:31:24,972 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:31:25,139 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:31:25,972 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:31:27,142 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:31:28,140 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:31:28,141 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:31:28,141 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:31:31,369 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:31:31,454 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:31:31,455 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:31:31,455 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:31:31,460 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:31:32,145 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:31:33,148 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:31:36,498 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:31:37,712 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:31:37,713 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:31:37,713 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:31:37,719 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:31:38,151 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:31:38,222 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 15:31:39,154 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:31:42,061 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:31:42,062 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:31:42,062 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:31:42,063 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:31:42,067 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:31:42,156 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:31:43,141 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:31:43,141 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:31:43,141 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:31:43,156 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:31:46,757 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:31:46,757 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:31:46,758 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:31:46,763 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:31:47,159 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:31:47,801 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:31:49,161 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:31:52,802 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:31:53,075 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:31:53,076 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:31:53,076 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:31:53,081 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:31:53,163 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:31:55,166 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:31:58,082 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:31:58,141 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:31:58,141 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:31:58,141 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:32:03,418 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:32:03,673 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:32:03,673 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:32:03,674 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:32:03,679 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:32:04,173 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:32:05,175 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:32:08,191 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:32:08,192 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:32:08,192 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:32:08,197 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:32:08,220 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 15:32:09,178 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:32:09,179 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:32:09,222 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:32:13,141 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:32:13,141 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:32:13,141 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:32:14,428 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:32:16,418 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:32:16,418 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:32:16,419 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:32:16,423 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:32:17,184 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:32:19,188 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:32:19,462 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:32:22,996 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:32:22,997 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:32:22,997 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:32:23,002 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:32:23,190 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:32:25,002 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:32:25,193 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:32:28,141 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:32:28,141 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:32:28,141 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:32:30,405 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:32:31,423 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:32:31,424 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:32:31,424 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:32:31,429 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:32:32,197 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:32:33,200 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:32:35,465 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:32:37,917 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:32:37,917 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:32:37,918 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:32:37,923 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:32:38,203 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:32:38,223 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 15:32:39,206 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:32:41,224 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:32:42,924 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:32:42,925 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:32:42,925 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:32:42,929 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:32:43,141 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:32:43,141 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:32:43,142 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:32:43,208 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:32:45,212 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:32:46,368 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:32:49,473 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:32:49,473 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:32:49,474 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:32:49,479 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:32:50,215 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:32:51,218 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:32:51,517 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:32:53,658 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:32:53,659 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:32:53,659 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:32:53,664 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:32:54,220 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:32:55,221 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:32:56,664 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:32:58,141 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:32:58,141 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:32:58,141 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:32:59,821 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:32:59,822 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:32:59,822 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:32:59,827 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:33:00,224 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:33:01,226 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:33:01,869 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:33:06,663 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:33:06,664 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:33:06,664 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:33:06,669 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:33:07,233 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:33:07,669 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:33:08,225 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 15:33:09,236 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:33:13,141 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:33:13,141 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:33:13,142 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:33:13,430 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:33:16,602 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:33:16,603 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:33:16,603 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:33:16,608 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:33:17,242 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:33:18,645 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:33:19,245 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:33:21,794 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:33:21,795 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:33:21,795 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:33:21,800 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:33:22,247 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:33:23,250 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:33:23,800 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:33:28,141 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:33:28,141 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:33:28,142 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:33:28,455 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:33:28,456 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:33:28,456 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:33:28,461 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:33:29,254 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:33:29,497 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:33:31,257 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:33:34,498 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:33:35,187 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:33:35,188 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:33:35,188 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:33:35,193 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:33:35,260 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:33:37,263 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:33:38,227 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 15:33:40,229 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:33:40,321 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:33:40,322 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:33:40,322 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:33:40,327 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:33:41,266 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:33:43,141 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:33:43,141 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:33:43,142 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:33:43,270 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:33:44,877 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:33:44,878 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:33:44,878 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:33:44,883 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:33:45,272 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:33:45,921 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:33:47,275 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:33:49,573 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:33:49,574 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:33:49,574 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:33:49,579 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:33:50,277 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:33:51,278 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:33:51,580 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:33:55,433 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:33:55,434 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:33:55,434 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:33:55,439 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:33:56,281 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:33:57,284 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:33:57,440 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:33:58,141 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:33:58,141 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:33:58,142 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:34:00,121 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:34:00,122 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:34:00,122 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:34:00,127 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:34:00,285 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:34:01,288 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:34:03,165 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:34:05,089 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:34:05,090 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:34:05,090 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:34:05,095 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:34:05,292 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:34:07,296 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:34:08,233 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 15:34:08,234 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:34:11,805 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:34:11,806 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:34:11,806 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:34:11,811 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:34:12,299 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:34:13,141 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:34:13,141 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:34:13,142 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:34:13,302 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:34:13,439 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:34:18,440 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:34:19,196 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:34:19,196 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:34:19,197 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:34:19,202 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:34:19,306 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:34:21,310 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:34:24,210 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:34:24,211 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:34:24,211 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:34:24,212 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:34:24,216 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:34:24,312 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:34:25,314 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:34:28,141 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:34:28,142 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:34:28,142 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:34:29,416 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:34:30,044 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:34:30,045 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:34:30,045 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:34:30,050 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:34:30,317 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:34:31,320 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:34:34,343 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:34:34,343 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:34:34,344 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:34:34,349 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:34:35,322 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:34:35,323 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:34:35,349 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:34:38,232 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 15:34:38,638 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:34:38,639 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:34:38,639 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:34:38,644 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:34:39,325 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:34:40,645 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:34:41,329 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:34:43,142 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:34:43,142 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:34:43,142 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:34:44,562 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:34:44,563 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:34:44,563 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:34:44,568 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:34:45,331 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:34:46,605 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:34:47,335 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:34:49,808 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:34:49,809 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:34:49,810 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:34:49,815 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:34:50,337 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:34:51,339 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:34:51,815 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:34:55,209 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:34:55,210 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:34:55,210 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:34:55,215 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:34:55,342 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:34:57,215 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:34:57,343 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:34:58,142 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:34:58,142 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:34:58,142 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:34:59,940 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:34:59,940 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:34:59,941 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:34:59,946 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:35:00,345 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:35:01,347 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:35:02,981 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:35:07,371 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:35:07,371 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:35:07,372 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:35:07,376 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:35:08,237 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 15:35:08,238 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:35:08,353 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:35:09,356 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:35:13,142 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:35:13,142 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:35:13,142 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:35:13,354 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:35:15,279 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:35:15,280 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:35:15,280 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:35:15,284 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:35:15,360 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:35:17,363 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:35:19,321 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:35:19,424 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:35:19,425 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:35:19,425 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:35:19,430 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:35:20,365 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:35:21,368 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:35:24,431 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:35:26,009 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:35:26,010 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:35:26,010 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:35:26,015 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:35:26,371 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:35:27,374 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:35:28,142 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:35:28,142 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:35:28,142 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:35:30,411 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:35:30,734 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:35:30,735 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:35:30,735 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:35:30,741 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:35:31,376 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:35:33,379 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:35:35,778 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:35:36,411 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:35:36,411 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:35:36,411 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:35:36,417 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:35:37,384 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:35:37,384 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:35:38,236 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 15:35:41,154 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:35:41,154 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:35:41,155 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:35:41,155 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:35:41,159 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:35:41,387 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:35:43,142 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:35:43,142 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:35:43,142 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:35:43,390 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:35:46,188 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:35:46,189 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:35:46,189 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:35:46,189 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:35:46,194 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:35:46,392 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:35:47,395 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:35:51,229 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:35:52,208 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:35:52,209 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:35:52,209 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:35:52,214 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:35:52,398 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:35:53,401 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:35:57,215 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:35:57,216 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:35:57,216 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:35:57,216 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:35:57,221 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:35:57,403 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:35:58,142 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:35:58,142 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:35:58,143 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:35:59,407 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:36:02,189 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:36:02,189 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:36:02,189 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:36:02,195 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:36:02,233 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:36:02,408 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:36:03,411 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:36:07,234 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:36:08,238 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 15:36:11,006 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:36:11,007 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:36:11,007 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:36:11,015 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:36:11,418 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:36:13,016 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:36:13,142 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:36:13,142 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:36:13,142 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:36:13,419 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:36:17,243 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:36:17,243 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:36:17,244 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:36:17,248 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:36:17,421 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:36:18,285 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:36:19,423 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:36:21,733 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:36:21,734 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:36:21,734 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:36:21,739 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:36:22,425 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:36:23,428 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:36:23,740 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:36:27,066 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:36:27,066 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:36:27,067 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:36:27,072 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:36:27,430 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:36:28,142 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:36:28,142 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:36:28,142 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:36:29,350 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:36:29,433 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:36:34,350 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:36:36,040 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:36:36,041 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:36:36,042 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:36:36,047 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:36:36,437 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:36:37,440 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:36:38,246 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 15:36:40,248 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:36:40,527 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:36:40,527 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:36:40,528 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:36:40,532 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:36:41,443 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:36:43,142 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:36:43,142 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:36:43,143 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:36:43,446 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:36:45,131 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:36:45,131 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:36:45,132 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:36:45,136 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:36:45,447 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:36:46,173 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:36:47,451 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:36:51,174 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:36:53,687 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:36:53,688 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:36:53,688 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:36:53,692 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:36:54,455 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:36:55,458 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:36:56,693 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:36:58,142 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:36:58,142 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:36:58,143 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:36:59,378 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:36:59,379 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:36:59,379 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:36:59,383 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:36:59,460 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:37:01,463 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:37:02,421 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:37:07,422 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:37:08,009 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:37:08,009 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:37:08,010 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:37:08,015 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:37:08,242 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 15:37:08,469 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:37:09,472 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:37:12,742 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:37:12,743 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:37:12,743 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:37:12,744 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:37:12,748 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:37:13,142 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:37:13,142 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:37:13,143 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:37:13,475 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:37:15,478 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:37:17,425 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:37:17,426 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:37:17,426 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:37:17,431 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:37:17,479 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:37:18,469 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:37:19,483 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:37:22,183 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:37:22,184 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:37:22,184 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:37:22,189 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:37:22,484 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:37:23,487 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:37:24,190 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:37:28,142 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:37:28,142 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:37:28,143 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:37:28,475 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:37:28,476 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:37:28,476 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:37:28,481 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:37:28,490 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:37:29,490 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:37:29,517 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:37:33,126 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:37:33,127 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:37:33,128 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:37:33,133 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:37:33,493 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:37:35,133 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:37:35,494 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:37:37,806 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:37:37,807 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:37:37,807 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:37:37,812 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:37:38,247 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 15:37:38,496 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:37:39,498 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:37:40,248 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:37:42,115 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:37:42,116 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:37:42,116 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:37:42,121 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:37:42,501 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:37:43,142 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:37:43,143 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:37:43,143 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:37:43,503 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:37:45,571 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:37:48,270 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:37:48,270 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:37:48,271 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:37:48,275 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:37:48,507 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:37:49,509 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:37:51,313 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:37:54,233 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:37:54,234 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:37:54,234 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:37:54,240 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:37:54,512 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:37:55,515 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:37:57,240 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:37:58,142 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:37:58,143 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:37:58,143 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:37:59,673 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:37:59,674 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:37:59,674 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:37:59,679 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:38:00,518 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:38:01,520 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:38:02,679 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:38:05,312 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:38:05,312 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:38:05,313 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:38:05,317 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:38:05,524 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:38:07,527 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:38:08,246 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 15:38:08,248 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:38:13,143 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:38:13,143 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:38:13,143 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:38:13,381 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:38:13,663 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:38:13,664 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:38:13,664 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:38:13,669 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:38:14,532 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:38:15,534 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:38:17,886 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:38:17,887 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:38:17,887 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:38:17,892 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:38:18,537 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:38:18,893 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:38:19,539 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:38:23,893 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:38:25,773 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:38:25,774 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:38:25,774 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:38:25,780 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:38:26,543 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:38:27,546 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:38:28,143 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:38:28,143 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:38:28,143 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:38:29,405 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:38:30,093 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:38:30,094 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:38:30,094 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:38:30,099 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:38:30,548 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:38:31,550 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:38:35,137 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:38:36,222 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:38:36,223 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:38:36,223 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:38:36,228 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:38:36,553 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:38:37,556 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:38:38,248 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 15:38:40,250 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:38:41,694 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:38:41,695 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:38:41,695 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:38:41,700 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:38:42,560 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:38:43,143 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:38:43,143 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:38:43,143 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:38:43,562 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:38:45,417 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:38:46,154 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:38:46,155 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:38:46,155 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:38:46,160 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:38:46,564 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:38:47,567 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:38:49,948 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:38:49,949 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:38:49,949 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:38:49,954 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:38:50,569 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:38:50,955 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:38:51,571 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:38:55,009 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:38:55,009 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:38:55,010 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:38:55,015 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:38:55,574 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:38:56,015 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:38:57,577 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:38:58,143 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:38:58,143 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:38:58,143 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:39:01,344 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:39:03,512 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:39:03,513 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:39:03,513 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:39:03,519 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:39:03,581 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:39:05,585 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:39:06,553 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:39:08,254 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 15:39:08,566 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:39:08,567 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:39:08,567 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:39:08,572 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:39:08,587 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:39:09,590 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:39:11,573 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:39:13,143 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:39:13,143 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:39:13,143 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:39:13,244 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:39:13,503 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:39:13,503 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:39:13,507 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:39:13,593 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:39:15,596 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:39:17,509 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:39:17,938 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:39:17,939 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:39:17,939 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:39:17,944 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:39:18,599 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:39:19,601 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:39:22,945 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:39:24,929 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:39:24,929 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:39:24,930 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:39:24,935 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:39:25,605 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:39:27,608 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:39:28,143 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:39:28,143 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:39:28,143 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:39:28,364 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:39:30,489 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:39:30,490 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:39:30,490 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:39:30,495 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:39:30,610 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:39:31,613 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:39:33,533 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:39:36,960 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:39:36,961 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:39:36,961 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:39:36,966 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:39:37,616 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:39:38,253 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 15:39:39,254 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:39:39,620 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:39:43,143 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:39:43,143 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:39:43,144 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:39:43,512 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:39:43,513 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:39:43,513 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:39:43,518 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:39:43,622 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:39:44,553 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:39:45,626 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:39:49,554 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:39:49,730 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:39:49,731 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:39:49,731 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:39:49,736 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:39:50,629 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:39:51,632 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:39:54,737 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:39:54,866 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:39:54,867 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:39:54,868 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:39:54,872 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:39:55,634 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:39:57,637 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:39:58,143 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:39:58,143 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:39:58,144 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:39:59,205 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:39:59,206 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:39:59,206 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:39:59,211 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:39:59,639 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:40:00,249 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:40:01,642 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:40:04,574 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:40:04,574 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:40:04,575 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:40:04,580 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:40:04,645 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:40:05,580 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:40:05,646 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:40:08,258 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 15:40:11,260 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:40:11,794 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:40:11,795 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:40:11,795 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:40:11,800 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:40:12,651 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:40:13,143 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:40:13,143 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:40:13,144 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:40:13,653 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:40:16,403 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:40:18,311 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:40:18,312 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:40:18,312 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:40:18,317 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:40:18,657 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:40:19,659 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:40:22,354 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:40:24,644 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:40:24,645 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:40:24,646 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:40:24,651 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:40:24,662 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:40:25,665 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:40:27,651 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:40:28,143 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:40:28,143 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:40:28,144 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:40:31,487 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:40:31,487 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:40:31,487 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:40:31,492 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:40:31,669 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:40:33,529 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:40:33,672 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:40:36,140 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:40:36,141 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:40:36,141 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:40:36,146 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:40:36,674 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:40:37,676 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:40:38,258 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 15:40:39,259 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:40:42,070 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:40:42,071 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:40:42,071 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:40:42,076 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:40:42,680 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:40:43,143 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:40:43,144 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:40:43,144 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:40:43,682 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:40:44,418 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:40:49,418 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:40:50,520 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:40:50,521 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:40:50,521 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:40:50,526 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:40:50,687 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:40:51,689 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:40:54,561 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:40:56,840 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:40:56,841 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:40:56,841 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:40:56,847 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:40:57,693 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:40:58,144 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:40:58,144 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:40:58,144 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:40:59,696 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:41:00,385 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:41:05,385 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:41:05,500 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:41:05,501 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:41:05,501 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:41:05,507 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:41:05,701 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:41:07,704 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:41:08,260 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 15:41:11,261 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:41:13,144 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:41:13,144 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:41:13,144 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:41:16,437 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:41:16,943 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:41:16,943 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:41:16,943 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:41:16,948 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:41:17,711 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:41:19,714 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:41:21,985 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:41:23,549 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:41:23,550 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:41:23,550 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:41:23,555 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:41:23,716 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:41:25,720 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:41:27,556 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:41:28,144 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:41:28,144 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:41:28,144 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:41:30,737 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:41:30,738 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:41:30,738 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:41:30,743 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:41:31,725 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:41:31,726 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:41:32,784 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:41:35,202 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:41:35,203 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:41:35,203 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:41:35,208 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:41:35,728 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:41:37,731 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:41:38,209 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:41:38,262 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 15:41:42,970 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:41:42,971 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:41:42,971 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:41:42,976 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:41:43,144 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:41:43,144 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:41:43,144 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:41:43,428 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:41:43,735 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:41:45,739 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:41:47,441 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:41:47,441 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:41:47,442 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:41:47,447 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:41:47,740 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:41:48,485 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:41:49,743 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:41:51,938 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:41:51,939 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:41:51,939 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:41:51,944 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:41:52,745 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:41:53,748 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:41:53,945 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:41:58,144 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:41:58,144 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:41:58,144 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:41:59,364 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:41:59,941 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:41:59,941 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:41:59,942 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:41:59,946 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:42:00,752 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:42:01,755 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:42:04,557 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:42:04,558 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:42:04,558 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:42:04,559 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:42:04,563 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:42:04,758 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:42:07,762 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:42:08,264 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 15:42:09,455 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:42:09,456 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:42:09,456 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:42:09,461 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:42:09,763 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:42:10,462 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:42:11,768 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:42:13,144 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:42:13,144 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:42:13,144 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:42:15,383 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:42:15,384 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:42:15,384 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:42:15,389 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:42:15,770 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:42:16,425 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:42:17,773 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:42:20,138 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:42:20,139 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:42:20,139 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:42:20,144 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:42:20,776 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:42:21,778 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:42:22,145 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:42:26,624 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:42:26,625 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:42:26,625 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:42:26,630 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:42:26,781 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:42:27,631 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:42:28,144 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:42:28,144 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:42:28,144 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:42:29,785 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:42:31,096 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:42:31,097 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:42:31,097 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:42:31,102 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:42:31,786 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:42:33,138 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:42:33,790 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:42:36,131 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:42:36,131 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:42:36,132 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:42:36,137 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:42:36,791 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:42:37,792 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:42:38,270 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 15:42:38,271 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:42:41,197 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:42:41,198 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:42:41,198 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:42:41,203 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:42:41,795 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:42:43,144 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:42:43,144 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:42:43,145 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:42:43,369 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:42:43,798 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:42:46,636 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:42:46,637 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:42:46,637 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:42:46,642 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:42:46,800 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:42:48,643 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:42:49,804 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:42:51,631 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:42:51,631 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:42:51,632 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:42:51,636 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:42:51,805 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:42:53,807 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:42:54,637 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:42:58,144 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:42:58,144 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:42:58,145 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:42:59,258 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:42:59,258 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:42:59,259 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:42:59,264 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:42:59,810 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:43:00,264 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:43:01,813 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:43:04,345 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:43:04,346 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:43:04,346 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:43:04,353 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:43:04,816 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:43:05,354 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:43:05,819 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:43:08,269 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 15:43:10,834 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:43:10,834 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:43:10,835 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:43:10,835 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:43:10,839 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:43:11,823 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:43:13,144 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:43:13,144 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:43:13,145 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:43:13,827 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:43:15,342 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:43:15,343 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:43:15,343 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:43:15,348 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:43:15,828 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:43:16,385 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:43:17,831 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:43:21,137 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:43:21,138 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:43:21,139 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:43:21,143 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:43:21,834 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:43:22,144 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:43:23,837 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:43:27,144 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:43:28,144 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:43:28,145 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:43:28,145 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:43:28,296 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:43:28,349 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:43:28,350 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:43:28,354 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:43:28,840 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:43:29,842 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:43:32,355 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:43:32,567 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:43:32,568 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:43:32,568 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:43:32,572 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:43:32,844 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:43:35,848 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:43:37,610 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:43:38,274 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 15:43:39,564 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:43:39,565 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:43:39,565 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:43:39,570 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:43:39,851 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:43:41,854 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:43:43,144 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:43:43,145 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:43:43,145 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:43:43,416 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:43:44,552 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:43:44,553 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:43:44,553 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:43:44,559 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:43:44,857 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:43:47,861 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:43:48,593 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:43:49,106 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:43:49,107 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:43:49,107 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:43:49,112 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:43:49,862 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:43:51,866 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:43:53,569 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:43:53,569 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:43:53,570 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:43:53,575 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:43:53,867 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:43:54,575 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:43:55,870 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:43:57,710 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:43:57,711 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:43:57,711 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:43:57,716 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:43:57,871 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:43:58,145 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:43:58,145 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:43:58,145 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:43:59,874 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:44:00,430 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:44:02,614 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:44:02,615 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:44:02,615 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:44:02,620 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:44:02,876 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:44:05,657 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:44:05,881 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:44:07,455 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:44:07,456 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:44:07,457 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:44:07,462 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:44:07,883 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:44:08,273 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 15:44:09,886 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:44:11,274 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:44:12,415 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:44:12,416 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:44:12,417 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:44:12,422 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:44:12,888 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:44:13,145 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:44:13,145 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:44:13,145 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:44:13,891 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:44:16,442 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:44:16,987 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:44:16,987 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:44:16,988 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:44:16,993 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:44:17,894 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:44:19,897 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:44:21,973 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:44:21,974 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:44:21,974 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:44:21,975 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:44:21,979 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:44:22,899 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:44:23,901 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:44:26,980 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:44:28,145 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:44:28,145 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:44:28,145 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:44:30,348 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:44:30,349 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:44:30,349 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:44:30,353 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:44:30,906 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:44:31,908 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:44:32,389 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:44:34,841 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:44:34,842 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:44:34,843 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:44:34,847 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:44:34,910 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:44:37,848 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:44:37,914 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:44:38,275 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 15:44:40,610 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:44:40,611 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:44:40,611 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:44:40,616 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:44:40,916 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:44:43,145 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:44:43,145 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:44:43,145 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:44:43,374 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:44:43,921 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:44:47,239 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:44:47,240 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:44:47,240 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:44:47,244 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:44:47,924 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:44:49,281 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:44:49,927 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:44:52,314 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:44:52,315 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:44:52,316 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:44:52,320 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:44:52,929 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:44:53,931 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:44:54,321 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:44:57,017 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:44:57,018 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:44:57,018 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:44:57,023 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:44:57,934 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:44:58,145 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:44:58,145 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:44:58,146 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:44:59,356 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:44:59,937 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:45:01,711 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:45:01,712 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:45:01,713 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:45:01,717 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:45:01,939 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:45:03,940 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:45:04,753 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:45:07,124 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:45:07,125 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:45:07,126 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:45:07,130 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:45:07,946 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:45:08,281 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 15:45:09,947 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:45:10,282 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:45:11,992 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:45:11,993 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:45:11,993 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:45:11,998 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:45:12,950 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:45:13,145 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:45:13,145 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:45:13,146 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:45:13,953 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:45:15,414 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:45:16,481 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:45:16,482 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:45:16,482 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:45:16,487 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:45:16,955 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:45:17,958 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:45:20,525 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:45:24,643 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:45:24,644 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:45:24,644 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:45:24,649 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:45:24,962 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:45:25,649 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:45:27,966 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:45:28,145 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:45:28,145 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:45:28,145 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:45:30,838 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:45:30,838 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:45:30,838 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:45:30,839 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:45:30,843 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:45:30,968 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:45:33,972 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:45:35,736 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:45:35,737 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:45:35,737 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:45:35,742 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:45:35,973 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:45:36,742 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:45:37,976 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:45:38,280 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 15:45:42,281 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:45:43,145 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:45:43,145 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:45:43,145 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:45:43,786 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:45:43,787 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:45:43,787 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:45:43,792 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:45:43,981 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:45:45,985 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:45:47,829 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:45:48,865 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:45:48,866 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:45:48,866 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:45:48,871 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:45:48,987 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:45:51,990 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:45:52,871 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:45:57,321 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:45:57,322 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:45:57,323 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:45:57,328 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:45:57,994 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:45:58,145 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:45:58,145 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:45:58,146 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:45:58,396 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:45:59,997 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:46:02,945 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:46:02,946 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:46:02,947 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:46:02,952 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:46:02,999 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:46:03,989 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:46:06,004 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:46:08,287 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 15:46:08,972 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:46:08,973 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:46:08,973 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:46:08,979 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:46:09,006 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:46:09,979 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:46:12,010 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:46:13,145 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:46:13,145 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:46:13,146 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:46:13,390 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:46:13,424 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:46:13,424 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:46:13,429 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:46:14,012 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:46:15,429 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:46:16,015 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:46:19,321 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:46:19,322 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:46:19,322 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:46:19,327 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:46:20,018 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:46:21,365 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:46:22,021 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:46:24,888 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:46:24,889 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:46:24,889 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:46:24,895 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:46:25,023 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:46:26,896 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:46:28,026 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:46:28,145 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:46:28,145 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:46:28,146 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:46:30,511 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:46:30,512 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:46:30,512 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:46:30,517 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:46:31,028 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:46:32,031 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:46:32,553 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:46:35,733 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:46:35,734 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:46:35,734 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:46:35,739 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:46:36,033 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:46:37,740 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:46:38,034 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:46:38,284 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 15:46:41,192 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:46:41,193 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:46:41,193 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:46:41,198 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:46:42,037 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:46:43,145 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:46:43,146 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:46:43,146 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:46:43,447 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:46:44,040 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:46:46,380 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:46:46,381 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:46:46,381 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:46:46,386 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:46:47,043 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:46:48,046 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:46:49,421 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:46:52,442 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:46:52,443 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:46:52,443 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:46:52,448 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:46:53,049 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:46:54,051 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:46:54,449 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:46:56,943 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:46:56,944 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:46:56,944 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:46:56,949 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:46:57,053 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:46:58,146 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:46:58,146 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:46:58,147 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:47:00,057 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:47:00,340 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:47:01,167 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:47:01,167 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:47:01,168 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:47:01,173 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:47:02,058 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:47:04,062 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:47:05,400 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:47:05,401 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:47:05,401 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:47:05,401 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:47:05,406 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:47:06,065 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:47:08,070 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:47:08,286 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 15:47:11,288 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:47:13,145 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:47:13,146 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:47:13,146 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:47:14,641 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:47:14,642 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:47:14,642 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:47:14,647 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:47:15,075 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:47:16,078 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:47:16,685 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:47:19,783 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:47:19,783 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:47:19,784 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:47:19,789 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:47:20,080 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:47:21,790 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:47:22,085 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:47:24,309 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:47:24,313 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:47:24,313 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:47:24,319 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:47:25,087 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:47:26,090 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:47:27,319 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:47:28,146 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:47:28,146 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:47:28,146 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:47:30,212 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:47:30,212 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:47:30,213 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:47:30,218 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:47:31,093 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:47:32,096 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:47:33,253 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:47:35,986 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:47:35,987 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:47:35,988 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:47:35,993 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:47:36,098 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:47:38,102 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:47:38,288 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 15:47:38,290 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:47:40,502 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:47:40,503 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:47:40,503 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:47:40,508 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:47:41,104 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:47:42,107 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:47:43,146 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:47:43,146 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:47:43,146 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:47:43,434 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:47:45,478 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:47:45,479 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:47:45,479 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:47:45,484 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:47:46,110 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:47:48,113 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:47:48,521 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:47:51,440 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:47:51,441 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:47:51,441 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:47:51,446 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:47:52,115 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:47:54,119 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:47:54,447 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:47:58,146 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:47:58,146 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:47:58,146 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:48:00,357 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:48:00,821 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:48:00,821 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:48:00,822 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:48:00,826 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:48:01,123 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:48:04,127 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:48:05,548 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:48:05,549 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:48:05,549 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:48:05,549 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:48:05,554 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:48:06,129 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:48:08,133 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:48:08,291 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 15:48:10,288 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:48:10,289 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:48:10,289 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:48:10,294 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:48:11,135 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:48:11,295 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:48:12,138 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:48:13,146 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:48:13,146 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:48:13,146 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:48:14,913 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:48:14,914 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:48:14,914 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:48:14,919 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:48:15,140 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:48:16,957 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:48:18,144 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:48:21,700 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:48:21,700 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:48:21,701 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:48:21,706 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:48:22,146 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:48:22,706 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:48:24,149 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:48:26,518 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:48:26,519 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:48:26,519 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:48:26,524 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:48:27,151 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:48:28,146 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:48:28,146 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:48:28,146 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:48:28,154 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:48:28,363 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:48:31,436 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:48:31,437 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:48:31,437 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:48:31,442 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:48:32,156 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:48:33,477 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:48:34,160 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:48:36,306 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:48:36,307 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:48:36,308 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:48:36,312 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:48:37,162 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:48:38,165 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:48:38,296 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 15:48:39,297 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:48:42,403 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:48:42,404 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:48:42,404 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:48:42,409 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:48:43,147 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:48:43,147 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:48:43,148 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:48:43,168 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:48:44,171 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:48:44,457 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:48:47,214 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:48:47,215 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:48:47,215 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:48:47,220 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:48:48,174 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:48:50,175 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:48:50,257 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:48:54,500 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:48:54,501 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:48:54,501 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:48:54,505 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:48:55,178 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:48:55,506 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:48:56,178 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:48:58,146 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:48:58,146 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:48:58,147 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:48:59,336 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:48:59,336 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:48:59,336 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:48:59,341 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:49:00,181 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:49:01,377 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:49:02,184 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:49:03,896 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:49:03,897 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:49:03,898 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:49:03,903 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:49:04,186 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:49:06,191 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:49:06,903 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:49:08,295 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 15:49:08,677 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:49:08,677 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:49:08,677 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:49:08,682 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:49:09,193 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:49:10,196 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:49:12,683 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:49:13,146 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:49:13,146 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:49:13,146 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:49:16,114 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:49:16,114 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:49:16,115 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:49:16,120 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:49:16,200 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:49:18,157 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:49:18,201 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:49:21,284 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:49:21,285 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:49:21,285 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:49:21,290 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:49:22,204 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:49:23,290 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:49:24,207 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:49:26,366 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:49:26,366 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:49:26,366 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:49:26,372 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:49:27,209 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:49:28,146 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:49:28,146 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:49:28,147 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:49:28,212 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:49:28,366 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:49:31,238 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:49:31,238 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:49:31,238 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:49:31,243 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:49:32,214 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:49:34,217 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:49:34,281 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:49:35,325 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:49:35,326 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:49:35,326 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:49:35,331 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:49:36,219 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:49:38,222 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:49:38,300 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 15:49:39,301 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:49:40,634 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:49:40,635 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:49:40,635 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:49:40,640 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:49:41,224 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:49:42,227 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:49:43,146 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:49:43,146 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:49:43,147 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:49:44,406 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:49:46,777 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:49:46,778 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:49:46,778 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:49:46,783 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:49:47,230 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:49:48,233 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:49:49,821 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:49:51,246 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:49:51,247 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:49:51,247 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:49:51,252 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:49:52,235 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:49:54,238 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:49:55,252 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:49:57,182 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:49:57,183 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:49:57,183 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:49:57,188 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:49:57,240 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:49:58,146 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:49:58,147 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:49:58,147 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:50:00,244 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:50:00,534 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:50:02,272 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:50:02,273 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:50:02,273 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:50:02,278 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:50:03,246 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:50:04,249 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:50:06,313 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:50:08,299 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 15:50:10,297 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:50:10,298 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:50:10,298 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:50:10,303 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:50:11,256 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:50:12,257 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:50:12,304 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:50:13,146 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:50:13,147 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:50:13,147 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:50:15,666 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:50:15,667 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:50:15,667 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:50:15,672 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:50:16,259 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:50:17,673 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:50:18,262 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:50:22,510 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:50:22,511 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:50:22,511 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:50:22,516 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:50:23,266 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:50:23,516 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:50:24,268 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:50:26,737 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:50:26,738 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:50:26,739 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:50:26,743 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:50:27,270 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:50:28,147 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:50:28,147 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:50:28,147 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:50:28,271 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:50:29,391 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:50:30,967 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:50:30,968 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:50:30,968 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:50:30,973 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:50:31,273 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:50:34,277 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:50:35,009 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:50:35,346 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:50:35,347 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:50:35,347 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:50:35,352 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:50:36,278 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:50:38,281 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:50:38,301 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 15:50:40,302 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:50:43,147 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:50:43,147 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:50:43,147 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:50:43,784 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:50:43,785 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:50:43,785 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:50:43,790 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:50:44,286 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:50:45,791 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:50:46,287 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:50:49,664 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:50:49,665 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:50:49,666 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:50:49,670 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:50:50,290 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:50:51,671 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:50:52,293 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:50:56,671 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:50:58,110 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:50:58,111 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:50:58,111 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:50:58,117 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:50:58,147 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:50:58,147 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:50:58,147 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:50:58,297 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:51:00,300 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:51:02,422 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:51:02,614 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:51:02,614 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:51:02,614 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:51:02,620 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:51:03,302 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:51:04,304 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:51:07,658 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:51:08,307 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 15:51:08,324 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:51:08,325 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:51:08,325 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:51:08,331 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:51:09,310 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:51:10,312 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:51:13,077 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:51:13,078 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:51:13,078 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:51:13,078 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:51:13,083 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:51:13,147 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:51:13,147 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:51:13,147 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:51:13,315 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:51:16,319 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:51:18,352 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:51:18,619 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:51:18,620 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:51:18,620 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:51:18,625 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:51:19,321 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:51:20,323 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:51:23,662 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:51:23,725 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:51:23,726 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:51:23,726 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:51:23,732 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:51:24,326 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:51:26,329 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:51:27,968 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:51:27,969 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:51:27,969 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:51:27,974 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:51:28,147 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:51:28,147 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:51:28,148 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:51:28,330 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:51:29,409 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:51:30,334 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:51:33,009 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:51:33,010 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:51:33,010 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:51:33,014 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:51:33,336 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:51:35,053 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:51:36,340 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:51:38,306 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 15:51:40,043 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:51:40,044 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:51:40,044 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:51:40,049 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:51:40,342 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:51:41,049 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:51:42,346 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:51:43,147 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:51:43,147 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:51:43,147 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:51:44,884 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:51:44,885 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:51:44,885 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:51:44,890 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:51:45,348 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:51:46,351 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:51:46,890 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:51:51,883 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:51:51,884 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:51:51,884 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:51:51,889 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:51:52,355 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:51:52,889 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:51:54,358 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:51:57,890 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:51:58,147 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:51:58,147 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:51:58,147 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:52:00,222 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:52:00,223 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:52:00,223 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:52:00,228 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:52:00,361 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:52:02,365 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:52:03,229 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:52:08,229 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:52:08,311 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 15:52:13,147 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:52:13,147 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:52:13,148 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:52:13,369 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:52:16,123 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:52:16,123 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:52:16,124 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:52:16,129 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:52:16,375 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:52:18,379 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:52:19,165 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:52:21,129 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:52:21,130 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:52:21,130 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:52:21,135 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:52:21,381 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:52:24,383 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:52:25,136 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:52:25,780 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:52:25,781 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:52:25,782 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:52:25,786 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:52:26,384 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:52:28,147 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:52:28,147 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:52:28,148 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:52:28,388 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:52:30,414 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:52:31,489 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:52:31,490 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:52:31,490 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:52:31,495 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:52:32,390 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:52:34,394 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:52:35,533 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:52:38,310 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 15:52:39,286 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:52:39,287 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:52:39,287 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:52:39,292 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:52:39,397 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:52:41,293 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:52:42,401 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:52:43,147 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:52:43,147 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:52:43,148 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:52:44,431 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:52:44,431 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:52:44,432 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:52:44,437 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:52:45,403 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:52:46,406 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:52:46,437 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:52:49,378 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:52:49,379 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:52:49,379 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:52:49,385 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:52:49,408 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:52:52,385 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:52:52,412 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:52:53,715 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:52:53,716 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:52:53,716 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:52:53,721 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:52:54,413 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:52:56,416 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:52:57,721 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:52:58,013 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:52:58,014 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:52:58,014 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:52:58,019 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:52:58,147 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:52:58,148 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:52:58,148 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:52:58,417 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:53:00,421 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:53:02,594 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:53:02,595 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:53:02,596 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:53:02,601 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:53:03,423 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:53:03,602 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:53:04,425 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:53:08,082 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:53:08,083 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:53:08,083 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:53:08,088 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:53:08,312 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 15:53:08,429 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:53:09,314 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:53:10,432 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:53:13,118 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:53:13,118 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:53:13,119 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:53:13,123 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:53:13,148 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:53:13,148 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:53:13,148 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:53:13,434 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:53:14,352 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:53:14,437 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:53:17,863 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:53:17,864 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:53:17,865 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:53:17,870 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:53:18,440 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:53:19,870 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:53:20,443 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:53:23,440 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:53:23,441 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:53:23,441 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:53:23,446 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:53:23,447 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:53:25,447 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:53:26,450 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:53:28,148 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:53:28,148 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:53:28,148 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:53:29,860 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:53:29,861 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:53:29,861 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:53:29,866 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:53:30,453 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:53:30,901 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:53:32,456 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:53:34,897 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:53:34,898 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:53:34,898 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:53:34,903 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:53:35,459 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:53:35,904 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:53:36,464 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:53:38,315 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 15:53:39,046 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:53:39,046 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:53:39,047 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:53:39,052 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:53:39,466 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:53:40,468 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:53:41,052 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:53:43,148 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:53:43,148 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:53:43,148 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:53:44,396 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:53:44,396 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:53:44,397 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:53:44,402 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:53:44,471 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:53:46,437 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:53:46,473 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:53:51,438 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:53:55,253 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:53:55,254 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:53:55,254 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:53:55,259 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:53:55,479 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:53:57,260 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:53:58,148 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:53:58,148 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:53:58,148 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:53:58,483 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:54:00,283 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:54:00,284 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:54:00,284 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:54:00,290 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:54:00,484 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:54:02,325 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:54:02,487 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:54:07,326 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:54:07,352 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:54:07,353 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:54:07,353 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:54:07,359 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:54:07,492 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:54:08,317 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 15:54:10,496 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:54:13,139 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:54:13,139 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:54:13,140 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:54:13,140 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:54:13,145 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:54:13,148 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:54:13,148 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:54:13,149 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:54:13,499 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:54:14,501 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:54:17,677 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:54:17,677 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:54:17,678 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:54:17,683 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:54:18,504 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:54:18,721 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:54:20,507 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:54:22,322 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:54:22,323 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:54:22,323 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:54:22,328 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:54:22,509 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:54:24,329 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:54:24,512 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:54:27,694 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:54:27,695 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:54:27,695 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:54:27,701 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:54:28,148 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:54:28,148 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:54:28,149 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:54:28,514 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:54:29,444 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:54:30,518 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:54:33,248 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:54:33,249 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:54:33,249 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:54:33,254 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:54:33,519 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:54:35,289 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:54:36,523 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:54:38,322 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 15:54:40,324 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:54:40,672 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:54:40,673 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:54:40,674 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:54:40,678 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:54:41,526 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:54:42,529 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:54:43,148 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:54:43,148 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:54:43,148 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:54:45,428 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:54:46,356 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:54:46,357 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:54:46,358 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:54:46,363 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:54:46,532 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:54:48,535 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:54:50,848 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:54:50,849 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:54:50,849 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:54:50,849 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:54:50,855 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:54:51,537 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:54:52,540 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:54:54,780 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:54:54,781 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:54:54,781 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:54:54,786 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:54:55,542 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:54:56,544 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:54:56,787 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:54:58,148 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:54:58,148 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:54:58,148 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:54:59,456 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:54:59,457 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:54:59,457 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:54:59,462 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:54:59,546 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:55:02,463 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:55:02,548 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:55:05,020 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:55:05,020 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:55:05,021 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:55:05,026 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:55:05,551 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:55:06,554 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:55:08,027 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:55:08,321 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 15:55:09,134 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:55:09,135 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:55:09,136 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:55:09,142 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:55:09,556 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:55:10,559 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:55:13,142 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:55:13,148 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:55:13,148 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:55:13,148 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:55:14,436 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:55:14,436 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:55:14,436 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:55:14,442 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:55:14,562 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:55:16,565 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:55:18,477 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:55:21,450 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:55:21,450 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:55:21,451 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:55:21,456 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:55:21,569 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:55:24,457 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:55:24,572 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:55:26,974 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:55:26,975 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:55:26,975 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:55:26,980 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:55:27,574 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:55:28,148 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:55:28,148 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:55:28,149 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:55:28,577 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:55:30,358 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:55:31,227 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:55:31,228 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:55:31,228 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:55:31,233 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:55:31,579 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:55:32,579 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:55:36,270 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:55:37,211 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:55:37,212 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:55:37,212 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:55:37,217 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:55:37,582 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:55:38,330 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 15:55:38,583 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:55:41,332 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:55:41,579 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:55:41,580 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:55:41,580 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:55:41,587 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:55:41,587 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:55:43,148 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:55:43,148 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:55:43,149 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:55:44,591 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:55:46,492 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:55:46,689 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:55:46,690 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:55:46,690 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:55:46,695 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:55:47,593 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:55:48,596 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:55:51,253 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:55:51,254 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:55:51,254 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:55:51,259 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:55:51,598 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:55:52,259 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:55:52,601 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:55:55,742 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:55:55,743 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:55:55,743 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:55:55,749 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:55:56,603 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:55:57,749 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:55:58,148 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:55:58,148 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:55:58,149 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:55:58,606 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:56:00,432 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:56:00,433 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:56:00,433 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:56:00,438 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:56:00,608 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:56:02,611 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:56:03,439 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:56:06,130 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:56:06,130 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:56:06,131 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:56:06,136 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:56:06,615 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:56:08,325 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 15:56:08,619 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:56:09,327 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:56:10,939 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:56:10,940 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:56:10,940 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:56:10,945 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:56:11,621 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:56:12,623 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:56:13,148 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:56:13,149 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:56:13,149 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:56:14,408 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:56:14,892 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:56:14,893 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:56:14,893 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:56:14,898 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:56:15,625 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:56:16,628 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:56:19,934 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:56:21,091 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:56:21,092 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:56:21,092 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:56:21,097 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:56:21,631 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:56:22,634 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:56:25,098 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:56:27,792 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:56:27,793 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:56:27,794 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:56:27,799 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:56:28,149 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:56:28,149 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:56:28,149 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:56:28,638 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:56:30,405 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:56:30,641 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:56:34,777 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:56:34,778 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:56:34,779 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:56:34,784 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:56:35,644 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:56:35,821 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:56:36,647 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:56:38,328 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 15:56:41,329 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:56:43,149 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:56:43,149 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:56:43,149 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:56:43,744 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:56:43,745 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:56:43,745 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:56:43,751 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:56:44,652 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:56:46,654 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:56:46,752 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:56:47,825 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:56:47,825 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:56:47,826 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:56:47,831 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:56:48,655 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:56:50,659 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:56:51,832 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:56:52,567 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:56:52,568 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:56:52,568 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:56:52,573 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:56:52,660 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:56:54,663 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:56:56,416 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:56:56,418 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:56:56,418 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:56:56,424 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:56:56,665 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:56:57,424 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:56:58,149 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:56:58,149 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:56:58,149 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:56:58,668 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:57:01,780 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:57:01,780 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:57:01,781 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:57:01,786 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:57:02,671 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:57:02,787 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:57:04,674 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:57:06,220 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:57:06,220 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:57:06,221 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:57:06,226 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:57:06,677 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:57:08,226 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:57:08,333 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 15:57:08,681 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:57:10,726 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:57:10,726 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:57:10,727 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:57:10,732 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:57:11,683 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:57:12,686 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:57:13,149 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:57:13,149 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:57:13,149 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:57:13,458 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:57:15,127 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:57:15,128 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:57:15,128 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:57:15,134 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:57:15,688 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:57:16,691 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:57:19,170 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:57:19,732 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:57:19,733 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:57:19,733 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:57:19,740 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:57:20,693 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:57:22,697 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:57:24,741 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:57:25,566 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:57:25,567 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:57:25,567 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:57:25,572 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:57:25,698 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:57:28,149 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:57:28,149 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:57:28,149 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:57:28,702 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:57:30,375 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:57:31,419 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:57:31,420 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:57:31,420 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:57:31,425 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:57:31,704 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:57:34,708 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:57:35,462 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:57:35,756 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:57:35,757 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:57:35,757 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:57:35,762 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:57:36,710 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:57:38,332 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 15:57:38,713 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:57:40,404 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:57:40,405 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:57:40,405 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:57:40,410 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:57:40,714 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:57:41,411 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:57:42,725 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:57:43,149 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:57:43,150 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:57:43,150 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:57:45,049 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:57:45,050 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:57:45,050 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:57:45,055 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:57:45,727 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:57:46,728 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:57:47,093 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:57:50,293 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:57:50,294 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:57:50,294 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:57:50,299 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:57:50,730 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:57:52,300 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:57:52,732 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:57:54,786 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:57:54,787 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:57:54,787 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:57:54,792 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:57:55,734 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:57:56,737 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:57:57,793 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:57:58,149 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:57:58,149 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:57:58,149 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:57:59,803 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:57:59,804 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:57:59,804 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:57:59,809 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:58:00,739 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:58:02,742 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:58:02,810 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:58:05,088 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:58:05,089 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:58:05,089 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:58:05,094 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:58:05,746 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:58:06,749 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:58:08,095 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:58:08,337 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 15:58:10,834 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:58:10,835 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:58:10,836 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:58:10,841 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:58:11,753 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:58:12,756 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:58:13,149 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:58:13,149 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:58:13,149 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:58:13,447 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:58:18,304 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:58:18,304 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:58:18,305 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:58:18,310 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:58:18,760 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:58:19,311 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:58:20,765 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:58:23,300 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:58:23,301 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:58:23,302 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:58:23,307 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:58:23,767 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:58:24,770 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:58:25,307 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:58:28,149 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:58:28,149 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:58:28,150 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:58:30,388 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:58:30,402 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:58:30,403 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:58:30,403 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:58:30,408 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:58:30,774 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:58:32,777 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:58:35,409 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:58:36,347 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:58:36,347 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:58:36,348 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:58:36,353 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:58:36,779 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:58:38,337 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 15:58:38,783 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:58:41,339 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:58:43,149 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:58:43,149 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:58:43,150 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:58:44,175 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:58:44,176 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:58:44,176 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:58:44,181 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:58:44,787 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:58:46,791 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:58:47,182 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:58:52,182 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:58:52,452 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:58:52,453 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:58:52,453 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:58:52,460 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:58:52,795 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:58:54,798 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:58:57,311 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:58:57,312 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:58:57,313 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:58:57,313 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:58:57,318 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:58:57,800 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:58:58,149 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:58:58,149 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:58:58,150 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:58:58,803 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:59:02,404 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:59:03,540 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:59:03,541 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:59:03,541 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:59:03,546 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:59:03,806 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:59:04,808 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:59:07,547 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:59:08,339 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 15:59:10,494 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:59:10,495 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:59:10,496 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:59:10,500 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:59:10,814 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:59:12,816 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:59:13,149 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:59:13,150 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:59:13,150 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:59:13,355 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:59:15,799 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:59:15,800 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:59:15,800 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:59:15,806 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:59:15,818 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:59:18,806 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:59:18,820 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:59:21,245 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:59:21,246 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:59:21,247 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:59:21,251 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:59:21,822 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:59:22,825 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:59:24,252 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:59:25,129 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:59:25,130 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:59:25,131 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:59:25,136 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:59:25,827 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:59:26,830 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:59:28,149 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:59:28,150 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:59:28,150 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:59:29,387 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:59:30,249 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:59:30,250 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:59:30,250 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:59:30,255 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:59:30,833 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:59:32,836 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:59:34,613 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:59:34,614 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:59:34,615 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:59:34,615 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:59:34,620 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:59:34,837 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:59:36,841 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:59:38,341 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 15:59:38,985 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:59:38,985 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:59:38,986 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:59:38,991 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:59:39,843 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:59:39,991 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:59:40,845 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:59:43,150 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:59:43,150 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:59:43,150 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:59:43,486 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:59:43,487 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:59:43,488 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:59:43,493 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:59:43,848 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:59:44,850 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:59:45,494 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:59:48,848 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:59:48,849 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:59:48,849 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:59:48,855 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:59:48,855 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:59:50,856 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:59:50,859 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:59:54,889 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:59:54,890 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:59:54,891 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:59:54,896 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:59:55,865 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 15:59:55,896 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:59:56,868 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 15:59:58,150 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:59:58,150 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:59:58,150 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:59:59,446 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:59:59,447 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 15:59:59,447 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:59:59,452 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:59:59,870 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:00:00,870 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:00:01,453 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:00:04,567 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:00:04,568 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:00:04,568 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:00:04,573 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:00:04,873 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:00:06,574 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:00:06,876 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:00:08,343 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 16:00:10,989 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:00:10,990 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:00:10,991 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:00:10,995 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:00:11,881 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:00:11,996 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:00:12,884 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:00:13,150 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:00:13,150 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:00:13,150 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:00:16,114 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:00:16,115 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:00:16,115 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:00:16,120 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:00:16,886 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:00:17,120 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:00:18,890 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:00:21,609 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:00:21,610 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:00:21,611 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:00:21,616 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:00:21,892 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:00:22,616 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:00:22,895 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:00:26,993 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:00:26,994 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:00:26,994 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:00:27,000 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:00:27,898 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:00:28,000 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:00:28,150 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:00:28,150 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:00:28,150 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:00:28,905 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:00:31,927 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:00:31,929 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:00:31,929 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:00:31,934 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:00:32,907 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:00:33,935 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:00:34,911 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:00:36,432 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:00:36,433 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:00:36,433 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:00:36,438 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:00:36,912 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:00:38,346 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 16:00:38,913 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:00:39,347 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:00:43,150 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:00:43,150 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:00:43,150 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:00:44,381 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:00:45,078 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:00:45,079 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:00:45,079 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:00:45,084 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:00:45,918 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:00:46,921 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:00:50,085 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:00:50,880 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:00:50,881 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:00:50,881 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:00:50,887 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:00:50,923 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:00:52,927 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:00:55,291 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:00:55,292 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:00:55,293 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:00:55,293 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:00:55,298 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:00:55,929 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:00:56,934 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:00:58,150 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:00:58,150 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:00:58,150 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:01:00,591 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:01:00,592 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:01:00,592 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:01:00,593 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:01:00,598 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:01:00,936 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:01:02,940 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:01:05,445 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:01:05,446 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:01:05,446 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:01:05,451 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:01:05,943 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:01:06,452 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:01:06,945 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:01:08,348 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 16:01:12,273 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:01:12,274 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:01:12,275 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:01:12,275 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:01:12,280 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:01:12,950 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:01:13,150 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:01:13,150 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:01:13,151 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:01:14,953 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:01:16,534 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:01:16,535 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:01:16,536 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:01:16,540 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:01:16,955 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:01:17,541 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:01:18,958 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:01:20,456 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:01:20,457 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:01:20,457 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:01:20,462 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:01:20,960 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:01:22,961 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:01:23,463 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:01:28,150 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:01:28,150 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:01:28,151 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:01:28,993 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:01:28,994 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:01:28,994 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:01:28,994 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:01:28,999 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:01:29,966 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:01:30,969 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:01:34,000 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:01:34,350 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:01:34,351 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:01:34,352 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:01:34,358 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:01:34,972 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:01:36,975 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:01:38,353 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 16:01:39,355 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:01:40,879 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:01:40,880 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:01:40,881 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:01:40,886 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:01:40,978 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:01:42,982 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:01:43,150 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:01:43,150 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:01:43,151 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:01:44,417 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:01:45,337 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:01:45,338 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:01:45,338 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:01:45,343 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:01:45,985 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:01:46,987 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:01:50,344 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:01:50,404 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:01:50,405 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:01:50,405 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:01:50,410 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:01:50,990 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:01:52,993 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:01:55,411 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:01:57,402 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:01:57,403 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:01:57,403 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:01:57,408 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:01:57,996 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:01:58,150 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:01:58,150 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:01:58,151 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:01:58,999 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:02:00,425 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:02:03,093 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:02:03,094 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:02:03,094 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:02:03,100 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:02:04,002 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:02:05,004 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:02:06,101 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:02:08,352 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 16:02:08,388 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:02:08,389 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:02:08,389 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:02:08,394 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:02:09,009 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:02:11,012 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:02:11,394 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:02:13,151 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:02:13,151 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:02:13,151 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:02:16,407 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:02:16,840 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:02:16,840 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:02:16,841 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:02:16,846 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:02:17,017 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:02:19,020 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:02:21,311 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:02:21,312 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:02:21,312 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:02:21,317 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:02:22,023 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:02:22,317 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:02:23,026 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:02:25,410 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:02:25,411 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:02:25,411 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:02:25,416 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:02:26,029 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:02:27,032 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:02:27,416 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:02:28,151 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:02:28,151 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:02:28,152 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:02:31,254 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:02:31,255 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:02:31,255 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:02:31,260 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:02:32,038 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:02:33,039 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:02:33,297 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:02:35,576 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:02:35,577 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:02:35,577 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:02:35,582 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:02:36,044 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:02:37,047 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:02:38,354 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 16:02:38,356 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:02:40,907 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:02:40,908 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:02:40,908 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:02:40,913 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:02:41,050 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:02:43,055 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:02:43,151 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:02:43,151 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:02:43,151 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:02:44,353 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:02:49,354 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:02:49,494 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:02:49,494 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:02:49,495 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:02:49,499 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:02:50,064 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:02:51,067 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:02:53,832 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:02:53,833 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:02:53,833 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:02:53,838 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:02:54,071 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:02:54,839 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:02:57,077 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:02:58,151 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:02:58,151 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:02:58,151 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:02:58,476 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:02:58,494 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:02:58,494 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:02:58,499 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:02:59,079 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:03:00,500 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:03:01,082 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:03:05,500 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:03:08,183 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:03:08,184 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:03:08,184 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:03:08,189 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:03:08,357 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 16:03:09,092 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:03:11,095 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:03:11,358 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:03:13,151 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:03:13,151 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:03:13,151 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:03:13,365 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:03:13,430 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:03:13,430 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:03:13,435 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:03:14,100 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:03:15,102 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:03:16,435 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:03:17,922 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:03:17,923 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:03:17,923 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:03:17,928 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:03:18,106 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:03:21,113 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:03:21,965 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:03:23,702 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:03:23,702 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:03:23,703 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:03:23,708 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:03:24,115 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:03:25,117 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:03:27,708 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:03:28,151 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:03:28,151 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:03:28,151 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:03:29,487 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:03:29,488 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:03:29,488 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:03:29,493 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:03:30,124 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:03:31,127 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:03:33,493 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:03:36,403 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:03:36,404 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:03:36,404 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:03:36,409 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:03:37,133 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:03:38,359 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 16:03:39,136 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:03:39,360 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:03:41,472 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:03:41,472 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:03:41,473 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:03:41,477 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:03:42,143 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:03:43,145 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:03:43,151 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:03:43,151 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:03:43,151 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:03:44,411 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:03:47,245 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:03:47,246 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:03:47,246 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:03:47,251 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:03:48,154 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:03:49,156 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:03:50,252 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:03:52,730 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:03:52,731 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:03:52,731 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:03:52,736 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:03:53,160 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:03:55,167 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:03:55,737 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:03:58,151 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:03:58,151 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:03:58,151 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:04:01,269 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:04:01,270 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:04:01,270 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:04:01,270 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:04:01,275 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:04:02,176 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:04:03,179 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:04:06,030 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:04:06,030 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:04:06,030 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:04:06,035 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:04:06,186 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:04:07,036 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:04:08,365 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 16:04:09,191 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:04:12,366 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:04:12,958 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:04:12,959 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:04:12,959 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:04:12,964 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:04:13,151 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:04:13,151 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:04:13,151 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:04:13,195 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:04:15,202 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:04:17,454 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:04:18,487 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:04:18,488 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:04:18,488 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:04:18,493 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:04:19,205 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:04:21,213 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:04:22,494 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:04:23,821 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:04:23,822 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:04:23,822 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:04:23,827 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:04:24,216 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:04:25,219 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:04:27,827 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:04:28,151 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:04:28,151 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:04:28,151 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:04:33,156 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:04:33,156 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:04:33,157 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:04:33,157 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:04:33,162 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:04:33,230 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:04:34,236 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:04:38,162 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:04:38,363 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 16:04:40,814 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:04:40,815 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:04:40,815 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:04:40,820 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:04:41,243 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:04:42,248 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:04:43,152 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:04:43,152 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:04:43,152 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:04:43,369 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:04:46,108 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:04:46,109 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:04:46,109 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:04:46,114 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:04:46,253 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:04:48,258 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:04:49,149 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:04:52,001 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:04:52,001 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:04:52,002 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:04:52,006 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:04:52,264 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:04:54,269 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:04:55,007 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:04:57,210 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:04:57,211 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:04:57,211 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:04:57,216 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:04:57,271 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:04:58,152 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:04:58,153 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:04:58,153 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:04:58,278 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:05:00,416 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:05:03,085 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:05:03,086 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:05:03,086 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:05:03,091 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:05:03,281 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:05:04,285 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:05:06,092 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:05:07,327 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:05:07,328 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:05:07,328 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:05:07,333 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:05:08,293 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:05:08,293 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:05:08,372 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 16:05:11,373 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:05:11,973 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:05:11,974 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:05:11,974 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:05:11,979 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:05:12,298 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:05:13,152 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:05:13,152 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:05:13,152 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:05:14,302 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:05:16,435 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:05:16,927 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:05:16,928 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:05:16,928 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:05:16,933 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:05:17,304 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:05:18,309 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:05:21,122 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:05:21,123 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:05:21,123 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:05:21,128 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:05:21,312 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:05:22,129 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:05:22,316 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:05:26,215 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:05:26,216 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:05:26,216 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:05:26,221 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:05:26,322 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:05:27,221 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:05:28,152 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:05:28,152 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:05:28,153 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:05:28,327 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:05:32,362 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:05:35,330 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:05:35,331 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:05:35,331 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:05:35,336 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:05:35,336 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:05:36,339 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:05:38,338 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:05:38,368 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 16:05:40,642 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:05:40,643 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:05:40,644 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:05:40,649 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:05:41,342 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:05:42,347 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:05:43,152 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:05:43,152 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:05:43,152 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:05:43,419 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:05:47,478 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:05:47,479 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:05:47,479 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:05:47,484 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:05:48,356 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:05:48,356 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:05:48,521 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:05:52,829 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:05:52,830 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:05:52,830 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:05:52,835 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:05:53,360 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:05:53,836 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:05:54,365 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:05:58,152 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:05:58,152 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:05:58,152 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:05:58,271 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:05:58,485 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:05:58,485 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:05:58,490 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:05:59,370 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:05:59,490 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:06:00,374 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:06:02,910 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:06:02,911 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:06:02,911 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:06:02,916 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:06:03,376 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:06:04,380 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:06:04,917 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:06:07,172 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:06:07,173 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:06:07,173 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:06:07,178 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:06:07,383 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:06:08,370 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 16:06:08,386 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:06:10,372 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:06:13,152 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:06:13,152 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:06:13,153 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:06:13,321 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:06:13,413 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:06:13,413 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:06:13,418 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:06:14,394 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:06:14,394 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:06:15,418 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:06:19,261 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:06:19,262 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:06:19,262 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:06:19,267 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:06:19,399 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:06:20,404 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:06:21,267 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:06:24,581 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:06:24,582 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:06:24,582 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:06:24,587 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:06:25,408 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:06:26,412 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:06:26,588 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:06:28,152 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:06:28,153 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:06:28,153 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:06:31,017 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:06:31,018 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:06:31,018 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:06:31,023 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:06:31,417 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:06:32,061 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:06:32,421 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:06:36,171 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:06:36,172 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:06:36,172 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:06:36,177 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:06:36,425 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:06:37,177 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:06:38,376 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 16:06:38,431 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:06:40,642 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:06:40,642 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:06:40,643 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:06:40,647 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:06:41,433 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:06:42,436 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:06:42,648 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:06:43,152 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:06:43,153 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:06:43,153 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:06:44,815 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:06:44,815 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:06:44,816 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:06:44,821 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:06:45,438 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:06:46,442 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:06:47,857 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:06:48,725 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:06:48,726 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:06:48,726 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:06:48,731 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:06:49,444 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:06:50,448 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:06:53,428 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:06:53,429 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:06:53,430 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:06:53,430 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:06:53,434 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:06:53,451 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:06:54,455 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:06:58,152 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:06:58,153 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:06:58,153 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:06:59,404 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:06:59,872 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:06:59,872 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:06:59,873 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:06:59,878 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:07:00,463 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:07:00,464 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:07:04,914 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:07:04,945 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:07:04,945 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:07:04,946 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:07:04,950 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:07:05,467 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:07:06,473 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:07:08,375 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 16:07:09,488 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:07:09,489 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:07:09,489 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:07:09,494 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:07:10,478 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:07:10,478 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:07:10,495 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:07:13,152 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:07:13,153 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:07:13,153 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:07:14,330 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:07:14,331 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:07:14,331 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:07:14,336 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:07:14,481 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:07:16,373 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:07:16,487 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:07:19,658 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:07:19,659 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:07:19,659 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:07:19,664 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:07:20,493 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:07:20,493 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:07:21,664 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:07:24,106 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:07:24,107 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:07:24,108 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:07:24,112 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:07:24,497 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:07:26,501 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:07:27,113 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:07:28,152 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:07:28,153 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:07:28,153 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:07:28,555 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:07:28,556 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:07:28,556 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:07:28,561 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:07:29,503 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:07:30,507 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:07:32,597 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:07:32,886 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:07:32,887 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:07:32,887 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:07:32,892 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:07:33,509 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:07:34,513 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:07:37,893 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:07:38,380 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 16:07:38,473 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:07:38,474 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:07:38,474 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:07:38,479 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:07:38,517 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:07:40,522 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:07:43,153 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:07:43,153 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:07:43,153 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:07:43,421 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:07:43,423 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:07:43,423 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:07:43,423 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:07:43,428 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:07:43,524 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:07:44,528 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:07:48,465 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:07:51,508 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:07:51,509 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:07:51,509 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:07:51,515 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:07:51,535 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:07:52,540 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:07:53,515 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:07:55,790 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:07:55,791 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:07:55,791 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:07:55,796 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:07:56,546 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:07:56,546 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:07:58,153 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:07:58,153 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:07:58,153 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:07:59,391 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:08:00,280 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:08:00,280 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:08:00,281 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:08:00,286 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:08:00,550 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:08:02,555 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:08:05,322 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:08:05,548 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:08:05,549 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:08:05,549 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:08:05,554 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:08:05,557 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:08:06,562 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:08:08,379 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 16:08:10,335 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:08:10,336 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:08:10,336 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:08:10,336 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:08:10,341 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:08:10,565 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:08:12,570 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:08:13,153 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:08:13,153 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:08:13,153 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:08:15,151 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:08:15,152 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:08:15,152 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:08:15,157 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:08:15,572 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:08:16,193 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:08:16,576 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:08:21,194 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:08:21,464 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:08:21,465 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:08:21,465 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:08:21,470 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:08:21,580 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:08:22,585 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:08:26,000 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:08:26,001 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:08:26,001 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:08:26,006 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:08:26,591 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:08:26,591 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:08:27,006 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:08:28,153 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:08:28,153 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:08:28,153 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:08:30,625 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:08:30,625 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:08:30,626 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:08:30,630 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:08:31,595 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:08:32,599 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:08:32,669 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:08:35,701 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:08:35,702 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:08:35,702 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:08:35,707 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:08:36,605 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:08:36,606 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:08:37,708 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:08:38,381 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 16:08:39,968 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:08:39,969 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:08:39,970 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:08:39,974 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:08:40,611 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:08:40,611 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:08:42,975 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:08:43,153 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:08:43,153 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:08:43,153 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:08:45,316 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:08:45,319 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:08:45,320 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:08:45,324 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:08:45,616 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:08:46,620 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:08:48,357 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:08:50,328 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:08:50,329 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:08:50,329 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:08:50,334 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:08:50,625 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:08:52,630 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:08:54,335 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:08:57,373 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:08:57,374 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:08:57,374 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:08:57,379 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:08:57,633 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:08:58,153 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:08:58,153 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:08:58,154 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:08:58,638 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:08:59,426 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:09:01,605 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:09:01,606 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:09:01,606 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:09:01,611 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:09:01,640 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:09:02,644 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:09:04,649 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:09:08,022 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:09:08,022 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:09:08,023 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:09:08,028 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:09:08,383 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 16:09:08,652 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:09:08,653 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:09:10,385 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:09:12,556 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:09:12,556 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:09:12,557 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:09:12,561 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:09:12,656 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:09:13,153 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:09:13,153 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:09:13,154 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:09:14,661 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:09:15,433 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:09:16,777 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:09:16,778 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:09:16,778 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:09:16,783 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:09:17,663 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:09:18,667 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:09:20,822 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:09:21,513 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:09:21,514 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:09:21,514 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:09:21,519 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:09:21,670 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:09:22,674 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:09:26,520 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:09:27,762 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:09:27,763 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:09:27,763 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:09:27,768 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:09:28,153 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:09:28,153 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:09:28,154 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:09:28,682 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:09:28,683 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:09:31,989 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:09:31,989 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:09:31,990 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:09:31,990 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:09:31,995 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:09:32,688 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:09:32,689 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:09:37,034 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:09:38,385 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 16:09:38,463 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:09:38,464 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:09:38,464 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:09:38,469 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:09:38,695 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:09:40,700 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:09:42,470 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:09:42,760 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:09:42,761 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:09:42,762 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:09:42,767 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:09:43,153 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:09:43,153 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:09:43,154 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:09:43,703 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:09:44,706 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:09:48,406 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:09:48,406 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:09:48,407 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:09:48,407 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:09:48,412 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:09:48,711 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:09:50,716 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:09:53,450 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:09:53,451 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:09:53,452 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:09:53,452 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:09:53,458 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:09:53,717 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:09:54,721 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:09:57,713 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:09:57,714 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:09:57,715 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:09:57,720 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:09:57,723 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:09:58,153 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:09:58,154 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:09:58,154 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:09:58,456 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:09:58,727 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:10:02,481 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:10:02,482 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:10:02,482 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:10:02,487 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:10:02,731 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:10:03,525 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:10:04,736 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:10:06,961 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:10:06,962 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:10:06,962 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:10:06,967 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:10:07,739 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:10:08,388 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 16:10:08,742 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:10:09,389 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:10:13,153 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:10:13,154 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:10:13,154 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:10:13,453 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:10:13,458 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:10:13,459 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:10:13,464 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:10:13,745 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:10:14,464 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:10:14,749 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:10:19,465 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:10:20,435 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:10:20,436 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:10:20,436 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:10:20,441 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:10:20,757 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:10:22,762 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:10:24,478 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:10:25,467 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:10:25,468 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:10:25,469 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:10:25,476 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:10:25,764 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:10:26,768 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:10:28,154 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:10:28,154 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:10:28,154 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:10:30,448 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:10:30,478 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:10:30,479 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:10:30,479 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:10:30,484 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:10:30,772 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:10:32,776 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:10:35,296 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:10:35,297 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:10:35,297 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:10:35,302 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:10:35,778 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:10:36,302 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:10:36,782 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:10:38,390 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 16:10:39,756 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:10:39,757 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:10:39,757 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:10:39,762 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:10:39,784 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:10:40,788 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:10:41,763 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:10:43,154 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:10:43,154 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:10:43,154 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:10:45,008 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:10:45,009 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:10:45,009 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:10:45,014 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:10:45,792 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:10:46,797 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:10:47,049 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:10:50,813 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:10:50,814 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:10:50,814 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:10:50,820 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:10:51,801 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:10:52,806 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:10:52,820 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:10:56,682 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:10:56,683 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:10:56,684 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:10:56,689 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:10:56,810 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:10:58,154 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:10:58,154 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:10:58,154 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:10:58,363 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:10:58,815 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:11:02,223 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:11:02,224 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:11:02,225 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:11:02,229 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:11:02,820 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:11:02,821 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:11:04,265 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:11:06,663 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:11:06,664 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:11:06,664 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:11:06,669 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:11:06,826 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:11:08,395 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 16:11:08,830 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:11:09,397 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:11:11,159 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:11:11,160 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:11:11,160 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:11:11,165 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:11:11,831 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:11:12,835 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:11:13,154 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:11:13,154 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:11:13,154 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:11:15,365 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:11:16,979 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:11:16,980 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:11:16,980 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:11:16,985 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:11:17,840 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:11:18,844 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:11:21,023 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:11:21,277 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:11:21,278 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:11:21,278 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:11:21,283 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:11:21,847 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:11:22,851 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:11:25,381 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:11:25,382 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:11:25,382 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:11:25,388 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:11:25,852 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:11:26,388 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:11:26,856 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:11:28,154 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:11:28,154 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:11:28,154 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:11:29,894 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:11:29,894 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:11:29,895 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:11:29,900 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:11:30,862 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:11:30,862 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:11:31,937 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:11:34,012 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:11:34,013 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:11:34,013 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:11:34,019 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:11:34,868 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:11:34,868 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:11:37,019 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:11:38,394 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 16:11:42,396 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:11:42,790 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:11:42,791 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:11:42,792 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:11:42,799 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:11:42,877 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:11:43,154 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:11:43,154 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:11:43,154 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:11:44,882 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:11:47,445 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:11:48,871 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:11:48,872 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:11:48,872 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:11:48,877 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:11:48,887 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:11:50,892 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:11:52,914 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:11:53,364 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:11:53,364 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:11:53,365 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:11:53,370 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:11:53,894 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:11:54,898 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:11:58,154 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:11:58,154 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:11:58,154 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:11:58,469 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:11:58,628 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:11:58,628 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:11:58,629 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:11:58,634 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:11:58,901 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:12:00,907 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:12:03,144 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:12:03,145 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:12:03,146 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:12:03,151 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:12:03,909 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:12:04,151 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:12:04,913 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:12:08,399 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 16:12:09,324 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:12:09,324 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:12:09,325 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:12:09,325 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:12:09,330 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:12:09,917 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:12:10,921 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:12:13,154 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:12:13,154 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:12:13,154 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:12:14,194 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:12:14,194 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:12:14,195 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:12:14,199 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:12:14,927 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:12:14,927 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:12:15,237 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:12:18,891 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:12:18,892 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:12:18,892 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:12:18,897 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:12:18,931 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:12:20,897 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:12:20,937 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:12:25,486 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:12:25,486 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:12:25,487 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:12:25,491 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:12:25,942 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:12:26,492 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:12:26,945 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:12:28,154 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:12:28,154 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:12:28,155 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:12:32,195 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:12:32,196 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:12:32,196 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:12:32,197 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:12:32,202 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:12:32,954 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:12:32,954 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:12:37,128 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:12:37,129 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:12:37,129 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:12:37,134 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:12:37,958 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:12:38,134 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:12:38,399 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 16:12:38,963 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:12:41,436 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:12:41,436 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:12:41,437 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:12:41,442 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:12:41,965 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:12:42,969 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:12:43,154 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:12:43,154 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:12:43,155 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:12:43,410 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:12:45,935 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:12:45,936 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:12:45,936 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:12:45,941 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:12:45,971 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:12:46,975 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:12:48,977 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:12:50,065 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:12:50,065 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:12:50,066 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:12:50,071 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:12:50,981 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:12:50,981 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:12:54,071 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:12:55,583 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:12:55,584 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:12:55,585 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:12:55,589 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:12:55,986 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:12:56,989 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:12:58,154 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:12:58,154 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:12:58,155 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:12:59,369 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:12:59,408 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:12:59,409 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:12:59,409 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:12:59,414 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:12:59,991 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:13:00,994 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:13:04,450 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:13:05,821 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:13:05,822 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:13:05,822 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:13:05,828 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:13:06,001 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:13:07,006 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:13:08,401 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 16:13:10,106 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:13:10,107 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:13:10,107 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:13:10,107 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:13:10,112 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:13:11,011 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:13:11,011 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:13:13,154 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:13:13,155 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:13:13,155 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:13:14,766 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:13:14,767 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:13:14,767 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:13:14,772 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:13:15,015 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:13:15,809 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:13:17,020 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:13:19,193 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:13:19,194 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:13:19,194 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:13:19,199 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:13:20,022 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:13:21,026 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:13:21,200 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:13:23,885 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:13:23,886 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:13:23,886 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:13:23,892 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:13:24,028 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:13:25,032 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:13:26,892 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:13:28,154 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:13:28,155 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:13:28,155 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:13:28,960 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:13:28,961 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:13:28,961 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:13:28,966 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:13:29,037 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:13:31,041 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:13:32,007 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:13:33,899 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:13:33,899 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:13:33,900 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:13:33,905 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:13:34,043 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:13:35,048 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:13:37,905 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:13:38,403 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 16:13:38,438 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:13:38,438 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:13:38,439 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:13:38,444 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:13:39,054 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:13:39,054 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:13:43,155 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:13:43,155 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:13:43,155 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:13:43,406 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:13:44,140 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:13:44,140 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:13:44,141 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:13:44,146 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:13:45,062 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:13:45,063 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:13:49,182 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:13:50,001 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:13:50,002 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:13:50,002 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:13:50,007 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:13:50,067 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:13:51,071 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:13:54,076 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:13:54,076 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:13:54,077 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:13:54,082 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:13:55,077 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:13:55,077 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:13:55,082 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:13:58,155 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:13:58,155 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:13:58,155 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:13:59,957 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:13:59,958 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:13:59,958 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:13:59,963 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:14:00,081 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:14:01,001 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:14:01,086 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:14:06,002 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:14:08,328 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:14:08,329 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:14:08,329 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:14:08,334 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:14:08,405 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 16:14:09,098 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:14:09,098 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:14:11,407 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:14:13,155 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:14:13,155 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:14:13,155 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:14:14,499 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:14:14,500 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:14:14,500 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:14:14,505 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:14:15,107 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:14:15,107 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:14:16,541 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:14:19,344 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:14:19,345 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:14:19,345 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:14:19,350 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:14:20,111 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:14:21,116 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:14:22,350 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:14:24,031 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:14:24,032 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:14:24,032 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:14:24,037 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:14:24,118 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:14:25,122 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:14:28,038 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:14:28,155 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:14:28,155 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:14:28,155 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:14:29,293 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:14:29,294 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:14:29,294 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:14:29,299 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:14:30,126 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:14:31,130 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:14:33,338 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:14:34,819 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:14:34,820 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:14:34,821 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:14:34,826 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:14:35,134 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:14:37,139 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:14:38,420 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 16:14:38,422 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:14:43,155 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:14:43,155 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:14:43,155 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:14:43,463 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:14:43,463 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:14:43,464 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:14:43,464 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:14:43,469 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:14:44,146 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:14:45,150 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:14:48,506 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:14:49,218 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:14:49,219 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:14:49,219 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:14:49,224 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:14:50,155 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:14:51,160 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:14:53,773 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:14:53,774 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:14:53,775 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:14:53,775 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:14:53,780 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:14:54,161 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:14:55,166 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:14:58,155 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:14:58,155 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:14:58,155 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:14:59,333 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:14:59,333 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:14:59,334 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:14:59,334 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:14:59,339 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:15:00,170 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:15:01,174 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:15:04,378 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:15:06,551 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:15:06,552 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:15:06,552 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:15:06,557 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:15:07,184 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:15:07,184 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:15:08,410 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 16:15:09,411 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:15:13,155 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:15:13,155 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:15:13,156 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:15:13,373 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:15:13,444 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:15:13,445 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:15:13,449 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:15:14,190 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:15:14,450 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:15:15,194 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:15:19,450 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:15:21,826 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:15:21,827 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:15:21,827 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:15:21,832 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:15:22,202 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:15:23,207 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:15:24,869 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:15:28,155 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:15:28,155 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:15:28,156 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:15:30,140 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:15:30,140 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:15:30,141 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:15:30,141 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:15:30,145 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:15:30,214 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:15:31,218 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:15:34,442 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:15:34,443 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:15:34,443 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:15:34,448 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:15:35,224 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:15:35,224 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:15:35,449 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:15:38,412 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 16:15:39,021 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:15:39,022 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:15:39,022 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:15:39,027 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:15:39,228 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:15:41,027 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:15:41,233 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:15:43,155 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:15:43,155 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:15:43,156 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:15:44,073 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:15:44,073 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:15:44,074 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:15:44,079 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:15:44,235 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:15:45,239 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:15:46,117 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:15:48,638 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:15:48,639 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:15:48,640 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:15:48,645 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:15:49,245 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:15:49,246 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:15:51,645 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:15:56,646 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:15:57,236 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:15:57,237 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:15:57,237 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:15:57,242 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:15:57,255 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:15:58,155 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:15:58,155 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:15:58,156 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:15:59,260 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:16:02,415 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:16:03,954 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:16:03,955 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:16:03,955 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:16:03,960 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:16:04,264 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:16:05,268 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:16:07,998 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:16:08,414 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 16:16:09,609 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:16:09,610 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:16:09,610 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:16:09,615 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:16:10,273 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:16:11,276 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:16:13,155 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:16:13,155 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:16:13,156 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:16:13,480 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:16:16,547 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:16:16,548 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:16:16,548 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:16:16,554 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:16:17,284 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:16:17,284 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:16:18,589 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:16:21,570 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:16:21,571 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:16:21,571 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:16:21,576 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:16:22,291 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:16:23,295 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:16:24,577 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:16:28,155 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:16:28,156 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:16:28,156 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:16:29,856 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:16:29,857 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:16:29,858 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:16:29,858 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:16:29,863 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:16:30,303 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:16:31,307 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:16:34,897 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:16:35,171 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:16:35,171 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:16:35,172 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:16:35,177 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:16:35,310 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:16:37,315 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:16:38,416 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 16:16:40,417 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:16:41,736 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:16:41,737 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:16:41,737 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:16:41,742 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:16:42,319 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:16:43,155 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:16:43,156 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:16:43,156 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:16:43,324 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:16:45,456 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:16:46,556 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:16:46,556 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:16:46,557 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:16:46,563 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:16:47,331 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:16:47,331 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:16:50,598 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:16:50,989 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:16:50,990 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:16:50,990 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:16:50,995 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:16:51,334 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:16:53,339 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:16:55,996 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:16:58,156 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:16:58,156 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:16:58,156 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:16:59,416 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:16:59,417 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:16:59,417 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:16:59,422 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:17:00,346 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:17:01,350 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:17:01,457 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:17:03,482 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:17:03,483 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:17:03,483 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:17:03,488 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:17:04,352 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:17:05,356 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:17:06,489 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:17:08,135 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:17:08,135 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:17:08,136 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:17:08,141 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:17:08,359 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:17:08,422 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 16:17:09,363 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:17:12,424 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:17:12,872 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:17:12,873 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:17:12,873 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:17:12,878 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:17:13,156 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:17:13,156 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:17:13,156 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:17:13,367 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:17:15,371 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:17:18,418 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:17:21,049 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:17:21,050 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:17:21,050 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:17:21,055 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:17:21,378 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:17:23,383 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:17:24,093 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:17:27,523 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:17:27,524 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:17:27,524 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:17:27,529 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:17:28,156 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:17:28,156 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:17:28,156 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:17:28,387 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:17:29,392 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:17:29,424 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:17:32,239 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:17:32,239 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:17:32,240 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:17:32,244 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:17:32,394 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:17:33,398 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:17:35,281 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:17:38,421 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 16:17:40,422 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:17:40,748 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:17:40,748 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:17:40,749 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:17:40,754 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:17:41,409 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:17:41,409 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:17:43,156 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:17:43,156 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:17:43,156 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:17:45,422 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:17:49,051 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:17:49,051 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:17:49,052 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:17:49,057 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:17:49,419 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:17:51,093 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:17:51,424 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:17:54,113 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:17:54,114 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:17:54,114 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:17:54,120 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:17:54,426 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:17:55,430 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:17:56,120 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:17:58,156 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:17:58,156 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:17:58,156 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:18:01,471 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:18:01,475 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:18:01,476 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:18:01,476 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:18:01,481 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:18:02,437 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:18:03,440 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:18:05,713 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:18:05,713 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:18:05,714 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:18:05,718 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:18:06,443 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:18:06,719 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:18:07,449 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:18:08,426 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 16:18:11,019 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:18:11,020 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:18:11,020 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:18:11,025 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:18:11,452 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:18:12,025 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:18:13,156 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:18:13,156 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:18:13,156 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:18:13,457 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:18:15,800 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:18:15,800 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:18:15,801 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:18:15,806 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:18:16,459 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:18:17,463 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:18:17,841 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:18:22,842 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:18:23,718 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:18:23,718 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:18:23,719 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:18:23,724 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:18:24,470 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:18:25,475 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:18:28,156 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:18:28,156 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:18:28,157 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:18:28,394 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:18:28,685 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:18:28,686 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:18:28,686 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:18:28,691 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:18:29,481 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:18:29,481 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:18:33,430 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:18:33,430 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:18:33,431 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:18:33,431 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:18:33,436 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:18:33,485 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:18:35,489 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:18:38,425 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 16:18:39,427 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:18:42,094 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:18:42,094 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:18:42,095 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:18:42,099 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:18:42,496 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:18:43,157 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:18:43,157 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:18:43,157 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:18:43,500 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:18:44,520 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:18:47,073 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:18:47,074 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:18:47,074 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:18:47,079 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:18:47,505 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:18:49,510 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:18:50,117 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:18:52,343 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:18:52,343 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:18:52,344 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:18:52,348 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:18:52,512 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:18:53,516 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:18:55,349 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:18:56,866 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:18:56,866 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:18:56,867 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:18:56,871 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:18:57,520 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:18:57,520 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:18:58,156 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:18:58,156 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:18:58,157 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:19:00,399 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:19:01,390 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:19:01,390 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:19:01,390 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:19:01,395 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:19:01,526 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:19:03,531 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:19:05,433 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:19:06,803 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:19:06,804 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:19:06,804 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:19:06,809 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:19:07,538 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:19:07,538 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:19:08,427 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 16:19:11,134 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:19:11,134 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:19:11,134 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:19:11,135 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:19:11,139 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:19:11,540 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:19:13,156 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:19:13,156 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:19:13,157 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:19:13,543 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:19:16,501 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:19:17,111 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:19:17,112 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:19:17,112 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:19:17,117 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:19:17,547 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:19:19,552 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:19:21,313 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:19:21,313 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:19:21,314 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:19:21,319 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:19:21,551 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:19:22,319 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:19:23,555 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:19:25,971 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:19:25,971 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:19:25,972 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:19:25,977 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:19:26,556 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:19:27,559 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:19:27,977 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:19:28,156 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:19:28,157 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:19:28,157 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:19:29,830 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:19:29,831 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:19:29,831 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:19:29,836 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:19:30,561 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:19:31,562 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:19:33,873 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:19:34,829 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:19:34,834 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:19:34,835 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:19:34,845 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:19:35,566 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:19:35,566 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:19:38,430 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 16:19:38,785 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:19:38,785 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:19:38,786 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:19:38,791 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:19:39,570 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:19:39,570 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:19:39,791 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:19:43,156 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:19:43,157 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:19:43,157 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:19:43,672 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:19:43,673 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:19:43,673 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:19:43,678 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:19:44,573 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:19:45,576 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:19:45,679 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:19:49,394 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:19:49,394 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:19:49,395 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:19:49,399 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:19:49,579 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:19:51,399 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:19:51,582 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:19:54,790 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:19:54,790 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:19:54,790 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:19:54,796 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:19:55,586 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:19:55,587 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:19:56,796 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:19:58,157 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:19:58,157 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:19:58,157 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:20:00,496 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:20:00,497 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:20:00,497 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:20:00,502 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:20:00,589 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:20:01,592 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:20:02,502 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:20:06,050 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:20:06,051 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:20:06,051 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:20:06,056 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:20:06,595 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:20:07,599 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:20:08,056 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:20:08,432 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 16:20:12,881 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:20:12,882 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:20:12,882 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:20:12,887 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:20:13,157 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:20:13,157 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:20:13,157 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:20:13,356 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:20:13,605 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:20:13,605 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:20:16,937 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:20:16,937 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:20:16,937 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:20:16,942 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:20:17,610 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:20:17,610 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:20:18,981 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:20:22,231 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:20:22,231 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:20:22,232 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:20:22,236 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:20:22,613 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:20:23,616 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:20:24,237 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:20:27,627 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:20:27,627 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:20:27,628 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:20:27,632 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:20:28,157 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:20:28,157 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:20:28,157 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:20:28,619 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:20:29,480 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:20:29,619 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:20:32,258 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:20:32,259 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:20:32,259 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:20:32,265 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:20:32,621 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:20:33,624 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:20:35,301 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:20:36,535 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:20:36,536 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:20:36,537 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:20:36,541 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:20:36,625 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:20:37,628 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:20:38,437 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 16:20:40,439 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:20:41,148 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:20:41,149 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:20:41,149 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:20:41,154 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:20:41,630 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:20:43,157 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:20:43,157 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:20:43,157 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:20:43,634 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:20:46,027 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:20:46,028 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:20:46,028 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:20:46,028 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:20:46,033 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:20:46,636 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:20:47,639 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:20:50,459 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:20:50,459 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:20:50,460 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:20:50,465 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:20:50,640 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:20:51,465 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:20:51,643 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:20:56,445 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:20:56,445 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:20:56,446 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:20:56,452 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:20:56,646 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:20:57,452 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:20:57,648 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:20:58,157 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:20:58,157 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:20:58,157 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:21:01,181 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:21:01,182 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:21:01,182 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:21:01,187 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:21:01,651 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:21:03,225 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:21:03,654 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:21:08,226 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:21:08,436 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 16:21:09,524 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:21:09,525 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:21:09,525 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:21:09,530 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:21:09,659 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:21:11,662 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:21:13,157 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:21:13,157 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:21:13,157 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:21:13,422 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:21:14,063 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:21:14,064 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:21:14,064 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:21:14,069 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:21:14,664 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:21:15,667 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:21:18,677 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:21:18,678 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:21:18,678 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:21:18,678 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:21:18,684 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:21:19,671 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:21:19,671 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:21:23,685 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:21:27,822 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:21:27,823 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:21:27,823 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:21:27,828 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:21:28,157 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:21:28,157 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:21:28,157 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:21:28,678 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:21:29,370 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:21:29,680 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:21:31,885 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:21:31,885 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:21:31,886 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:21:31,890 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:21:32,682 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:21:33,685 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:21:34,891 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:21:38,100 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:21:38,100 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:21:38,101 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:21:38,105 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:21:38,438 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 16:21:38,688 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:21:39,691 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:21:40,439 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:21:43,157 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:21:43,157 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:21:43,158 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:21:44,393 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:21:44,396 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:21:44,397 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:21:44,402 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:21:44,694 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:21:45,697 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:21:46,437 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:21:48,655 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:21:48,656 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:21:48,656 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:21:48,661 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:21:48,699 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:21:49,700 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:21:51,662 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:21:53,931 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:21:53,932 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:21:53,932 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:21:53,937 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:21:54,703 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:21:55,705 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:21:56,937 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:21:58,157 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:21:58,157 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:21:58,158 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:21:58,501 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:21:58,501 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:21:58,502 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:21:58,506 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:21:58,707 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:21:59,710 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:22:02,507 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:22:04,254 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:22:04,255 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:22:04,255 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:22:04,260 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:22:04,712 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:22:05,715 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:22:08,260 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:22:08,440 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 16:22:09,634 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:22:09,635 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:22:09,635 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:22:09,640 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:22:09,719 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:22:11,722 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:22:13,157 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:22:13,157 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:22:13,158 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:22:13,373 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:22:15,997 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:22:15,998 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:22:15,998 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:22:16,003 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:22:16,726 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:22:17,729 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:22:19,041 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:22:21,018 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:22:21,019 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:22:21,019 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:22:21,024 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:22:21,734 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:22:21,734 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:22:25,025 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:22:28,157 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:22:28,157 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:22:28,158 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:22:30,311 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:22:30,312 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:22:30,312 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:22:30,312 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:22:30,317 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:22:30,740 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:22:31,742 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:22:34,756 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:22:34,757 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:22:34,757 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:22:34,762 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:22:35,746 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:22:35,746 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:22:35,763 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:22:38,442 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 16:22:39,065 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:22:39,066 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:22:39,066 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:22:39,071 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:22:39,751 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:22:39,751 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:22:41,072 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:22:43,157 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:22:43,158 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:22:43,158 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:22:43,161 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:22:43,407 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:22:43,407 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:22:43,413 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:22:43,753 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:22:45,757 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:22:46,414 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:22:49,856 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:22:49,857 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:22:49,857 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:22:49,862 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:22:50,760 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:22:51,761 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:22:51,862 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:22:54,128 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:22:54,129 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:22:54,129 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:22:54,137 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:22:54,762 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:22:55,765 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:22:57,137 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:22:58,156 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:22:58,157 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:22:58,157 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:22:58,158 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:22:58,158 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:22:58,162 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:22:58,163 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:22:58,767 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:22:59,770 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:23:02,468 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:23:04,276 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:23:04,277 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:23:04,277 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:23:04,282 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:23:04,773 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:23:05,775 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:23:08,318 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:23:08,448 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 16:23:08,991 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:23:08,992 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:23:08,993 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:23:08,998 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:23:09,781 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:23:09,781 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:23:13,158 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:23:13,158 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:23:13,158 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:23:13,378 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:23:14,817 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:23:14,818 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:23:14,818 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:23:14,823 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:23:15,787 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:23:15,788 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:23:18,862 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:23:19,094 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:23:19,095 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:23:19,096 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:23:19,101 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:23:19,792 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:23:19,792 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:23:23,767 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:23:23,767 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:23:23,768 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:23:23,773 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:23:23,795 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:23:24,773 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:23:25,798 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:23:28,158 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:23:28,158 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:23:28,159 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:23:28,501 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:23:28,502 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:23:28,502 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:23:28,507 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:23:28,800 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:23:29,802 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:23:30,545 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:23:33,778 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:23:33,778 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:23:33,778 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:23:33,783 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:23:33,805 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:23:35,783 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:23:35,806 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:23:38,447 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 16:23:38,624 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:23:38,625 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:23:38,625 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:23:38,630 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:23:38,807 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:23:39,810 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:23:41,630 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:23:42,696 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:23:42,696 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:23:42,697 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:23:42,702 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:23:42,812 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:23:43,158 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:23:43,158 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:23:43,159 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:23:43,815 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:23:47,449 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:23:49,393 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:23:49,394 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:23:49,394 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:23:49,399 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:23:49,819 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:23:51,822 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:23:53,437 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:23:53,802 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:23:53,803 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:23:53,803 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:23:53,808 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:23:53,823 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:23:55,827 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:23:58,158 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:23:58,158 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:23:58,159 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:23:58,451 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:23:59,328 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:23:59,328 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:23:59,328 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:23:59,333 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:23:59,829 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:24:01,832 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:24:04,020 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:24:04,020 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:24:04,021 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:24:04,021 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:24:04,026 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:24:04,834 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:24:05,837 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:24:08,082 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:24:08,082 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:24:08,083 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:24:08,088 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:24:08,455 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 16:24:08,840 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:24:09,457 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:24:09,843 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:24:13,023 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:24:13,024 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:24:13,024 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:24:13,030 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:24:13,158 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:24:13,158 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:24:13,158 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:24:13,848 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:24:13,848 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:24:14,652 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:24:17,317 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:24:17,318 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:24:17,318 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:24:17,324 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:24:17,851 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:24:19,854 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:24:20,361 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:24:24,046 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:24:24,047 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:24:24,047 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:24:24,052 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:24:24,857 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:24:25,860 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:24:26,053 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:24:28,158 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:24:28,158 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:24:28,159 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:24:29,786 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:24:29,786 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:24:29,787 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:24:29,791 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:24:29,862 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:24:31,829 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:24:31,865 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:24:34,200 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:24:34,201 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:24:34,201 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:24:34,206 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:24:34,867 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:24:35,870 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:24:37,206 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:24:38,452 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 16:24:39,259 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:24:39,260 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:24:39,261 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:24:39,266 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:24:39,874 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:24:39,874 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:24:42,266 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:24:43,158 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:24:43,158 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:24:43,159 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:24:46,427 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:24:46,428 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:24:46,428 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:24:46,432 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:24:46,879 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:24:47,469 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:24:47,882 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:24:51,376 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:24:51,377 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:24:51,377 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:24:51,382 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:24:51,884 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:24:53,382 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:24:53,887 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:24:57,096 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:24:57,097 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:24:57,097 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:24:57,102 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:24:57,894 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:24:57,894 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:24:58,158 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:24:58,158 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:24:58,159 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:24:58,429 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:25:02,096 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:25:02,097 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:25:02,097 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:25:02,102 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:25:02,897 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:25:03,900 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:25:04,137 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:25:06,385 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:25:06,386 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:25:06,386 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:25:06,391 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:25:06,901 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:25:07,905 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:25:08,457 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 16:25:09,458 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:25:10,538 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:25:10,539 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:25:10,539 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:25:10,544 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:25:10,908 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:25:11,910 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:25:13,158 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:25:13,159 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:25:13,159 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:25:15,183 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:25:15,184 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:25:15,184 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:25:15,184 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:25:15,189 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:25:15,915 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:25:15,916 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:25:19,905 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:25:19,906 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:25:19,906 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:25:19,911 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:25:19,918 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:25:20,911 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:25:21,921 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:25:25,912 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:25:28,159 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:25:28,159 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:25:28,159 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:25:28,981 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:25:28,981 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:25:28,982 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:25:28,986 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:25:29,928 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:25:29,929 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:25:31,025 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:25:35,344 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:25:35,344 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:25:35,345 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:25:35,350 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:25:35,934 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:25:35,934 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:25:36,350 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:25:38,456 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 16:25:39,321 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:25:39,322 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:25:39,322 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:25:39,327 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:25:39,938 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:25:39,938 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:25:42,328 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:25:43,159 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:25:43,159 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:25:43,159 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:25:43,978 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:25:43,978 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:25:43,978 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:25:43,983 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:25:44,942 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:25:45,944 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:25:48,021 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:25:48,057 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:25:48,058 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:25:48,058 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:25:48,063 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:25:48,946 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:25:49,949 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:25:53,064 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:25:53,675 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:25:53,676 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:25:53,676 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:25:53,681 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:25:53,951 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:25:55,955 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:25:57,991 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:25:57,992 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:25:57,993 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:25:57,998 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:25:58,159 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:25:58,159 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:25:58,159 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:25:58,362 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:25:58,956 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:25:59,959 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:26:03,363 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:26:03,945 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:26:03,946 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:26:03,946 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:26:03,951 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:26:03,961 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:26:05,964 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:26:08,458 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 16:26:08,460 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:26:09,858 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:26:09,858 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:26:09,859 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:26:09,864 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:26:09,969 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:26:11,972 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:26:13,159 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:26:13,159 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:26:13,159 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:26:14,046 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:26:14,046 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:26:14,047 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:26:14,047 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:26:14,052 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:26:14,974 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:26:15,977 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:26:18,573 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:26:18,574 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:26:18,574 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:26:18,579 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:26:18,979 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:26:19,579 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:26:19,981 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:26:23,305 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:26:23,306 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:26:23,306 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:26:23,312 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:26:23,984 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:26:23,984 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:26:25,312 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:26:28,159 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:26:28,159 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:26:28,159 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:26:29,513 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:26:29,514 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:26:29,514 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:26:29,519 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:26:29,987 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:26:30,560 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:26:31,991 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:26:34,892 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:26:34,893 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:26:34,893 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:26:34,898 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:26:34,993 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:26:35,899 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:26:35,995 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:26:38,460 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 16:26:40,010 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:26:40,010 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:26:40,010 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:26:40,016 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:26:40,998 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:26:41,016 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:26:42,004 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:26:43,159 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:26:43,159 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:26:43,159 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:26:44,614 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:26:44,615 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:26:44,615 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:26:44,621 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:26:45,006 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:26:46,009 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:26:46,657 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:26:50,136 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:26:50,137 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:26:50,137 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:26:50,142 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:26:51,014 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:26:52,017 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:26:52,142 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:26:54,803 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:26:54,804 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:26:54,804 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:26:54,809 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:26:55,019 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:26:56,021 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:26:57,810 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:26:58,159 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:26:58,159 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:26:58,159 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:27:00,708 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:27:00,708 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:27:00,709 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:27:00,714 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:27:01,024 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:27:02,027 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:27:03,749 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:27:05,425 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:27:05,426 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:27:05,426 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:27:05,431 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:27:06,031 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:27:06,031 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:27:08,462 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 16:27:09,463 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:27:10,005 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:27:10,005 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:27:10,005 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:27:10,011 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:27:10,035 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:27:12,038 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:27:13,159 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:27:13,159 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:27:13,160 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:27:15,450 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:27:16,972 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:27:16,973 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:27:16,973 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:27:16,978 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:27:17,041 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:27:18,044 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:27:21,013 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:27:21,453 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:27:21,454 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:27:21,454 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:27:21,459 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:27:22,049 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:27:22,049 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:27:26,460 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:27:27,913 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:27:27,913 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:27:27,913 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:27:27,918 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:27:28,053 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:27:28,159 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:27:28,160 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:27:28,161 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:27:30,056 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:27:32,450 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:27:32,573 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:27:32,574 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:27:32,574 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:27:32,579 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:27:33,058 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:27:34,060 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:27:36,788 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:27:36,788 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:27:36,789 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:27:36,794 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:27:37,062 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:27:37,794 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:27:38,065 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:27:38,468 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 16:27:43,159 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:27:43,159 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:27:43,160 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:27:43,374 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:27:43,422 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:27:43,422 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:27:43,422 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:27:43,428 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:27:44,071 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:27:44,071 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:27:48,357 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:27:48,358 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:27:48,358 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:27:48,363 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:27:49,074 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:27:49,401 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:27:50,077 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:27:52,809 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:27:52,810 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:27:52,810 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:27:52,816 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:27:53,079 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:27:54,082 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:27:54,816 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:27:58,159 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:27:58,159 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:27:58,160 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:27:58,541 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:27:58,541 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:27:58,542 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:27:58,547 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:27:59,085 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:28:00,087 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:28:00,548 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:28:04,852 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:28:04,853 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:28:04,853 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:28:04,859 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:28:05,091 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:28:05,859 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:28:06,093 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:28:08,467 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 16:28:10,287 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:28:10,288 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:28:10,288 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:28:10,294 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:28:11,098 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:28:11,294 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:28:12,101 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:28:13,159 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:28:13,160 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:28:13,160 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:28:16,390 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:28:16,514 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:28:16,515 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:28:16,515 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:28:16,520 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:28:17,104 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:28:18,107 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:28:20,545 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:28:20,546 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:28:20,546 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:28:20,551 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:28:21,109 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:28:21,552 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:28:22,112 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:28:25,219 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:28:25,220 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:28:25,220 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:28:25,225 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:28:26,116 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:28:26,116 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:28:27,226 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:28:28,159 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:28:28,160 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:28:28,160 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:28:29,542 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:28:29,542 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:28:29,542 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:28:29,548 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:28:30,119 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:28:32,122 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:28:32,585 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:28:36,565 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:28:36,566 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:28:36,566 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:28:36,571 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:28:37,125 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:28:38,126 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:28:38,472 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 16:28:38,473 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:28:43,160 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:28:43,160 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:28:43,160 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:28:43,555 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:28:43,555 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:28:43,556 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:28:43,556 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:28:43,561 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:28:44,130 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:28:46,133 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:28:47,913 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:28:47,914 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:28:47,914 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:28:47,919 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:28:48,136 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:28:48,919 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:28:50,139 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:28:52,643 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:28:52,644 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:28:52,644 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:28:52,649 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:28:53,141 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:28:54,141 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:28:54,649 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:28:57,752 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:28:57,753 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:28:57,753 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:28:57,758 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:28:58,144 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:28:58,160 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:28:58,160 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:28:58,160 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:29:00,147 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:29:00,435 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:29:04,839 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:29:04,840 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:29:04,840 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:29:04,845 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:29:05,152 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:29:05,881 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:29:06,155 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:29:08,471 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 16:29:09,100 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:29:09,100 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:29:09,101 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:29:09,106 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:29:09,158 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:29:10,161 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:29:11,106 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:29:13,160 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:29:13,160 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:29:13,160 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:29:14,328 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:29:14,328 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:29:14,328 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:29:14,334 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:29:15,165 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:29:16,165 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:29:16,369 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:29:20,856 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:29:20,857 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:29:20,857 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:29:20,862 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:29:21,168 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:29:21,862 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:29:22,171 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:29:25,928 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:29:25,929 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:29:25,929 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:29:25,934 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:29:26,174 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:29:26,935 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:29:28,160 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:29:28,160 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:29:28,160 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:29:28,177 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:29:31,030 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:29:31,030 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:29:31,030 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:29:31,036 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:29:31,179 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:29:32,073 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:29:32,180 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:29:37,074 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:29:38,201 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:29:38,201 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:29:38,201 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:29:38,206 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:29:38,473 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 16:29:39,184 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:29:40,186 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:29:42,475 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:29:43,160 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:29:43,160 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:29:43,160 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:29:43,385 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:29:43,676 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:29:43,676 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:29:43,681 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:29:44,189 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:29:46,193 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:29:47,682 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:29:47,772 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:29:47,773 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:29:47,773 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:29:47,779 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:29:48,194 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:29:50,197 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:29:52,656 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:29:52,657 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:29:52,657 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:29:52,662 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:29:53,199 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:29:53,662 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:29:54,202 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:29:58,160 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:29:58,160 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:29:58,160 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:29:58,198 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:29:58,465 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:29:58,466 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:29:58,471 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:29:59,205 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:29:59,471 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:30:00,207 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:30:02,466 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:30:02,467 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:30:02,467 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:30:02,472 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:30:03,209 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:30:04,212 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:30:04,472 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:30:07,115 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:30:07,119 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:30:07,120 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:30:07,125 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:30:07,214 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:30:08,218 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:30:08,475 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 16:30:09,477 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:30:11,369 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:30:11,370 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:30:11,370 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:30:11,375 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:30:12,222 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:30:12,223 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:30:13,160 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:30:13,160 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:30:13,160 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:30:15,440 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:30:16,885 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:30:16,886 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:30:16,886 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:30:16,891 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:30:17,226 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:30:18,228 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:30:20,929 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:30:22,316 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:30:22,317 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:30:22,317 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:30:22,322 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:30:23,232 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:30:24,234 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:30:26,323 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:30:27,006 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:30:27,007 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:30:27,007 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:30:27,012 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:30:27,236 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:30:28,160 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:30:28,160 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:30:28,161 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:30:28,239 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:30:31,302 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:30:31,303 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:30:31,303 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:30:31,307 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:30:31,345 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:30:32,243 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:30:32,243 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:30:36,242 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:30:36,242 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:30:36,243 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:30:36,247 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:30:36,248 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:30:37,248 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:30:38,250 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:30:38,478 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 16:30:42,337 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:30:42,338 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:30:42,338 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:30:42,338 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:30:42,343 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:30:43,160 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:30:43,160 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:30:43,161 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:30:43,253 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:30:44,256 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:30:46,453 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:30:46,454 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:30:46,454 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:30:46,458 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:30:47,258 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:30:47,497 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:30:48,261 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:30:52,498 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:30:55,206 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:30:55,207 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:30:55,207 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:30:55,212 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:30:55,265 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:30:56,268 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:30:58,160 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:30:58,160 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:30:58,161 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:30:58,386 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:30:59,456 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:30:59,457 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:30:59,457 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:30:59,462 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:31:00,272 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:31:00,272 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:31:03,497 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:31:05,011 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:31:05,012 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:31:05,012 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:31:05,017 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:31:05,275 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:31:06,278 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:31:08,483 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 16:31:09,381 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:31:09,382 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:31:09,382 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:31:09,382 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:31:09,386 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:31:10,284 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:31:10,284 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:31:13,160 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:31:13,161 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:31:13,161 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:31:13,651 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:31:13,652 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:31:13,652 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:31:13,657 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:31:14,289 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:31:14,289 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:31:14,693 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:31:19,636 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:31:19,636 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:31:19,636 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:31:19,642 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:31:20,294 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:31:20,295 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:31:20,642 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:31:24,760 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:31:24,761 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:31:24,761 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:31:24,766 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:31:25,298 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:31:25,766 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:31:26,300 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:31:28,160 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:31:28,161 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:31:28,161 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:31:29,302 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:31:29,303 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:31:29,303 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:31:29,308 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:31:30,305 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:31:30,305 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:31:31,345 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:31:33,553 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:31:33,554 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:31:33,554 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:31:33,559 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:31:34,309 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:31:34,309 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:31:36,559 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:31:38,433 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:31:38,434 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:31:38,434 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:31:38,439 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:31:38,482 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 16:31:39,312 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:31:40,314 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:31:42,484 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:31:43,161 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:31:43,161 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:31:43,161 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:31:43,371 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:31:43,463 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:31:43,463 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:31:43,468 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:31:44,319 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:31:44,319 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:31:48,469 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:31:51,869 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:31:51,870 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:31:51,870 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:31:51,875 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:31:52,324 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:31:53,876 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:31:54,327 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:31:57,204 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:31:57,205 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:31:57,205 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:31:57,210 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:31:57,329 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:31:58,161 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:31:58,161 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:31:58,161 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:31:58,331 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:31:59,378 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:32:02,913 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:32:02,914 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:32:02,914 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:32:02,918 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:32:03,334 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:32:04,337 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:32:04,957 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:32:07,529 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:32:07,530 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:32:07,530 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:32:07,535 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:32:08,342 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:32:08,343 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:32:08,487 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 16:32:10,489 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:32:13,161 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:32:13,161 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:32:13,161 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:32:13,199 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:32:13,772 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:32:13,773 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:32:13,778 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:32:14,350 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:32:14,350 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:32:15,778 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:32:20,779 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:32:20,825 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:32:20,826 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:32:20,826 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:32:20,831 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:32:21,354 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:32:22,357 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:32:25,831 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:32:25,975 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:32:25,976 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:32:25,976 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:32:25,981 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:32:26,359 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:32:28,161 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:32:28,161 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:32:28,161 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:32:28,362 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:32:30,528 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:32:30,529 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:32:30,529 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:32:30,533 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:32:31,364 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:32:31,569 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:32:32,367 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:32:35,390 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:32:35,391 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:32:35,392 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:32:35,396 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:32:36,371 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:32:36,371 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:32:37,397 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:32:38,486 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 16:32:42,112 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:32:42,113 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:32:42,113 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:32:42,118 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:32:42,375 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:32:43,119 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:32:43,161 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:32:43,161 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:32:43,161 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:32:44,380 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:32:48,458 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:32:50,478 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:32:50,478 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:32:50,478 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:32:50,483 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:32:51,385 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:32:52,387 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:32:53,521 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:32:54,259 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:32:54,260 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:32:54,260 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:32:54,265 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:32:54,388 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:32:56,392 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:32:58,161 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:32:58,161 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:32:58,161 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:32:59,358 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:32:59,691 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:32:59,691 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:32:59,691 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:32:59,696 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:33:00,396 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:33:00,396 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:33:04,432 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:33:04,432 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:33:04,433 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:33:04,433 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:33:04,437 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:33:05,399 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:33:06,401 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:33:08,489 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 16:33:09,490 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:33:13,161 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:33:13,161 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:33:13,161 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:33:13,682 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:33:13,683 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:33:13,683 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:33:13,688 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:33:14,410 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:33:14,410 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:33:14,725 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:33:18,294 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:33:18,295 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:33:18,295 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:33:18,300 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:33:18,413 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:33:20,300 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:33:20,416 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:33:24,978 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:33:24,979 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:33:24,979 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:33:24,984 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:33:25,419 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:33:25,984 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:33:26,422 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:33:28,161 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:33:28,161 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:33:28,161 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:33:30,278 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:33:30,278 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:33:30,278 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:33:30,284 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:33:30,424 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:33:31,321 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:33:32,427 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:33:35,211 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:33:35,212 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:33:35,212 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:33:35,217 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:33:35,429 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:33:36,431 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:33:37,218 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:33:38,490 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 16:33:41,140 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:33:41,140 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:33:41,141 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:33:41,146 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:33:41,434 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:33:42,437 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:33:43,147 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:33:43,161 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:33:43,161 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:33:43,161 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:33:45,625 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:33:45,626 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:33:45,626 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:33:45,631 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:33:46,439 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:33:46,440 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:33:48,669 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:33:51,840 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:33:51,841 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:33:51,841 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:33:51,846 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:33:52,445 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:33:52,445 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:33:53,847 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:33:57,231 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:33:57,232 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:33:57,232 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:33:57,237 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:33:57,448 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:33:58,161 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:33:58,162 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:33:58,162 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:33:58,451 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:33:59,421 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:34:01,426 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:34:01,427 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:34:01,427 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:34:01,432 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:34:01,453 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:34:02,453 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:34:04,469 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:34:05,918 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:34:05,918 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:34:05,918 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:34:05,924 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:34:06,458 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:34:06,458 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:34:08,493 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 16:34:09,494 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:34:10,282 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:34:10,283 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:34:10,283 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:34:10,288 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:34:10,462 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:34:12,465 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:34:13,161 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:34:13,162 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:34:13,162 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:34:15,428 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:34:17,943 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:34:17,944 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:34:17,944 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:34:17,949 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:34:18,471 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:34:18,471 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:34:20,950 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:34:22,932 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:34:22,933 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:34:22,933 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:34:22,938 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:34:23,475 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:34:24,477 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:34:26,877 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:34:26,878 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:34:26,878 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:34:26,878 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:34:26,883 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:34:27,479 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:34:28,161 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:34:28,162 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:34:28,162 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:34:28,482 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:34:32,441 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:34:34,486 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:34:34,487 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:34:34,487 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:34:34,496 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:34:35,486 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:34:36,488 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:34:37,529 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:34:38,498 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 16:34:38,783 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:34:38,783 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:34:38,784 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:34:38,789 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:34:39,490 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:34:40,493 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:34:42,789 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:34:43,161 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:34:43,162 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:34:43,162 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:34:45,403 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:34:45,404 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:34:45,404 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:34:45,409 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:34:45,497 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:34:46,499 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:34:48,410 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:34:50,328 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:34:50,329 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:34:50,329 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:34:50,334 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:34:50,502 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:34:52,506 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:34:54,334 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:34:55,696 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:34:55,697 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:34:55,697 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:34:55,702 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:34:56,510 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:34:56,510 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:34:58,162 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:34:58,162 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:34:58,162 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:34:59,440 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:35:04,124 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:35:04,125 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:35:04,125 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:35:04,130 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:35:04,514 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:35:05,130 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:35:06,523 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:35:08,192 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:35:08,192 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:35:08,193 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:35:08,199 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:35:08,497 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 16:35:08,526 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:35:10,499 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:35:10,530 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:35:13,162 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:35:13,162 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:35:13,162 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:35:13,738 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:35:13,739 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:35:13,739 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:35:13,744 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:35:14,535 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:35:14,535 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:35:15,744 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:35:17,813 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:35:17,814 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:35:17,814 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:35:17,819 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:35:18,539 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:35:18,539 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:35:20,820 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:35:25,820 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:35:26,166 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:35:26,167 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:35:26,167 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:35:26,172 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:35:26,544 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:35:28,162 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:35:28,162 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:35:28,162 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:35:28,547 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:35:31,406 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:35:32,023 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:35:32,024 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:35:32,024 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:35:32,029 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:35:32,551 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:35:32,552 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:35:37,030 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:35:38,500 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:35:38,501 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 16:35:38,502 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:35:38,502 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:35:38,506 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:35:38,555 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:35:40,558 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:35:42,507 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:35:43,162 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:35:43,162 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:35:43,162 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:35:45,472 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:35:45,473 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:35:45,473 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:35:45,478 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:35:45,561 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:35:46,564 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:35:48,478 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:35:49,764 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:35:49,768 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:35:49,768 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:35:49,773 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:35:50,569 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:35:50,569 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:35:53,774 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:35:54,671 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:35:54,671 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:35:54,671 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:35:54,676 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:35:55,572 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:35:56,574 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:35:58,162 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:35:58,162 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:35:58,162 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:35:59,466 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:35:59,580 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:35:59,581 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:35:59,581 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:35:59,586 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:36:00,579 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:36:00,579 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:36:04,586 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:36:04,587 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:36:04,587 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:36:04,587 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:36:04,592 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:36:05,581 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:36:06,584 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:36:08,502 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 16:36:09,845 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:36:09,845 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:36:09,846 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:36:09,846 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:36:09,850 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:36:10,590 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:36:10,590 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:36:13,162 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:36:13,162 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:36:13,162 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:36:14,057 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:36:14,058 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:36:14,058 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:36:14,063 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:36:14,594 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:36:14,595 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:36:15,101 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:36:19,782 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:36:19,782 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:36:19,782 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:36:19,787 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:36:20,600 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:36:20,600 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:36:20,788 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:36:24,830 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:36:24,831 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:36:24,831 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:36:24,836 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:36:25,604 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:36:25,836 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:36:26,607 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:36:28,162 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:36:28,162 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:36:28,162 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:36:29,105 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:36:29,106 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:36:29,106 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:36:29,111 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:36:29,609 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:36:30,611 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:36:31,111 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:36:33,573 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:36:33,573 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:36:33,573 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:36:33,579 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:36:33,613 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:36:34,616 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:36:36,579 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:36:38,504 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 16:36:38,863 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:36:38,864 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:36:38,864 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:36:38,869 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:36:39,619 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:36:40,621 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:36:41,870 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:36:43,162 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:36:43,162 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:36:43,163 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:36:45,528 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:36:45,528 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:36:45,529 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:36:45,534 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:36:45,625 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:36:46,627 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:36:47,534 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:36:52,535 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:36:54,365 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:36:54,365 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:36:54,366 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:36:54,371 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:36:54,632 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:36:56,639 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:36:58,162 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:36:58,162 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:36:58,163 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:36:58,418 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:37:03,419 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:37:03,695 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:37:03,696 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:37:03,696 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:37:03,701 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:37:04,644 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:37:06,647 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:37:08,353 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:37:08,354 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:37:08,354 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:37:08,359 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:37:08,510 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 16:37:08,512 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:37:08,649 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:37:10,653 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:37:12,650 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:37:12,651 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:37:12,651 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:37:12,656 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:37:12,656 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:37:13,162 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:37:13,162 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:37:13,163 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:37:14,372 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:37:14,660 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:37:18,915 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:37:18,915 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:37:18,916 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:37:18,921 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:37:19,663 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:37:19,921 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:37:20,665 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:37:24,921 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:37:26,041 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:37:26,042 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:37:26,042 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:37:26,048 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:37:26,669 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:37:28,162 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:37:28,163 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:37:28,163 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:37:28,673 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:37:30,369 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:37:30,370 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:37:30,370 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:37:30,370 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:37:30,375 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:37:30,674 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:37:32,677 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:37:35,413 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:37:35,504 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:37:35,504 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:37:35,504 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:37:35,509 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:37:35,679 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:37:36,681 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:37:38,509 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 16:37:40,510 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:37:40,897 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:37:40,897 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:37:40,898 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:37:40,902 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:37:41,685 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:37:42,687 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:37:43,163 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:37:43,163 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:37:43,163 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:37:46,427 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:37:46,508 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:37:46,509 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:37:46,509 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:37:46,514 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:37:46,690 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:37:48,693 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:37:51,514 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:37:54,872 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:37:54,873 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:37:54,873 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:37:54,878 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:37:55,697 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:37:56,698 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:37:56,878 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:37:58,162 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:37:58,163 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:37:58,163 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:37:59,395 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:37:59,396 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:37:59,396 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:37:59,401 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:37:59,700 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:38:00,703 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:38:02,402 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:38:03,698 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:38:03,699 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:38:03,699 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:38:03,704 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:38:03,705 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:38:04,707 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:38:07,704 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:38:08,514 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 16:38:09,729 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:38:09,730 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:38:09,730 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:38:09,735 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:38:10,715 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:38:10,715 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:38:12,735 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:38:13,163 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:38:13,163 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:38:13,163 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:38:14,241 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:38:14,242 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:38:14,242 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:38:14,248 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:38:14,718 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:38:16,721 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:38:18,249 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:38:22,654 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:38:22,655 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:38:22,655 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:38:22,660 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:38:22,724 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:38:23,661 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:38:24,728 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:38:27,553 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:38:27,553 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:38:27,554 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:38:27,559 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:38:27,730 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:38:28,163 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:38:28,163 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:38:28,163 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:38:28,733 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:38:29,366 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:38:32,088 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:38:32,088 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:38:32,089 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:38:32,093 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:38:32,736 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:38:34,739 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:38:35,094 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:38:38,513 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 16:38:40,369 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:38:40,370 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:38:40,370 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:38:40,370 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:38:40,375 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:38:40,742 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:38:42,746 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:38:43,163 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:38:43,163 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:38:43,163 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:38:44,846 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:38:44,846 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:38:44,846 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:38:44,851 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:38:45,748 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:38:45,889 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:38:46,750 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:38:49,515 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:38:49,516 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:38:49,516 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:38:49,521 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:38:49,753 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:38:50,755 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:38:51,521 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:38:54,585 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:38:54,585 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:38:54,586 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:38:54,590 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:38:54,758 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:38:56,591 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:38:56,759 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:38:58,163 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:38:58,163 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:38:58,163 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:39:00,310 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:39:00,311 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:39:00,311 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:39:00,316 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:39:00,762 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:39:02,317 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:39:02,765 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:39:05,316 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:39:05,317 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:39:05,317 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:39:05,322 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:39:05,767 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:39:06,769 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:39:07,322 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:39:08,515 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 16:39:10,172 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:39:10,172 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:39:10,173 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:39:10,177 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:39:10,773 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:39:12,777 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:39:13,163 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:39:13,163 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:39:13,163 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:39:13,434 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:39:14,738 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:39:14,739 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:39:14,739 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:39:14,745 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:39:14,778 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:39:16,781 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:39:18,746 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:39:20,506 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:39:20,507 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:39:20,507 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:39:20,511 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:39:20,783 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:39:22,787 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:39:24,512 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:39:25,709 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:39:25,710 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:39:25,710 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:39:25,715 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:39:25,789 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:39:26,792 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:39:28,163 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:39:28,163 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:39:28,163 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:39:29,965 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:39:29,965 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:39:29,966 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:39:29,966 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:39:29,971 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:39:30,794 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:39:32,797 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:39:34,971 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:39:35,130 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:39:35,131 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:39:35,131 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:39:35,136 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:39:35,799 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:39:36,802 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:39:38,517 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 16:39:40,359 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:39:40,360 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:39:40,360 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:39:40,360 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:39:40,365 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:39:40,804 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:39:42,808 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:39:43,163 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:39:43,163 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:39:43,164 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:39:45,445 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:39:45,469 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:39:45,470 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:39:45,470 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:39:45,475 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:39:45,810 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:39:46,812 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:39:50,475 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:39:50,812 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:39:50,813 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:39:50,813 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:39:50,818 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:39:50,818 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:39:52,822 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:39:55,819 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:39:56,322 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:39:56,323 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:39:56,323 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:39:56,329 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:39:56,824 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:39:58,163 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:39:58,163 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:39:58,163 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:39:58,827 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:40:01,406 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:40:02,027 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:40:02,028 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:40:02,028 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:40:02,033 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:40:02,830 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:40:04,833 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:40:06,565 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:40:06,566 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:40:06,567 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:40:06,567 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:40:06,573 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:40:06,835 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:40:08,519 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 16:40:08,839 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:40:11,789 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:40:11,790 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:40:11,790 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:40:11,790 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:40:11,796 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:40:11,842 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:40:12,844 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:40:13,163 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:40:13,164 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:40:13,164 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:40:17,410 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:40:17,558 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:40:17,559 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:40:17,559 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:40:17,564 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:40:17,848 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:40:18,851 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:40:22,565 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:40:23,430 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:40:23,430 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:40:23,431 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:40:23,435 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:40:23,854 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:40:24,857 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:40:28,163 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:40:28,163 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:40:28,164 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:40:28,467 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:40:29,186 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:40:29,186 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:40:29,186 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:40:29,191 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:40:29,860 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:40:30,862 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:40:34,229 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:40:34,327 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:40:34,328 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:40:34,328 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:40:34,334 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:40:34,865 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:40:36,868 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:40:38,525 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 16:40:39,499 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:40:39,500 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:40:39,500 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:40:39,500 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:40:39,505 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:40:39,870 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:40:40,873 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:40:43,163 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:40:43,164 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:40:43,164 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:40:45,459 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:40:47,846 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:40:47,846 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:40:47,847 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:40:47,851 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:40:47,877 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:40:48,880 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:40:50,852 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:40:53,090 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:40:53,091 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:40:53,091 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:40:53,096 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:40:53,883 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:40:54,886 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:40:56,096 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:40:58,163 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:40:58,164 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:40:58,164 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:41:00,271 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:41:00,272 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:41:00,272 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:41:00,277 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:41:00,890 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:41:01,277 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:41:02,893 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:41:04,722 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:41:04,723 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:41:04,723 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:41:04,728 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:41:04,894 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:41:06,729 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:41:06,895 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:41:08,524 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 16:41:09,157 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:41:09,158 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:41:09,158 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:41:09,163 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:41:09,898 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:41:10,901 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:41:12,164 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:41:13,164 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:41:13,164 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:41:13,164 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:41:13,817 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:41:13,818 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:41:13,818 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:41:13,823 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:41:13,903 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:41:14,906 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:41:17,824 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:41:21,680 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:41:21,680 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:41:21,680 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:41:21,685 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:41:21,910 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:41:22,913 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:41:23,686 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:41:26,403 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:41:26,404 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:41:26,404 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:41:26,409 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:41:26,916 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:41:28,164 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:41:28,164 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:41:28,164 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:41:28,919 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:41:29,412 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:41:31,483 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:41:31,483 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:41:31,483 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:41:31,488 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:41:31,921 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:41:32,923 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:41:34,525 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:41:36,772 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:41:36,773 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:41:36,774 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:41:36,778 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:41:36,926 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:41:38,529 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 16:41:38,929 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:41:39,530 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:41:42,060 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:41:42,060 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:41:42,060 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:41:42,065 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:41:42,931 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:41:43,164 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:41:43,164 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:41:43,164 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:41:44,935 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:41:45,395 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:41:46,620 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:41:46,621 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:41:46,621 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:41:46,626 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:41:46,937 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:41:48,940 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:41:50,661 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:41:53,171 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:41:53,172 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:41:53,172 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:41:53,177 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:41:53,943 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:41:54,946 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:41:56,177 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:41:58,164 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:41:58,164 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:41:58,164 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:41:58,716 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:41:58,717 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:41:58,717 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:41:58,722 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:41:58,948 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:42:00,951 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:42:01,722 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:42:03,048 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:42:03,049 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:42:03,050 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:42:03,055 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:42:03,955 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:42:04,957 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:42:07,055 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:42:08,181 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:42:08,181 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:42:08,181 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:42:08,186 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:42:08,528 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 16:42:08,961 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:42:10,964 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:42:12,530 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:42:13,164 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:42:13,164 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:42:13,164 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:42:17,404 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:42:17,405 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:42:17,405 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:42:17,410 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:42:17,969 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:42:18,410 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:42:18,971 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:42:22,087 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:42:22,088 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:42:22,089 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:42:22,094 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:42:22,975 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:42:24,094 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:42:24,978 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:42:25,954 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:42:25,954 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:42:25,955 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:42:25,959 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:42:25,979 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:42:26,982 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:42:28,164 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:42:28,164 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:42:28,164 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:42:29,418 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:42:31,418 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:42:31,418 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:42:31,418 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:42:31,423 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:42:31,985 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:42:32,990 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:42:34,424 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:42:35,910 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:42:35,911 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:42:35,911 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:42:35,917 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:42:35,992 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:42:36,994 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:42:38,530 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 16:42:39,532 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:42:41,153 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:42:41,154 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:42:41,154 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:42:41,159 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:42:41,997 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:42:43,001 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:42:43,164 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:42:43,164 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:42:43,164 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:42:45,440 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:42:45,630 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:42:45,631 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:42:45,631 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:42:45,636 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:42:46,003 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:42:47,006 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:42:50,637 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:42:54,167 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:42:54,168 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:42:54,168 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:42:54,173 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:42:55,011 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:42:56,173 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:42:57,014 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:42:58,164 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:42:58,164 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:42:58,165 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:42:58,210 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:42:58,387 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:42:58,387 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:42:58,392 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:42:59,015 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:43:01,018 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:43:01,393 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:43:03,547 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:43:03,548 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:43:03,548 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:43:03,553 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:43:04,020 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:43:05,023 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:43:06,554 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:43:08,207 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:43:08,208 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:43:08,209 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:43:08,214 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:43:08,533 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 16:43:09,027 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:43:11,030 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:43:12,460 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:43:12,461 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:43:12,461 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:43:12,461 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:43:12,466 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:43:13,032 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:43:13,164 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:43:13,165 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:43:13,165 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:43:15,035 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:43:17,315 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:43:17,316 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:43:17,317 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:43:17,321 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:43:18,037 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:43:18,322 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:43:19,040 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:43:21,768 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:43:21,769 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:43:21,769 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:43:21,774 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:43:22,041 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:43:23,045 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:43:23,775 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:43:27,176 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:43:27,176 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:43:27,177 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:43:27,181 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:43:28,048 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:43:28,164 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:43:28,165 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:43:28,165 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:43:29,048 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:43:29,432 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:43:31,275 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:43:31,275 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:43:31,276 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:43:31,281 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:43:32,050 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:43:33,053 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:43:35,281 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:43:36,453 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:43:36,454 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:43:36,454 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:43:36,459 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:43:37,055 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:43:38,535 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 16:43:39,059 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:43:40,536 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:43:43,164 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:43:43,165 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:43:43,165 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:43:44,075 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:43:44,076 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:43:44,076 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:43:44,081 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:43:45,066 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:43:45,066 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:43:46,082 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:43:48,834 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:43:48,835 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:43:48,835 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:43:48,840 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:43:49,068 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:43:51,072 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:43:51,840 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:43:56,841 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:43:58,165 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:43:58,165 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:43:58,165 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:43:58,327 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:43:58,431 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:43:58,431 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:43:58,436 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:43:59,078 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:44:01,081 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:44:02,436 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:44:02,458 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:44:02,458 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:44:02,459 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:44:02,464 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:44:03,082 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:44:05,085 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:44:06,726 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:44:06,727 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:44:06,727 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:44:06,732 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:44:07,087 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:44:07,732 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:44:08,541 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 16:44:09,091 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:44:10,978 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:44:10,978 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:44:10,979 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:44:10,984 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:44:11,093 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:44:12,984 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:44:13,096 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:44:13,165 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:44:13,165 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:44:13,165 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:44:16,335 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:44:16,336 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:44:16,336 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:44:16,341 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:44:17,099 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:44:18,341 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:44:19,103 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:44:20,492 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:44:20,493 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:44:20,493 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:44:20,498 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:44:21,104 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:44:23,108 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:44:23,498 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:44:26,255 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:44:26,256 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:44:26,256 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:44:26,262 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:44:27,110 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:44:28,165 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:44:28,165 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:44:28,165 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:44:29,113 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:44:29,382 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:44:31,063 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:44:31,064 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:44:31,064 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:44:31,069 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:44:31,115 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:44:33,118 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:44:35,106 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:44:38,539 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 16:44:39,373 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:44:39,374 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:44:39,374 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:44:39,379 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:44:40,122 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:44:40,379 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:44:41,125 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:44:43,165 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:44:43,165 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:44:43,165 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:44:43,874 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:44:43,874 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:44:43,875 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:44:43,880 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:44:44,127 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:44:45,130 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:44:45,880 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:44:50,376 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:44:50,377 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:44:50,377 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:44:50,382 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:44:51,134 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:44:51,383 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:44:53,137 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:44:54,882 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:44:54,883 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:44:54,884 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:44:54,889 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:44:55,138 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:44:56,889 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:44:57,141 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:44:58,165 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:44:58,165 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:44:58,165 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:45:00,224 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:45:00,225 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:45:00,225 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:45:00,231 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:45:01,146 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:45:01,146 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:45:02,231 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:45:05,232 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:45:05,233 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:45:05,234 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:45:05,238 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:45:06,149 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:45:07,152 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:45:07,239 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:45:08,545 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 16:45:12,546 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:45:13,165 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:45:13,165 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:45:13,165 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:45:13,636 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:45:13,637 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:45:13,637 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:45:13,642 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:45:14,157 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:45:15,160 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:45:17,643 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:45:18,111 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:45:18,111 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:45:18,112 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:45:18,117 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:45:18,162 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:45:19,163 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:45:22,564 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:45:22,565 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:45:22,565 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:45:22,570 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:45:23,166 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:45:23,571 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:45:25,169 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:45:28,165 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:45:28,165 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:45:28,165 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:45:29,404 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:45:31,032 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:45:31,033 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:45:31,033 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:45:31,039 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:45:31,172 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:45:33,176 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:45:35,040 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:45:37,209 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:45:37,210 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:45:37,211 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:45:37,216 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:45:38,179 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:45:38,544 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 16:45:39,181 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:45:40,545 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:45:41,701 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:45:41,702 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:45:41,703 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:45:41,707 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:45:42,183 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:45:43,165 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:45:43,165 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:45:43,166 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:45:43,186 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:45:46,453 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:45:47,649 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:45:47,650 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:45:47,650 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:45:47,655 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:45:48,190 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:45:49,192 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:45:51,656 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:45:55,461 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:45:55,462 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:45:55,462 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:45:55,469 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:45:56,197 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:45:57,199 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:45:57,469 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:45:58,165 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:45:58,165 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:45:58,166 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:46:02,227 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:46:02,228 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:46:02,228 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:46:02,234 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:46:03,205 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:46:03,205 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:46:03,234 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:46:06,284 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:46:06,285 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:46:06,285 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:46:06,290 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:46:07,209 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:46:07,209 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:46:08,290 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:46:08,546 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 16:46:13,165 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:46:13,165 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:46:13,166 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:46:13,445 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:46:14,653 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:46:14,654 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:46:14,654 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:46:14,659 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:46:15,217 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:46:17,220 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:46:18,660 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:46:19,128 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:46:19,129 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:46:19,129 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:46:19,134 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:46:19,221 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:46:21,224 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:46:24,135 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:46:24,838 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:46:24,839 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:46:24,839 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:46:24,844 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:46:25,228 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:46:27,232 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:46:28,165 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:46:28,165 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:46:28,166 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:46:29,161 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:46:29,162 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:46:29,163 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:46:29,163 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:46:29,168 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:46:29,233 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:46:31,236 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:46:34,169 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:46:34,315 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:46:34,315 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:46:34,316 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:46:34,321 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:46:35,242 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:46:35,242 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:46:38,551 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 16:46:39,553 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:46:39,660 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:46:39,660 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:46:39,661 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:46:39,666 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:46:40,245 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:46:41,247 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:46:43,165 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:46:43,166 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:46:43,166 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:46:45,421 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:46:48,142 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:46:48,143 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:46:48,143 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:46:48,148 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:46:48,252 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:46:49,255 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:46:51,149 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:46:56,149 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:46:57,203 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:46:57,204 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:46:57,204 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:46:57,209 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:46:57,260 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:46:58,165 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:46:58,166 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:46:58,166 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:46:59,263 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:47:01,376 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:47:05,434 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:47:05,434 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:47:05,435 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:47:05,440 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:47:06,268 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:47:06,477 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:47:07,271 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:47:08,550 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 16:47:10,101 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:47:10,102 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:47:10,102 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:47:10,106 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:47:10,274 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:47:11,277 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:47:12,107 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:47:13,165 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:47:13,166 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:47:13,166 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:47:14,423 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:47:14,424 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:47:14,424 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:47:14,429 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:47:15,280 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:47:17,283 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:47:17,430 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:47:20,452 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:47:20,453 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:47:20,453 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:47:20,458 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:47:21,286 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:47:22,458 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:47:23,289 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:47:24,392 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:47:24,393 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:47:24,393 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:47:24,398 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:47:25,293 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:47:25,293 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:47:28,166 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:47:28,166 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:47:28,166 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:47:28,399 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:47:29,520 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:47:29,521 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:47:29,521 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:47:29,526 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:47:30,296 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:47:31,299 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:47:33,561 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:47:33,727 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:47:33,728 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:47:33,729 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:47:33,734 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:47:34,301 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:47:35,303 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:47:38,556 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 16:47:38,576 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:47:38,577 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:47:38,577 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:47:38,577 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:47:38,583 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:47:39,306 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:47:41,309 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:47:42,678 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:47:42,678 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:47:42,679 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:47:42,684 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:47:43,166 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:47:43,166 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:47:43,166 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:47:43,311 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:47:44,458 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:47:45,314 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:47:47,406 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:47:47,407 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:47:47,408 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:47:47,412 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:47:48,317 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:47:49,320 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:47:50,413 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:47:52,383 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:47:52,383 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:47:52,384 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:47:52,388 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:47:53,324 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:47:53,325 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:47:56,389 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:47:58,115 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:47:58,115 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:47:58,116 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:47:58,120 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:47:58,166 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:47:58,166 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:47:58,166 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:47:58,327 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:47:59,330 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:48:01,462 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:48:02,558 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:48:02,559 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:48:02,559 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:48:02,564 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:48:03,333 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:48:05,337 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:48:06,565 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:48:08,554 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 16:48:08,866 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:48:08,867 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:48:08,867 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:48:08,873 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:48:09,341 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:48:11,345 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:48:11,874 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:48:13,166 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:48:13,166 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:48:13,166 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:48:13,603 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:48:13,604 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:48:13,604 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:48:13,610 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:48:14,347 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:48:15,350 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:48:17,610 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:48:18,341 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:48:18,342 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:48:18,342 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:48:18,347 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:48:18,351 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:48:19,354 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:48:23,348 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:48:23,591 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:48:23,591 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:48:23,592 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:48:23,597 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:48:24,358 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:48:25,360 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:48:28,028 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:48:28,029 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:48:28,029 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:48:28,034 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:48:28,166 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:48:28,166 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:48:28,166 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:48:28,362 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:48:28,406 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:48:29,365 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:48:32,852 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:48:32,853 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:48:32,853 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:48:32,858 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:48:33,367 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:48:33,859 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:48:35,370 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:48:38,557 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 16:48:39,558 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:48:41,110 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:48:41,110 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:48:41,111 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:48:41,116 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:48:41,374 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:48:43,166 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:48:43,166 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:48:43,166 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:48:43,377 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:48:45,384 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:48:46,351 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:48:46,352 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:48:46,353 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:48:46,357 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:48:46,379 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:48:47,383 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:48:51,358 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:48:52,471 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:48:52,471 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:48:52,472 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:48:52,476 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:48:53,389 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:48:53,389 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:48:56,477 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:48:58,166 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:48:58,166 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:48:58,166 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:48:58,574 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:48:58,575 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:48:58,575 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:48:58,580 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:48:59,392 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:49:01,402 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:49:01,580 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:49:06,490 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:49:06,490 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:49:06,491 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:49:06,496 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:49:07,407 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:49:07,408 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:49:07,496 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:49:08,559 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 16:49:12,092 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:49:12,093 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:49:12,093 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:49:12,098 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:49:12,413 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:49:13,099 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:49:13,166 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:49:13,166 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:49:13,167 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:49:13,415 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:49:16,348 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:49:16,349 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:49:16,349 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:49:16,354 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:49:16,417 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:49:17,418 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:49:18,354 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:49:20,804 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:49:20,805 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:49:20,805 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:49:20,810 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:49:21,421 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:49:23,425 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:49:23,811 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:49:28,166 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:49:28,166 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:49:28,167 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:49:29,280 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:49:29,280 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:49:29,281 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:49:29,281 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:49:29,286 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:49:29,428 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:49:31,432 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:49:33,788 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:49:33,789 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:49:33,789 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:49:33,794 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:49:34,433 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:49:34,794 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:49:35,436 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:49:38,561 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 16:49:40,533 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:49:40,534 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:49:40,534 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:49:40,534 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:49:40,539 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:49:41,442 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:49:41,442 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:49:43,166 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:49:43,166 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:49:43,167 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:49:45,601 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:49:45,602 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:49:45,603 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:49:45,603 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:49:45,608 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:49:46,445 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:49:47,448 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:49:50,608 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:49:51,036 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:49:51,037 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:49:51,038 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:49:51,043 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:49:51,452 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:49:53,455 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:49:56,044 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:49:56,207 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:49:56,208 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:49:56,208 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:49:56,213 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:49:56,457 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:49:57,460 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:49:58,166 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:49:58,167 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:49:58,167 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:50:01,440 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:50:01,724 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:50:01,725 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:50:01,726 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:50:01,731 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:50:02,463 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:50:03,464 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:50:06,413 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:50:06,414 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:50:06,414 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:50:06,419 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:50:06,466 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:50:07,420 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:50:07,468 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:50:08,567 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 16:50:12,093 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:50:12,094 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:50:12,094 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:50:12,099 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:50:12,473 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:50:13,099 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:50:13,166 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:50:13,167 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:50:13,167 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:50:13,476 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:50:16,655 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:50:16,655 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:50:16,656 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:50:16,661 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:50:17,478 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:50:18,661 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:50:19,483 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:50:23,662 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:50:25,107 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:50:25,108 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:50:25,108 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:50:25,113 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:50:25,487 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:50:27,491 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:50:28,167 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:50:28,167 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:50:28,167 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:50:29,407 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:50:29,579 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:50:29,580 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:50:29,580 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:50:29,585 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:50:30,492 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:50:31,495 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:50:34,227 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:50:34,227 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:50:34,228 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:50:34,233 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:50:34,497 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:50:35,233 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:50:35,500 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:50:38,565 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 16:50:39,469 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:50:39,469 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:50:39,469 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:50:39,474 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:50:39,503 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:50:40,475 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:50:41,506 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:50:43,167 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:50:43,167 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:50:43,167 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:50:45,337 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:50:45,338 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:50:45,338 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:50:45,343 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:50:45,509 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:50:46,343 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:50:47,513 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:50:49,853 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:50:49,854 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:50:49,854 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:50:49,859 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:50:50,515 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:50:51,515 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:50:51,860 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:50:54,499 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:50:54,500 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:50:54,500 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:50:54,505 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:50:54,517 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:50:55,520 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:50:57,506 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:50:58,167 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:50:58,167 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:50:58,167 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:50:59,413 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:50:59,414 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:50:59,414 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:50:59,419 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:50:59,522 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:51:01,525 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:51:03,420 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:51:04,421 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:51:04,422 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:51:04,422 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:51:04,426 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:51:04,528 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:51:05,530 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:51:08,427 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:51:08,571 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 16:51:09,522 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:51:09,523 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:51:09,523 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:51:09,528 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:51:09,534 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:51:11,538 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:51:13,167 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:51:13,167 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:51:13,167 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:51:14,381 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:51:15,014 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:51:15,015 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:51:15,015 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:51:15,020 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:51:15,540 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:51:17,544 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:51:20,020 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:51:23,382 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:51:23,383 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:51:23,383 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:51:23,388 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:51:23,547 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:51:25,389 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:51:25,551 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:51:28,167 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:51:28,167 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:51:28,167 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:51:29,023 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:51:29,024 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:51:29,024 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:51:29,029 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:51:29,553 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:51:31,030 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:51:31,556 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:51:33,450 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:51:33,451 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:51:33,451 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:51:33,456 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:51:33,558 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:51:35,561 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:51:36,456 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:51:38,166 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:51:38,166 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:51:38,167 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:51:38,171 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:51:38,563 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:51:38,570 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 16:51:39,566 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:51:41,571 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:51:43,167 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:51:43,167 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:51:43,167 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:51:45,768 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:51:45,768 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:51:45,768 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:51:45,773 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:51:46,571 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:51:46,774 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:51:47,573 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:51:50,973 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:51:50,974 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:51:50,974 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:51:50,979 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:51:51,576 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:51:51,979 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:51:53,579 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:51:55,833 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:51:55,833 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:51:55,834 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:51:55,839 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:51:56,581 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:51:57,584 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:51:57,839 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:51:58,167 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:51:58,167 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:51:58,167 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:52:01,804 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:52:01,805 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:52:01,805 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:52:01,810 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:52:02,587 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:52:03,589 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:52:03,811 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:52:08,159 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:52:08,160 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:52:08,160 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:52:08,165 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:52:08,572 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 16:52:08,592 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:52:09,573 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:52:09,596 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:52:12,691 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:52:12,691 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:52:12,692 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:52:12,697 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:52:13,167 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:52:13,167 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:52:13,168 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:52:13,599 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:52:13,599 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:52:15,439 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:52:18,017 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:52:18,017 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:52:18,018 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:52:18,023 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:52:18,602 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:52:19,605 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:52:21,024 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:52:23,807 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:52:23,808 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:52:23,808 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:52:23,813 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:52:24,608 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:52:25,611 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:52:26,814 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:52:28,167 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:52:28,167 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:52:28,168 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:52:28,632 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:52:28,632 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:52:28,633 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:52:28,638 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:52:29,615 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:52:29,616 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:52:32,639 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:52:34,194 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:52:34,195 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:52:34,195 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:52:34,200 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:52:34,619 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:52:35,621 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:52:38,200 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:52:38,574 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 16:52:42,364 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:52:42,365 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:52:42,365 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:52:42,369 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:52:42,625 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:52:43,167 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:52:43,168 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:52:43,168 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:52:43,386 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:52:43,626 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:52:47,063 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:52:47,064 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:52:47,064 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:52:47,069 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:52:47,629 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:52:49,069 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:52:49,632 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:52:51,387 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:52:51,388 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:52:51,388 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:52:51,393 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:52:51,634 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:52:53,637 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:52:54,394 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:52:56,737 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:52:56,738 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:52:56,738 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:52:56,743 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:52:57,641 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:52:57,641 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:52:58,167 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:52:58,168 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:52:58,168 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:52:59,435 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:53:02,179 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:53:02,180 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:53:02,180 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:53:02,185 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:53:02,644 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:53:03,647 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:53:05,185 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:53:06,461 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:53:06,461 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:53:06,461 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:53:06,466 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:53:06,649 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:53:07,649 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:53:08,576 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 16:53:10,577 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:53:12,691 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:53:12,691 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:53:12,692 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:53:12,696 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:53:13,168 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:53:13,168 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:53:13,168 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:53:13,656 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:53:13,657 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:53:16,378 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:53:17,527 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:53:17,527 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:53:17,527 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:53:17,533 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:53:17,659 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:53:19,662 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:53:21,569 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:53:22,241 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:53:22,242 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:53:22,242 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:53:22,247 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:53:22,664 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:53:23,667 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:53:26,732 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:53:26,733 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:53:26,733 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:53:26,734 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:53:26,738 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:53:27,672 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:53:27,672 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:53:28,167 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:53:28,168 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:53:28,168 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:53:32,403 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:53:33,336 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:53:33,337 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:53:33,337 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:53:33,342 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:53:33,676 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:53:35,679 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:53:37,800 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:53:37,800 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:53:37,801 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:53:37,801 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:53:37,806 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:53:38,582 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 16:53:38,681 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:53:39,683 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:53:43,168 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:53:43,168 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:53:43,168 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:53:43,412 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:53:43,896 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:53:43,897 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:53:43,897 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:53:43,902 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:53:44,686 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:53:45,689 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:53:48,696 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:53:48,697 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:53:48,697 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:53:48,697 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:53:48,702 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:53:49,694 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:53:49,694 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:53:52,623 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:53:52,624 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:53:52,624 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:53:52,630 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:53:52,696 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:53:53,698 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:53:54,630 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:53:56,507 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:53:56,508 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:53:56,508 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:53:56,514 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:53:56,700 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:53:57,703 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:53:58,168 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:53:58,168 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:53:58,168 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:54:00,410 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:54:01,959 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:54:01,960 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:54:01,960 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:54:01,965 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:54:02,706 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:54:03,709 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:54:05,965 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:54:08,581 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 16:54:10,304 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:54:10,305 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:54:10,305 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:54:10,309 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:54:10,714 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:54:11,309 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:54:11,717 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:54:13,168 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:54:13,168 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:54:13,168 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:54:15,145 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:54:15,146 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:54:15,146 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:54:15,151 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:54:15,720 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:54:17,151 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:54:17,724 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:54:20,341 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:54:20,342 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:54:20,342 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:54:20,348 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:54:20,726 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:54:21,728 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:54:22,349 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:54:25,282 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:54:25,283 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:54:25,283 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:54:25,288 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:54:25,731 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:54:27,732 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:54:28,168 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:54:28,168 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:54:28,168 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:54:28,475 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:54:29,979 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:54:29,980 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:54:29,980 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:54:29,985 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:54:30,735 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:54:31,737 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:54:33,986 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:54:34,545 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:54:34,546 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:54:34,546 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:54:34,551 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:54:34,739 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:54:35,742 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:54:38,586 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 16:54:39,587 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:54:43,168 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:54:43,168 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:54:43,168 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:54:45,388 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:54:46,080 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:54:46,081 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:54:46,081 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:54:46,086 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:54:46,749 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:54:47,751 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:54:50,511 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:54:50,512 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:54:50,512 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:54:50,513 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:54:50,518 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:54:50,754 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:54:51,756 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:54:55,518 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:54:56,573 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:54:56,574 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:54:56,574 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:54:56,579 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:54:56,759 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:54:57,762 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:54:58,168 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:54:58,168 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:54:58,168 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:55:01,469 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:55:02,763 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:55:02,764 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:55:02,764 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:55:02,769 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:55:03,767 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:55:03,768 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:55:06,769 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:55:08,585 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 16:55:08,975 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:55:08,976 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:55:08,976 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:55:08,981 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:55:09,773 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:55:11,777 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:55:11,981 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:55:13,168 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:55:13,168 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:55:13,169 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:55:15,340 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:55:15,340 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:55:15,341 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:55:15,345 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:55:15,779 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:55:17,346 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:55:17,783 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:55:21,037 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:55:21,038 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:55:21,038 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:55:21,044 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:55:21,785 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:55:23,044 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:55:23,789 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:55:25,536 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:55:25,536 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:55:25,537 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:55:25,541 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:55:25,790 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:55:27,793 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:55:28,168 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:55:28,168 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:55:28,169 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:55:28,462 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:55:31,431 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:55:31,432 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:55:31,432 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:55:31,437 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:55:31,795 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:55:33,799 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:55:34,438 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:55:36,189 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:55:36,190 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:55:36,190 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:55:36,195 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:55:36,800 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:55:37,803 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:55:38,587 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 16:55:39,589 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:55:41,535 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:55:41,536 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:55:41,536 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:55:41,541 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:55:41,805 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:55:43,168 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:55:43,168 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:55:43,169 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:55:43,809 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:55:45,385 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:55:46,832 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:55:46,832 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:55:46,833 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:55:46,837 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:55:47,813 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:55:47,814 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:55:50,838 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:55:52,628 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:55:52,629 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:55:52,629 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:55:52,635 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:55:52,817 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:55:53,819 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:55:56,636 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:55:58,168 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:55:58,168 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:55:58,169 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:56:01,080 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:56:01,081 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:56:01,081 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:56:01,086 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:56:01,824 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:56:02,086 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:56:03,827 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:56:05,756 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:56:05,757 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:56:05,757 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:56:05,762 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:56:05,829 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:56:07,763 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:56:07,832 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:56:08,590 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 16:56:13,168 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:56:13,169 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:56:13,169 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:56:13,454 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:56:15,306 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:56:15,307 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:56:15,307 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:56:15,312 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:56:15,839 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:56:17,842 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:56:19,349 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:56:20,637 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:56:20,638 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:56:20,638 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:56:20,643 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:56:20,844 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:56:21,847 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:56:24,644 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:56:25,820 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:56:25,821 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:56:25,821 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:56:25,827 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:56:25,849 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:56:27,853 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:56:28,168 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:56:28,169 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:56:28,169 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:56:30,476 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:56:30,642 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:56:30,643 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:56:30,644 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:56:30,649 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:56:30,855 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:56:31,857 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:56:35,649 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:56:36,153 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:56:36,154 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:56:36,154 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:56:36,159 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:56:36,860 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:56:37,863 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:56:38,592 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 16:56:41,593 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:56:43,169 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:56:43,169 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:56:43,169 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:56:44,581 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:56:44,582 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:56:44,582 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:56:44,587 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:56:44,867 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:56:45,870 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:56:47,588 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:56:49,513 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:56:49,514 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:56:49,514 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:56:49,519 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:56:49,874 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:56:51,878 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:56:53,520 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:56:57,928 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:56:57,928 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:56:57,929 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:56:57,933 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:56:58,169 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:56:58,169 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:56:58,170 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:56:58,882 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:56:59,471 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:56:59,885 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:57:02,604 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:57:02,604 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:57:02,604 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:57:02,609 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:57:02,887 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:57:03,889 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:57:04,645 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:57:07,103 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:57:07,104 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:57:07,104 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:57:07,109 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:57:07,892 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:57:08,598 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 16:57:09,894 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:57:10,599 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:57:13,169 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:57:13,169 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:57:13,170 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:57:13,715 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:57:13,715 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:57:13,715 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:57:13,720 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:57:13,897 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:57:15,757 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:57:15,901 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:57:19,266 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:57:19,267 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:57:19,267 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:57:19,272 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:57:19,904 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:57:21,272 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:57:21,907 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:57:25,203 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:57:25,204 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:57:25,204 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:57:25,209 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:57:25,910 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:57:27,210 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:57:27,913 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:57:28,169 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:57:28,169 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:57:28,170 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:57:29,583 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:57:29,583 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:57:29,583 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:57:29,588 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:57:29,914 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:57:31,918 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:57:32,625 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:57:34,357 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:57:34,358 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:57:34,358 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:57:34,364 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:57:34,920 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:57:35,923 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:57:38,364 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:57:38,596 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 16:57:38,720 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:57:38,720 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:57:38,720 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:57:38,725 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:57:38,924 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:57:39,927 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:57:43,169 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:57:43,169 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:57:43,170 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:57:43,457 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:57:44,256 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:57:44,256 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:57:44,256 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:57:44,261 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:57:44,930 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:57:45,933 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:57:48,770 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:57:48,771 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:57:48,771 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:57:48,771 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:57:48,777 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:57:48,935 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:57:49,938 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:57:53,777 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:57:54,905 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:57:54,905 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:57:54,906 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:57:54,910 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:57:54,942 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:57:55,944 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:57:58,169 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:57:58,169 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:57:58,171 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:57:59,211 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:57:59,211 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:57:59,211 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:57:59,211 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:57:59,216 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:57:59,947 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:58:01,950 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:58:04,253 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:58:04,346 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:58:04,347 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:58:04,347 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:58:04,352 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:58:04,952 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:58:05,954 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:58:08,598 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 16:58:09,294 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:58:09,295 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:58:09,295 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:58:09,295 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:58:09,300 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:58:09,958 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:58:11,962 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:58:13,169 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:58:13,169 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:58:13,170 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:58:14,308 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:58:14,309 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:58:14,309 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:58:14,309 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:58:14,314 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:58:14,964 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:58:15,966 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:58:18,817 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:58:18,818 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:58:18,818 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:58:18,823 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:58:18,968 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:58:19,823 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:58:19,969 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:58:23,937 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:58:23,938 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:58:23,938 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:58:23,943 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:58:23,971 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:58:24,944 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:58:25,975 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:58:28,169 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:58:28,169 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:58:28,171 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:58:28,771 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:58:28,771 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:58:28,771 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:58:28,776 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:58:28,976 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:58:29,979 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:58:30,813 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:58:32,963 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:58:32,964 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:58:32,964 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:58:32,969 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:58:32,981 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:58:33,983 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:58:35,970 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:58:37,279 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:58:37,280 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:58:37,280 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:58:37,285 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:58:37,986 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:58:38,601 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 16:58:39,989 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:58:41,602 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:58:42,496 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:58:42,497 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:58:42,497 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:58:42,502 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:58:42,991 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:58:43,169 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:58:43,170 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:58:43,171 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:58:43,994 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:58:46,763 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:58:46,764 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:58:46,764 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:58:46,764 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:58:46,769 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:58:46,996 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:58:47,999 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:58:51,495 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:58:51,496 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:58:51,496 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:58:51,501 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:58:52,002 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:58:52,501 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:58:54,005 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:58:56,252 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:58:56,253 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:58:56,253 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:58:56,258 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:58:57,007 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:58:58,009 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:58:58,169 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:58:58,170 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:58:58,171 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:58:58,404 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:59:00,363 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:59:00,363 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:59:00,363 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:59:00,368 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:59:01,011 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:59:02,014 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:59:03,406 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:59:04,476 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:59:04,477 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:59:04,477 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:59:04,483 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:59:05,016 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:59:06,020 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:59:08,483 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:59:08,603 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 16:59:11,494 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:59:11,494 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:59:11,495 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:59:11,499 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:59:12,026 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:59:13,169 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:59:13,170 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:59:13,171 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:59:14,029 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:59:14,450 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:59:15,877 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:59:15,878 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:59:15,878 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:59:15,885 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:59:16,031 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:59:18,034 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:59:19,921 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:59:20,149 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:59:20,150 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:59:20,150 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:59:20,155 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:59:21,036 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:59:22,039 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:59:25,140 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:59:25,141 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:59:25,141 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:59:25,141 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:59:25,146 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:59:26,044 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:59:26,044 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:59:28,169 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:59:28,170 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:59:28,171 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:59:29,651 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:59:29,652 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:59:29,652 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:59:29,657 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:59:30,046 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:59:30,693 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:59:32,056 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:59:33,954 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:59:33,955 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:59:33,955 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:59:33,960 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:59:34,058 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:59:35,961 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:59:36,059 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:59:38,069 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:59:38,069 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:59:38,070 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:59:38,077 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:59:38,605 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 16:59:39,061 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:59:40,063 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:59:41,607 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:59:43,003 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:59:43,003 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:59:43,003 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:59:43,008 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:59:43,065 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:59:43,170 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:59:43,170 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:59:43,171 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:59:44,068 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:59:47,452 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:59:47,873 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:59:47,873 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:59:47,873 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:59:47,878 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:59:48,071 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:59:50,074 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:59:52,917 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:59:55,770 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:59:55,771 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 16:59:55,771 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:59:55,776 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:59:56,078 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 16:59:58,079 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 16:59:58,170 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:59:58,170 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:59:58,171 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:59:58,436 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:00:01,376 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:00:01,376 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:00:01,377 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:00:01,382 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:00:02,082 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:00:04,085 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:00:04,417 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:00:06,217 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:00:06,218 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:00:06,218 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:00:06,223 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:00:07,087 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:00:08,090 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:00:08,608 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 17:00:09,609 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:00:12,586 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:00:12,587 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:00:12,588 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:00:12,595 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:00:13,095 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:00:13,170 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:00:13,170 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:00:13,171 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:00:14,098 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:00:15,381 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:00:17,585 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:00:17,586 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:00:17,586 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:00:17,591 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:00:18,101 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:00:20,104 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:00:20,629 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:00:22,376 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:00:22,377 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:00:22,377 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:00:22,382 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:00:23,108 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:00:24,112 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:00:26,383 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:00:27,006 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:00:27,007 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:00:27,007 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:00:27,011 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:00:27,114 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:00:28,117 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:00:28,170 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:00:28,170 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:00:28,171 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:00:31,411 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:00:33,339 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:00:33,339 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:00:33,339 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:00:33,344 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:00:34,123 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:00:36,128 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:00:37,381 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:00:38,613 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 17:00:39,206 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:00:39,206 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:00:39,207 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:00:39,212 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:00:40,134 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:00:40,134 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:00:43,170 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:00:43,170 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:00:43,171 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:00:43,455 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:00:43,706 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:00:43,706 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:00:43,706 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:00:43,711 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:00:44,137 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:00:46,142 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:00:48,749 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:00:50,285 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:00:50,286 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:00:50,286 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:00:50,291 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:00:51,149 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:00:52,152 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:00:54,292 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:00:58,170 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:00:58,170 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:00:58,171 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:00:58,569 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:00:58,570 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:00:58,570 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:00:58,575 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:00:59,160 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:00:59,613 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:01:00,162 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:01:03,449 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:01:03,450 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:01:03,450 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:01:03,455 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:01:04,166 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:01:05,455 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:01:06,171 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:01:07,752 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:01:07,753 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:01:07,753 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:01:07,758 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:01:08,172 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:01:08,612 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 17:01:10,178 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:01:10,613 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:01:12,543 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:01:12,544 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:01:12,544 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:01:12,549 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:01:13,170 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:01:13,170 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:01:13,172 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:01:13,181 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:01:14,183 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:01:16,390 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:01:17,609 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:01:17,610 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:01:17,610 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:01:17,615 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:01:18,186 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:01:20,190 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:01:21,653 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:01:22,786 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:01:22,787 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:01:22,787 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:01:22,792 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:01:23,194 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:01:24,198 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:01:26,792 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:01:27,300 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:01:27,300 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:01:27,301 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:01:27,306 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:01:28,170 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:01:28,170 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:01:28,172 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:01:28,203 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:01:28,203 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:01:32,405 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:01:32,757 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:01:32,758 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:01:32,758 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:01:32,763 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:01:33,208 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:01:34,211 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:01:37,801 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:01:38,621 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 17:01:38,687 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:01:38,688 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:01:38,688 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:01:38,693 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:01:39,217 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:01:40,219 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:01:43,170 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:01:43,170 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:01:43,172 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:01:43,388 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:01:43,430 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:01:43,431 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:01:43,431 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:01:43,437 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:01:44,224 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:01:46,229 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:01:47,970 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:01:47,971 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:01:47,971 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:01:47,976 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:01:48,230 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:01:48,977 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:01:50,235 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:01:51,777 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:01:51,778 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:01:51,778 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:01:51,783 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:01:52,236 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:01:54,240 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:01:54,784 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:01:56,059 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:01:56,060 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:01:56,060 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:01:56,065 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:01:56,242 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:01:58,170 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:01:58,171 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:01:58,172 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:01:58,247 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:02:00,398 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:02:01,332 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:02:01,333 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:02:01,333 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:02:01,338 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:02:02,252 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:02:02,252 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:02:06,052 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:02:06,053 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:02:06,053 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:02:06,053 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:02:06,059 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:02:06,256 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:02:08,261 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:02:08,617 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 17:02:10,303 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:02:10,304 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:02:10,304 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:02:10,309 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:02:11,265 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:02:11,310 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:02:12,268 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:02:13,170 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:02:13,171 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:02:13,172 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:02:15,812 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:02:15,813 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:02:15,813 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:02:15,818 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:02:16,271 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:02:16,853 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:02:18,275 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:02:21,854 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:02:22,552 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:02:22,552 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:02:22,553 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:02:22,558 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:02:23,282 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:02:24,285 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:02:27,559 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:02:28,159 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:02:28,160 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:02:28,160 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:02:28,165 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:02:28,170 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:02:28,171 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:02:28,172 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:02:28,288 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:02:30,293 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:02:33,423 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:02:33,974 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:02:33,974 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:02:33,975 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:02:33,980 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:02:34,297 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:02:36,302 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:02:38,619 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 17:02:38,621 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:02:39,596 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:02:39,597 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:02:39,597 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:02:39,602 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:02:40,305 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:02:42,310 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:02:43,170 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:02:43,171 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:02:43,172 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:02:44,400 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:02:44,438 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:02:44,438 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:02:44,439 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:02:44,444 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:02:45,313 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:02:46,317 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:02:48,808 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:02:48,809 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:02:48,809 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:02:48,814 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:02:49,319 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:02:49,815 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:02:50,322 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:02:53,474 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:02:53,474 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:02:53,475 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:02:53,480 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:02:54,326 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:02:55,480 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:02:56,331 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:02:58,171 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:02:58,171 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:02:58,171 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:03:01,462 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:03:02,039 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:03:02,039 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:03:02,040 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:03:02,044 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:03:02,338 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:03:04,343 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:03:07,082 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:03:07,534 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:03:07,535 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:03:07,535 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:03:07,540 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:03:08,346 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:03:08,622 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 17:03:10,354 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:03:12,194 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:03:12,195 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:03:12,195 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:03:12,196 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:03:12,201 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:03:12,356 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:03:13,171 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:03:13,171 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:03:13,172 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:03:14,359 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:03:17,408 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:03:17,801 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:03:17,802 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:03:17,802 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:03:17,807 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:03:18,362 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:03:20,366 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:03:22,846 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:03:24,269 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:03:24,271 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:03:24,271 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:03:24,276 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:03:24,370 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:03:26,375 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:03:28,171 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:03:28,171 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:03:28,171 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:03:28,410 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:03:29,192 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:03:29,193 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:03:29,193 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:03:29,199 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:03:29,378 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:03:30,381 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:03:34,234 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:03:35,973 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:03:35,974 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:03:35,974 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:03:35,980 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:03:36,387 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:03:38,392 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:03:38,624 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 17:03:39,626 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:03:42,005 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:03:42,006 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:03:42,006 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:03:42,011 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:03:42,396 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:03:43,171 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:03:43,171 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:03:43,171 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:03:44,401 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:03:45,444 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:03:46,992 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:03:46,993 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:03:46,993 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:03:46,998 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:03:47,404 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:03:48,407 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:03:51,034 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:03:53,012 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:03:53,013 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:03:53,013 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:03:53,018 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:03:53,413 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:03:54,415 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:03:57,019 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:03:58,171 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:03:58,171 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:03:58,173 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:04:01,417 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:04:01,418 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:04:01,418 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:04:01,424 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:04:01,425 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:04:02,426 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:04:02,461 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:04:05,799 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:04:05,800 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:04:05,800 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:04:05,806 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:04:06,432 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:04:07,806 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:04:08,436 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:04:08,629 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 17:04:11,004 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:04:11,005 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:04:11,005 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:04:11,010 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:04:11,440 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:04:12,443 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:04:13,011 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:04:13,171 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:04:13,171 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:04:13,173 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:04:18,470 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:04:18,481 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:04:18,481 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:04:18,481 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:04:18,487 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:04:19,452 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:04:20,454 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:04:23,526 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:04:25,199 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:04:25,200 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:04:25,200 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:04:25,206 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:04:25,461 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:04:26,463 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:04:28,171 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:04:28,171 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:04:28,172 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:04:29,472 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:04:31,543 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:04:31,544 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:04:31,544 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:04:31,549 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:04:32,470 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:04:34,475 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:04:34,585 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:04:38,343 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:04:38,344 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:04:38,344 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:04:38,349 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:04:38,479 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:04:38,628 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 17:04:39,629 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:04:40,484 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:04:42,408 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:04:42,409 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:04:42,409 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:04:42,415 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:04:42,485 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:04:43,171 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:04:43,172 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:04:43,173 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:04:44,490 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:04:45,444 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:04:46,872 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:04:46,873 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:04:46,873 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:04:46,878 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:04:47,493 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:04:48,495 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:04:50,913 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:04:51,947 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:04:51,948 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:04:51,948 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:04:51,954 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:04:52,500 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:04:54,504 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:04:55,955 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:04:56,448 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:04:56,449 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:04:56,449 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:04:56,454 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:04:56,506 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:04:58,171 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:04:58,171 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:04:58,172 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:04:58,510 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:05:01,410 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:05:02,144 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:05:02,145 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:05:02,145 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:05:02,151 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:05:02,514 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:05:04,519 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:05:06,831 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:05:06,832 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:05:06,832 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:05:06,833 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:05:06,838 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:05:07,522 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:05:08,525 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:05:08,633 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 17:05:11,226 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:05:11,226 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:05:11,227 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:05:11,232 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:05:11,528 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:05:12,233 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:05:12,532 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:05:13,171 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:05:13,172 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:05:13,173 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:05:17,460 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:05:18,214 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:05:18,215 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:05:18,215 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:05:18,220 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:05:18,537 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:05:20,542 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:05:23,258 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:05:27,182 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:05:27,183 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:05:27,183 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:05:27,189 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:05:27,551 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:05:28,171 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:05:28,172 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:05:28,173 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:05:28,456 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:05:28,554 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:05:31,295 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:05:31,295 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:05:31,296 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:05:31,301 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:05:31,557 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:05:32,558 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:05:34,337 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:05:35,531 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:05:35,531 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:05:35,532 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:05:35,537 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:05:35,563 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:05:36,565 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:05:38,632 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 17:05:39,634 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:05:40,550 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:05:40,551 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:05:40,551 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:05:40,556 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:05:40,569 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:05:42,575 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:05:43,171 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:05:43,172 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:05:43,172 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:05:45,394 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:05:46,352 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:05:46,353 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:05:46,353 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:05:46,358 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:05:46,578 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:05:48,583 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:05:50,871 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:05:50,872 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:05:50,872 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:05:50,872 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:05:50,878 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:05:51,587 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:05:52,590 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:05:55,879 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:05:56,635 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:05:56,636 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:05:56,637 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:05:56,642 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:05:57,595 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:05:58,172 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:05:58,172 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:05:58,173 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:05:58,598 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:06:01,443 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:06:02,021 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:06:02,021 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:06:02,022 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:06:02,027 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:06:02,601 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:06:04,606 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:06:06,834 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:06:06,835 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:06:06,836 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:06:06,836 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:06:06,841 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:06:07,610 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:06:08,613 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:06:08,634 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 17:06:11,325 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:06:11,326 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:06:11,326 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:06:11,331 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:06:11,617 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:06:12,332 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:06:12,620 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:06:13,172 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:06:13,172 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:06:13,172 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:06:16,431 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:06:16,432 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:06:16,432 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:06:16,437 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:06:16,623 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:06:17,473 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:06:18,625 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:06:22,014 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:06:22,015 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:06:22,015 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:06:22,020 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:06:22,630 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:06:23,020 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:06:24,636 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:06:26,541 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:06:26,542 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:06:26,542 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:06:26,547 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:06:26,637 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:06:28,172 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:06:28,172 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:06:28,173 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:06:28,452 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:06:28,642 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:06:33,426 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:06:33,427 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:06:33,427 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:06:33,432 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:06:33,469 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:06:33,647 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:06:34,650 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:06:37,734 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:06:37,735 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:06:37,735 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:06:37,741 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:06:38,637 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 17:06:38,638 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:06:38,654 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:06:40,659 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:06:42,390 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:06:42,391 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:06:42,391 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:06:42,396 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:06:42,660 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:06:43,172 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:06:43,172 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:06:43,172 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:06:44,437 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:06:44,665 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:06:47,909 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:06:47,909 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:06:47,910 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:06:47,915 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:06:48,668 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:06:49,953 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:06:50,678 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:06:52,533 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:06:52,534 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:06:52,534 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:06:52,539 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:06:52,679 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:06:54,683 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:06:55,539 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:06:57,223 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:06:57,223 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:06:57,224 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:06:57,228 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:06:57,685 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:06:58,172 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:06:58,172 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:06:58,172 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:06:58,687 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:07:01,442 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:07:04,005 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:07:04,006 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:07:04,006 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:07:04,011 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:07:04,692 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:07:06,695 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:07:07,049 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:07:08,430 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:07:08,431 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:07:08,431 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:07:08,436 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:07:08,639 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 17:07:08,698 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:07:10,704 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:07:12,478 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:07:12,479 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:07:12,479 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:07:12,479 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:07:12,485 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:07:12,706 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:07:13,172 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:07:13,172 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:07:13,172 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:07:14,709 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:07:17,332 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:07:17,333 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:07:17,333 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:07:17,338 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:07:17,712 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:07:18,373 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:07:18,715 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:07:23,374 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:07:23,820 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:07:23,820 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:07:23,821 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:07:23,826 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:07:24,722 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:07:26,726 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:07:28,126 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:07:28,127 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:07:28,127 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:07:28,132 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:07:28,172 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:07:28,172 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:07:28,173 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:07:28,379 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:07:28,727 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:07:30,732 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:07:33,104 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:07:33,104 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:07:33,105 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:07:33,110 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:07:33,735 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:07:34,145 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:07:34,738 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:07:37,422 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:07:37,423 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:07:37,423 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:07:37,428 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:07:37,741 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:07:38,645 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 17:07:38,743 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:07:39,647 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:07:42,245 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:07:42,246 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:07:42,246 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:07:42,253 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:07:42,747 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:07:43,172 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:07:43,172 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:07:43,174 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:07:44,750 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:07:45,416 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:07:48,859 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:07:48,860 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:07:48,860 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:07:48,865 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:07:49,758 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:07:50,761 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:07:50,901 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:07:53,332 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:07:53,333 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:07:53,334 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:07:53,339 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:07:53,764 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:07:54,767 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:07:56,340 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:07:58,172 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:07:58,172 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:07:58,174 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:07:59,670 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:07:59,670 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:07:59,671 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:07:59,675 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:07:59,773 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:08:00,776 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:08:01,713 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:08:04,585 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:08:04,586 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:08:04,586 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:08:04,591 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:08:04,779 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:08:06,782 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:08:07,592 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:08:08,643 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 17:08:09,085 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:08:09,086 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:08:09,086 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:08:09,091 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:08:09,788 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:08:10,792 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:08:13,092 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:08:13,172 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:08:13,172 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:08:13,174 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:08:13,552 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:08:13,553 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:08:13,553 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:08:13,558 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:08:13,794 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:08:14,797 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:08:18,593 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:08:21,641 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:08:21,642 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:08:21,642 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:08:21,647 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:08:21,802 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:08:22,805 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:08:23,648 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:08:25,466 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:08:25,467 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:08:25,467 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:08:25,472 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:08:25,811 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:08:26,813 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:08:28,172 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:08:28,173 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:08:28,173 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:08:29,467 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:08:32,285 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:08:32,286 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:08:32,286 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:08:32,291 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:08:32,820 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:08:34,825 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:08:35,329 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:08:37,031 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:08:37,032 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:08:37,032 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:08:37,037 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:08:37,828 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:08:38,649 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 17:08:38,831 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:08:40,650 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:08:42,468 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:08:42,468 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:08:42,469 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:08:42,474 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:08:42,834 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:08:43,172 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:08:43,173 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:08:43,173 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:08:44,839 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:08:46,441 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:08:49,237 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:08:49,238 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:08:49,238 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:08:49,243 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:08:49,845 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:08:50,848 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:08:52,281 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:08:53,711 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:08:53,711 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:08:53,712 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:08:53,717 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:08:53,852 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:08:54,852 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:08:57,718 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:08:58,172 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:08:58,173 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:08:58,174 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:08:58,827 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:08:58,828 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:08:58,828 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:08:58,833 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:08:58,858 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:09:00,863 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:09:02,869 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:09:03,285 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:09:03,286 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:09:03,286 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:09:03,291 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:09:03,866 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:09:04,869 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:09:08,292 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:09:08,648 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 17:09:12,426 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:09:12,427 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:09:12,427 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:09:12,432 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:09:12,879 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:09:13,173 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:09:13,173 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:09:13,173 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:09:13,443 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:09:14,883 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:09:18,156 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:09:18,157 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:09:18,157 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:09:18,162 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:09:18,886 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:09:19,197 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:09:20,891 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:09:23,171 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:09:23,172 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:09:23,172 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:09:23,177 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:09:23,896 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:09:24,898 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:09:25,178 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:09:28,173 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:09:28,173 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:09:28,173 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:09:30,381 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:09:30,390 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:09:30,390 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:09:30,391 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:09:30,396 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:09:30,905 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:09:32,910 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:09:35,434 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:09:35,881 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:09:35,882 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:09:35,882 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:09:35,888 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:09:35,913 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:09:36,915 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:09:38,650 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 17:09:40,651 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:09:41,047 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:09:41,047 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:09:41,048 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:09:41,053 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:09:41,922 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:09:42,924 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:09:43,173 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:09:43,173 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:09:43,173 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:09:45,872 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:09:45,873 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:09:45,873 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:09:45,874 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:09:45,879 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:09:45,927 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:09:48,933 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:09:50,918 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:09:51,083 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:09:51,084 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:09:51,085 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:09:51,089 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:09:51,937 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:09:52,939 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:09:56,090 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:09:56,807 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:09:56,807 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:09:56,808 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:09:56,813 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:09:56,943 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:09:58,325 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:09:58,325 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:09:58,325 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:09:58,948 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:10:01,626 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:10:02,267 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:10:02,268 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:10:02,268 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:10:02,273 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:10:02,952 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:10:04,957 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:10:07,310 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:10:08,652 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 17:10:10,789 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:10:10,790 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:10:10,790 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:10:10,795 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:10:10,964 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:10:12,796 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:10:12,967 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:10:13,325 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:10:13,325 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:10:13,325 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:10:15,653 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:10:15,654 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:10:15,654 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:10:15,659 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:10:15,972 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:10:16,975 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:10:18,697 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:10:21,075 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:10:21,076 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:10:21,076 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:10:21,082 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:10:21,978 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:10:22,981 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:10:24,082 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:10:25,521 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:10:25,522 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:10:25,522 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:10:25,528 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:10:25,983 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:10:26,986 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:10:28,324 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:10:28,325 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:10:28,325 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:10:29,526 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:10:30,236 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:10:30,237 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:10:30,237 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:10:30,243 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:10:30,988 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:10:32,991 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:10:35,277 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:10:35,694 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:10:35,694 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:10:35,695 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:10:35,701 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:10:35,993 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:10:36,996 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:10:38,654 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 17:10:40,655 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:10:42,639 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:10:42,640 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:10:42,641 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:10:42,649 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:10:43,000 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:10:43,325 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:10:43,325 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:10:43,325 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:10:45,003 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:10:46,631 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:10:49,611 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:10:49,612 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:10:49,612 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:10:49,617 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:10:50,006 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:10:51,009 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:10:51,653 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:10:55,549 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:10:55,550 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:10:55,550 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:10:55,556 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:10:56,012 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:10:57,015 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:10:57,557 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:10:58,325 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:10:58,325 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:10:58,325 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:11:00,787 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:11:00,787 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:11:00,788 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:11:00,793 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:11:01,018 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:11:02,829 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:11:03,021 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:11:05,367 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:11:05,368 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:11:05,369 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:11:05,380 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:11:06,023 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:11:07,025 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:11:08,380 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:11:08,659 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 17:11:09,793 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:11:09,794 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:11:09,795 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:11:09,800 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:11:10,027 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:11:11,031 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:11:13,325 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:11:13,325 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:11:13,325 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:11:13,610 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:11:15,786 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:11:15,786 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:11:15,787 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:11:15,791 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:11:16,034 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:11:17,037 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:11:18,829 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:11:21,528 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:11:21,529 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:11:21,529 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:11:21,534 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:11:22,040 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:11:23,043 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:11:24,535 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:11:26,310 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:11:26,311 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:11:26,311 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:11:26,316 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:11:27,046 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:11:28,325 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:11:28,325 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:11:28,325 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:11:29,049 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:11:29,582 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:11:30,856 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:11:30,857 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:11:30,857 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:11:30,862 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:11:31,050 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:11:33,053 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:11:34,897 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:11:36,787 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:11:36,788 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:11:36,788 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:11:36,793 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:11:37,055 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:11:38,658 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 17:11:39,059 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:11:40,660 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:11:43,325 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:11:43,325 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:11:43,326 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:11:43,332 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:11:43,589 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:11:43,589 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:11:43,594 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:11:44,061 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:11:45,064 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:11:46,595 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:11:48,177 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:11:48,178 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:11:48,178 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:11:48,183 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:11:49,067 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:11:51,070 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:11:52,184 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:11:52,867 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:11:52,868 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:11:52,868 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:11:52,873 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:11:53,072 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:11:55,075 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:11:56,985 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:11:56,986 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:11:56,986 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:11:56,991 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:11:57,076 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:11:57,992 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:11:58,325 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:11:58,325 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:11:58,325 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:11:59,079 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:12:01,891 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:12:01,892 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:12:01,892 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:12:01,897 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:12:02,081 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:12:03,083 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:12:03,933 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:12:06,215 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:12:06,216 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:12:06,216 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:12:06,221 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:12:07,086 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:12:08,663 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 17:12:09,089 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:12:09,665 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:12:10,923 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:12:10,924 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:12:10,924 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:12:10,929 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:12:11,091 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:12:13,095 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:12:13,325 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:12:13,326 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:12:13,326 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:12:15,624 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:12:15,679 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:12:15,680 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:12:15,680 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:12:15,686 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:12:16,097 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:12:17,100 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:12:19,983 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:12:19,984 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:12:19,985 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:12:19,990 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:12:20,102 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:12:20,990 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:12:21,102 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:12:24,438 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:12:24,439 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:12:24,440 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:12:24,445 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:12:25,105 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:12:26,446 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:12:27,108 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:12:28,325 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:12:28,325 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:12:28,326 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:12:29,953 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:12:29,954 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:12:29,954 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:12:29,959 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:12:30,110 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:12:31,113 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:12:31,997 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:12:36,037 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:12:36,038 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:12:36,039 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:12:36,044 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:12:36,116 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:12:37,044 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:12:37,116 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:12:38,662 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 17:12:40,533 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:12:40,533 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:12:40,534 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:12:40,539 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:12:41,119 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:12:42,540 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:12:43,122 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:12:43,325 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:12:43,326 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:12:43,326 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:12:46,955 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:12:46,956 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:12:46,956 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:12:46,962 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:12:47,125 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:12:47,997 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:12:49,129 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:12:51,487 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:12:51,488 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:12:51,488 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:12:51,493 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:12:52,131 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:12:53,134 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:12:53,494 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:12:58,325 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:12:58,326 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:12:58,326 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:12:58,547 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:13:00,110 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:13:00,110 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:13:00,110 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:13:00,116 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:13:00,138 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:13:03,142 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:13:04,153 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:13:05,872 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:13:05,873 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:13:05,873 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:13:05,879 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:13:06,144 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:13:07,146 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:13:08,665 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 17:13:09,666 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:13:10,813 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:13:10,813 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:13:10,814 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:13:10,819 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:13:11,150 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:13:13,154 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:13:13,325 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:13:13,326 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:13:13,327 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:13:15,614 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:13:15,682 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:13:15,683 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:13:15,683 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:13:15,688 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:13:16,155 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:13:17,158 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:13:20,726 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:13:22,050 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:13:22,051 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:13:22,051 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:13:22,057 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:13:22,161 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:13:23,164 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:13:26,058 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:13:26,604 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:13:26,609 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:13:26,609 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:13:26,614 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:13:27,167 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:13:28,325 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:13:28,326 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:13:28,328 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:13:29,170 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:13:31,599 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:13:35,028 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:13:35,028 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:13:35,029 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:13:35,034 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:13:35,173 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:13:37,069 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:13:37,177 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:13:38,667 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 17:13:39,290 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:13:39,291 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:13:39,292 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:13:39,296 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:13:40,179 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:13:41,181 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:13:42,297 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:13:43,325 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:13:43,326 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:13:43,326 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:13:44,007 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:13:44,008 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:13:44,008 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:13:44,013 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:13:44,183 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:13:45,186 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:13:48,049 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:13:48,692 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:13:48,693 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:13:48,693 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:13:48,698 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:13:49,189 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:13:51,192 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:13:53,177 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:13:53,178 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:13:53,178 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:13:53,178 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:13:53,183 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:13:53,193 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:13:55,196 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:13:58,184 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:13:58,326 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:13:58,326 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:13:58,327 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:13:59,097 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:13:59,098 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:13:59,098 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:13:59,103 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:13:59,199 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:14:01,202 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:14:04,141 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:14:05,251 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:14:05,251 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:14:05,251 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:14:05,256 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:14:06,205 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:14:07,208 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:14:08,669 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 17:14:09,670 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:14:11,367 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:14:11,367 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:14:11,367 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:14:11,373 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:14:12,212 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:14:13,215 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:14:13,326 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:14:13,326 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:14:13,326 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:14:15,628 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:14:18,584 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:14:18,584 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:14:18,585 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:14:18,589 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:14:19,219 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:14:21,220 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:14:21,625 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:14:23,875 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:14:23,876 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:14:23,876 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:14:23,880 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:14:24,222 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:14:25,225 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:14:26,881 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:14:28,326 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:14:28,326 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:14:28,326 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:14:29,316 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:14:29,317 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:14:29,317 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:14:29,321 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:14:30,228 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:14:31,231 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:14:32,357 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:14:35,497 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:14:35,498 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:14:35,498 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:14:35,503 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:14:36,234 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:14:37,237 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:14:37,503 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:14:38,674 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 17:14:40,026 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:14:40,026 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:14:40,026 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:14:40,031 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:14:40,239 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:14:41,241 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:14:43,032 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:14:43,326 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:14:43,326 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:14:43,327 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:14:44,306 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:14:44,306 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:14:44,306 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:14:44,311 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:14:45,244 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:14:47,247 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:14:48,349 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:14:48,824 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:14:48,824 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:14:48,825 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:14:48,830 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:14:49,248 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:14:51,252 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:14:53,830 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:14:53,927 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:14:53,927 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:14:53,927 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:14:53,933 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:14:54,254 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:14:55,257 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:14:58,326 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:14:58,326 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:14:58,327 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:14:58,954 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:14:58,955 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:14:58,955 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:14:58,955 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:14:58,960 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:14:59,259 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:15:01,262 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:15:03,997 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:15:04,704 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:15:04,704 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:15:04,705 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:15:04,709 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:15:05,265 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:15:07,268 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:15:08,673 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 17:15:09,675 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:15:10,962 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:15:10,963 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:15:10,963 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:15:10,968 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:15:11,272 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:15:13,275 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:15:13,326 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:15:13,326 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:15:13,326 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:15:15,436 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:15:15,437 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:15:15,437 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:15:15,437 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:15:15,442 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:15:16,277 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:15:17,281 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:15:20,241 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:15:20,242 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:15:20,242 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:15:20,247 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:15:20,283 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:15:21,248 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:15:21,285 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:15:25,773 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:15:25,774 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:15:25,774 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:15:25,780 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:15:26,289 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:15:26,780 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:15:27,291 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:15:28,326 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:15:28,326 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:15:28,328 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:15:30,454 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:15:30,454 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:15:30,454 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:15:30,459 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:15:31,294 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:15:32,497 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:15:33,297 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:15:35,637 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:15:35,638 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:15:35,638 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:15:35,643 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:15:36,299 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:15:37,299 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:15:37,643 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:15:38,679 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 17:15:41,209 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:15:41,210 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:15:41,210 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:15:41,215 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:15:41,302 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:15:43,215 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:15:43,305 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:15:43,326 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:15:43,326 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:15:43,328 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:15:45,856 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:15:45,856 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:15:45,857 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:15:45,861 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:15:46,307 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:15:47,310 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:15:48,897 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:15:50,711 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:15:50,711 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:15:50,712 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:15:50,716 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:15:51,312 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:15:53,316 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:15:54,717 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:15:55,234 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:15:55,235 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:15:55,235 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:15:55,240 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:15:55,317 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:15:57,320 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:15:58,326 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:15:58,326 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:15:58,328 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:16:00,373 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:16:00,374 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:16:00,374 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:16:00,374 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:16:00,379 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:16:01,322 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:16:03,326 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:16:04,866 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:16:04,867 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:16:04,867 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:16:04,872 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:16:05,327 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:16:05,872 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:16:07,330 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:16:08,678 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 17:16:09,527 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:16:09,528 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:16:09,528 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:16:09,533 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:16:10,332 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:16:11,334 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:16:11,533 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:16:13,326 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:16:13,327 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:16:13,327 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:16:14,668 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:16:14,668 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:16:14,668 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:16:14,673 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:16:15,337 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:16:16,709 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:16:17,340 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:16:21,710 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:16:23,334 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:16:23,335 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:16:23,335 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:16:23,340 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:16:23,344 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:16:25,348 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:16:27,341 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:16:28,326 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:16:28,327 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:16:28,328 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:16:28,797 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:16:28,797 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:16:28,797 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:16:28,802 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:16:29,351 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:16:31,354 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:16:32,841 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:16:34,019 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:16:34,019 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:16:34,020 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:16:34,024 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:16:34,356 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:16:35,358 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:16:38,025 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:16:38,680 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 17:16:40,560 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:16:40,561 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:16:40,561 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:16:40,567 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:16:41,362 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:16:43,326 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:16:43,327 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:16:43,328 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:16:43,363 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:16:43,626 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:16:45,691 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:16:45,692 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:16:45,692 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:16:45,697 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:16:46,365 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:16:47,368 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:16:48,736 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:16:50,477 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:16:50,477 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:16:50,477 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:16:50,482 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:16:51,371 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:16:53,374 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:16:54,483 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:16:57,099 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:16:57,100 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:16:57,100 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:16:57,105 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:16:57,377 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:16:58,326 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:16:58,327 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:16:58,328 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:16:59,380 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:16:59,595 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:17:01,883 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:17:01,883 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:17:01,883 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:17:01,889 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:17:02,382 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:17:03,384 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:17:04,925 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:17:08,051 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:17:08,052 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:17:08,052 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:17:08,057 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:17:08,387 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:17:08,682 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 17:17:09,390 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:17:10,683 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:17:13,226 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:17:13,227 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:17:13,227 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:17:13,231 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:17:13,327 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:17:13,328 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:17:13,328 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:17:13,394 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:17:15,397 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:17:16,612 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:17:18,041 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:17:18,042 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:17:18,042 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:17:18,047 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:17:18,399 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:17:19,402 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:17:22,048 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:17:22,825 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:17:22,826 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:17:22,826 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:17:22,832 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:17:23,404 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:17:25,408 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:17:27,833 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:17:28,327 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:17:28,327 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:17:28,328 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:17:28,513 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:17:28,632 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:17:28,632 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:17:28,637 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:17:29,410 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:17:31,413 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:17:33,638 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:17:34,520 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:17:34,521 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:17:34,521 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:17:34,527 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:17:35,416 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:17:37,419 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:17:38,684 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 17:17:38,685 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:17:39,366 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:17:39,367 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:17:39,367 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:17:39,372 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:17:39,420 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:17:41,424 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:17:43,327 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:17:43,328 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:17:43,328 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:17:44,237 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:17:44,238 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:17:44,238 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:17:44,238 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:17:44,244 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:17:44,426 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:17:45,428 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:17:49,281 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:17:50,445 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:17:50,446 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:17:50,446 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:17:50,451 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:17:51,435 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:17:51,435 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:17:54,452 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:17:56,359 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:17:56,360 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:17:56,360 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:17:56,365 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:17:56,438 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:17:57,441 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:17:58,327 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:17:58,328 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:17:58,328 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:17:59,529 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:18:01,866 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:18:01,866 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:18:01,867 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:18:01,871 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:18:02,443 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:18:03,446 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:18:04,909 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:18:06,371 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:18:06,372 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:18:06,372 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:18:06,377 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:18:06,448 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:18:07,448 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:18:08,689 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 17:18:10,691 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:18:12,060 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:18:12,061 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:18:12,061 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:18:12,066 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:18:12,452 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:18:13,327 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:18:13,328 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:18:13,328 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:18:13,453 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:18:16,552 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:18:16,553 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:18:16,553 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:18:16,553 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:18:16,558 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:18:17,456 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:18:19,459 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:18:20,399 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:18:20,399 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:18:20,400 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:18:20,404 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:18:20,460 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:18:21,463 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:18:22,405 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:18:25,864 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:18:25,864 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:18:25,864 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:18:25,869 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:18:26,466 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:18:27,468 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:18:27,870 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:18:28,327 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:18:28,328 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:18:28,328 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:18:33,097 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:18:33,098 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:18:33,098 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:18:33,098 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:18:33,103 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:18:33,472 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:18:35,475 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:18:37,651 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:18:37,652 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:18:37,652 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:18:37,657 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:18:38,477 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:18:38,657 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:18:38,689 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 17:18:39,479 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:18:41,756 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:18:41,756 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:18:41,757 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:18:41,761 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:18:42,481 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:18:43,327 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:18:43,328 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:18:43,328 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:18:43,484 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:18:44,642 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:18:45,796 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:18:45,797 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:18:45,797 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:18:45,802 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:18:46,485 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:18:47,489 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:18:49,837 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:18:51,210 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:18:51,210 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:18:51,210 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:18:51,215 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:18:51,494 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:18:53,497 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:18:55,216 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:18:55,461 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:18:55,462 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:18:55,462 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:18:55,467 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:18:55,498 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:18:57,502 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:18:58,327 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:18:58,328 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:18:58,328 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:19:00,633 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:19:05,018 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:19:05,019 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:19:05,019 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:19:05,024 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:19:05,506 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:19:06,061 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:19:07,509 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:19:08,694 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 17:19:10,405 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:19:10,405 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:19:10,405 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:19:10,410 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:19:10,511 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:19:11,410 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:19:11,513 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:19:13,327 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:19:13,328 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:19:13,328 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:19:14,492 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:19:14,492 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:19:14,492 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:19:14,497 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:19:14,515 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:19:15,518 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:19:16,533 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:19:19,643 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:19:19,644 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:19:19,644 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:19:19,649 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:19:20,521 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:19:21,524 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:19:21,650 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:19:24,157 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:19:24,158 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:19:24,158 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:19:24,163 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:19:24,525 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:19:25,528 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:19:27,163 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:19:28,328 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:19:28,328 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:19:28,328 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:19:28,465 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:19:28,632 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:19:28,632 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:19:28,637 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:19:29,531 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:19:31,534 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:19:32,638 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:19:33,334 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:19:33,334 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:19:33,335 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:19:33,339 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:19:33,535 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:19:35,538 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:19:38,340 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:19:38,393 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:19:38,394 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:19:38,394 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:19:38,399 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:19:38,540 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:19:38,693 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 17:19:39,542 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:19:43,298 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:19:43,299 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:19:43,299 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:19:43,303 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:19:43,328 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:19:43,328 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:19:43,329 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:19:43,545 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:19:43,582 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:19:45,548 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:19:48,583 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:19:49,069 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:19:49,070 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:19:49,070 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:19:49,075 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:19:49,550 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:19:51,554 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:19:53,764 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:19:53,765 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:19:53,765 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:19:53,765 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:19:53,770 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:19:54,556 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:19:55,558 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:19:58,328 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:19:58,328 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:19:58,328 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:19:59,594 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:20:00,768 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:20:00,769 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:20:00,769 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:20:00,774 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:20:01,562 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:20:03,563 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:20:04,809 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:20:05,238 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:20:05,238 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:20:05,239 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:20:05,243 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:20:05,564 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:20:07,567 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:20:08,695 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 17:20:09,674 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:20:09,675 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:20:09,675 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:20:09,681 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:20:10,569 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:20:10,681 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:20:11,573 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:20:13,328 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:20:13,328 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:20:13,328 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:20:16,619 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:20:18,185 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:20:18,186 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:20:18,186 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:20:18,192 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:20:18,578 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:20:19,580 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:20:22,233 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:20:23,262 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:20:23,263 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:20:23,263 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:20:23,268 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:20:23,583 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:20:25,586 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:20:27,269 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:20:28,328 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:20:28,328 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:20:28,329 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:20:31,460 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:20:31,461 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:20:31,461 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:20:31,466 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:20:31,590 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:20:32,501 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:20:33,591 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:20:36,144 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:20:36,145 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:20:36,145 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:20:36,150 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:20:36,593 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:20:37,595 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:20:38,150 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:20:38,697 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 17:20:42,059 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:20:42,060 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:20:42,060 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:20:42,066 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:20:42,598 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:20:43,328 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:20:43,328 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:20:43,328 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:20:43,585 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:20:43,600 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:20:46,089 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:20:46,090 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:20:46,090 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:20:46,095 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:20:46,602 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:20:47,605 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:20:49,133 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:20:51,710 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:20:51,711 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:20:51,711 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:20:51,716 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:20:52,609 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:20:53,611 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:20:54,716 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:20:57,302 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:20:57,303 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:20:57,303 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:20:57,308 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:20:57,613 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:20:58,328 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:20:58,328 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:20:58,329 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:20:59,617 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:21:00,559 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:21:01,688 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:21:01,689 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:21:01,689 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:21:01,694 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:21:02,618 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:21:03,621 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:21:05,729 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:21:08,700 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 17:21:10,162 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:21:10,163 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:21:10,163 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:21:10,168 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:21:10,625 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:21:11,168 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:21:11,629 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:21:13,328 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:21:13,328 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:21:13,329 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:21:16,612 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:21:16,613 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:21:16,613 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:21:16,614 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:21:16,618 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:21:16,632 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:21:17,635 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:21:21,313 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:21:21,314 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:21:21,314 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:21:21,319 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:21:21,638 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:21:22,320 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:21:23,641 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:21:27,320 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:21:27,378 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:21:27,378 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:21:27,379 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:21:27,384 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:21:27,643 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:21:28,328 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:21:28,329 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:21:28,329 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:21:29,646 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:21:32,565 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:21:35,855 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:21:35,855 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:21:35,855 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:21:35,860 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:21:36,651 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:21:37,653 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:21:37,897 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:21:38,705 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 17:21:41,361 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:21:41,361 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:21:41,362 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:21:41,367 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:21:41,655 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:21:43,328 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:21:43,329 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:21:43,329 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:21:43,586 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:21:43,656 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:21:46,752 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:21:46,753 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:21:46,753 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:21:46,758 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:21:47,659 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:21:48,793 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:21:49,662 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:21:52,190 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:21:52,191 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:21:52,191 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:21:52,196 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:21:52,664 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:21:53,667 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:21:54,196 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:21:57,313 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:21:57,314 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:21:57,314 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:21:57,318 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:21:57,669 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:21:58,328 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:21:58,329 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:21:58,329 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:21:59,547 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:21:59,670 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:22:02,792 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:22:02,793 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:22:02,793 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:22:02,798 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:22:03,673 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:22:04,833 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:22:05,676 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:22:07,318 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:22:07,319 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:22:07,319 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:22:07,324 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:22:07,677 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:22:08,704 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 17:22:09,681 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:22:10,705 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:22:11,638 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:22:11,639 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:22:11,639 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:22:11,644 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:22:11,683 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:22:13,328 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:22:13,329 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:22:13,329 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:22:13,687 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:22:16,104 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:22:16,105 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:22:16,105 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:22:16,105 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:22:16,110 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:22:16,688 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:22:17,692 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:22:21,145 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:22:22,245 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:22:22,245 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:22:22,245 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:22:22,251 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:22:22,695 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:22:23,697 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:22:26,251 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:22:26,354 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:22:26,354 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:22:26,355 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:22:26,360 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:22:26,699 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:22:27,700 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:22:28,329 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:22:28,329 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:22:28,329 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:22:30,904 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:22:30,905 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:22:30,905 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:22:30,910 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:22:31,702 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:22:31,945 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:22:33,706 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:22:36,283 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:22:36,284 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:22:36,284 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:22:36,289 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:22:36,708 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:22:37,290 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:22:37,710 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:22:38,710 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 17:22:41,281 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:22:41,282 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:22:41,282 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:22:41,287 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:22:41,713 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:22:43,287 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:22:43,329 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:22:43,329 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:22:43,330 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:22:43,716 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:22:47,001 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:22:47,002 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:22:47,002 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:22:47,007 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:22:47,719 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:22:49,045 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:22:49,722 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:22:54,046 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:22:54,172 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:22:54,173 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:22:54,173 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:22:54,178 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:22:54,726 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:22:55,728 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:22:58,329 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:22:58,329 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:22:58,329 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:22:58,981 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:22:58,981 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:22:58,981 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:22:58,986 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:22:59,731 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:23:00,025 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:23:01,734 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:23:04,396 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:23:04,397 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:23:04,397 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:23:04,402 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:23:04,736 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:23:05,402 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:23:05,738 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:23:08,708 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 17:23:09,785 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:23:09,786 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:23:09,786 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:23:09,791 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:23:10,741 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:23:10,791 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:23:11,746 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:23:13,329 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:23:13,329 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:23:13,329 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:23:16,546 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:23:17,041 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:23:17,041 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:23:17,042 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:23:17,046 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:23:17,750 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:23:19,753 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:23:22,085 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:23:22,472 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:23:22,472 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:23:22,473 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:23:22,478 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:23:22,755 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:23:23,758 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:23:27,478 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:23:28,329 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:23:28,329 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:23:28,330 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:23:30,824 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:23:30,824 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:23:30,825 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:23:30,829 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:23:31,763 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:23:32,865 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:23:33,766 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:23:37,245 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:23:37,246 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:23:37,246 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:23:37,251 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:23:37,769 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:23:38,252 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:23:38,710 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 17:23:39,772 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:23:43,329 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:23:43,329 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:23:43,330 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:23:43,578 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:23:43,584 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:23:43,584 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:23:43,585 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:23:43,590 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:23:43,775 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:23:45,778 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:23:48,030 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:23:48,031 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:23:48,031 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:23:48,036 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:23:48,780 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:23:49,073 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:23:49,783 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:23:52,852 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:23:52,852 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:23:52,853 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:23:52,857 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:23:53,788 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:23:54,858 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:23:55,791 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:23:57,435 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:23:57,436 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:23:57,436 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:23:57,441 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:23:57,792 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:23:58,329 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:23:58,329 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:23:58,330 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:23:59,796 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:24:00,551 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:24:02,671 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:24:02,674 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:24:02,675 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:24:02,680 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:24:02,798 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:24:03,800 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:24:05,713 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:24:07,194 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:24:07,195 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:24:07,195 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:24:07,200 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:24:07,803 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:24:08,713 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 17:24:09,806 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:24:10,714 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:24:11,414 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:24:11,415 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:24:11,416 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:24:11,420 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:24:11,809 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:24:13,329 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:24:13,330 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:24:13,330 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:24:13,813 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:24:15,566 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:24:15,567 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:24:15,567 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:24:15,572 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:24:15,815 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:24:16,609 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:24:17,819 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:24:21,610 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:24:24,177 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:24:24,177 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:24:24,178 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:24:24,182 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:24:24,823 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:24:25,826 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:24:27,183 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:24:28,329 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:24:28,329 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:24:28,331 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:24:28,697 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:24:28,697 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:24:28,697 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:24:28,702 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:24:28,828 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:24:29,830 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:24:32,741 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:24:33,299 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:24:33,300 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:24:33,300 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:24:33,305 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:24:33,833 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:24:35,836 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:24:37,549 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:24:37,550 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:24:37,550 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:24:37,555 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:24:37,837 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:24:38,555 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:24:38,715 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 17:24:39,841 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:24:42,016 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:24:42,017 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:24:42,017 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:24:42,022 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:24:42,842 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:24:43,330 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:24:43,330 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:24:43,331 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:24:43,611 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:24:43,843 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:24:48,110 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:24:48,111 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:24:48,111 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:24:48,116 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:24:48,847 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:24:49,117 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:24:49,850 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:24:52,520 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:24:52,521 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:24:52,521 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:24:52,526 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:24:52,852 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:24:53,854 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:24:54,527 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:24:58,329 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:24:58,330 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:24:58,331 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:24:59,603 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:25:01,615 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:25:01,615 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:25:01,615 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:25:01,621 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:25:01,859 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:25:03,862 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:25:04,657 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:25:06,056 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:25:06,057 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:25:06,057 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:25:06,062 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:25:06,864 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:25:07,866 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:25:08,721 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 17:25:09,722 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:25:10,193 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:25:10,194 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:25:10,194 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:25:10,199 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:25:10,868 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:25:11,872 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:25:13,329 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:25:13,330 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:25:13,330 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:25:14,517 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:25:14,517 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:25:14,518 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:25:14,522 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:25:14,874 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:25:15,561 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:25:15,876 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:25:20,562 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:25:22,546 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:25:22,547 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:25:22,547 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:25:22,552 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:25:22,881 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:25:23,884 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:25:26,553 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:25:28,330 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:25:28,330 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:25:28,331 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:25:31,119 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:25:31,120 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:25:31,120 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:25:31,125 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:25:31,889 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:25:32,161 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:25:33,892 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:25:35,550 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:25:35,550 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:25:35,551 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:25:35,556 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:25:35,893 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:25:37,556 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:25:37,894 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:25:38,719 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 17:25:40,652 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:25:40,653 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:25:40,653 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:25:40,658 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:25:40,896 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:25:41,899 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:25:42,658 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:25:43,330 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:25:43,330 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:25:43,331 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:25:44,917 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:25:44,917 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:25:44,918 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:25:44,922 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:25:45,903 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:25:45,903 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:25:47,961 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:25:49,372 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:25:49,373 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:25:49,373 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:25:49,378 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:25:49,906 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:25:51,909 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:25:53,378 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:25:56,356 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:25:56,356 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:25:56,357 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:25:56,362 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:25:56,912 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:25:57,915 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:25:58,330 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:25:58,330 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:25:58,330 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:25:58,583 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:26:01,009 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:26:01,009 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:26:01,009 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:26:01,014 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:26:01,917 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:26:03,920 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:26:04,053 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:26:05,493 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:26:05,494 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:26:05,494 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:26:05,499 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:26:05,922 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:26:07,925 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:26:08,724 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 17:26:09,726 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:26:10,877 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:26:10,878 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:26:10,878 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:26:10,883 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:26:10,926 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:26:11,930 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:26:13,330 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:26:13,330 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:26:13,330 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:26:15,623 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:26:16,053 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:26:16,054 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:26:16,054 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:26:16,059 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:26:16,934 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:26:17,937 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:26:20,515 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:26:20,515 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:26:20,516 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:26:20,521 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:26:20,939 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:26:21,522 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:26:21,941 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:26:26,522 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:26:28,330 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:26:28,330 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:26:28,330 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:26:28,861 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:26:28,861 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:26:28,862 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:26:28,867 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:26:28,945 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:26:29,948 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:26:31,905 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:26:33,272 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:26:33,272 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:26:33,273 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:26:33,279 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:26:33,950 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:26:35,954 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:26:37,280 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:26:38,724 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 17:26:39,839 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:26:39,839 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:26:39,840 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:26:39,845 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:26:39,956 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:26:41,959 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:26:42,845 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:26:43,330 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:26:43,330 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:26:43,332 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:26:44,634 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:26:44,635 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:26:44,635 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:26:44,640 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:26:44,961 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:26:45,963 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:26:48,677 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:26:49,280 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:26:49,281 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:26:49,281 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:26:49,286 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:26:49,966 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:26:51,970 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:26:54,287 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:26:54,654 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:26:54,654 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:26:54,655 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:26:54,660 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:26:54,971 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:26:55,974 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:26:58,330 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:26:58,331 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:26:58,331 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:26:59,542 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:27:00,131 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:27:00,132 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:27:00,132 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:27:00,137 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:27:00,977 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:27:01,979 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:27:05,174 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:27:08,672 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:27:08,673 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:27:08,673 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:27:08,678 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:27:08,726 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 17:27:08,983 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:27:09,986 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:27:10,728 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:27:13,330 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:27:13,330 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:27:13,332 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:27:14,899 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:27:14,900 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:27:14,900 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:27:14,905 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:27:14,990 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:27:15,941 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:27:15,991 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:27:19,359 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:27:19,360 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:27:19,360 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:27:19,365 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:27:19,994 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:27:21,366 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:27:21,997 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:27:24,742 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:27:24,742 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:27:24,743 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:27:24,748 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:27:24,999 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:27:26,002 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:27:26,748 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:27:28,330 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:27:28,331 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:27:28,332 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:27:29,307 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:27:29,307 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:27:29,308 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:27:29,312 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:27:30,005 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:27:32,006 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:27:32,349 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:27:33,438 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:27:33,439 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:27:33,439 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:27:33,445 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:27:34,007 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:27:36,010 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:27:37,446 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:27:38,728 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 17:27:39,036 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:27:39,037 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:27:39,037 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:27:39,042 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:27:40,014 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:27:40,014 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:27:42,910 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:27:42,910 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:27:42,911 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:27:42,911 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:27:42,916 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:27:43,016 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:27:43,330 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:27:43,331 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:27:43,332 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:27:44,018 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:27:48,556 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:27:49,691 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:27:49,691 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:27:49,691 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:27:49,696 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:27:50,022 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:27:52,026 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:27:53,733 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:27:55,914 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:27:55,914 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:27:55,915 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:27:55,920 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:27:56,028 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:27:58,031 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:27:58,330 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:27:58,331 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:27:58,332 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:27:59,538 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:28:02,634 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:28:02,635 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:28:02,635 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:28:02,640 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:28:03,034 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:28:04,037 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:28:04,677 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:28:08,730 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 17:28:08,847 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:28:08,848 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:28:08,848 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:28:08,853 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:28:09,040 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:28:09,853 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:28:10,041 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:28:13,119 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:28:13,120 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:28:13,120 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:28:13,125 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:28:13,331 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:28:13,331 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:28:13,331 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:28:14,046 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:28:14,046 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:28:15,616 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:28:17,156 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:28:17,157 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:28:17,157 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:28:17,162 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:28:18,049 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:28:20,052 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:28:21,197 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:28:22,586 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:28:22,587 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:28:22,587 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:28:22,592 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:28:23,054 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:28:24,056 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:28:26,593 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:28:27,932 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:28:27,932 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:28:27,933 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:28:27,938 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:28:28,059 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:28:28,330 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:28:28,331 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:28:28,332 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:28:30,062 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:28:32,570 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:28:33,783 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:28:33,783 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:28:33,784 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:28:33,788 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:28:34,065 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:28:36,068 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:28:37,826 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:28:38,736 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 17:28:42,230 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:28:42,231 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:28:42,231 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:28:42,236 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:28:43,071 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:28:43,236 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:28:43,331 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:28:43,331 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:28:43,333 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:28:44,074 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:28:48,578 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:28:50,531 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:28:50,531 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:28:50,532 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:28:50,537 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:28:51,078 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:28:52,081 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:28:54,574 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:28:56,625 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:28:56,626 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:28:56,626 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:28:56,631 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:28:57,084 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:28:58,087 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:28:58,331 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:28:58,331 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:28:58,332 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:28:59,579 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:29:01,261 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:29:01,262 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:29:01,262 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:29:01,267 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:29:02,089 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:29:04,092 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:29:05,305 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:29:05,427 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:29:05,428 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:29:05,428 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:29:05,434 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:29:06,094 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:29:08,097 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:29:08,735 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 17:29:09,511 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:29:09,512 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:29:09,513 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:29:09,518 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:29:10,098 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:29:10,519 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:29:12,103 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:29:13,331 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:29:13,331 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:29:13,333 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:29:13,793 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:29:13,794 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:29:13,794 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:29:13,799 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:29:14,104 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:29:15,837 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:29:16,107 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:29:20,838 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:29:22,258 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:29:22,259 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:29:22,259 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:29:22,264 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:29:23,112 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:29:24,115 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:29:26,265 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:29:26,521 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:29:26,522 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:29:26,522 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:29:26,527 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:29:27,117 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:29:28,119 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:29:28,331 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:29:28,331 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:29:28,332 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:29:31,616 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:29:34,669 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:29:34,670 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:29:34,670 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:29:34,675 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:29:35,124 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:29:36,126 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:29:36,713 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:29:38,740 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 17:29:39,127 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:29:39,128 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:29:39,129 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:29:39,134 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:29:40,132 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:29:40,133 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:29:42,134 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:29:43,331 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:29:43,331 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:29:43,333 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:29:47,635 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:29:48,061 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:29:48,062 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:29:48,062 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:29:48,068 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:29:48,137 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:29:50,140 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:29:52,457 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:29:52,458 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:29:52,458 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:29:52,463 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:29:53,142 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:29:53,464 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:29:54,148 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:29:56,858 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:29:56,859 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:29:56,859 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:29:56,864 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:29:57,150 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:29:58,153 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:29:58,331 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:29:58,331 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:29:58,333 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:29:58,597 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:30:01,746 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:30:01,747 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:30:01,747 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:30:01,752 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:30:02,155 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:30:03,789 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:30:04,156 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:30:06,607 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:30:06,608 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:30:06,608 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:30:06,613 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:30:07,158 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:30:08,160 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:30:08,739 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 17:30:09,740 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:30:11,176 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:30:11,177 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:30:11,177 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:30:11,182 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:30:12,166 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:30:12,166 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:30:13,331 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:30:13,331 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:30:13,333 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:30:15,564 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:30:19,377 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:30:19,377 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:30:19,377 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:30:19,382 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:30:20,171 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:30:21,421 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:30:22,174 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:30:25,590 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:30:25,591 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:30:25,591 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:30:25,596 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:30:26,177 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:30:26,597 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:30:28,180 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:30:28,331 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:30:28,332 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:30:28,333 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:30:29,810 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:30:29,811 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:30:29,811 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:30:29,816 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:30:30,181 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:30:31,853 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:30:32,183 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:30:34,861 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:30:34,862 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:30:34,862 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:30:34,867 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:30:35,185 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:30:36,187 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:30:36,868 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:30:38,741 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 17:30:39,153 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:30:39,154 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:30:39,154 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:30:39,160 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:30:39,189 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:30:40,192 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:30:42,160 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:30:43,331 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:30:43,332 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:30:43,333 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:30:43,442 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:30:43,586 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:30:43,586 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:30:43,591 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:30:44,194 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:30:46,197 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:30:47,592 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:30:47,934 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:30:47,935 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:30:47,935 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:30:47,940 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:30:48,199 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:30:50,200 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:30:52,151 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:30:52,152 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:30:52,152 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:30:52,157 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:30:52,201 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:30:53,158 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:30:54,204 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:30:58,158 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:30:58,331 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:30:58,332 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:30:58,333 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:31:00,261 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:31:00,262 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:31:00,262 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:31:00,267 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:31:01,208 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:31:02,211 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:31:03,305 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:31:05,272 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:31:05,276 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:31:05,277 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:31:05,282 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:31:06,215 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:31:06,215 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:31:08,743 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 17:31:08,745 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:31:10,004 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:31:10,004 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:31:10,005 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:31:10,010 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:31:10,217 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:31:12,222 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:31:13,331 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:31:13,332 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:31:13,333 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:31:14,302 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:31:14,302 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:31:14,303 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:31:14,303 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:31:14,308 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:31:15,224 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:31:16,227 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:31:18,580 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:31:18,581 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:31:18,581 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:31:18,586 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:31:19,229 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:31:19,587 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:31:20,231 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:31:23,082 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:31:23,083 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:31:23,084 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:31:23,089 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:31:23,233 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:31:24,236 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:31:25,089 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:31:28,332 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:31:28,332 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:31:28,334 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:31:28,440 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:31:28,639 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:31:28,639 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:31:28,644 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:31:29,239 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:31:30,240 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:31:30,645 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:31:33,786 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:31:33,787 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:31:33,788 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:31:33,793 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:31:34,242 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:31:35,794 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:31:36,245 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:31:38,746 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 17:31:38,914 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:31:38,915 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:31:38,915 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:31:38,921 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:31:39,247 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:31:40,249 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:31:40,921 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:31:43,332 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:31:43,332 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:31:43,332 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:31:44,338 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:31:44,339 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:31:44,339 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:31:44,344 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:31:45,252 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:31:46,253 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:31:46,381 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:31:49,062 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:31:49,063 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:31:49,063 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:31:49,068 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:31:49,255 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:31:50,258 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:31:52,069 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:31:53,565 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:31:53,566 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:31:53,566 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:31:53,571 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:31:54,260 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:31:56,263 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:31:57,572 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:31:58,332 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:31:58,332 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:31:58,334 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:32:02,057 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:32:02,058 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:32:02,058 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:32:02,063 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:32:02,267 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:32:03,101 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:32:04,270 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:32:08,102 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:32:08,751 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 17:32:10,086 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:32:10,087 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:32:10,088 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:32:10,093 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:32:10,273 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:32:12,278 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:32:13,332 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:32:13,332 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:32:13,332 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:32:13,582 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:32:15,066 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:32:15,067 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:32:15,067 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:32:15,072 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:32:15,280 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:32:16,282 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:32:19,109 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:32:19,432 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:32:19,433 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:32:19,433 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:32:19,438 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:32:20,285 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:32:22,290 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:32:24,439 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:32:25,220 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:32:25,220 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:32:25,221 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:32:25,226 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:32:25,291 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:32:26,294 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:32:28,332 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:32:28,332 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:32:28,333 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:32:29,642 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:32:29,888 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:32:29,888 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:32:29,889 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:32:29,894 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:32:30,297 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:32:32,300 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:32:34,645 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:32:34,646 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:32:34,646 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:32:34,646 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:32:34,651 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:32:35,302 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:32:36,304 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:32:38,751 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 17:32:38,914 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:32:38,915 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:32:38,915 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:32:38,920 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:32:39,306 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:32:39,920 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:32:40,308 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:32:43,332 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:32:43,332 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:32:43,332 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:32:44,083 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:32:44,083 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:32:44,084 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:32:44,089 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:32:44,311 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:32:45,125 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:32:46,314 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:32:49,244 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:32:49,245 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:32:49,245 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:32:49,250 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:32:49,316 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:32:50,251 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:32:50,318 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:32:53,689 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:32:53,690 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:32:53,690 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:32:53,695 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:32:54,322 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:32:55,696 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:32:56,325 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:32:58,332 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:32:58,332 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:32:58,333 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:32:59,794 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:32:59,795 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:32:59,795 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:32:59,800 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:33:00,327 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:33:00,837 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:33:02,330 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:33:05,020 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:33:05,021 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:33:05,021 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:33:05,025 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:33:05,332 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:33:06,026 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:33:06,332 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:33:08,756 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 17:33:09,384 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:33:09,385 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:33:09,385 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:33:09,390 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:33:10,337 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:33:10,337 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:33:11,391 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:33:13,332 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:33:13,333 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:33:13,334 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:33:16,603 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:33:16,846 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:33:16,847 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:33:16,847 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:33:16,852 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:33:17,342 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:33:18,345 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:33:21,889 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:33:23,343 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:33:23,344 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:33:23,344 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:33:23,349 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:33:23,349 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:33:24,352 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:33:27,350 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:33:28,332 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:33:28,333 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:33:28,333 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:33:31,850 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:33:31,851 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:33:31,851 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:33:31,856 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:33:32,357 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:33:32,893 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:33:34,360 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:33:37,894 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:33:38,755 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 17:33:40,347 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:33:40,348 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:33:40,348 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:33:40,353 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:33:40,363 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:33:42,367 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:33:43,332 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:33:43,333 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:33:43,333 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:33:43,540 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:33:46,447 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:33:46,448 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:33:46,448 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:33:46,452 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:33:47,370 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:33:48,373 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:33:49,489 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:33:51,143 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:33:51,143 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:33:51,144 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:33:51,148 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:33:51,374 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:33:52,377 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:33:55,149 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:33:56,441 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:33:56,442 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:33:56,443 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:33:56,448 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:33:57,380 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:33:58,332 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:33:58,333 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:33:58,333 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:33:58,383 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:34:00,570 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:34:01,303 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:34:01,303 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:34:01,304 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:34:01,309 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:34:01,384 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:34:02,387 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:34:06,346 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:34:07,592 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:34:07,593 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:34:07,593 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:34:07,598 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:34:08,390 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:34:08,757 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 17:34:10,393 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:34:11,759 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:34:12,662 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:34:12,662 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:34:12,663 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:34:12,667 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:34:13,332 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:34:13,333 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:34:13,334 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:34:13,396 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:34:14,399 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:34:17,642 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:34:20,339 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:34:20,340 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:34:20,340 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:34:20,345 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:34:20,403 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:34:22,406 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:34:23,381 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:34:24,975 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:34:24,976 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:34:24,976 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:34:24,981 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:34:25,408 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:34:26,411 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:34:28,333 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:34:28,333 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:34:28,333 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:34:28,621 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:34:31,438 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:34:31,442 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:34:31,442 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:34:31,447 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:34:32,416 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:34:32,416 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:34:34,481 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:34:35,432 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:34:35,432 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:34:35,433 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:34:35,437 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:34:36,420 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:34:36,421 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:34:38,760 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 17:34:39,761 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:34:39,992 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:34:39,993 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:34:39,993 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:34:39,998 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:34:40,423 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:34:42,426 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:34:43,333 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:34:43,333 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:34:43,334 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:34:45,563 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:34:46,539 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:34:46,539 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:34:46,540 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:34:46,545 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:34:47,429 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:34:48,432 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:34:50,581 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:34:51,100 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:34:51,100 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:34:51,100 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:34:51,105 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:34:51,433 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:34:52,436 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:34:56,107 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:34:58,333 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:34:58,333 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:34:58,334 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:35:00,210 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:35:00,210 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:35:00,210 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:35:00,215 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:35:00,441 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:35:01,253 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:35:02,444 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:35:04,874 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:35:04,874 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:35:04,875 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:35:04,880 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:35:05,446 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:35:06,448 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:35:06,880 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:35:08,762 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 17:35:09,695 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:35:09,696 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:35:09,696 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:35:09,701 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:35:10,451 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:35:12,455 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:35:12,701 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:35:13,333 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:35:13,333 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:35:13,334 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:35:14,545 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:35:14,546 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:35:14,546 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:35:14,551 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:35:15,457 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:35:16,458 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:35:18,589 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:35:19,534 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:35:19,535 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:35:19,535 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:35:19,540 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:35:20,463 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:35:20,463 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:35:24,541 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:35:27,011 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:35:27,012 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:35:27,012 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:35:27,017 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:35:27,467 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:35:28,333 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:35:28,333 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:35:28,334 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:35:28,470 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:35:29,602 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:35:33,130 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:35:33,131 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:35:33,131 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:35:33,136 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:35:33,473 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:35:34,475 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:35:35,173 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:35:38,767 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 17:35:39,847 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:35:39,848 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:35:39,848 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:35:39,853 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:35:40,479 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:35:40,853 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:35:42,482 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:35:43,333 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:35:43,333 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:35:43,335 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:35:44,726 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:35:44,726 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:35:44,727 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:35:44,731 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:35:45,483 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:35:46,486 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:35:46,769 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:35:49,177 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:35:49,178 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:35:49,178 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:35:49,183 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:35:49,488 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:35:50,489 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:35:52,184 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:35:53,635 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:35:53,636 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:35:53,636 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:35:53,641 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:35:54,494 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:35:54,494 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:35:57,642 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:35:58,333 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:35:58,333 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:35:58,335 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:36:03,342 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:36:03,343 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:36:03,343 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:36:03,343 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:36:03,348 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:36:03,499 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:36:04,501 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:36:07,705 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:36:07,706 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:36:07,706 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:36:07,710 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:36:08,504 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:36:08,711 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:36:08,766 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 17:36:10,507 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:36:12,444 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:36:12,445 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:36:12,445 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:36:12,450 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:36:12,510 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:36:13,333 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:36:13,334 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:36:13,335 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:36:14,513 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:36:14,596 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:36:16,966 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:36:16,967 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:36:16,967 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:36:16,972 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:36:17,515 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:36:18,518 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:36:20,009 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:36:23,029 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:36:23,030 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:36:23,030 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:36:23,035 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:36:23,520 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:36:24,523 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:36:25,036 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:36:27,291 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:36:27,292 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:36:27,292 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:36:27,297 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:36:27,525 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:36:28,333 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:36:28,334 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:36:28,334 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:36:28,528 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:36:30,626 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:36:32,179 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:36:32,180 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:36:32,180 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:36:32,185 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:36:32,530 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:36:34,533 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:36:36,221 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:36:36,853 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:36:36,853 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:36:36,854 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:36:36,859 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:36:37,535 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:36:38,538 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:36:38,772 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 17:36:41,773 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:36:43,092 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:36:43,093 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:36:43,093 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:36:43,098 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:36:43,333 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:36:43,334 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:36:43,335 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:36:43,541 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:36:44,544 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:36:47,602 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:36:49,145 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:36:49,146 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:36:49,146 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:36:49,151 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:36:49,547 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:36:50,550 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:36:53,189 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:36:54,232 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:36:54,233 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:36:54,233 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:36:54,238 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:36:54,552 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:36:56,556 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:36:58,239 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:36:58,333 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:36:58,334 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:36:58,335 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:36:59,914 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:36:59,915 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:36:59,915 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:36:59,919 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:37:00,558 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:37:02,561 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:37:03,957 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:37:04,466 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:37:04,467 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:37:04,467 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:37:04,471 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:37:04,563 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:37:06,566 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:37:08,771 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 17:37:08,914 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:37:08,915 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:37:08,915 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:37:08,920 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:37:09,568 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:37:09,921 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:37:10,570 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:37:13,333 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:37:13,334 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:37:13,335 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:37:14,146 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:37:14,147 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:37:14,147 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:37:14,152 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:37:14,574 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:37:15,189 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:37:16,578 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:37:18,582 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:37:18,583 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:37:18,583 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:37:18,589 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:37:19,580 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:37:20,583 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:37:20,589 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:37:24,765 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:37:24,766 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:37:24,766 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:37:24,771 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:37:25,586 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:37:25,771 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:37:26,589 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:37:28,334 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:37:28,334 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:37:28,335 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:37:30,483 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:37:30,484 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:37:30,484 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:37:30,489 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:37:30,591 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:37:31,525 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:37:32,594 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:37:35,746 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:37:35,747 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:37:35,747 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:37:35,752 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:37:36,599 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:37:36,599 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:37:36,752 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:37:38,773 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 17:37:40,180 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:37:40,181 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:37:40,181 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:37:40,186 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:37:40,601 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:37:42,186 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:37:42,604 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:37:43,334 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:37:43,334 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:37:43,335 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:37:44,699 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:37:44,699 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:37:44,700 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:37:44,704 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:37:45,606 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:37:46,609 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:37:47,741 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:37:49,148 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:37:49,149 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:37:49,149 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:37:49,154 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:37:49,611 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:37:50,614 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:37:53,154 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:37:53,453 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:37:53,454 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:37:53,454 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:37:53,459 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:37:53,616 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:37:54,618 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:37:58,334 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:37:58,334 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:37:58,335 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:37:58,569 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:38:00,568 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:38:00,568 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:38:00,568 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:38:00,574 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:38:00,622 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:38:02,626 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:38:03,612 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:38:06,043 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:38:06,043 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:38:06,043 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:38:06,048 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:38:06,628 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:38:08,631 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:38:08,775 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 17:38:08,776 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:38:13,334 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:38:13,334 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:38:13,335 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:38:14,412 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:38:14,413 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:38:14,413 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:38:14,413 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:38:14,418 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:38:14,644 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:38:16,647 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:38:19,003 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:38:19,004 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:38:19,004 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:38:19,009 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:38:19,649 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:38:20,009 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:38:20,651 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:38:23,509 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:38:23,510 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:38:23,510 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:38:23,515 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:38:23,653 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:38:24,656 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:38:25,516 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:38:27,699 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:38:27,700 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:38:27,700 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:38:27,704 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:38:28,334 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:38:28,334 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:38:28,335 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:38:28,661 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:38:28,661 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:38:30,553 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:38:33,920 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:38:33,921 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:38:33,921 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:38:33,926 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:38:34,665 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:38:35,961 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:38:36,668 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:38:38,777 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 17:38:39,387 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:38:39,387 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:38:39,387 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:38:39,392 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:38:39,670 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:38:40,672 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:38:41,392 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:38:43,334 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:38:43,334 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:38:43,335 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:38:43,690 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:38:43,690 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:38:43,690 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:38:43,698 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:38:44,675 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:38:44,675 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:38:46,733 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:38:50,318 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:38:50,319 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:38:50,319 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:38:50,324 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:38:50,679 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:38:52,324 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:38:52,682 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:38:55,245 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:38:55,246 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:38:55,246 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:38:55,251 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:38:55,684 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:38:56,687 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:38:58,252 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:38:58,334 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:38:58,334 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:38:58,336 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:38:59,756 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:38:59,756 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:38:59,756 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:38:59,761 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:39:00,692 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:39:00,692 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:39:03,797 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:39:06,723 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:39:06,724 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:39:06,724 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:39:06,729 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:39:07,696 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:39:08,698 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:39:08,783 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 17:39:09,784 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:39:11,296 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:39:11,297 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:39:11,297 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:39:11,302 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:39:11,700 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:39:12,704 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:39:13,334 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:39:13,334 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:39:13,336 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:39:15,406 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:39:15,407 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:39:15,407 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:39:15,407 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:39:15,412 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:39:15,706 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:39:16,709 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:39:20,449 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:39:21,069 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:39:21,070 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:39:21,070 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:39:21,075 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:39:21,712 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:39:22,714 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:39:25,997 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:39:25,998 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:39:25,998 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:39:25,998 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:39:26,003 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:39:26,717 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:39:28,334 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:39:28,334 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:39:28,336 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:39:28,721 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:39:31,637 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:39:32,694 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:39:32,695 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:39:32,695 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:39:32,700 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:39:32,723 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:39:34,726 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:39:36,737 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:39:37,255 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:39:37,255 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:39:37,256 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:39:37,260 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:39:37,728 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:39:38,731 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:39:38,782 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 17:39:41,748 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:39:41,749 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:39:41,749 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:39:41,749 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:39:41,754 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:39:42,733 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:39:42,733 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:39:43,334 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:39:43,335 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:39:43,336 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:39:46,563 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:39:46,563 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:39:46,564 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:39:46,568 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:39:46,735 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:39:47,605 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:39:48,739 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:39:52,606 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:39:54,949 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:39:54,949 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:39:54,950 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:39:54,955 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:39:55,744 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:39:56,746 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:39:57,955 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:39:58,334 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:39:58,335 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:39:58,336 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:39:59,470 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:39:59,471 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:39:59,471 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:39:59,476 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:39:59,748 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:40:00,749 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:40:03,514 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:40:03,532 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:40:03,533 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:40:03,533 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:40:03,539 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:40:03,750 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:40:04,753 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:40:07,698 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:40:07,698 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:40:07,699 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:40:07,704 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:40:07,755 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:40:08,704 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:40:08,757 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:40:08,787 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 17:40:13,334 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:40:13,335 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:40:13,336 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:40:13,577 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:40:13,617 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:40:13,617 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:40:13,623 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:40:13,762 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:40:14,623 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:40:14,764 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:40:19,624 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:40:22,007 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:40:22,008 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:40:22,008 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:40:22,013 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:40:22,769 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:40:24,770 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:40:25,049 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:40:26,943 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:40:26,944 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:40:26,944 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:40:26,949 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:40:27,773 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:40:28,334 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:40:28,335 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:40:28,336 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:40:28,775 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:40:30,576 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:40:31,339 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:40:31,339 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:40:31,340 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:40:31,344 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:40:31,777 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:40:32,780 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:40:35,724 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:40:35,725 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:40:35,726 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:40:35,726 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:40:35,731 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:40:35,782 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:40:36,784 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:40:38,787 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 17:40:40,018 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:40:40,018 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:40:40,019 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:40:40,024 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:40:40,789 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:40:41,024 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:40:42,793 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:40:43,335 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:40:43,335 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:40:43,336 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:40:44,674 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:40:44,674 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:40:44,674 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:40:44,679 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:40:44,794 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:40:46,717 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:40:46,795 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:40:49,185 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:40:49,186 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:40:49,186 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:40:49,191 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:40:49,797 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:40:50,800 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:40:52,192 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:40:53,578 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:40:53,579 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:40:53,579 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:40:53,584 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:40:53,802 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:40:54,804 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:40:57,585 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:40:58,335 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:40:58,335 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:40:58,336 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:40:58,956 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:40:58,956 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:40:58,957 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:40:58,962 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:40:59,807 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:41:00,810 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:41:02,997 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:41:07,443 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:41:07,444 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:41:07,444 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:41:07,449 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:41:07,814 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:41:08,449 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:41:08,789 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 17:41:08,814 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:41:12,601 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:41:12,602 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:41:12,602 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:41:12,607 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:41:12,818 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:41:13,335 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:41:13,335 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:41:13,336 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:41:13,567 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:41:14,821 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:41:17,035 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:41:17,035 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:41:17,035 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:41:17,040 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:41:17,823 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:41:18,826 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:41:19,080 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:41:21,853 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:41:21,854 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:41:21,854 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:41:21,859 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:41:22,830 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:41:22,830 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:41:24,860 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:41:28,335 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:41:28,335 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:41:28,336 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:41:30,386 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:41:30,386 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:41:30,387 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:41:30,388 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:41:30,392 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:41:30,836 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:41:32,839 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:41:35,430 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:41:36,163 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:41:36,164 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:41:36,165 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:41:36,170 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:41:36,841 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:41:38,791 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 17:41:38,845 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:41:40,792 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:41:42,361 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:41:42,361 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:41:42,361 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:41:42,366 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:41:42,847 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:41:43,335 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:41:43,335 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:41:43,336 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:41:44,850 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:41:46,577 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:41:46,846 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:41:46,847 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:41:46,847 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:41:46,852 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:41:46,852 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:41:48,856 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:41:50,974 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:41:50,975 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:41:50,975 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:41:50,980 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:41:51,857 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:41:51,981 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:41:52,861 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:41:55,674 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:41:55,674 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:41:55,675 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:41:55,680 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:41:55,862 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:41:56,865 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:41:57,680 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:41:58,335 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:41:58,335 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:41:58,337 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:42:00,227 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:42:00,227 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:42:00,227 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:42:00,232 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:42:00,867 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:42:02,869 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:42:03,269 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:42:06,298 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:42:06,299 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:42:06,299 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:42:06,304 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:42:06,871 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:42:08,305 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:42:08,793 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 17:42:08,872 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:42:12,588 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:42:12,589 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:42:12,589 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:42:12,594 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:42:12,876 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:42:13,335 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:42:13,335 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:42:13,337 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:42:13,594 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:42:14,879 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:42:18,595 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:42:20,943 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:42:20,944 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:42:20,944 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:42:20,949 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:42:21,883 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:42:22,886 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:42:23,985 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:42:25,196 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:42:25,196 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:42:25,196 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:42:25,201 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:42:25,888 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:42:26,891 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:42:28,335 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:42:28,335 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:42:28,337 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:42:29,637 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:42:31,608 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:42:31,609 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:42:31,609 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:42:31,614 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:42:31,894 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:42:32,896 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:42:34,649 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:42:36,058 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:42:36,059 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:42:36,059 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:42:36,064 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:42:36,899 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:42:36,899 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:42:38,798 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 17:42:39,799 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:42:43,335 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:42:43,336 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:42:43,337 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:42:44,533 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:42:44,534 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:42:44,534 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:42:44,539 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:42:44,903 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:42:45,577 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:42:46,906 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:42:50,578 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:42:51,314 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:42:51,314 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:42:51,315 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:42:51,319 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:42:51,910 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:42:52,913 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:42:55,178 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:42:55,179 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:42:55,179 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:42:55,184 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:42:55,914 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:42:56,184 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:42:56,917 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:42:58,335 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:42:58,336 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:42:58,337 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:43:01,143 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:43:01,144 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:43:01,144 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:43:01,149 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:43:01,185 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:43:01,920 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:43:02,922 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:43:06,186 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:43:08,797 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 17:43:09,673 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:43:09,674 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:43:09,674 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:43:09,679 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:43:09,926 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:43:10,929 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:43:11,680 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:43:13,335 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:43:13,336 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:43:13,337 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:43:14,035 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:43:14,036 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:43:14,036 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:43:14,041 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:43:14,935 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:43:14,935 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:43:17,077 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:43:19,298 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:43:19,299 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:43:19,299 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:43:19,303 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:43:19,939 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:43:20,941 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:43:22,304 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:43:23,339 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:43:23,340 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:43:23,340 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:43:23,345 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:43:23,943 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:43:24,943 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:43:27,346 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:43:28,335 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:43:28,336 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:43:28,337 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:43:28,382 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:43:28,566 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:43:28,566 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:43:28,571 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:43:28,946 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:43:30,949 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:43:32,572 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:43:34,748 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:43:34,748 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:43:34,749 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:43:34,754 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:43:34,952 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:43:36,955 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:43:37,754 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:43:38,802 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 17:43:38,925 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:43:38,926 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:43:38,926 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:43:38,931 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:43:38,956 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:43:40,959 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:43:42,931 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:43:43,336 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:43:43,336 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:43:43,337 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:43:43,672 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:43:43,672 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:43:43,673 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:43:43,677 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:43:43,961 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:43:46,965 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:43:48,713 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:43:52,206 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:43:52,207 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:43:52,207 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:43:52,211 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:43:52,969 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:43:54,212 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:43:54,983 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:43:58,336 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:43:58,336 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:43:58,337 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:43:59,624 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:44:00,596 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:44:00,597 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:44:00,597 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:44:00,601 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:44:00,986 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:44:02,990 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:44:04,637 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:44:04,704 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:44:04,705 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:44:04,705 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:44:04,709 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:44:04,991 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:44:06,994 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:44:08,802 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 17:44:09,202 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:44:09,203 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:44:09,203 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:44:09,208 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:44:09,996 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:44:10,208 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:44:10,996 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:44:13,336 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:44:13,336 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:44:13,337 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:44:14,346 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:44:14,347 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:44:14,347 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:44:14,352 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:44:15,001 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:44:15,389 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:44:17,004 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:44:18,760 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:44:18,761 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:44:18,761 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:44:18,766 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:44:19,005 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:44:20,766 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:44:21,009 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:44:24,456 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:44:24,457 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:44:24,457 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:44:24,462 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:44:25,012 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:44:26,463 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:44:27,015 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:44:28,320 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:44:28,320 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:44:28,320 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:44:28,326 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:44:28,336 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:44:28,336 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:44:28,337 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:44:29,016 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:44:31,019 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:44:31,629 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:44:32,538 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:44:32,538 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:44:32,539 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:44:32,543 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:44:33,020 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:44:35,023 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:44:37,581 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:44:38,804 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 17:44:41,043 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:44:41,044 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:44:41,044 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:44:41,049 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:44:42,028 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:44:43,028 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:44:43,050 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:44:43,336 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:44:43,336 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:44:43,337 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:44:46,095 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:44:46,096 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:44:46,096 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:44:46,101 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:44:47,031 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:44:48,137 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:44:49,034 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:44:53,138 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:44:53,185 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:44:53,186 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:44:53,186 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:44:53,190 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:44:54,038 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:44:55,040 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:44:57,950 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:44:57,951 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:44:57,951 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:44:57,955 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:44:58,042 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:44:58,336 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:44:58,336 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:44:58,338 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:44:58,638 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:45:01,046 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:45:03,029 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:45:03,030 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:45:03,030 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:45:03,035 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:45:03,048 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:45:04,073 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:45:05,051 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:45:08,740 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:45:08,741 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:45:08,741 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:45:08,746 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:45:08,806 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 17:45:09,054 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:45:09,807 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:45:11,057 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:45:13,336 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:45:13,336 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:45:13,338 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:45:15,250 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:45:15,250 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:45:15,251 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:45:15,251 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:45:15,256 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:45:16,062 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:45:17,065 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:45:20,293 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:45:21,220 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:45:21,220 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:45:21,221 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:45:21,225 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:45:22,068 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:45:23,071 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:45:26,226 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:45:28,032 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:45:28,032 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:45:28,033 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:45:28,037 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:45:28,075 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:45:28,336 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:45:28,337 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:45:28,338 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:45:31,079 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:45:31,560 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:45:34,644 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:45:34,645 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:45:34,645 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:45:34,651 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:45:35,081 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:45:36,685 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:45:37,084 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:45:38,809 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 17:45:40,120 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:45:40,121 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:45:40,121 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:45:40,127 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:45:41,087 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:45:42,127 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:45:43,090 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:45:43,336 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:45:43,336 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:45:43,338 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:45:45,236 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:45:45,236 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:45:45,237 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:45:45,241 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:45:46,092 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:45:47,094 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:45:47,277 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:45:49,524 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:45:49,525 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:45:49,525 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:45:49,530 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:45:50,097 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:45:51,098 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:45:52,530 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:45:54,957 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:45:54,958 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:45:54,958 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:45:54,963 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:45:55,100 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:45:57,103 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:45:57,964 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:45:58,336 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:45:58,337 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:45:58,338 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:46:00,636 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:46:00,636 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:46:00,636 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:46:00,641 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:46:01,106 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:46:03,110 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:46:03,677 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:46:06,684 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:46:06,685 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:46:06,685 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:46:06,690 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:46:07,112 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:46:08,690 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:46:08,814 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 17:46:09,116 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:46:11,211 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:46:11,211 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:46:11,212 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:46:11,217 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:46:12,118 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:46:13,122 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:46:13,336 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:46:13,337 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:46:13,338 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:46:14,628 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:46:16,396 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:46:16,396 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:46:16,397 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:46:16,401 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:46:17,125 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:46:19,128 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:46:20,437 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:46:21,854 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:46:21,855 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:46:21,855 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:46:21,860 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:46:22,130 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:46:25,134 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:46:25,860 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:46:28,336 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:46:28,337 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:46:28,338 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:46:28,521 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:46:28,642 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:46:28,642 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:46:28,647 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:46:29,137 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:46:31,138 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:46:31,647 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:46:34,313 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:46:34,313 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:46:34,314 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:46:34,318 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:46:35,140 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:46:37,142 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:46:37,319 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:46:38,810 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:46:38,813 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:46:38,813 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:46:38,818 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:46:38,818 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 17:46:39,143 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:46:41,146 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:46:42,819 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:46:43,337 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:46:43,337 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:46:43,338 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:46:45,469 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:46:45,469 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:46:45,470 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:46:45,474 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:46:46,149 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:46:47,151 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:46:48,513 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:46:49,710 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:46:49,710 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:46:49,711 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:46:49,715 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:46:50,154 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:46:51,156 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:46:53,716 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:46:53,867 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:46:53,868 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:46:53,868 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:46:53,873 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:46:54,158 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:46:55,161 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:46:58,337 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:46:58,337 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:46:58,338 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:46:59,370 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:46:59,371 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:46:59,371 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:46:59,371 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:46:59,376 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:47:00,164 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:47:01,167 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:47:04,413 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:47:04,913 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:47:04,914 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:47:04,914 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:47:04,919 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:47:05,169 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:47:07,172 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:47:08,818 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 17:47:09,328 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:47:09,329 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:47:09,329 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:47:09,333 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:47:10,174 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:47:10,334 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:47:11,177 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:47:13,337 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:47:13,337 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:47:13,338 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:47:15,626 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:47:17,315 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:47:17,315 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:47:17,316 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:47:17,320 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:47:18,183 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:47:19,187 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:47:21,357 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:47:22,234 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:47:22,234 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:47:22,234 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:47:22,239 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:47:23,191 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:47:25,195 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:47:26,622 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:47:26,622 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:47:26,623 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:47:26,623 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:47:26,628 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:47:27,198 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:47:28,337 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:47:28,337 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:47:28,338 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:47:29,203 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:47:30,940 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:47:30,941 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:47:30,941 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:47:30,946 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:47:31,205 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:47:31,981 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:47:33,209 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:47:35,205 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:47:35,206 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:47:35,206 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:47:35,212 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:47:35,213 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:47:37,213 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:47:37,214 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:47:38,817 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 17:47:39,725 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:47:39,726 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:47:39,726 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:47:39,731 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:47:40,217 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:47:41,222 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:47:42,731 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:47:43,337 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:47:43,337 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:47:43,338 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:47:45,700 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:47:45,701 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:47:45,701 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:47:45,706 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:47:46,226 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:47:47,230 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:47:47,741 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:47:50,208 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:47:50,209 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:47:50,209 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:47:50,215 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:47:50,233 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:47:53,215 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:47:53,239 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:47:56,789 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:47:56,790 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:47:56,790 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:47:56,795 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:47:57,243 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:47:58,337 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:47:58,337 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:47:58,339 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:47:58,601 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:47:59,248 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:48:02,687 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:48:02,687 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:48:02,688 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:48:02,692 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:48:03,252 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:48:03,729 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:48:05,257 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:48:08,730 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:48:08,819 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 17:48:09,620 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:48:09,621 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:48:09,621 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:48:09,626 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:48:10,261 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:48:11,266 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:48:13,337 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:48:13,337 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:48:13,338 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:48:14,262 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:48:14,263 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:48:14,263 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:48:14,263 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:48:14,268 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:48:14,269 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:48:17,274 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:48:19,181 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:48:19,181 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:48:19,181 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:48:19,187 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:48:19,275 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:48:20,187 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:48:21,280 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:48:23,605 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:48:23,606 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:48:23,606 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:48:23,611 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:48:24,282 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:48:25,286 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:48:25,612 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:48:28,337 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:48:28,337 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:48:28,339 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:48:29,846 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:48:29,847 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:48:29,847 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:48:29,852 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:48:30,291 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:48:30,889 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:48:31,293 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:48:34,727 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:48:34,732 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:48:34,732 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:48:34,737 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:48:35,299 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:48:36,737 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:48:37,304 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:48:38,821 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 17:48:39,106 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:48:39,106 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:48:39,107 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:48:39,111 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:48:39,305 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:48:41,310 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:48:42,112 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:48:43,337 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:48:43,337 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:48:43,339 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:48:44,020 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:48:44,021 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:48:44,021 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:48:44,026 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:48:44,312 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:48:47,317 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:48:48,061 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:48:51,269 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:48:51,270 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:48:51,270 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:48:51,275 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:48:51,324 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:48:53,275 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:48:53,329 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:48:58,276 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:48:58,282 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:48:58,283 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:48:58,284 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:48:58,288 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:48:58,334 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:48:58,337 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:48:58,337 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:48:58,339 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:49:01,341 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:49:02,749 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:49:02,749 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:49:02,749 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:49:02,754 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:49:03,343 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:49:03,793 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:49:05,347 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:49:06,581 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:49:06,581 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:49:06,582 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:49:06,587 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:49:07,348 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:49:08,824 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 17:49:08,825 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:49:09,353 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:49:13,337 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:49:13,338 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:49:13,338 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:49:14,626 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:49:15,428 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:49:15,429 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:49:15,429 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:49:15,435 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:49:16,361 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:49:17,364 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:49:20,469 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:49:24,131 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:49:24,132 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:49:24,132 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:49:24,136 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:49:24,371 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:49:26,137 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:49:27,377 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:49:28,338 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:49:28,338 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:49:28,339 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:49:29,630 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:49:29,630 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:49:29,631 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:49:29,635 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:49:30,380 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:49:31,383 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:49:31,673 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:49:35,260 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:49:35,261 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:49:35,261 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:49:35,266 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:49:35,389 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:49:37,266 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:49:37,392 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:49:38,829 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 17:49:41,513 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:49:41,514 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:49:41,514 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:49:41,519 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:49:42,398 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:49:42,520 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:49:43,337 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:49:43,338 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:49:43,339 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:49:43,403 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:49:46,219 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:49:46,220 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:49:46,220 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:49:46,225 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:49:46,405 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:49:48,261 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:49:49,411 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:49:51,107 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:49:51,108 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:49:51,108 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:49:51,113 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:49:51,413 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:49:53,417 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:49:54,114 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:49:55,868 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:49:55,869 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:49:55,869 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:49:55,874 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:49:56,419 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:49:57,424 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:49:58,338 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:49:58,338 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:49:58,338 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:49:59,571 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:50:00,314 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:50:00,314 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:50:00,315 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:50:00,320 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:50:00,426 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:50:03,432 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:50:05,358 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:50:06,007 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:50:06,008 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:50:06,008 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:50:06,013 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:50:06,434 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:50:07,438 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:50:08,828 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 17:50:10,510 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:50:10,510 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:50:10,511 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:50:10,511 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:50:10,516 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:50:11,442 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:50:13,338 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:50:13,339 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:50:13,339 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:50:13,448 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:50:15,643 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:50:15,880 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:50:15,880 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:50:15,881 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:50:15,886 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:50:16,451 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:50:17,453 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:50:20,196 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:50:20,197 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:50:20,197 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:50:20,202 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:50:20,456 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:50:21,203 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:50:23,461 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:50:26,203 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:50:26,815 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:50:26,816 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:50:26,816 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:50:26,821 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:50:27,466 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:50:28,338 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:50:28,338 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:50:28,339 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:50:29,470 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:50:31,627 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:50:35,159 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:50:35,159 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:50:35,160 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:50:35,165 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:50:35,477 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:50:37,201 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:50:37,481 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:50:38,833 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 17:50:39,032 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:50:39,033 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:50:39,034 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:50:39,038 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:50:39,483 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:50:41,487 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:50:43,039 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:50:43,338 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:50:43,338 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:50:43,340 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:50:44,311 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:50:44,312 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:50:44,312 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:50:44,316 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:50:44,489 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:50:47,496 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:50:48,353 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:50:50,241 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:50:50,242 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:50:50,242 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:50:50,247 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:50:50,498 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:50:53,504 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:50:54,247 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:50:58,338 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:50:58,338 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:50:58,338 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:50:58,613 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:50:58,613 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:50:58,614 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:50:58,618 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:50:59,512 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:50:59,657 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:51:01,517 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:51:04,657 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:51:08,832 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 17:51:09,834 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:51:10,034 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:51:10,035 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:51:10,035 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:51:10,039 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:51:10,527 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:51:11,531 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:51:13,338 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:51:13,338 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:51:13,340 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:51:15,087 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:51:15,088 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:51:15,088 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:51:15,088 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:51:15,092 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:51:15,536 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:51:17,540 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:51:20,129 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:51:20,585 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:51:20,586 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:51:20,586 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:51:20,591 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:51:21,544 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:51:23,549 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:51:25,277 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:51:25,278 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:51:25,278 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:51:25,278 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:51:25,283 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:51:25,552 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:51:27,557 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:51:28,338 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:51:28,339 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:51:28,340 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:51:30,602 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:51:33,677 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:51:33,677 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:51:33,678 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:51:33,682 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:51:34,564 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:51:35,568 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:51:35,721 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:51:38,835 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 17:51:40,836 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:51:41,300 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:51:41,301 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:51:41,301 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:51:41,306 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:51:41,575 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:51:43,338 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:51:43,339 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:51:43,340 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:51:43,580 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:51:46,551 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:51:48,341 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:51:48,342 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:51:48,342 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:51:48,347 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:51:48,584 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:51:51,591 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:51:52,385 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:51:53,432 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:51:53,433 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:51:53,433 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:51:53,438 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:51:53,594 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:51:55,599 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:51:57,439 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:51:58,338 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:51:58,339 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:51:58,339 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:51:58,924 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:51:58,925 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:51:58,925 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:51:58,930 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:51:59,604 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:52:01,608 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:52:02,965 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:52:03,694 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:52:03,695 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:52:03,695 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:52:03,700 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:52:04,610 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:52:05,614 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:52:08,208 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:52:08,209 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:52:08,209 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:52:08,209 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:52:08,214 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:52:08,616 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:52:08,840 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 17:52:09,620 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:52:13,338 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:52:13,339 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:52:13,339 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:52:13,461 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:52:13,573 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:52:13,573 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:52:13,573 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:52:13,578 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:52:13,627 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:52:15,631 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:52:18,578 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:52:19,257 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:52:19,258 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:52:19,258 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:52:19,262 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:52:19,635 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:52:21,639 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:52:24,263 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:52:28,339 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:52:28,339 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:52:28,339 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:52:28,643 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:52:28,643 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:52:28,643 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:52:28,648 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:52:28,649 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:52:29,685 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:52:31,654 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:52:33,103 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:52:33,104 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:52:33,105 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:52:33,109 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:52:33,655 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:52:35,110 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:52:35,658 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:52:38,839 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 17:52:39,603 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:52:39,603 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:52:39,603 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:52:39,608 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:52:39,665 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:52:40,609 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:52:41,671 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:52:43,339 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:52:43,339 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:52:43,339 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:52:45,034 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:52:45,035 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:52:45,035 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:52:45,040 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:52:45,675 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:52:46,077 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:52:47,679 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:52:50,313 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:52:50,314 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:52:50,314 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:52:50,319 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:52:50,681 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:52:51,319 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:52:53,688 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:52:54,968 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:52:54,969 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:52:54,969 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:52:54,974 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:52:55,690 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:52:56,974 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:52:57,694 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:52:58,339 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:52:58,339 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:52:58,340 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:53:00,722 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:53:00,723 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:53:00,723 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:53:00,728 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:53:01,698 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:53:02,765 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:53:03,703 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:53:06,137 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:53:06,138 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:53:06,138 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:53:06,143 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:53:06,705 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:53:07,709 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:53:08,143 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:53:08,844 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 17:53:11,200 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:53:11,201 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:53:11,201 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:53:11,206 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:53:11,713 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:53:13,206 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:53:13,339 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:53:13,339 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:53:13,340 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:53:13,717 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:53:16,074 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:53:16,075 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:53:16,075 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:53:16,080 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:53:16,720 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:53:17,723 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:53:19,117 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:53:20,369 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:53:20,369 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:53:20,370 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:53:20,375 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:53:20,726 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:53:23,732 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:53:24,376 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:53:25,242 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:53:25,243 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:53:25,243 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:53:25,249 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:53:25,734 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:53:27,739 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:53:28,339 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:53:28,339 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:53:28,339 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:53:29,649 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:53:29,762 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:53:29,762 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:53:29,763 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:53:29,768 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:53:30,741 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:53:31,745 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:53:34,436 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:53:34,437 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:53:34,437 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:53:34,442 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:53:34,747 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:53:35,442 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:53:37,754 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:53:38,844 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 17:53:40,364 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:53:40,364 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:53:40,365 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:53:40,369 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:53:40,756 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:53:41,370 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:53:43,339 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:53:43,339 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:53:43,339 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:53:43,762 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:53:45,235 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:53:45,236 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:53:45,236 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:53:45,241 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:53:45,763 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:53:47,277 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:53:47,768 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:53:50,833 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:53:50,834 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:53:50,834 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:53:50,839 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:53:51,772 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:53:52,839 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:53:53,777 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:53:55,635 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:53:55,635 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:53:55,636 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:53:55,641 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:53:55,779 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:53:57,784 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:53:58,339 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:53:58,339 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:53:58,339 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:53:58,581 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:54:00,409 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:54:00,409 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:54:00,410 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:54:00,415 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:54:00,785 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:54:03,793 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:54:04,454 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:54:06,936 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:54:06,937 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:54:06,937 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:54:06,942 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:54:07,797 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:54:08,849 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 17:54:09,801 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:54:09,850 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:54:11,447 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:54:11,448 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:54:11,448 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:54:11,453 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:54:11,803 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:54:13,339 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:54:13,340 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:54:13,340 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:54:13,809 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:54:15,585 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:54:16,006 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:54:16,007 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:54:16,007 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:54:16,012 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:54:16,814 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:54:16,814 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:54:21,050 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:54:21,370 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:54:21,371 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:54:21,371 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:54:21,376 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:54:21,818 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:54:22,820 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:54:25,625 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:54:25,626 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:54:25,626 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:54:25,631 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:54:25,824 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:54:26,632 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:54:26,826 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:54:28,339 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:54:28,340 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:54:28,340 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:54:30,371 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:54:30,372 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:54:30,372 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:54:30,377 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:54:30,831 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:54:32,413 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:54:32,836 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:54:34,593 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:54:34,594 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:54:34,594 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:54:34,599 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:54:34,837 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:54:36,841 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:54:37,599 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:54:38,848 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 17:54:38,878 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:54:38,878 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:54:38,879 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:54:38,884 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:54:39,845 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:54:40,848 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:54:42,884 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:54:43,105 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:54:43,106 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:54:43,106 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:54:43,111 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:54:43,339 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:54:43,340 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:54:43,340 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:54:43,851 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:54:44,853 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:54:48,883 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:54:48,918 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:54:48,919 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:54:48,919 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:54:48,924 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:54:49,859 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:54:50,862 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:54:52,971 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:54:52,972 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:54:52,972 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:54:52,977 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:54:53,865 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:54:53,978 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:54:54,868 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:54:58,003 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:54:58,004 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:54:58,004 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:54:58,009 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:54:58,339 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:54:58,340 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:54:58,341 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:54:58,874 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:54:58,874 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:54:59,744 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:55:04,744 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:55:05,270 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:55:05,270 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:55:05,271 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:55:05,276 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:55:05,883 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:55:06,886 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:55:08,850 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 17:55:09,492 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:55:09,493 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:55:09,493 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:55:09,498 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:55:09,889 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:55:10,498 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:55:10,892 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:55:13,339 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:55:13,340 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:55:13,341 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:55:15,649 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:55:17,818 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:55:17,819 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:55:17,819 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:55:17,824 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:55:17,900 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:55:18,903 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:55:20,861 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:55:21,867 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:55:21,868 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:55:21,869 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:55:21,874 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:55:21,906 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:55:22,909 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:55:25,875 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:55:26,804 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:55:26,805 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:55:26,805 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:55:26,811 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:55:26,914 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:55:28,339 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:55:28,340 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:55:28,340 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:55:28,919 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:55:31,338 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:55:31,339 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:55:31,339 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:55:31,340 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:55:31,345 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:55:31,923 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:55:32,925 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:55:36,108 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:55:36,112 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:55:36,113 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:55:36,118 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:55:36,931 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:55:36,931 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:55:37,118 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:55:38,852 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 17:55:40,753 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:55:40,754 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:55:40,754 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:55:40,759 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:55:40,935 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:55:42,760 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:55:42,940 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:55:43,340 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:55:43,340 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:55:43,340 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:55:45,000 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:55:45,000 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:55:45,000 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:55:45,005 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:55:45,943 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:55:46,945 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:55:48,041 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:55:49,243 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:55:49,243 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:55:49,243 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:55:49,248 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:55:49,949 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:55:50,951 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:55:53,249 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:55:55,101 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:55:55,102 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:55:55,102 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:55:55,107 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:55:55,957 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:55:56,960 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:55:58,340 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:55:58,340 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:55:58,340 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:55:58,563 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:55:59,661 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:55:59,662 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:55:59,662 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:55:59,667 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:55:59,964 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:56:00,966 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:56:03,705 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:56:04,602 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:56:04,602 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:56:04,602 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:56:04,607 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:56:04,970 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:56:06,975 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:56:08,854 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 17:56:08,855 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:56:09,012 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:56:09,013 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:56:09,013 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:56:09,018 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:56:09,978 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:56:10,980 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:56:13,340 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:56:13,340 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:56:13,340 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:56:13,685 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:56:13,685 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:56:13,686 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:56:13,690 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:56:13,985 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:56:14,729 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:56:14,988 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:56:18,551 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:56:18,551 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:56:18,551 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:56:18,556 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:56:18,991 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:56:20,557 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:56:20,996 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:56:23,720 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:56:23,721 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:56:23,721 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:56:23,726 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:56:23,998 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:56:25,001 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:56:25,727 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:56:28,296 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:56:28,296 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:56:28,296 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:56:28,301 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:56:28,340 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:56:28,340 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:56:28,341 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:56:29,007 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:56:29,007 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:56:31,632 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:56:32,645 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:56:32,645 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:56:32,645 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:56:32,650 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:56:33,011 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:56:35,016 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:56:36,689 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:56:38,170 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:56:38,171 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:56:38,171 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:56:38,176 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:56:38,860 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 17:56:39,022 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:56:39,022 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:56:41,861 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:56:43,340 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:56:43,340 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:56:43,340 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:56:44,352 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:56:44,353 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:56:44,353 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:56:44,358 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:56:45,030 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:56:45,030 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:56:47,393 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:56:49,075 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:56:49,076 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:56:49,076 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:56:49,081 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:56:50,036 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:56:51,039 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:56:53,082 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:56:54,262 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:56:54,262 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:56:54,263 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:56:54,267 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:56:55,045 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:56:55,045 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:56:58,268 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:56:58,340 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:56:58,340 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:56:58,340 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:56:58,557 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:56:58,558 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:56:58,558 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:56:58,563 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:56:59,049 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:57:01,054 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:57:03,564 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:57:06,592 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:57:06,592 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:57:06,593 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:57:06,597 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:57:07,062 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:57:08,633 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:57:08,858 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 17:57:09,065 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:57:10,991 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:57:10,992 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:57:10,992 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:57:10,997 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:57:11,067 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:57:13,072 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:57:13,340 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:57:13,340 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:57:13,341 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:57:14,576 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:57:15,714 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:57:15,715 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:57:15,715 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:57:15,720 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:57:16,076 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:57:17,079 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:57:19,757 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:57:20,649 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:57:20,650 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:57:20,650 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:57:20,655 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:57:21,083 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:57:23,087 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:57:25,099 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:57:25,100 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:57:25,100 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:57:25,100 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:57:25,104 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:57:26,091 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:57:27,093 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:57:28,340 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:57:28,341 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:57:28,341 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:57:29,397 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:57:29,397 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:57:29,397 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:57:29,402 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:57:30,097 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:57:30,441 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:57:31,100 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:57:34,108 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:57:34,108 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:57:34,109 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:57:34,113 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:57:35,105 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:57:35,105 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:57:36,113 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:57:38,812 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:57:38,812 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:57:38,813 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:57:38,817 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:57:38,864 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 17:57:39,109 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:57:41,114 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:57:41,865 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:57:43,209 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:57:43,210 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:57:43,210 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:57:43,215 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:57:43,341 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:57:43,341 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:57:43,341 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:57:44,118 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:57:45,121 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:57:47,582 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:57:48,496 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:57:48,496 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:57:48,497 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:57:48,501 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:57:49,126 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:57:49,126 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:57:53,537 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:57:56,312 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:57:56,313 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:57:56,313 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:57:56,318 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:57:57,139 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:57:57,139 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:57:58,341 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:57:58,341 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:57:58,341 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:57:58,579 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:58:02,167 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:58:02,168 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:58:02,168 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:58:02,174 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:58:03,149 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:58:03,149 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:58:04,209 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:58:08,585 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:58:08,586 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:58:08,586 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:58:08,591 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:58:08,863 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 17:58:09,158 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:58:09,158 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:58:09,864 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:58:13,336 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:58:13,337 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:58:13,337 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:58:13,341 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:58:13,341 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:58:13,341 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:58:13,342 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:58:14,166 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:58:15,168 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:58:15,633 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:58:18,850 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:58:18,850 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:58:18,851 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:58:18,855 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:58:19,171 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:58:20,893 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:58:21,176 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:58:24,726 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:58:24,727 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:58:24,727 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:58:24,732 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:58:25,179 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:58:26,732 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:58:27,184 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:58:28,341 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:58:28,341 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:58:28,341 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:58:29,276 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:58:29,277 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:58:29,277 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:58:29,282 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:58:30,188 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:58:31,190 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:58:32,317 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:58:36,422 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:58:36,423 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:58:36,423 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:58:36,428 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:58:37,199 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:58:37,199 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:58:37,428 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:58:38,865 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 17:58:41,863 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:58:41,864 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:58:41,864 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:58:41,868 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:58:42,205 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:58:42,869 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:58:43,208 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:58:43,341 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:58:43,341 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:58:43,341 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:58:47,812 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:58:47,816 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:58:47,816 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:58:47,820 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:58:48,214 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:58:48,853 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:58:49,217 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:58:52,464 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:58:52,465 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:58:52,465 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:58:52,470 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:58:53,222 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:58:53,223 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:58:54,470 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:58:58,341 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:58:58,341 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:58:58,341 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:58:59,179 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:58:59,179 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:58:59,179 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:58:59,185 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:58:59,231 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:59:00,221 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:59:01,235 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:59:03,587 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:59:03,587 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:59:03,587 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:59:03,592 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:59:04,238 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:59:05,241 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:59:05,593 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:59:08,133 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:59:08,134 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:59:08,134 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:59:08,139 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:59:08,245 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:59:08,868 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 17:59:09,248 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:59:10,869 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:59:12,812 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:59:12,813 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:59:12,813 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:59:12,818 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:59:13,252 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:59:13,341 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:59:13,341 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:59:13,341 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:59:15,258 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:59:16,565 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:59:18,517 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:59:18,518 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:59:18,518 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:59:18,524 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:59:19,263 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:59:19,263 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:59:22,561 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:59:23,762 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:59:23,762 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:59:23,762 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:59:23,768 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:59:24,268 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:59:25,271 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:59:27,768 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:59:28,341 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:59:28,341 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:59:28,342 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:59:28,618 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:59:28,619 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:59:28,619 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:59:28,624 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:59:29,277 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:59:29,278 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:59:33,661 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:59:33,669 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:59:33,670 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:59:33,670 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:59:33,675 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:59:34,284 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:59:35,287 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:59:38,242 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:59:38,243 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:59:38,243 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:59:38,249 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:59:38,291 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:59:38,870 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 17:59:38,871 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:59:39,293 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:59:43,341 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:59:43,341 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:59:43,341 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:59:43,968 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:59:43,969 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:59:43,969 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:59:43,969 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:59:43,982 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:59:44,300 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:59:45,302 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:59:48,265 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:59:48,266 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:59:48,266 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:59:48,271 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:59:48,306 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:59:49,271 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:59:49,309 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:59:53,080 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:59:53,081 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:59:53,082 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:59:53,086 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:59:53,313 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:59:55,087 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:59:55,318 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 17:59:57,307 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:59:57,308 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 17:59:57,308 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:59:57,313 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:59:57,319 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 17:59:58,341 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:59:58,341 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:59:58,342 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:59:59,325 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:00:00,546 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:00:02,503 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:00:02,503 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:00:02,504 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:00:02,508 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:00:03,330 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:00:03,331 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:00:06,545 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:00:06,899 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:00:06,903 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:00:06,904 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:00:06,908 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:00:07,335 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:00:08,872 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 18:00:09,340 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:00:11,053 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:00:11,054 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:00:11,054 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:00:11,059 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:00:11,341 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:00:12,059 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:00:13,341 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:00:13,341 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:00:13,343 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:00:13,346 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:00:15,864 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:00:15,865 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:00:15,865 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:00:15,870 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:00:16,350 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:00:17,352 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:00:17,905 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:00:22,906 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:00:23,432 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:00:23,433 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:00:23,433 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:00:23,438 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:00:24,361 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:00:25,363 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:00:28,341 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:00:28,341 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:00:28,342 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:00:28,567 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:00:28,624 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:00:28,624 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:00:28,624 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:00:28,629 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:00:29,370 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:00:29,370 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:00:33,524 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:00:33,525 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:00:33,525 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:00:33,530 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:00:34,376 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:00:34,565 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:00:35,379 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:00:37,996 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:00:37,997 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:00:37,997 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:00:38,002 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:00:38,382 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:00:38,874 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 18:00:39,385 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:00:39,875 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:00:42,237 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:00:42,237 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:00:42,238 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:00:42,242 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:00:42,388 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:00:43,341 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:00:43,342 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:00:43,342 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:00:43,390 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:00:45,580 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:00:47,264 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:00:47,265 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:00:47,265 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:00:47,270 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:00:47,394 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:00:49,399 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:00:51,305 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:00:51,379 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:00:51,379 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:00:51,379 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:00:51,384 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:00:51,401 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:00:53,405 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:00:56,385 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:00:56,475 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:00:56,475 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:00:56,476 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:00:56,480 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:00:57,412 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:00:57,412 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:00:58,341 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:00:58,342 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:00:58,343 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:01:01,538 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:01:01,714 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:01:01,714 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:01:01,715 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:01:01,720 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:01:02,418 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:01:03,421 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:01:06,757 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:01:07,453 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:01:07,454 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:01:07,454 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:01:07,459 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:01:08,427 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:01:08,879 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 18:01:09,429 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:01:11,881 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:01:12,324 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:01:12,325 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:01:12,326 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:01:12,331 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:01:12,433 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:01:13,342 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:01:13,342 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:01:13,342 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:01:13,436 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:01:17,612 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:01:18,533 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:01:18,533 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:01:18,533 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:01:18,538 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:01:19,444 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:01:19,444 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:01:22,941 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:01:22,941 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:01:22,942 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:01:22,942 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:01:22,946 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:01:23,448 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:01:25,453 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:01:27,947 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:01:28,242 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:01:28,242 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:01:28,243 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:01:28,248 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:01:28,342 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:01:28,342 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:01:28,343 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:01:28,457 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:01:29,459 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:01:33,603 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:01:36,807 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:01:36,808 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:01:36,808 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:01:36,814 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:01:37,470 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:01:37,471 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:01:38,849 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:01:38,878 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 18:01:43,170 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:01:43,171 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:01:43,171 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:01:43,176 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:01:43,342 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:01:43,342 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:01:43,342 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:01:43,477 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:01:44,723 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:01:45,482 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:01:48,308 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:01:48,309 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:01:48,309 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:01:48,314 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:01:48,485 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:01:49,488 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:01:50,349 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:01:53,022 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:01:53,022 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:01:53,022 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:01:53,028 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:01:53,492 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:01:55,495 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:01:56,028 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:01:58,342 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:01:58,342 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:01:58,342 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:01:58,537 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:01:58,553 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:01:58,553 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:01:58,558 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:01:59,499 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:01:59,500 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:02:01,558 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:02:04,162 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:02:04,162 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:02:04,162 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:02:04,167 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:02:04,502 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:02:05,505 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:02:07,205 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:02:08,881 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 18:02:10,379 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:02:10,383 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:02:10,383 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:02:10,388 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:02:10,508 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:02:11,511 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:02:12,388 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:02:13,342 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:02:13,342 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:02:13,342 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:02:17,065 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:02:17,065 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:02:17,066 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:02:17,071 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:02:17,517 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:02:18,109 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:02:19,520 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:02:23,021 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:02:23,022 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:02:23,022 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:02:23,028 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:02:23,523 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:02:24,028 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:02:25,527 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:02:27,664 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:02:27,664 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:02:27,664 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:02:27,670 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:02:28,342 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:02:28,342 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:02:28,343 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:02:28,529 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:02:29,532 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:02:29,616 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:02:34,294 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:02:34,295 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:02:34,295 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:02:34,300 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:02:34,535 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:02:35,337 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:02:35,538 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:02:38,883 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 18:02:39,039 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:02:39,040 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:02:39,040 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:02:39,045 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:02:39,540 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:02:41,046 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:02:41,544 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:02:43,342 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:02:43,342 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:02:43,342 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:02:43,544 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:02:43,580 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:02:43,580 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:02:43,585 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:02:44,546 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:02:45,549 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:02:46,586 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:02:50,564 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:02:50,565 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:02:50,565 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:02:50,570 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:02:51,556 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:02:51,556 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:02:51,605 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:02:56,452 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:02:56,453 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:02:56,453 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:02:56,458 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:02:56,559 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:02:57,458 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:02:57,562 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:02:58,342 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:02:58,342 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:02:58,343 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:03:02,545 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:03:04,953 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:03:04,953 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:03:04,954 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:03:04,959 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:03:05,567 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:03:07,570 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:03:07,997 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:03:08,885 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 18:03:13,342 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:03:13,343 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:03:13,343 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:03:13,370 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:03:13,600 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:03:13,600 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:03:13,601 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:03:13,606 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:03:14,575 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:03:15,578 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:03:18,606 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:03:19,541 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:03:19,542 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:03:19,542 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:03:19,547 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:03:19,581 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:03:21,585 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:03:24,548 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:03:27,048 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:03:27,049 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:03:27,049 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:03:27,054 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:03:27,588 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:03:28,342 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:03:28,343 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:03:28,343 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:03:29,578 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:03:29,592 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:03:30,886 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:03:30,887 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:03:30,887 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:03:30,892 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:03:31,595 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:03:31,596 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:03:34,929 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:03:35,438 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:03:35,439 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:03:35,440 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:03:35,445 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:03:35,598 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:03:37,601 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:03:38,891 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 18:03:40,204 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:03:40,205 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:03:40,205 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:03:40,205 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:03:40,210 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:03:40,603 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:03:41,606 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:03:43,342 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:03:43,343 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:03:43,343 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:03:45,557 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:03:46,258 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:03:46,259 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:03:46,259 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:03:46,265 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:03:46,609 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:03:47,611 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:03:51,302 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:03:51,627 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:03:51,628 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:03:51,628 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:03:51,634 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:03:52,615 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:03:53,618 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:03:56,634 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:03:56,839 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:03:56,840 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:03:56,840 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:03:56,845 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:03:57,623 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:03:57,623 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:03:58,342 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:03:58,343 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:03:58,344 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:04:01,637 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:04:01,733 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:04:01,734 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:04:01,734 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:04:01,746 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:04:02,626 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:04:03,629 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:04:06,777 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:04:08,094 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:04:08,095 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:04:08,095 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:04:08,100 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:04:08,632 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:04:08,890 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 18:04:09,634 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:04:11,891 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:04:13,163 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:04:13,164 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:04:13,164 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:04:13,169 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:04:13,343 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:04:13,343 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:04:13,344 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:04:13,637 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:04:15,642 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:04:17,472 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:04:17,473 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:04:17,473 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:04:17,473 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:04:17,479 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:04:17,643 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:04:19,647 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:04:22,156 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:04:22,157 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:04:22,157 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:04:22,162 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:04:22,649 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:04:23,162 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:04:23,652 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:04:27,258 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:04:27,259 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:04:27,259 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:04:27,264 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:04:27,654 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:04:28,265 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:04:28,343 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:04:28,343 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:04:28,344 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:04:29,658 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:04:32,046 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:04:32,047 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:04:32,047 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:04:32,052 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:04:32,659 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:04:33,662 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:04:34,089 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:04:36,502 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:04:36,503 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:04:36,503 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:04:36,508 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:04:36,664 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:04:37,666 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:04:38,895 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 18:04:39,896 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:04:43,343 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:04:43,343 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:04:43,344 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:04:44,819 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:04:44,820 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:04:44,820 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:04:44,825 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:04:45,673 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:04:45,673 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:04:45,861 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:04:49,347 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:04:49,348 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:04:49,348 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:04:49,353 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:04:49,676 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:04:51,353 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:04:51,679 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:04:54,510 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:04:54,511 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:04:54,511 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:04:54,517 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:04:54,682 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:04:55,685 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:04:56,517 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:04:58,343 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:04:58,343 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:04:58,344 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:04:58,471 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:04:58,653 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:04:58,653 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:04:58,658 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:04:58,687 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:04:59,689 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:05:01,659 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:05:04,835 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:05:04,836 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:05:04,837 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:05:04,842 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:05:05,695 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:05:05,696 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:05:06,842 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:05:08,894 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 18:05:09,288 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:05:09,289 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:05:09,289 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:05:09,295 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:05:09,698 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:05:11,701 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:05:12,295 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:05:13,343 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:05:13,343 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:05:13,343 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:05:13,440 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:05:13,561 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:05:13,561 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:05:13,566 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:05:13,702 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:05:15,707 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:05:17,567 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:05:19,555 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:05:19,555 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:05:19,556 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:05:19,561 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:05:19,710 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:05:21,714 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:05:23,561 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:05:24,737 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:05:24,738 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:05:24,738 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:05:24,743 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:05:25,719 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:05:25,719 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:05:28,343 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:05:28,343 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:05:28,343 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:05:28,635 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:05:28,807 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:05:28,807 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:05:28,808 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:05:28,813 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:05:29,725 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:05:29,725 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:05:33,703 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:05:33,704 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:05:33,705 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:05:33,705 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:05:33,710 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:05:33,727 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:05:35,731 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:05:38,198 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:05:38,199 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:05:38,199 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:05:38,203 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:05:38,733 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:05:38,900 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 18:05:38,901 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:05:39,735 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:05:42,887 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:05:42,888 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:05:42,888 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:05:42,894 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:05:43,343 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:05:43,344 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:05:43,344 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:05:43,740 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:05:43,740 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:05:44,583 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:05:47,389 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:05:47,389 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:05:47,390 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:05:47,395 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:05:47,742 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:05:49,744 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:05:50,433 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:05:54,475 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:05:54,476 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:05:54,476 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:05:54,481 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:05:54,747 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:05:55,481 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:05:55,750 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:05:58,343 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:05:58,343 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:05:58,344 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:05:58,870 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:05:58,870 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:05:58,870 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:05:58,875 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:05:59,755 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:05:59,755 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:06:00,913 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:06:05,092 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:06:05,092 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:06:05,093 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:06:05,097 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:06:05,760 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:06:05,761 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:06:06,098 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:06:08,899 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 18:06:09,784 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:06:09,785 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:06:09,785 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:06:09,791 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:06:10,764 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:06:11,766 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:06:11,791 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:06:13,343 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:06:13,343 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:06:13,345 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:06:14,162 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:06:14,163 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:06:14,163 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:06:14,168 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:06:14,769 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:06:15,772 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:06:17,205 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:06:18,701 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:06:18,702 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:06:18,702 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:06:18,708 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:06:18,774 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:06:19,777 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:06:22,708 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:06:25,055 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:06:25,055 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:06:25,056 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:06:25,061 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:06:25,783 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:06:25,784 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:06:28,061 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:06:28,343 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:06:28,343 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:06:28,345 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:06:30,054 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:06:30,055 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:06:30,055 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:06:30,061 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:06:30,787 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:06:31,790 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:06:33,097 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:06:34,554 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:06:34,555 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:06:34,555 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:06:34,560 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:06:34,792 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:06:35,795 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:06:38,561 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:06:38,900 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 18:06:40,933 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:06:40,934 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:06:40,934 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:06:40,939 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:06:41,801 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:06:41,801 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:06:43,343 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:06:43,344 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:06:43,344 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:06:44,544 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:06:47,137 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:06:47,137 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:06:47,138 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:06:47,143 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:06:47,806 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:06:47,806 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:06:50,181 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:06:51,468 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:06:51,469 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:06:51,469 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:06:51,474 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:06:51,809 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:06:53,813 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:06:55,474 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:06:56,766 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:06:56,767 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:06:56,767 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:06:56,772 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:06:56,815 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:06:57,817 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:06:58,343 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:06:58,344 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:06:58,344 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:07:00,573 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:07:01,882 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:07:01,883 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:07:01,883 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:07:01,888 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:07:02,821 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:07:03,823 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:07:05,925 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:07:07,801 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:07:07,805 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:07:07,805 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:07:07,810 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:07:07,826 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:07:08,903 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 18:07:09,829 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:07:11,904 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:07:13,343 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:07:13,344 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:07:13,345 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:07:14,914 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:07:14,915 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:07:14,915 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:07:14,920 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:07:15,836 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:07:15,836 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:07:16,957 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:07:19,366 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:07:19,367 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:07:19,367 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:07:19,372 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:07:19,839 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:07:21,843 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:07:22,372 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:07:24,836 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:07:24,837 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:07:24,837 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:07:24,842 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:07:24,845 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:07:25,847 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:07:27,842 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:07:28,344 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:07:28,344 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:07:28,344 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:07:29,126 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:07:29,126 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:07:29,127 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:07:29,132 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:07:29,852 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:07:29,852 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:07:33,169 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:07:33,792 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:07:33,793 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:07:33,794 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:07:33,799 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:07:33,855 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:07:35,859 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:07:38,800 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:07:38,905 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 18:07:40,266 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:07:40,266 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:07:40,266 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:07:40,272 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:07:40,861 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:07:41,864 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:07:43,344 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:07:43,344 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:07:43,344 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:07:44,579 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:07:49,580 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:07:49,877 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:07:49,877 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:07:49,877 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:07:49,883 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:07:50,871 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:07:51,873 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:07:54,921 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:07:55,048 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:07:55,048 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:07:55,048 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:07:55,053 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:07:55,878 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:07:55,878 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:07:58,344 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:07:58,344 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:07:58,345 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:07:59,010 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:07:59,010 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:07:59,010 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:07:59,015 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:07:59,882 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:07:59,882 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:08:00,053 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:08:02,848 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:08:02,848 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:08:02,848 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:08:02,854 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:08:02,884 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:08:03,887 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:08:05,854 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:08:07,963 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:08:07,964 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:08:07,964 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:08:07,969 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:08:08,890 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:08:08,911 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 18:08:09,892 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:08:10,912 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:08:12,812 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:08:12,813 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:08:12,813 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:08:12,818 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:08:12,894 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:08:13,344 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:08:13,344 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:08:13,344 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:08:13,897 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:08:16,548 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:08:16,927 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:08:16,928 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:08:16,928 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:08:16,933 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:08:17,903 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:08:17,903 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:08:21,969 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:08:22,450 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:08:22,450 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:08:22,451 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:08:22,455 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:08:22,907 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:08:23,909 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:08:27,456 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:08:28,344 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:08:28,344 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:08:28,345 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:08:29,131 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:08:29,132 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:08:29,132 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:08:29,137 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:08:29,915 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:08:29,916 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:08:33,173 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:08:34,609 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:08:34,610 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:08:34,610 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:08:34,615 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:08:34,918 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:08:35,921 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:08:38,616 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:08:38,909 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 18:08:39,151 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:08:39,152 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:08:39,152 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:08:39,157 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:08:39,926 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:08:39,926 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:08:43,344 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:08:43,344 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:08:43,345 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:08:43,473 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:08:43,629 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:08:43,629 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:08:43,629 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:08:43,635 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:08:43,928 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:08:45,931 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:08:48,103 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:08:48,104 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:08:48,104 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:08:48,109 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:08:48,933 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:08:49,109 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:08:49,936 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:08:54,110 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:08:55,146 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:08:55,147 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:08:55,147 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:08:55,152 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:08:55,943 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:08:55,943 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:08:58,344 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:08:58,344 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:08:58,344 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:08:59,556 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:09:00,938 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:09:00,938 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:09:00,938 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:09:00,943 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:09:00,946 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:09:01,948 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:09:04,981 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:09:05,641 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:09:05,642 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:09:05,642 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:09:05,647 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:09:05,951 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:09:07,955 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:09:08,915 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 18:09:10,916 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:09:11,190 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:09:11,191 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:09:11,191 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:09:11,196 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:09:11,959 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:09:11,959 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:09:13,344 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:09:13,344 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:09:13,346 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:09:16,617 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:09:16,678 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:09:16,679 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:09:16,679 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:09:16,684 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:09:16,964 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:09:17,967 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:09:21,721 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:09:22,355 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:09:22,356 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:09:22,356 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:09:22,361 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:09:22,970 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:09:23,973 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:09:26,843 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:09:26,843 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:09:26,844 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:09:26,844 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:09:26,849 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:09:26,974 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:09:27,977 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:09:28,344 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:09:28,345 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:09:28,346 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:09:30,778 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:09:30,778 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:09:30,779 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:09:30,783 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:09:30,980 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:09:31,980 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:09:32,821 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:09:35,577 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:09:35,578 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:09:35,578 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:09:35,583 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:09:35,983 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:09:37,986 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:09:38,584 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:09:38,914 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 18:09:41,439 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:09:41,440 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:09:41,440 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:09:41,445 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:09:41,988 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:09:43,344 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:09:43,345 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:09:43,345 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:09:43,593 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:09:43,991 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:09:47,047 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:09:47,047 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:09:47,048 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:09:47,052 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:09:47,996 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:09:47,996 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:09:49,089 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:09:54,089 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:09:55,459 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:09:55,460 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:09:55,460 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:09:55,464 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:09:56,002 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:09:58,005 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:09:58,344 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:09:58,345 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:09:58,346 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:09:59,655 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:10:03,488 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:10:03,489 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:10:03,489 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:10:03,494 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:10:04,008 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:10:05,529 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:10:06,012 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:10:07,565 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:10:07,566 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:10:07,566 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:10:07,571 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:10:08,013 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:10:08,916 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 18:10:10,016 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:10:10,917 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:10:13,344 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:10:13,345 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:10:13,346 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:10:14,770 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:10:14,771 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:10:14,771 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:10:14,776 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:10:15,020 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:10:16,023 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:10:16,813 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:10:20,688 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:10:20,689 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:10:20,689 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:10:20,694 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:10:21,027 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:10:22,029 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:10:22,694 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:10:26,487 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:10:26,488 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:10:26,488 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:10:26,493 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:10:27,032 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:10:28,035 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:10:28,345 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:10:28,345 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:10:28,346 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:10:28,642 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:10:30,884 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:10:30,885 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:10:30,885 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:10:30,890 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:10:31,037 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:10:32,040 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:10:33,925 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:10:36,790 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:10:36,790 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:10:36,791 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:10:36,795 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:10:37,043 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:10:38,045 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:10:38,918 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 18:10:39,919 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:10:43,345 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:10:43,345 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:10:43,346 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:10:45,223 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:10:45,223 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:10:45,223 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:10:45,223 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:10:45,228 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:10:46,052 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:10:46,053 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:10:49,735 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:10:49,736 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:10:49,736 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:10:49,741 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:10:50,055 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:10:50,741 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:10:52,058 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:10:54,254 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:10:54,255 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:10:54,255 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:10:54,260 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:10:55,061 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:10:56,063 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:10:56,260 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:10:58,345 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:10:58,345 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:10:58,346 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:11:00,633 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:11:00,634 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:11:00,634 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:11:00,639 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:11:01,066 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:11:01,677 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:11:02,069 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:11:06,677 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:11:08,727 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:11:08,727 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:11:08,728 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:11:08,733 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:11:08,920 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 18:11:09,073 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:11:10,075 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:11:11,922 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:11:13,179 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:11:13,180 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:11:13,181 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:11:13,185 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:11:13,345 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:11:13,345 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:11:13,345 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:11:14,080 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:11:14,080 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:11:17,608 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:11:19,185 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:11:19,186 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:11:19,186 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:11:19,191 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:11:20,089 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:11:20,089 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:11:23,229 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:11:23,433 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:11:23,434 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:11:23,434 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:11:23,440 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:11:24,095 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:11:24,095 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:11:28,093 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:11:28,094 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:11:28,094 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:11:28,100 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:11:28,101 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:11:28,345 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:11:28,345 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:11:28,345 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:11:28,602 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:11:30,105 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:11:33,534 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:11:33,535 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:11:33,535 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:11:33,541 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:11:34,107 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:11:34,577 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:11:36,111 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:11:37,886 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:11:37,886 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:11:37,886 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:11:37,892 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:11:38,112 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:11:38,926 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 18:11:39,927 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:11:40,116 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:11:42,969 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:11:42,969 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:11:42,969 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:11:42,975 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:11:43,118 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:11:43,345 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:11:43,346 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:11:43,346 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:11:44,121 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:11:45,601 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:11:47,593 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:11:47,593 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:11:47,594 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:11:47,599 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:11:48,124 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:11:50,127 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:11:50,637 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:11:52,435 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:11:52,436 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:11:52,436 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:11:52,442 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:11:53,130 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:11:54,134 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:11:56,443 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:11:57,364 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:11:57,365 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:11:57,365 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:11:57,372 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:11:58,139 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:11:58,139 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:11:58,345 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:11:58,346 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:11:58,346 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:12:01,644 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:12:01,989 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:12:01,990 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:12:01,990 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:12:01,996 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:12:02,142 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:12:04,146 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:12:06,322 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:12:06,323 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:12:06,323 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:12:06,329 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:12:07,148 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:12:07,329 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:12:08,151 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:12:08,925 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 18:12:11,600 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:12:11,601 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:12:11,601 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:12:11,606 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:12:12,154 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:12:12,607 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:12:13,345 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:12:13,345 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:12:13,347 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:12:14,157 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:12:16,726 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:12:16,726 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:12:16,726 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:12:16,732 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:12:17,161 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:12:17,769 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:12:18,164 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:12:22,769 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:12:25,150 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:12:25,150 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:12:25,151 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:12:25,156 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:12:25,169 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:12:26,172 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:12:28,157 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:12:28,346 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:12:28,346 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:12:28,346 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:12:29,463 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:12:29,463 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:12:29,463 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:12:29,469 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:12:30,178 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:12:30,178 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:12:33,505 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:12:35,293 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:12:35,294 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:12:35,294 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:12:35,300 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:12:36,184 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:12:36,185 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:12:38,931 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 18:12:38,932 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:12:39,909 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:12:39,910 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:12:39,910 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:12:39,916 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:12:40,188 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:12:42,191 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:12:43,345 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:12:43,346 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:12:43,347 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:12:44,599 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:12:44,629 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:12:44,630 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:12:44,630 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:12:44,636 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:12:45,194 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:12:46,197 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:12:49,673 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:12:53,075 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:12:53,076 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:12:53,076 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:12:53,082 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:12:53,203 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:12:54,207 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:12:55,083 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:12:58,346 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:12:58,346 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:12:58,347 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:13:00,558 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:13:01,642 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:13:01,643 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:13:01,643 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:13:01,650 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:13:02,214 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:13:04,218 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:13:05,685 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:13:06,913 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:13:06,914 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:13:06,914 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:13:06,921 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:13:07,221 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:13:08,224 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:13:08,929 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 18:13:10,930 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:13:13,346 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:13:13,346 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:13:13,347 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:13:14,505 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:13:14,505 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:13:14,506 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:13:14,512 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:13:15,232 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:13:16,233 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:13:16,549 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:13:20,434 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:13:20,435 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:13:20,435 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:13:20,442 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:13:21,238 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:13:22,241 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:13:22,442 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:13:25,124 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:13:25,125 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:13:25,125 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:13:25,131 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:13:25,244 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:13:26,247 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:13:28,132 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:13:28,346 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:13:28,346 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:13:28,347 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:13:29,002 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:13:29,002 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:13:29,002 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:13:29,009 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:13:29,250 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:13:30,253 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:13:33,312 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:13:33,313 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:13:33,313 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:13:33,313 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:13:33,319 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:13:34,258 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:13:34,258 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:13:38,319 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:13:38,931 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 18:13:39,588 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:13:39,589 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:13:39,589 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:13:39,595 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:13:40,266 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:13:40,266 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:13:43,346 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:13:43,346 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:13:43,346 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:13:43,575 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:13:44,463 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:13:44,464 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:13:44,464 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:13:44,470 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:13:45,270 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:13:46,275 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:13:49,505 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:13:49,824 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:13:49,824 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:13:49,824 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:13:49,831 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:13:50,278 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:13:52,283 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:13:54,832 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:13:56,144 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:13:56,145 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:13:56,145 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:13:56,151 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:13:56,287 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:13:58,290 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:13:58,346 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:13:58,346 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:13:58,346 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:14:00,412 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:14:00,413 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:14:00,413 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:14:00,413 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:14:00,419 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:14:01,293 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:14:02,296 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:14:05,457 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:14:05,816 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:14:05,817 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:14:05,817 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:14:05,823 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:14:06,298 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:14:08,302 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:14:08,934 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 18:14:09,813 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:14:09,814 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:14:09,814 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:14:09,820 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:14:10,304 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:14:10,821 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:14:12,307 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:14:13,346 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:14:13,346 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:14:13,348 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:14:14,881 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:14:14,882 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:14:14,882 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:14:14,894 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:14:15,313 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:14:15,925 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:14:16,323 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:14:19,652 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:14:19,653 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:14:19,653 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:14:19,660 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:14:20,337 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:14:20,338 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:14:21,660 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:14:24,064 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:14:24,065 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:14:24,065 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:14:24,074 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:14:24,343 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:14:26,352 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:14:27,075 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:14:28,346 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:14:28,346 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:14:28,347 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:14:30,555 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:14:30,556 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:14:30,556 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:14:30,563 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:14:31,359 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:14:32,365 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:14:32,597 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:14:35,433 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:14:35,434 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:14:35,434 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:14:35,444 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:14:36,373 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:14:36,373 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:14:38,444 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:14:38,936 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 18:14:40,755 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:14:40,755 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:14:40,755 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:14:40,768 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:14:41,378 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:14:42,384 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:14:43,346 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:14:43,346 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:14:43,348 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:14:43,540 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:14:44,883 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:14:44,884 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:14:44,884 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:14:44,893 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:14:45,387 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:14:46,391 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:14:48,925 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:14:49,099 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:14:49,100 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:14:49,100 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:14:49,108 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:14:49,393 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:14:50,394 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:14:53,779 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:14:53,780 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:14:53,780 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:14:53,794 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:14:54,398 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:14:54,795 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:14:56,408 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:14:58,346 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:14:58,346 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:14:58,348 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:14:58,855 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:14:58,856 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:14:58,856 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:14:58,868 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:14:59,411 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:14:59,897 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:15:00,418 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:15:03,569 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:15:03,570 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:15:03,570 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:15:03,581 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:15:04,431 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:15:04,432 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:15:05,581 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:15:07,948 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:15:07,949 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:15:07,949 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:15:07,957 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:15:08,435 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:15:08,949 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 18:15:10,442 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:15:10,950 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:15:13,346 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:15:13,346 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:15:13,348 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:15:13,728 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:15:13,729 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:15:13,729 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:15:13,741 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:15:14,445 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:15:14,446 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:15:16,769 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:15:18,182 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:15:18,182 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:15:18,183 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:15:18,195 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:15:18,452 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:15:20,463 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:15:22,196 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:15:24,469 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:15:24,470 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:15:24,471 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:15:24,480 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:15:25,506 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:15:26,510 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:15:27,480 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:15:28,346 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:15:28,347 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:15:28,348 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:15:29,970 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:15:29,971 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:15:29,971 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:15:29,978 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:15:30,515 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:15:32,521 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:15:33,013 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:15:38,014 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:15:38,940 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 18:15:39,391 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:15:39,392 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:15:39,392 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:15:39,399 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:15:39,528 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:15:40,537 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:15:43,346 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:15:43,347 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:15:43,347 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:15:43,618 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:15:44,052 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:15:44,053 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:15:44,054 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:15:44,061 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:15:44,541 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:15:46,549 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:15:49,098 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:15:49,419 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:15:49,420 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:15:49,420 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:15:49,429 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:15:49,553 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:15:50,559 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:15:54,430 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:15:55,122 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:15:55,123 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:15:55,123 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:15:55,131 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:15:55,566 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:15:56,570 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:15:58,346 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:15:58,347 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:15:58,347 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:15:59,621 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:16:00,132 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:16:00,133 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:16:00,133 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:16:00,151 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:16:00,574 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:16:02,578 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:16:05,174 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:16:05,866 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:16:05,867 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:16:05,867 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:16:05,874 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:16:06,582 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:16:08,587 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:16:08,953 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 18:16:10,954 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:16:13,347 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:16:13,347 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:16:13,348 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:16:14,007 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:16:14,008 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:16:14,008 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:16:14,019 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:16:14,594 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:16:16,049 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:16:16,604 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:16:18,496 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:16:18,497 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:16:18,497 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:16:18,510 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:16:18,607 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:16:20,614 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:16:21,511 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:16:25,966 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:16:25,967 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:16:25,967 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:16:25,974 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:16:26,620 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:16:26,975 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:16:28,347 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:16:28,347 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:16:28,348 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:16:28,632 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:16:32,640 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:16:34,459 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:16:34,459 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:16:34,460 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:16:34,471 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:16:34,642 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:16:36,648 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:16:38,501 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:16:38,945 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 18:16:39,936 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:16:39,937 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:16:39,938 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:16:39,948 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:16:40,652 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:16:42,659 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:16:43,347 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:16:43,347 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:16:43,348 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:16:43,657 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:16:44,621 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:16:44,621 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:16:44,622 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:16:44,631 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:16:44,662 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:16:46,670 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:16:48,666 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:16:49,750 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:16:49,751 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:16:49,751 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:16:49,762 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:16:50,681 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:16:50,681 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:16:53,763 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:16:54,439 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:16:54,440 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:16:54,440 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:16:54,449 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:16:54,687 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:16:56,693 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:16:58,347 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:16:58,347 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:16:58,349 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:16:58,705 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:16:58,705 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:16:58,706 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:16:58,714 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:16:59,699 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:16:59,749 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:17:00,706 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:17:04,750 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:17:05,490 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:17:05,491 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:17:05,492 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:17:05,499 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:17:05,712 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:17:06,718 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:17:08,947 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 18:17:09,948 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:17:10,223 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:17:10,224 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:17:10,224 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:17:10,235 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:17:10,721 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:17:12,726 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:17:13,348 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:17:13,348 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:17:13,348 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:17:14,925 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:17:14,926 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:17:14,926 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:17:14,940 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:17:14,969 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:17:15,730 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:17:16,737 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:17:19,978 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:17:21,091 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:17:21,092 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:17:21,092 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:17:21,101 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:17:21,743 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:17:22,751 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:17:25,102 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:17:25,635 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:17:25,636 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:17:25,636 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:17:25,647 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:17:25,753 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:17:26,754 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:17:28,347 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:17:28,347 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:17:28,349 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:17:30,292 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:17:30,292 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:17:30,293 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:17:30,293 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:17:30,300 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:17:30,758 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:17:32,764 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:17:34,709 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:17:34,710 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:17:34,710 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:17:34,719 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:17:34,766 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:17:35,720 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:17:36,771 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:17:38,949 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 18:17:39,568 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:17:39,569 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:17:39,569 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:17:39,580 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:17:39,773 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:17:40,780 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:17:41,580 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:17:43,347 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:17:43,347 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:17:43,349 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:17:44,886 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:17:44,886 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:17:44,886 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:17:44,895 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:17:45,786 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:17:46,793 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:17:46,929 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:17:49,126 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:17:49,127 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:17:49,127 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:17:49,136 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:17:49,796 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:17:50,801 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:17:52,136 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:17:54,167 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:17:54,168 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:17:54,168 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:17:54,176 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:17:54,805 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:17:56,810 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:17:57,177 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:17:58,347 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:17:58,348 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:17:58,349 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:17:58,943 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:17:58,944 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:17:58,944 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:17:58,951 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:17:59,813 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:18:00,816 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:18:02,985 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:18:04,132 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:18:04,133 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:18:04,133 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:18:04,142 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:18:04,820 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:18:06,825 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:18:08,143 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:18:08,951 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 18:18:08,979 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:18:08,980 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:18:08,980 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:18:08,987 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:18:09,828 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:18:10,833 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:18:13,255 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:18:13,256 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:18:13,256 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:18:13,257 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:18:13,263 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:18:13,347 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:18:13,348 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:18:13,349 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:18:13,836 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:18:14,841 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:18:18,652 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:18:19,022 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:18:19,023 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:18:19,023 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:18:19,029 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:18:19,848 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:18:20,853 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:18:24,066 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:18:24,567 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:18:24,567 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:18:24,568 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:18:24,575 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:18:24,857 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:18:26,861 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:18:28,347 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:18:28,348 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:18:28,348 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:18:29,609 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:18:29,753 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:18:29,753 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:18:29,753 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:18:29,762 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:18:29,864 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:18:30,867 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:18:34,798 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:18:35,559 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:18:35,560 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:18:35,560 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:18:35,568 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:18:35,871 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:18:36,876 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:18:38,954 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 18:18:39,955 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:18:42,234 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:18:42,234 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:18:42,235 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:18:42,241 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:18:42,880 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:18:43,348 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:18:43,348 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:18:43,349 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:18:44,885 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:18:45,648 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:18:47,198 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:18:47,198 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:18:47,199 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:18:47,205 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:18:47,887 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:18:48,891 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:18:51,241 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:18:52,079 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:18:52,080 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:18:52,080 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:18:52,087 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:18:52,895 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:18:54,899 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:18:57,087 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:18:58,238 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:18:58,239 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:18:58,239 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:18:58,245 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:18:58,348 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:18:58,348 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:18:58,348 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:18:58,903 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:19:00,908 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:19:02,628 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:19:03,121 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:19:03,122 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:19:03,122 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:19:03,128 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:19:03,910 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:19:04,914 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:19:07,650 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:19:07,650 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:19:07,651 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:19:07,651 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:19:07,657 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:19:07,916 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:19:08,920 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:19:08,955 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 18:19:12,957 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:19:13,348 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:19:13,348 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:19:13,349 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:19:16,036 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:19:16,036 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:19:16,037 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:19:16,042 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:19:16,927 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:19:18,077 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:19:18,931 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:19:22,778 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:19:22,778 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:19:22,779 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:19:22,784 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:19:22,934 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:19:23,785 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:19:24,938 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:19:28,094 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:19:28,094 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:19:28,095 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:19:28,101 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:19:28,348 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:19:28,348 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:19:28,349 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:19:28,941 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:19:29,563 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:19:30,946 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:19:32,023 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:19:32,024 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:19:32,024 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:19:32,030 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:19:32,947 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:19:34,951 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:19:35,065 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:19:37,537 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:19:37,538 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:19:37,538 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:19:37,544 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:19:37,953 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:19:38,957 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:19:38,961 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 18:19:40,963 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:19:42,990 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:19:42,990 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:19:42,991 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:19:42,996 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:19:43,348 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:19:43,348 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:19:43,349 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:19:43,961 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:19:44,964 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:19:46,597 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:19:49,040 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:19:49,041 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:19:49,041 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:19:49,046 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:19:49,968 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:19:50,971 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:19:52,081 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:19:55,385 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:19:55,385 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:19:55,386 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:19:55,391 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:19:55,975 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:19:56,978 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:19:57,392 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:19:58,348 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:19:58,349 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:19:58,349 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:20:02,603 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:20:03,829 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:20:03,829 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:20:03,829 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:20:03,835 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:20:03,983 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:20:04,986 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:20:07,873 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:20:08,798 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:20:08,799 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:20:08,799 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:20:08,805 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:20:08,960 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 18:20:08,989 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:20:10,992 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:20:12,961 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:20:13,348 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:20:13,348 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:20:13,350 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:20:14,133 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:20:14,134 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:20:14,134 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:20:14,139 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:20:14,995 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:20:16,999 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:20:18,177 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:20:20,849 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:20:20,850 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:20:20,850 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:20:20,856 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:20:21,002 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:20:23,006 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:20:23,856 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:20:25,497 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:20:25,498 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:20:25,498 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:20:25,504 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:20:26,008 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:20:27,011 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:20:28,348 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:20:28,348 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:20:28,349 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:20:29,626 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:20:33,906 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:20:33,906 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:20:33,907 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:20:33,912 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:20:34,016 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:20:34,949 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:20:35,017 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:20:38,962 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 18:20:39,031 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:20:39,031 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:20:39,032 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:20:39,037 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:20:40,020 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:20:40,037 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:20:41,023 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:20:43,348 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:20:43,348 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:20:43,349 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:20:45,618 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:20:45,853 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:20:45,854 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:20:45,854 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:20:45,859 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:20:46,027 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:20:47,030 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:20:50,897 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:20:52,030 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:20:52,031 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:20:52,031 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:20:52,037 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:20:53,036 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:20:53,036 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:20:56,037 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:20:58,076 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:20:58,077 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:20:58,077 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:20:58,082 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:20:58,348 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:20:58,349 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:20:58,349 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:20:59,042 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:20:59,043 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:21:01,592 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:21:02,707 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:21:02,708 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:21:02,708 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:21:02,715 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:21:03,046 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:21:05,049 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:21:06,749 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:21:07,671 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:21:07,672 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:21:07,672 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:21:07,678 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:21:08,051 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:21:08,964 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 18:21:09,054 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:21:11,966 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:21:13,348 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:21:13,349 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:21:13,350 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:21:14,926 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:21:14,927 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:21:14,927 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:21:14,933 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:21:15,058 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:21:16,969 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:21:17,063 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:21:19,474 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:21:19,475 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:21:19,475 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:21:19,481 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:21:20,066 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:21:21,069 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:21:22,481 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:21:24,255 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:21:24,258 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:21:24,258 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:21:24,264 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:21:25,072 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:21:27,075 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:21:28,265 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:21:28,348 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:21:28,349 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:21:28,350 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:21:30,742 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:21:30,743 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:21:30,743 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:21:30,748 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:21:31,078 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:21:33,082 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:21:33,785 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:21:35,547 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:21:35,548 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:21:35,548 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:21:35,554 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:21:36,084 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:21:37,087 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:21:38,966 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 18:21:38,968 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:21:43,349 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:21:43,349 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:21:43,350 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:21:43,851 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:21:43,852 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:21:43,852 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:21:43,857 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:21:44,091 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:21:44,893 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:21:45,094 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:21:48,933 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:21:48,934 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:21:48,934 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:21:48,940 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:21:49,098 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:21:49,941 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:21:51,101 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:21:53,388 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:21:53,389 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:21:53,389 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:21:53,394 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:21:54,104 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:21:55,106 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:21:55,395 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:21:58,349 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:21:58,349 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:21:58,349 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:22:00,596 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:22:01,867 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:22:01,867 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:22:01,867 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:22:01,873 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:22:02,111 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:22:03,114 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:22:05,909 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:22:06,499 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:22:06,499 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:22:06,499 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:22:06,505 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:22:07,117 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:22:08,972 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 18:22:09,120 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:22:10,973 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:22:11,765 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:22:11,765 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:22:11,765 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:22:11,771 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:22:12,122 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:22:13,125 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:22:13,349 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:22:13,349 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:22:13,350 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:22:16,295 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:22:16,296 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:22:16,296 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:22:16,296 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:22:16,302 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:22:17,129 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:22:19,133 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:22:21,337 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:22:22,343 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:22:22,343 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:22:22,344 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:22:22,349 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:22:23,136 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:22:25,140 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:22:26,350 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:22:28,349 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:22:28,349 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:22:28,349 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:22:30,551 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:22:30,551 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:22:30,552 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:22:30,557 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:22:31,144 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:22:31,593 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:22:33,148 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:22:36,593 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:22:38,971 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 18:22:39,008 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:22:39,009 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:22:39,009 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:22:39,015 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:22:39,152 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:22:41,155 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:22:42,015 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:22:43,349 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:22:43,349 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:22:43,351 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:22:43,494 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:22:43,642 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:22:43,642 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:22:43,648 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:22:44,157 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:22:45,160 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:22:47,649 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:22:47,750 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:22:47,751 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:22:47,751 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:22:47,757 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:22:48,162 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:22:49,165 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:22:51,993 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:22:51,994 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:22:51,994 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:22:51,999 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:22:52,167 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:22:53,000 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:22:53,168 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:22:56,269 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:22:56,270 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:22:56,270 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:22:56,275 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:22:57,174 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:22:57,174 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:22:58,276 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:22:58,349 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:22:58,349 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:22:58,350 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:23:03,428 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:23:03,429 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:23:03,429 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:23:03,429 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:23:03,435 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:23:04,178 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:23:05,181 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:23:08,115 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:23:08,116 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:23:08,116 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:23:08,121 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:23:08,183 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:23:08,976 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 18:23:08,978 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:23:09,186 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:23:12,645 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:23:12,646 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:23:12,646 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:23:12,652 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:23:13,189 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:23:13,349 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:23:13,349 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:23:13,351 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:23:14,548 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:23:15,193 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:23:17,786 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:23:17,787 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:23:17,787 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:23:17,793 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:23:18,197 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:23:19,200 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:23:19,829 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:23:22,464 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:23:22,465 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:23:22,466 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:23:22,472 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:23:23,203 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:23:25,204 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:23:25,472 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:23:27,223 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:23:27,223 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:23:27,224 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:23:27,230 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:23:28,206 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:23:28,349 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:23:28,349 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:23:28,351 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:23:29,209 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:23:30,617 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:23:32,350 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:23:32,350 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:23:32,350 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:23:32,356 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:23:33,212 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:23:35,216 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:23:36,393 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:23:37,767 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:23:37,768 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:23:37,768 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:23:37,774 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:23:38,218 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:23:38,975 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 18:23:39,221 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:23:41,977 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:23:43,174 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:23:43,174 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:23:43,175 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:23:43,180 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:23:43,223 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:23:43,349 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:23:43,349 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:23:43,351 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:23:45,227 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:23:47,415 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:23:47,416 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:23:47,416 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:23:47,416 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:23:47,422 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:23:48,229 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:23:49,232 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:23:52,397 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:23:52,397 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:23:52,398 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:23:52,404 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:23:53,235 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:23:53,404 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:23:55,239 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:23:57,318 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:23:57,318 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:23:57,319 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:23:57,325 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:23:58,241 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:23:58,349 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:23:58,350 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:23:58,350 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:23:58,545 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:23:59,244 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:24:02,115 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:24:02,116 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:24:02,116 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:24:02,122 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:24:02,246 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:24:03,249 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:24:04,157 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:24:07,598 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:24:07,599 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:24:07,599 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:24:07,604 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:24:08,253 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:24:08,978 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 18:24:09,253 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:24:09,979 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:24:13,349 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:24:13,350 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:24:13,351 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:24:14,734 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:24:14,737 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:24:14,738 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:24:14,743 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:24:15,257 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:24:15,777 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:24:17,262 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:24:19,438 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:24:19,439 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:24:19,439 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:24:19,445 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:24:20,265 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:24:21,270 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:24:21,445 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:24:24,315 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:24:24,316 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:24:24,316 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:24:24,321 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:24:25,276 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:24:25,276 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:24:27,322 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:24:28,349 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:24:28,350 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:24:28,350 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:24:29,661 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:24:29,661 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:24:29,662 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:24:29,668 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:24:30,280 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:24:31,282 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:24:32,705 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:24:34,117 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:24:34,118 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:24:34,118 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:24:34,124 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:24:34,285 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:24:35,288 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:24:38,125 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:24:38,753 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:24:38,754 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:24:38,754 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:24:38,760 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:24:38,980 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 18:24:39,290 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:24:41,294 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:24:43,350 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:24:43,350 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:24:43,350 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:24:43,613 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:24:44,631 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:24:44,632 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:24:44,632 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:24:44,638 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:24:45,297 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:24:47,300 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:24:48,673 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:24:49,962 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:24:49,963 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:24:49,963 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:24:49,969 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:24:50,302 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:24:51,305 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:24:53,970 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:24:54,508 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:24:54,509 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:24:54,509 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:24:54,515 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:24:55,309 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:24:57,313 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:24:58,350 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:24:58,350 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:24:58,350 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:24:59,622 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:25:00,290 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:25:00,290 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:25:00,290 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:25:00,297 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:25:00,315 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:25:01,318 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:25:04,201 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:25:04,202 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:25:04,202 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:25:04,208 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:25:04,320 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:25:05,209 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:25:05,321 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:25:08,893 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:25:08,894 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:25:08,894 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:25:08,899 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:25:08,982 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 18:25:09,324 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:25:10,984 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:25:11,327 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:25:13,350 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:25:13,350 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:25:13,351 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:25:13,386 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:25:13,599 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:25:13,599 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:25:13,604 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:25:14,329 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:25:15,332 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:25:16,605 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:25:18,373 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:25:18,374 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:25:18,374 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:25:18,379 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:25:19,339 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:25:19,339 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:25:22,380 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:25:24,065 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:25:24,065 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:25:24,066 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:25:24,071 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:25:24,343 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:25:25,345 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:25:28,072 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:25:28,350 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:25:28,350 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:25:28,351 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:25:30,903 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:25:30,904 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:25:30,904 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:25:30,910 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:25:31,350 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:25:33,351 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:25:33,945 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:25:35,595 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:25:35,599 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:25:35,599 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:25:35,604 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:25:36,353 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:25:37,356 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:25:38,984 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 18:25:38,985 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:25:43,350 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:25:43,350 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:25:43,351 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:25:44,670 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:25:45,148 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:25:45,149 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:25:45,149 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:25:45,154 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:25:45,361 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:25:47,365 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:25:50,190 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:25:50,370 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:25:50,371 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:25:50,371 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:25:50,377 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:25:51,370 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:25:51,370 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:25:55,378 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:25:57,952 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:25:57,953 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:25:57,953 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:25:57,959 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:25:58,351 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:25:58,351 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:25:58,351 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:25:58,375 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:25:59,377 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:26:00,592 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:26:04,075 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:26:04,076 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:26:04,076 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:26:04,082 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:26:04,381 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:26:05,383 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:26:06,082 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:26:08,987 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 18:26:09,267 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:26:09,268 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:26:09,268 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:26:09,274 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:26:09,386 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:26:11,275 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:26:11,390 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:26:13,350 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:26:13,350 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:26:13,352 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:26:14,776 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:26:14,777 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:26:14,777 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:26:14,783 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:26:15,392 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:26:16,817 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:26:17,398 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:26:20,048 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:26:20,049 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:26:20,049 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:26:20,054 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:26:20,400 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:26:21,403 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:26:22,055 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:26:27,055 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:26:27,368 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:26:27,369 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:26:27,369 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:26:27,375 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:26:27,407 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:26:28,350 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:26:28,351 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:26:28,351 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:26:29,411 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:26:32,148 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:26:32,149 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:26:32,149 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:26:32,149 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:26:32,154 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:26:32,413 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:26:33,417 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:26:36,958 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:26:36,959 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:26:36,959 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:26:36,965 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:26:37,419 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:26:37,965 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:26:38,992 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 18:26:39,423 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:26:42,439 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:26:42,439 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:26:42,440 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:26:42,445 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:26:43,350 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:26:43,351 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:26:43,351 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:26:43,426 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:26:43,426 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:26:43,660 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:26:48,660 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:26:48,727 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:26:48,727 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:26:48,727 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:26:48,733 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:26:49,430 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:26:51,434 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:26:53,235 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:26:53,236 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:26:53,236 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:26:53,242 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:26:53,436 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:26:54,242 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:26:55,439 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:26:58,350 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:26:58,351 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:26:58,351 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:26:58,401 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:26:58,636 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:26:58,636 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:26:58,642 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:26:59,442 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:26:59,642 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:27:01,446 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:27:03,189 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:27:03,189 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:27:03,190 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:27:03,195 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:27:03,447 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:27:05,195 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:27:05,451 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:27:08,991 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 18:27:10,992 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:27:11,377 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:27:11,378 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:27:11,378 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:27:11,384 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:27:11,455 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:27:13,350 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:27:13,351 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:27:13,351 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:27:13,458 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:27:16,601 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:27:18,875 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:27:18,875 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:27:18,875 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:27:18,881 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:27:19,464 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:27:21,468 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:27:21,917 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:27:24,222 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:27:24,222 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:27:24,223 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:27:24,233 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:27:24,471 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:27:25,474 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:27:27,234 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:27:28,351 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:27:28,351 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:27:28,351 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:27:30,065 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:27:30,065 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:27:30,066 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:27:30,072 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:27:30,477 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:27:31,480 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:27:33,109 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:27:36,124 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:27:36,125 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:27:36,125 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:27:36,131 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:27:36,484 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:27:37,486 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:27:38,132 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:27:38,993 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 18:27:42,354 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:27:42,355 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:27:42,355 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:27:42,360 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:27:42,490 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:27:43,351 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:27:43,351 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:27:43,352 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:27:43,492 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:27:43,641 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:27:46,510 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:27:46,511 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:27:46,511 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:27:46,520 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:27:47,498 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:27:47,498 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:27:49,556 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:27:51,278 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:27:51,278 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:27:51,278 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:27:51,284 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:27:51,500 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:27:53,504 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:27:55,285 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:27:56,050 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:27:56,051 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:27:56,051 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:27:56,057 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:27:56,507 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:27:57,511 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:27:58,351 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:27:58,351 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:27:58,351 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:28:00,576 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:28:01,553 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:28:01,554 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:28:01,554 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:28:01,560 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:28:02,514 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:28:03,515 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:28:05,597 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:28:06,420 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:28:06,421 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:28:06,421 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:28:06,427 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:28:06,517 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:28:07,520 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:28:08,995 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 18:28:10,997 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:28:13,351 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:28:13,351 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:28:13,351 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:28:14,768 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:28:14,769 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:28:14,769 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:28:14,775 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:28:15,525 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:28:16,809 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:28:17,530 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:28:20,076 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:28:20,076 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:28:20,077 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:28:20,083 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:28:20,533 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:28:21,536 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:28:22,083 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:28:24,767 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:28:24,768 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:28:24,768 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:28:24,773 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:28:25,539 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:28:27,542 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:28:27,774 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:28:28,351 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:28:28,351 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:28:28,352 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:28:28,867 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:28:28,868 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:28:28,868 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:28:28,874 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:28:29,543 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:28:31,547 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:28:32,909 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:28:33,669 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:28:33,670 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:28:33,670 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:28:33,676 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:28:34,549 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:28:35,552 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:28:37,702 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:28:37,702 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:28:37,702 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:28:37,708 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:28:38,554 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:28:38,708 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:28:38,998 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 18:28:39,557 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:28:43,351 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:28:43,351 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:28:43,351 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:28:44,600 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:28:44,616 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:28:44,616 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:28:44,616 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:28:44,622 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:28:45,563 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:28:45,564 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:28:49,657 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:28:50,010 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:28:50,011 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:28:50,011 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:28:50,016 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:28:50,567 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:28:51,569 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:28:55,017 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:28:56,444 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:28:56,444 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:28:56,444 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:28:56,451 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:28:56,573 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:28:57,577 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:28:58,351 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:28:58,351 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:28:58,352 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:29:00,632 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:29:04,966 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:29:04,967 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:29:04,967 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:29:04,972 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:29:05,582 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:29:06,009 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:29:07,585 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:29:08,964 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:29:08,965 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:29:08,965 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:29:08,970 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:29:09,003 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 18:29:09,586 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:29:11,588 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:29:12,004 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:29:13,351 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:29:13,352 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:29:13,352 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:29:13,436 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:29:13,619 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:29:13,619 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:29:13,625 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:29:14,590 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:29:15,593 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:29:17,626 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:29:20,025 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:29:20,025 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:29:20,026 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:29:20,031 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:29:20,598 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:29:21,601 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:29:23,032 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:29:24,486 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:29:24,487 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:29:24,487 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:29:24,493 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:29:24,604 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:29:25,607 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:29:28,351 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:29:28,352 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:29:28,352 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:29:28,620 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:29:28,944 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:29:28,945 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:29:28,945 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:29:28,951 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:29:29,610 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:29:31,614 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:29:33,643 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:29:33,644 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:29:33,644 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:29:33,645 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:29:33,650 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:29:34,616 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:29:35,619 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:29:38,175 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:29:38,175 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:29:38,176 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:29:38,182 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:29:38,621 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:29:39,002 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 18:29:39,004 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:29:39,624 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:29:43,351 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:29:43,352 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:29:43,352 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:29:43,370 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:29:43,627 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:29:43,628 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:29:43,634 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:29:44,627 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:29:44,635 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:29:45,630 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:29:47,299 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:29:47,300 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:29:47,300 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:29:47,306 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:29:47,631 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:29:49,635 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:29:50,306 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:29:52,502 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:29:52,503 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:29:52,503 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:29:52,509 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:29:52,637 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:29:53,640 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:29:55,509 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:29:57,872 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:29:57,873 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:29:57,874 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:29:57,879 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:29:58,352 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:29:58,352 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:29:58,352 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:29:58,644 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:29:59,647 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:30:00,648 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:30:04,950 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:30:04,951 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:30:04,951 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:30:04,957 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:30:05,651 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:30:05,993 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:30:07,655 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:30:09,007 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 18:30:09,413 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:30:09,414 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:30:09,414 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:30:09,420 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:30:09,656 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:30:11,420 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:30:11,660 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:30:13,351 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:30:13,352 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:30:13,352 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:30:16,580 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:30:17,676 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:30:17,677 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:30:17,677 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:30:17,683 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:30:18,666 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:30:19,669 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:30:21,717 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:30:21,772 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:30:21,773 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:30:21,773 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:30:21,779 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:30:22,671 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:30:23,674 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:30:26,309 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:30:26,310 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:30:26,310 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:30:26,316 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:30:26,676 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:30:27,316 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:30:27,679 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:30:28,352 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:30:28,352 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:30:28,352 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:30:32,590 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:30:32,641 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:30:32,642 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:30:32,642 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:30:32,649 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:30:32,683 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:30:33,687 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:30:37,686 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:30:39,007 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 18:30:41,165 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:30:41,166 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:30:41,166 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:30:41,172 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:30:41,693 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:30:43,173 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:30:43,352 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:30:43,352 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:30:43,353 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:30:43,696 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:30:46,463 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:30:46,464 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:30:46,464 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:30:46,470 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:30:46,698 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:30:47,701 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:30:48,505 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:30:50,904 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:30:50,905 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:30:50,906 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:30:50,911 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:30:51,704 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:30:53,708 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:30:53,912 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:30:56,509 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:30:56,510 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:30:56,510 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:30:56,516 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:30:56,710 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:30:57,713 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:30:58,352 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:30:58,352 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:30:58,352 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:30:59,642 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:31:02,539 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:31:02,539 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:31:02,540 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:31:02,546 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:31:02,716 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:31:03,719 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:31:05,581 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:31:09,009 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 18:31:09,212 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:31:09,215 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:31:09,215 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:31:09,221 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:31:09,723 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:31:11,221 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:31:11,727 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:31:13,352 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:31:13,352 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:31:13,352 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:31:13,779 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:31:13,780 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:31:13,780 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:31:13,786 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:31:14,729 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:31:15,732 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:31:16,821 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:31:18,556 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:31:18,557 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:31:18,557 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:31:18,563 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:31:18,736 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:31:19,739 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:31:22,564 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:31:23,306 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:31:23,306 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:31:23,307 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:31:23,313 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:31:23,742 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:31:25,745 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:31:28,314 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:31:28,352 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:31:28,352 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:31:28,353 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:31:30,307 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:31:30,307 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:31:30,307 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:31:30,313 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:31:30,749 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:31:31,752 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:31:33,349 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:31:38,350 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:31:38,754 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:31:38,755 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:31:38,755 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:31:38,762 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:31:39,011 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 18:31:39,761 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:31:39,761 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:31:43,352 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:31:43,352 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:31:43,352 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:31:43,599 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:31:44,641 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:31:44,642 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:31:44,642 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:31:44,648 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:31:44,764 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:31:45,767 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:31:48,685 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:31:49,286 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:31:49,287 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:31:49,287 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:31:49,293 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:31:49,770 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:31:51,774 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:31:54,294 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:31:55,709 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:31:55,710 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:31:55,710 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:31:55,717 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:31:55,777 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:31:57,781 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:31:58,352 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:31:58,352 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:31:58,353 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:31:59,585 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:32:02,978 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:32:02,979 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:32:02,979 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:32:02,985 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:32:03,785 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:32:05,021 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:32:05,788 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:32:07,936 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:32:07,937 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:32:07,937 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:32:07,944 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:32:08,790 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:32:09,013 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 18:32:09,793 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:32:11,015 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:32:12,626 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:32:12,627 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:32:12,628 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:32:12,634 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:32:12,795 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:32:13,352 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:32:13,352 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:32:13,353 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:32:13,796 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:32:16,619 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:32:17,867 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:32:17,868 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:32:17,868 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:32:17,874 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:32:18,802 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:32:19,804 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:32:21,909 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:32:22,637 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:32:22,638 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:32:22,638 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:32:22,644 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:32:22,807 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:32:23,810 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:32:27,645 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:32:28,352 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:32:28,353 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:32:28,353 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:32:28,589 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:32:28,589 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:32:28,590 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:32:28,596 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:32:28,813 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:32:29,816 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:32:33,634 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:32:37,039 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:32:37,039 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:32:37,040 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:32:37,045 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:32:37,822 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:32:39,018 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 18:32:39,021 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:32:39,826 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:32:42,530 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:32:42,531 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:32:42,531 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:32:42,537 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:32:42,828 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:32:43,352 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:32:43,353 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:32:43,353 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:32:43,830 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:32:44,645 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:32:48,741 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:32:48,742 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:32:48,742 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:32:48,748 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:32:48,834 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:32:49,785 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:32:49,837 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:32:52,667 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:32:52,668 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:32:52,669 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:32:52,675 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:32:52,839 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:32:53,842 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:32:55,675 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:32:57,189 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:32:57,189 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:32:57,190 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:32:57,196 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:32:57,845 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:32:58,352 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:32:58,353 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:32:58,353 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:32:59,849 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:33:01,605 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:33:01,693 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:33:01,693 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:33:01,694 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:33:01,700 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:33:01,851 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:33:03,854 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:33:06,274 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:33:06,274 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:33:06,275 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:33:06,280 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:33:06,856 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:33:07,281 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:33:07,859 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:33:09,017 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 18:33:11,177 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:33:11,178 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:33:11,178 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:33:11,185 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:33:11,862 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:33:13,185 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:33:13,352 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:33:13,353 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:33:13,354 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:33:13,866 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:33:17,570 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:33:17,571 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:33:17,571 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:33:17,578 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:33:17,870 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:33:18,613 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:33:19,874 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:33:22,447 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:33:22,448 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:33:22,448 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:33:22,454 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:33:22,877 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:33:23,880 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:33:24,455 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:33:28,353 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:33:28,353 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:33:28,353 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:33:28,923 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:33:28,924 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:33:28,924 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:33:28,930 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:33:29,887 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:33:29,887 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:33:29,965 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:33:33,993 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:33:33,993 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:33:33,994 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:33:33,999 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:33:34,891 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:33:35,000 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:33:35,893 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:33:38,838 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:33:38,839 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:33:38,839 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:33:38,845 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:33:38,895 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:33:39,024 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 18:33:39,898 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:33:40,025 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:33:43,285 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:33:43,287 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:33:43,287 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:33:43,294 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:33:43,353 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:33:43,353 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:33:43,354 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:33:43,901 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:33:45,736 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:33:45,904 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:33:47,740 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:33:47,741 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:33:47,741 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:33:47,747 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:33:47,906 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:33:49,909 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:33:50,781 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:33:52,859 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:33:52,860 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:33:52,860 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:33:52,866 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:33:52,911 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:33:53,914 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:33:55,867 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:33:58,287 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:33:58,288 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:33:58,288 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:33:58,294 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:33:58,353 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:33:58,353 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:33:58,353 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:33:58,918 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:33:59,921 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:34:01,555 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:34:06,556 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:34:06,657 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:34:06,658 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:34:06,658 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:34:06,665 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:34:06,926 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:34:07,928 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:34:09,023 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 18:34:11,789 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:34:11,790 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:34:11,790 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:34:11,790 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:34:11,796 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:34:11,931 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:34:13,353 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:34:13,353 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:34:13,353 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:34:13,935 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:34:16,032 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:34:16,033 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:34:16,033 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:34:16,040 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:34:16,939 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:34:17,073 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:34:17,942 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:34:20,582 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:34:20,583 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:34:20,583 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:34:20,589 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:34:20,944 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:34:21,947 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:34:22,590 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:34:25,297 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:34:25,298 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:34:25,298 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:34:25,304 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:34:25,950 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:34:27,952 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:34:28,305 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:34:28,353 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:34:28,353 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:34:28,353 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:34:29,845 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:34:29,846 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:34:29,846 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:34:29,852 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:34:29,953 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:34:31,957 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:34:33,890 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:34:34,315 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:34:34,316 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:34:34,316 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:34:34,322 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:34:34,959 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:34:35,962 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:34:39,028 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 18:34:39,030 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:34:39,038 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:34:39,038 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:34:39,039 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:34:39,045 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:34:39,966 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:34:39,967 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:34:43,348 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:34:43,349 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:34:43,349 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:34:43,353 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:34:43,353 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:34:43,355 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:34:43,355 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:34:43,969 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:34:44,565 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:34:45,973 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:34:49,565 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:34:51,894 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:34:51,895 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:34:51,895 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:34:51,902 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:34:51,977 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:34:53,981 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:34:54,937 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:34:56,135 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:34:56,136 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:34:56,136 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:34:56,142 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:34:56,983 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:34:57,986 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:34:58,353 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:34:58,354 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:34:58,354 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:35:00,658 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:35:00,709 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:35:00,710 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:35:00,710 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:35:00,716 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:35:00,989 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:35:01,992 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:35:05,177 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:35:05,178 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:35:05,178 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:35:05,184 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:35:05,994 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:35:06,184 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:35:07,998 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:35:09,028 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 18:35:10,413 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:35:10,414 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:35:10,414 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:35:10,420 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:35:11,000 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:35:11,420 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:35:12,003 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:35:13,353 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:35:13,353 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:35:13,354 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:35:15,802 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:35:15,802 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:35:15,803 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:35:15,812 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:35:16,005 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:35:16,845 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:35:18,011 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:35:21,846 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:35:22,640 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:35:22,640 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:35:22,641 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:35:22,650 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:35:23,015 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:35:24,018 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:35:27,281 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:35:27,282 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:35:27,282 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:35:27,283 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:35:27,290 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:35:28,021 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:35:28,353 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:35:28,354 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:35:28,354 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:35:30,025 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:35:31,598 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:35:31,598 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:35:31,599 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:35:31,604 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:35:32,027 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:35:32,641 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:35:34,030 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:35:37,642 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:35:38,409 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:35:38,410 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:35:38,410 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:35:38,416 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:35:39,030 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 18:35:39,033 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:35:40,036 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:35:43,031 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:35:43,123 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:35:43,124 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:35:43,124 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:35:43,130 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:35:43,353 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:35:43,354 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:35:43,354 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:35:44,039 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:35:44,039 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:35:48,612 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:35:51,414 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:35:51,414 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:35:51,415 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:35:51,420 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:35:52,044 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:35:54,048 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:35:54,457 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:35:57,157 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:35:57,158 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:35:57,158 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:35:57,164 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:35:58,051 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:35:58,353 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:35:58,354 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:35:58,355 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:35:59,634 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:36:00,055 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:36:04,084 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:36:04,085 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:36:04,085 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:36:04,091 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:36:05,059 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:36:05,128 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:36:06,062 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:36:09,032 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 18:36:10,504 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:36:10,505 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:36:10,505 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:36:10,505 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:36:10,511 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:36:11,065 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:36:12,068 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:36:13,353 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:36:13,354 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:36:13,354 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:36:15,576 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:36:15,820 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:36:15,820 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:36:15,820 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:36:15,826 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:36:16,071 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:36:18,076 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:36:20,861 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:36:21,282 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:36:21,283 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:36:21,283 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:36:21,289 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:36:22,079 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:36:24,083 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:36:25,451 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:36:25,452 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:36:25,452 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:36:25,458 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:36:26,085 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:36:26,459 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:36:28,088 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:36:28,354 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:36:28,354 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:36:28,354 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:36:30,989 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:36:30,990 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:36:30,990 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:36:30,996 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:36:31,090 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:36:32,033 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:36:32,093 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:36:36,914 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:36:36,915 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:36:36,915 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:36:36,921 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:36:37,097 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:36:37,921 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:36:38,101 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:36:39,034 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 18:36:43,036 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:36:43,354 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:36:43,354 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:36:43,355 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:36:45,498 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:36:45,498 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:36:45,499 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:36:45,504 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:36:46,106 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:36:48,109 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:36:48,541 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:36:50,076 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:36:50,077 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:36:50,077 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:36:50,083 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:36:50,111 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:36:52,114 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:36:54,084 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:36:56,392 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:36:56,393 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:36:56,393 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:36:56,399 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:36:57,118 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:36:58,121 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:36:58,354 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:36:58,354 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:36:58,355 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:36:59,599 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:37:01,917 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:37:01,918 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:37:01,918 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:37:01,923 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:37:02,124 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:37:04,128 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:37:04,961 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:37:06,421 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:37:06,422 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:37:06,422 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:37:06,428 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:37:07,130 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:37:08,133 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:37:09,039 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 18:37:10,041 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:37:10,409 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:37:10,410 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:37:10,410 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:37:10,421 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:37:11,135 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:37:12,138 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:37:13,354 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:37:13,354 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:37:13,354 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:37:15,639 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:37:15,913 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:37:15,914 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:37:15,914 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:37:15,920 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:37:16,141 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:37:18,146 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:37:20,378 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:37:20,379 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:37:20,379 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:37:20,385 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:37:21,148 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:37:21,386 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:37:22,151 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:37:25,045 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:37:25,046 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:37:25,046 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:37:25,053 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:37:25,154 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:37:26,157 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:37:27,054 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:37:28,354 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:37:28,354 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:37:28,354 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:37:29,888 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:37:29,888 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:37:29,888 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:37:29,896 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:37:30,160 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:37:32,164 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:37:32,929 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:37:34,465 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:37:34,466 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:37:34,467 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:37:34,472 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:37:35,166 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:37:36,169 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:37:38,473 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:37:39,038 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 18:37:41,521 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:37:41,521 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:37:41,522 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:37:41,527 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:37:42,173 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:37:43,354 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:37:43,354 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:37:43,355 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:37:43,601 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:37:44,175 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:37:46,460 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:37:46,460 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:37:46,461 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:37:46,467 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:37:47,177 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:37:48,180 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:37:49,501 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:37:51,630 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:37:51,631 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:37:51,631 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:37:51,636 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:37:52,182 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:37:54,186 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:37:54,637 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:37:56,886 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:37:56,887 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:37:56,887 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:37:56,893 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:37:57,189 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:37:58,192 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:37:58,354 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:37:58,354 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:37:58,354 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:38:00,578 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:38:01,271 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:38:01,271 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:38:01,272 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:38:01,278 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:38:02,194 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:38:04,198 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:38:06,313 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:38:07,653 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:38:07,654 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:38:07,654 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:38:07,660 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:38:08,201 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:38:09,043 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 18:38:10,205 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:38:12,045 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:38:12,796 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:38:12,796 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:38:12,797 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:38:12,802 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:38:13,207 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:38:13,354 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:38:13,354 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:38:13,355 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:38:14,210 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:38:17,367 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:38:17,368 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:38:17,368 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:38:17,369 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:38:17,374 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:38:18,215 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:38:20,219 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:38:22,409 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:38:26,207 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:38:26,208 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:38:26,208 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:38:26,214 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:38:26,223 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:38:28,214 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:38:28,227 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:38:28,354 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:38:28,354 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:38:28,356 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:38:31,303 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:38:31,304 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:38:31,304 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:38:31,310 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:38:32,230 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:38:33,345 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:38:34,234 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:38:36,240 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:38:36,241 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:38:36,241 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:38:36,247 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:38:37,236 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:38:38,239 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:38:39,043 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 18:38:39,044 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:38:40,766 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:38:40,767 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:38:40,767 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:38:40,773 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:38:41,241 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:38:42,244 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:38:43,354 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:38:43,355 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:38:43,356 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:38:44,597 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:38:45,630 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:38:45,630 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:38:45,630 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:38:45,637 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:38:46,247 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:38:48,250 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:38:49,673 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:38:50,756 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:38:50,757 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:38:50,757 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:38:50,763 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:38:51,252 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:38:52,255 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:38:54,764 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:38:55,358 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:38:55,358 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:38:55,359 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:38:55,364 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:38:56,258 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:38:58,262 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:38:58,354 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:38:58,355 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:38:58,356 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:39:00,611 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:39:03,619 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:39:03,620 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:39:03,620 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:39:03,626 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:39:04,266 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:39:05,661 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:39:06,267 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:39:09,045 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 18:39:09,961 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:39:09,962 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:39:09,962 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:39:09,967 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:39:10,270 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:39:10,968 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:39:12,273 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:39:13,354 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:39:13,355 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:39:13,355 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:39:14,739 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:39:14,739 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:39:14,740 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:39:14,745 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:39:15,276 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:39:16,279 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:39:16,781 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:39:19,892 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:39:19,893 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:39:19,893 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:39:19,899 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:39:20,284 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:39:21,899 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:39:22,285 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:39:25,273 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:39:25,274 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:39:25,274 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:39:25,280 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:39:25,288 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:39:26,290 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:39:27,280 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:39:28,355 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:39:28,355 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:39:28,356 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:39:30,113 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:39:30,113 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:39:30,114 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:39:30,125 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:39:30,293 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:39:32,297 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:39:33,160 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:39:35,647 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:39:35,648 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:39:35,648 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:39:35,653 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:39:36,300 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:39:38,302 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:39:38,654 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:39:39,047 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 18:39:40,038 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:39:40,038 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:39:40,038 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:39:40,044 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:39:40,303 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:39:42,307 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:39:43,355 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:39:43,355 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:39:43,355 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:39:44,351 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:39:44,352 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:39:44,352 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:39:44,352 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:39:44,357 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:39:45,309 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:39:46,313 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:39:48,863 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:39:48,864 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:39:48,864 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:39:48,870 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:39:49,315 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:39:49,871 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:39:50,317 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:39:54,871 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:39:54,998 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:39:54,999 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:39:54,999 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:39:55,005 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:39:55,321 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:39:56,324 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:39:58,355 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:39:58,355 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:39:58,355 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:40:00,627 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:40:01,800 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:40:01,801 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:40:01,801 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:40:01,807 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:40:02,329 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:40:04,332 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:40:05,845 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:40:06,942 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:40:06,942 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:40:06,943 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:40:06,948 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:40:07,334 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:40:08,337 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:40:09,049 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 18:40:11,051 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:40:11,310 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:40:11,311 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:40:11,311 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:40:11,317 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:40:11,339 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:40:12,340 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:40:13,355 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:40:13,355 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:40:13,355 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:40:15,827 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:40:15,827 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:40:15,827 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:40:15,833 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:40:16,343 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:40:16,869 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:40:18,348 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:40:21,330 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:40:21,331 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:40:21,331 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:40:21,337 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:40:21,351 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:40:22,337 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:40:22,354 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:40:26,682 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:40:26,682 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:40:26,683 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:40:26,688 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:40:27,358 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:40:27,689 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:40:28,355 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:40:28,355 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:40:28,356 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:40:28,360 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:40:31,349 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:40:31,350 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:40:31,350 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:40:31,356 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:40:31,363 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:40:32,363 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:40:33,393 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:40:35,638 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:40:35,638 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:40:35,639 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:40:35,645 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:40:36,366 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:40:38,370 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:40:38,646 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:40:39,054 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 18:40:43,355 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:40:43,355 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:40:43,355 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:40:44,045 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:40:44,045 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:40:44,045 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:40:44,045 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:40:44,051 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:40:44,374 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:40:46,378 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:40:49,089 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:40:50,649 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:40:50,650 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:40:50,650 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:40:50,656 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:40:51,381 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:40:52,384 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:40:54,656 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:40:58,355 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:40:58,355 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:40:58,357 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:40:59,357 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:40:59,358 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:40:59,358 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:40:59,365 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:40:59,390 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:41:00,390 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:41:00,401 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:41:03,820 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:41:03,820 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:41:03,821 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:41:03,827 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:41:04,393 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:41:05,828 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:41:06,397 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:41:08,399 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:41:08,400 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:41:08,400 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:41:08,407 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:41:09,054 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 18:41:09,399 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:41:10,402 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:41:11,055 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:41:12,286 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:41:12,287 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:41:12,287 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:41:12,293 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:41:12,404 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:41:13,355 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:41:13,356 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:41:13,356 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:41:14,407 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:41:16,603 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:41:20,953 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:41:20,953 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:41:20,953 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:41:20,959 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:41:21,414 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:41:21,997 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:41:22,417 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:41:26,997 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:41:27,163 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:41:27,164 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:41:27,164 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:41:27,170 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:41:27,421 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:41:28,355 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:41:28,356 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:41:28,356 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:41:28,424 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:41:31,611 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:41:31,612 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:41:31,612 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:41:31,618 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:41:32,427 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:41:32,653 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:41:34,430 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:41:36,953 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:41:36,954 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:41:36,954 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:41:36,959 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:41:37,432 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:41:37,960 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:41:38,436 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:41:39,059 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 18:41:43,060 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:41:43,355 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:41:43,355 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:41:43,357 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:41:45,481 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:41:45,482 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:41:45,482 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:41:45,488 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:41:46,443 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:41:46,444 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:41:48,525 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:41:50,550 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:41:50,551 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:41:50,551 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:41:50,557 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:41:51,447 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:41:52,450 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:41:53,558 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:41:56,809 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:41:56,810 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:41:56,810 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:41:56,815 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:41:57,454 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:41:58,355 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:41:58,356 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:41:58,357 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:41:58,457 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:41:58,635 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:42:01,343 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:42:01,344 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:42:01,344 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:42:01,350 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:42:01,460 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:42:02,460 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:42:04,385 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:42:06,593 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:42:06,594 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:42:06,594 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:42:06,600 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:42:07,464 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:42:08,465 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:42:09,058 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 18:42:10,059 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:42:13,162 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:42:13,163 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:42:13,163 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:42:13,169 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:42:13,355 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:42:13,356 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:42:13,357 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:42:13,468 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:42:14,474 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:42:15,622 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:42:18,663 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:42:18,664 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:42:18,664 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:42:18,670 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:42:19,480 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:42:20,484 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:42:20,705 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:42:23,127 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:42:23,128 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:42:23,128 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:42:23,133 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:42:23,486 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:42:24,489 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:42:26,134 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:42:27,498 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:42:27,498 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:42:27,499 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:42:27,504 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:42:28,356 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:42:28,356 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:42:28,356 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:42:28,494 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:42:28,494 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:42:31,616 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:42:35,829 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:42:35,830 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:42:35,830 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:42:35,835 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:42:36,500 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:42:36,873 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:42:38,504 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:42:39,060 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 18:42:40,150 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:42:40,151 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:42:40,151 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:42:40,157 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:42:40,505 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:42:42,157 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:42:42,507 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:42:43,356 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:42:43,356 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:42:43,357 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:42:45,248 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:42:45,248 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:42:45,249 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:42:45,254 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:42:45,509 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:42:46,512 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:42:47,289 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:42:49,619 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:42:49,620 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:42:49,620 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:42:49,626 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:42:50,515 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:42:52,519 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:42:52,626 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:42:54,276 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:42:54,280 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:42:54,280 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:42:54,285 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:42:54,521 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:42:56,524 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:42:58,286 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:42:58,356 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:42:58,356 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:42:58,357 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:42:59,754 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:42:59,754 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:42:59,755 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:42:59,760 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:43:00,528 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:43:02,531 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:43:03,797 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:43:04,383 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:43:04,383 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:43:04,383 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:43:04,389 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:43:04,533 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:43:06,536 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:43:09,063 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 18:43:09,064 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:43:09,477 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:43:09,477 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:43:09,477 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:43:09,483 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:43:09,538 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:43:10,542 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:43:13,356 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:43:13,356 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:43:13,356 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:43:14,624 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:43:14,813 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:43:14,814 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:43:14,814 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:43:14,820 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:43:15,546 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:43:16,548 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:43:19,857 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:43:21,224 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:43:21,225 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:43:21,225 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:43:21,231 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:43:21,554 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:43:22,557 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:43:25,232 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:43:27,400 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:43:27,400 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:43:27,401 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:43:27,406 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:43:27,561 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:43:28,356 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:43:28,356 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:43:28,356 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:43:28,564 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:43:30,590 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:43:33,174 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:43:33,174 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:43:33,174 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:43:33,180 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:43:33,568 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:43:34,571 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:43:36,217 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:43:37,542 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:43:37,543 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:43:37,543 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:43:37,548 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:43:37,573 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:43:38,576 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:43:39,065 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 18:43:42,066 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:43:42,510 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:43:42,511 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:43:42,511 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:43:42,517 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:43:42,579 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:43:43,356 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:43:43,356 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:43:43,357 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:43:44,582 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:43:47,572 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:43:48,494 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:43:48,494 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:43:48,494 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:43:48,501 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:43:48,585 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:43:50,589 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:43:53,537 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:43:55,307 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:43:55,308 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:43:55,308 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:43:55,314 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:43:55,593 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:43:56,596 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:43:58,356 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:43:58,356 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:43:58,358 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:43:58,603 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:43:59,888 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:43:59,888 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:43:59,888 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:43:59,894 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:44:00,599 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:44:02,602 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:44:03,929 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:44:07,061 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:44:07,062 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:44:07,062 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:44:07,068 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:44:07,606 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:44:08,609 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:44:09,068 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:44:09,070 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 18:44:13,356 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:44:13,356 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:44:13,358 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:44:13,414 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:44:13,637 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:44:13,638 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:44:13,643 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:44:14,613 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:44:14,613 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:44:14,644 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:44:17,879 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:44:17,880 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:44:17,880 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:44:17,886 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:44:18,619 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:44:19,886 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:44:20,623 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:44:24,503 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:44:24,504 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:44:24,504 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:44:24,510 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:44:24,626 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:44:25,510 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:44:26,629 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:44:28,356 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:44:28,357 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:44:28,357 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:44:29,811 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:44:29,811 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:44:29,812 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:44:29,817 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:44:30,632 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:44:30,853 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:44:32,637 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:44:35,854 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:44:36,468 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:44:36,468 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:44:36,469 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:44:36,474 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:44:36,639 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:44:38,643 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:44:39,069 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 18:44:41,070 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:44:43,254 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:44:43,255 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:44:43,255 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:44:43,261 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:44:43,356 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:44:43,357 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:44:43,358 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:44:43,647 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:44:44,650 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:44:46,582 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:44:51,582 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:44:52,444 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:44:52,445 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:44:52,445 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:44:52,452 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:44:52,657 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:44:54,661 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:44:57,489 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:44:57,642 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:44:57,643 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:44:57,643 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:44:57,649 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:44:57,663 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:44:58,356 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:44:58,357 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:44:58,358 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:44:58,666 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:45:02,564 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:45:03,283 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:45:03,284 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:45:03,284 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:45:03,290 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:45:03,670 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:45:04,673 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:45:08,326 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:45:08,599 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:45:08,600 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:45:08,600 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:45:08,605 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:45:08,676 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:45:09,074 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 18:45:10,680 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:45:13,357 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:45:13,357 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:45:13,358 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:45:13,582 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:45:14,306 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:45:14,306 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:45:14,306 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:45:14,312 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:45:14,683 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:45:16,686 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:45:19,198 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:45:19,198 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:45:19,198 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:45:19,199 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:45:19,204 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:45:19,690 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:45:20,693 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:45:23,345 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:45:23,346 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:45:23,346 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:45:23,352 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:45:23,696 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:45:24,352 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:45:24,698 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:45:27,620 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:45:27,620 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:45:27,620 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:45:27,626 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:45:27,700 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:45:28,357 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:45:28,357 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:45:28,357 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:45:28,703 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:45:29,644 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:45:32,923 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:45:32,924 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:45:32,924 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:45:32,930 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:45:33,708 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:45:34,711 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:45:34,965 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:45:38,418 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:45:38,419 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:45:38,419 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:45:38,425 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:45:38,713 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:45:39,074 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 18:45:40,075 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:45:40,717 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:45:43,357 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:45:43,357 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:45:43,357 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:45:44,576 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:45:44,577 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:45:44,577 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:45:44,582 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:45:44,719 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:45:45,617 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:45:46,723 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:45:50,617 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:45:53,016 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:45:53,016 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:45:53,017 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:45:53,022 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:45:53,729 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:45:54,731 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:45:56,023 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:45:57,920 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:45:57,921 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:45:57,921 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:45:57,926 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:45:58,357 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:45:58,357 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:45:58,359 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:45:58,735 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:46:00,739 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:46:01,558 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:46:03,327 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:46:03,328 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:46:03,328 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:46:03,337 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:46:03,741 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:46:04,744 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:46:07,369 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:46:07,784 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:46:07,785 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:46:07,785 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:46:07,794 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:46:08,749 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:46:08,749 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:46:09,076 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 18:46:12,398 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:46:12,399 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:46:12,399 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:46:12,399 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:46:12,405 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:46:12,751 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:46:13,357 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:46:13,357 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:46:13,357 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:46:14,755 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:46:17,544 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:46:20,705 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:46:20,705 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:46:20,705 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:46:20,711 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:46:20,761 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:46:22,749 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:46:22,765 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:46:25,494 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:46:25,495 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:46:25,495 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:46:25,501 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:46:25,768 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:46:26,770 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:46:28,357 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:46:28,357 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:46:28,357 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:46:28,555 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:46:29,939 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:46:29,940 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:46:29,940 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:46:29,946 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:46:30,773 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:46:32,777 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:46:33,981 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:46:35,239 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:46:35,240 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:46:35,240 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:46:35,246 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:46:35,780 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:46:36,782 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:46:39,078 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 18:46:39,080 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:46:40,457 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:46:40,457 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:46:40,457 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:46:40,463 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:46:40,785 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:46:42,789 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:46:43,357 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:46:43,357 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:46:43,358 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:46:44,600 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:46:44,985 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:46:44,986 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:46:44,986 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:46:44,992 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:46:45,792 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:46:46,795 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:46:49,530 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:46:49,531 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:46:49,531 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:46:49,536 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:46:49,798 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:46:50,537 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:46:50,800 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:46:55,537 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:46:55,661 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:46:55,662 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:46:55,662 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:46:55,667 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:46:55,804 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:46:56,807 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:46:58,357 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:46:58,358 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:46:58,358 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:47:00,334 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:47:00,334 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:47:00,334 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:47:00,340 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:47:00,812 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:47:01,377 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:47:02,816 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:47:06,378 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:47:08,879 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:47:08,880 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:47:08,880 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:47:08,885 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:47:09,081 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 18:47:09,821 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:47:10,824 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:47:12,082 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:47:13,357 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:47:13,357 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:47:13,359 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:47:13,964 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:47:13,964 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:47:13,965 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:47:13,970 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:47:14,827 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:47:16,831 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:47:18,005 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:47:18,487 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:47:18,487 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:47:18,488 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:47:18,493 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:47:18,835 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:47:20,839 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:47:23,068 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:47:23,069 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:47:23,069 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:47:23,069 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:47:23,075 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:47:23,841 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:47:24,844 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:47:27,641 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:47:27,642 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:47:27,642 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:47:27,648 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:47:27,846 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:47:28,357 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:47:28,358 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:47:28,358 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:47:28,647 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:47:28,849 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:47:32,354 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:47:32,354 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:47:32,354 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:47:32,360 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:47:32,852 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:47:34,397 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:47:34,856 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:47:38,432 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:47:38,433 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:47:38,433 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:47:38,439 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:47:38,859 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:47:39,086 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 18:47:40,088 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:47:40,863 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:47:43,213 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:47:43,214 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:47:43,214 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:47:43,220 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:47:43,357 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:47:43,358 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:47:43,358 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:47:43,865 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:47:44,868 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:47:45,569 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:47:48,021 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:47:48,022 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:47:48,022 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:47:48,028 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:47:48,871 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:47:50,875 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:47:51,065 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:47:53,782 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:47:53,783 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:47:53,783 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:47:53,789 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:47:53,877 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:47:54,880 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:47:56,789 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:47:58,357 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:47:58,358 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:47:58,359 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:48:00,057 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:48:00,058 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:48:00,058 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:48:00,063 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:48:00,885 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:48:02,101 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:48:02,889 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:48:07,102 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:48:08,507 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:48:08,508 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:48:08,508 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:48:08,513 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:48:08,893 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:48:09,085 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 18:48:10,897 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:48:12,937 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:48:12,938 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:48:12,938 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:48:12,938 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:48:12,944 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:48:13,357 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:48:13,358 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:48:13,359 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:48:13,899 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:48:14,902 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:48:17,619 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:48:17,619 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:48:17,620 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:48:17,626 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:48:17,906 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:48:18,661 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:48:18,909 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:48:21,884 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:48:21,885 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:48:21,885 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:48:21,891 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:48:21,911 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:48:22,914 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:48:23,891 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:48:26,010 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:48:26,011 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:48:26,011 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:48:26,017 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:48:26,918 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:48:28,358 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:48:28,358 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:48:28,359 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:48:28,924 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:48:29,655 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:48:31,265 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:48:31,265 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:48:31,266 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:48:31,272 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:48:31,927 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:48:32,930 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:48:35,309 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:48:35,497 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:48:35,498 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:48:35,498 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:48:35,505 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:48:35,932 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:48:36,935 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:48:39,090 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 18:48:40,335 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:48:40,336 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:48:40,336 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:48:40,336 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:48:40,342 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:48:40,937 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:48:42,941 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:48:43,358 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:48:43,358 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:48:43,358 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:48:45,053 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:48:45,054 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:48:45,054 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:48:45,060 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:48:45,943 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:48:46,097 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:48:46,946 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:48:49,709 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:48:49,710 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:48:49,710 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:48:49,715 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:48:49,948 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:48:50,951 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:48:51,716 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:48:55,291 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:48:55,292 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:48:55,292 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:48:55,298 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:48:55,955 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:48:56,956 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:48:57,298 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:48:58,358 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:48:58,358 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:48:58,359 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:49:01,210 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:49:01,211 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:49:01,211 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:49:01,216 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:49:01,959 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:49:02,960 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:49:03,253 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:49:05,629 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:49:05,629 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:49:05,630 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:49:05,636 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:49:05,963 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:49:06,965 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:49:08,636 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:49:09,090 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 18:49:11,996 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:49:11,997 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:49:11,997 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:49:12,003 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:49:12,972 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:49:12,972 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:49:13,358 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:49:13,358 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:49:13,360 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:49:13,687 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:49:17,397 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:49:17,398 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:49:17,398 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:49:17,404 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:49:17,977 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:49:18,980 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:49:19,441 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:49:23,325 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:49:23,326 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:49:23,326 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:49:23,332 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:49:23,984 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:49:24,987 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:49:25,333 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:49:27,605 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:49:27,606 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:49:27,606 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:49:27,612 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:49:27,989 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:49:28,358 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:49:28,358 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:49:28,360 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:49:28,990 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:49:30,631 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:49:31,650 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:49:31,651 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:49:31,651 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:49:31,657 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:49:31,993 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:49:32,996 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:49:35,693 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:49:36,926 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:49:36,927 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:49:36,927 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:49:36,933 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:49:36,999 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:49:39,003 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:49:39,095 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 18:49:41,096 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:49:41,431 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:49:41,432 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:49:41,432 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:49:41,438 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:49:42,005 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:49:43,008 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:49:43,358 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:49:43,358 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:49:43,359 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:49:46,654 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:49:47,590 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:49:47,590 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:49:47,590 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:49:47,596 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:49:48,012 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:49:49,015 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:49:52,633 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:49:53,288 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:49:53,289 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:49:53,289 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:49:53,295 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:49:54,019 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:49:55,023 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:49:58,295 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:49:58,358 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:49:58,358 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:49:58,359 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:50:00,275 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:50:00,276 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:50:00,276 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:50:00,282 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:50:01,027 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:50:03,031 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:50:03,317 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:50:04,938 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:50:04,939 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:50:04,939 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:50:04,944 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:50:05,033 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:50:07,036 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:50:08,945 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:50:09,094 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 18:50:09,115 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:50:09,115 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:50:09,116 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:50:09,121 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:50:10,038 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:50:11,041 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:50:13,358 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:50:13,359 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:50:13,360 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:50:14,209 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:50:14,210 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:50:14,210 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:50:14,210 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:50:14,216 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:50:15,044 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:50:17,048 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:50:19,254 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:50:20,006 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:50:20,007 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:50:20,008 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:50:20,013 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:50:20,052 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:50:21,055 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:50:24,910 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:50:24,911 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:50:24,911 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:50:24,911 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:50:24,917 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:50:25,058 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:50:27,062 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:50:28,358 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:50:28,359 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:50:28,360 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:50:30,598 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:50:32,078 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:50:32,079 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:50:32,079 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:50:32,084 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:50:33,068 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:50:33,069 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:50:36,121 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:50:36,342 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:50:36,343 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:50:36,343 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:50:36,349 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:50:37,071 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:50:39,075 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:50:39,096 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 18:50:41,231 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:50:41,232 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:50:41,232 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:50:41,232 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:50:41,238 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:50:42,078 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:50:43,080 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:50:43,358 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:50:43,359 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:50:43,359 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:50:45,863 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:50:45,863 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:50:45,863 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:50:45,869 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:50:46,082 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:50:46,905 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:50:47,085 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:50:51,906 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:50:54,750 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:50:54,750 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:50:54,751 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:50:54,757 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:50:55,092 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:50:57,095 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:50:57,757 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:50:58,358 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:50:58,359 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:50:58,360 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:50:59,593 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:50:59,594 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:50:59,594 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:50:59,600 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:51:00,098 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:51:01,101 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:51:03,637 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:51:04,265 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:51:04,266 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:51:04,266 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:51:04,272 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:51:05,104 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:51:07,107 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:51:09,101 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 18:51:09,102 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:51:13,071 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:51:13,072 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:51:13,073 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:51:13,078 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:51:13,111 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:51:13,359 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:51:13,359 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:51:13,360 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:51:14,630 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:51:15,113 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:51:19,631 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:51:22,354 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:51:22,354 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:51:22,355 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:51:22,361 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:51:23,122 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:51:25,126 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:51:25,397 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:51:26,617 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:51:26,617 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:51:26,618 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:51:26,623 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:51:27,127 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:51:28,359 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:51:28,359 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:51:28,360 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:51:29,131 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:51:30,610 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:51:30,981 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:51:30,982 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:51:30,982 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:51:30,988 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:51:31,132 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:51:33,137 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:51:35,681 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:51:35,682 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:51:35,682 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:51:35,683 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:51:35,688 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:51:36,139 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:51:37,142 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:51:39,100 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 18:51:41,102 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:51:43,359 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:51:43,359 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:51:43,359 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:51:44,083 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:51:44,083 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:51:44,084 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:51:44,089 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:51:44,148 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:51:45,151 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:51:46,125 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:51:49,028 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:51:49,029 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:51:49,029 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:51:49,035 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:51:49,154 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:51:51,158 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:51:52,035 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:51:55,360 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:51:55,361 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:51:55,361 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:51:55,367 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:51:56,162 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:51:57,164 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:51:57,368 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:51:58,359 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:51:58,359 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:51:58,359 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:52:00,019 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:52:00,020 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:52:00,020 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:52:00,026 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:52:00,167 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:52:01,168 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:52:03,061 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:52:05,273 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:52:05,274 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:52:05,274 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:52:05,279 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:52:06,172 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:52:07,174 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:52:08,280 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:52:09,106 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 18:52:10,928 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:52:10,929 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:52:10,929 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:52:10,935 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:52:11,177 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:52:13,181 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:52:13,359 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:52:13,359 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:52:13,360 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:52:13,651 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:52:15,155 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:52:15,156 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:52:15,156 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:52:15,162 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:52:15,183 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:52:17,187 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:52:19,198 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:52:20,514 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:52:20,515 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:52:20,515 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:52:20,522 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:52:21,192 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:52:23,196 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:52:24,523 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:52:24,917 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:52:24,918 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:52:24,918 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:52:24,924 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:52:25,197 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:52:27,201 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:52:28,359 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:52:28,359 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:52:28,360 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:52:29,560 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:52:29,741 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:52:29,742 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:52:29,742 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:52:29,748 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:52:30,203 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:52:31,206 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:52:34,785 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:52:37,126 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:52:37,127 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:52:37,128 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:52:37,134 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:52:37,210 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:52:39,105 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 18:52:39,214 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:52:40,106 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:52:41,704 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:52:41,704 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:52:41,705 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:52:41,710 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:52:42,216 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:52:43,219 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:52:43,359 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:52:43,359 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:52:43,361 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:52:45,601 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:52:49,139 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:52:49,139 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:52:49,140 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:52:49,145 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:52:49,223 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:52:51,181 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:52:51,225 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:52:54,298 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:52:54,298 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:52:54,299 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:52:54,304 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:52:55,228 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:52:56,304 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:52:57,229 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:52:58,359 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:52:58,359 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:52:58,360 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:52:59,238 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:52:59,238 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:52:59,238 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:52:59,244 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:53:00,232 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:53:01,235 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:53:02,281 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:53:07,282 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:53:07,716 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:53:07,716 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:53:07,717 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:53:07,722 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:53:08,240 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:53:09,107 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 18:53:09,243 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:53:12,474 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:53:12,474 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:53:12,475 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:53:12,475 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:53:12,480 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:53:13,246 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:53:13,359 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:53:13,359 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:53:13,360 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:53:15,249 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:53:16,771 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:53:16,771 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:53:16,772 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:53:16,778 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:53:17,251 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:53:17,813 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:53:19,257 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:53:22,814 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:53:23,163 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:53:23,164 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:53:23,164 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:53:23,170 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:53:23,260 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:53:25,264 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:53:27,583 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:53:27,584 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:53:27,584 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:53:27,590 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:53:28,266 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:53:28,360 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:53:28,360 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:53:28,360 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:53:28,658 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:53:29,267 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:53:32,532 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:53:32,532 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:53:32,532 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:53:32,538 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:53:33,270 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:53:34,573 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:53:35,274 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:53:37,047 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:53:37,048 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:53:37,048 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:53:37,053 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:53:37,275 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:53:39,109 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 18:53:39,279 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:53:40,111 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:53:41,902 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:53:41,903 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:53:41,903 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:53:41,909 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:53:42,281 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:53:43,284 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:53:43,359 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:53:43,360 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:53:43,361 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:53:45,546 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:53:46,351 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:53:46,352 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:53:46,352 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:53:46,358 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:53:47,286 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:53:49,290 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:53:50,887 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:53:50,888 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:53:50,889 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:53:50,889 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:53:50,894 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:53:51,291 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:53:53,295 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:53:55,895 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:53:56,348 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:53:56,349 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:53:56,349 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:53:56,356 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:53:57,299 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:53:58,359 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:53:58,360 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:53:58,360 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:53:59,304 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:54:01,384 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:54:01,384 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:54:01,385 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:54:01,385 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:54:01,391 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:54:02,306 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:54:03,309 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:54:06,425 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:54:06,951 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:54:06,952 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:54:06,952 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:54:06,957 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:54:07,312 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:54:09,111 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 18:54:09,315 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:54:12,113 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:54:13,360 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:54:13,360 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:54:13,360 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:54:14,238 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:54:14,239 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:54:14,239 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:54:14,245 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:54:14,319 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:54:15,322 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:54:17,281 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:54:18,966 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:54:18,967 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:54:18,967 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:54:18,974 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:54:19,327 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:54:21,331 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:54:22,974 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:54:23,718 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:54:23,718 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:54:23,719 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:54:23,724 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:54:24,334 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:54:25,337 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:54:28,339 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:54:28,340 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:54:28,340 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:54:28,340 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:54:28,346 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:54:28,360 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:54:28,360 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:54:28,361 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:54:29,342 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:54:29,342 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:54:33,157 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:54:33,158 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:54:33,158 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:54:33,164 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:54:33,345 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:54:34,204 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:54:35,349 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:54:37,629 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:54:37,630 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:54:37,630 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:54:37,636 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:54:38,351 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:54:39,114 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 18:54:39,352 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:54:40,115 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:54:43,334 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:54:43,335 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:54:43,336 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:54:43,342 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:54:43,355 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:54:43,360 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:54:43,360 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:54:43,361 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:54:45,358 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:54:45,606 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:54:47,647 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:54:47,648 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:54:47,648 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:54:47,655 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:54:48,361 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:54:49,364 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:54:50,689 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:54:52,564 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:54:52,565 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:54:52,565 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:54:52,571 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:54:53,367 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:54:55,371 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:54:56,572 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:54:57,102 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:54:57,103 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:54:57,103 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:54:57,109 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:54:57,372 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:54:58,360 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:54:58,360 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:54:58,360 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:54:59,376 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:55:01,658 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:55:02,313 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:55:02,314 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:55:02,314 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:55:02,320 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:55:02,378 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:55:03,381 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:55:06,369 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:55:06,370 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:55:06,370 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:55:06,376 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:55:06,383 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:55:07,376 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:55:07,384 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:55:09,116 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 18:55:13,118 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:55:13,360 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:55:13,360 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:55:13,360 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:55:14,117 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:55:14,118 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:55:14,118 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:55:14,124 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:55:14,390 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:55:15,393 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:55:18,162 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:55:19,877 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:55:19,878 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:55:19,878 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:55:19,885 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:55:20,398 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:55:21,400 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:55:23,886 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:55:24,199 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:55:24,200 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:55:24,200 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:55:24,205 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:55:24,403 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:55:25,404 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:55:28,360 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:55:28,360 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:55:28,362 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:55:29,643 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:55:29,687 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:55:29,688 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:55:29,688 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:55:29,694 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:55:30,407 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:55:31,408 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:55:34,730 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:55:34,897 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:55:34,897 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:55:34,898 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:55:34,904 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:55:35,411 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:55:37,414 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:55:39,122 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 18:55:39,326 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:55:39,327 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:55:39,327 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:55:39,333 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:55:39,416 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:55:40,333 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:55:41,419 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:55:43,360 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:55:43,360 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:55:43,362 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:55:45,565 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:55:47,833 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:55:47,833 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:55:47,834 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:55:47,839 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:55:48,424 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:55:49,428 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:55:50,877 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:55:54,550 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:55:54,551 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:55:54,551 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:55:54,557 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:55:55,432 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:55:56,558 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:55:57,436 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:55:58,360 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:55:58,360 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:55:58,362 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:56:00,301 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:56:00,302 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:56:00,302 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:56:00,308 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:56:00,439 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:56:01,441 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:56:02,345 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:56:07,346 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:56:08,995 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:56:08,996 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:56:08,996 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:56:09,002 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:56:09,121 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 18:56:09,447 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:56:11,450 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:56:13,122 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:56:13,360 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:56:13,361 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:56:13,362 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:56:13,494 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:56:13,668 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:56:13,668 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:56:13,673 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:56:14,453 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:56:15,455 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:56:18,337 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:56:18,338 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:56:18,339 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:56:18,339 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:56:18,345 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:56:18,460 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:56:19,463 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:56:23,346 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:56:26,242 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:56:26,242 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:56:26,243 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:56:26,248 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:56:26,469 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:56:27,472 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:56:28,360 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:56:28,361 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:56:28,361 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:56:28,633 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:56:32,712 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:56:32,713 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:56:32,713 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:56:32,719 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:56:33,476 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:56:33,753 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:56:35,480 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:56:38,754 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:56:39,122 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 18:56:40,173 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:56:40,174 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:56:40,174 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:56:40,179 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:56:40,484 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:56:41,487 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:56:43,360 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:56:43,361 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:56:43,362 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:56:44,602 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:56:44,935 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:56:44,935 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:56:44,936 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:56:44,941 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:56:45,490 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:56:47,494 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:56:49,732 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:56:49,733 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:56:49,733 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:56:49,733 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:56:49,739 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:56:50,496 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:56:51,499 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:56:54,740 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:56:55,000 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:56:55,001 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:56:55,001 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:56:55,007 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:56:55,502 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:56:57,506 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:56:58,361 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:56:58,361 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:56:58,361 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:56:59,543 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:56:59,544 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:56:59,544 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:56:59,550 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:57:00,509 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:57:00,585 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:57:01,511 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:57:05,226 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:57:05,227 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:57:05,227 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:57:05,233 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:57:05,515 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:57:06,233 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:57:07,518 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:57:09,125 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 18:57:09,734 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:57:09,735 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:57:09,735 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:57:09,741 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:57:10,520 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:57:11,521 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:57:11,741 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:57:13,361 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:57:13,361 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:57:13,361 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:57:15,217 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:57:15,218 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:57:15,218 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:57:15,224 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:57:15,524 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:57:17,261 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:57:17,525 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:57:22,008 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:57:22,009 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:57:22,009 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:57:22,015 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:57:22,530 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:57:23,015 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:57:23,534 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:57:26,700 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:57:26,701 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:57:26,701 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:57:26,707 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:57:27,538 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:57:28,361 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:57:28,361 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:57:28,361 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:57:28,577 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:57:29,542 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:57:31,038 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:57:31,039 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:57:31,039 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:57:31,045 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:57:31,543 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:57:33,545 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:57:34,081 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:57:36,243 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:57:36,244 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:57:36,244 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:57:36,250 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:57:36,548 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:57:37,551 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:57:39,127 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 18:57:39,128 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:57:40,948 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:57:40,949 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:57:40,950 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:57:40,956 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:57:41,553 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:57:43,361 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:57:43,361 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:57:43,361 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:57:43,557 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:57:44,646 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:57:47,816 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:57:47,817 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:57:47,817 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:57:47,823 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:57:48,560 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:57:49,563 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:57:49,857 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:57:52,833 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:57:52,834 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:57:52,834 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:57:52,840 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:57:53,566 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:57:55,568 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:57:55,841 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:57:58,234 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:57:58,234 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:57:58,235 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:57:58,241 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:57:58,361 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:57:58,361 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:57:58,363 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:57:58,571 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:57:59,574 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:58:01,626 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:58:03,565 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:58:03,566 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:58:03,566 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:58:03,572 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:58:03,577 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:58:05,581 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:58:07,612 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:58:09,129 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 18:58:09,930 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:58:09,931 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:58:09,931 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:58:09,937 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:58:10,584 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:58:11,587 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:58:12,937 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:58:13,361 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:58:13,361 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:58:13,361 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:58:14,045 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:58:14,046 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:58:14,046 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:58:14,053 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:58:14,589 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:58:15,592 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:58:18,089 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:58:21,494 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:58:21,495 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:58:21,495 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:58:21,501 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:58:21,598 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:58:23,501 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:58:23,600 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:58:27,374 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:58:27,375 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:58:27,375 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:58:27,381 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:58:27,603 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:58:28,361 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:58:28,361 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:58:28,363 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:58:28,646 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:58:29,604 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:58:32,296 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:58:32,297 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:58:32,297 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:58:32,303 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:58:32,607 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:58:33,610 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:58:34,337 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:58:36,887 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:58:36,888 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:58:36,888 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:58:36,894 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:58:37,613 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:58:39,131 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 18:58:39,616 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:58:40,132 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:58:43,361 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:58:43,361 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:58:43,363 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:58:45,462 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:58:45,462 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:58:45,463 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:58:45,463 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:58:45,469 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:58:45,620 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:58:47,629 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:58:50,356 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:58:50,357 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:58:50,357 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:58:50,363 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:58:50,632 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:58:51,364 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:58:51,635 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:58:54,637 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:58:54,638 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:58:54,638 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:58:54,644 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:58:55,639 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:58:55,640 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:58:56,645 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:58:58,361 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:58:58,361 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:58:58,363 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:58:59,988 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:58:59,989 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:58:59,989 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:58:59,995 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:59:00,643 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:59:01,646 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:59:02,029 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:59:05,485 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:59:05,486 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:59:05,486 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:59:05,492 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:59:05,649 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:59:07,492 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:59:07,653 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:59:09,136 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 18:59:09,715 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:59:09,716 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:59:09,716 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:59:09,722 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:59:10,655 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:59:11,658 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:59:12,722 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:59:13,361 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:59:13,361 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:59:13,363 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:59:14,333 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:59:14,333 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:59:14,334 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:59:14,339 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:59:14,660 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:59:15,663 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:59:18,377 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:59:19,521 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:59:19,522 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:59:19,522 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:59:19,528 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:59:19,668 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:59:21,673 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:59:23,528 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:59:27,866 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:59:27,867 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:59:27,867 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:59:27,873 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:59:28,361 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:59:28,362 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:59:28,363 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:59:28,628 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:59:28,679 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:59:29,682 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:59:33,375 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:59:33,376 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:59:33,376 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:59:33,382 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:59:33,685 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:59:34,417 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:59:35,689 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:59:37,705 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:59:37,706 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:59:37,706 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:59:37,712 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:59:38,691 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:59:39,135 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 18:59:39,692 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:59:40,137 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:59:43,212 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:59:43,213 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:59:43,213 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:59:43,218 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:59:43,361 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:59:43,362 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:59:43,363 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:59:43,696 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:59:45,639 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:59:45,700 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:59:49,024 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:59:49,025 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:59:49,025 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:59:49,031 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:59:49,702 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:59:51,065 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:59:51,706 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:59:53,944 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:59:53,945 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:59:53,945 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:59:53,951 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:59:54,708 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:59:55,711 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 18:59:56,951 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:59:57,734 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:59:57,735 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 18:59:57,735 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:59:57,741 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:59:58,362 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:59:58,362 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:59:58,362 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:59:58,714 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 18:59:59,717 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:00:02,635 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:00:02,882 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:00:02,883 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:00:02,883 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:00:02,889 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:00:03,719 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:00:05,723 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:00:07,925 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:00:09,140 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 19:00:09,634 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:00:09,635 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:00:09,635 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:00:09,641 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:00:09,725 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:00:11,729 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:00:13,362 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:00:13,362 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:00:13,362 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:00:13,576 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:00:14,922 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:00:14,923 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:00:14,923 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:00:14,929 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:00:15,732 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:00:17,735 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:00:18,965 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:00:19,327 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:00:19,328 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:00:19,328 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:00:19,334 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:00:19,738 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:00:21,742 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:00:24,335 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:00:25,520 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:00:25,521 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:00:25,521 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:00:25,527 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:00:25,745 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:00:27,749 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:00:28,362 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:00:28,362 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:00:28,362 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:00:29,617 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:00:31,073 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:00:31,074 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:00:31,074 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:00:31,080 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:00:31,752 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:00:33,756 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:00:35,117 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:00:35,391 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:00:35,392 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:00:35,393 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:00:35,398 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:00:35,757 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:00:37,761 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:00:39,140 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 19:00:40,141 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:00:41,737 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:00:41,738 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:00:41,738 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:00:41,744 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:00:41,764 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:00:43,362 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:00:43,362 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:00:43,362 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:00:43,768 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:00:45,646 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:00:46,369 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:00:46,370 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:00:46,370 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:00:46,376 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:00:46,770 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:00:47,770 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:00:51,413 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:00:54,925 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:00:54,926 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:00:54,926 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:00:54,932 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:00:55,777 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:00:56,933 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:00:57,780 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:00:58,362 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:00:58,362 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:00:58,362 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:01:00,257 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:01:00,257 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:01:00,258 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:01:00,263 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:01:00,783 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:01:01,785 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:01:02,301 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:01:05,854 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:01:05,855 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:01:05,855 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:01:05,861 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:01:06,789 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:01:07,791 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:01:07,861 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:01:09,142 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 19:01:10,306 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:01:10,307 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:01:10,307 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:01:10,313 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:01:10,794 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:01:11,794 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:01:13,314 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:01:13,362 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:01:13,362 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:01:13,363 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:01:15,645 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:01:15,646 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:01:15,646 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:01:15,652 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:01:15,798 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:01:17,799 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:01:18,689 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:01:21,234 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:01:21,238 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:01:21,238 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:01:21,244 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:01:21,805 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:01:23,806 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:01:24,244 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:01:28,362 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:01:28,362 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:01:28,363 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:01:29,127 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:01:29,128 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:01:29,128 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:01:29,134 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:01:29,810 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:01:30,169 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:01:31,814 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:01:34,455 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:01:34,456 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:01:34,456 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:01:34,462 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:01:34,817 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:01:35,462 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:01:37,821 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:01:38,934 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:01:38,935 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:01:38,935 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:01:38,941 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:01:39,144 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 19:01:39,823 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:01:41,146 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:01:41,827 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:01:43,362 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:01:43,362 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:01:43,363 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:01:45,247 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:01:45,247 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:01:45,248 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:01:45,254 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:01:45,829 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:01:46,289 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:01:47,833 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:01:50,449 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:01:50,449 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:01:50,450 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:01:50,455 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:01:50,835 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:01:51,456 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:01:53,839 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:01:56,456 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:01:58,362 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:01:58,362 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:01:58,363 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:01:58,843 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:01:58,844 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:01:58,844 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:01:58,851 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:01:59,845 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:02:01,846 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:02:01,885 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:02:05,571 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:02:05,572 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:02:05,572 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:02:05,578 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:02:05,850 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:02:07,579 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:02:07,851 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:02:09,147 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 19:02:11,181 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:02:11,182 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:02:11,182 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:02:11,188 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:02:11,854 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:02:13,188 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:02:13,362 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:02:13,362 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:02:13,363 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:02:13,858 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:02:16,484 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:02:16,485 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:02:16,485 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:02:16,491 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:02:16,860 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:02:18,526 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:02:19,866 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:02:20,649 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:02:20,650 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:02:20,650 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:02:20,657 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:02:20,868 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:02:23,657 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:02:23,870 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:02:25,470 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:02:25,471 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:02:25,471 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:02:25,477 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:02:25,872 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:02:27,876 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:02:28,362 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:02:28,363 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:02:28,363 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:02:29,642 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:02:31,766 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:02:31,767 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:02:31,767 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:02:31,774 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:02:31,879 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:02:33,883 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:02:34,809 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:02:35,925 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:02:35,926 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:02:35,926 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:02:35,932 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:02:36,885 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:02:37,888 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:02:39,152 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 19:02:39,754 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:02:39,755 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:02:39,755 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:02:39,761 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:02:39,890 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:02:40,762 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:02:41,893 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:02:43,362 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:02:43,363 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:02:43,363 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:02:45,032 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:02:45,033 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:02:45,033 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:02:45,040 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:02:45,896 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:02:46,073 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:02:47,900 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:02:51,074 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:02:51,579 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:02:51,580 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:02:51,580 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:02:51,586 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:02:51,903 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:02:53,907 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:02:56,587 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:02:57,190 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:02:57,191 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:02:57,192 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:02:57,197 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:02:57,911 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:02:58,363 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:02:58,363 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:02:58,363 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:02:59,915 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:03:01,807 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:03:02,974 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:03:02,975 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:03:02,975 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:03:02,982 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:03:03,918 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:03:05,922 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:03:07,017 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:03:08,742 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:03:08,743 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:03:08,744 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:03:08,750 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:03:08,924 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:03:09,151 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 19:03:11,929 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:03:12,152 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:03:13,363 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:03:13,363 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:03:13,363 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:03:13,787 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:03:13,788 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:03:13,788 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:03:13,794 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:03:13,931 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:03:15,935 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:03:17,829 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:03:18,908 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:03:18,909 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:03:18,909 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:03:18,915 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:03:18,939 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:03:21,944 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:03:22,916 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:03:23,704 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:03:23,705 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:03:23,705 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:03:23,711 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:03:23,945 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:03:25,950 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:03:28,363 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:03:28,363 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:03:28,363 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:03:28,560 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:03:28,692 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:03:28,693 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:03:28,693 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:03:28,700 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:03:28,952 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:03:31,957 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:03:33,234 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:03:33,234 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:03:33,235 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:03:33,241 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:03:33,958 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:03:34,241 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:03:35,962 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:03:39,156 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 19:03:40,157 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:03:40,241 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:03:40,242 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:03:40,242 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:03:40,248 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:03:40,966 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:03:41,969 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:03:43,363 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:03:43,363 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:03:43,363 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:03:45,673 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:03:45,994 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:03:45,995 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:03:45,995 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:03:46,001 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:03:46,973 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:03:47,976 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:03:50,506 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:03:50,507 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:03:50,508 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:03:50,514 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:03:50,978 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:03:51,514 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:03:53,982 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:03:54,913 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:03:54,914 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:03:54,914 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:03:54,921 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:03:54,983 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:03:56,921 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:03:57,988 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:03:58,363 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:03:58,364 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:03:58,365 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:03:59,314 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:03:59,314 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:03:59,314 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:03:59,320 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:03:59,990 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:04:01,994 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:04:02,357 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:04:03,837 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:04:03,838 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:04:03,838 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:04:03,844 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:04:03,996 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:04:06,000 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:04:07,845 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:04:08,277 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:04:08,278 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:04:08,278 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:04:08,284 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:04:09,002 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:04:09,155 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 19:04:10,005 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:04:12,091 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:04:12,092 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:04:12,092 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:04:12,098 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:04:13,008 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:04:13,099 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:04:13,363 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:04:13,363 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:04:13,365 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:04:14,008 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:04:16,210 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:04:16,210 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:04:16,211 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:04:16,217 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:04:17,011 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:04:18,014 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:04:18,253 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:04:21,523 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:04:21,524 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:04:21,524 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:04:21,530 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:04:22,020 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:04:23,530 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:04:24,021 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:04:27,862 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:04:27,862 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:04:27,863 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:04:27,868 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:04:28,025 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:04:28,363 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:04:28,363 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:04:28,365 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:04:28,587 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:04:30,029 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:04:33,252 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:04:33,253 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:04:33,253 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:04:33,259 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:04:34,032 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:04:34,293 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:04:36,036 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:04:38,420 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:04:38,421 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:04:38,421 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:04:38,428 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:04:39,038 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:04:39,157 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 19:04:40,039 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:04:40,158 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:04:42,699 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:04:42,700 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:04:42,700 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:04:42,707 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:04:43,041 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:04:43,364 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:04:43,364 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:04:43,365 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:04:45,666 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:04:46,043 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:04:47,206 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:04:47,207 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:04:47,207 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:04:47,213 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:04:48,045 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:04:50,048 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:04:51,249 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:04:51,267 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:04:51,268 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:04:51,268 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:04:51,274 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:04:52,050 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:04:54,054 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:04:56,131 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:04:56,132 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:04:56,132 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:04:56,138 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:04:57,057 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:04:57,139 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:04:58,060 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:04:58,363 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:04:58,364 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:04:58,364 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:05:01,130 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:05:01,130 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:05:01,131 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:05:01,137 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:05:02,063 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:05:02,173 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:05:04,068 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:05:06,120 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:05:06,120 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:05:06,121 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:05:06,127 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:05:07,070 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:05:08,073 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:05:08,127 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:05:09,159 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 19:05:11,459 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:05:11,460 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:05:11,460 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:05:11,466 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:05:12,076 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:05:13,364 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:05:13,364 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:05:13,364 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:05:13,612 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:05:14,081 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:05:15,863 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:05:15,864 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:05:15,864 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:05:15,870 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:05:16,082 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:05:18,086 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:05:18,905 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:05:20,930 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:05:20,931 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:05:20,931 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:05:20,937 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:05:21,091 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:05:23,095 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:05:23,938 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:05:26,290 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:05:26,291 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:05:26,291 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:05:26,302 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:05:27,101 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:05:27,101 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:05:28,364 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:05:28,364 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:05:28,364 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:05:29,644 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:05:31,656 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:05:31,656 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:05:31,657 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:05:31,662 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:05:32,104 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:05:33,108 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:05:34,697 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:05:35,986 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:05:35,987 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:05:35,987 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:05:35,993 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:05:36,110 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:05:37,113 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:05:39,162 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 19:05:40,163 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:05:42,183 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:05:42,183 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:05:42,184 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:05:42,189 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:05:43,119 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:05:43,120 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:05:43,364 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:05:43,364 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:05:43,364 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:05:45,648 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:05:46,475 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:05:46,475 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:05:46,476 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:05:46,482 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:05:47,125 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:05:47,125 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:05:51,517 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:05:51,839 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:05:51,840 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:05:51,840 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:05:51,847 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:05:52,129 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:05:53,132 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:05:56,075 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:05:56,076 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:05:56,076 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:05:56,082 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:05:56,136 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:05:57,083 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:05:57,139 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:05:58,364 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:05:58,364 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:05:58,364 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:06:00,351 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:06:00,355 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:06:00,355 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:06:00,361 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:06:01,145 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:06:01,145 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:06:02,393 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:06:05,110 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:06:05,111 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:06:05,111 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:06:05,117 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:06:05,148 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:06:07,152 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:06:08,117 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:06:09,164 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 19:06:12,342 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:06:12,343 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:06:12,343 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:06:12,349 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:06:13,160 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:06:13,160 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:06:13,350 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:06:13,364 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:06:13,364 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:06:13,365 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:06:17,043 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:06:17,044 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:06:17,044 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:06:17,050 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:06:17,163 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:06:19,085 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:06:19,168 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:06:21,702 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:06:21,702 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:06:21,703 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:06:21,708 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:06:22,171 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:06:23,174 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:06:24,709 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:06:26,365 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:06:26,366 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:06:26,366 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:06:26,373 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:06:27,180 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:06:27,181 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:06:28,364 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:06:28,364 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:06:28,365 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:06:30,622 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:06:30,623 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:06:30,623 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:06:30,623 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:06:30,629 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:06:31,186 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:06:31,186 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:06:35,420 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:06:35,421 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:06:35,421 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:06:35,427 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:06:36,189 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:06:36,428 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:06:37,193 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:06:39,166 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 19:06:39,996 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:06:39,997 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:06:39,997 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:06:40,003 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:06:40,195 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:06:41,198 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:06:42,003 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:06:43,364 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:06:43,364 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:06:43,366 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:06:47,600 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:06:48,393 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:06:48,393 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:06:48,394 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:06:48,400 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:06:49,206 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:06:49,206 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:06:53,437 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:06:55,794 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:06:55,795 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:06:55,795 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:06:55,801 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:06:56,212 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:06:57,215 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:06:58,364 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:06:58,365 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:06:58,365 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:06:58,627 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:07:01,364 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:07:01,364 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:07:01,364 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:07:01,371 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:07:02,219 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:07:03,222 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:07:04,405 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:07:05,821 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:07:05,822 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:07:05,822 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:07:05,828 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:07:06,224 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:07:07,228 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:07:09,171 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 19:07:10,173 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:07:11,733 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:07:11,734 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:07:11,734 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:07:11,740 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:07:12,231 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:07:13,234 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:07:13,364 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:07:13,365 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:07:13,365 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:07:15,653 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:07:16,224 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:07:16,224 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:07:16,225 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:07:16,231 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:07:16,237 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:07:17,240 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:07:20,809 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:07:20,810 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:07:20,810 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:07:20,810 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:07:20,816 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:07:21,246 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:07:23,249 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:07:25,346 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:07:25,347 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:07:25,347 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:07:25,353 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:07:26,252 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:07:26,354 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:07:27,255 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:07:28,364 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:07:28,365 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:07:28,365 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:07:30,773 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:07:30,774 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:07:30,774 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:07:30,780 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:07:31,259 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:07:31,817 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:07:33,263 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:07:35,712 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:07:35,713 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:07:35,713 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:07:35,719 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:07:36,265 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:07:37,268 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:07:37,719 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:07:39,171 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 19:07:40,913 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:07:40,914 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:07:40,914 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:07:40,920 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:07:41,271 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:07:42,921 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:07:43,274 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:07:43,364 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:07:43,365 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:07:43,366 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:07:45,584 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:07:45,585 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:07:45,585 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:07:45,591 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:07:46,277 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:07:47,280 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:07:48,625 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:07:50,286 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:07:50,287 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:07:50,287 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:07:50,293 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:07:51,287 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:07:51,287 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:07:54,294 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:07:54,376 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:07:54,376 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:07:54,377 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:07:54,382 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:07:55,295 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:07:55,295 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:07:58,365 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:07:58,365 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:07:58,365 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:07:59,578 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:08:00,668 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:08:00,668 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:08:00,669 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:08:00,675 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:08:01,305 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:08:01,306 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:08:04,709 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:08:05,133 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:08:05,134 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:08:05,134 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:08:05,139 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:08:05,312 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:08:07,318 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:08:09,176 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 19:08:10,177 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:08:10,584 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:08:10,585 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:08:10,585 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:08:10,591 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:08:11,327 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:08:11,327 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:08:13,365 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:08:13,365 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:08:13,365 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:08:15,595 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:08:15,839 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:08:15,840 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:08:15,840 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:08:15,846 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:08:16,335 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:08:17,338 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:08:20,881 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:08:21,056 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:08:21,057 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:08:21,057 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:08:21,063 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:08:21,345 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:08:23,350 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:08:26,063 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:08:28,365 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:08:28,365 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:08:28,365 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:08:29,198 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:08:29,199 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:08:29,199 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:08:29,205 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:08:29,360 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:08:31,241 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:08:31,366 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:08:33,523 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:08:33,524 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:08:33,524 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:08:33,530 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:08:34,372 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:08:35,375 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:08:36,531 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:08:38,129 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:08:38,130 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:08:38,130 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:08:38,136 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:08:38,379 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:08:39,175 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 19:08:39,382 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:08:42,176 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:08:43,365 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:08:43,365 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:08:43,366 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:08:44,887 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:08:44,888 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:08:44,888 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:08:44,893 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:08:45,393 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:08:47,398 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:08:47,929 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:08:52,930 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:08:53,558 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:08:53,559 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:08:53,559 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:08:53,565 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:08:54,410 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:08:55,415 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:08:58,141 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:08:58,142 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:08:58,142 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:08:58,142 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:08:58,147 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:08:58,365 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:08:58,365 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:08:58,366 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:08:58,418 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:08:59,421 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:09:03,657 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:09:04,333 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:09:04,334 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:09:04,334 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:09:04,343 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:09:04,424 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:09:05,427 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:09:09,177 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 19:09:09,179 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:09:10,093 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:09:10,094 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:09:10,094 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:09:10,100 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:09:10,431 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:09:11,434 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:09:13,365 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:09:13,365 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:09:13,365 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:09:14,628 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:09:14,844 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:09:14,845 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:09:14,845 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:09:14,850 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:09:15,436 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:09:17,440 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:09:19,885 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:09:19,893 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:09:19,894 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:09:19,894 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:09:19,900 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:09:20,444 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:09:21,447 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:09:24,901 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:09:25,565 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:09:25,566 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:09:25,566 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:09:25,572 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:09:26,452 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:09:27,455 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:09:28,365 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:09:28,365 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:09:28,366 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:09:30,641 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:09:30,655 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:09:30,655 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:09:30,655 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:09:30,661 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:09:31,461 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:09:31,461 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:09:35,054 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:09:35,055 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:09:35,056 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:09:35,061 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:09:35,464 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:09:36,061 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:09:37,468 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:09:39,179 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 19:09:41,180 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:09:43,365 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:09:43,365 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:09:43,367 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:09:44,346 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:09:44,346 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:09:44,346 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:09:44,352 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:09:44,473 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:09:45,476 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:09:46,389 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:09:48,475 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:09:48,476 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:09:48,476 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:09:48,482 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:09:48,483 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:09:49,486 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:09:51,483 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:09:53,367 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:09:53,367 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:09:53,368 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:09:53,373 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:09:53,489 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:09:55,493 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:09:57,374 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:09:57,519 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:09:57,520 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:09:57,520 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:09:57,526 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:09:58,365 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:09:58,366 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:09:58,366 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:09:58,495 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:09:59,498 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:10:02,616 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:10:05,010 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:10:05,011 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:10:05,011 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:10:05,016 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:10:05,503 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:10:07,507 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:10:08,053 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:10:09,181 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 19:10:10,484 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:10:10,484 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:10:10,485 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:10:10,490 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:10:10,509 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:10:11,512 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:10:13,365 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:10:13,366 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:10:13,366 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:10:13,583 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:10:15,239 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:10:15,239 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:10:15,239 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:10:15,245 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:10:15,515 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:10:17,519 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:10:19,281 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:10:19,795 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:10:19,796 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:10:19,796 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:10:19,802 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:10:20,522 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:10:21,525 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:10:24,802 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:10:25,952 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:10:25,953 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:10:25,953 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:10:25,958 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:10:26,529 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:10:27,531 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:10:28,365 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:10:28,366 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:10:28,366 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:10:30,622 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:10:31,769 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:10:31,769 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:10:31,770 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:10:31,775 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:10:32,536 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:10:33,539 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:10:35,813 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:10:35,961 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:10:35,962 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:10:35,962 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:10:35,968 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:10:36,541 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:10:37,543 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:10:39,187 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 19:10:41,064 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:10:41,064 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:10:41,065 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:10:41,065 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:10:41,070 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:10:41,546 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:10:43,365 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:10:43,366 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:10:43,367 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:10:43,550 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:10:46,068 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:10:46,069 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:10:46,069 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:10:46,069 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:10:46,075 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:10:46,552 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:10:47,555 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:10:51,109 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:10:51,358 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:10:51,358 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:10:51,359 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:10:51,364 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:10:51,558 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:10:53,561 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:10:56,301 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:10:56,302 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:10:56,302 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:10:56,302 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:10:56,307 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:10:56,564 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:10:57,567 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:10:58,365 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:10:58,366 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:10:58,367 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:11:00,795 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:11:00,795 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:11:00,795 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:11:00,801 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:11:01,572 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:11:01,572 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:11:01,837 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:11:06,838 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:11:07,122 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:11:07,123 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:11:07,124 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:11:07,129 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:11:07,576 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:11:09,186 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 19:11:09,580 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:11:12,187 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:11:13,366 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:11:13,366 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:11:13,367 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:11:15,664 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:11:15,665 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:11:15,665 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:11:15,670 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:11:16,584 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:11:17,587 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:11:17,705 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:11:20,410 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:11:20,411 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:11:20,411 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:11:20,427 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:11:20,592 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:11:21,595 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:11:23,428 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:11:24,876 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:11:24,877 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:11:24,877 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:11:24,883 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:11:25,600 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:11:25,601 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:11:28,366 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:11:28,366 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:11:28,366 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:11:28,644 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:11:29,316 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:11:29,317 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:11:29,317 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:11:29,328 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:11:29,603 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:11:31,607 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:11:33,899 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:11:33,900 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:11:33,900 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:11:33,900 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:11:33,906 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:11:34,609 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:11:35,612 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:11:38,906 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:11:39,191 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 19:11:42,444 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:11:42,445 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:11:42,445 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:11:42,451 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:11:42,617 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:11:43,366 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:11:43,366 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:11:43,367 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:11:43,620 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:11:44,655 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:11:47,186 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:11:47,187 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:11:47,187 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:11:47,192 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:11:47,623 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:11:49,627 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:11:50,229 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:11:53,172 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:11:53,172 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:11:53,173 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:11:53,178 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:11:53,630 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:11:55,633 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:11:56,179 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:11:57,727 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:11:57,728 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:11:57,728 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:11:57,734 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:11:58,366 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:11:58,366 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:11:58,368 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:11:58,636 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:11:59,639 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:12:01,642 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:12:04,329 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:12:04,330 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:12:04,330 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:12:04,336 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:12:04,642 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:12:05,645 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:12:07,373 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:12:09,190 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 19:12:09,931 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:12:09,932 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:12:09,932 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:12:09,938 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:12:10,648 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:12:11,651 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:12:12,938 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:12:13,366 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:12:13,366 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:12:13,367 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:12:16,062 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:12:16,063 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:12:16,063 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:12:16,069 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:12:16,655 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:12:17,659 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:12:18,105 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:12:20,784 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:12:20,785 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:12:20,785 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:12:20,791 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:12:21,666 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:12:21,666 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:12:23,792 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:12:25,172 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:12:25,173 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:12:25,173 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:12:25,179 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:12:25,669 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:12:27,672 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:12:28,366 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:12:28,366 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:12:28,367 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:12:29,541 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:12:29,542 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:12:29,542 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:12:29,542 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:12:29,547 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:12:29,674 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:12:31,677 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:12:34,021 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:12:34,022 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:12:34,022 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:12:34,027 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:12:34,680 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:12:35,028 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:12:35,683 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:12:38,690 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:12:38,691 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:12:38,691 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:12:38,697 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:12:39,195 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 19:12:39,688 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:12:39,689 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:12:40,197 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:12:43,366 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:12:43,366 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:12:43,368 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:12:43,435 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:12:43,605 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:12:43,605 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:12:43,611 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:12:43,691 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:12:45,611 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:12:45,695 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:12:47,618 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:12:47,619 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:12:47,619 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:12:47,625 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:12:47,696 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:12:49,700 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:12:50,626 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:12:54,009 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:12:54,010 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:12:54,010 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:12:54,016 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:12:54,703 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:12:55,707 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:12:56,016 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:12:58,314 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:12:58,315 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:12:58,315 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:12:58,320 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:12:58,366 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:12:58,366 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:12:58,367 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:12:58,709 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:12:59,712 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:13:01,646 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:13:04,530 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:13:04,531 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:13:04,531 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:13:04,536 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:13:04,716 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:13:05,719 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:13:07,573 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:13:09,195 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 19:13:09,719 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:13:09,719 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:13:09,720 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:13:09,725 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:13:09,725 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:13:11,729 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:13:12,726 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:13:13,366 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:13:13,367 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:13:13,368 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:13:14,182 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:13:14,183 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:13:14,183 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:13:14,189 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:13:14,731 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:13:15,734 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:13:18,225 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:13:23,151 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:13:23,152 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:13:23,152 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:13:23,158 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:13:23,742 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:13:24,158 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:13:25,746 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:13:27,902 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:13:27,902 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:13:27,903 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:13:27,908 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:13:28,366 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:13:28,367 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:13:28,367 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:13:28,748 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:13:29,750 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:13:29,973 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:13:34,974 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:13:36,241 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:13:36,241 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:13:36,241 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:13:36,247 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:13:36,756 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:13:37,759 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:13:39,197 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 19:13:40,199 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:13:41,128 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:13:41,129 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:13:41,129 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:13:41,134 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:13:41,764 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:13:41,764 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:13:43,366 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:13:43,367 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:13:43,368 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:13:45,660 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:13:46,655 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:13:46,655 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:13:46,655 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:13:46,662 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:13:46,767 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:13:47,770 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:13:50,697 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:13:52,210 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:13:52,211 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:13:52,211 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:13:52,217 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:13:52,774 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:13:53,777 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:13:56,218 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:13:56,291 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:13:56,292 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:13:56,292 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:13:56,298 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:13:56,779 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:13:57,782 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:13:58,366 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:13:58,367 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:13:58,368 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:14:00,933 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:14:00,933 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:14:00,934 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:14:00,940 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:14:01,788 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:14:01,788 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:14:01,980 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:14:05,389 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:14:05,390 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:14:05,390 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:14:05,396 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:14:05,790 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:14:07,397 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:14:07,794 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:14:09,200 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 19:14:10,309 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:14:10,310 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:14:10,310 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:14:10,316 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:14:10,796 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:14:11,799 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:14:13,317 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:14:13,367 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:14:13,367 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:14:13,368 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:14:14,888 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:14:14,888 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:14:14,888 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:14:14,894 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:14:15,804 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:14:15,804 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:14:18,929 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:14:20,232 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:14:20,233 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:14:20,233 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:14:20,238 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:14:20,809 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:14:21,813 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:14:24,239 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:14:28,367 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:14:28,367 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:14:28,368 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:14:29,559 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:14:29,746 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:14:29,747 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:14:29,747 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:14:29,753 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:14:29,818 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:14:31,822 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:14:34,789 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:14:35,156 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:14:35,157 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:14:35,157 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:14:35,163 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:14:35,827 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:14:35,828 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:14:39,202 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 19:14:40,203 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:14:41,853 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:14:41,854 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:14:41,854 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:14:41,860 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:14:42,832 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:14:43,367 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:14:43,367 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:14:43,367 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:14:43,835 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:14:45,625 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:14:47,227 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:14:47,228 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:14:47,228 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:14:47,234 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:14:47,838 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:14:49,841 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:14:51,269 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:14:55,630 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:14:55,631 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:14:55,631 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:14:55,637 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:14:55,846 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:14:56,638 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:14:57,850 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:14:58,367 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:14:58,367 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:14:58,367 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:15:00,234 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:15:00,234 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:15:00,235 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:15:00,241 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:15:00,852 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:15:01,855 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:15:02,277 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:15:06,368 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:15:06,369 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:15:06,369 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:15:06,375 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:15:06,858 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:15:07,375 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:15:07,861 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:15:09,207 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 19:15:11,951 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:15:11,952 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:15:11,952 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:15:11,958 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:15:12,865 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:15:12,958 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:15:13,367 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:15:13,367 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:15:13,368 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:15:13,868 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:15:16,466 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:15:16,467 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:15:16,467 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:15:16,473 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:15:16,870 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:15:17,873 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:15:18,509 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:15:22,951 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:15:22,952 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:15:22,952 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:15:22,959 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:15:23,881 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:15:23,881 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:15:23,959 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:15:28,367 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:15:28,368 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:15:28,368 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:15:28,472 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:15:28,643 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:15:28,643 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:15:28,649 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:15:28,885 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:15:29,649 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:15:29,888 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:15:33,699 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:15:33,700 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:15:33,700 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:15:33,707 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:15:33,891 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:15:34,707 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:15:35,895 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:15:39,049 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:15:39,050 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:15:39,051 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:15:39,056 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:15:39,206 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 19:15:39,900 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:15:39,900 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:15:40,207 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:15:43,335 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:15:43,335 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:15:43,336 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:15:43,341 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:15:43,367 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:15:43,368 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:15:43,368 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:15:43,903 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:15:45,578 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:15:45,906 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:15:49,246 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:15:49,247 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:15:49,247 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:15:49,253 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:15:49,911 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:15:49,911 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:15:51,289 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:15:55,473 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:15:55,473 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:15:55,474 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:15:55,479 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:15:55,916 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:15:56,480 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:15:57,920 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:15:58,367 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:15:58,368 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:15:58,368 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:15:59,793 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:15:59,793 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:15:59,793 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:15:59,799 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:15:59,921 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:16:01,837 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:16:01,925 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:16:04,303 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:16:04,304 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:16:04,304 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:16:04,309 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:16:04,927 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:16:05,930 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:16:07,310 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:16:08,649 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:16:08,650 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:16:08,650 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:16:08,656 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:16:08,932 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:16:09,211 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 19:16:09,935 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:16:13,213 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:16:13,367 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:16:13,368 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:16:13,368 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:16:14,101 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:16:14,102 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:16:14,102 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:16:14,108 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:16:14,939 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:16:15,941 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:16:19,145 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:16:20,807 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:16:20,808 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:16:20,808 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:16:20,814 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:16:20,947 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:16:21,950 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:16:24,814 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:16:28,367 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:16:28,368 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:16:28,369 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:16:29,246 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:16:29,247 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:16:29,247 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:16:29,254 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:16:29,958 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:16:29,958 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:16:30,289 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:16:33,780 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:16:33,781 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:16:33,781 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:16:33,786 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:16:33,961 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:16:35,787 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:16:35,965 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:16:38,945 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:16:38,946 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:16:38,946 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:16:38,952 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:16:38,967 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:16:39,211 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 19:16:39,970 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:16:41,213 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:16:43,367 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:16:43,368 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:16:43,368 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:16:43,698 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:16:43,699 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:16:43,699 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:16:43,704 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:16:43,973 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:16:45,977 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:16:46,741 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:16:48,109 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:16:48,110 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:16:48,110 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:16:48,116 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:16:48,979 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:16:49,982 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:16:52,116 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:16:52,144 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:16:52,145 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:16:52,146 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:16:52,151 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:16:52,984 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:16:53,987 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:16:56,693 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:16:56,694 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:16:56,694 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:16:56,700 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:16:56,990 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:16:57,700 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:16:57,993 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:16:58,368 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:16:58,368 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:16:58,369 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:17:01,380 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:17:01,380 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:17:01,380 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:17:01,389 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:17:01,996 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:17:03,421 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:17:04,000 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:17:06,565 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:17:06,566 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:17:06,566 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:17:06,572 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:17:07,002 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:17:08,004 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:17:08,573 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:17:09,213 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 19:17:13,368 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:17:13,368 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:17:13,369 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:17:13,650 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:17:15,043 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:17:15,044 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:17:15,044 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:17:15,050 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:17:16,013 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:17:16,013 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:17:19,085 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:17:19,810 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:17:19,811 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:17:19,811 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:17:19,818 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:17:20,017 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:17:22,021 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:17:23,918 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:17:23,919 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:17:23,919 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:17:23,925 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:17:24,023 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:17:24,925 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:17:26,027 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:17:28,368 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:17:28,368 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:17:28,368 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:17:30,607 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:17:30,626 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:17:30,629 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:17:30,630 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:17:30,635 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:17:31,031 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:17:32,034 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:17:35,669 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:17:38,887 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:17:38,888 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:17:38,888 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:17:38,894 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:17:39,040 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:17:39,215 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 19:17:40,043 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:17:41,217 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:17:43,368 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:17:43,368 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:17:43,369 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:17:44,999 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:17:45,000 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:17:45,000 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:17:45,006 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:17:45,046 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:17:46,050 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:17:47,041 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:17:49,241 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:17:49,242 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:17:49,242 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:17:49,248 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:17:50,056 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:17:50,056 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:17:52,248 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:17:54,120 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:17:54,121 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:17:54,121 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:17:54,126 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:17:55,059 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:17:56,063 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:17:58,127 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:17:58,368 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:17:58,368 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:17:58,369 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:17:58,882 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:17:58,882 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:17:58,882 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:17:58,888 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:17:59,065 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:18:00,068 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:18:03,236 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:18:03,238 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:18:03,238 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:18:03,238 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:18:03,243 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:18:04,073 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:18:04,074 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:18:08,244 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:18:08,286 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:18:08,286 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:18:08,287 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:18:08,292 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:18:09,077 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:18:09,218 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 19:18:10,080 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:18:13,368 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:18:13,368 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:18:13,368 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:18:13,587 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:18:16,484 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:18:16,485 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:18:16,485 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:18:16,490 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:18:17,085 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:18:18,088 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:18:19,525 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:18:22,043 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:18:22,044 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:18:22,044 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:18:22,050 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:18:22,092 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:18:24,096 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:18:25,051 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:18:26,735 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:18:26,736 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:18:26,737 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:18:26,742 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:18:27,098 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:18:28,101 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:18:28,368 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:18:28,368 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:18:28,369 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:18:30,649 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:18:31,214 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:18:31,214 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:18:31,214 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:18:31,220 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:18:32,107 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:18:32,107 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:18:36,257 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:18:38,196 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:18:38,197 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:18:38,197 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:18:38,204 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:18:39,112 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:18:39,223 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 19:18:40,115 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:18:42,224 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:18:43,368 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:18:43,369 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:18:43,370 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:18:43,411 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:18:43,561 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:18:43,561 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:18:43,567 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:18:44,117 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:18:46,121 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:18:47,567 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:18:48,061 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:18:48,062 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:18:48,062 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:18:48,068 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:18:48,122 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:18:50,125 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:18:52,902 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:18:52,902 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:18:52,903 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:18:52,903 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:18:52,908 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:18:53,127 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:18:54,130 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:18:57,887 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:18:57,888 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:18:57,888 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:18:57,893 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:18:58,134 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:18:58,368 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:18:58,369 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:18:58,370 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:18:58,619 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:19:00,138 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:19:03,620 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:19:03,897 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:19:03,897 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:19:03,898 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:19:03,904 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:19:04,140 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:19:06,144 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:19:08,941 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:19:09,222 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 19:19:10,012 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:19:10,013 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:19:10,013 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:19:10,019 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:19:10,147 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:19:12,150 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:19:13,368 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:19:13,369 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:19:13,369 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:19:14,521 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:19:14,522 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:19:14,522 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:19:14,522 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:19:14,528 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:19:15,152 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:19:16,155 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:19:19,132 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:19:19,133 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:19:19,133 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:19:19,139 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:19:19,157 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:19:20,139 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:19:20,162 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:19:24,217 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:19:24,218 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:19:24,218 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:19:24,224 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:19:25,166 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:19:25,224 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:19:26,170 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:19:28,368 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:19:28,369 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:19:28,369 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:19:28,782 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:19:28,783 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:19:28,783 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:19:28,789 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:19:29,172 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:19:30,175 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:19:30,825 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:19:34,261 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:19:34,261 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:19:34,262 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:19:34,267 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:19:35,179 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:19:36,182 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:19:36,267 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:19:39,181 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:19:39,182 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:19:39,182 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:19:39,188 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:19:39,227 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 19:19:40,186 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:19:40,187 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:19:42,228 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:19:43,368 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:19:43,369 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:19:43,370 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:19:44,847 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:19:44,847 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:19:44,847 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:19:44,853 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:19:45,190 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:19:46,193 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:19:47,889 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:19:49,469 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:19:49,470 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:19:49,470 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:19:49,476 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:19:50,198 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:19:50,198 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:19:53,476 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:19:55,692 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:19:55,693 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:19:55,693 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:19:55,699 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:19:56,202 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:19:58,206 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:19:58,369 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:19:58,369 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:19:58,369 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:19:58,652 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:19:59,924 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:19:59,925 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:19:59,925 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:19:59,930 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:20:00,208 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:20:02,211 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:20:03,965 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:20:04,914 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:20:04,915 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:20:04,915 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:20:04,921 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:20:05,213 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:20:06,216 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:20:09,227 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 19:20:09,228 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:20:12,511 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:20:12,512 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:20:12,512 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:20:12,522 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:20:13,220 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:20:13,369 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:20:13,369 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:20:13,370 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:20:14,223 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:20:14,640 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:20:19,214 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:20:19,215 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:20:19,215 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:20:19,220 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:20:19,227 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:20:20,231 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:20:20,257 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:20:23,908 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:20:23,909 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:20:23,909 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:20:23,914 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:20:24,234 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:20:25,914 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:20:26,238 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:20:28,369 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:20:28,369 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:20:28,369 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:20:29,168 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:20:29,169 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:20:29,169 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:20:29,175 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:20:29,239 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:20:30,243 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:20:31,209 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:20:35,148 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:20:35,149 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:20:35,149 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:20:35,155 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:20:35,247 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:20:36,250 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:20:37,155 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:20:39,229 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 19:20:40,799 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:20:40,800 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:20:40,800 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:20:40,806 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:20:41,254 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:20:42,257 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:20:42,806 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:20:43,369 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:20:43,369 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:20:43,370 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:20:45,434 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:20:45,434 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:20:45,434 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:20:45,440 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:20:46,262 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:20:46,262 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:20:48,477 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:20:51,204 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:20:51,205 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:20:51,205 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:20:51,211 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:20:51,265 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:20:52,268 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:20:54,211 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:20:56,240 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:20:56,241 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:20:56,241 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:20:56,247 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:20:56,271 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:20:58,275 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:20:58,369 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:20:58,369 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:20:58,369 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:20:59,600 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:21:00,615 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:21:00,616 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:21:00,616 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:21:00,621 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:21:01,277 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:21:02,280 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:21:04,657 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:21:05,915 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:21:05,916 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:21:05,916 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:21:05,921 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:21:06,283 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:21:08,287 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:21:09,231 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 19:21:09,921 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:21:09,922 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:21:09,922 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:21:09,922 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:21:09,928 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:21:10,289 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:21:12,292 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:21:13,369 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:21:13,369 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:21:13,369 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:21:14,572 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:21:14,572 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:21:14,573 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:21:14,578 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:21:15,295 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:21:15,613 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:21:16,298 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:21:19,538 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:21:19,539 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:21:19,539 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:21:19,545 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:21:20,305 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:21:20,305 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:21:21,545 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:21:23,865 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:21:23,865 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:21:23,866 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:21:23,871 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:21:24,308 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:21:26,312 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:21:26,872 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:21:28,228 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:21:28,229 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:21:28,229 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:21:28,235 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:21:28,313 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:21:28,369 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:21:28,369 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:21:28,371 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:21:30,317 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:21:32,599 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:21:35,662 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:21:35,662 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:21:35,662 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:21:35,668 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:21:36,322 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:21:37,705 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:21:38,325 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:21:39,233 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 19:21:40,246 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:21:40,246 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:21:40,246 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:21:40,253 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:21:40,327 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:21:42,331 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:21:43,253 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:21:43,369 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:21:43,369 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:21:43,371 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:21:45,405 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:21:45,405 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:21:45,405 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:21:45,411 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:21:46,336 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:21:46,336 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:21:48,449 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:21:49,814 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:21:49,815 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:21:49,815 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:21:49,820 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:21:50,339 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:21:52,343 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:21:53,821 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:21:55,196 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:21:55,196 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:21:55,197 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:21:55,202 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:21:55,345 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:21:56,348 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:21:58,369 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:21:58,370 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:21:58,370 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:21:59,669 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:22:03,835 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:22:03,836 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:22:03,836 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:22:03,842 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:22:04,355 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:22:04,877 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:22:06,358 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:22:07,865 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:22:07,866 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:22:07,866 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:22:07,872 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:22:08,360 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:22:09,238 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 19:22:10,240 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:22:10,364 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:22:13,369 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:22:13,370 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:22:13,370 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:22:14,479 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:22:14,480 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:22:14,480 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:22:14,486 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:22:15,368 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:22:15,521 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:22:16,371 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:22:20,522 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:22:20,618 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:22:20,619 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:22:20,619 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:22:20,625 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:22:21,376 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:22:22,379 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:22:25,626 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:22:28,370 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:22:28,370 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:22:28,370 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:22:28,601 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:22:28,636 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:22:28,636 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:22:28,642 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:22:29,385 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:22:30,388 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:22:30,642 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:22:32,436 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:22:32,437 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:22:32,437 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:22:32,442 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:22:33,390 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:22:34,393 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:22:36,481 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:22:36,709 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:22:36,710 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:22:36,710 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:22:36,716 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:22:37,395 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:22:38,398 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:22:39,238 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 19:22:41,002 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:22:41,003 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:22:41,003 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:22:41,009 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:22:41,401 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:22:42,009 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:22:42,404 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:22:43,369 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:22:43,370 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:22:43,370 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:22:45,455 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:22:45,455 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:22:45,456 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:22:45,461 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:22:46,409 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:22:46,409 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:22:47,497 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:22:50,751 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:22:50,752 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:22:50,752 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:22:50,758 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:22:51,413 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:22:52,415 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:22:52,758 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:22:57,758 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:22:58,370 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:22:58,370 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:22:58,371 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:22:58,868 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:22:58,868 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:22:58,868 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:22:58,875 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:22:59,421 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:23:00,424 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:23:02,909 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:23:03,927 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:23:03,927 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:23:03,927 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:23:03,933 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:23:04,427 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:23:06,431 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:23:07,934 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:23:08,765 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:23:08,766 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:23:08,766 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:23:08,772 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:23:09,243 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 19:23:09,433 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:23:10,436 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:23:13,244 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:23:13,370 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:23:13,370 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:23:13,371 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:23:13,775 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:23:13,776 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:23:13,776 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:23:13,781 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:23:14,439 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:23:16,443 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:23:18,270 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:23:18,271 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:23:18,271 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:23:18,271 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:23:18,276 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:23:18,444 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:23:20,449 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:23:23,277 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:23:24,336 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:23:24,336 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:23:24,337 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:23:24,342 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:23:24,452 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:23:26,457 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:23:28,342 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:23:28,370 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:23:28,370 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:23:28,371 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:23:28,612 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:23:28,671 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:23:28,672 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:23:28,677 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:23:29,459 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:23:30,462 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:23:33,678 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:23:33,994 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:23:33,994 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:23:33,995 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:23:34,000 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:23:34,465 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:23:36,469 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:23:39,037 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:23:39,242 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 19:23:39,792 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:23:39,793 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:23:39,794 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:23:39,799 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:23:40,474 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:23:40,474 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:23:43,370 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:23:43,370 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:23:43,370 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:23:44,582 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:23:47,684 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:23:47,684 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:23:47,685 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:23:47,690 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:23:48,482 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:23:48,482 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:23:49,725 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:23:51,948 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:23:51,948 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:23:51,949 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:23:51,955 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:23:52,485 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:23:54,489 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:23:54,955 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:23:56,819 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:23:56,820 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:23:56,820 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:23:56,826 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:23:57,492 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:23:58,370 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:23:58,370 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:23:58,372 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:23:58,495 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:24:00,570 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:24:03,950 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:24:03,951 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:24:03,951 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:24:03,958 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:24:04,500 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:24:05,993 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:24:06,504 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:24:09,090 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:24:09,091 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:24:09,091 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:24:09,097 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:24:09,249 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 19:24:09,506 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:24:10,509 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:24:11,250 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:24:13,370 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:24:13,370 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:24:13,370 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:24:14,458 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:24:14,458 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:24:14,459 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:24:14,465 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:24:14,512 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:24:16,501 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:24:16,516 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:24:18,969 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:24:18,970 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:24:18,970 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:24:18,976 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:24:19,519 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:24:20,523 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:24:21,976 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:24:26,977 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:24:27,334 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:24:27,335 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:24:27,335 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:24:27,341 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:24:27,529 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:24:28,370 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:24:28,370 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:24:28,371 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:24:28,532 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:24:32,440 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:24:32,440 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:24:32,441 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:24:32,441 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:24:32,446 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:24:32,535 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:24:34,539 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:24:36,882 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:24:36,883 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:24:36,883 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:24:36,888 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:24:37,541 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:24:37,889 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:24:38,544 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:24:39,248 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 19:24:42,444 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:24:42,445 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:24:42,445 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:24:42,451 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:24:42,547 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:24:43,370 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:24:43,371 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:24:43,371 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:24:43,653 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:24:44,551 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:24:47,189 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:24:47,190 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:24:47,190 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:24:47,196 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:24:47,553 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:24:48,555 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:24:49,233 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:24:51,472 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:24:51,473 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:24:51,473 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:24:51,479 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:24:51,558 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:24:52,560 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:24:54,480 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:24:55,910 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:24:55,911 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:24:55,911 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:24:55,916 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:24:56,564 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:24:58,370 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:24:58,371 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:24:58,371 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:24:58,567 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:24:59,606 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:25:01,401 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:25:01,402 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:25:01,402 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:25:01,408 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:25:01,570 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:25:02,573 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:25:05,445 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:25:06,119 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:25:06,120 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:25:06,120 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:25:06,126 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:25:06,575 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:25:08,579 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:25:09,250 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 19:25:10,871 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:25:10,872 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:25:10,872 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:25:10,872 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:25:10,877 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:25:11,582 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:25:12,584 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:25:13,370 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:25:13,371 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:25:13,371 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:25:16,249 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:25:16,249 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:25:16,250 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:25:16,250 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:25:16,255 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:25:16,587 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:25:18,591 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:25:20,344 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:25:20,345 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:25:20,345 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:25:20,351 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:25:20,594 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:25:21,351 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:25:22,597 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:25:26,352 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:25:26,849 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:25:26,853 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:25:26,853 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:25:26,859 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:25:27,601 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:25:28,370 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:25:28,371 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:25:28,371 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:25:28,604 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:25:31,601 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:25:34,947 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:25:34,948 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:25:34,948 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:25:34,954 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:25:35,610 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:25:36,613 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:25:36,989 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:25:39,253 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 19:25:40,181 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:25:40,182 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:25:40,182 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:25:40,187 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:25:40,615 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:25:42,188 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:25:42,619 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:25:43,370 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:25:43,371 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:25:43,372 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:25:45,349 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:25:45,350 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:25:45,350 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:25:45,355 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:25:45,621 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:25:46,624 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:25:47,393 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:25:49,868 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:25:49,869 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:25:49,869 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:25:49,875 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:25:50,629 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:25:50,629 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:25:52,876 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:25:56,548 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:25:56,549 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:25:56,549 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:25:56,555 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:25:56,634 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:25:58,371 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:25:58,371 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:25:58,371 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:25:58,602 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:25:58,637 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:26:01,727 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:26:01,728 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:26:01,728 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:26:01,734 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:26:02,643 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:26:02,643 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:26:03,769 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:26:07,679 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:26:07,680 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:26:07,680 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:26:07,686 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:26:08,649 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:26:08,649 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:26:09,255 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 19:26:09,256 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:26:12,944 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:26:12,945 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:26:12,945 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:26:12,951 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:26:13,371 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:26:13,371 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:26:13,371 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:26:13,653 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:26:14,605 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:26:14,656 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:26:19,326 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:26:19,326 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:26:19,326 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:26:19,332 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:26:19,659 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:26:20,369 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:26:20,664 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:26:25,370 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:26:26,090 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:26:26,091 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:26:26,091 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:26:26,097 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:26:26,670 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:26:28,371 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:26:28,371 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:26:28,372 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:26:28,673 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:26:30,624 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:26:31,437 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:26:31,438 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:26:31,438 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:26:31,444 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:26:31,676 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:26:32,679 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:26:35,863 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:26:35,864 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:26:35,864 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:26:35,864 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:26:35,870 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:26:36,685 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:26:36,685 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:26:39,260 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 19:26:41,199 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:26:41,200 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:26:41,200 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:26:41,200 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:26:41,206 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:26:41,688 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:26:42,691 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:26:43,371 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:26:43,371 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:26:43,371 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:26:46,584 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:26:46,585 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:26:46,585 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:26:46,585 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:26:46,591 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:26:46,694 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:26:48,698 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:26:50,893 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:26:50,894 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:26:50,894 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:26:50,900 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:26:51,700 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:26:51,900 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:26:52,703 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:26:55,713 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:26:55,714 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:26:55,714 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:26:55,720 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:26:56,709 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:26:56,709 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:26:57,720 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:26:58,371 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:26:58,371 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:26:58,371 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:27:02,053 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:27:02,054 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:27:02,054 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:27:02,060 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:27:02,714 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:27:03,097 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:27:04,718 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:27:06,525 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:27:06,526 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:27:06,526 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:27:06,532 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:27:06,720 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:27:08,532 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:27:08,723 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:27:09,259 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 19:27:11,326 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:27:11,327 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:27:11,327 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:27:11,333 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:27:11,725 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:27:12,728 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:27:13,371 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:27:13,371 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:27:13,372 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:27:13,644 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:27:15,809 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:27:15,810 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:27:15,810 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:27:15,816 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:27:16,734 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:27:16,734 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:27:18,853 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:27:20,070 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:27:20,070 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:27:20,070 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:27:20,076 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:27:20,740 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:27:22,744 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:27:24,077 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:27:28,123 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:27:28,124 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:27:28,124 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:27:28,129 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:27:28,371 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:27:28,371 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:27:28,372 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:27:28,748 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:27:29,650 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:27:30,752 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:27:33,401 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:27:33,401 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:27:33,402 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:27:33,407 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:27:33,754 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:27:34,757 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:27:35,445 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:27:39,264 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 19:27:41,265 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:27:41,718 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:27:41,719 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:27:41,719 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:27:41,725 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:27:41,763 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:27:42,766 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:27:43,371 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:27:43,371 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:27:43,372 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:27:46,597 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:27:47,003 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:27:47,004 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:27:47,004 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:27:47,009 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:27:47,769 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:27:48,772 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:27:51,679 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:27:51,680 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:27:51,680 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:27:51,680 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:27:51,687 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:27:51,774 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:27:52,777 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:27:56,051 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:27:56,052 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:27:56,052 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:27:56,058 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:27:56,782 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:27:56,783 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:27:57,058 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:27:58,371 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:27:58,371 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:27:58,372 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:28:01,071 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:28:01,072 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:28:01,072 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:28:01,078 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:28:01,786 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:28:02,113 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:28:02,787 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:28:07,114 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:28:07,167 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:28:07,168 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:28:07,168 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:28:07,173 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:28:07,790 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:28:08,793 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:28:09,263 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 19:28:11,848 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:28:11,849 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:28:11,849 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:28:11,854 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:28:12,799 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:28:12,799 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:28:12,855 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:28:13,371 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:28:13,371 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:28:13,373 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:28:16,491 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:28:16,491 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:28:16,491 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:28:16,497 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:28:16,802 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:28:18,533 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:28:18,806 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:28:20,722 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:28:20,723 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:28:20,723 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:28:20,728 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:28:20,809 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:28:22,813 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:28:23,729 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:28:28,371 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:28:28,372 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:28:28,372 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:28:29,183 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:28:29,183 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:28:29,183 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:28:29,184 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:28:29,190 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:28:29,819 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:28:30,822 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:28:34,225 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:28:34,283 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:28:34,284 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:28:34,284 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:28:34,290 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:28:34,825 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:28:36,829 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:28:38,603 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:28:38,604 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:28:38,604 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:28:38,610 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:28:38,830 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:28:39,266 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 19:28:39,267 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:28:40,834 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:28:43,371 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:28:43,372 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:28:43,372 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:28:43,885 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:28:43,886 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:28:43,886 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:28:43,892 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:28:44,839 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:28:44,839 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:28:44,929 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:28:49,560 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:28:49,561 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:28:49,561 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:28:49,567 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:28:49,842 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:28:50,567 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:28:50,846 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:28:54,369 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:28:54,369 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:28:54,370 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:28:54,375 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:28:54,848 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:28:56,376 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:28:56,852 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:28:58,372 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:28:58,372 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:28:58,372 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:28:58,689 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:28:58,690 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:28:58,690 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:28:58,696 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:28:58,854 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:29:00,858 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:29:01,733 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:29:03,045 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:29:03,046 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:29:03,046 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:29:03,052 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:29:03,860 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:29:04,863 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:29:07,052 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:29:08,653 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:29:08,654 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:29:08,654 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:29:08,660 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:29:08,865 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:29:09,268 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 19:29:10,869 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:29:12,269 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:29:12,914 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:29:12,914 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:29:12,915 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:29:12,921 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:29:13,372 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:29:13,372 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:29:13,373 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:29:13,871 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:29:14,874 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:29:17,644 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:29:18,077 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:29:18,078 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:29:18,078 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:29:18,083 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:29:18,880 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:29:18,880 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:29:23,121 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:29:26,202 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:29:26,203 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:29:26,203 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:29:26,208 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:29:26,890 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:29:26,890 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:29:28,209 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:29:28,372 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:29:28,372 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:29:28,373 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:29:30,905 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:29:30,905 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:29:30,905 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:29:30,912 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:29:31,893 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:29:32,896 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:29:33,949 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:29:36,028 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:29:36,028 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:29:36,029 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:29:36,034 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:29:36,902 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:29:36,902 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:29:39,035 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:29:39,270 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 19:29:42,967 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:29:42,968 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:29:42,968 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:29:42,974 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:29:43,372 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:29:43,372 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:29:43,372 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:29:43,907 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:29:44,660 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:29:44,910 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:29:47,514 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:29:47,514 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:29:47,514 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:29:47,520 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:29:47,911 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:29:48,914 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:29:50,557 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:29:55,558 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:29:56,018 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:29:56,018 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:29:56,018 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:29:56,025 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:29:56,922 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:29:56,923 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:29:58,372 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:29:58,372 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:29:58,372 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:30:00,666 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:30:00,723 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:30:00,724 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:30:00,724 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:30:00,730 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:30:00,926 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:30:02,929 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:30:05,765 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:30:06,306 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:30:06,307 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:30:06,307 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:30:06,313 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:30:06,932 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:30:08,935 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:30:09,278 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 19:30:11,196 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:30:11,196 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:30:11,196 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:30:11,197 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:30:11,202 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:30:11,937 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:30:12,940 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:30:13,372 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:30:13,372 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:30:13,372 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:30:16,342 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:30:16,343 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:30:16,343 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:30:16,344 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:30:16,350 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:30:16,943 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:30:18,946 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:30:21,385 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:30:22,393 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:30:22,394 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:30:22,394 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:30:22,400 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:30:22,951 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:30:24,955 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:30:26,401 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:30:26,916 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:30:26,917 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:30:26,917 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:30:26,923 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:30:26,956 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:30:28,372 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:30:28,372 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:30:28,372 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:30:28,960 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:30:31,204 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:30:31,205 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:30:31,205 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:30:31,211 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:30:31,962 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:30:32,245 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:30:32,964 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:30:36,041 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:30:36,042 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:30:36,042 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:30:36,048 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:30:36,970 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:30:36,970 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:30:38,048 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:30:39,274 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 19:30:41,316 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:30:41,317 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:30:41,317 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:30:41,322 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:30:41,973 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:30:42,977 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:30:43,323 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:30:43,372 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:30:43,372 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:30:43,374 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:30:47,251 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:30:47,251 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:30:47,252 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:30:47,258 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:30:47,981 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:30:48,984 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:30:49,293 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:30:52,000 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:30:52,001 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:30:52,001 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:30:52,007 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:30:52,989 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:30:52,989 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:30:55,008 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:30:57,626 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:30:57,627 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:30:57,627 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:30:57,634 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:30:57,993 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:30:58,372 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:30:58,373 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:30:58,374 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:30:58,997 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:31:00,758 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:31:02,324 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:31:02,324 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:31:02,325 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:31:02,330 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:31:03,000 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:31:05,003 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:31:06,366 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:31:09,280 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 19:31:10,457 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:31:10,458 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:31:10,458 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:31:10,464 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:31:11,008 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:31:11,464 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:31:13,011 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:31:13,372 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:31:13,373 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:31:13,373 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:31:14,954 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:31:14,955 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:31:14,955 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:31:14,961 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:31:15,014 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:31:16,997 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:31:17,018 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:31:19,490 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:31:19,491 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:31:19,491 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:31:19,497 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:31:20,020 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:31:21,024 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:31:22,497 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:31:24,780 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:31:24,781 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:31:24,781 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:31:24,787 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:31:25,028 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:31:27,032 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:31:27,787 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:31:28,372 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:31:28,373 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:31:28,374 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:31:29,028 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:31:29,029 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:31:29,029 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:31:29,034 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:31:29,035 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:31:31,038 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:31:33,069 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:31:38,020 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:31:38,021 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:31:38,021 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:31:38,027 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:31:38,044 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:31:39,027 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:31:39,047 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:31:39,279 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 19:31:43,372 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:31:43,373 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:31:43,373 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:31:43,841 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:31:43,841 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:31:43,842 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:31:43,847 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:31:44,051 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:31:44,885 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:31:45,053 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:31:48,099 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:31:48,100 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:31:48,100 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:31:48,106 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:31:49,059 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:31:49,059 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:31:50,107 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:31:52,807 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:31:52,808 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:31:52,808 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:31:52,815 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:31:53,062 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:31:55,065 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:31:55,815 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:31:57,067 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:31:57,068 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:31:57,068 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:31:57,074 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:31:58,068 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:31:58,372 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:31:58,373 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:31:58,374 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:31:59,071 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:32:01,643 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:32:03,372 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:32:03,373 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:32:03,373 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:32:03,379 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:32:04,076 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:32:05,079 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:32:07,414 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:32:09,284 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 19:32:09,931 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:32:09,932 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:32:09,932 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:32:09,938 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:32:10,082 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:32:11,085 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:32:12,938 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:32:13,373 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:32:13,373 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:32:13,373 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:32:14,607 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:32:14,608 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:32:14,608 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:32:14,614 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:32:15,088 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:32:17,092 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:32:18,650 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:32:19,981 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:32:19,982 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:32:19,983 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:32:19,989 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:32:20,094 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:32:21,098 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:32:23,989 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:32:25,887 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:32:25,888 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:32:25,888 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:32:25,894 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:32:26,103 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:32:27,106 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:32:28,373 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:32:28,373 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:32:28,373 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:32:29,669 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:32:30,727 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:32:30,728 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:32:30,728 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:32:30,734 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:32:31,109 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:32:33,113 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:32:34,770 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:32:37,141 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:32:37,142 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:32:37,142 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:32:37,148 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:32:38,116 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:32:39,119 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:32:39,283 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 19:32:40,285 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:32:43,373 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:32:43,373 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:32:43,373 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:32:43,524 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:32:43,626 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:32:43,626 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:32:43,632 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:32:44,123 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:32:45,126 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:32:45,633 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:32:50,525 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:32:50,526 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:32:50,526 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:32:50,532 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:32:51,131 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:32:51,533 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:32:53,135 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:32:54,674 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:32:54,675 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:32:54,675 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:32:54,681 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:32:55,136 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:32:56,682 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:32:57,140 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:32:58,373 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:32:58,373 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:32:58,375 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:32:59,052 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:32:59,052 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:32:59,053 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:32:59,059 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:32:59,142 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:33:01,146 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:33:02,093 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:33:03,470 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:33:03,471 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:33:03,471 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:33:03,477 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:33:04,149 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:33:05,152 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:33:07,478 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:33:08,140 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:33:08,141 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:33:08,141 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:33:08,147 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:33:08,154 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:33:09,157 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:33:09,285 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 19:33:12,622 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:33:12,623 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:33:12,624 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:33:12,624 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:33:12,630 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:33:13,160 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:33:13,373 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:33:13,373 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:33:13,375 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:33:15,164 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:33:18,161 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:33:18,162 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:33:18,162 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:33:18,162 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:33:18,168 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:33:18,168 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:33:19,171 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:33:23,020 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:33:23,021 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:33:23,021 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:33:23,027 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:33:23,176 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:33:24,027 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:33:25,180 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:33:27,479 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:33:27,480 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:33:27,480 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:33:27,487 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:33:28,183 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:33:28,373 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:33:28,373 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:33:28,374 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:33:29,186 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:33:29,664 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:33:32,497 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:33:32,498 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:33:32,498 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:33:32,504 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:33:33,188 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:33:35,193 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:33:35,541 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:33:37,362 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:33:37,363 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:33:37,363 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:33:37,369 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:33:38,195 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:33:39,198 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:33:39,288 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 19:33:41,289 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:33:41,482 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:33:41,483 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:33:41,483 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:33:41,488 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:33:42,200 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:33:43,203 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:33:43,373 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:33:43,373 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:33:43,375 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:33:46,013 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:33:46,014 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:33:46,014 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:33:46,020 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:33:46,205 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:33:47,057 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:33:47,208 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:33:51,395 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:33:51,396 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:33:51,396 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:33:51,416 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:33:52,212 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:33:52,416 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:33:53,215 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:33:57,417 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:33:58,373 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:33:58,373 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:33:58,375 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:33:59,730 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:33:59,731 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:33:59,731 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:33:59,738 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:34:00,221 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:34:01,224 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:34:02,773 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:34:03,832 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:34:03,833 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:34:03,833 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:34:03,839 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:34:04,226 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:34:05,229 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:34:07,839 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:34:08,278 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:34:08,278 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:34:08,279 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:34:08,285 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:34:09,234 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:34:09,234 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:34:09,290 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 19:34:13,291 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:34:13,332 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:34:13,333 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:34:13,333 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:34:13,339 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:34:13,373 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:34:13,374 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:34:13,375 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:34:14,238 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:34:15,241 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:34:18,187 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:34:18,188 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:34:18,188 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:34:18,194 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:34:18,243 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:34:19,229 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:34:19,246 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:34:23,375 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:34:23,376 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:34:23,376 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:34:23,382 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:34:24,252 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:34:24,383 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:34:25,255 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:34:28,373 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:34:28,374 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:34:28,375 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:34:29,274 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:34:29,274 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:34:29,274 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:34:29,280 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:34:30,259 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:34:30,317 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:34:31,262 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:34:34,636 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:34:34,636 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:34:34,637 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:34:34,642 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:34:35,265 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:34:35,643 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:34:37,268 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:34:39,295 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 19:34:41,297 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:34:41,615 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:34:41,616 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:34:41,617 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:34:41,623 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:34:42,272 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:34:43,275 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:34:43,373 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:34:43,374 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:34:43,375 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:34:45,941 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:34:45,942 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:34:45,942 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:34:45,948 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:34:46,277 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:34:46,985 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:34:47,280 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:34:51,010 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:34:51,011 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:34:51,011 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:34:51,017 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:34:51,282 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:34:52,018 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:34:53,286 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:34:55,503 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:34:55,504 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:34:55,505 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:34:55,515 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:34:56,288 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:34:57,291 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:34:57,516 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:34:58,374 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:34:58,374 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:34:58,375 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:35:00,202 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:35:00,202 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:35:00,202 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:35:00,211 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:35:00,294 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:35:01,297 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:35:03,245 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:35:05,282 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:35:05,282 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:35:05,283 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:35:05,288 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:35:05,300 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:35:07,304 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:35:08,289 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:35:09,294 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 19:35:09,733 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:35:09,734 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:35:09,734 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:35:09,740 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:35:10,306 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:35:11,308 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:35:13,374 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:35:13,374 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:35:13,375 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:35:13,603 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:35:14,112 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:35:14,113 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:35:14,113 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:35:14,118 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:35:14,310 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:35:15,313 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:35:19,154 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:35:21,258 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:35:21,259 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:35:21,259 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:35:21,265 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:35:21,319 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:35:23,322 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:35:24,266 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:35:26,205 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:35:26,206 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:35:26,206 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:35:26,211 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:35:26,324 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:35:27,328 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:35:28,374 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:35:28,374 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:35:28,375 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:35:29,590 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:35:31,285 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:35:31,286 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:35:31,286 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:35:31,292 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:35:31,331 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:35:33,335 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:35:35,330 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:35:35,761 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:35:35,762 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:35:35,762 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:35:35,768 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:35:36,337 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:35:37,340 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:35:39,299 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 19:35:41,301 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:35:42,094 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:35:42,095 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:35:42,095 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:35:42,101 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:35:42,344 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:35:43,347 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:35:43,374 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:35:43,374 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:35:43,375 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:35:46,593 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:35:48,005 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:35:48,006 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:35:48,006 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:35:48,012 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:35:48,351 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:35:49,354 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:35:52,050 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:35:52,718 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:35:52,719 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:35:52,719 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:35:52,724 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:35:53,357 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:35:55,361 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:35:57,725 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:35:58,374 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:35:58,374 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:35:58,374 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:36:00,969 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:36:00,969 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:36:00,969 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:36:00,975 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:36:01,366 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:36:03,013 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:36:03,369 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:36:05,615 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:36:05,616 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:36:05,616 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:36:05,622 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:36:06,372 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:36:07,375 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:36:08,623 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:36:09,298 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 19:36:10,067 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:36:10,068 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:36:10,069 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:36:10,074 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:36:10,377 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:36:11,381 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:36:13,374 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:36:13,374 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:36:13,375 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:36:13,681 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:36:17,850 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:36:17,851 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:36:17,851 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:36:17,857 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:36:18,386 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:36:18,893 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:36:19,389 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:36:23,736 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:36:23,737 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:36:23,737 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:36:23,743 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:36:24,395 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:36:24,743 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:36:25,398 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:36:28,374 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:36:28,375 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:36:28,375 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:36:29,844 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:36:29,845 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:36:29,845 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:36:29,845 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:36:29,851 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:36:30,402 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:36:31,405 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:36:34,886 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:36:34,970 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:36:34,971 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:36:34,971 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:36:34,977 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:36:35,408 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:36:37,412 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:36:39,301 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 19:36:39,438 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:36:39,439 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:36:39,439 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:36:39,444 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:36:40,414 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:36:40,445 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:36:41,417 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:36:43,374 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:36:43,374 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:36:43,376 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:36:43,885 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:36:43,889 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:36:43,890 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:36:43,895 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:36:44,420 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:36:45,423 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:36:45,929 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:36:49,402 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:36:49,403 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:36:49,403 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:36:49,409 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:36:49,426 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:36:51,409 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:36:51,429 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:36:55,363 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:36:55,363 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:36:55,364 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:36:55,370 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:36:55,432 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:36:57,371 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:36:57,437 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:36:58,374 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:36:58,374 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:36:58,376 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:37:00,267 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:37:00,268 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:37:00,268 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:37:00,274 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:37:00,439 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:37:01,442 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:37:03,309 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:37:05,118 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:37:05,119 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:37:05,120 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:37:05,126 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:37:05,446 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:37:07,449 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:37:09,127 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:37:09,303 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 19:37:09,764 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:37:09,765 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:37:09,765 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:37:09,771 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:37:10,452 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:37:11,454 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:37:13,374 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:37:13,375 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:37:13,375 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:37:14,492 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:37:14,493 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:37:14,493 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:37:14,494 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:37:14,499 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:37:15,460 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:37:15,460 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:37:18,924 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:37:18,925 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:37:18,925 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:37:18,931 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:37:19,462 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:37:19,932 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:37:21,469 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:37:23,563 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:37:23,564 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:37:23,565 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:37:23,570 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:37:24,471 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:37:25,474 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:37:25,571 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:37:28,160 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:37:28,161 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:37:28,161 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:37:28,166 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:37:28,374 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:37:28,375 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:37:28,375 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:37:28,477 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:37:29,480 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:37:30,658 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:37:33,008 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:37:33,009 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:37:33,009 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:37:33,015 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:37:33,483 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:37:35,490 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:37:36,050 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:37:37,250 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:37:37,251 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:37:37,251 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:37:37,257 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:37:37,492 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:37:39,305 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 19:37:39,495 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:37:41,306 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:37:41,741 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:37:41,741 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:37:41,742 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:37:41,748 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:37:42,498 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:37:43,374 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:37:43,375 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:37:43,375 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:37:43,501 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:37:46,637 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:37:47,010 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:37:47,010 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:37:47,011 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:37:47,017 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:37:47,504 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:37:49,508 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:37:52,006 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:37:52,007 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:37:52,007 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:37:52,007 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:37:52,013 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:37:52,510 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:37:53,513 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:37:56,148 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:37:56,148 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:37:56,149 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:37:56,154 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:37:56,515 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:37:57,155 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:37:57,519 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:37:58,375 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:37:58,375 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:37:58,376 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:38:01,509 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:38:01,509 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:38:01,510 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:38:01,516 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:38:01,522 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:38:02,553 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:38:03,527 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:38:06,160 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:38:06,161 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:38:06,161 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:38:06,167 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:38:06,529 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:38:07,532 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:38:08,168 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:38:09,307 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 19:38:13,309 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:38:13,375 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:38:13,375 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:38:13,376 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:38:13,554 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:38:13,565 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:38:13,565 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:38:13,571 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:38:14,537 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:38:15,541 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:38:17,799 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:38:17,799 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:38:17,800 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:38:17,806 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:38:18,543 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:38:18,841 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:38:19,546 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:38:22,263 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:38:22,264 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:38:22,265 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:38:22,270 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:38:22,550 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:38:23,553 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:38:24,271 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:38:27,423 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:38:27,424 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:38:27,424 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:38:27,431 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:38:27,557 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:38:28,375 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:38:28,375 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:38:28,375 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:38:29,560 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:38:29,574 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:38:32,515 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:38:32,516 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:38:32,517 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:38:32,523 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:38:32,562 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:38:33,565 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:38:35,558 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:38:37,990 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:38:37,991 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:38:37,991 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:38:37,997 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:38:38,570 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:38:39,309 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 19:38:39,573 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:38:41,311 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:38:42,519 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:38:42,520 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:38:42,520 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:38:42,526 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:38:42,575 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:38:43,375 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:38:43,375 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:38:43,375 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:38:43,578 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:38:46,652 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:38:46,778 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:38:46,778 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:38:46,779 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:38:46,784 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:38:47,583 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:38:47,583 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:38:51,146 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:38:51,147 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:38:51,147 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:38:51,153 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:38:51,586 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:38:52,153 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:38:53,589 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:38:55,681 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:38:55,681 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:38:55,682 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:38:55,688 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:38:56,591 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:38:57,594 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:38:57,688 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:38:58,375 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:38:58,375 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:38:58,376 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:39:00,006 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:39:00,007 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:39:00,007 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:39:00,013 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:39:00,597 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:39:01,600 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:39:03,049 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:39:08,050 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:39:09,315 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 19:39:13,316 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:39:13,375 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:39:13,375 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:39:13,377 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:39:13,588 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:39:13,588 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:39:13,589 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:39:13,595 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:39:13,609 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:39:15,613 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:39:18,148 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:39:18,149 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:39:18,149 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:39:18,154 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:39:18,615 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:39:19,155 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:39:19,618 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:39:22,666 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:39:22,667 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:39:22,667 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:39:22,673 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:39:23,623 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:39:23,623 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:39:24,674 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:39:27,368 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:39:27,369 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:39:27,369 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:39:27,375 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:39:27,626 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:39:28,375 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:39:28,376 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:39:28,376 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:39:29,630 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:39:30,646 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:39:31,785 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:39:31,785 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:39:31,786 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:39:31,792 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:39:32,632 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:39:33,635 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:39:35,830 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:39:36,456 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:39:36,458 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:39:36,458 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:39:36,464 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:39:36,638 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:39:37,641 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:39:39,314 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 19:39:40,923 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:39:40,924 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:39:40,925 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:39:40,925 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:39:40,932 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:39:41,643 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:39:43,375 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:39:43,376 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:39:43,376 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:39:43,648 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:39:45,559 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:39:45,559 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:39:45,559 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:39:45,565 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:39:45,649 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:39:46,601 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:39:47,653 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:39:50,864 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:39:50,865 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:39:50,866 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:39:50,872 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:39:51,658 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:39:51,658 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:39:51,872 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:39:55,840 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:39:55,841 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:39:55,842 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:39:55,847 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:39:56,661 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:39:57,665 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:39:57,848 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:39:58,375 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:39:58,375 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:39:58,377 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:40:01,169 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:40:01,170 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:40:01,170 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:40:01,176 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:40:01,668 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:40:03,213 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:40:03,672 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:40:08,214 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:40:09,318 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 19:40:09,550 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:40:09,550 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:40:09,551 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:40:09,556 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:40:09,676 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:40:11,680 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:40:13,375 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:40:13,376 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:40:13,376 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:40:13,676 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:40:14,694 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:40:14,695 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:40:14,695 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:40:14,701 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:40:15,685 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:40:15,685 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:40:18,738 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:40:20,550 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:40:20,552 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:40:20,552 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:40:20,558 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:40:20,689 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:40:21,693 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:40:24,558 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:40:25,108 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:40:25,108 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:40:25,109 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:40:25,114 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:40:25,697 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:40:27,701 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:40:28,375 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:40:28,376 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:40:28,377 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:40:29,573 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:40:33,312 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:40:33,313 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:40:33,313 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:40:33,319 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:40:33,705 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:40:35,353 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:40:35,709 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:40:38,018 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:40:38,019 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:40:38,019 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:40:38,025 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:40:38,711 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:40:39,318 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 19:40:39,714 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:40:41,319 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:40:43,375 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:40:43,376 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:40:43,376 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:40:45,280 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:40:45,281 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:40:45,281 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:40:45,287 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:40:45,718 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:40:46,321 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:40:47,722 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:40:49,806 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:40:49,807 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:40:49,807 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:40:49,813 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:40:50,724 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:40:51,727 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:40:51,813 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:40:56,814 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:40:58,202 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:40:58,202 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:40:58,203 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:40:58,209 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:40:58,376 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:40:58,376 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:40:58,376 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:40:58,732 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:40:59,735 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:41:02,669 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:41:03,447 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:41:03,448 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:41:03,448 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:41:03,454 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:41:03,738 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:41:05,742 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:41:08,149 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:41:08,150 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:41:08,150 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:41:08,151 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:41:08,156 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:41:08,744 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:41:09,320 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 19:41:09,747 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:41:13,278 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:41:13,279 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:41:13,279 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:41:13,279 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:41:13,285 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:41:13,376 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:41:13,376 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:41:13,378 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:41:13,750 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:41:15,754 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:41:18,689 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:41:18,977 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:41:18,977 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:41:18,978 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:41:18,984 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:41:19,757 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:41:21,762 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:41:24,025 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:41:24,127 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:41:24,127 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:41:24,128 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:41:24,134 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:41:24,765 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:41:25,768 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:41:28,270 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:41:28,271 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:41:28,271 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:41:28,276 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:41:28,376 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:41:28,376 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:41:28,378 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:41:28,771 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:41:29,627 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:41:29,774 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:41:33,327 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:41:33,328 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:41:33,328 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:41:33,334 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:41:33,777 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:41:35,369 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:41:35,781 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:41:39,322 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 19:41:39,525 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:41:39,526 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:41:39,526 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:41:39,532 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:41:39,784 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:41:40,533 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:41:41,785 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:41:43,376 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:41:43,376 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:41:43,376 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:41:45,612 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:41:46,270 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:41:46,271 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:41:46,271 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:41:46,277 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:41:46,790 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:41:47,792 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:41:51,314 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:41:53,771 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:41:53,772 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:41:53,772 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:41:53,778 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:41:53,797 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:41:55,801 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:41:56,779 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:41:58,376 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:41:58,376 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:41:58,378 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:42:00,987 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:42:00,987 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:42:00,987 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:42:00,993 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:42:01,808 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:42:01,809 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:42:02,029 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:42:05,463 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:42:05,463 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:42:05,464 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:42:05,469 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:42:05,811 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:42:07,470 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:42:07,815 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:42:09,324 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 19:42:09,941 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:42:09,941 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:42:09,942 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:42:09,947 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:42:10,817 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:42:11,820 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:42:12,948 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:42:13,376 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:42:13,376 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:42:13,378 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:42:14,464 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:42:14,465 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:42:14,465 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:42:14,471 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:42:14,822 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:42:15,825 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:42:18,506 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:42:20,769 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:42:20,770 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:42:20,770 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:42:20,776 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:42:20,829 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:42:21,833 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:42:23,776 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:42:25,702 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:42:25,703 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:42:25,703 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:42:25,709 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:42:25,837 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:42:27,841 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:42:28,376 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:42:28,376 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:42:28,377 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:42:29,655 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:42:32,727 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:42:32,728 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:42:32,728 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:42:32,734 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:42:32,845 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:42:33,848 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:42:34,769 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:42:38,182 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:42:38,183 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:42:38,183 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:42:38,189 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:42:38,852 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:42:39,330 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 19:42:39,855 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:42:40,331 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:42:43,376 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:42:43,376 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:42:43,378 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:42:43,944 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:42:43,944 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:42:43,945 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:42:43,950 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:42:44,859 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:42:45,861 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:42:45,985 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:42:48,363 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:42:48,364 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:42:48,364 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:42:48,370 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:42:48,864 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:42:49,866 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:42:51,370 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:42:54,956 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:42:54,956 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:42:54,957 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:42:54,963 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:42:55,873 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:42:55,873 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:42:56,963 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:42:58,376 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:42:58,377 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:42:58,377 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:43:00,475 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:43:00,475 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:43:00,476 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:43:00,482 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:43:00,877 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:43:01,881 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:43:02,517 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:43:04,968 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:43:04,969 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:43:04,969 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:43:04,975 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:43:05,886 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:43:05,886 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:43:07,975 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:43:09,329 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 19:43:09,913 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:43:09,914 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:43:09,914 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:43:09,919 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:43:10,890 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:43:11,893 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:43:13,376 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:43:13,377 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:43:13,377 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:43:13,616 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:43:14,087 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:43:14,088 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:43:14,088 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:43:14,094 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:43:14,895 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:43:15,898 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:43:19,130 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:43:20,929 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:43:20,929 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:43:20,930 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:43:20,936 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:43:21,906 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:43:21,907 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:43:24,937 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:43:25,033 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:43:25,034 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:43:25,034 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:43:25,040 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:43:25,912 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:43:25,912 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:43:28,376 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:43:28,377 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:43:28,377 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:43:30,284 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:43:30,285 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:43:30,285 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:43:30,285 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:43:30,291 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:43:30,917 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:43:31,920 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:43:35,326 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:43:35,447 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:43:35,448 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:43:35,449 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:43:35,454 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:43:35,924 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:43:37,927 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:43:39,334 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 19:43:40,336 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:43:41,190 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:43:41,191 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:43:41,191 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:43:41,197 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:43:41,930 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:43:43,376 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:43:43,377 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:43:43,377 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:43:43,937 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:43:45,634 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:43:49,837 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:43:49,838 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:43:49,838 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:43:49,844 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:43:49,942 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:43:50,881 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:43:51,946 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:43:54,110 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:43:54,111 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:43:54,111 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:43:54,117 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:43:54,949 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:43:55,952 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:43:56,117 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:43:58,377 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:43:58,377 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:43:58,378 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:43:59,272 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:43:59,273 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:43:59,273 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:43:59,279 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:43:59,955 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:44:01,313 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:44:01,960 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:44:06,314 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:44:06,665 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:44:06,666 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:44:06,666 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:44:06,672 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:44:06,964 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:44:07,967 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:44:09,333 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 19:44:11,335 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:44:11,746 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:44:11,746 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:44:11,747 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:44:11,752 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:44:11,970 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:44:13,377 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:44:13,377 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:44:13,377 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:44:13,973 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:44:16,138 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:44:16,139 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:44:16,139 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:44:16,145 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:44:16,976 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:44:17,181 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:44:17,979 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:44:22,182 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:44:25,084 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:44:25,085 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:44:25,085 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:44:25,091 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:44:25,989 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:44:25,989 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:44:28,091 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:44:28,377 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:44:28,377 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:44:28,378 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:44:29,003 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:44:29,003 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:44:29,004 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:44:29,009 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:44:29,995 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:44:29,995 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:44:34,046 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:44:34,744 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:44:34,745 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:44:34,745 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:44:34,750 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:44:34,999 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:44:36,002 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:44:39,242 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:44:39,243 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:44:39,243 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:44:39,243 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:44:39,248 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:44:39,338 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 19:44:40,005 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:44:42,009 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:44:43,377 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:44:43,377 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:44:43,377 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:44:44,177 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:44:44,178 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:44:44,178 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:44:44,184 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:44:45,012 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:44:45,221 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:44:46,015 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:44:48,838 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:44:48,839 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:44:48,839 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:44:48,846 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:44:49,017 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:44:50,020 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:44:50,846 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:44:53,100 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:44:53,101 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:44:53,101 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:44:53,107 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:44:54,025 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:44:54,025 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:44:56,107 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:44:58,377 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:44:58,377 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:44:58,377 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:44:59,200 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:44:59,201 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:44:59,201 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:44:59,207 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:45:00,032 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:45:00,032 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:45:01,241 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:45:05,122 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:45:05,123 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:45:05,123 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:45:05,130 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:45:06,039 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:45:06,039 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:45:07,130 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:45:09,337 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 19:45:10,241 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:45:10,242 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:45:10,242 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:45:10,248 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:45:11,042 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:45:12,043 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:45:12,248 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:45:13,377 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:45:13,377 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:45:13,379 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:45:15,512 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:45:15,513 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:45:15,513 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:45:15,519 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:45:16,046 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:45:17,557 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:45:18,049 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:45:20,989 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:45:20,990 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:45:20,990 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:45:20,996 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:45:21,051 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:45:22,055 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:45:22,997 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:45:27,871 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:45:27,872 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:45:27,873 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:45:27,878 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:45:28,060 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:45:28,377 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:45:28,378 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:45:28,378 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:45:28,662 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:45:30,064 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:45:32,637 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:45:32,638 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:45:32,638 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:45:32,643 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:45:33,066 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:45:33,681 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:45:34,069 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:45:37,957 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:45:37,958 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:45:37,958 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:45:37,971 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:45:38,072 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:45:38,972 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:45:39,340 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 19:45:40,076 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:45:43,377 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:45:43,378 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:45:43,378 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:45:43,534 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:45:43,601 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:45:43,601 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:45:43,607 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:45:44,078 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:45:44,607 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:45:46,082 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:45:49,608 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:45:50,660 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:45:50,660 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:45:50,660 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:45:50,666 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:45:51,085 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:45:52,088 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:45:54,701 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:45:54,890 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:45:54,891 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:45:54,891 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:45:54,896 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:45:55,090 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:45:56,093 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:45:58,377 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:45:58,378 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:45:58,378 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:45:59,168 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:45:59,169 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:45:59,169 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:45:59,175 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:46:00,099 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:46:00,099 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:46:00,212 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:46:03,403 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:46:03,404 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:46:03,404 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:46:03,410 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:46:04,102 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:46:05,410 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:46:06,106 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:46:07,501 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:46:07,501 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:46:07,502 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:46:07,508 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:46:08,108 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:46:09,342 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 19:46:10,111 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:46:11,344 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:46:11,801 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:46:11,802 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:46:11,802 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:46:11,808 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:46:12,113 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:46:13,377 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:46:13,378 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:46:13,378 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:46:14,116 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:46:16,652 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:46:21,652 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:46:21,735 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:46:21,735 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:46:21,735 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:46:21,741 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:46:22,124 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:46:24,128 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:46:26,777 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:46:28,377 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:46:28,378 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:46:28,378 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:46:28,405 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:46:28,598 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:46:28,598 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:46:28,604 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:46:29,133 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:46:30,136 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:46:32,605 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:46:32,751 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:46:32,752 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:46:32,752 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:46:32,758 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:46:33,138 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:46:34,141 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:46:37,021 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:46:37,022 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:46:37,023 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:46:37,028 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:46:37,143 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:46:38,029 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:46:38,146 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:46:39,344 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 19:46:42,028 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:46:42,029 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:46:42,029 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:46:42,035 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:46:42,149 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:46:43,035 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:46:43,377 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:46:43,378 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:46:43,379 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:46:44,153 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:46:46,362 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:46:46,362 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:46:46,363 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:46:46,368 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:46:47,156 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:46:48,158 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:46:48,405 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:46:51,278 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:46:51,278 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:46:51,279 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:46:51,284 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:46:52,164 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:46:52,164 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:46:54,285 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:46:55,938 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:46:55,939 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:46:55,939 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:46:55,945 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:46:56,167 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:46:58,171 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:46:58,378 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:46:58,378 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:46:58,378 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:46:59,584 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:47:00,411 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:47:00,412 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:47:00,412 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:47:00,418 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:47:01,174 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:47:02,177 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:47:05,454 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:47:06,128 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:47:06,129 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:47:06,129 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:47:06,135 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:47:06,180 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:47:08,184 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:47:09,350 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 19:47:11,351 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:47:11,418 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:47:11,419 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:47:11,419 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:47:11,425 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:47:12,186 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:47:13,378 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:47:13,378 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:47:13,378 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:47:14,190 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:47:15,723 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:47:15,724 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:47:15,724 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:47:15,730 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:47:16,191 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:47:16,765 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:47:18,195 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:47:20,345 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:47:20,345 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:47:20,346 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:47:20,352 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:47:21,198 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:47:22,202 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:47:22,352 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:47:25,693 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:47:25,694 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:47:25,694 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:47:25,700 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:47:26,205 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:47:27,701 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:47:28,209 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:47:28,378 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:47:28,378 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:47:28,379 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:47:31,475 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:47:31,476 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:47:31,476 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:47:31,482 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:47:32,212 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:47:33,517 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:47:34,216 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:47:36,399 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:47:36,400 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:47:36,401 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:47:36,407 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:47:37,218 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:47:38,221 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:47:39,348 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 19:47:39,350 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:47:41,734 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:47:41,734 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:47:41,735 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:47:41,741 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:47:42,224 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:47:43,378 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:47:43,378 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:47:43,378 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:47:44,228 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:47:44,584 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:47:46,261 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:47:46,262 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:47:46,262 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:47:46,268 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:47:47,230 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:47:48,233 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:47:50,305 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:47:50,591 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:47:50,592 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:47:50,592 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:47:50,599 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:47:51,235 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:47:52,238 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:47:55,599 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:47:55,861 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:47:55,862 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:47:55,862 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:47:55,868 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:47:56,241 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:47:58,245 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:47:58,378 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:47:58,378 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:47:58,378 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:48:00,300 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:48:00,301 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:48:00,301 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:48:00,307 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:48:01,247 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:48:01,341 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:48:02,250 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:48:04,858 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:48:04,859 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:48:04,859 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:48:04,865 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:48:05,252 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:48:06,253 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:48:06,865 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:48:09,353 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 19:48:09,919 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:48:09,920 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:48:09,920 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:48:09,926 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:48:10,256 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:48:11,926 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:48:12,260 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:48:13,378 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:48:13,378 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:48:13,378 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:48:15,448 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:48:15,448 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:48:15,449 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:48:15,455 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:48:16,262 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:48:17,489 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:48:18,266 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:48:22,192 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:48:22,193 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:48:22,193 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:48:22,199 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:48:22,271 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:48:23,200 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:48:24,275 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:48:26,461 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:48:26,462 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:48:26,462 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:48:26,468 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:48:27,277 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:48:28,283 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:48:28,378 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:48:28,379 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:48:28,379 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:48:28,601 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:48:32,762 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:48:32,763 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:48:32,763 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:48:32,768 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:48:33,287 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:48:33,805 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:48:34,290 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:48:37,346 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:48:37,347 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:48:37,347 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:48:37,353 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:48:38,296 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:48:38,296 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:48:39,353 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 19:48:39,354 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:48:42,713 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:48:42,714 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:48:42,714 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:48:42,720 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:48:43,299 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:48:43,378 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:48:43,378 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:48:43,379 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:48:44,302 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:48:44,579 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:48:47,101 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:48:47,101 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:48:47,101 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:48:47,109 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:48:47,304 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:48:48,307 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:48:50,145 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:48:51,636 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:48:51,637 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:48:51,637 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:48:51,643 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:48:52,310 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:48:54,313 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:48:55,644 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:48:56,447 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:48:56,447 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:48:56,448 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:48:56,453 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:48:57,315 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:48:58,319 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:48:58,378 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:48:58,379 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:48:58,379 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:49:00,651 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:49:01,578 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:49:01,578 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:49:01,579 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:49:01,585 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:49:02,322 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:49:04,325 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:49:06,141 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:49:06,142 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:49:06,142 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:49:06,142 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:49:06,147 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:49:06,327 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:49:08,330 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:49:09,358 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 19:49:11,360 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:49:11,760 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:49:11,761 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:49:11,761 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:49:11,766 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:49:12,334 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:49:13,378 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:49:13,379 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:49:13,380 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:49:14,337 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:49:16,569 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:49:20,621 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:49:20,622 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:49:20,622 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:49:20,628 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:49:21,343 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:49:21,665 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:49:22,347 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:49:26,666 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:49:27,804 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:49:27,805 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:49:27,805 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:49:27,811 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:49:28,352 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:49:28,378 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:49:28,379 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:49:28,379 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:49:30,356 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:49:32,577 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:49:32,662 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:49:32,662 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:49:32,662 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:49:32,669 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:49:33,358 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:49:34,361 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:49:36,628 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:49:36,629 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:49:36,629 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:49:36,636 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:49:37,364 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:49:37,636 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:49:38,367 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:49:39,357 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 19:49:43,359 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:49:43,378 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:49:43,379 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:49:43,380 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:49:43,454 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:49:43,597 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:49:43,598 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:49:43,604 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:49:44,372 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:49:46,375 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:49:48,605 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:49:50,908 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:49:50,909 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:49:50,909 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:49:50,915 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:49:51,379 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:49:52,382 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:49:53,916 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:49:57,237 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:49:57,238 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:49:57,238 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:49:57,244 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:49:57,385 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:49:58,379 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:49:58,379 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:49:58,380 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:49:58,389 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:49:59,633 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:50:02,873 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:50:02,874 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:50:02,874 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:50:02,880 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:50:03,393 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:50:04,396 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:50:04,917 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:50:08,302 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:50:08,304 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:50:08,304 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:50:08,310 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:50:08,399 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:50:09,359 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 19:50:10,361 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:50:10,403 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:50:12,171 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:50:12,172 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:50:12,172 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:50:12,178 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:50:12,405 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:50:13,379 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:50:13,379 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:50:13,379 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:50:14,408 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:50:15,616 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:50:16,698 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:50:16,698 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:50:16,699 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:50:16,704 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:50:17,411 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:50:18,414 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:50:20,741 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:50:22,227 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:50:22,228 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:50:22,228 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:50:22,234 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:50:22,420 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:50:24,424 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:50:26,235 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:50:27,567 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:50:27,568 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:50:27,569 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:50:27,574 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:50:28,379 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:50:28,379 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:50:28,379 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:50:28,430 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:50:28,430 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:50:31,608 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:50:32,039 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:50:32,039 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:50:32,040 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:50:32,046 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:50:32,432 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:50:34,434 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:50:37,082 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:50:37,971 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:50:37,975 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:50:37,975 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:50:37,981 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:50:38,437 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:50:39,362 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 19:50:40,441 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:50:42,363 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:50:43,379 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:50:43,379 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:50:43,380 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:50:46,477 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:50:46,477 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:50:46,478 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:50:46,483 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:50:47,446 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:50:47,521 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:50:48,449 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:50:52,075 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:50:52,076 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:50:52,076 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:50:52,081 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:50:52,451 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:50:53,082 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:50:54,455 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:50:58,082 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:50:58,379 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:50:58,379 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:50:58,379 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:51:00,541 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:51:00,542 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:51:00,542 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:51:00,548 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:51:01,461 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:51:02,464 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:51:03,585 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:51:04,849 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:51:04,850 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:51:04,850 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:51:04,855 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:51:05,466 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:51:06,469 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:51:08,856 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:51:09,364 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 19:51:10,285 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:51:10,286 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:51:10,286 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:51:10,292 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:51:10,473 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:51:12,476 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:51:13,379 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:51:13,379 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:51:13,379 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:51:14,629 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:51:15,237 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:51:15,237 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:51:15,238 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:51:15,243 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:51:15,479 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:51:16,482 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:51:20,164 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:51:20,165 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:51:20,165 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:51:20,165 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:51:20,171 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:51:20,485 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:51:22,491 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:51:25,172 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:51:25,277 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:51:25,278 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:51:25,278 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:51:25,285 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:51:25,493 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:51:26,496 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:51:28,379 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:51:28,379 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:51:28,381 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:51:29,335 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:51:29,335 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:51:29,335 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:51:29,341 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:51:29,499 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:51:30,377 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:51:30,502 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:51:35,378 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:51:35,648 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:51:35,649 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:51:35,649 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:51:35,655 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:51:36,509 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:51:36,510 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:51:39,370 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 19:51:41,371 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:51:42,606 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:51:42,607 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:51:42,607 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:51:42,614 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:51:43,379 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:51:43,379 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:51:43,381 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:51:43,515 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:51:44,518 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:51:46,635 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:51:47,007 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:51:47,007 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:51:47,008 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:51:47,014 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:51:47,520 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:51:48,523 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:51:52,050 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:51:55,473 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:51:55,474 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:51:55,474 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:51:55,480 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:51:55,528 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:51:56,531 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:51:57,481 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:51:58,379 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:51:58,379 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:51:58,381 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:52:00,826 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:52:00,827 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:52:00,827 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:52:00,833 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:52:01,536 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:52:02,539 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:52:02,869 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:52:04,991 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:52:04,992 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:52:04,992 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:52:04,998 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:52:05,541 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:52:06,544 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:52:07,999 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:52:09,368 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 19:52:10,963 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:52:10,964 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:52:10,964 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:52:10,970 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:52:11,548 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:52:12,551 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:52:13,379 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:52:13,380 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:52:13,380 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:52:13,671 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:52:15,495 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:52:15,496 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:52:15,496 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:52:15,502 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:52:15,553 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:52:16,556 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:52:19,537 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:52:24,055 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:52:24,056 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:52:24,056 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:52:24,062 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:52:24,563 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:52:25,063 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:52:26,567 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:52:28,379 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:52:28,380 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:52:28,380 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:52:30,579 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:52:30,897 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:52:30,898 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:52:30,898 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:52:30,904 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:52:31,571 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:52:32,574 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:52:34,948 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:52:34,949 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:52:34,949 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:52:34,955 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:52:35,576 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:52:35,956 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:52:36,579 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:52:39,370 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 19:52:41,208 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:52:41,209 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:52:41,209 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:52:41,209 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:52:41,215 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:52:41,583 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:52:42,587 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:52:43,379 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:52:43,380 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:52:43,381 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:52:46,317 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:52:46,321 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:52:46,321 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:52:46,321 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:52:46,327 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:52:46,590 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:52:48,594 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:52:51,133 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:52:51,134 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:52:51,134 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:52:51,140 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:52:51,596 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:52:52,140 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:52:52,598 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:52:57,141 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:52:58,379 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:52:58,380 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:52:58,381 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:53:00,359 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:53:00,360 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:53:00,360 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:53:00,366 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:53:00,605 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:53:02,401 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:53:02,609 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:53:05,354 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:53:05,355 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:53:05,355 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:53:05,361 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:53:05,611 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:53:06,614 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:53:08,362 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:53:09,372 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 19:53:09,656 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:53:09,657 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:53:09,657 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:53:09,663 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:53:10,619 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:53:10,619 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:53:13,380 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:53:13,380 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:53:13,380 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:53:13,613 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:53:14,545 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:53:14,546 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:53:14,546 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:53:14,552 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:53:14,622 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:53:16,625 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:53:18,695 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:53:18,696 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:53:18,696 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:53:18,696 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:53:18,702 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:53:19,628 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:53:20,631 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:53:23,333 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:53:23,334 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:53:23,334 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:53:23,340 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:53:23,635 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:53:24,340 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:53:24,638 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:53:27,644 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:53:27,645 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:53:27,645 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:53:27,652 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:53:28,380 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:53:28,380 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:53:28,380 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:53:28,644 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:53:28,644 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:53:29,590 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:53:32,359 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:53:32,359 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:53:32,359 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:53:32,366 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:53:32,647 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:53:34,651 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:53:35,401 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:53:37,219 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:53:37,220 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:53:37,220 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:53:37,226 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:53:37,654 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:53:38,657 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:53:39,375 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 19:53:41,376 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:53:42,097 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:53:42,098 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:53:42,098 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:53:42,104 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:53:42,659 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:53:43,380 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:53:43,380 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:53:43,380 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:53:44,663 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:53:46,493 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:53:46,494 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:53:46,494 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:53:46,494 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:53:46,500 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:53:46,665 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:53:48,668 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:53:51,537 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:53:53,962 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:53:53,963 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:53:53,963 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:53:53,969 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:53:54,672 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:53:56,677 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:53:56,970 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:53:58,380 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:53:58,380 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:53:58,381 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:53:59,542 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:53:59,543 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:53:59,543 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:53:59,549 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:53:59,679 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:54:00,682 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:54:02,585 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:54:05,011 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:54:05,011 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:54:05,011 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:54:05,018 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:54:05,686 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:54:06,689 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:54:08,018 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:54:09,377 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 19:54:09,697 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:54:09,698 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:54:09,698 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:54:09,704 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:54:10,694 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:54:10,695 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:54:13,380 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:54:13,380 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:54:13,380 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:54:13,640 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:54:14,530 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:54:14,530 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:54:14,531 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:54:14,536 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:54:14,698 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:54:16,701 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:54:19,088 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:54:19,089 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:54:19,089 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:54:19,089 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:54:19,096 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:54:19,703 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:54:20,706 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:54:23,364 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:54:23,365 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:54:23,365 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:54:23,371 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:54:23,710 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:54:24,372 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:54:24,713 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:54:28,380 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:54:28,380 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:54:28,380 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:54:28,645 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:54:28,657 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:54:28,658 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:54:28,664 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:54:28,716 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:54:29,664 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:54:30,720 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:54:33,346 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:54:33,346 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:54:33,347 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:54:33,352 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:54:33,722 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:54:34,725 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:54:35,389 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:54:38,020 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:54:38,021 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:54:38,021 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:54:38,026 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:54:38,728 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:54:39,379 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 19:54:40,732 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:54:41,381 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:54:43,380 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:54:43,380 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:54:43,380 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:54:44,519 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:54:44,519 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:54:44,519 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:54:44,525 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:54:44,735 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:54:46,561 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:54:46,738 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:54:49,401 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:54:49,402 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:54:49,402 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:54:49,408 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:54:49,740 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:54:50,743 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:54:52,409 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:54:53,638 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:54:53,639 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:54:53,639 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:54:53,645 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:54:53,746 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:54:54,749 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:54:57,646 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:54:58,380 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:54:58,380 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:54:58,382 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:54:59,175 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:54:59,176 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:54:59,176 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:54:59,182 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:54:59,753 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:55:00,757 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:55:03,217 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:55:05,592 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:55:05,593 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:55:05,593 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:55:05,599 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:55:05,761 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:55:06,764 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:55:08,600 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:55:09,385 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 19:55:11,514 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:55:11,514 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:55:11,515 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:55:11,520 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:55:11,768 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:55:12,770 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:55:13,380 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:55:13,381 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:55:13,381 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:55:14,580 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:55:16,199 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:55:16,200 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:55:16,200 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:55:16,206 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:55:16,773 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:55:18,777 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:55:20,241 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:55:20,834 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:55:20,835 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:55:20,835 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:55:20,840 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:55:21,779 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:55:22,783 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:55:25,841 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:55:26,041 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:55:26,042 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:55:26,042 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:55:26,047 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:55:26,786 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:55:28,380 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:55:28,381 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:55:28,382 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:55:28,790 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:55:31,614 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:55:33,153 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:55:33,154 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:55:33,154 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:55:33,159 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:55:33,795 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:55:34,798 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:55:37,197 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:55:37,391 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:55:37,392 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:55:37,392 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:55:37,399 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:55:37,800 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:55:38,803 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:55:39,384 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 19:55:41,920 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:55:41,921 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:55:41,921 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:55:41,927 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:55:42,809 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:55:42,809 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:55:42,928 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:55:43,380 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:55:43,381 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:55:43,382 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:55:48,035 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:55:48,036 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:55:48,036 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:55:48,036 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:55:48,042 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:55:48,813 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:55:50,817 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:55:52,450 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:55:52,452 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:55:52,452 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:55:52,457 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:55:52,818 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:55:53,458 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:55:54,822 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:55:57,222 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:55:57,224 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:55:57,224 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:55:57,229 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:55:57,824 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:55:58,381 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:55:58,381 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:55:58,381 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:55:58,619 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:55:58,833 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:56:02,572 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:56:02,573 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:56:02,573 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:56:02,579 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:56:02,836 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:56:04,613 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:56:04,840 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:56:07,016 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:56:07,017 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:56:07,017 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:56:07,023 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:56:07,842 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:56:08,845 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:56:09,389 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 19:56:10,391 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:56:13,197 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:56:13,198 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:56:13,198 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:56:13,205 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:56:13,381 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:56:13,381 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:56:13,381 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:56:13,849 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:56:14,851 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:56:15,615 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:56:18,923 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:56:18,924 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:56:18,924 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:56:18,930 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:56:19,855 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:56:20,858 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:56:20,965 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:56:23,173 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:56:23,174 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:56:23,174 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:56:23,180 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:56:23,862 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:56:24,865 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:56:26,180 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:56:28,381 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:56:28,381 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:56:28,381 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:56:31,607 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:56:31,624 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:56:31,625 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:56:31,625 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:56:31,631 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:56:31,871 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:56:32,873 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:56:36,665 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:56:38,426 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:56:38,426 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:56:38,427 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:56:38,433 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:56:38,878 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:56:39,388 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 19:56:40,882 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:56:42,390 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:56:42,671 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:56:42,672 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:56:42,673 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:56:42,679 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:56:42,883 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:56:43,381 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:56:43,381 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:56:43,382 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:56:44,887 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:56:47,589 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:56:51,062 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:56:51,062 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:56:51,063 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:56:51,069 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:56:51,891 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:56:52,895 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:56:53,105 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:56:55,741 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:56:55,742 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:56:55,742 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:56:55,748 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:56:55,897 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:56:56,900 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:56:58,381 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:56:58,381 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:56:58,381 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:56:58,678 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:56:59,879 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:56:59,880 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:56:59,880 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:56:59,886 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:56:59,903 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:57:00,906 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:57:03,922 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:57:05,034 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:57:05,035 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:57:05,035 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:57:05,041 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:57:05,910 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:57:06,913 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:57:09,042 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:57:09,390 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 19:57:10,191 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:57:10,192 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:57:10,192 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:57:10,197 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:57:10,916 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:57:12,919 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:57:13,381 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:57:13,381 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:57:13,381 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:57:14,690 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:57:15,628 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:57:15,629 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:57:15,629 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:57:15,636 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:57:15,922 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:57:16,925 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:57:20,669 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:57:21,344 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:57:21,345 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:57:21,345 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:57:21,351 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:57:21,928 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:57:22,933 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:57:26,077 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:57:26,078 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:57:26,078 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:57:26,078 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:57:26,084 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:57:26,936 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:57:28,381 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:57:28,381 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:57:28,381 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:57:28,940 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:57:31,634 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:57:33,839 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:57:33,839 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:57:33,840 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:57:33,846 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:57:33,944 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:57:34,947 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:57:36,881 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:57:38,078 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:57:38,079 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:57:38,079 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:57:38,085 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:57:38,952 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:57:38,952 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:57:39,393 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 19:57:42,327 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:57:42,328 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:57:42,328 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:57:42,328 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:57:42,334 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:57:42,955 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:57:43,382 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:57:43,382 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:57:43,383 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:57:44,959 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:57:47,634 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:57:50,749 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:57:50,750 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:57:50,750 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:57:50,756 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:57:50,963 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:57:52,757 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:57:52,967 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:57:57,079 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:57:57,080 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:57:57,080 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:57:57,086 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:57:57,970 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:57:58,086 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:57:58,381 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:57:58,381 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:57:58,383 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:57:58,974 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:58:01,553 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:58:01,554 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:58:01,554 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:58:01,560 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:58:01,976 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:58:02,979 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:58:03,597 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:58:06,295 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:58:06,296 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:58:06,297 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:58:06,303 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:58:06,982 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:58:08,987 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:58:09,303 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:58:09,395 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 19:58:11,297 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:58:11,298 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:58:11,298 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:58:11,304 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:58:11,989 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:58:12,992 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:58:13,381 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:58:13,382 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:58:13,382 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:58:14,637 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:58:16,090 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:58:16,091 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:58:16,091 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:58:16,097 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:58:16,998 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:58:16,998 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:58:20,133 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:58:22,019 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:58:22,020 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:58:22,020 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:58:22,026 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:58:23,007 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:58:23,008 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:58:26,027 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:58:27,715 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:58:27,716 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:58:27,716 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:58:27,722 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:58:28,011 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:58:28,381 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:58:28,382 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:58:28,382 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:58:29,015 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:58:31,609 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:58:31,960 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:58:31,961 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:58:31,961 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:58:31,967 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:58:32,017 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:58:33,020 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:58:36,374 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:58:36,374 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:58:36,374 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:58:36,381 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:58:37,024 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:58:37,381 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:58:39,029 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:58:39,397 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 19:58:40,923 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:58:40,924 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:58:40,924 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:58:40,929 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:58:41,030 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:58:42,929 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:58:43,037 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:58:43,381 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:58:43,382 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:58:43,382 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:58:45,015 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:58:45,015 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:58:45,016 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:58:45,030 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:58:45,039 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:58:47,043 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:58:48,057 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:58:49,841 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:58:49,842 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:58:49,842 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:58:49,849 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:58:50,045 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:58:51,048 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:58:53,850 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:58:55,049 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:58:55,049 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:58:55,050 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:58:55,056 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:58:55,056 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:58:57,060 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:58:58,382 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:58:58,382 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:58:58,382 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:58:59,654 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:58:59,731 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:58:59,731 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:58:59,732 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:58:59,737 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:59:00,064 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:59:01,066 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:59:04,172 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:59:04,173 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:59:04,173 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:59:04,179 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:59:05,072 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:59:05,072 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:59:05,179 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:59:09,399 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 19:59:10,400 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:59:11,325 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:59:11,326 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:59:11,326 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:59:11,332 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:59:12,077 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:59:13,080 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:59:13,382 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:59:13,382 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:59:13,382 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:59:15,648 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:59:16,669 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:59:16,670 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:59:16,670 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:59:16,676 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:59:17,083 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:59:19,088 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:59:20,713 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:59:22,566 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:59:22,567 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:59:22,567 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:59:22,573 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:59:23,093 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:59:25,097 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:59:26,573 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:59:27,165 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:59:27,166 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:59:27,166 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:59:27,172 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:59:28,099 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:59:28,382 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:59:28,382 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:59:28,383 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:59:29,103 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:59:31,585 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:59:33,385 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:59:33,385 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:59:33,386 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:59:33,391 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:59:34,106 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:59:35,110 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:59:37,429 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:59:38,325 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:59:38,326 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:59:38,326 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:59:38,332 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:59:39,114 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:59:39,405 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 19:59:41,119 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:59:42,613 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:59:42,614 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:59:42,614 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:59:42,614 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:59:42,620 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:59:43,120 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:59:43,382 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:59:43,382 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:59:43,382 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:59:45,125 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:59:47,197 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:59:47,198 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:59:47,198 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:59:47,204 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:59:48,127 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:59:48,241 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:59:49,130 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:59:51,890 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:59:51,891 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:59:51,891 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:59:51,897 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:59:52,133 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:59:53,136 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:59:53,897 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:59:56,078 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:59:56,079 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 19:59:56,079 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:59:56,085 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:59:56,138 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 19:59:57,142 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 19:59:58,382 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:59:58,382 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:59:58,382 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:59:59,617 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:00:00,604 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:00:00,604 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:00:00,605 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:00:00,610 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:00:01,145 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:00:03,151 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:00:04,645 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:00:04,887 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:00:04,887 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:00:04,887 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:00:04,893 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:00:05,153 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:00:07,157 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:00:08,870 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:00:08,870 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:00:08,871 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:00:08,876 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:00:09,158 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:00:09,404 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 20:00:10,405 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:00:11,163 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:00:13,382 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:00:13,382 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:00:13,383 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:00:15,605 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:00:17,281 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:00:17,282 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:00:17,282 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:00:17,288 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:00:18,170 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:00:19,174 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:00:21,325 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:00:22,167 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:00:22,168 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:00:22,168 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:00:22,173 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:00:22,176 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:00:23,180 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:00:27,174 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:00:27,344 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:00:27,345 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:00:27,345 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:00:27,351 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:00:28,184 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:00:28,382 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:00:28,382 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:00:28,384 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:00:29,189 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:00:32,292 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:00:32,293 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:00:32,293 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:00:32,294 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:00:32,299 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:00:33,193 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:00:35,197 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:00:36,612 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:00:36,612 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:00:36,613 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:00:36,619 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:00:37,200 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:00:37,619 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:00:39,204 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:00:39,409 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 20:00:41,370 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:00:41,371 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:00:41,371 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:00:41,376 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:00:42,206 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:00:43,210 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:00:43,377 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:00:43,382 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:00:43,382 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:00:43,383 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:00:47,679 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:00:47,680 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:00:47,680 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:00:47,689 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:00:48,214 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:00:48,721 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:00:49,219 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:00:52,098 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:00:52,099 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:00:52,099 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:00:52,105 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:00:52,221 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:00:53,225 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:00:54,106 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:00:57,184 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:00:57,185 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:00:57,186 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:00:57,191 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:00:57,229 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:00:58,382 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:00:58,383 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:00:58,383 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:00:59,232 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:00:59,744 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:01:01,727 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:01:01,728 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:01:01,728 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:01:01,733 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:01:02,235 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:01:03,239 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:01:04,769 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:01:06,709 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:01:06,710 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:01:06,710 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:01:06,716 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:01:07,244 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:01:09,249 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:01:09,408 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 20:01:10,409 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:01:13,382 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:01:13,383 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:01:13,384 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:01:13,484 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:01:13,682 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:01:13,683 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:01:13,689 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:01:14,253 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:01:15,258 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:01:15,689 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:01:18,883 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:01:18,884 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:01:18,884 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:01:18,889 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:01:19,261 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:01:20,890 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:01:21,266 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:01:23,474 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:01:23,475 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:01:23,475 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:01:23,481 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:01:24,270 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:01:25,273 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:01:26,482 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:01:28,382 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:01:28,383 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:01:28,383 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:01:31,419 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:01:31,419 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:01:31,420 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:01:31,425 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:01:32,279 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:01:32,461 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:01:33,284 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:01:35,883 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:01:35,883 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:01:35,884 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:01:35,890 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:01:36,286 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:01:37,291 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:01:37,890 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:01:39,410 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 20:01:40,959 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:01:40,960 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:01:40,960 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:01:40,966 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:01:41,294 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:01:42,967 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:01:43,299 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:01:43,382 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:01:43,383 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:01:43,383 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:01:45,823 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:01:45,823 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:01:45,824 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:01:45,829 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:01:46,302 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:01:47,305 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:01:48,865 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:01:52,368 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:01:52,369 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:01:52,369 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:01:52,375 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:01:53,314 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:01:53,314 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:01:54,376 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:01:57,300 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:01:57,301 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:01:57,301 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:01:57,307 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:01:57,317 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:01:58,382 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:01:58,383 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:01:58,384 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:01:59,323 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:01:59,684 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:02:01,578 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:02:01,578 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:02:01,579 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:02:01,585 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:02:02,326 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:02:03,330 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:02:05,621 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:02:06,073 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:02:06,074 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:02:06,074 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:02:06,080 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:02:06,332 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:02:07,335 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:02:09,412 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 20:02:10,451 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:02:10,451 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:02:10,451 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:02:10,458 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:02:11,338 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:02:11,458 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:02:13,343 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:02:13,383 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:02:13,383 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:02:13,384 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:02:14,734 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:02:14,734 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:02:14,734 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:02:14,740 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:02:15,345 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:02:16,777 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:02:17,349 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:02:20,665 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:02:20,666 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:02:20,666 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:02:20,672 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:02:21,353 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:02:22,672 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:02:23,360 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:02:24,985 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:02:24,986 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:02:24,986 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:02:24,991 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:02:25,362 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:02:27,366 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:02:27,992 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:02:28,383 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:02:28,383 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:02:28,384 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:02:29,287 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:02:29,288 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:02:29,288 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:02:29,294 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:02:29,367 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:02:31,371 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:02:33,329 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:02:37,756 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:02:37,757 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:02:37,757 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:02:37,763 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:02:38,377 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:02:38,763 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:02:39,381 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:02:39,414 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 20:02:41,601 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:02:41,602 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:02:41,602 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:02:41,608 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:02:42,383 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:02:43,384 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:02:43,385 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:02:43,385 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:02:43,389 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:02:44,671 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:02:46,278 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:02:46,279 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:02:46,279 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:02:46,285 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:02:46,392 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:02:47,395 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:02:50,321 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:02:50,601 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:02:50,601 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:02:50,602 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:02:50,608 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:02:51,398 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:02:53,403 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:02:55,180 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:02:55,181 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:02:55,181 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:02:55,187 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:02:55,405 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:02:56,188 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:02:57,413 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:02:58,383 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:02:58,383 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:02:58,383 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:03:00,570 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:03:00,574 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:03:00,574 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:03:00,581 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:03:01,417 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:03:01,613 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:03:03,422 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:03:05,580 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:03:05,581 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:03:05,581 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:03:05,588 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:03:06,424 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:03:07,428 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:03:07,588 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:03:09,417 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 20:03:09,857 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:03:09,858 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:03:09,858 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:03:09,865 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:03:10,430 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:03:11,433 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:03:12,865 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:03:13,383 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:03:13,383 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:03:13,384 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:03:18,508 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:03:18,508 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:03:18,508 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:03:18,509 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:03:18,514 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:03:19,439 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:03:21,444 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:03:23,549 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:03:24,878 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:03:24,879 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:03:24,880 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:03:24,886 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:03:25,449 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:03:27,453 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:03:28,383 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:03:28,383 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:03:28,384 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:03:28,678 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:03:29,393 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:03:29,394 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:03:29,394 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:03:29,401 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:03:29,455 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:03:31,459 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:03:34,141 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:03:34,142 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:03:34,142 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:03:34,142 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:03:34,148 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:03:34,461 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:03:35,464 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:03:39,149 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:03:39,400 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:03:39,401 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:03:39,401 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:03:39,407 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:03:39,419 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 20:03:39,468 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:03:41,473 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:03:43,383 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:03:43,383 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:03:43,384 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:03:44,159 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:03:44,159 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:03:44,159 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:03:44,160 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:03:44,166 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:03:44,475 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:03:45,479 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:03:48,023 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:03:48,024 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:03:48,024 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:03:48,030 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:03:48,482 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:03:49,485 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:03:50,030 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:03:53,448 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:03:53,449 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:03:53,449 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:03:53,455 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:03:53,488 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:03:55,455 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:03:55,494 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:03:58,263 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:03:58,264 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:03:58,265 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:03:58,271 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:03:58,383 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:03:58,384 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:03:58,385 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:03:58,496 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:03:59,500 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:04:00,655 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:04:05,298 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:04:05,299 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:04:05,299 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:04:05,305 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:04:05,507 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:04:06,341 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:04:07,513 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:04:09,425 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 20:04:09,776 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:04:09,777 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:04:09,777 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:04:09,783 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:04:10,515 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:04:11,519 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:04:11,784 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:04:13,383 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:04:13,384 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:04:13,385 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:04:14,073 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:04:14,074 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:04:14,074 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:04:14,080 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:04:14,521 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:04:15,525 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:04:17,117 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:04:18,654 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:04:18,655 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:04:18,655 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:04:18,661 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:04:19,528 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:04:21,534 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:04:22,662 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:04:23,975 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:04:23,976 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:04:23,976 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:04:23,983 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:04:24,538 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:04:25,542 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:04:27,984 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:04:28,383 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:04:28,384 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:04:28,385 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:04:28,566 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:04:28,588 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:04:28,588 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:04:28,594 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:04:29,546 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:04:31,550 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:04:33,297 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:04:33,298 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:04:33,298 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:04:33,298 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:04:33,304 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:04:33,551 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:04:35,555 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:04:38,341 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:04:39,423 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 20:04:41,802 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:04:41,803 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:04:41,804 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:04:41,810 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:04:42,561 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:04:43,383 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:04:43,384 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:04:43,385 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:04:43,565 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:04:43,640 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:04:45,845 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:04:45,846 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:04:45,846 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:04:45,852 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:04:46,568 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:04:47,571 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:04:48,889 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:04:50,634 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:04:50,635 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:04:50,635 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:04:50,642 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:04:51,575 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:04:53,580 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:04:54,642 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:04:58,067 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:04:58,068 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:04:58,068 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:04:58,074 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:04:58,384 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:04:58,384 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:04:58,384 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:04:58,584 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:04:59,588 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:05:00,611 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:05:02,194 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:05:02,195 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:05:02,195 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:05:02,201 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:05:02,590 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:05:03,595 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:05:06,237 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:05:07,645 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:05:07,646 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:05:07,646 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:05:07,652 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:05:08,599 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:05:09,428 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 20:05:09,604 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:05:11,429 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:05:13,384 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:05:13,384 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:05:13,385 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:05:15,088 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:05:15,089 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:05:15,089 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:05:15,096 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:05:15,611 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:05:17,129 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:05:17,616 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:05:20,606 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:05:20,607 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:05:20,607 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:05:20,613 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:05:20,618 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:05:21,622 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:05:22,614 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:05:25,145 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:05:25,146 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:05:25,146 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:05:25,152 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:05:25,627 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:05:27,631 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:05:28,152 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:05:28,384 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:05:28,384 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:05:28,385 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:05:29,811 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:05:29,811 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:05:29,812 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:05:29,817 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:05:30,634 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:05:31,638 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:05:33,853 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:05:35,086 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:05:35,086 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:05:35,087 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:05:35,093 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:05:35,641 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:05:37,646 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:05:38,975 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:05:38,976 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:05:38,976 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:05:38,976 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:05:38,983 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:05:39,427 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 20:05:39,648 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:05:41,653 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:05:43,283 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:05:43,285 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:05:43,285 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:05:43,291 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:05:43,384 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:05:43,384 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:05:43,385 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:05:43,654 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:05:44,629 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:05:45,659 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:05:48,630 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:05:48,631 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:05:48,631 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:05:48,638 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:05:48,661 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:05:49,673 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:05:51,669 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:05:52,862 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:05:52,863 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:05:52,863 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:05:52,869 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:05:53,671 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:05:54,869 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:05:55,675 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:05:58,070 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:05:58,071 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:05:58,071 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:05:58,077 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:05:58,384 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:05:58,384 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:05:58,386 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:05:58,678 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:05:59,681 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:06:00,664 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:06:05,552 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:06:05,553 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:06:05,553 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:06:05,560 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:06:05,687 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:06:06,593 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:06:07,691 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:06:09,429 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 20:06:12,431 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:06:12,533 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:06:12,534 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:06:12,534 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:06:12,541 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:06:12,695 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:06:13,384 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:06:13,384 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:06:13,384 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:06:13,700 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:06:17,666 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:06:17,732 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:06:17,737 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:06:17,737 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:06:17,743 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:06:18,704 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:06:19,708 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:06:22,022 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:06:22,023 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:06:22,023 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:06:22,029 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:06:22,711 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:06:23,029 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:06:23,716 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:06:28,030 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:06:28,158 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:06:28,159 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:06:28,159 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:06:28,165 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:06:28,384 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:06:28,384 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:06:28,384 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:06:28,720 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:06:29,724 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:06:32,667 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:06:32,668 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:06:32,668 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:06:32,674 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:06:32,726 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:06:33,709 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:06:35,732 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:06:38,709 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:06:39,432 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 20:06:40,950 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:06:40,951 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:06:40,951 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:06:40,957 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:06:41,739 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:06:43,384 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:06:43,384 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:06:43,386 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:06:43,744 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:06:44,677 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:06:46,588 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:06:46,588 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:06:46,589 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:06:46,595 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:06:46,746 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:06:47,748 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:06:50,629 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:06:51,280 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:06:51,281 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:06:51,281 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:06:51,288 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:06:51,753 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:06:53,759 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:06:56,288 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:06:58,384 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:06:58,384 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:06:58,386 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:06:59,448 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:06:59,449 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:06:59,449 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:06:59,455 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:06:59,765 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:07:01,489 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:07:01,770 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:07:06,490 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:07:07,386 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:07:07,387 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:07:07,387 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:07:07,393 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:07:07,777 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:07:09,434 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 20:07:09,782 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:07:12,334 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:07:12,335 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:07:12,335 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:07:12,335 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:07:12,341 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:07:12,784 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:07:13,384 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:07:13,385 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:07:13,385 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:07:13,788 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:07:17,590 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:07:19,134 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:07:19,135 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:07:19,135 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:07:19,141 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:07:19,794 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:07:21,799 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:07:23,177 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:07:24,901 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:07:24,902 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:07:24,902 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:07:24,908 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:07:25,805 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:07:27,809 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:07:28,384 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:07:28,385 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:07:28,385 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:07:28,658 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:07:29,430 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:07:29,431 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:07:29,431 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:07:29,438 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:07:29,812 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:07:31,816 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:07:34,473 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:07:35,262 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:07:35,263 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:07:35,263 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:07:35,269 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:07:35,819 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:07:37,824 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:07:39,439 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 20:07:40,441 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:07:40,620 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:07:40,621 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:07:40,622 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:07:40,628 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:07:40,830 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:07:40,830 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:07:43,384 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:07:43,385 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:07:43,386 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:07:44,940 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:07:44,941 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:07:44,941 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:07:44,947 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:07:45,834 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:07:45,981 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:07:46,837 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:07:49,888 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:07:49,889 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:07:49,889 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:07:49,896 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:07:50,842 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:07:50,842 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:07:51,896 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:07:56,896 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:07:58,231 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:07:58,232 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:07:58,232 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:07:58,239 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:07:58,385 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:07:58,385 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:07:58,385 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:07:58,853 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:07:58,853 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:08:02,677 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:08:03,051 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:08:03,052 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:08:03,052 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:08:03,058 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:08:03,859 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:08:04,862 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:08:08,093 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:08:08,545 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:08:08,546 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:08:08,546 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:08:08,551 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:08:08,868 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:08:08,868 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:08:09,438 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 20:08:13,385 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:08:13,385 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:08:13,386 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:08:13,633 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:08:14,337 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:08:14,338 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:08:14,338 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:08:14,344 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:08:14,877 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:08:14,877 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:08:19,381 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:08:22,190 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:08:22,191 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:08:22,191 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:08:22,197 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:08:22,888 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:08:22,888 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:08:25,197 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:08:27,282 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:08:27,283 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:08:27,283 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:08:27,289 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:08:27,894 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:08:28,385 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:08:28,385 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:08:28,386 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:08:28,897 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:08:30,651 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:08:32,261 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:08:32,262 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:08:32,262 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:08:32,268 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:08:32,904 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:08:32,904 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:08:36,305 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:08:38,576 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:08:38,577 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:08:38,577 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:08:38,583 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:08:38,912 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:08:38,912 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:08:39,444 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 20:08:41,445 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:08:43,385 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:08:43,385 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:08:43,385 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:08:43,420 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:08:43,608 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:08:43,609 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:08:43,614 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:08:43,918 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:08:44,922 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:08:46,615 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:08:48,297 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:08:48,298 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:08:48,298 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:08:48,304 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:08:48,927 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:08:48,927 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:08:52,228 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:08:52,229 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:08:52,229 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:08:52,229 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:08:52,235 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:08:52,933 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:08:52,933 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:08:57,236 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:08:58,385 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:08:58,385 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:08:58,386 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:08:58,781 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:08:58,782 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:08:58,782 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:08:58,788 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:08:58,942 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:08:58,942 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:09:02,826 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:09:04,197 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:09:04,198 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:09:04,198 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:09:04,204 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:09:04,951 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:09:04,952 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:09:08,204 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:09:08,670 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:09:08,670 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:09:08,671 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:09:08,677 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:09:08,956 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:09:08,957 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:09:09,443 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 20:09:12,832 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:09:12,833 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:09:12,833 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:09:12,839 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:09:12,963 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:09:12,963 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:09:13,385 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:09:13,385 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:09:13,386 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:09:13,578 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:09:17,252 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:09:17,253 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:09:17,253 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:09:17,259 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:09:17,968 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:09:18,971 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:09:19,293 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:09:21,502 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:09:21,502 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:09:21,503 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:09:21,508 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:09:21,974 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:09:22,978 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:09:24,509 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:09:28,385 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:09:28,385 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:09:28,386 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:09:29,645 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:09:30,611 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:09:30,612 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:09:30,612 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:09:30,618 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:09:30,989 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:09:30,989 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:09:34,653 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:09:38,936 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:09:38,937 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:09:38,937 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:09:38,943 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:09:38,998 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:09:39,448 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 20:09:40,450 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:09:41,004 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:09:43,385 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:09:43,385 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:09:43,385 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:09:45,679 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:09:47,230 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:09:47,231 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:09:47,231 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:09:47,237 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:09:48,013 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:09:49,015 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:09:51,273 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:09:51,751 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:09:51,751 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:09:51,751 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:09:51,757 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:09:52,019 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:09:53,023 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:09:56,758 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:09:58,385 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:09:58,385 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:09:58,387 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:09:58,547 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:09:58,608 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:09:58,609 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:09:58,614 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:09:59,031 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:09:59,031 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:10:02,615 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:10:04,331 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:10:04,332 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:10:04,332 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:10:04,339 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:10:05,039 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:10:05,039 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:10:08,373 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:10:09,447 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 20:10:09,724 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:10:09,725 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:10:09,725 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:10:09,730 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:10:10,045 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:10:11,049 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:10:13,385 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:10:13,386 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:10:13,387 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:10:13,659 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:10:15,085 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:10:15,086 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:10:15,086 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:10:15,092 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:10:16,054 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:10:17,057 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:10:19,129 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:10:20,691 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:10:20,692 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:10:20,692 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:10:20,698 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:10:21,062 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:10:21,063 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:10:24,699 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:10:25,689 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:10:25,690 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:10:25,690 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:10:25,696 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:10:26,069 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:10:27,072 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:10:28,385 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:10:28,386 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:10:28,387 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:10:30,670 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:10:32,172 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:10:32,173 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:10:32,173 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:10:32,179 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:10:33,080 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:10:33,080 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:10:36,213 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:10:37,058 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:10:37,059 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:10:37,059 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:10:37,065 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:10:37,083 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:10:39,089 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:10:39,449 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 20:10:41,451 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:10:42,586 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:10:42,587 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:10:42,587 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:10:42,593 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:10:43,095 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:10:43,096 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:10:43,385 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:10:43,386 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:10:43,387 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:10:46,618 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:10:48,100 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:10:48,101 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:10:48,101 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:10:48,106 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:10:49,104 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:10:49,104 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:10:52,141 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:10:53,712 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:10:53,713 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:10:53,713 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:10:53,719 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:10:54,110 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:10:55,112 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:10:57,720 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:10:58,385 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:10:58,386 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:10:58,387 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:11:00,076 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:11:00,076 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:11:00,077 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:11:00,083 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:11:00,118 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:11:01,121 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:11:03,118 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:11:04,382 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:11:04,383 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:11:04,383 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:11:04,389 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:11:05,128 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:11:05,128 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:11:08,390 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:11:09,452 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 20:11:09,987 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:11:09,988 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:11:09,988 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:11:09,994 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:11:10,134 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:11:11,138 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:11:13,386 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:11:13,386 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:11:13,387 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:11:13,631 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:11:15,753 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:11:15,754 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:11:15,754 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:11:15,760 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:11:16,143 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:11:17,146 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:11:18,797 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:11:21,168 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:11:21,169 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:11:21,169 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:11:21,174 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:11:22,151 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:11:23,154 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:11:24,175 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:11:25,923 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:11:25,924 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:11:25,924 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:11:25,930 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:11:26,158 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:11:27,161 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:11:28,386 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:11:28,386 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:11:28,387 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:11:29,681 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:11:30,123 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:11:30,123 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:11:30,123 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:11:30,130 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:11:30,164 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:11:31,166 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:11:35,166 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:11:36,394 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:11:36,395 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:11:36,395 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:11:36,401 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:11:37,175 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:11:37,175 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:11:39,453 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 20:11:40,455 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:11:41,175 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:11:41,176 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:11:41,176 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:11:41,182 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:11:41,182 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:11:43,186 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:11:43,386 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:11:43,386 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:11:43,386 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:11:45,572 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:11:45,572 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:11:45,572 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:11:45,572 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:11:45,578 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:11:46,188 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:11:47,192 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:11:50,613 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:11:51,195 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:11:51,196 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:11:51,197 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:11:51,203 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:11:52,196 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:11:53,199 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:11:56,203 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:11:58,386 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:11:58,386 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:11:58,387 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:11:59,710 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:11:59,711 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:11:59,711 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:11:59,717 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:12:00,207 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:12:01,210 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:12:01,753 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:12:03,815 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:12:03,816 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:12:03,816 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:12:03,822 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:12:04,213 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:12:05,216 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:12:06,823 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:12:08,267 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:12:08,268 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:12:08,268 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:12:08,273 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:12:09,222 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:12:09,222 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:12:09,459 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 20:12:12,461 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:12:13,368 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:12:13,369 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:12:13,369 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:12:13,375 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:12:13,386 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:12:13,386 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:12:13,387 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:12:14,228 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:12:15,231 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:12:17,631 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:12:21,824 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:12:21,825 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:12:21,825 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:12:21,830 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:12:22,239 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:12:22,865 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:12:23,243 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:12:27,865 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:12:28,386 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:12:28,386 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:12:28,386 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:12:29,007 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:12:29,008 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:12:29,008 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:12:29,013 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:12:29,251 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:12:29,251 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:12:33,049 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:12:34,060 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:12:34,061 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:12:34,061 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:12:34,067 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:12:34,256 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:12:35,259 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:12:38,067 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:12:39,458 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 20:12:39,493 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:12:39,493 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:12:39,493 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:12:39,499 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:12:40,266 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:12:41,269 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:12:43,386 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:12:43,386 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:12:43,388 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:12:43,664 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:12:48,095 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:12:48,096 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:12:48,096 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:12:48,103 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:12:48,277 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:12:49,137 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:12:49,280 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:12:54,031 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:12:54,032 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:12:54,033 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:12:54,038 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:12:54,286 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:12:55,039 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:12:55,289 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:12:58,386 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:12:58,386 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:12:58,388 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:12:59,981 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:12:59,982 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:12:59,982 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:12:59,987 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:13:00,295 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:13:01,025 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:13:01,298 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:13:04,854 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:13:04,855 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:13:04,856 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:13:04,861 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:13:05,304 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:13:05,304 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:13:06,862 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:13:08,997 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:13:08,998 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:13:08,998 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:13:09,004 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:13:09,310 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:13:09,310 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:13:09,463 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 20:13:12,465 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:13:13,386 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:13:13,386 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:13:13,388 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:13:13,480 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:13:13,672 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:13:13,673 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:13:13,679 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:13:14,316 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:13:15,319 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:13:17,680 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:13:18,194 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:13:18,195 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:13:18,196 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:13:18,201 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:13:18,322 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:13:19,325 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:13:23,202 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:13:23,995 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:13:23,995 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:13:23,996 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:13:24,002 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:13:24,332 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:13:25,334 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:13:28,386 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:13:28,387 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:13:28,388 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:13:28,614 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:13:28,679 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:13:28,679 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:13:28,679 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:13:28,685 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:13:29,339 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:13:29,340 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:13:33,685 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:13:34,533 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:13:34,534 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:13:34,534 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:13:34,540 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:13:35,348 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:13:35,348 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:13:39,462 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 20:13:39,464 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:13:39,868 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:13:39,869 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:13:39,869 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:13:39,875 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:13:40,355 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:13:41,357 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:13:43,386 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:13:43,387 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:13:43,387 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:13:44,599 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:13:44,860 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:13:44,861 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:13:44,861 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:13:44,867 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:13:45,363 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:13:45,363 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:13:49,901 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:13:51,681 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:13:51,682 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:13:51,682 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:13:51,688 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:13:52,371 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:13:53,374 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:13:55,688 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:13:56,987 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:13:56,988 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:13:56,988 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:13:56,994 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:13:57,380 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:13:57,380 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:13:58,386 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:13:58,387 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:13:58,388 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:14:01,195 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:14:01,196 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:14:01,196 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:14:01,196 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:14:01,202 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:14:01,384 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:14:03,389 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:14:06,237 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:14:09,465 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 20:14:09,522 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:14:09,523 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:14:09,523 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:14:09,529 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:14:10,397 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:14:11,404 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:14:11,529 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:14:13,387 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:14:13,387 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:14:13,387 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:14:16,248 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:14:16,249 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:14:16,249 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:14:16,255 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:14:16,408 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:14:17,289 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:14:17,411 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:14:20,486 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:14:20,487 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:14:20,487 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:14:20,492 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:14:21,416 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:14:21,416 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:14:22,493 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:14:26,728 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:14:26,729 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:14:26,729 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:14:26,735 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:14:27,423 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:14:27,736 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:14:28,387 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:14:28,387 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:14:28,388 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:14:29,426 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:14:31,266 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:14:31,270 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:14:31,270 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:14:31,276 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:14:31,428 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:14:33,309 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:14:33,432 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:14:36,398 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:14:36,400 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:14:36,400 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:14:36,405 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:14:36,434 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:14:37,437 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:14:38,406 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:14:39,467 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 20:14:40,359 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:14:40,360 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:14:40,360 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:14:40,366 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:14:40,441 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:14:41,444 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:14:43,387 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:14:43,387 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:14:43,387 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:14:43,683 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:14:46,230 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:14:46,230 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:14:46,230 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:14:46,236 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:14:46,450 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:14:47,453 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:14:49,273 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:14:51,154 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:14:51,154 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:14:51,155 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:14:51,160 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:14:51,456 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:14:53,462 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:14:55,161 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:14:56,161 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:14:56,162 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:14:56,162 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:14:56,168 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:14:56,465 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:14:57,469 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:14:58,387 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:14:58,387 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:14:58,387 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:15:00,614 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:15:00,614 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:15:00,615 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:15:00,615 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:15:00,620 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:15:01,474 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:15:01,474 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:15:04,862 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:15:04,863 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:15:04,863 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:15:04,869 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:15:05,477 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:15:05,869 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:15:07,482 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:15:09,194 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:15:09,195 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:15:09,195 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:15:09,201 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:15:09,469 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 20:15:09,484 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:15:11,470 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:15:11,488 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:15:13,387 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:15:13,387 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:15:13,388 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:15:15,857 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:15:15,858 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:15:15,858 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:15:15,864 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:15:16,494 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:15:16,901 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:15:17,497 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:15:21,902 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:15:22,177 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:15:22,178 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:15:22,178 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:15:22,183 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:15:22,503 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:15:23,506 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:15:27,184 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:15:27,770 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:15:27,771 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:15:27,771 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:15:27,776 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:15:28,387 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:15:28,388 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:15:28,388 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:15:28,511 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:15:29,514 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:15:32,638 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:15:33,445 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:15:33,446 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:15:33,446 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:15:33,452 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:15:33,518 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:15:35,523 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:15:38,489 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:15:39,166 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:15:39,167 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:15:39,167 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:15:39,173 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:15:39,475 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 20:15:39,527 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:15:41,532 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:15:43,387 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:15:43,388 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:15:43,388 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:15:43,541 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:15:43,625 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:15:43,625 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:15:43,625 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:15:43,632 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:15:44,535 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:15:45,538 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:15:48,633 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:15:49,114 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:15:49,115 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:15:49,115 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:15:49,121 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:15:49,541 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:15:51,546 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:15:54,122 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:15:55,444 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:15:55,445 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:15:55,445 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:15:55,451 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:15:55,550 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:15:57,555 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:15:58,387 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:15:58,388 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:15:58,388 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:15:59,602 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:15:59,602 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:15:59,603 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:15:59,603 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:15:59,608 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:16:00,558 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:16:01,565 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:16:03,660 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:16:03,661 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:16:03,661 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:16:03,667 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:16:04,568 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:16:04,668 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:16:05,571 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:16:09,474 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 20:16:10,475 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:16:10,610 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:16:10,611 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:16:10,611 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:16:10,617 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:16:11,577 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:16:11,577 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:16:13,388 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:16:13,388 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:16:13,389 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:16:15,372 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:16:15,373 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:16:15,373 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:16:15,378 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:16:15,580 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:16:16,413 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:16:17,584 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:16:20,250 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:16:20,251 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:16:20,251 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:16:20,258 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:16:20,587 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:16:21,590 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:16:22,258 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:16:24,129 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:16:24,130 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:16:24,130 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:16:24,136 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:16:24,595 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:16:25,598 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:16:28,136 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:16:28,387 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:16:28,387 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:16:28,389 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:16:29,036 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:16:29,037 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:16:29,037 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:16:29,043 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:16:29,602 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:16:31,606 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:16:33,186 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:16:33,187 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:16:33,187 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:16:33,187 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:16:33,192 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:16:33,607 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:16:35,611 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:16:38,193 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:16:38,660 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:16:38,661 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:16:38,662 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:16:38,667 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:16:39,479 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 20:16:39,617 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:16:39,617 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:16:43,387 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:16:43,388 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:16:43,389 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:16:43,492 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:16:43,626 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:16:43,627 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:16:43,627 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:16:43,632 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:16:44,622 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:16:45,625 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:16:47,979 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:16:47,981 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:16:47,981 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:16:47,986 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:16:48,628 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:16:48,987 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:16:49,631 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:16:52,534 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:16:52,535 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:16:52,535 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:16:52,542 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:16:52,634 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:16:53,638 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:16:54,542 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:16:57,277 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:16:57,278 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:16:57,278 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:16:57,284 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:16:57,641 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:16:58,388 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:16:58,388 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:16:58,389 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:16:59,646 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:16:59,660 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:17:02,343 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:17:02,344 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:17:02,344 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:17:02,350 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:17:02,649 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:17:03,652 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:17:05,385 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:17:09,478 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 20:17:10,479 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:17:11,006 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:17:11,007 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:17:11,007 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:17:11,013 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:17:11,662 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:17:13,388 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:17:13,388 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:17:13,389 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:17:13,667 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:17:15,590 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:17:16,203 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:17:16,204 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:17:16,204 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:17:16,210 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:17:16,670 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:17:17,673 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:17:21,246 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:17:24,727 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:17:24,728 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:17:24,728 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:17:24,734 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:17:25,686 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:17:25,686 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:17:26,735 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:17:28,388 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:17:28,388 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:17:28,389 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:17:29,767 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:17:29,768 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:17:29,768 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:17:29,774 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:17:30,690 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:17:31,693 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:17:31,809 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:17:35,003 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:17:35,004 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:17:35,004 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:17:35,010 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:17:35,696 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:17:37,011 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:17:37,701 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:17:39,480 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 20:17:40,059 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:17:40,060 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:17:40,060 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:17:40,066 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:17:40,705 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:17:41,707 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:17:42,066 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:17:43,388 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:17:43,388 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:17:43,388 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:17:47,174 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:17:47,175 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:17:47,175 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:17:47,175 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:17:47,181 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:17:47,714 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:17:49,719 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:17:51,597 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:17:51,598 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:17:51,598 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:17:51,604 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:17:51,720 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:17:52,604 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:17:53,725 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:17:56,674 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:17:56,675 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:17:56,675 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:17:56,681 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:17:56,727 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:17:57,682 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:17:57,730 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:17:58,388 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:17:58,388 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:17:58,389 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:18:00,861 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:18:00,862 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:18:00,862 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:18:00,868 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:18:01,736 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:18:01,737 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:18:02,905 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:18:06,037 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:18:06,038 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:18:06,038 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:18:06,043 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:18:06,742 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:18:07,745 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:18:08,044 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:18:09,482 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 20:18:10,794 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:18:10,795 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:18:10,796 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:18:10,802 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:18:11,751 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:18:11,751 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:18:13,388 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:18:13,388 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:18:13,388 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:18:13,689 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:18:18,689 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:18:18,842 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:18:18,842 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:18:18,843 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:18:18,848 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:18:19,762 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:18:19,763 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:18:23,381 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:18:23,382 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:18:23,382 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:18:23,388 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:18:23,766 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:18:24,388 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:18:25,773 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:18:27,775 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:18:27,776 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:18:27,776 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:18:27,782 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:18:28,388 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:18:28,388 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:18:28,389 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:18:28,775 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:18:29,626 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:18:29,778 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:18:32,986 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:18:32,987 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:18:32,987 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:18:32,993 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:18:33,781 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:18:35,029 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:18:35,785 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:18:38,731 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:18:38,732 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:18:38,732 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:18:38,739 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:18:38,789 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:18:39,485 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 20:18:39,792 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:18:40,486 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:18:43,388 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:18:43,388 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:18:43,389 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:18:43,899 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:18:43,899 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:18:43,900 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:18:43,906 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:18:44,798 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:18:45,801 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:18:45,941 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:18:48,359 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:18:48,360 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:18:48,360 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:18:48,366 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:18:48,803 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:18:49,807 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:18:51,367 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:18:52,742 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:18:52,743 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:18:52,743 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:18:52,749 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:18:52,809 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:18:53,812 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:18:56,749 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:18:57,307 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:18:57,308 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:18:57,308 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:18:57,314 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:18:57,816 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:18:58,388 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:18:58,389 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:18:58,390 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:18:59,821 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:19:02,689 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:19:05,729 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:19:05,733 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:19:05,733 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:19:05,739 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:19:05,828 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:19:07,773 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:19:07,832 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:19:09,487 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 20:19:12,120 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:19:12,121 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:19:12,121 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:19:12,127 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:19:12,839 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:19:13,128 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:19:13,388 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:19:13,388 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:19:13,390 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:19:13,842 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:19:18,243 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:19:18,244 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:19:18,244 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:19:18,244 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:19:18,250 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:19:18,848 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:19:19,851 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:19:22,639 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:19:22,640 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:19:22,640 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:19:22,646 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:19:22,853 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:19:23,646 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:19:23,856 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:19:27,814 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:19:27,815 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:19:27,815 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:19:27,821 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:19:27,861 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:19:28,388 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:19:28,389 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:19:28,390 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:19:28,674 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:19:29,865 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:19:33,675 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:19:35,977 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:19:35,978 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:19:35,978 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:19:35,983 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:19:36,873 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:19:37,876 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:19:39,021 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:19:39,489 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 20:19:41,782 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:19:41,783 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:19:41,783 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:19:41,789 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:19:41,879 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:19:43,388 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:19:43,389 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:19:43,390 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:19:43,885 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:19:44,672 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:19:48,514 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:19:48,515 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:19:48,515 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:19:48,521 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:19:48,891 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:19:49,894 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:19:50,557 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:19:53,761 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:19:53,762 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:19:53,762 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:19:53,768 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:19:53,898 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:19:55,768 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:19:55,903 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:19:58,389 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:19:58,389 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:19:58,390 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:20:01,578 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:20:02,075 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:20:02,076 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:20:02,076 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:20:02,082 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:20:02,911 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:20:03,914 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:20:07,117 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:20:09,495 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 20:20:10,530 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:20:10,531 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:20:10,531 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:20:10,537 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:20:10,923 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:20:11,926 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:20:12,537 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:20:13,389 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:20:13,389 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:20:13,390 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:20:14,624 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:20:14,625 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:20:14,625 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:20:14,631 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:20:14,929 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:20:15,932 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:20:17,665 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:20:18,786 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:20:18,787 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:20:18,787 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:20:18,793 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:20:18,935 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:20:19,940 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:20:22,794 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:20:23,209 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:20:23,210 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:20:23,210 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:20:23,216 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:20:23,943 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:20:25,949 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:20:27,920 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:20:27,921 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:20:27,921 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:20:27,921 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:20:27,927 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:20:27,951 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:20:28,389 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:20:28,389 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:20:28,390 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:20:29,954 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:20:33,608 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:20:34,002 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:20:34,002 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:20:34,002 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:20:34,008 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:20:34,958 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:20:35,965 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:20:38,287 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:20:38,288 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:20:38,288 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:20:38,294 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:20:38,968 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:20:39,294 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:20:39,493 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 20:20:39,971 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:20:43,389 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:20:43,389 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:20:43,390 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:20:44,577 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:20:47,117 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:20:47,118 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:20:47,118 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:20:47,123 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:20:47,976 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:20:49,981 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:20:50,161 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:20:51,432 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:20:51,432 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:20:51,432 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:20:51,438 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:20:51,982 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:20:53,987 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:20:55,439 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:20:55,958 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:20:55,959 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:20:55,959 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:20:55,965 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:20:55,989 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:20:57,993 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:20:58,389 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:20:58,389 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:20:58,390 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:21:00,595 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:21:00,595 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:21:00,596 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:21:00,596 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:21:00,601 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:21:00,996 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:21:02,000 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:21:05,245 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:21:05,246 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:21:05,246 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:21:05,252 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:21:06,003 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:21:06,252 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:21:08,007 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:21:09,499 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 20:21:11,501 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:21:12,589 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:21:12,590 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:21:12,590 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:21:12,595 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:21:13,014 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:21:13,389 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:21:13,389 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:21:13,390 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:21:14,017 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:21:16,639 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:21:17,061 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:21:17,062 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:21:17,062 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:21:17,068 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:21:18,023 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:21:18,024 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:21:22,105 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:21:25,879 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:21:25,880 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:21:25,880 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:21:25,886 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:21:26,034 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:21:27,887 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:21:28,038 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:21:28,389 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:21:28,389 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:21:28,390 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:21:31,440 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:21:31,441 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:21:31,441 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:21:31,447 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:21:32,041 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:21:33,481 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:21:34,046 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:21:35,943 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:21:35,944 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:21:35,944 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:21:35,950 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:21:36,047 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:21:38,049 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:21:38,951 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:21:39,498 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 20:21:40,471 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:21:40,472 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:21:40,472 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:21:40,478 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:21:41,055 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:21:42,058 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:21:43,389 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:21:43,389 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:21:43,391 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:21:44,599 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:21:44,599 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:21:44,599 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:21:44,599 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:21:44,605 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:21:45,062 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:21:46,064 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:21:49,641 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:21:50,952 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:21:50,953 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:21:50,953 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:21:50,959 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:21:51,070 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:21:52,073 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:21:54,960 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:21:58,389 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:21:58,390 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:21:58,390 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:21:59,458 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:21:59,458 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:21:59,458 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:21:59,464 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:22:00,082 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:22:00,501 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:22:02,087 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:22:05,502 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:22:08,345 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:22:08,346 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:22:08,346 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:22:08,352 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:22:09,096 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:22:09,500 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 20:22:10,099 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:22:11,502 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:22:12,862 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:22:12,863 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:22:12,863 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:22:12,869 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:22:13,102 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:22:13,389 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:22:13,390 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:22:13,391 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:22:14,103 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:22:16,620 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:22:18,363 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:22:18,364 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:22:18,364 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:22:18,370 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:22:19,111 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:22:20,114 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:22:22,405 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:22:23,778 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:22:23,778 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:22:23,779 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:22:23,784 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:22:24,117 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:22:26,124 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:22:27,785 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:22:28,389 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:22:28,390 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:22:28,391 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:22:28,466 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:22:28,613 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:22:28,613 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:22:28,619 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:22:29,126 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:22:30,129 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:22:33,620 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:22:33,855 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:22:33,855 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:22:33,856 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:22:33,861 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:22:34,132 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:22:36,137 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:22:38,862 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:22:39,502 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 20:22:40,183 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:22:40,184 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:22:40,184 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:22:40,190 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:22:41,144 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:22:42,147 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:22:43,389 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:22:43,390 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:22:43,390 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:22:44,678 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:22:44,974 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:22:44,975 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:22:44,975 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:22:44,980 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:22:45,149 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:22:46,152 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:22:49,434 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:22:49,435 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:22:49,436 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:22:49,443 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:22:50,156 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:22:50,444 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:22:52,161 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:22:54,081 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:22:54,082 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:22:54,082 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:22:54,087 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:22:54,163 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:22:56,088 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:22:56,167 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:22:58,390 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:22:58,390 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:22:58,391 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:22:58,711 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:22:58,711 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:22:58,711 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:22:58,717 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:22:59,170 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:23:00,173 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:23:01,753 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:23:03,874 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:23:03,875 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:23:03,875 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:23:03,881 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:23:04,177 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:23:06,181 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:23:06,882 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:23:09,504 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 20:23:10,629 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:23:10,630 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:23:10,630 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:23:10,635 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:23:11,188 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:23:12,191 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:23:12,636 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:23:13,390 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:23:13,390 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:23:13,391 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:23:15,270 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:23:15,271 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:23:15,271 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:23:15,276 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:23:16,194 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:23:18,199 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:23:18,313 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:23:23,314 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:23:23,598 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:23:23,599 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:23:23,599 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:23:23,604 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:23:24,206 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:23:26,213 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:23:28,161 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:23:28,162 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:23:28,162 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:23:28,168 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:23:28,215 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:23:28,390 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:23:28,390 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:23:28,391 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:23:28,685 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:23:30,218 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:23:32,381 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:23:32,382 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:23:32,382 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:23:32,387 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:23:33,221 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:23:34,224 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:23:34,425 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:23:36,488 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:23:36,489 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:23:36,489 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:23:36,495 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:23:37,226 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:23:38,229 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:23:39,496 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:23:39,510 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 20:23:41,030 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:23:41,030 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:23:41,031 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:23:41,036 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:23:41,232 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:23:42,235 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:23:43,390 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:23:43,390 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:23:43,390 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:23:44,634 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:23:47,200 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:23:47,201 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:23:47,201 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:23:47,207 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:23:47,241 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:23:48,243 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:23:50,242 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:23:52,175 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:23:52,176 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:23:52,176 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:23:52,182 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:23:52,247 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:23:54,253 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:23:56,183 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:23:58,390 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:23:58,390 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:23:58,392 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:23:58,478 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:23:58,615 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:23:58,615 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:23:58,621 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:23:59,258 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:24:00,261 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:24:01,621 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:24:03,691 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:24:03,692 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:24:03,692 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:24:03,698 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:24:04,265 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:24:06,270 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:24:06,698 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:24:08,298 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:24:08,298 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:24:08,298 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:24:08,305 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:24:09,274 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:24:09,508 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 20:24:10,277 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:24:12,510 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:24:12,587 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:24:12,587 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:24:12,588 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:24:12,593 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:24:13,280 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:24:13,390 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:24:13,390 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:24:13,391 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:24:14,283 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:24:17,622 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:24:18,495 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:24:18,496 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:24:18,496 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:24:18,502 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:24:19,289 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:24:20,292 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:24:23,537 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:24:24,788 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:24:24,789 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:24:24,790 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:24:24,796 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:24:25,299 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:24:26,302 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:24:28,390 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:24:28,390 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:24:28,391 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:24:28,664 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:24:33,665 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:24:34,368 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:24:34,368 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:24:34,368 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:24:34,374 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:24:35,311 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:24:36,314 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:24:38,540 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:24:38,540 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:24:38,541 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:24:38,546 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:24:39,317 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:24:39,514 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 20:24:39,515 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:24:40,320 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:24:43,390 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:24:43,391 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:24:43,391 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:24:44,254 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:24:44,255 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:24:44,255 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:24:44,260 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:24:44,324 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:24:45,297 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:24:46,330 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:24:49,241 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:24:49,242 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:24:49,242 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:24:49,248 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:24:49,332 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:24:50,335 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:24:51,249 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:24:53,758 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:24:53,759 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:24:53,760 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:24:53,765 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:24:54,339 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:24:56,344 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:24:56,766 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:24:58,390 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:24:58,391 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:24:58,391 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:24:58,701 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:24:58,702 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:24:58,702 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:24:58,707 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:24:59,347 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:25:00,350 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:25:02,745 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:25:04,905 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:25:04,906 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:25:04,906 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:25:04,911 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:25:05,356 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:25:06,358 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:25:07,912 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:25:09,513 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 20:25:10,045 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:25:10,046 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:25:10,046 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:25:10,052 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:25:10,363 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:25:12,368 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:25:13,053 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:25:13,390 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:25:13,391 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:25:13,391 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:25:14,814 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:25:14,814 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:25:14,815 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:25:14,820 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:25:15,371 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:25:16,374 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:25:18,857 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:25:19,774 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:25:19,775 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:25:19,775 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:25:19,781 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:25:20,377 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:25:22,382 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:25:24,781 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:25:25,542 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:25:25,543 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:25:25,543 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:25:25,549 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:25:26,388 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:25:28,390 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:25:28,391 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:25:28,392 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:25:28,392 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:25:30,596 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:25:32,472 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:25:32,473 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:25:32,473 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:25:32,479 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:25:33,397 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:25:34,400 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:25:36,513 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:25:36,943 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:25:36,944 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:25:36,944 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:25:36,949 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:25:37,403 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:25:38,406 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:25:39,515 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 20:25:41,517 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:25:42,570 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:25:42,571 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:25:42,571 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:25:42,577 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:25:43,390 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:25:43,391 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:25:43,392 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:25:43,412 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:25:44,415 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:25:46,653 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:25:51,451 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:25:51,452 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:25:51,452 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:25:51,458 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:25:52,425 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:25:52,493 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:25:54,431 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:25:57,494 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:25:58,390 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:25:58,391 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:25:58,392 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:25:59,378 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:25:59,378 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:25:59,378 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:25:59,384 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:25:59,436 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:26:00,439 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:26:03,421 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:26:04,711 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:26:04,712 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:26:04,712 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:26:04,718 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:26:05,444 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:26:06,447 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:26:08,719 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:26:09,517 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 20:26:11,592 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:26:11,593 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:26:11,593 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:26:11,599 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:26:12,454 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:26:13,391 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:26:13,391 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:26:13,392 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:26:14,459 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:26:14,678 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:26:16,685 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:26:16,686 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:26:16,686 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:26:16,692 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:26:17,462 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:26:18,465 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:26:19,729 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:26:21,251 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:26:21,252 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:26:21,252 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:26:21,258 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:26:21,468 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:26:22,471 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:26:25,259 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:26:25,865 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:26:25,866 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:26:25,866 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:26:25,872 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:26:26,476 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:26:28,391 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:26:28,391 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:26:28,392 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:26:28,480 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:26:30,681 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:26:32,630 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:26:32,631 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:26:32,631 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:26:32,636 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:26:33,486 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:26:34,489 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:26:36,673 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:26:37,571 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:26:37,572 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:26:37,572 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:26:37,578 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:26:38,492 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:26:39,520 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 20:26:40,497 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:26:42,170 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:26:42,171 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:26:42,171 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:26:42,171 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:26:42,176 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:26:42,499 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:26:43,391 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:26:43,391 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:26:43,392 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:26:44,504 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:26:47,311 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:26:47,312 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:26:47,312 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:26:47,312 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:26:47,318 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:26:47,507 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:26:48,511 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:26:51,667 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:26:51,668 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:26:51,668 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:26:51,675 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:26:52,514 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:26:52,675 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:26:54,519 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:26:57,676 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:26:57,886 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:26:57,886 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:26:57,886 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:26:57,892 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:26:58,391 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:26:58,391 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:26:58,393 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:26:58,522 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:27:00,526 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:27:03,635 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:27:05,590 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:27:05,591 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:27:05,591 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:27:05,596 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:27:06,534 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:27:08,539 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:27:09,523 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 20:27:09,524 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:27:10,190 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:27:10,190 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:27:10,191 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:27:10,196 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:27:10,541 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:27:12,546 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:27:13,391 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:27:13,391 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:27:13,393 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:27:14,629 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:27:16,114 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:27:16,114 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:27:16,114 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:27:16,121 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:27:16,550 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:27:18,555 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:27:20,157 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:27:20,773 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:27:20,774 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:27:20,774 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:27:20,780 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:27:21,558 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:27:22,561 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:27:25,778 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:27:25,779 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:27:25,779 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:27:25,780 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:27:25,785 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:27:26,566 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:27:28,391 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:27:28,391 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:27:28,393 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:27:28,570 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:27:30,081 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:27:30,082 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:27:30,082 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:27:30,088 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:27:30,572 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:27:31,125 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:27:32,576 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:27:35,814 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:27:35,815 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:27:35,815 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:27:35,821 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:27:36,579 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:27:36,822 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:27:38,583 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:27:39,522 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 20:27:40,547 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:27:40,548 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:27:40,548 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:27:40,554 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:27:40,585 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:27:42,555 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:27:42,590 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:27:43,391 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:27:43,391 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:27:43,393 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:27:45,222 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:27:45,223 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:27:45,223 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:27:45,229 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:27:45,593 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:27:46,595 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:27:48,265 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:27:50,072 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:27:50,072 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:27:50,073 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:27:50,079 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:27:50,599 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:27:52,604 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:27:54,079 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:27:57,806 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:27:57,807 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:27:57,807 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:27:57,812 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:27:58,391 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:27:58,391 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:27:58,393 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:27:58,610 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:27:59,661 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:28:00,615 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:28:02,558 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:28:02,558 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:28:02,559 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:28:02,564 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:28:02,617 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:28:04,622 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:28:05,601 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:28:09,525 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 20:28:09,579 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:28:09,580 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:28:09,580 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:28:09,586 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:28:09,627 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:28:10,631 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:28:11,587 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:28:13,391 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:28:13,392 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:28:13,393 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:28:15,302 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:28:15,302 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:28:15,303 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:28:15,308 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:28:15,637 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:28:16,639 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:28:17,345 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:28:21,060 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:28:21,061 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:28:21,061 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:28:21,067 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:28:21,645 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:28:22,648 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:28:23,068 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:28:28,068 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:28:28,391 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:28:28,392 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:28:28,393 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:28:30,030 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:28:30,031 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:28:30,031 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:28:30,036 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:28:30,657 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:28:32,662 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:28:33,073 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:28:35,096 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:28:35,097 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:28:35,098 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:28:35,103 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:28:35,665 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:28:36,668 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:28:38,104 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:28:39,432 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:28:39,432 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:28:39,433 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:28:39,438 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:28:39,527 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 20:28:39,673 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:28:40,676 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:28:43,391 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:28:43,392 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:28:43,393 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:28:43,611 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:28:44,494 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:28:44,494 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:28:44,495 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:28:44,500 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:28:44,679 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:28:46,683 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:28:49,537 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:28:50,113 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:28:50,114 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:28:50,114 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:28:50,119 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:28:50,687 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:28:52,692 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:28:54,571 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:28:54,572 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:28:54,572 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:28:54,572 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:28:54,578 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:28:54,693 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:28:56,698 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:28:58,392 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:28:58,392 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:28:58,393 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:28:59,575 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:28:59,576 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:28:59,576 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:28:59,576 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:28:59,581 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:28:59,701 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:29:00,704 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:29:03,481 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:29:03,482 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:29:03,482 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:29:03,488 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:29:03,708 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:29:04,711 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:29:05,488 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:29:08,623 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:29:08,624 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:29:08,624 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:29:08,630 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:29:08,714 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:29:09,530 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 20:29:10,531 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:29:10,720 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:29:13,141 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:29:13,142 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:29:13,142 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:29:13,147 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:29:13,392 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:29:13,392 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:29:13,393 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:29:13,723 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:29:14,726 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:29:15,759 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:29:17,804 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:29:17,804 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:29:17,805 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:29:17,810 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:29:18,729 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:29:20,735 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:29:20,845 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:29:24,619 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:29:24,620 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:29:24,621 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:29:24,627 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:29:24,738 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:29:26,628 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:29:26,745 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:29:28,392 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:29:28,392 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:29:28,393 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:29:30,067 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:29:30,067 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:29:30,067 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:29:30,074 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:29:30,748 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:29:32,109 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:29:32,752 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:29:34,962 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:29:34,963 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:29:34,963 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:29:34,969 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:29:35,755 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:29:36,758 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:29:37,969 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:29:39,535 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 20:29:40,341 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:29:40,341 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:29:40,342 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:29:40,347 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:29:40,762 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:29:42,767 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:29:43,348 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:29:43,392 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:29:43,392 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:29:43,393 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:29:46,693 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:29:46,694 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:29:46,694 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:29:46,700 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:29:46,770 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:29:48,737 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:29:48,775 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:29:51,985 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:29:51,986 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:29:51,986 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:29:51,992 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:29:52,779 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:29:53,993 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:29:54,784 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:29:58,392 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:29:58,392 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:29:58,394 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:29:58,648 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:29:58,680 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:29:58,680 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:29:58,686 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:29:58,787 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:29:59,687 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:30:00,793 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:30:04,687 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:30:07,125 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:30:07,126 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:30:07,126 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:30:07,131 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:30:07,801 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:30:08,804 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:30:09,534 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 20:30:10,535 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:30:11,655 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:30:11,657 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:30:11,657 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:30:11,662 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:30:11,808 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:30:12,811 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:30:13,392 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:30:13,392 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:30:13,394 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:30:15,687 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:30:15,977 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:30:15,977 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:30:15,978 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:30:15,983 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:30:16,814 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:30:18,820 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:30:21,022 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:30:23,553 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:30:23,554 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:30:23,554 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:30:23,559 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:30:23,826 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:30:24,828 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:30:26,560 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:30:28,225 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:30:28,226 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:30:28,226 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:30:28,231 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:30:28,392 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:30:28,392 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:30:28,393 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:30:28,833 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:30:30,835 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:30:31,670 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:30:32,893 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:30:32,894 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:30:32,894 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:30:32,899 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:30:33,841 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:30:34,843 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:30:36,937 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:30:37,628 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:30:37,628 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:30:37,629 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:30:37,634 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:30:37,846 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:30:38,849 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:30:39,539 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 20:30:41,755 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:30:41,756 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:30:41,756 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:30:41,762 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:30:41,853 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:30:42,762 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:30:42,856 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:30:43,392 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:30:43,392 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:30:43,394 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:30:46,552 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:30:46,553 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:30:46,553 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:30:46,559 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:30:46,859 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:30:48,593 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:30:48,865 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:30:50,914 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:30:50,915 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:30:50,915 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:30:50,921 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:30:51,868 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:30:52,871 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:30:53,921 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:30:56,899 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:30:56,900 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:30:56,900 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:30:56,905 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:30:57,876 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:30:58,392 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:30:58,392 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:30:58,394 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:30:58,879 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:30:59,599 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:31:03,418 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:31:03,419 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:31:03,419 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:31:03,425 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:31:03,885 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:31:04,888 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:31:05,461 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:31:08,389 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:31:08,390 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:31:08,390 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:31:08,396 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:31:08,892 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:31:09,538 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 20:31:10,540 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:31:10,897 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:31:13,233 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:31:13,234 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:31:13,234 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:31:13,240 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:31:13,392 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:31:13,392 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:31:13,394 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:31:13,901 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:31:14,903 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:31:15,653 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:31:19,329 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:31:19,330 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:31:19,330 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:31:19,337 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:31:19,909 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:31:20,912 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:31:21,373 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:31:23,756 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:31:23,757 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:31:23,757 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:31:23,762 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:31:23,916 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:31:24,918 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:31:26,763 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:31:28,392 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:31:28,392 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:31:28,394 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:31:28,794 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:31:28,795 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:31:28,795 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:31:28,801 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:31:28,923 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:31:30,927 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:31:31,837 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:31:32,826 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:31:32,826 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:31:32,826 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:31:32,832 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:31:32,929 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:31:34,933 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:31:37,833 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:31:39,541 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 20:31:41,317 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:31:41,321 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:31:41,321 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:31:41,327 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:31:41,942 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:31:42,945 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:31:43,328 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:31:43,392 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:31:43,392 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:31:43,394 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:31:46,018 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:31:46,018 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:31:46,018 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:31:46,024 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:31:46,949 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:31:48,954 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:31:49,061 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:31:50,279 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:31:50,280 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:31:50,280 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:31:50,286 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:31:50,956 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:31:52,960 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:31:54,287 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:31:55,375 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:31:55,376 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:31:55,376 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:31:55,381 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:31:55,963 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:31:56,966 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:31:58,392 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:31:58,393 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:31:58,394 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:31:59,594 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:31:59,953 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:31:59,953 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:31:59,954 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:31:59,959 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:31:59,969 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:32:02,975 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:32:04,260 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:32:04,261 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:32:04,261 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:32:04,267 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:32:04,977 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:32:05,268 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:32:06,981 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:32:09,543 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 20:32:10,284 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:32:10,285 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:32:10,285 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:32:10,285 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:32:10,291 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:32:10,985 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:32:12,988 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:32:13,392 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:32:13,393 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:32:13,394 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:32:15,354 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:32:15,354 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:32:15,355 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:32:15,355 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:32:15,361 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:32:15,993 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:32:16,997 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:32:20,398 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:32:22,507 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:32:22,508 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:32:22,508 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:32:22,514 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:32:23,003 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:32:25,008 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:32:25,514 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:32:27,024 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:32:27,025 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:32:27,025 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:32:27,031 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:32:28,012 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:32:28,393 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:32:28,393 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:32:28,394 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:32:29,015 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:32:30,686 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:32:32,969 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:32:32,970 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:32:32,970 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:32:32,976 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:32:33,018 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:32:35,023 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:32:36,013 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:32:37,510 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:32:37,511 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:32:37,511 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:32:37,517 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:32:38,026 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:32:39,028 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:32:39,545 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 20:32:41,547 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:32:41,925 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:32:41,926 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:32:41,926 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:32:41,932 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:32:42,032 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:32:43,035 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:32:43,393 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:32:43,393 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:32:43,394 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:32:46,318 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:32:46,319 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:32:46,319 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:32:46,325 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:32:47,039 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:32:47,361 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:32:49,042 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:32:51,641 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:32:51,641 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:32:51,642 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:32:51,647 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:32:52,047 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:32:52,648 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:32:53,050 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:32:57,648 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:32:58,059 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:32:58,063 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:32:58,063 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:32:58,068 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:32:58,393 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:32:58,393 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:32:58,395 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:32:59,056 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:33:01,062 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:33:03,063 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:33:03,063 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:33:03,064 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:33:03,064 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:33:03,069 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:33:04,065 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:33:05,068 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:33:08,106 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:33:09,547 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 20:33:10,038 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:33:10,039 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:33:10,039 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:33:10,045 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:33:10,074 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:33:13,079 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:33:13,393 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:33:13,393 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:33:13,395 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:33:13,701 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:33:15,849 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:33:15,850 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:33:15,850 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:33:15,856 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:33:16,082 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:33:17,085 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:33:18,893 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:33:21,963 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:33:21,964 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:33:21,965 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:33:21,970 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:33:22,091 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:33:23,094 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:33:23,971 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:33:26,669 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:33:26,670 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:33:26,670 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:33:26,676 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:33:27,099 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:33:28,393 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:33:28,393 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:33:28,395 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:33:29,104 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:33:29,678 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:33:31,095 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:33:31,096 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:33:31,096 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:33:31,102 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:33:31,105 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:33:33,109 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:33:35,137 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:33:35,317 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:33:35,317 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:33:35,318 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:33:35,323 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:33:36,112 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:33:37,115 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:33:39,550 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 20:33:40,551 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:33:43,393 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:33:43,393 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:33:43,395 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:33:43,592 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:33:43,703 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:33:43,703 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:33:43,709 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:33:44,124 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:33:45,127 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:33:45,709 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:33:47,882 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:33:47,883 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:33:47,883 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:33:47,888 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:33:48,130 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:33:49,133 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:33:50,889 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:33:52,808 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:33:52,809 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:33:52,809 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:33:52,815 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:33:53,136 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:33:55,142 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:33:56,815 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:33:58,393 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:33:58,393 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:33:58,395 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:33:59,198 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:33:59,198 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:33:59,198 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:33:59,204 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:34:00,148 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:34:01,150 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:34:02,241 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:34:05,285 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:34:05,286 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:34:05,287 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:34:05,292 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:34:06,156 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:34:07,159 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:34:07,293 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:34:09,555 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 20:34:12,155 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:34:12,156 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:34:12,156 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:34:12,161 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:34:12,165 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:34:13,162 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:34:13,393 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:34:13,393 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:34:13,395 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:34:15,171 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:34:17,730 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:34:17,731 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:34:17,731 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:34:17,737 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:34:18,174 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:34:18,773 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:34:19,177 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:34:23,774 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:34:25,726 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:34:25,727 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:34:25,727 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:34:25,733 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:34:26,187 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:34:27,190 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:34:28,393 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:34:28,393 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:34:28,395 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:34:29,682 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:34:30,408 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:34:30,409 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:34:30,409 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:34:30,416 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:34:31,194 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:34:33,198 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:34:35,247 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:34:35,248 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:34:35,248 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:34:35,248 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:34:35,254 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:34:36,201 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:34:37,204 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:34:39,554 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 20:34:40,107 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:34:40,108 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:34:40,108 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:34:40,113 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:34:40,207 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:34:41,114 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:34:43,213 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:34:43,393 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:34:43,394 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:34:43,395 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:34:46,673 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:34:48,575 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:34:48,576 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:34:48,576 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:34:48,582 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:34:49,219 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:34:51,224 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:34:52,617 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:34:52,782 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:34:52,783 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:34:52,783 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:34:52,788 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:34:53,225 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:34:55,230 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:34:57,224 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:34:57,225 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:34:57,225 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:34:57,231 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:34:57,232 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:34:58,233 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:34:58,393 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:34:58,394 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:34:58,395 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:34:59,236 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:35:02,937 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:35:02,937 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:35:02,938 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:35:02,943 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:35:03,241 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:35:03,981 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:35:05,245 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:35:07,893 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:35:07,894 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:35:07,894 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:35:07,899 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:35:08,248 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:35:09,251 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:35:09,559 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 20:35:09,561 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:35:12,458 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:35:12,459 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:35:12,459 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:35:12,465 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:35:13,255 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:35:13,393 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:35:13,394 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:35:13,395 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:35:14,687 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:35:15,261 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:35:17,980 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:35:17,981 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:35:17,981 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:35:17,986 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:35:18,263 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:35:19,266 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:35:20,021 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:35:22,503 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:35:22,504 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:35:22,505 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:35:22,510 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:35:23,270 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:35:25,275 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:35:25,511 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:35:27,065 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:35:27,066 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:35:27,066 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:35:27,072 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:35:27,278 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:35:28,393 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:35:28,394 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:35:28,395 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:35:29,282 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:35:30,655 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:35:31,458 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:35:31,459 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:35:31,459 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:35:31,465 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:35:32,284 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:35:33,287 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:35:35,950 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:35:35,951 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:35:35,951 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:35:35,951 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:35:35,956 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:35:36,290 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:35:37,293 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:35:39,558 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 20:35:40,441 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:35:40,442 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:35:40,442 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:35:40,448 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:35:41,297 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:35:41,449 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:35:43,300 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:35:43,394 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:35:43,394 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:35:43,395 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:35:46,607 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:35:48,590 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:35:48,591 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:35:48,591 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:35:48,597 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:35:49,308 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:35:51,314 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:35:51,633 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:35:52,916 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:35:52,917 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:35:52,918 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:35:52,923 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:35:53,315 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:35:55,319 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:35:56,924 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:35:57,969 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:35:57,970 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:35:57,970 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:35:57,975 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:35:58,323 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:35:58,394 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:35:58,394 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:35:58,395 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:35:59,326 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:36:02,717 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:36:02,718 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:36:02,718 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:36:02,718 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:36:02,723 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:36:03,330 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:36:05,334 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:36:07,202 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:36:07,202 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:36:07,203 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:36:07,208 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:36:07,336 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:36:08,208 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:36:09,340 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:36:09,564 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 20:36:12,209 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:36:12,209 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:36:12,209 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:36:12,215 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:36:12,344 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:36:13,216 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:36:13,394 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:36:13,394 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:36:13,395 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:36:15,350 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:36:16,944 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:36:16,945 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:36:16,945 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:36:16,951 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:36:17,359 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:36:18,985 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:36:19,363 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:36:22,708 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:36:22,709 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:36:22,709 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:36:22,716 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:36:23,366 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:36:24,716 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:36:25,370 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:36:28,394 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:36:28,394 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:36:28,396 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:36:30,695 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:36:31,043 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:36:31,044 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:36:31,044 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:36:31,050 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:36:31,375 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:36:33,380 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:36:36,085 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:36:37,795 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:36:37,796 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:36:37,796 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:36:37,801 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:36:38,383 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:36:39,387 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:36:39,562 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 20:36:41,564 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:36:42,623 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:36:42,624 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:36:42,624 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:36:42,630 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:36:43,390 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:36:43,394 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:36:43,394 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:36:43,396 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:36:45,394 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:36:46,614 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:36:47,032 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:36:47,033 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:36:47,033 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:36:47,039 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:36:47,396 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:36:49,399 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:36:52,074 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:36:54,210 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:36:54,211 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:36:54,211 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:36:54,216 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:36:54,405 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:36:55,408 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:36:57,217 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:36:58,394 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:36:58,394 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:36:58,396 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:36:59,498 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:36:59,499 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:36:59,499 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:36:59,504 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:37:00,413 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:37:01,416 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:37:02,541 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:37:04,260 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:37:04,261 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:37:04,261 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:37:04,266 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:37:04,419 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:37:07,425 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:37:08,267 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:37:09,564 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 20:37:12,608 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:37:12,609 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:37:12,609 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:37:12,615 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:37:13,394 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:37:13,394 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:37:13,396 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:37:13,432 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:37:13,696 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:37:15,437 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:37:16,677 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:37:16,677 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:37:16,677 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:37:16,683 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:37:17,438 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:37:18,721 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:37:19,443 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:37:22,607 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:37:22,607 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:37:22,608 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:37:22,613 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:37:23,446 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:37:24,614 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:37:25,452 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:37:28,394 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:37:28,394 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:37:28,396 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:37:29,640 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:37:30,922 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:37:30,923 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:37:30,923 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:37:30,929 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:37:31,458 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:37:33,463 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:37:34,965 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:37:36,700 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:37:36,701 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:37:36,701 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:37:36,707 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:37:37,466 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:37:39,471 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:37:39,570 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 20:37:40,572 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:37:41,169 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:37:41,170 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:37:41,170 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:37:41,176 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:37:41,473 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:37:43,394 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:37:43,394 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:37:43,396 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:37:43,477 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:37:45,597 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:37:46,289 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:37:46,290 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:37:46,290 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:37:46,296 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:37:46,481 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:37:49,487 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:37:51,334 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:37:53,953 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:37:53,953 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:37:53,954 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:37:53,960 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:37:54,492 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:37:55,495 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:37:56,960 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:37:58,394 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:37:58,394 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:37:58,396 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:38:02,239 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:38:02,240 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:38:02,240 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:38:02,240 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:38:02,246 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:38:02,504 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:38:03,507 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:38:07,281 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:38:07,997 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:38:07,998 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:38:07,998 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:38:08,004 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:38:08,512 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:38:09,515 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:38:09,569 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 20:38:12,570 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:38:13,156 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:38:13,157 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:38:13,157 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:38:13,162 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:38:13,394 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:38:13,394 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:38:13,396 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:38:13,519 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:38:15,524 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:38:17,606 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:38:20,767 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:38:20,767 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:38:20,768 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:38:20,774 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:38:21,530 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:38:22,809 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:38:23,536 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:38:24,928 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:38:24,929 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:38:24,929 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:38:24,934 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:38:25,537 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:38:27,543 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:38:27,935 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:38:28,394 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:38:28,395 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:38:28,396 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:38:29,023 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:38:29,024 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:38:29,024 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:38:29,030 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:38:29,545 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:38:31,548 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:38:33,065 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:38:35,187 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:38:35,188 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:38:35,188 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:38:35,194 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:38:35,551 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:38:37,556 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:38:38,194 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:38:39,574 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 20:38:41,707 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:38:41,708 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:38:41,708 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:38:41,713 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:38:42,562 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:38:43,394 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:38:43,395 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:38:43,396 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:38:43,566 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:38:43,593 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:38:48,456 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:38:48,457 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:38:48,457 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:38:48,463 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:38:48,571 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:38:49,497 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:38:51,577 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:38:53,502 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:38:53,502 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:38:53,503 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:38:53,508 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:38:53,579 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:38:54,509 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:38:55,583 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:38:58,395 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:38:58,395 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:38:58,396 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:38:59,692 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:39:01,917 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:39:01,918 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:39:01,918 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:39:01,923 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:39:02,591 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:39:03,594 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:39:04,961 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:39:09,573 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 20:39:10,575 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:39:10,608 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:39:10,609 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:39:10,609 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:39:10,615 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:39:11,603 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:39:13,395 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:39:13,395 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:39:13,396 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:39:13,608 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:39:15,605 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:39:16,329 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:39:16,329 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:39:16,330 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:39:16,335 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:39:16,612 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:39:17,615 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:39:21,214 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:39:21,215 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:39:21,215 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:39:21,215 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:39:21,220 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:39:21,618 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:39:23,621 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:39:26,221 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:39:26,778 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:39:26,779 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:39:26,779 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:39:26,785 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:39:27,629 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:39:28,395 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:39:28,395 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:39:28,397 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:39:29,634 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:39:31,604 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:39:31,788 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:39:31,788 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:39:31,788 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:39:31,794 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:39:32,636 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:39:33,639 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:39:36,829 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:39:38,081 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:39:38,082 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:39:38,082 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:39:38,087 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:39:38,643 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:39:39,575 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 20:39:39,646 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:39:42,307 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:39:42,311 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:39:42,311 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:39:42,311 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:39:42,317 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:39:42,650 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:39:43,395 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:39:43,395 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:39:43,396 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:39:43,653 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:39:47,608 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:39:50,837 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:39:50,838 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:39:50,838 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:39:50,844 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:39:51,661 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:39:52,881 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:39:53,666 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:39:56,784 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:39:56,786 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:39:56,786 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:39:56,791 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:39:57,670 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:39:58,395 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:39:58,395 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:39:58,397 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:39:58,687 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:39:59,675 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:40:01,660 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:40:01,660 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:40:01,660 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:40:01,666 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:40:01,676 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:40:03,681 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:40:03,701 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:40:05,745 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:40:05,746 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:40:05,746 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:40:05,751 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:40:06,684 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:40:07,687 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:40:08,752 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:40:09,578 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 20:40:10,029 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:40:10,030 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:40:10,030 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:40:10,036 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:40:10,691 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:40:11,693 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:40:13,395 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:40:13,395 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:40:13,397 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:40:14,697 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:40:16,844 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:40:16,844 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:40:16,845 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:40:16,850 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:40:17,700 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:40:19,705 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:40:19,885 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:40:23,593 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:40:23,594 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:40:23,594 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:40:23,600 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:40:23,708 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:40:25,600 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:40:25,713 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:40:28,355 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:40:28,356 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:40:28,357 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:40:28,362 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:40:28,395 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:40:28,395 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:40:28,397 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:40:28,718 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:40:29,721 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:40:30,602 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:40:35,455 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:40:35,456 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:40:35,456 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:40:35,462 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:40:35,725 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:40:36,497 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:40:37,731 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:40:39,580 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 20:40:41,153 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:40:41,154 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:40:41,154 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:40:41,160 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:40:41,736 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:40:42,160 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:40:43,395 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:40:43,395 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:40:43,397 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:40:43,740 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:40:47,698 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:40:48,322 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:40:48,322 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:40:48,323 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:40:48,328 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:40:48,746 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:40:49,749 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:40:53,366 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:40:53,591 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:40:53,592 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:40:53,592 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:40:53,598 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:40:53,752 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:40:55,757 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:40:57,831 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:40:57,832 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:40:57,832 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:40:57,838 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:40:58,395 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:40:58,396 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:40:58,397 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:40:58,691 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:40:58,761 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:40:59,763 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:41:03,692 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:41:06,310 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:41:06,311 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:41:06,311 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:41:06,317 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:41:06,772 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:41:07,775 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:41:09,354 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:41:09,585 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 20:41:12,911 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:41:12,912 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:41:12,912 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:41:12,918 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:41:13,395 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:41:13,396 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:41:13,397 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:41:13,782 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:41:14,677 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:41:15,785 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:41:18,451 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:41:18,452 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:41:18,452 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:41:18,457 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:41:18,790 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:41:19,793 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:41:20,493 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:41:23,722 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:41:23,722 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:41:23,723 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:41:23,728 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:41:23,797 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:41:25,729 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:41:25,803 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:41:28,395 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:41:28,396 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:41:28,397 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:41:28,582 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:41:28,687 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:41:28,687 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:41:28,693 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:41:28,807 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:41:31,693 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:41:31,811 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:41:33,640 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:41:33,641 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:41:33,642 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:41:33,647 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:41:33,814 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:41:35,817 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:41:37,648 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:41:38,012 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:41:38,013 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:41:38,013 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:41:38,019 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:41:38,819 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:41:39,584 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 20:41:39,822 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:41:43,395 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:41:43,396 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:41:43,397 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:41:43,612 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:41:45,275 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:41:45,275 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:41:45,276 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:41:45,281 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:41:45,828 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:41:47,833 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:41:49,318 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:41:50,781 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:41:50,782 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:41:50,782 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:41:50,788 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:41:50,836 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:41:53,842 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:41:54,789 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:41:55,567 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:41:55,568 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:41:55,568 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:41:55,574 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:41:55,843 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:41:57,849 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:41:58,395 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:41:58,396 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:41:58,397 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:42:00,269 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:42:00,269 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:42:00,270 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:42:00,270 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:42:00,275 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:42:00,851 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:42:01,854 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:42:05,314 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:42:05,770 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:42:05,770 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:42:05,771 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:42:05,776 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:42:05,857 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:42:07,863 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:42:09,590 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 20:42:10,592 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:42:10,678 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:42:10,678 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:42:10,679 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:42:10,685 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:42:10,866 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:42:13,396 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:42:13,396 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:42:13,398 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:42:13,872 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:42:15,652 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:42:16,380 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:42:16,381 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:42:16,381 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:42:16,387 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:42:16,874 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:42:17,877 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:42:21,422 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:42:22,079 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:42:22,080 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:42:22,080 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:42:22,086 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:42:22,883 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:42:23,886 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:42:27,087 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:42:27,163 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:42:27,164 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:42:27,164 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:42:27,170 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:42:27,891 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:42:28,396 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:42:28,396 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:42:28,398 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:42:29,895 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:42:31,900 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:42:31,901 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:42:31,901 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:42:31,907 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:42:32,898 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:42:32,941 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:42:33,901 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:42:37,942 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:42:38,648 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:42:38,649 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:42:38,649 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:42:38,655 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:42:38,907 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:42:39,589 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 20:42:41,913 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:42:43,396 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:42:43,396 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:42:43,398 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:42:43,698 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:42:43,726 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:42:43,727 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:42:43,727 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:42:43,733 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:42:43,914 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:42:45,919 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:42:48,403 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:42:48,403 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:42:48,404 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:42:48,409 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:42:48,922 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:42:49,410 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:42:49,925 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:42:52,819 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:42:52,821 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:42:52,821 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:42:52,826 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:42:52,928 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:42:54,827 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:42:55,934 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:42:58,396 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:42:58,396 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:42:58,398 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:42:58,544 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:42:58,608 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:42:58,608 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:42:58,613 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:42:58,937 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:42:59,940 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:43:00,614 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:43:03,306 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:43:03,307 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:43:03,307 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:43:03,313 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:43:03,944 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:43:05,948 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:43:06,349 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:43:08,396 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:43:08,397 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:43:08,397 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:43:08,403 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:43:08,952 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:43:09,597 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 20:43:09,955 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:43:11,598 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:43:13,396 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:43:13,396 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:43:13,398 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:43:13,648 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:43:13,648 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:43:13,649 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:43:13,655 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:43:13,959 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:43:14,964 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:43:16,689 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:43:19,129 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:43:19,130 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:43:19,130 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:43:19,136 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:43:19,969 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:43:20,973 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:43:22,136 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:43:25,055 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:43:25,056 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:43:25,056 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:43:25,062 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:43:25,978 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:43:26,984 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:43:28,062 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:43:28,396 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:43:28,396 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:43:28,398 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:43:29,967 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:43:29,968 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:43:29,968 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:43:29,974 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:43:29,987 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:43:30,990 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:43:34,009 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:43:36,122 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:43:36,124 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:43:36,124 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:43:36,130 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:43:36,997 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:43:36,997 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:43:39,130 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:43:39,593 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 20:43:40,604 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:43:40,605 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:43:40,605 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:43:40,611 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:43:41,003 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:43:41,004 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:43:43,396 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:43:43,397 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:43:43,398 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:43:44,633 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:43:45,160 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:43:45,161 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:43:45,161 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:43:45,166 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:43:46,008 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:43:47,012 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:43:49,670 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:43:49,671 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:43:49,671 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:43:49,671 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:43:49,677 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:43:50,015 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:43:51,018 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:43:54,678 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:43:56,898 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:43:56,899 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:43:56,899 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:43:56,905 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:43:57,043 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:43:58,396 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:43:58,397 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:43:58,398 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:43:59,048 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:44:00,601 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:44:02,365 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:44:02,366 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:44:02,366 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:44:02,372 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:44:03,055 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:44:03,056 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:44:06,410 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:44:06,642 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:44:06,643 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:44:06,643 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:44:06,649 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:44:07,061 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:44:07,061 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:44:09,595 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 20:44:11,059 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:44:11,060 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:44:11,060 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:44:11,066 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:44:11,066 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:44:12,066 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:44:13,070 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:44:13,396 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:44:13,397 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:44:13,398 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:44:16,997 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:44:16,997 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:44:16,998 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:44:17,004 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:44:17,073 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:44:18,041 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:44:19,078 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:44:22,242 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:44:22,242 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:44:22,242 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:44:22,249 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:44:23,084 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:44:23,084 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:44:23,249 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:44:28,051 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:44:28,052 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:44:28,052 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:44:28,058 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:44:28,090 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:44:28,396 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:44:28,397 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:44:28,398 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:44:28,628 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:44:29,093 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:44:33,629 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:44:34,127 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:44:34,127 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:44:34,128 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:44:34,133 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:44:35,101 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:44:35,101 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:44:38,806 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:44:38,807 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:44:38,807 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:44:38,807 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:44:38,813 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:44:39,105 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:44:39,597 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 20:44:41,110 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:44:43,396 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:44:43,397 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:44:43,398 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:44:43,647 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:44:43,647 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:44:43,647 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:44:43,653 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:44:44,112 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:44:44,689 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:44:45,117 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:44:48,160 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:44:48,161 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:44:48,161 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:44:48,167 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:44:49,123 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:44:49,123 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:44:50,168 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:44:52,095 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:44:52,096 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:44:52,097 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:44:52,103 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:44:52,125 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:44:53,129 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:44:56,104 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:44:56,522 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:44:56,523 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:44:56,523 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:44:56,529 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:44:57,134 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:44:57,134 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:44:58,397 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:44:58,397 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:44:58,398 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:45:01,012 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:45:01,012 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:45:01,013 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:45:01,018 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:45:01,138 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:45:02,053 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:45:03,143 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:45:05,623 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:45:05,624 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:45:05,624 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:45:05,629 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:45:06,145 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:45:07,149 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:45:07,630 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:45:09,599 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 20:45:10,845 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:45:10,845 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:45:10,846 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:45:10,852 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:45:11,153 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:45:12,852 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:45:13,158 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:45:13,397 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:45:13,397 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:45:13,398 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:45:15,681 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:45:15,681 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:45:15,682 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:45:15,688 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:45:16,161 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:45:17,164 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:45:18,725 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:45:21,122 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:45:21,122 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:45:21,123 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:45:21,128 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:45:21,168 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:45:23,173 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:45:24,129 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:45:27,435 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:45:27,436 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:45:27,436 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:45:27,442 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:45:28,179 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:45:28,397 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:45:28,397 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:45:28,399 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:45:29,182 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:45:29,624 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:45:31,909 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:45:31,910 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:45:31,910 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:45:31,916 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:45:32,184 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:45:33,188 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:45:34,953 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:45:36,203 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:45:36,204 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:45:36,204 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:45:36,210 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:45:37,194 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:45:37,194 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:45:39,602 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 20:45:40,603 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:45:40,861 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:45:40,862 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:45:40,862 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:45:40,867 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:45:41,198 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:45:43,202 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:45:43,397 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:45:43,397 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:45:43,399 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:45:45,427 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:45:45,428 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:45:45,428 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:45:45,433 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:45:46,204 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:45:46,469 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:45:47,208 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:45:49,571 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:45:49,572 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:45:49,572 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:45:49,577 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:45:50,211 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:45:51,214 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:45:51,578 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:45:55,196 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:45:55,197 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:45:55,197 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:45:55,203 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:45:55,218 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:45:57,203 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:45:57,223 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:45:58,397 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:45:58,397 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:45:58,399 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:46:02,646 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:46:03,444 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:46:03,444 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:46:03,444 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:46:03,450 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:46:04,230 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:46:05,234 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:46:08,434 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:46:08,435 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:46:08,435 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:46:08,435 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:46:08,441 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:46:09,241 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:46:09,241 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:46:09,604 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 20:46:12,575 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:46:12,576 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:46:12,576 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:46:12,583 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:46:13,247 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:46:13,248 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:46:13,397 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:46:13,397 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:46:13,399 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:46:13,614 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:46:18,615 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:46:20,966 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:46:20,967 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:46:20,967 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:46:20,973 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:46:21,257 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:46:23,262 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:46:24,009 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:46:26,769 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:46:26,770 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:46:26,770 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:46:26,776 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:46:27,270 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:46:27,270 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:46:28,397 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:46:28,398 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:46:28,399 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:46:29,613 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:46:31,379 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:46:31,380 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:46:31,380 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:46:31,386 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:46:32,274 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:46:33,278 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:46:35,421 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:46:36,334 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:46:36,335 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:46:36,335 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:46:36,341 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:46:37,286 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:46:37,286 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:46:39,609 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 20:46:40,611 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:46:41,517 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:46:41,518 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:46:41,518 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:46:41,524 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:46:42,291 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:46:43,294 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:46:43,397 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:46:43,398 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:46:43,399 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:46:45,643 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:46:45,834 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:46:45,835 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:46:45,835 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:46:45,842 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:46:46,296 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:46:47,299 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:46:50,877 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:46:53,966 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:46:53,966 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:46:53,967 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:46:53,972 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:46:54,304 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:46:55,309 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:46:55,973 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:46:58,397 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:46:58,398 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:46:58,399 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:46:59,170 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:46:59,170 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:46:59,171 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:46:59,176 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:46:59,313 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:47:01,213 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:47:01,318 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:47:04,620 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:47:04,621 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:47:04,621 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:47:04,626 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:47:05,324 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:47:05,324 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:47:06,627 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:47:08,779 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:47:08,780 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:47:08,780 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:47:08,786 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:47:09,330 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:47:09,330 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:47:09,608 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 20:47:12,610 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:47:13,397 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:47:13,398 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:47:13,399 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:47:13,907 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:47:13,908 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:47:13,908 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:47:13,914 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:47:14,334 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:47:15,339 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:47:17,949 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:47:18,591 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:47:18,592 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:47:18,593 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:47:18,599 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:47:19,345 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:47:19,345 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:47:22,928 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:47:22,929 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:47:22,929 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:47:22,935 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:47:23,348 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:47:23,936 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:47:25,353 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:47:27,614 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:47:27,615 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:47:27,615 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:47:27,621 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:47:28,358 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:47:28,397 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:47:28,398 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:47:28,399 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:47:29,361 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:47:29,602 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:47:33,038 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:47:33,038 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:47:33,039 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:47:33,044 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:47:33,365 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:47:35,081 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:47:35,369 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:47:37,348 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:47:37,349 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:47:37,349 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:47:37,355 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:47:37,370 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:47:39,374 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:47:39,613 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 20:47:40,615 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:47:41,556 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:47:41,557 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:47:41,557 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:47:41,563 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:47:42,377 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:47:43,381 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:47:43,397 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:47:43,398 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:47:43,399 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:47:45,684 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:47:46,054 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:47:46,055 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:47:46,055 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:47:46,062 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:47:46,383 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:47:47,386 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:47:50,703 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:47:50,704 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:47:50,704 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:47:50,704 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:47:50,710 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:47:51,392 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:47:51,392 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:47:55,711 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:47:57,105 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:47:57,106 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:47:57,106 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:47:57,112 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:47:57,399 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:47:58,398 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:47:58,398 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:47:58,400 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:47:59,404 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:48:01,663 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:48:02,667 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:48:02,667 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:48:02,667 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:48:02,673 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:48:03,410 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:48:03,410 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:48:06,709 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:48:07,677 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:48:07,678 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:48:07,678 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:48:07,684 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:48:08,414 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:48:09,418 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:48:09,612 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 20:48:12,614 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:48:12,641 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:48:12,642 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:48:12,642 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:48:12,648 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:48:13,398 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:48:13,398 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:48:13,399 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:48:13,425 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:48:13,425 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:48:17,502 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:48:17,503 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:48:17,503 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:48:17,510 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:48:18,429 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:48:18,545 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:48:19,434 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:48:22,069 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:48:22,070 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:48:22,070 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:48:22,076 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:48:22,436 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:48:23,440 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:48:24,077 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:48:28,398 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:48:28,398 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:48:28,400 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:48:29,652 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:48:30,486 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:48:30,487 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:48:30,487 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:48:30,493 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:48:31,451 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:48:31,451 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:48:35,529 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:48:35,905 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:48:35,906 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:48:35,906 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:48:35,912 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:48:36,455 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:48:37,460 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:48:39,618 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 20:48:40,620 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:48:40,813 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:48:40,813 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:48:40,814 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:48:40,819 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:48:41,466 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:48:41,466 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:48:43,398 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:48:43,398 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:48:43,400 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:48:45,691 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:48:47,365 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:48:47,365 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:48:47,365 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:48:47,372 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:48:47,472 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:48:49,478 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:48:51,410 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:48:53,174 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:48:53,176 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:48:53,176 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:48:53,182 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:48:53,481 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:48:55,486 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:48:57,183 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:48:58,398 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:48:58,398 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:48:58,400 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:49:01,265 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:49:01,266 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:49:01,266 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:49:01,272 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:49:01,492 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:49:02,309 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:49:03,498 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:49:06,493 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:49:06,494 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:49:06,494 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:49:06,500 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:49:06,501 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:49:07,501 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:49:07,504 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:49:09,616 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 20:49:12,618 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:49:12,702 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:49:12,702 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:49:12,702 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:49:12,708 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:49:13,398 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:49:13,398 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:49:13,400 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:49:13,513 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:49:13,514 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:49:17,661 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:49:19,683 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:49:19,684 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:49:19,684 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:49:19,690 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:49:20,519 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:49:21,522 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:49:22,725 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:49:24,410 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:49:24,411 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:49:24,411 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:49:24,416 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:49:24,525 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:49:25,530 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:49:28,398 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:49:28,398 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:49:28,400 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:49:28,641 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:49:31,214 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:49:31,214 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:49:31,214 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:49:31,220 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:49:31,537 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:49:33,542 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:49:34,257 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:49:36,341 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:49:36,342 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:49:36,342 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:49:36,348 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:49:36,544 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:49:37,547 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:49:39,349 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:49:39,619 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 20:49:41,890 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:49:41,891 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:49:41,891 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:49:41,897 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:49:42,552 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:49:43,398 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:49:43,398 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:49:43,400 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:49:43,557 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:49:44,685 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:49:47,658 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:49:47,659 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:49:47,659 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:49:47,665 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:49:48,562 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:49:49,566 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:49:49,701 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:49:52,044 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:49:52,045 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:49:52,045 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:49:52,052 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:49:52,568 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:49:53,572 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:49:55,053 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:49:56,149 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:49:56,150 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:49:56,150 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:49:56,156 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:49:56,574 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:49:57,577 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:49:58,398 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:49:58,399 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:49:58,400 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:50:00,671 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:50:00,715 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:50:00,715 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:50:00,715 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:50:00,722 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:50:01,583 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:50:01,584 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:50:05,757 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:50:06,619 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:50:06,620 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:50:06,620 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:50:06,626 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:50:07,592 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:50:07,592 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:50:09,624 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 20:50:11,141 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:50:11,142 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:50:11,142 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:50:11,142 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:50:11,148 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:50:11,596 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:50:13,398 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:50:13,399 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:50:13,400 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:50:13,602 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:50:16,666 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:50:17,114 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:50:17,114 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:50:17,114 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:50:17,120 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:50:17,605 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:50:19,610 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:50:22,157 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:50:22,315 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:50:22,316 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:50:22,316 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:50:22,323 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:50:22,613 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:50:23,616 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:50:27,320 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:50:27,321 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:50:27,321 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:50:27,321 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:50:27,327 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:50:27,622 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:50:28,398 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:50:28,399 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:50:28,400 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:50:29,626 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:50:32,440 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:50:32,440 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:50:32,441 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:50:32,441 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:50:32,447 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:50:32,629 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:50:33,632 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:50:36,851 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:50:36,852 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:50:36,852 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:50:36,858 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:50:37,637 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:50:37,638 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:50:37,858 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:50:39,623 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 20:50:43,398 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:50:43,399 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:50:43,400 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:50:43,684 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:50:45,168 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:50:45,168 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:50:45,168 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:50:45,174 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:50:45,647 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:50:47,652 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:50:49,209 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:50:49,997 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:50:49,998 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:50:49,998 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:50:50,004 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:50:50,654 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:50:51,658 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:50:55,005 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:50:55,769 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:50:55,769 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:50:55,770 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:50:55,776 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:50:56,662 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:50:57,668 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:50:58,399 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:50:58,399 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:50:58,400 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:51:00,279 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:51:00,280 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:51:00,280 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:51:00,280 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:51:00,286 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:51:00,670 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:51:01,673 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:51:05,322 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:51:07,718 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:51:07,719 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:51:07,719 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:51:07,725 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:51:08,681 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:51:09,628 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 20:51:09,684 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:51:10,630 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:51:11,800 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:51:11,801 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:51:11,801 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:51:11,807 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:51:12,687 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:51:13,399 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:51:13,399 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:51:13,401 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:51:13,691 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:51:15,657 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:51:16,503 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:51:16,504 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:51:16,504 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:51:16,510 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:51:16,693 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:51:17,697 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:51:21,545 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:51:21,967 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:51:21,968 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:51:21,968 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:51:21,974 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:51:22,702 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:51:23,707 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:51:26,975 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:51:28,123 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:51:28,124 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:51:28,124 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:51:28,131 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:51:28,399 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:51:28,399 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:51:28,401 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:51:28,712 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:51:29,715 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:51:32,582 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:51:32,582 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:51:32,582 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:51:32,583 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:51:32,588 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:51:32,718 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:51:33,721 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:51:37,049 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:51:37,050 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:51:37,050 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:51:37,056 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:51:37,727 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:51:37,727 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:51:38,057 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:51:39,628 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 20:51:41,310 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:51:41,311 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:51:41,311 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:51:41,318 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:51:41,731 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:51:43,318 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:51:43,399 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:51:43,399 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:51:43,400 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:51:43,736 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:51:45,700 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:51:45,700 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:51:45,701 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:51:45,706 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:51:45,738 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:51:47,742 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:51:48,741 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:51:50,293 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:51:50,294 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:51:50,294 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:51:50,300 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:51:50,744 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:51:51,748 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:51:54,301 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:51:55,835 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:51:55,836 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:51:55,836 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:51:55,841 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:51:56,752 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:51:57,757 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:51:58,399 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:51:58,399 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:51:58,401 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:51:59,656 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:52:00,114 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:52:00,114 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:52:00,114 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:52:00,121 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:52:00,759 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:52:01,762 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:52:04,612 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:52:04,612 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:52:04,613 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:52:04,620 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:52:04,765 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:52:05,620 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:52:05,768 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:52:09,239 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:52:09,240 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:52:09,240 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:52:09,246 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:52:09,633 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 20:52:09,774 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:52:09,774 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:52:10,634 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:52:13,399 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:52:13,399 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:52:13,401 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:52:14,155 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:52:14,155 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:52:14,155 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:52:14,161 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:52:14,779 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:52:15,783 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:52:16,197 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:52:19,173 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:52:19,174 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:52:19,174 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:52:19,180 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:52:19,789 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:52:19,790 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:52:22,181 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:52:24,944 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:52:24,945 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:52:24,945 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:52:24,951 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:52:25,798 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:52:25,798 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:52:27,952 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:52:28,399 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:52:28,399 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:52:28,401 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:52:29,240 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:52:29,241 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:52:29,241 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:52:29,246 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:52:29,803 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:52:31,807 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:52:33,281 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:52:38,282 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:52:38,461 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:52:38,462 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:52:38,462 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:52:38,468 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:52:38,813 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:52:39,632 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 20:52:39,818 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:52:42,681 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:52:42,682 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:52:42,682 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:52:42,689 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:52:42,820 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:52:43,399 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:52:43,400 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:52:43,401 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:52:43,599 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:52:43,825 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:52:47,367 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:52:47,368 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:52:47,368 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:52:47,374 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:52:47,828 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:52:49,409 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:52:49,833 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:52:51,792 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:52:51,793 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:52:51,793 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:52:51,799 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:52:51,835 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:52:53,839 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:52:54,800 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:52:56,847 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:52:56,848 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:52:56,848 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:52:56,854 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:52:57,846 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:52:57,846 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:52:58,399 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:52:58,400 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:52:58,401 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:53:00,697 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:53:00,974 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:53:00,975 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:53:00,975 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:53:00,981 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:53:01,851 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:53:01,852 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:53:06,008 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:53:06,009 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:53:06,009 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:53:06,009 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:53:06,015 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:53:06,855 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:53:07,860 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:53:09,634 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 20:53:11,273 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:53:11,277 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:53:11,278 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:53:11,278 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:53:11,284 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:53:11,867 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:53:11,867 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:53:13,399 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:53:13,400 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:53:13,401 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:53:15,606 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:53:15,606 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:53:15,606 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:53:15,612 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:53:15,870 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:53:16,649 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:53:17,875 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:53:21,156 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:53:21,157 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:53:21,157 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:53:21,163 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:53:21,882 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:53:21,882 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:53:22,164 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:53:25,891 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:53:25,892 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:53:25,892 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:53:25,898 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:53:26,886 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:53:27,891 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:53:27,899 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:53:28,399 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:53:28,400 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:53:28,401 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:53:30,821 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:53:30,822 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:53:30,822 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:53:30,828 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:53:30,894 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:53:31,898 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:53:33,865 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:53:35,770 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:53:35,771 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:53:35,771 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:53:35,777 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:53:35,901 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:53:37,905 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:53:39,637 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 20:53:39,638 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:53:40,788 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:53:40,789 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:53:40,789 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:53:40,794 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:53:40,907 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:53:41,911 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:53:43,399 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:53:43,400 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:53:43,401 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:53:44,677 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:53:45,350 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:53:45,351 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:53:45,351 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:53:45,357 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:53:45,915 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:53:47,920 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:53:50,394 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:53:52,967 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:53:52,968 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:53:52,968 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:53:52,974 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:53:53,929 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:53:53,929 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:53:55,974 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:53:58,399 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:53:58,400 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:53:58,401 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:54:01,645 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:54:03,997 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:54:03,998 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:54:03,998 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:54:04,004 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:54:04,941 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:54:05,946 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:54:07,041 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:54:09,500 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:54:09,501 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:54:09,501 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:54:09,508 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:54:09,639 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 20:54:09,949 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:54:11,955 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:54:12,640 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:54:13,399 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:54:13,400 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:54:13,401 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:54:15,976 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:54:15,977 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:54:15,977 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:54:15,983 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:54:16,960 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:54:17,964 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:54:18,017 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:54:21,215 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:54:21,216 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:54:21,216 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:54:21,222 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:54:21,970 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:54:21,970 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:54:23,223 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:54:26,153 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:54:26,154 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:54:26,154 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:54:26,160 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:54:26,974 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:54:27,980 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:54:28,400 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:54:28,400 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:54:28,401 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:54:28,635 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:54:31,869 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:54:31,870 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:54:31,870 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:54:31,877 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:54:31,984 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:54:33,913 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:54:33,988 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:54:36,353 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:54:36,357 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:54:36,357 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:54:36,363 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:54:36,990 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:54:37,993 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:54:39,364 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:54:39,641 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 20:54:40,813 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:54:40,814 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:54:40,814 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:54:40,820 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:54:40,995 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:54:41,999 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:54:43,400 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:54:43,400 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:54:43,402 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:54:44,685 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:54:45,115 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:54:45,116 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:54:45,116 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:54:45,122 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:54:46,006 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:54:46,006 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:54:49,920 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:54:49,921 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:54:49,921 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:54:49,921 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:54:49,927 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:54:50,009 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:54:52,014 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:54:54,548 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:54:54,549 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:54:54,549 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:54:54,555 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:54:55,016 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:54:55,556 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:54:56,020 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:54:58,400 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:54:58,400 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:54:58,402 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:54:58,840 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:54:58,841 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:54:58,841 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:54:58,847 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:54:59,022 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:55:00,026 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:55:00,881 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:55:05,882 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:55:06,015 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:55:06,015 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:55:06,015 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:55:06,021 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:55:06,032 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:55:08,038 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:55:09,643 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 20:55:10,472 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:55:10,473 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:55:10,473 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:55:10,479 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:55:11,040 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:55:11,479 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:55:12,044 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:55:13,400 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:55:13,400 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:55:13,402 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:55:16,316 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:55:16,317 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:55:16,317 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:55:16,323 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:55:17,048 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:55:17,357 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:55:18,052 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:55:22,358 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:55:25,159 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:55:25,160 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:55:25,160 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:55:25,166 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:55:26,064 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:55:26,064 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:55:28,166 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:55:28,400 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:55:28,400 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:55:28,402 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:55:30,468 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:55:30,469 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:55:30,469 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:55:30,475 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:55:31,070 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:55:32,073 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:55:33,509 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:55:38,510 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:55:38,762 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:55:38,763 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:55:38,763 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:55:38,769 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:55:39,079 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:55:39,648 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 20:55:40,083 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:55:43,400 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:55:43,400 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:55:43,402 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:55:43,622 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:55:43,720 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:55:43,721 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:55:43,721 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:55:43,727 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:55:44,088 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:55:46,093 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:55:48,762 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:55:49,046 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:55:49,047 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:55:49,047 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:55:49,053 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:55:49,095 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:55:50,099 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:55:53,705 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:55:53,706 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:55:53,706 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:55:53,712 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:55:54,103 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:55:54,713 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:55:56,108 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:55:58,400 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:55:58,400 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:55:58,402 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:55:58,592 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:55:58,696 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:55:58,696 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:55:58,702 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:55:59,111 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:56:00,114 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:56:00,703 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:56:05,703 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:56:06,818 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:56:06,818 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:56:06,819 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:56:06,824 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:56:07,121 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:56:08,126 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:56:09,647 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 20:56:11,649 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:56:12,717 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:56:12,718 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:56:12,718 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:56:12,724 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:56:13,131 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:56:13,400 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:56:13,401 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:56:13,402 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:56:14,136 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:56:17,611 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:56:18,153 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:56:18,154 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:56:18,154 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:56:18,160 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:56:19,140 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:56:20,145 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:56:23,197 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:56:24,582 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:56:24,583 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:56:24,583 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:56:24,589 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:56:25,149 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:56:26,153 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:56:28,400 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:56:28,401 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:56:28,402 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:56:28,701 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:56:29,697 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:56:29,698 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:56:29,698 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:56:29,704 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:56:30,159 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:56:32,163 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:56:33,741 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:56:34,355 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:56:34,356 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:56:34,356 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:56:34,363 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:56:35,165 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:56:36,169 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:56:39,363 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:56:39,652 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 20:56:40,068 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:56:40,069 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:56:40,069 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:56:40,075 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:56:40,172 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:56:42,177 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:56:43,400 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:56:43,401 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:56:43,402 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:56:44,473 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:56:44,474 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:56:44,474 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:56:44,474 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:56:44,481 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:56:45,179 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:56:46,183 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:56:49,135 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:56:49,136 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:56:49,136 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:56:49,142 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:56:49,185 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:56:50,142 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:56:50,189 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:56:54,210 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:56:54,212 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:56:54,212 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:56:54,218 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:56:55,193 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:56:55,218 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:56:56,197 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:56:58,400 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:56:58,401 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:56:58,402 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:56:58,417 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:56:58,615 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:56:58,615 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:56:58,621 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:56:59,199 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:57:00,203 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:57:00,621 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:57:04,187 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:57:04,188 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:57:04,188 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:57:04,194 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:57:04,207 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:57:06,195 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:57:06,212 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:57:09,652 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 20:57:09,781 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:57:09,782 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:57:09,782 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:57:09,788 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:57:10,216 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:57:11,789 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:57:12,221 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:57:13,401 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:57:13,401 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:57:13,402 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:57:14,273 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:57:14,274 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:57:14,274 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:57:14,280 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:57:15,224 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:57:16,227 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:57:17,317 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:57:20,199 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:57:20,200 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:57:20,200 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:57:20,206 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:57:20,231 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:57:22,237 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:57:23,207 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:57:24,640 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:57:24,641 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:57:24,641 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:57:24,647 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:57:25,239 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:57:26,242 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:57:28,401 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:57:28,401 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:57:28,402 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:57:28,671 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:57:33,544 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:57:33,544 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:57:33,545 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:57:33,550 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:57:34,254 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:57:34,254 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:57:34,585 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:57:38,626 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:57:38,626 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:57:38,627 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:57:38,632 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:57:39,257 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:57:39,633 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:57:39,655 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 20:57:40,260 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:57:43,401 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:57:43,401 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:57:43,402 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:57:45,621 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:57:46,828 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:57:46,829 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:57:46,829 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:57:46,835 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:57:47,270 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:57:48,275 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:57:50,869 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:57:51,383 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:57:51,384 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:57:51,384 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:57:51,389 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:57:52,281 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:57:52,281 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:57:56,390 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:57:56,506 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:57:56,510 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:57:56,510 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:57:56,516 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:57:57,285 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:57:58,289 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:57:58,401 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:57:58,401 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:57:58,402 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:58:01,196 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:58:01,197 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:58:01,197 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:58:01,203 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:58:01,291 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:58:02,237 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:58:02,295 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:58:07,238 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:58:07,537 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:58:07,538 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:58:07,538 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:58:07,544 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:58:08,304 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:58:08,304 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:58:09,657 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 20:58:12,175 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:58:12,175 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:58:12,176 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:58:12,181 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:58:12,308 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:58:13,182 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:58:13,401 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:58:13,401 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:58:13,402 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:58:14,313 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:58:18,703 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:58:19,346 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:58:19,346 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:58:19,346 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:58:19,352 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:58:20,322 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:58:20,322 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:58:24,389 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:58:24,900 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:58:24,902 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:58:24,902 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:58:24,907 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:58:25,326 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:58:26,330 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:58:28,401 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:58:28,401 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:58:28,403 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:58:29,707 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:58:29,752 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:58:29,752 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:58:29,752 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:58:29,758 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:58:30,335 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:58:32,339 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:58:34,448 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:58:34,449 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:58:34,449 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:58:34,455 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:58:35,342 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:58:35,456 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:58:36,346 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:58:39,351 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:58:39,352 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:58:39,352 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:58:39,358 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:58:39,659 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 20:58:40,351 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:58:40,351 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:58:40,661 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:58:43,401 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:58:43,401 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:58:43,403 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:58:44,153 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:58:44,154 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:58:44,154 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:58:44,160 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:58:44,356 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:58:46,197 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:58:46,361 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:58:49,458 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:58:49,459 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:58:49,459 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:58:49,465 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:58:50,366 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:58:50,367 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:58:51,466 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:58:53,587 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:58:53,588 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:58:53,588 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:58:53,594 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:58:54,372 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:58:54,373 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:58:56,594 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:58:58,401 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:58:58,401 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:58:58,403 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:59:01,607 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:59:02,645 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:59:02,646 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:59:02,646 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:59:02,653 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:59:03,382 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:59:04,387 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:59:06,690 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:59:07,049 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:59:07,050 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:59:07,050 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:59:07,056 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:59:07,389 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:59:08,393 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:59:09,665 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 20:59:11,741 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:59:11,742 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:59:11,742 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:59:11,742 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:59:11,748 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:59:12,397 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:59:13,401 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:59:13,401 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:59:13,403 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:59:14,402 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:59:17,648 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:59:17,830 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:59:17,830 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:59:17,830 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:59:17,836 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:59:18,406 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:59:20,411 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:59:22,459 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:59:22,460 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:59:22,460 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:59:22,466 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:59:23,413 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:59:23,467 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:59:24,417 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:59:27,321 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:59:27,321 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:59:27,322 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:59:27,328 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:59:27,419 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:59:28,401 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:59:28,402 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:59:28,403 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:59:28,424 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:59:28,705 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:59:31,663 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:59:31,664 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:59:31,664 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:59:31,670 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:59:32,430 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:59:32,430 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:59:33,705 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:59:36,656 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:59:36,657 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:59:36,657 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:59:36,664 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:59:37,434 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:59:38,438 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:59:39,664 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 20:59:39,665 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:59:42,544 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:59:42,545 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:59:42,545 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:59:42,551 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:59:43,401 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:59:43,401 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:59:43,403 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:59:43,443 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:59:44,447 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:59:45,614 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:59:49,594 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:59:49,594 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:59:49,594 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:59:49,600 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:59:50,455 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 20:59:50,456 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 20:59:50,637 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:59:55,638 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:59:58,029 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:59:58,030 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 20:59:58,030 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:59:58,036 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:59:58,401 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:59:58,402 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:59:58,403 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:59:58,464 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:00:00,470 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:00:00,658 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:00:02,254 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:00:02,254 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:00:02,255 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:00:02,260 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:00:02,471 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:00:04,476 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:00:06,297 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:00:07,080 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:00:07,081 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:00:07,081 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:00:07,087 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:00:07,478 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:00:08,482 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:00:09,669 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 21:00:11,670 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:00:11,838 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:00:11,838 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:00:11,838 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:00:11,844 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:00:12,486 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:00:13,401 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:00:13,402 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:00:13,403 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:00:14,491 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:00:17,560 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:00:17,560 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:00:17,561 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:00:17,561 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:00:17,566 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:00:18,497 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:00:18,497 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:00:22,601 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:00:22,946 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:00:22,947 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:00:22,947 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:00:22,953 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:00:23,502 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:00:24,506 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:00:27,954 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:00:28,401 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:00:28,402 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:00:28,403 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:00:31,349 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:00:31,349 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:00:31,349 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:00:31,355 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:00:31,513 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:00:32,517 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:00:33,393 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:00:36,031 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:00:36,032 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:00:36,032 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:00:36,038 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:00:36,521 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:00:38,526 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:00:39,039 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:00:39,668 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 21:00:40,559 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:00:40,560 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:00:40,560 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:00:40,565 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:00:41,529 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:00:42,533 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:00:43,402 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:00:43,402 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:00:43,403 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:00:44,637 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:00:49,026 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:00:49,027 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:00:49,027 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:00:49,033 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:00:49,539 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:00:50,069 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:00:50,544 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:00:54,917 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:00:54,918 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:00:54,918 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:00:54,924 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:00:55,548 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:00:55,924 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:00:56,552 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:00:58,402 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:00:58,402 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:00:58,403 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:01:00,404 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:01:00,405 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:01:00,405 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:01:00,410 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:01:00,556 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:01:01,445 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:01:02,562 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:01:06,446 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:01:06,497 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:01:06,498 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:01:06,498 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:01:06,503 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:01:06,565 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:01:08,571 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:01:09,673 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 21:01:11,586 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:01:11,587 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:01:11,588 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:01:11,588 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:01:11,593 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:01:12,577 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:01:12,577 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:01:13,402 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:01:13,402 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:01:13,404 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:01:16,670 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:01:16,996 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:01:16,996 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:01:16,996 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:01:17,003 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:01:17,580 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:01:18,585 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:01:22,037 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:01:26,118 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:01:26,119 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:01:26,119 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:01:26,126 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:01:26,594 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:01:27,126 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:01:28,402 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:01:28,402 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:01:28,404 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:01:28,601 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:01:31,357 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:01:31,357 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:01:31,358 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:01:31,364 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:01:31,603 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:01:32,401 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:01:32,607 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:01:36,423 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:01:36,423 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:01:36,424 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:01:36,430 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:01:36,610 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:01:37,430 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:01:38,614 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:01:39,672 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 21:01:42,275 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:01:42,276 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:01:42,276 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:01:42,282 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:01:42,618 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:01:43,282 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:01:43,402 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:01:43,402 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:01:43,403 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:01:44,623 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:01:46,800 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:01:46,800 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:01:46,800 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:01:46,807 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:01:47,626 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:01:48,629 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:01:48,841 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:01:51,043 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:01:51,044 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:01:51,044 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:01:51,051 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:01:51,632 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:01:52,635 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:01:54,051 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:01:57,559 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:01:57,560 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:01:57,560 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:01:57,566 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:01:57,639 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:01:58,402 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:01:58,402 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:01:58,403 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:01:58,643 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:01:59,698 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:02:01,804 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:02:01,804 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:02:01,805 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:02:01,811 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:02:02,649 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:02:02,649 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:02:04,845 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:02:08,658 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:02:08,659 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:02:08,659 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:02:08,665 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:02:09,656 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:02:09,674 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 21:02:10,660 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:02:10,676 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:02:13,402 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:02:13,402 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:02:13,404 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:02:14,439 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:02:14,440 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:02:14,440 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:02:14,447 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:02:14,665 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:02:16,481 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:02:16,671 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:02:19,434 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:02:19,435 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:02:19,435 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:02:19,442 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:02:19,673 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:02:20,676 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:02:22,442 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:02:25,514 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:02:25,515 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:02:25,515 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:02:25,521 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:02:25,680 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:02:26,685 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:02:27,522 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:02:28,402 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:02:28,402 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:02:28,404 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:02:30,760 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:02:30,761 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:02:30,761 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:02:30,768 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:02:31,692 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:02:32,695 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:02:32,801 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:02:36,039 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:02:36,040 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:02:36,040 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:02:36,046 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:02:36,699 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:02:38,046 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:02:38,702 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:02:39,680 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 21:02:40,562 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:02:40,563 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:02:40,563 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:02:40,569 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:02:40,704 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:02:42,709 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:02:43,402 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:02:43,403 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:02:43,404 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:02:43,667 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:02:46,675 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:02:46,676 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:02:46,676 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:02:46,682 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:02:46,712 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:02:48,717 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:02:48,717 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:02:51,819 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:02:51,820 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:02:51,821 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:02:51,827 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:02:52,723 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:02:52,724 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:02:53,827 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:02:58,402 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:02:58,403 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:02:58,404 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:02:58,616 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:02:58,698 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:02:58,699 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:02:58,704 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:02:58,729 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:02:59,705 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:03:00,734 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:03:03,319 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:03:03,320 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:03:03,321 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:03:03,327 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:03:03,737 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:03:04,741 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:03:05,328 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:03:08,167 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:03:08,168 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:03:08,168 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:03:08,174 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:03:08,744 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:03:09,679 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 21:03:10,680 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:03:10,747 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:03:12,588 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:03:12,589 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:03:12,589 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:03:12,595 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:03:12,750 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:03:13,402 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:03:13,403 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:03:13,404 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:03:14,756 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:03:16,663 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:03:17,059 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:03:17,060 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:03:17,060 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:03:17,066 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:03:17,758 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:03:18,762 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:03:21,712 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:03:21,712 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:03:21,713 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:03:21,713 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:03:21,718 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:03:21,764 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:03:22,768 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:03:26,306 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:03:26,307 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:03:26,308 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:03:26,313 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:03:26,771 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:03:27,314 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:03:28,402 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:03:28,403 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:03:28,404 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:03:28,778 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:03:30,557 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:03:30,557 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:03:30,558 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:03:30,564 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:03:30,780 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:03:32,601 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:03:32,781 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:03:35,617 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:03:35,617 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:03:35,618 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:03:35,624 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:03:35,784 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:03:36,788 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:03:37,625 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:03:39,685 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 21:03:41,915 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:03:41,916 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:03:41,917 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:03:41,923 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:03:42,797 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:03:42,797 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:03:42,923 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:03:43,402 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:03:43,403 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:03:43,404 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:03:48,605 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:03:49,281 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:03:49,282 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:03:49,282 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:03:49,288 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:03:49,804 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:03:50,809 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:03:53,912 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:03:53,913 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:03:53,913 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:03:53,913 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:03:53,920 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:03:54,815 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:03:54,815 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:03:58,403 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:03:58,403 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:03:58,404 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:03:58,808 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:03:58,808 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:03:58,808 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:03:58,815 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:03:58,819 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:03:59,849 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:04:00,825 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:04:04,850 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:04:05,032 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:04:05,033 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:04:05,033 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:04:05,039 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:04:05,829 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:04:06,833 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:04:09,683 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 21:04:09,723 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:04:09,724 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:04:09,724 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:04:09,731 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:04:09,835 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:04:10,732 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:04:10,839 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:04:13,403 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:04:13,403 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:04:13,404 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:04:16,639 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:04:19,892 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:04:19,892 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:04:19,892 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:04:19,899 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:04:20,853 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:04:20,853 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:04:21,933 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:04:24,487 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:04:24,488 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:04:24,488 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:04:24,494 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:04:24,856 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:04:26,861 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:04:27,495 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:04:28,403 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:04:28,403 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:04:28,404 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:04:29,943 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:04:29,944 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:04:29,944 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:04:29,950 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:04:30,869 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:04:30,869 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:04:32,985 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:04:34,449 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:04:34,450 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:04:34,450 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:04:34,456 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:04:34,873 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:04:36,877 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:04:38,457 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:04:39,238 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:04:39,238 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:04:39,239 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:04:39,244 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:04:39,689 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 21:04:39,879 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:04:40,883 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:04:43,403 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:04:43,403 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:04:43,405 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:04:43,628 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:04:44,053 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:04:44,054 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:04:44,054 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:04:44,060 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:04:44,886 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:04:46,891 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:04:48,951 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:04:48,952 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:04:48,952 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:04:48,952 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:04:48,958 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:04:49,893 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:04:50,897 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:04:53,557 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:04:53,558 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:04:53,558 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:04:53,564 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:04:53,899 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:04:54,565 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:04:54,903 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:04:58,403 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:04:58,403 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:04:58,405 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:04:59,624 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:05:00,276 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:05:00,277 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:05:00,277 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:05:00,284 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:05:00,909 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:05:02,914 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:05:04,633 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:05:04,634 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:05:04,634 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:05:04,634 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:05:04,640 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:05:04,916 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:05:06,920 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:05:09,149 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:05:09,150 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:05:09,150 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:05:09,156 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:05:09,688 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 21:05:09,689 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:05:09,923 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:05:10,926 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:05:13,403 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:05:13,403 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:05:13,405 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:05:13,615 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:05:13,659 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:05:13,659 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:05:13,665 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:05:13,929 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:05:14,932 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:05:15,666 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:05:18,323 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:05:18,324 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:05:18,324 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:05:18,330 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:05:18,936 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:05:20,941 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:05:21,365 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:05:22,878 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:05:22,879 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:05:22,879 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:05:22,885 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:05:22,943 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:05:24,945 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:05:26,886 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:05:27,591 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:05:27,592 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:05:27,592 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:05:27,599 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:05:27,949 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:05:28,404 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:05:28,404 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:05:28,405 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:05:28,955 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:05:32,151 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:05:32,151 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:05:32,151 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:05:32,152 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:05:32,157 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:05:32,958 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:05:34,962 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:05:37,154 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:05:37,155 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:05:37,155 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:05:37,156 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:05:37,161 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:05:37,965 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:05:38,968 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:05:39,694 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 21:05:41,684 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:05:41,685 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:05:41,685 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:05:41,691 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:05:41,970 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:05:42,692 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:05:42,975 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:05:43,403 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:05:43,403 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:05:43,405 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:05:46,192 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:05:46,193 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:05:46,193 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:05:46,199 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:05:46,978 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:05:48,233 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:05:48,983 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:05:53,095 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:05:53,096 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:05:53,096 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:05:53,103 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:05:53,987 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:05:54,103 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:05:54,991 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:05:58,403 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:05:58,404 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:05:58,405 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:05:58,612 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:05:58,709 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:05:58,710 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:05:58,715 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:05:58,995 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:05:59,716 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:06:00,998 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:06:03,834 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:06:03,835 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:06:03,835 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:06:03,841 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:06:04,002 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:06:04,841 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:06:05,007 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:06:09,692 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 21:06:10,694 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:06:11,036 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:06:11,037 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:06:11,037 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:06:11,043 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:06:12,013 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:06:13,018 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:06:13,403 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:06:13,404 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:06:13,405 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:06:15,704 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:06:17,421 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:06:17,421 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:06:17,421 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:06:17,428 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:06:18,023 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:06:19,027 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:06:21,465 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:06:22,458 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:06:22,459 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:06:22,459 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:06:22,465 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:06:23,031 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:06:25,036 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:06:26,466 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:06:27,563 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:06:27,564 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:06:27,564 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:06:27,571 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:06:28,038 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:06:28,404 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:06:28,404 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:06:28,405 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:06:29,044 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:06:31,660 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:06:31,839 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:06:31,840 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:06:31,840 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:06:31,847 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:06:32,047 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:06:33,050 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:06:36,881 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:06:37,349 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:06:37,349 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:06:37,350 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:06:37,356 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:06:38,055 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:06:39,058 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:06:39,695 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 21:06:42,002 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:06:42,003 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:06:42,003 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:06:42,003 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:06:42,009 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:06:42,060 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:06:43,064 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:06:43,403 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:06:43,404 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:06:43,405 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:06:46,376 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:06:46,376 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:06:46,377 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:06:46,383 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:06:47,067 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:06:47,417 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:06:49,072 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:06:52,418 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:06:54,883 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:06:54,884 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:06:54,884 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:06:54,890 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:06:55,078 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:06:57,082 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:06:57,890 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:06:58,404 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:06:58,404 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:06:58,405 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:07:00,310 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:07:00,314 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:07:00,314 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:07:00,321 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:07:01,086 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:07:03,091 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:07:03,353 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:07:04,742 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:07:04,743 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:07:04,743 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:07:04,749 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:07:05,094 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:07:07,098 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:07:08,750 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:07:09,697 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 21:07:10,443 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:07:10,443 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:07:10,444 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:07:10,450 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:07:11,101 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:07:13,107 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:07:13,404 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:07:13,404 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:07:13,405 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:07:14,617 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:07:17,190 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:07:17,191 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:07:17,191 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:07:17,197 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:07:18,112 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:07:19,116 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:07:20,233 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:07:21,662 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:07:21,663 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:07:21,663 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:07:21,669 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:07:22,119 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:07:23,122 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:07:25,670 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:07:28,211 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:07:28,212 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:07:28,212 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:07:28,218 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:07:28,404 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:07:28,404 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:07:28,405 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:07:29,132 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:07:29,132 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:07:30,695 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:07:34,302 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:07:34,302 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:07:34,303 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:07:34,308 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:07:35,138 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:07:36,345 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:07:37,141 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:07:39,615 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:07:39,616 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:07:39,617 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:07:39,622 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:07:39,699 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 21:07:40,144 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:07:41,149 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:07:41,700 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:07:43,404 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:07:43,404 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:07:43,406 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:07:46,051 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:07:46,052 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:07:46,052 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:07:46,057 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:07:46,153 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:07:47,093 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:07:47,157 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:07:51,165 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:07:51,166 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:07:51,166 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:07:51,171 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:07:52,161 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:07:52,172 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:07:53,166 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:07:57,172 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:07:57,736 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:07:57,737 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:07:57,737 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:07:57,743 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:07:58,170 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:07:58,404 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:07:58,404 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:07:58,406 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:07:59,175 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:08:02,413 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:08:02,413 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:08:02,413 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:08:02,414 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:08:02,419 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:08:03,178 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:08:05,183 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:08:06,540 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:08:06,541 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:08:06,541 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:08:06,547 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:08:07,185 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:08:07,547 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:08:09,189 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:08:09,704 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 21:08:12,706 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:08:13,404 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:08:13,404 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:08:13,406 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:08:14,793 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:08:14,794 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:08:14,794 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:08:14,800 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:08:15,196 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:08:17,201 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:08:17,837 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:08:18,922 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:08:18,922 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:08:18,923 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:08:18,928 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:08:19,203 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:08:21,207 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:08:22,929 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:08:24,018 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:08:24,019 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:08:24,019 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:08:24,025 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:08:24,210 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:08:25,214 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:08:28,026 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:08:28,404 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:08:28,405 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:08:28,406 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:08:28,470 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:08:28,684 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:08:28,684 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:08:28,690 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:08:29,219 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:08:31,223 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:08:33,417 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:08:33,418 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:08:33,418 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:08:33,418 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:08:33,424 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:08:34,225 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:08:35,229 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:08:38,425 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:08:38,662 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:08:38,663 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:08:38,663 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:08:38,669 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:08:39,232 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:08:39,703 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 21:08:41,237 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:08:43,404 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:08:43,404 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:08:43,406 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:08:43,637 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:08:47,303 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:08:47,303 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:08:47,303 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:08:47,310 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:08:48,244 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:08:49,249 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:08:49,345 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:08:53,925 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:08:53,926 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:08:53,926 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:08:53,932 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:08:54,253 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:08:54,932 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:08:55,255 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:08:58,404 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:08:58,405 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:08:58,406 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:08:59,667 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:08:59,667 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:08:59,667 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:08:59,673 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:09:00,262 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:09:00,709 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:09:01,266 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:09:04,325 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:09:04,326 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:09:04,326 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:09:04,332 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:09:05,273 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:09:05,273 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:09:06,332 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:09:09,708 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 21:09:09,822 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:09:09,822 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:09:09,822 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:09:09,828 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:09:10,277 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:09:11,281 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:09:11,829 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:09:13,405 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:09:13,405 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:09:13,406 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:09:17,704 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:09:18,361 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:09:18,362 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:09:18,362 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:09:18,368 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:09:19,291 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:09:21,296 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:09:23,405 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:09:23,715 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:09:23,716 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:09:23,716 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:09:23,722 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:09:24,298 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:09:25,302 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:09:28,281 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:09:28,282 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:09:28,282 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:09:28,288 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:09:28,304 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:09:28,404 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:09:28,405 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:09:28,406 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:09:28,651 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:09:29,309 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:09:33,651 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:09:33,768 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:09:33,769 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:09:33,769 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:09:33,775 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:09:34,313 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:09:35,317 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:09:38,810 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:09:39,708 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 21:09:42,342 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:09:42,344 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:09:42,344 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:09:42,350 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:09:43,328 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:09:43,328 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:09:43,404 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:09:43,405 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:09:43,406 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:09:44,661 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:09:49,274 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:09:49,275 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:09:49,275 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:09:49,280 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:09:49,334 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:09:50,317 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:09:51,340 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:09:53,607 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:09:53,608 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:09:53,608 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:09:53,613 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:09:54,342 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:09:55,346 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:09:55,614 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:09:58,405 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:09:58,405 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:09:58,407 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:09:58,426 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:09:58,644 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:09:58,644 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:09:58,649 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:09:59,350 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:10:00,650 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:10:01,354 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:10:04,955 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:10:04,956 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:10:04,956 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:10:04,962 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:10:05,358 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:10:05,962 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:10:07,363 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:10:09,434 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:10:09,435 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:10:09,435 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:10:09,441 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:10:09,710 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 21:10:10,365 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:10:11,369 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:10:11,711 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:10:13,405 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:10:13,405 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:10:13,406 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:10:14,084 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:10:14,085 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:10:14,085 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:10:14,090 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:10:14,371 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:10:15,375 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:10:17,125 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:10:18,652 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:10:18,653 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:10:18,653 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:10:18,658 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:10:19,379 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:10:21,384 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:10:22,659 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:10:23,409 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:10:23,413 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:10:23,414 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:10:23,419 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:10:24,387 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:10:25,390 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:10:27,872 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:10:27,873 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:10:27,874 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:10:27,874 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:10:27,879 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:10:28,392 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:10:28,405 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:10:28,405 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:10:28,406 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:10:29,399 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:10:32,339 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:10:32,340 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:10:32,341 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:10:32,346 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:10:32,401 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:10:33,381 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:10:33,404 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:10:36,741 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:10:36,742 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:10:36,742 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:10:36,748 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:10:37,408 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:10:38,748 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:10:39,412 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:10:39,712 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 21:10:41,049 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:10:41,050 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:10:41,050 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:10:41,055 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:10:41,413 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:10:43,405 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:10:43,405 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:10:43,406 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:10:43,418 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:10:44,653 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:10:46,001 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:10:46,001 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:10:46,002 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:10:46,007 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:10:46,420 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:10:47,423 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:10:50,046 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:10:51,383 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:10:51,384 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:10:51,384 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:10:51,390 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:10:51,427 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:10:53,432 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:10:55,391 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:10:57,573 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:10:57,574 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:10:57,574 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:10:57,580 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:10:58,405 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:10:58,405 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:10:58,407 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:10:58,437 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:10:59,441 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:11:00,630 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:11:02,365 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:11:02,366 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:11:02,366 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:11:02,372 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:11:02,443 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:11:03,449 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:11:06,409 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:11:07,146 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:11:07,146 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:11:07,147 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:11:07,152 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:11:07,452 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:11:09,456 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:11:09,714 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 21:11:11,716 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:11:12,232 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:11:12,233 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:11:12,233 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:11:12,239 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:11:12,458 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:11:13,405 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:11:13,405 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:11:13,407 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:11:13,462 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:11:17,705 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:11:18,711 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:11:18,712 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:11:18,712 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:11:18,718 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:11:19,469 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:11:21,473 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:11:22,754 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:11:24,311 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:11:24,312 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:11:24,313 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:11:24,319 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:11:24,476 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:11:25,479 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:11:28,320 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:11:28,405 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:11:28,405 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:11:28,407 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:11:29,201 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:11:29,202 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:11:29,202 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:11:29,207 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:11:29,485 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:11:31,489 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:11:34,246 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:11:34,295 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:11:34,296 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:11:34,296 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:11:34,302 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:11:34,491 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:11:35,495 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:11:39,303 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:11:39,720 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 21:11:42,919 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:11:42,920 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:11:42,920 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:11:42,926 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:11:43,405 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:11:43,406 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:11:43,407 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:11:43,504 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:11:44,647 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:11:45,509 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:11:49,647 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:11:49,822 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:11:49,823 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:11:49,823 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:11:49,830 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:11:50,513 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:11:51,518 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:11:54,074 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:11:54,075 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:11:54,075 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:11:54,081 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:11:54,520 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:11:55,081 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:11:55,521 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:11:58,405 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:11:58,405 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:11:58,407 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:12:00,712 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:12:00,928 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:12:00,928 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:12:00,929 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:12:00,934 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:12:01,530 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:12:03,535 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:12:05,970 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:12:06,286 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:12:06,286 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:12:06,287 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:12:06,292 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:12:06,538 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:12:07,541 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:12:09,719 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 21:12:10,834 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:12:10,834 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:12:10,835 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:12:10,840 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:12:11,544 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:12:11,841 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:12:13,405 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:12:13,406 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:12:13,407 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:12:13,550 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:12:17,617 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:12:20,737 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:12:20,738 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:12:20,738 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:12:20,744 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:12:21,559 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:12:22,781 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:12:23,562 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:12:26,894 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:12:26,895 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:12:26,895 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:12:26,900 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:12:27,568 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:12:27,901 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:12:28,405 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:12:28,406 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:12:28,407 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:12:29,576 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:12:32,180 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:12:32,181 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:12:32,181 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:12:32,187 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:12:32,578 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:12:33,221 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:12:33,580 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:12:38,222 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:12:39,287 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:12:39,288 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:12:39,289 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:12:39,294 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:12:39,585 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:12:39,724 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 21:12:41,591 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:12:43,405 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:12:43,406 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:12:43,407 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:12:43,711 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:12:44,132 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:12:44,132 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:12:44,133 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:12:44,138 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:12:44,593 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:12:45,597 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:12:48,858 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:12:48,858 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:12:48,859 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:12:48,859 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:12:48,864 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:12:49,601 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:12:51,606 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:12:53,865 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:12:55,454 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:12:55,455 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:12:55,455 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:12:55,461 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:12:55,609 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:12:57,614 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:12:58,406 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:12:58,406 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:12:58,407 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:12:59,698 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:13:00,897 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:13:00,897 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:13:00,897 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:13:00,903 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:13:01,618 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:13:03,623 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:13:04,941 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:13:05,566 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:13:05,567 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:13:05,567 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:13:05,573 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:13:05,625 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:13:07,629 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:13:09,723 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 21:13:10,265 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:13:10,266 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:13:10,266 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:13:10,266 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:13:10,272 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:13:10,632 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:13:11,635 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:13:13,406 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:13:13,406 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:13:13,408 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:13:15,179 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:13:15,179 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:13:15,180 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:13:15,185 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:13:15,639 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:13:16,221 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:13:17,644 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:13:20,943 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:13:20,945 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:13:20,945 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:13:20,950 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:13:21,648 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:13:21,951 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:13:23,653 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:13:26,951 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:13:27,289 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:13:27,290 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:13:27,290 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:13:27,296 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:13:27,657 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:13:28,406 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:13:28,406 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:13:28,408 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:13:29,664 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:13:31,554 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:13:31,555 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:13:31,555 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:13:31,561 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:13:31,666 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:13:32,597 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:13:33,670 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:13:36,065 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:13:36,066 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:13:36,067 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:13:36,072 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:13:36,673 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:13:37,676 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:13:38,073 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:13:39,728 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 21:13:41,302 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:13:41,303 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:13:41,303 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:13:41,309 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:13:41,679 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:13:43,309 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:13:43,406 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:13:43,406 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:13:43,407 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:13:43,685 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:13:45,726 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:13:45,727 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:13:45,727 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:13:45,733 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:13:46,687 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:13:47,690 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:13:48,769 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:13:50,231 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:13:50,231 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:13:50,231 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:13:50,238 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:13:50,693 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:13:51,696 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:13:54,238 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:13:54,983 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:13:54,983 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:13:54,984 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:13:54,990 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:13:55,699 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:13:57,704 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:13:58,406 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:13:58,406 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:13:58,408 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:13:59,399 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:13:59,399 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:13:59,399 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:13:59,400 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:13:59,406 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:13:59,706 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:14:01,710 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:14:03,809 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:14:03,810 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:14:03,810 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:14:03,816 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:14:04,712 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:14:04,816 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:14:05,716 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:14:09,727 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 21:14:10,463 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:14:10,465 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:14:10,465 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:14:10,465 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:14:10,471 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:14:10,721 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:14:11,725 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:14:13,406 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:14:13,406 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:14:13,408 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:14:15,695 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:14:18,854 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:14:18,855 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:14:18,855 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:14:18,861 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:14:19,735 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:14:20,897 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:14:21,740 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:14:24,582 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:14:24,583 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:14:24,583 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:14:24,589 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:14:24,743 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:14:25,746 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:14:26,589 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:14:28,406 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:14:28,406 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:14:28,408 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:14:29,452 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:14:29,452 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:14:29,452 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:14:29,458 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:14:29,752 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:14:31,756 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:14:32,493 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:14:35,863 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:14:35,864 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:14:35,865 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:14:35,870 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:14:36,760 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:14:37,764 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:14:37,871 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:14:39,730 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 21:14:40,757 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:14:40,758 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:14:40,758 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:14:40,764 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:14:40,766 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:14:41,770 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:14:43,406 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:14:43,407 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:14:43,408 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:14:43,714 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:14:47,009 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:14:47,010 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:14:47,010 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:14:47,015 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:14:47,777 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:14:49,053 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:14:49,782 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:14:53,163 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:14:53,164 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:14:53,164 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:14:53,170 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:14:53,786 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:14:54,171 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:14:55,791 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:14:58,406 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:14:58,407 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:14:58,408 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:14:58,425 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:14:58,693 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:14:58,694 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:14:58,699 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:14:58,794 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:14:59,700 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:14:59,797 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:15:02,977 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:15:02,978 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:15:02,978 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:15:02,984 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:15:03,800 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:15:04,984 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:15:05,806 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:15:07,468 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:15:07,469 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:15:07,469 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:15:07,475 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:15:07,807 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:15:09,732 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 21:15:09,812 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:15:10,733 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:15:13,406 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:15:13,407 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:15:13,408 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:15:13,409 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:15:13,653 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:15:13,653 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:15:13,659 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:15:13,816 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:15:15,821 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:15:16,660 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:15:19,598 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:15:19,599 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:15:19,599 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:15:19,605 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:15:19,825 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:15:21,830 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:15:22,605 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:15:26,321 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:15:26,322 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:15:26,323 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:15:26,328 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:15:26,834 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:15:27,838 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:15:28,329 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:15:28,406 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:15:28,406 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:15:28,408 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:15:30,772 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:15:30,773 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:15:30,773 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:15:30,779 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:15:30,843 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:15:31,846 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:15:33,813 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:15:35,203 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:15:35,204 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:15:35,204 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:15:35,211 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:15:35,849 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:15:37,853 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:15:39,211 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:15:39,418 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:15:39,419 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:15:39,419 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:15:39,426 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:15:39,734 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 21:15:39,854 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:15:41,859 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:15:43,406 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:15:43,407 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:15:43,408 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:15:44,649 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:15:44,653 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:15:44,654 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:15:44,654 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:15:44,660 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:15:44,862 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:15:45,866 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:15:49,338 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:15:49,339 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:15:49,339 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:15:49,344 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:15:49,869 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:15:50,345 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:15:51,875 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:15:53,875 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:15:53,875 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:15:53,876 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:15:53,881 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:15:54,877 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:15:55,881 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:15:55,882 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:15:58,406 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:15:58,407 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:15:58,408 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:15:59,443 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:15:59,444 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:15:59,444 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:15:59,450 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:15:59,885 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:16:01,485 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:16:01,888 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:16:03,685 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:16:03,685 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:16:03,686 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:16:03,691 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:16:03,891 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:16:05,896 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:16:06,692 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:16:08,000 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:16:08,001 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:16:08,002 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:16:08,008 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:16:08,898 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:16:09,739 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 21:16:09,902 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:16:11,741 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:16:12,566 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:16:12,567 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:16:12,567 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:16:12,573 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:16:12,904 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:16:13,407 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:16:13,407 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:16:13,408 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:16:13,909 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:16:17,710 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:16:19,400 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:16:19,401 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:16:19,401 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:16:19,406 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:16:19,915 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:16:21,921 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:16:23,442 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:16:24,532 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:16:24,533 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:16:24,533 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:16:24,538 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:16:24,923 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:16:25,927 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:16:28,407 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:16:28,407 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:16:28,408 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:16:28,689 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:16:29,119 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:16:29,119 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:16:29,120 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:16:29,125 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:16:29,932 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:16:31,935 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:16:33,569 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:16:33,570 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:16:33,570 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:16:33,576 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:16:33,937 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:16:34,577 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:16:35,941 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:16:39,100 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:16:39,101 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:16:39,101 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:16:39,108 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:16:39,739 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 21:16:39,740 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:16:39,944 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:16:41,950 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:16:43,407 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:16:43,407 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:16:43,409 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:16:44,208 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:16:44,209 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:16:44,209 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:16:44,215 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:16:44,952 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:16:45,249 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:16:45,956 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:16:48,967 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:16:48,967 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:16:48,968 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:16:48,973 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:16:49,960 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:16:50,974 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:16:51,965 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:16:55,472 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:16:55,473 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:16:55,473 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:16:55,479 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:16:55,969 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:16:56,480 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:16:57,974 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:16:58,407 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:16:58,407 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:16:58,409 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:17:00,329 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:17:00,330 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:17:00,330 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:17:00,336 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:17:00,976 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:17:01,980 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:17:02,373 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:17:07,076 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:17:07,077 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:17:07,077 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:17:07,083 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:17:07,986 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:17:08,083 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:17:09,744 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 21:17:09,992 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:17:11,721 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:17:11,722 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:17:11,722 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:17:11,729 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:17:11,993 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:17:13,407 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:17:13,407 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:17:13,409 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:17:13,691 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:17:13,996 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:17:16,213 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:17:16,214 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:17:16,214 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:17:16,219 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:17:17,000 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:17:18,004 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:17:19,257 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:17:24,258 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:17:24,811 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:17:24,812 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:17:24,812 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:17:24,818 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:17:25,013 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:17:25,013 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:17:28,407 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:17:28,407 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:17:28,409 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:17:29,611 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:17:30,058 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:17:30,058 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:17:30,058 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:17:30,064 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:17:31,023 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:17:31,024 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:17:35,101 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:17:35,631 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:17:35,632 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:17:35,632 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:17:35,638 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:17:36,028 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:17:37,031 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:17:39,743 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 21:17:40,745 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:17:41,600 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:17:41,601 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:17:41,601 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:17:41,607 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:17:42,037 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:17:43,039 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:17:43,407 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:17:43,407 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:17:43,409 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:17:46,139 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:17:46,140 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:17:46,140 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:17:46,140 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:17:46,145 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:17:47,046 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:17:47,046 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:17:51,146 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:17:51,147 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:17:51,147 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:17:51,147 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:17:51,154 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:17:52,052 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:17:53,055 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:17:56,154 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:17:56,863 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:17:56,864 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:17:56,864 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:17:56,869 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:17:57,060 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:17:57,060 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:17:58,407 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:17:58,407 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:17:58,409 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:18:01,690 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:18:01,863 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:18:01,864 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:18:01,864 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:18:01,870 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:18:02,067 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:18:03,070 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:18:06,905 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:18:06,949 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:18:06,950 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:18:06,950 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:18:06,956 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:18:07,076 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:18:07,076 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:18:09,748 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 21:18:12,229 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:18:12,230 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:18:12,230 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:18:12,230 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:18:12,236 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:18:13,085 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:18:13,085 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:18:13,407 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:18:13,408 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:18:13,409 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:18:17,672 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:18:18,674 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:18:18,674 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:18:18,675 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:18:18,680 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:18:19,093 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:18:19,093 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:18:22,717 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:18:23,929 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:18:23,929 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:18:23,930 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:18:23,935 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:18:24,099 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:18:25,102 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:18:27,936 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:18:28,407 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:18:28,408 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:18:28,409 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:18:28,955 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:18:28,956 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:18:28,956 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:18:28,962 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:18:29,108 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:18:29,108 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:18:32,997 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:18:34,440 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:18:34,440 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:18:34,441 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:18:34,446 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:18:35,117 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:18:35,117 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:18:38,447 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:18:39,716 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:18:39,717 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:18:39,717 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:18:39,723 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:18:39,748 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 21:18:40,122 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:18:41,125 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:18:43,407 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:18:43,408 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:18:43,409 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:18:43,673 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:18:44,768 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:18:44,769 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:18:44,769 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:18:44,775 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:18:45,132 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:18:45,132 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:18:48,809 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:18:49,093 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:18:49,094 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:18:49,094 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:18:49,100 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:18:49,136 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:18:51,141 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:18:53,963 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:18:53,964 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:18:53,964 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:18:53,964 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:18:53,970 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:18:54,144 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:18:55,147 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:18:58,366 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:18:58,367 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:18:58,367 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:18:58,374 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:18:58,407 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:18:58,408 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:18:58,409 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:18:59,152 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:18:59,153 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:18:59,697 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:19:02,277 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:19:02,277 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:19:02,278 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:19:02,284 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:19:03,158 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:19:03,158 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:19:05,321 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:19:09,750 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 21:19:10,551 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:19:10,552 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:19:10,552 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:19:10,552 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:19:10,558 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:19:11,170 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:19:11,170 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:19:13,408 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:19:13,408 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:19:13,410 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:19:15,647 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:19:15,658 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:19:15,658 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:19:15,658 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:19:15,665 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:19:16,176 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:19:17,179 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:19:20,705 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:19:21,297 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:19:21,298 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:19:21,298 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:19:21,303 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:19:22,185 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:19:23,188 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:19:25,871 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:19:25,872 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:19:25,872 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:19:25,872 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:19:25,878 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:19:26,191 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:19:27,194 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:19:28,408 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:19:28,408 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:19:28,409 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:19:30,745 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:19:30,746 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:19:30,746 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:19:30,752 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:19:31,201 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:19:31,201 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:19:31,789 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:19:36,790 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:19:38,675 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:19:38,676 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:19:38,676 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:19:38,682 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:19:39,211 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:19:39,211 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:19:39,752 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 21:19:42,754 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:19:43,408 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:19:43,408 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:19:43,410 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:19:44,235 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:19:44,236 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:19:44,236 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:19:44,241 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:19:45,221 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:19:45,221 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:19:48,277 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:19:49,441 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:19:49,442 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:19:49,442 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:19:49,448 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:19:50,227 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:19:51,229 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:19:53,449 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:19:53,977 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:19:53,978 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:19:53,978 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:19:53,984 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:19:54,233 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:19:55,235 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:19:58,321 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:19:58,322 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:19:58,322 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:19:58,328 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:19:58,408 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:19:58,408 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:19:58,409 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:19:58,704 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:19:59,241 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:19:59,242 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:20:03,070 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:20:03,071 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:20:03,071 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:20:03,077 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:20:03,247 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:20:03,247 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:20:04,113 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:20:07,386 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:20:07,386 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:20:07,387 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:20:07,392 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:20:08,253 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:20:09,256 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:20:09,393 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:20:09,755 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 21:20:11,798 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:20:11,798 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:20:11,798 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:20:11,804 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:20:12,259 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:20:13,262 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:20:13,408 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:20:13,408 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:20:13,410 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:20:14,670 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:20:17,086 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:20:17,087 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:20:17,087 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:20:17,093 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:20:17,269 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:20:17,269 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:20:20,129 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:20:22,028 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:20:22,029 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:20:22,029 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:20:22,036 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:20:22,274 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:20:23,277 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:20:26,037 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:20:27,361 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:20:27,361 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:20:27,362 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:20:27,367 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:20:28,283 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:20:28,408 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:20:28,408 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:20:28,410 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:20:29,286 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:20:31,613 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:20:32,514 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:20:32,515 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:20:32,515 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:20:32,521 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:20:33,293 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:20:33,293 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:20:37,289 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:20:37,289 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:20:37,290 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:20:37,290 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:20:37,296 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:20:37,296 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:20:39,300 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:20:39,760 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 21:20:42,160 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:20:42,161 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:20:42,162 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:20:42,168 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:20:42,304 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:20:43,168 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:20:43,306 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:20:43,408 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:20:43,408 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:20:43,410 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:20:46,937 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:20:46,938 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:20:46,938 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:20:46,944 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:20:47,312 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:20:47,313 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:20:48,981 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:20:53,929 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:20:53,930 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:20:53,930 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:20:53,936 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:20:54,322 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:20:54,937 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:20:55,325 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:20:58,255 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:20:58,256 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:20:58,256 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:20:58,262 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:20:58,328 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:20:58,408 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:20:58,408 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:20:58,410 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:20:59,330 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:21:00,674 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:21:03,192 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:21:03,193 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:21:03,193 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:21:03,199 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:21:03,334 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:21:05,339 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:21:06,233 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:21:08,147 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:21:08,148 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:21:08,148 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:21:08,154 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:21:08,342 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:21:09,345 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:21:09,759 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 21:21:11,761 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:21:13,295 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:21:13,295 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:21:13,296 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:21:13,301 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:21:13,349 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:21:13,408 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:21:13,408 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:21:13,410 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:21:15,355 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:21:17,699 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:21:19,322 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:21:19,323 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:21:19,323 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:21:19,329 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:21:19,359 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:21:21,364 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:21:23,365 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:21:24,307 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:21:24,308 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:21:24,308 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:21:24,314 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:21:24,367 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:21:25,370 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:21:28,408 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:21:28,409 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:21:28,410 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:21:28,664 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:21:32,733 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:21:32,734 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:21:32,734 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:21:32,740 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:21:33,381 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:21:33,382 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:21:33,777 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:21:38,440 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:21:38,441 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:21:38,441 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:21:38,447 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:21:39,390 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:21:39,390 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:21:39,448 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:21:39,764 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 21:21:43,408 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:21:43,409 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:21:43,410 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:21:44,324 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:21:44,325 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:21:44,325 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:21:44,331 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:21:44,396 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:21:45,369 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:21:45,399 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:21:48,821 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:21:48,822 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:21:48,822 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:21:48,828 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:21:49,405 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:21:49,405 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:21:50,829 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:21:53,901 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:21:53,902 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:21:53,902 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:21:53,908 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:21:54,411 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:21:55,414 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:21:55,909 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:21:58,408 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:21:58,409 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:21:58,410 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:21:59,286 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:21:59,287 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:21:59,287 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:21:59,293 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:21:59,418 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:22:01,329 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:22:01,423 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:22:06,330 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:22:07,710 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:22:07,711 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:22:07,711 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:22:07,717 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:22:08,431 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:22:09,434 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:22:09,763 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 21:22:11,765 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:22:13,409 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:22:13,409 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:22:13,410 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:22:14,878 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:22:14,879 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:22:14,879 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:22:14,885 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:22:15,443 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:22:15,443 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:22:16,921 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:22:19,358 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:22:19,359 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:22:19,359 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:22:19,365 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:22:19,447 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:22:21,452 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:22:22,365 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:22:23,987 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:22:23,988 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:22:23,988 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:22:23,994 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:22:24,456 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:22:25,459 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:22:27,995 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:22:28,409 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:22:28,409 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:22:28,411 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:22:29,090 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:22:29,091 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:22:29,091 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:22:29,097 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:22:29,465 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:22:29,465 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:22:33,133 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:22:35,957 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:22:35,958 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:22:35,958 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:22:35,964 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:22:36,473 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:22:37,476 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:22:38,964 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:22:39,768 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 21:22:40,731 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:22:40,732 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:22:40,732 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:22:40,738 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:22:41,482 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:22:41,482 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:22:43,409 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:22:43,409 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:22:43,411 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:22:44,694 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:22:44,969 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:22:44,970 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:22:44,970 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:22:44,976 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:22:45,488 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:22:45,489 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:22:50,013 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:22:55,014 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:22:57,364 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:22:57,365 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:22:57,366 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:22:57,372 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:22:57,503 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:22:58,409 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:22:58,409 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:22:58,410 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:22:59,509 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:23:00,700 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:23:03,069 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:23:03,070 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:23:03,070 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:23:03,076 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:23:03,515 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:23:03,515 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:23:06,114 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:23:07,560 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:23:07,561 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:23:07,561 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:23:07,567 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:23:08,520 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:23:09,523 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:23:09,768 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 21:23:11,770 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:23:12,427 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:23:12,428 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:23:12,428 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:23:12,434 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:23:12,526 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:23:13,409 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:23:13,410 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:23:13,411 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:23:13,529 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:23:17,634 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:23:18,476 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:23:18,477 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:23:18,477 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:23:18,483 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:23:18,536 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:23:19,539 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:23:22,853 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:23:22,854 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:23:22,854 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:23:22,854 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:23:22,860 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:23:23,544 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:23:23,545 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:23:26,891 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:23:26,891 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:23:26,892 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:23:26,898 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:23:27,550 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:23:27,550 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:23:27,898 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:23:28,409 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:23:28,409 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:23:28,410 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:23:33,615 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:23:33,870 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:23:33,870 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:23:33,871 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:23:33,876 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:23:34,559 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:23:35,562 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:23:38,914 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:23:38,933 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:23:38,934 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:23:38,934 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:23:38,940 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:23:39,568 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:23:39,568 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:23:39,770 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 21:23:43,409 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:23:43,409 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:23:43,411 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:23:43,793 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:23:43,794 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:23:43,794 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:23:43,800 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:23:44,574 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:23:44,837 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:23:45,577 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:23:49,838 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:23:50,219 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:23:50,220 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:23:50,220 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:23:50,226 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:23:50,583 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:23:51,586 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:23:55,226 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:23:55,890 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:23:55,890 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:23:55,890 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:23:55,896 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:23:56,592 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:23:57,595 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:23:58,409 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:23:58,410 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:23:58,411 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:24:00,451 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:24:00,451 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:24:00,452 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:24:00,452 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:24:00,458 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:24:00,598 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:24:01,601 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:24:05,494 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:24:05,685 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:24:05,686 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:24:05,686 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:24:05,692 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:24:06,607 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:24:07,610 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:24:09,772 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 21:24:10,432 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:24:10,433 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:24:10,433 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:24:10,438 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:24:10,613 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:24:11,439 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:24:11,616 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:24:13,409 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:24:13,410 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:24:13,411 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:24:16,681 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:24:18,852 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:24:18,853 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:24:18,853 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:24:18,859 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:24:19,627 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:24:19,627 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:24:21,893 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:24:24,194 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:24:24,195 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:24:24,195 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:24:24,201 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:24:24,634 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:24:25,637 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:24:27,202 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:24:28,409 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:24:28,410 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:24:28,411 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:24:29,010 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:24:29,010 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:24:29,010 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:24:29,017 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:24:29,642 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:24:29,642 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:24:33,053 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:24:33,919 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:24:33,920 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:24:33,920 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:24:33,926 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:24:34,648 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:24:35,651 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:24:38,397 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:24:38,398 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:24:38,398 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:24:38,398 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:24:38,404 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:24:38,654 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:24:39,657 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:24:39,774 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 21:24:42,925 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:24:42,926 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:24:42,926 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:24:42,931 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:24:43,410 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:24:43,410 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:24:43,411 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:24:43,411 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:24:43,663 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:24:43,663 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:24:48,477 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:24:48,478 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:24:48,478 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:24:48,478 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:24:48,484 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:24:48,669 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:24:49,672 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:24:53,191 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:24:53,192 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:24:53,192 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:24:53,198 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:24:53,678 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:24:53,678 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:24:54,199 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:24:58,410 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:24:58,410 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:24:58,411 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:24:59,657 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:24:59,695 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:24:59,695 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:24:59,696 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:24:59,701 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:25:00,686 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:25:01,689 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:25:04,738 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:25:04,969 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:25:04,970 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:25:04,970 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:25:04,976 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:25:05,695 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:25:05,695 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:25:09,780 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 21:25:09,781 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:25:11,440 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:25:11,441 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:25:11,441 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:25:11,447 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:25:11,703 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:25:11,703 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:25:13,410 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:25:13,410 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:25:13,411 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:25:15,649 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:25:15,736 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:25:15,737 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:25:15,737 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:25:15,744 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:25:16,709 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:25:17,712 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:25:20,778 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:25:22,070 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:25:22,071 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:25:22,071 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:25:22,077 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:25:22,718 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:25:23,721 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:25:26,077 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:25:28,410 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:25:28,410 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:25:28,411 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:25:28,505 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:25:28,679 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:25:28,679 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:25:28,685 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:25:28,727 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:25:29,730 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:25:31,685 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:25:33,959 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:25:33,960 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:25:33,960 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:25:33,967 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:25:34,735 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:25:35,736 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:25:36,968 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:25:38,829 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:25:38,830 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:25:38,830 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:25:38,837 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:25:39,744 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:25:39,745 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:25:39,779 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 21:25:42,780 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:25:43,410 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:25:43,410 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:25:43,412 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:25:47,992 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:25:47,993 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:25:47,993 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:25:47,993 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:25:47,999 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:25:48,756 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:25:49,759 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:25:53,033 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:25:56,511 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:25:56,512 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:25:56,512 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:25:56,518 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:25:56,768 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:25:57,771 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:25:58,410 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:25:58,410 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:25:58,412 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:25:58,698 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:26:02,653 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:26:02,654 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:26:02,654 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:26:02,660 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:26:02,776 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:26:03,779 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:26:04,697 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:26:08,053 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:26:08,054 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:26:08,055 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:26:08,060 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:26:08,785 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:26:09,784 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 21:26:09,786 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:26:09,789 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:26:12,727 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:26:12,728 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:26:12,728 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:26:12,734 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:26:12,791 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:26:13,410 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:26:13,410 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:26:13,412 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:26:13,794 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:26:15,694 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:26:18,401 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:26:18,401 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:26:18,401 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:26:18,407 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:26:18,800 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:26:19,803 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:26:21,445 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:26:23,191 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:26:23,192 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:26:23,192 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:26:23,198 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:26:23,807 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:26:25,812 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:26:27,198 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:26:28,328 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:26:28,329 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:26:28,329 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:26:28,335 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:26:28,410 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:26:28,410 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:26:28,412 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:26:28,815 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:26:29,818 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:26:32,636 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:26:32,950 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:26:32,950 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:26:32,950 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:26:32,956 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:26:33,826 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:26:33,826 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:26:37,994 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:26:39,783 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 21:26:40,563 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:26:40,564 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:26:40,565 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:26:40,570 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:26:40,832 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:26:41,835 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:26:43,410 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:26:43,411 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:26:43,412 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:26:43,666 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:26:45,121 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:26:45,122 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:26:45,122 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:26:45,128 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:26:45,840 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:26:47,844 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:26:49,165 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:26:49,741 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:26:49,742 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:26:49,742 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:26:49,748 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:26:49,846 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:26:51,850 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:26:53,884 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:26:53,886 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:26:53,886 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:26:53,891 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:26:54,853 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:26:54,892 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:26:55,857 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:26:58,410 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:26:58,411 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:26:58,412 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:27:00,700 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:27:02,147 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:27:02,148 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:27:02,148 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:27:02,154 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:27:02,865 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:27:03,867 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:27:06,189 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:27:07,253 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:27:07,254 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:27:07,254 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:27:07,260 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:27:07,871 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:27:09,785 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 21:27:09,876 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:27:11,787 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:27:13,411 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:27:13,411 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:27:13,412 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:27:15,467 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:27:15,467 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:27:15,467 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:27:15,474 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:27:15,883 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:27:17,513 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:27:17,888 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:27:19,961 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:27:19,962 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:27:19,962 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:27:19,968 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:27:20,891 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:27:21,894 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:27:22,968 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:27:25,440 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:27:25,441 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:27:25,441 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:27:25,447 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:27:25,897 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:27:27,903 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:27:28,410 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:27:28,411 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:27:28,412 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:27:28,625 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:27:30,458 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:27:30,459 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:27:30,459 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:27:30,465 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:27:30,907 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:27:31,910 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:27:34,502 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:27:34,733 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:27:34,734 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:27:34,734 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:27:34,739 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:27:34,913 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:27:35,915 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:27:39,487 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:27:39,488 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:27:39,488 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:27:39,494 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:27:39,787 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 21:27:39,789 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:27:39,919 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:27:41,923 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:27:43,410 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:27:43,411 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:27:43,412 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:27:44,857 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:27:44,857 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:27:44,858 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:27:44,858 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:27:44,863 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:27:44,927 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:27:45,930 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:27:49,867 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:27:49,868 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:27:49,868 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:27:49,868 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:27:49,874 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:27:49,933 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:27:51,938 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:27:54,613 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:27:54,614 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:27:54,614 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:27:54,620 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:27:54,941 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:27:55,620 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:27:55,944 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:27:58,411 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:27:58,411 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:27:58,412 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:28:00,503 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:28:00,503 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:28:00,503 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:28:00,509 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:28:00,950 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:28:01,545 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:28:01,953 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:28:04,773 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:28:04,774 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:28:04,774 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:28:04,780 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:28:04,956 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:28:05,959 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:28:06,781 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:28:09,790 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 21:28:11,791 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:28:13,308 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:28:13,309 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:28:13,309 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:28:13,315 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:28:13,411 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:28:13,411 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:28:13,412 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:28:13,967 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:28:15,973 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:28:17,680 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:28:18,734 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:28:18,735 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:28:18,735 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:28:18,741 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:28:18,976 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:28:19,979 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:28:22,778 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:28:23,184 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:28:23,185 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:28:23,185 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:28:23,191 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:28:23,983 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:28:25,988 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:28:27,777 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:28:27,778 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:28:27,778 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:28:27,778 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:28:27,784 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:28:27,989 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:28:28,411 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:28:28,411 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:28:28,413 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:28:29,994 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:28:32,294 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:28:32,299 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:28:32,299 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:28:32,305 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:28:32,998 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:28:33,337 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:28:34,001 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:28:36,571 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:28:36,571 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:28:36,572 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:28:36,578 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:28:37,004 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:28:38,006 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:28:38,578 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:28:39,792 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 21:28:43,411 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:28:43,411 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:28:43,412 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:28:43,673 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:28:45,303 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:28:45,304 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:28:45,304 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:28:45,310 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:28:46,014 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:28:48,020 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:28:49,345 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:28:49,637 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:28:49,638 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:28:49,638 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:28:49,644 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:28:50,021 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:28:52,026 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:28:54,645 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:28:57,440 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:28:57,441 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:28:57,442 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:28:57,448 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:28:58,033 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:28:58,411 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:28:58,411 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:28:58,412 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:28:59,711 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:29:00,037 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:29:02,864 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:29:02,865 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:29:02,865 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:29:02,871 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:29:03,041 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:29:04,044 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:29:04,905 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:29:07,403 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:29:07,404 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:29:07,404 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:29:07,410 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:29:08,047 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:29:09,794 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 21:29:10,052 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:29:10,796 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:29:11,666 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:29:11,667 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:29:11,667 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:29:11,673 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:29:12,054 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:29:13,411 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:29:13,411 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:29:13,413 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:29:14,058 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:29:16,211 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:29:16,211 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:29:16,212 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:29:16,212 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:29:16,217 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:29:17,062 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:29:18,065 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:29:21,104 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:29:21,105 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:29:21,106 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:29:21,112 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:29:22,070 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:29:22,071 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:29:22,112 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:29:25,929 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:29:25,930 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:29:25,930 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:29:25,936 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:29:26,074 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:29:27,936 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:29:28,079 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:29:28,411 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:29:28,411 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:29:28,413 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:29:31,383 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:29:31,384 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:29:31,384 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:29:31,390 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:29:32,084 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:29:33,425 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:29:34,088 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:29:35,911 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:29:35,912 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:29:35,912 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:29:35,918 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:29:36,090 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:29:38,095 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:29:38,919 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:29:39,799 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 21:29:40,537 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:29:40,538 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:29:40,538 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:29:40,544 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:29:41,097 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:29:42,100 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:29:43,411 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:29:43,411 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:29:43,413 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:29:44,666 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:29:45,275 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:29:45,275 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:29:45,276 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:29:45,281 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:29:46,104 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:29:48,109 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:29:49,533 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:29:49,534 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:29:49,534 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:29:49,540 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:29:50,110 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:29:50,540 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:29:52,115 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:29:55,482 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:29:55,482 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:29:55,483 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:29:55,488 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:29:56,118 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:29:56,489 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:29:58,124 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:29:58,411 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:29:58,411 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:29:58,413 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:30:01,677 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:30:04,047 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:30:04,048 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:30:04,048 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:30:04,054 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:30:04,130 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:30:06,135 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:30:07,089 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:30:09,798 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 21:30:10,338 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:30:10,339 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:30:10,339 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:30:10,345 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:30:11,141 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:30:12,144 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:30:12,346 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:30:13,411 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:30:13,412 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:30:13,413 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:30:16,272 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:30:16,272 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:30:16,273 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:30:16,279 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:30:17,150 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:30:18,151 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:30:18,313 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:30:21,183 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:30:21,184 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:30:21,184 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:30:21,191 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:30:22,159 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:30:22,159 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:30:24,191 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:30:27,315 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:30:27,316 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:30:27,316 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:30:27,322 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:30:28,165 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:30:28,411 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:30:28,412 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:30:28,413 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:30:29,708 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:30:30,171 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:30:34,075 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:30:34,076 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:30:34,077 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:30:34,082 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:30:34,176 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:30:35,117 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:30:36,180 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:30:39,804 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 21:30:40,805 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:30:42,129 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:30:42,130 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:30:42,130 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:30:42,136 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:30:42,186 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:30:43,411 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:30:43,412 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:30:43,413 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:30:44,191 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:30:46,703 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:30:48,131 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:30:48,131 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:30:48,131 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:30:48,137 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:30:48,195 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:30:50,200 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:30:52,173 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:30:53,680 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:30:53,681 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:30:53,681 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:30:53,687 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:30:54,204 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:30:56,209 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:30:57,688 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:30:58,411 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:30:58,412 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:30:58,413 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:31:00,499 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:31:00,499 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:31:00,499 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:31:00,506 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:31:01,215 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:31:02,218 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:31:03,541 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:31:05,212 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:31:05,212 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:31:05,213 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:31:05,220 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:31:05,221 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:31:06,224 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:31:09,221 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:31:09,803 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 21:31:10,063 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:31:10,064 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:31:10,064 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:31:10,070 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:31:10,227 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:31:12,232 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:31:13,412 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:31:13,412 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:31:13,413 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:31:14,636 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:31:15,199 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:31:15,200 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:31:15,200 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:31:15,207 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:31:15,236 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:31:16,239 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:31:20,242 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:31:23,223 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:31:23,224 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:31:23,224 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:31:23,230 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:31:23,248 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:31:24,250 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:31:26,231 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:31:28,110 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:31:28,111 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:31:28,111 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:31:28,116 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:31:28,254 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:31:28,994 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:31:28,994 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:31:28,994 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:31:30,259 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:31:31,262 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:31:33,381 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:31:33,381 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:31:33,382 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:31:33,388 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:31:34,264 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:31:36,268 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:31:36,425 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:31:38,141 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:31:38,142 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:31:38,142 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:31:38,148 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:31:38,271 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:31:39,808 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 21:31:40,274 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:31:41,810 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:31:43,762 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:31:43,763 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:31:43,763 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:31:43,769 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:31:43,993 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:31:43,993 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:31:43,995 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:31:44,278 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:31:46,284 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:31:47,283 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:31:52,225 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:31:52,225 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:31:52,226 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:31:52,231 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:31:52,290 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:31:53,269 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:31:54,295 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:31:56,652 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:31:56,653 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:31:56,653 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:31:56,659 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:31:57,298 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:31:58,301 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:31:58,659 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:31:58,993 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:31:58,993 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:31:58,995 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:32:03,314 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:32:03,315 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:32:03,315 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:32:03,321 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:32:04,309 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:32:04,310 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:32:04,357 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:32:08,334 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:32:08,335 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:32:08,335 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:32:08,341 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:32:09,315 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:32:09,807 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 21:32:09,809 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:32:10,318 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:32:13,421 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:32:13,422 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:32:13,422 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:32:13,428 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:32:13,993 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:32:13,994 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:32:13,995 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:32:14,325 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:32:14,325 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:32:15,288 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:32:17,973 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:32:17,973 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:32:17,974 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:32:17,979 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:32:18,328 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:32:20,334 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:32:21,017 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:32:24,332 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:32:24,333 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:32:24,333 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:32:24,339 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:32:24,339 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:32:26,340 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:32:26,343 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:32:28,638 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:32:28,639 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:32:28,639 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:32:28,646 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:32:28,993 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:32:28,994 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:32:28,995 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:32:29,345 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:32:30,348 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:32:32,282 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:32:33,149 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:32:33,149 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:32:33,150 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:32:33,156 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:32:33,353 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:32:34,356 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:32:38,194 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:32:39,809 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 21:32:41,633 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:32:41,634 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:32:41,634 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:32:41,640 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:32:42,363 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:32:43,640 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:32:43,993 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:32:43,994 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:32:43,995 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:32:44,368 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:32:47,009 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:32:47,010 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:32:47,010 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:32:47,016 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:32:47,371 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:32:48,374 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:32:49,053 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:32:51,970 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:32:51,971 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:32:51,971 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:32:51,978 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:32:52,378 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:32:54,383 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:32:54,978 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:32:58,993 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:32:58,994 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:32:58,995 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:33:00,271 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:33:00,551 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:33:00,552 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:33:00,552 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:33:00,558 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:33:01,392 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:33:02,395 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:33:05,126 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:33:05,127 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:33:05,127 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:33:05,133 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:33:05,397 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:33:06,133 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:33:06,400 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:33:09,481 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:33:09,482 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:33:09,482 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:33:09,488 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:33:09,815 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 21:33:10,406 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:33:10,407 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:33:11,816 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:33:13,994 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:33:13,994 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:33:13,995 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:33:14,340 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:33:14,340 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:33:14,340 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:33:14,347 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:33:14,410 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:33:16,415 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:33:17,381 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:33:21,254 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:33:21,255 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:33:21,256 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:33:21,261 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:33:21,421 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:33:22,424 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:33:23,262 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:33:26,493 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:33:26,494 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:33:26,494 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:33:26,500 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:33:27,429 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:33:28,432 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:33:28,501 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:33:28,994 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:33:28,994 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:33:28,995 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:33:31,610 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:33:31,611 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:33:31,611 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:33:31,617 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:33:32,437 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:33:33,653 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:33:34,442 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:33:35,884 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:33:35,885 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:33:35,886 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:33:35,891 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:33:36,443 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:33:38,448 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:33:38,892 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:33:39,813 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 21:33:40,182 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:33:40,183 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:33:40,183 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:33:40,189 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:33:40,450 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:33:42,454 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:33:43,994 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:33:43,994 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:33:43,995 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:33:44,212 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:33:44,707 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:33:44,708 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:33:44,708 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:33:44,714 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:33:45,457 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:33:46,460 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:33:49,749 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:33:51,492 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:33:51,492 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:33:51,493 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:33:51,499 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:33:52,468 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:33:52,468 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:33:55,289 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:33:55,290 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:33:55,290 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:33:55,290 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:33:55,296 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:33:55,470 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:33:56,473 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:33:58,994 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:33:58,994 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:33:58,996 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:34:01,094 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:34:01,095 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:34:01,095 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:34:01,096 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:34:01,102 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:34:01,479 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:34:02,482 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:34:06,138 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:34:09,604 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:34:09,605 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:34:09,605 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:34:09,611 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:34:09,819 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 21:34:10,491 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:34:11,820 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:34:12,496 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:34:13,923 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:34:13,924 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:34:13,924 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:34:13,929 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:34:13,994 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:34:13,994 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:34:13,995 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:34:14,497 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:34:16,502 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:34:17,202 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:34:17,947 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:34:17,948 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:34:17,948 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:34:17,954 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:34:18,504 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:34:20,508 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:34:22,990 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:34:24,877 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:34:24,878 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:34:24,878 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:34:24,883 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:34:25,514 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:34:26,517 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:34:28,884 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:34:28,994 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:34:28,994 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:34:28,996 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:34:29,718 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:34:29,719 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:34:29,719 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:34:29,725 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:34:30,520 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:34:32,527 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:34:34,761 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:34:37,521 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:34:37,522 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:34:37,522 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:34:37,530 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:34:37,531 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:34:38,534 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:34:39,818 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 21:34:39,819 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:34:42,020 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:34:42,020 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:34:42,021 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:34:42,026 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:34:42,537 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:34:43,994 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:34:43,994 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:34:43,996 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:34:44,543 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:34:45,201 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:34:46,465 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:34:46,465 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:34:46,465 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:34:46,471 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:34:46,545 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:34:48,549 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:34:50,510 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:34:50,873 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:34:50,874 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:34:50,874 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:34:50,880 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:34:51,552 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:34:52,555 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:34:55,881 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:34:58,994 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:34:58,994 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:34:58,996 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:34:59,220 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:34:59,253 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:34:59,253 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:34:59,259 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:34:59,563 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:35:00,566 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:35:01,260 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:35:04,020 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:35:04,021 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:35:04,021 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:35:04,028 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:35:04,569 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:35:06,575 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:35:07,061 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:35:09,014 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:35:09,015 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:35:09,015 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:35:09,021 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:35:09,578 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:35:09,823 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 21:35:10,580 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:35:12,824 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:35:13,994 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:35:13,994 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:35:13,996 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:35:15,516 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:35:15,517 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:35:15,517 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:35:15,523 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:35:15,586 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:35:16,589 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:35:18,557 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:35:21,015 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:35:21,016 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:35:21,016 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:35:21,022 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:35:21,596 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:35:22,599 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:35:24,022 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:35:26,054 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:35:26,055 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:35:26,055 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:35:26,061 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:35:26,602 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:35:28,607 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:35:28,994 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:35:28,995 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:35:28,996 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:35:29,255 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:35:30,863 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:35:30,863 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:35:30,864 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:35:30,870 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:35:31,612 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:35:32,615 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:35:34,905 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:35:36,089 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:35:36,090 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:35:36,090 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:35:36,096 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:35:36,619 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:35:38,623 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:35:39,822 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 21:35:40,824 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:35:42,229 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:35:42,230 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:35:42,230 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:35:42,236 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:35:42,627 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:35:43,994 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:35:43,995 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:35:43,996 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:35:44,631 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:35:46,695 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:35:46,759 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:35:46,759 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:35:46,759 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:35:46,765 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:35:47,655 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:35:48,658 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:35:51,107 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:35:51,108 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:35:51,108 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:35:51,118 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:35:51,671 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:35:52,118 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:35:52,674 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:35:55,149 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:35:55,150 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:35:55,150 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:35:55,156 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:35:55,678 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:35:56,682 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:35:57,157 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:35:58,994 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:35:58,995 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:35:58,996 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:36:01,711 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:36:01,712 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:36:01,712 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:36:01,718 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:36:02,687 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:36:02,753 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:36:04,691 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:36:07,165 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:36:07,166 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:36:07,166 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:36:07,172 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:36:07,693 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:36:08,172 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:36:08,696 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:36:09,824 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 21:36:11,881 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:36:11,882 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:36:11,882 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:36:11,888 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:36:12,699 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:36:13,889 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:36:13,994 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:36:13,995 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:36:13,996 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:36:14,703 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:36:18,054 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:36:18,055 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:36:18,055 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:36:18,060 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:36:18,706 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:36:19,097 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:36:20,710 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:36:23,010 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:36:23,011 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:36:23,011 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:36:23,018 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:36:23,712 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:36:24,715 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:36:25,018 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:36:28,143 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:36:28,144 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:36:28,144 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:36:28,150 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:36:28,718 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:36:28,995 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:36:28,995 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:36:28,996 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:36:30,207 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:36:30,721 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:36:34,042 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:36:34,042 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:36:34,042 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:36:34,049 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:36:34,726 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:36:36,085 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:36:36,729 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:36:38,259 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:36:38,260 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:36:38,260 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:36:38,267 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:36:38,731 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:36:39,826 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 21:36:40,735 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:36:41,829 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:36:43,506 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:36:43,506 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:36:43,507 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:36:43,512 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:36:43,738 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:36:43,995 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:36:43,995 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:36:43,996 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:36:44,741 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:36:47,206 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:36:49,393 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:36:49,393 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:36:49,393 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:36:49,399 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:36:49,746 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:36:50,749 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:36:52,437 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:36:54,100 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:36:54,101 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:36:54,101 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:36:54,108 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:36:54,751 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:36:56,755 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:36:58,109 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:36:58,995 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:36:58,995 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:36:58,997 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:36:59,109 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:36:59,288 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:36:59,288 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:36:59,294 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:36:59,758 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:37:00,761 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:37:03,295 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:37:04,371 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:37:04,372 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:37:04,372 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:37:04,378 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:37:04,764 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:37:06,769 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:37:08,379 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:37:08,463 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:37:08,464 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:37:08,465 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:37:08,471 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:37:08,771 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:37:09,829 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 21:37:10,776 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:37:13,470 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:37:13,471 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:37:13,472 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:37:13,472 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:37:13,477 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:37:13,779 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:37:13,995 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:37:13,995 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:37:13,996 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:37:14,782 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:37:18,368 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:37:18,368 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:37:18,369 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:37:18,375 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:37:18,785 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:37:19,409 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:37:20,792 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:37:22,398 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:37:22,400 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:37:22,400 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:37:22,406 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:37:22,794 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:37:24,798 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:37:25,406 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:37:27,863 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:37:27,867 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:37:27,867 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:37:27,873 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:37:28,801 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:37:28,995 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:37:28,995 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:37:28,996 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:37:30,805 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:37:31,251 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:37:32,395 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:37:32,396 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:37:32,396 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:37:32,402 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:37:32,808 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:37:34,812 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:37:36,437 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:37:39,154 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:37:39,155 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:37:39,155 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:37:39,161 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:37:39,816 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:37:39,831 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 21:37:40,820 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:37:41,832 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:37:43,995 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:37:43,995 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:37:43,997 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:37:44,684 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:37:44,685 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:37:44,685 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:37:44,691 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:37:44,823 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:37:46,827 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:37:47,729 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:37:49,441 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:37:49,442 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:37:49,442 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:37:49,448 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:37:49,829 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:37:50,832 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:37:53,448 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:37:56,037 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:37:56,038 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:37:56,038 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:37:56,044 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:37:56,837 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:37:58,840 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:37:58,995 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:37:58,995 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:37:58,997 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:37:59,274 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:38:01,178 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:38:01,178 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:38:01,178 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:38:01,184 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:38:01,843 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:38:02,845 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:38:05,221 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:38:06,673 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:38:06,673 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:38:06,673 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:38:06,679 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:38:06,848 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:38:08,852 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:38:09,833 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 21:38:10,835 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:38:11,686 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:38:11,687 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:38:11,687 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:38:11,693 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:38:11,855 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:38:12,858 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:38:13,995 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:38:13,995 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:38:13,997 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:38:16,256 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:38:16,257 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:38:16,257 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:38:16,257 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:38:16,263 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:38:16,861 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:38:18,864 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:38:20,451 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:38:20,452 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:38:20,452 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:38:20,458 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:38:20,866 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:38:21,458 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:38:22,870 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:38:26,459 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:38:26,826 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:38:26,826 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:38:26,827 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:38:26,833 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:38:26,873 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:38:28,877 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:38:28,995 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:38:28,996 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:38:28,997 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:38:30,930 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:38:30,930 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:38:30,930 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:38:30,937 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:38:31,880 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:38:31,973 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:38:32,883 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:38:36,974 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:38:39,838 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 21:38:39,846 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:38:39,846 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:38:39,846 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:38:39,852 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:38:39,889 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:38:40,893 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:38:42,853 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:38:43,995 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:38:43,996 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:38:43,997 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:38:45,411 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:38:45,412 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:38:45,412 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:38:45,418 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:38:45,897 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:38:46,900 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:38:48,453 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:38:50,050 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:38:50,051 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:38:50,051 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:38:50,057 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:38:50,904 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:38:52,908 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:38:54,057 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:38:54,309 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:38:54,310 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:38:54,310 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:38:54,316 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:38:54,909 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:38:56,913 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:38:58,995 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:38:58,996 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:38:58,997 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:38:59,307 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:39:02,551 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:39:02,552 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:39:02,552 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:39:02,558 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:39:02,917 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:39:04,593 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:39:04,921 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:39:07,802 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:39:07,802 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:39:07,803 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:39:07,808 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:39:07,924 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:39:08,927 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:39:09,809 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:39:09,837 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 21:39:12,383 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:39:12,383 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:39:12,384 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:39:12,390 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:39:12,930 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:39:13,995 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:39:13,996 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:39:13,997 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:39:14,934 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:39:15,275 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:39:16,769 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:39:16,769 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:39:16,770 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:39:16,775 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:39:16,936 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:39:18,940 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:39:20,813 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:39:21,823 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:39:21,824 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:39:21,824 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:39:21,829 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:39:21,942 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:39:22,945 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:39:25,830 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:39:26,362 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:39:26,363 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:39:26,364 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:39:26,370 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:39:26,948 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:39:28,952 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:39:28,995 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:39:28,996 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:39:28,997 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:39:31,198 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:39:31,259 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:39:31,259 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:39:31,259 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:39:31,265 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:39:31,957 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:39:32,960 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:39:36,301 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:39:36,973 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:39:36,974 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:39:36,974 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:39:36,980 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:39:37,965 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:39:38,968 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:39:39,842 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 21:39:41,237 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:39:41,237 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:39:41,238 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:39:41,243 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:39:41,971 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:39:42,244 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:39:42,974 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:39:43,996 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:39:43,996 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:39:43,997 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:39:45,122 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:39:45,122 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:39:45,123 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:39:45,129 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:39:45,977 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:39:46,980 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:39:48,165 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:39:51,374 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:39:51,375 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:39:51,375 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:39:51,381 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:39:51,984 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:39:52,987 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:39:53,381 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:39:55,921 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:39:55,922 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:39:55,922 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:39:55,927 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:39:55,990 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:39:56,993 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:39:58,928 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:39:58,996 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:39:58,996 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:39:58,997 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:40:01,741 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:40:01,741 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:40:01,741 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:40:01,747 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:40:01,998 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:40:03,001 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:40:04,785 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:40:08,121 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:40:08,122 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:40:08,122 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:40:08,128 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:40:09,007 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:40:09,841 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 21:40:09,843 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:40:11,012 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:40:13,600 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:40:13,601 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:40:13,602 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:40:13,607 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:40:13,996 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:40:13,996 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:40:13,997 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:40:14,015 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:40:15,018 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:40:15,215 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:40:18,217 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:40:18,217 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:40:18,218 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:40:18,223 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:40:19,022 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:40:20,261 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:40:21,027 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:40:22,939 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:40:22,940 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:40:22,940 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:40:22,946 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:40:23,028 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:40:25,033 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:40:25,947 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:40:28,457 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:40:28,458 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:40:28,458 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:40:28,464 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:40:28,996 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:40:28,996 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:40:28,997 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:40:29,036 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:40:31,041 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:40:31,234 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:40:35,195 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:40:35,196 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:40:35,196 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:40:35,202 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:40:36,047 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:40:36,237 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:40:37,050 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:40:39,847 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 21:40:41,755 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:40:41,756 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:40:41,756 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:40:41,757 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:40:41,762 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:40:42,055 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:40:43,058 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:40:43,996 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:40:43,996 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:40:43,998 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:40:46,208 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:40:46,209 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:40:46,209 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:40:46,215 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:40:47,062 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:40:47,249 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:40:49,067 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:40:52,250 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:40:53,214 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:40:53,215 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:40:53,215 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:40:53,221 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:40:54,072 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:40:55,075 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:40:58,222 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:40:58,996 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:40:58,996 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:40:58,998 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:41:01,201 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:41:01,202 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:41:01,202 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:41:01,208 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:41:02,083 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:41:03,086 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:41:03,245 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:41:05,938 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:41:05,939 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:41:05,939 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:41:05,945 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:41:06,089 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:41:07,091 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:41:08,946 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:41:09,846 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 21:41:10,790 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:41:10,791 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:41:10,791 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:41:10,797 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:41:11,095 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:41:13,100 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:41:13,996 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:41:13,996 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:41:13,998 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:41:14,219 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:41:14,858 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:41:14,858 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:41:14,858 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:41:14,864 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:41:15,102 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:41:17,107 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:41:19,846 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:41:19,847 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:41:19,847 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:41:19,847 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:41:19,853 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:41:20,110 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:41:21,113 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:41:24,854 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:41:26,568 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:41:26,569 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:41:26,570 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:41:26,576 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:41:27,119 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:41:28,996 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:41:28,996 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:41:28,998 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:41:29,122 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:41:30,258 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:41:31,393 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:41:31,394 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:41:31,394 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:41:31,399 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:41:32,128 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:41:33,131 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:41:35,437 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:41:36,256 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:41:36,257 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:41:36,258 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:41:36,263 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:41:37,136 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:41:39,140 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:41:39,848 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 21:41:40,849 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:41:41,534 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:41:41,534 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:41:41,535 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:41:41,541 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:41:42,143 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:41:43,147 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:41:43,996 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:41:43,997 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:41:43,998 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:41:46,282 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:41:47,129 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:41:47,130 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:41:47,130 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:41:47,135 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:41:47,150 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:41:49,154 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:41:52,174 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:41:52,251 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:41:52,252 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:41:52,252 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:41:52,257 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:41:53,157 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:41:55,161 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:41:56,951 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:41:56,955 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:41:56,955 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:41:56,961 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:41:57,162 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:41:57,962 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:41:58,996 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:41:58,997 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:41:58,998 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:41:59,166 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:42:01,939 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:42:01,940 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:42:01,940 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:42:01,946 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:42:02,169 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:42:02,981 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:42:03,172 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:42:06,773 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:42:06,774 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:42:06,774 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:42:06,780 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:42:07,175 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:42:08,780 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:42:09,180 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:42:09,850 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 21:42:11,277 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:42:11,278 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:42:11,278 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:42:11,284 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:42:12,184 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:42:13,187 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:42:13,996 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:42:13,997 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:42:13,998 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:42:14,278 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:42:18,307 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:42:18,308 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:42:18,308 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:42:18,314 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:42:19,193 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:42:19,349 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:42:21,198 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:42:23,018 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:42:23,019 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:42:23,020 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:42:23,025 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:42:23,200 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:42:25,026 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:42:25,204 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:42:28,506 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:42:28,507 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:42:28,507 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:42:28,513 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:42:28,997 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:42:28,997 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:42:28,998 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:42:29,207 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:42:30,300 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:42:31,212 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:42:33,695 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:42:33,695 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:42:33,696 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:42:33,702 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:42:34,216 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:42:35,219 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:42:35,737 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:42:38,491 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:42:38,492 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:42:38,493 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:42:38,498 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:42:39,223 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:42:39,853 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 21:42:40,854 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:42:41,226 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:42:43,890 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:42:43,891 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:42:43,891 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:42:43,897 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:42:43,997 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:42:43,997 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:42:43,998 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:42:44,229 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:42:45,232 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:42:46,267 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:42:48,780 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:42:48,780 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:42:48,781 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:42:48,787 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:42:49,236 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:42:51,240 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:42:51,821 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:42:54,230 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:42:54,230 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:42:54,231 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:42:54,236 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:42:54,243 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:42:57,237 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:42:57,249 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:42:58,607 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:42:58,608 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:42:58,608 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:42:58,614 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:42:58,997 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:42:58,997 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:42:58,998 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:42:59,251 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:43:01,255 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:43:02,284 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:43:03,833 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:43:03,834 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:43:03,834 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:43:03,840 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:43:04,258 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:43:05,261 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:43:07,878 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:43:08,401 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:43:08,402 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:43:08,402 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:43:08,408 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:43:09,265 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:43:09,855 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 21:43:11,267 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:43:13,856 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:43:13,997 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:43:13,997 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:43:13,998 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:43:14,509 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:43:14,510 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:43:14,510 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:43:14,516 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:43:15,273 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:43:17,278 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:43:19,554 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:43:19,710 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:43:19,711 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:43:19,711 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:43:19,717 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:43:20,281 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:43:21,284 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:43:24,581 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:43:24,582 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:43:24,582 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:43:24,583 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:43:24,588 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:43:25,288 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:43:27,292 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:43:28,997 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:43:28,997 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:43:28,998 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:43:29,074 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:43:29,221 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:43:29,221 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:43:29,227 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:43:29,294 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:43:30,227 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:43:31,299 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:43:35,228 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:43:36,478 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:43:36,479 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:43:36,479 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:43:36,484 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:43:37,306 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:43:39,310 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:43:39,857 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 21:43:40,858 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:43:41,170 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:43:41,171 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:43:41,171 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:43:41,177 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:43:41,312 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:43:43,316 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:43:43,997 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:43:43,997 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:43:43,998 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:43:46,207 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:43:50,157 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:43:50,158 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:43:50,158 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:43:50,164 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:43:50,325 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:43:51,328 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:43:52,201 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:43:54,633 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:43:54,634 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:43:54,634 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:43:54,640 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:43:55,332 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:43:57,337 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:43:57,640 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:43:58,997 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:43:58,997 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:43:58,998 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:43:59,761 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:43:59,762 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:43:59,762 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:43:59,767 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:44:00,340 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:44:01,343 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:44:02,805 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:44:05,369 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:44:05,370 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:44:05,370 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:44:05,376 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:44:06,349 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:44:07,352 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:44:08,377 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:44:09,862 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 21:44:10,980 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:44:10,981 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:44:10,981 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:44:10,987 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:44:11,356 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:44:13,361 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:44:13,987 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:44:13,997 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:44:13,997 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:44:13,998 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:44:19,227 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:44:19,349 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:44:19,349 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:44:19,349 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:44:19,355 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:44:19,366 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:44:21,371 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:44:23,766 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:44:23,767 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:44:23,767 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:44:23,773 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:44:24,374 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:44:24,773 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:44:25,377 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:44:28,531 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:44:28,532 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:44:28,532 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:44:28,538 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:44:28,997 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:44:28,997 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:44:28,999 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:44:29,380 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:44:30,255 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:44:31,385 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:44:33,177 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:44:33,178 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:44:33,178 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:44:33,184 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:44:33,388 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:44:35,392 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:44:36,221 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:44:37,448 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:44:37,449 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:44:37,449 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:44:37,455 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:44:38,395 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:44:39,398 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:44:39,861 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 21:44:41,863 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:44:43,997 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:44:43,997 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:44:43,999 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:44:45,498 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:44:45,499 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:44:45,499 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:44:45,505 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:44:46,406 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:44:47,409 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:44:47,541 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:44:50,613 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:44:50,614 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:44:50,614 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:44:50,620 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:44:51,412 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:44:52,620 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:44:53,417 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:44:55,791 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:44:55,792 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:44:55,792 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:44:55,798 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:44:56,420 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:44:57,424 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:44:57,798 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:44:58,997 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:44:58,998 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:44:58,999 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:45:03,325 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:45:04,238 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:45:04,239 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:45:04,239 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:45:04,245 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:45:04,432 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:45:05,434 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:45:08,954 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:45:08,955 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:45:08,956 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:45:08,956 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:45:08,961 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:45:09,439 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:45:09,866 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 21:45:11,443 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:45:13,935 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:45:13,936 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:45:13,936 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:45:13,942 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:45:13,997 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:45:13,997 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:45:13,999 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:45:14,236 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:45:14,446 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:45:15,449 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:45:18,386 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:45:18,386 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:45:18,386 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:45:18,392 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:45:18,452 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:45:19,429 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:45:21,458 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:45:24,430 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:45:26,264 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:45:26,265 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:45:26,266 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:45:26,271 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:45:26,464 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:45:27,467 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:45:28,997 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:45:28,998 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:45:28,999 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:45:30,209 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:45:31,284 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:45:31,284 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:45:31,284 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:45:31,290 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:45:31,470 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:45:33,478 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:45:35,325 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:45:35,928 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:45:35,929 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:45:35,929 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:45:35,935 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:45:36,481 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:45:37,483 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:45:39,865 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 21:45:40,866 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:45:42,347 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:45:42,348 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:45:42,348 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:45:42,354 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:45:42,488 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:45:43,997 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:45:43,998 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:45:43,999 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:45:45,493 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:45:46,286 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:45:47,031 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:45:47,032 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:45:47,032 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:45:47,037 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:45:47,495 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:45:49,500 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:45:52,073 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:45:52,513 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:45:52,514 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:45:52,514 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:45:52,521 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:45:53,503 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:45:55,508 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:45:56,377 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:45:56,378 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:45:56,378 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:45:56,384 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:45:56,508 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:45:57,385 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:45:58,998 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:45:58,998 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:45:58,999 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:45:59,514 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:46:00,829 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:46:00,829 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:46:00,829 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:46:00,835 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:46:01,515 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:46:02,873 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:46:03,520 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:46:05,991 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:46:05,992 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:46:05,992 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:46:05,998 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:46:06,523 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:46:07,525 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:46:07,999 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:46:09,867 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 21:46:10,709 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:46:10,710 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:46:10,710 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:46:10,716 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:46:11,529 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:46:13,534 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:46:13,717 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:46:13,998 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:46:13,998 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:46:13,999 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:46:15,910 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:46:15,911 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:46:15,911 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:46:15,917 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:46:16,538 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:46:17,540 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:46:18,953 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:46:21,153 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:46:21,154 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:46:21,154 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:46:21,160 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:46:21,544 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:46:23,549 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:46:24,160 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:46:26,186 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:46:26,187 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:46:26,187 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:46:26,193 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:46:26,552 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:46:27,555 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:46:28,998 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:46:28,998 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:46:28,999 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:46:29,228 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:46:30,694 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:46:30,695 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:46:30,695 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:46:30,701 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:46:31,559 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:46:33,566 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:46:34,737 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:46:35,755 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:46:35,755 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:46:35,756 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:46:35,761 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:46:36,568 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:46:37,571 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:46:39,762 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:46:39,870 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 21:46:40,045 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:46:40,046 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:46:40,046 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:46:40,052 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:46:40,574 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:46:41,577 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:46:43,998 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:46:43,998 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:46:43,999 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:46:45,219 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:46:47,065 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:46:47,065 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:46:47,065 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:46:47,071 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:46:47,582 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:46:49,588 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:46:51,109 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:46:51,764 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:46:51,765 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:46:51,765 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:46:51,771 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:46:52,591 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:46:53,594 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:46:56,771 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:46:58,998 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:46:58,998 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:46:58,999 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:47:00,326 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:47:00,326 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:47:00,326 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:47:00,332 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:47:00,602 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:47:01,605 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:47:02,369 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:47:05,337 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:47:05,338 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:47:05,338 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:47:05,344 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:47:05,609 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:47:07,614 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:47:08,344 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:47:09,530 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:47:09,531 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:47:09,531 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:47:09,536 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:47:09,615 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:47:09,872 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 21:47:11,620 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:47:13,874 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:47:13,998 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:47:13,998 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:47:13,999 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:47:15,906 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:47:15,907 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:47:15,907 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:47:15,913 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:47:16,625 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:47:17,628 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:47:18,949 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:47:20,547 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:47:20,548 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:47:20,548 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:47:20,554 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:47:20,631 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:47:23,637 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:47:24,555 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:47:25,978 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:47:25,979 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:47:25,979 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:47:25,985 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:47:26,640 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:47:27,641 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:47:28,998 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:47:28,998 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:47:28,999 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:47:30,222 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:47:30,612 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:47:30,613 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:47:30,613 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:47:30,619 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:47:30,646 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:47:33,654 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:47:35,651 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:47:35,652 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:47:35,653 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:47:35,653 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:47:35,659 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:47:35,659 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:47:37,662 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:47:39,877 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 21:47:40,214 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:47:40,215 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:47:40,215 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:47:40,221 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:47:40,664 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:47:41,221 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:47:41,668 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:47:43,998 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:47:43,998 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:47:43,999 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:47:45,573 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:47:45,573 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:47:45,574 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:47:45,580 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:47:45,671 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:47:46,617 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:47:47,675 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:47:50,381 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:47:50,382 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:47:50,382 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:47:50,388 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:47:50,677 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:47:51,680 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:47:52,388 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:47:57,388 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:47:58,334 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:47:58,335 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:47:58,335 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:47:58,340 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:47:58,687 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:47:58,998 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:47:58,998 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:47:59,000 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:47:59,690 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:48:03,088 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:48:03,089 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:48:03,089 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:48:03,090 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:48:03,095 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:48:03,693 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:48:05,698 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:48:08,130 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:48:08,156 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:48:08,157 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:48:08,157 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:48:08,164 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:48:08,702 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:48:09,705 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:48:09,876 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 21:48:12,791 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:48:12,791 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:48:12,792 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:48:12,797 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:48:13,708 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:48:13,798 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:48:13,998 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:48:13,999 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:48:14,000 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:48:15,713 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:48:18,299 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:48:18,299 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:48:18,300 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:48:18,305 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:48:18,717 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:48:19,341 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:48:19,719 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:48:24,175 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:48:24,176 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:48:24,176 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:48:24,182 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:48:24,725 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:48:25,182 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:48:25,728 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:48:28,998 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:48:28,999 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:48:29,000 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:48:29,188 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:48:29,267 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:48:29,268 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:48:29,273 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:48:29,733 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:48:30,274 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:48:31,737 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:48:35,274 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:48:38,379 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:48:38,380 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:48:38,380 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:48:38,386 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:48:38,746 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:48:39,749 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:48:39,882 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 21:48:40,883 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:48:43,998 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:48:43,999 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:48:44,000 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:48:46,266 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:48:46,952 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:48:46,952 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:48:46,953 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:48:46,958 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:48:47,759 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:48:49,764 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:48:51,994 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:48:52,276 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:48:52,276 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:48:52,277 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:48:52,282 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:48:52,767 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:48:53,770 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:48:56,806 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:48:56,807 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:48:56,807 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:48:56,814 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:48:57,774 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:48:57,814 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:48:58,998 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:48:58,999 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:48:59,000 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:48:59,779 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:49:01,120 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:49:01,120 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:49:01,121 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:49:01,126 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:49:01,780 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:49:03,161 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:49:03,785 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:49:07,010 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:49:07,011 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:49:07,011 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:49:07,017 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:49:07,788 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:49:09,018 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:49:09,791 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:49:09,881 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 21:49:11,385 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:49:11,386 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:49:11,387 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:49:11,392 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:49:11,794 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:49:13,800 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:49:13,999 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:49:13,999 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:49:14,000 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:49:14,279 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:49:15,663 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:49:15,664 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:49:15,664 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:49:15,670 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:49:15,801 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:49:17,806 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:49:19,706 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:49:20,576 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:49:20,577 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:49:20,577 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:49:20,585 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:49:20,809 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:49:21,812 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:49:25,586 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:49:25,933 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:49:25,933 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:49:25,934 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:49:25,940 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:49:26,818 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:49:27,821 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:49:28,999 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:49:28,999 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:49:29,001 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:49:31,290 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:49:31,739 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:49:31,740 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:49:31,740 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:49:31,746 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:49:31,824 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:49:33,831 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:49:36,782 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:49:38,507 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:49:38,507 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:49:38,508 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:49:38,514 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:49:38,835 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:49:39,838 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:49:39,886 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 21:49:41,887 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:49:43,147 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:49:43,147 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:49:43,148 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:49:43,154 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:49:43,841 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:49:43,999 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:49:43,999 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:49:44,001 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:49:45,844 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:49:47,303 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:49:50,280 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:49:50,281 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:49:50,281 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:49:50,287 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:49:50,852 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:49:51,855 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:49:52,321 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:49:54,680 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:49:54,681 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:49:54,681 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:49:54,687 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:49:54,857 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:49:57,688 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:49:57,863 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:49:58,999 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:49:58,999 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:49:59,000 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:50:02,544 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:50:02,545 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:50:02,545 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:50:02,551 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:50:02,868 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:50:03,585 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:50:03,871 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:50:08,586 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:50:09,885 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 21:50:10,763 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:50:10,764 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:50:10,764 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:50:10,770 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:50:10,879 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:50:13,771 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:50:13,885 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:50:13,999 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:50:13,999 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:50:14,001 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:50:14,961 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:50:14,961 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:50:14,962 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:50:14,967 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:50:15,887 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:50:17,891 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:50:19,005 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:50:22,513 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:50:22,514 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:50:22,514 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:50:22,520 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:50:22,897 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:50:23,900 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:50:24,521 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:50:28,220 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:50:28,221 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:50:28,221 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:50:28,228 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:50:28,906 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:50:28,999 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:50:28,999 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:50:29,001 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:50:29,909 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:50:30,243 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:50:33,971 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:50:33,972 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:50:33,972 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:50:33,978 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:50:34,915 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:50:35,918 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:50:36,013 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:50:39,888 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 21:50:41,171 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:50:41,172 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:50:41,173 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:50:41,173 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:50:41,178 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:50:41,923 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:50:43,927 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:50:43,999 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:50:43,999 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:50:44,001 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:50:46,217 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:50:47,278 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:50:47,278 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:50:47,278 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:50:47,284 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:50:47,932 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:50:49,937 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:50:51,322 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:50:52,235 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:50:52,236 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:50:52,236 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:50:52,242 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:50:52,939 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:50:53,942 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:50:56,759 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:50:56,760 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:50:56,760 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:50:56,761 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:50:56,766 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:50:56,945 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:50:58,999 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:50:58,999 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:50:59,001 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:50:59,951 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:51:01,126 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:51:01,126 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:51:01,127 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:51:01,133 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:51:01,952 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:51:02,169 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:51:03,957 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:51:05,506 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:51:05,507 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:51:05,507 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:51:05,513 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:51:05,958 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:51:07,514 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:51:07,962 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:51:09,889 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 21:51:11,623 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:51:11,624 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:51:11,624 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:51:11,630 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:51:11,966 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:51:12,630 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:51:13,971 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:51:13,999 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:51:13,999 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:51:14,001 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:51:15,556 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:51:15,557 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:51:15,557 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:51:15,563 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:51:15,972 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:51:17,977 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:51:18,597 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:51:19,652 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:51:19,653 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:51:19,654 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:51:19,659 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:51:19,978 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:51:21,982 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:51:23,660 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:51:25,438 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:51:25,440 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:51:25,440 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:51:25,446 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:51:25,986 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:51:27,991 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:51:28,999 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:51:29,000 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:51:29,001 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:51:29,290 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:51:29,764 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:51:29,765 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:51:29,765 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:51:29,771 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:51:29,992 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:51:31,996 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:51:34,806 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:51:34,999 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:51:35,000 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:51:35,000 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:51:35,006 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:51:36,002 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:51:38,005 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:51:39,891 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 21:51:39,894 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:51:41,401 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:51:41,402 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:51:41,403 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:51:41,408 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:51:42,008 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:51:43,999 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:51:44,000 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:51:44,001 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:51:44,013 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:51:45,291 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:51:46,414 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:51:46,415 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:51:46,415 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:51:46,421 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:51:47,017 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:51:48,020 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:51:50,458 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:51:50,878 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:51:50,878 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:51:50,879 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:51:50,885 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:51:51,022 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:51:54,028 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:51:55,886 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:51:58,999 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:51:59,000 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:51:59,001 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:51:59,451 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:51:59,451 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:51:59,452 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:51:59,458 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:52:00,033 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:52:01,493 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:52:02,039 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:52:04,131 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:52:04,132 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:52:04,132 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:52:04,138 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:52:05,041 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:52:06,044 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:52:07,139 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:52:09,897 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 21:52:10,222 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:52:10,223 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:52:10,223 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:52:10,229 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:52:11,050 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:52:12,052 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:52:12,230 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:52:13,999 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:52:14,000 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:52:14,002 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:52:15,138 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:52:15,139 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:52:15,139 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:52:15,145 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:52:16,056 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:52:18,061 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:52:18,181 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:52:21,626 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:52:21,626 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:52:21,627 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:52:21,633 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:52:22,065 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:52:23,633 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:52:24,070 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:52:26,083 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:52:26,084 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:52:26,084 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:52:26,090 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:52:27,073 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:52:28,076 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:52:29,000 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:52:29,000 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:52:29,001 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:52:29,298 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:52:32,712 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:52:32,712 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:52:32,712 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:52:32,718 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:52:33,082 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:52:34,085 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:52:34,753 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:52:38,625 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:52:38,626 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:52:38,627 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:52:38,632 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:52:39,090 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:52:39,896 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 21:52:39,897 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:52:40,093 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:52:44,000 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:52:44,000 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:52:44,001 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:52:44,634 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:52:44,634 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:52:44,634 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:52:44,641 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:52:45,098 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:52:45,677 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:52:46,101 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:52:48,484 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:52:48,484 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:52:48,485 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:52:48,490 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:52:49,104 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:52:50,107 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:52:51,491 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:52:52,786 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:52:52,787 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:52:52,787 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:52:52,793 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:52:53,110 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:52:54,113 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:52:56,794 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:52:57,883 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:52:57,884 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:52:57,884 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:52:57,890 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:52:58,116 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:52:59,000 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:52:59,000 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:52:59,001 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:53:00,121 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:53:02,225 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:53:02,226 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:53:02,226 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:53:02,226 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:53:02,232 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:53:03,124 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:53:04,127 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:53:06,501 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:53:06,502 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:53:06,502 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:53:06,507 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:53:07,130 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:53:07,508 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:53:08,133 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:53:09,901 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 21:53:10,653 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:53:10,654 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:53:10,654 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:53:10,659 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:53:11,136 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:53:12,138 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:53:12,660 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:53:14,000 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:53:14,000 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:53:14,001 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:53:16,209 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:53:16,209 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:53:16,210 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:53:16,215 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:53:17,143 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:53:18,146 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:53:18,253 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:53:20,671 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:53:20,672 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:53:20,673 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:53:20,679 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:53:21,152 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:53:22,227 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:53:23,679 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:53:26,088 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:53:26,088 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:53:26,089 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:53:26,095 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:53:26,230 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:53:27,233 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:53:29,000 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:53:29,000 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:53:29,002 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:53:29,247 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:53:31,311 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:53:31,312 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:53:31,312 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:53:31,318 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:53:32,236 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:53:33,241 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:53:34,354 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:53:36,991 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:53:36,992 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:53:36,992 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:53:36,998 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:53:37,244 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:53:39,248 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:53:39,900 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 21:53:39,902 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:53:42,731 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:53:42,732 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:53:42,732 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:53:42,738 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:53:43,252 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:53:43,253 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:53:44,000 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:53:44,000 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:53:44,002 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:53:45,226 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:53:48,479 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:53:48,480 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:53:48,480 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:53:48,486 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:53:49,259 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:53:49,259 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:53:50,521 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:53:53,520 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:53:53,521 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:53:53,521 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:53:53,527 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:53:54,262 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:53:55,265 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:53:55,527 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:53:59,000 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:53:59,000 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:53:59,002 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:54:00,531 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:54:00,532 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:54:00,532 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:54:00,532 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:54:00,538 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:54:01,271 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:54:01,272 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:54:05,573 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:54:05,723 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:54:05,724 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:54:05,724 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:54:05,730 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:54:06,275 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:54:07,278 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:54:09,906 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 21:54:10,372 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:54:10,373 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:54:10,373 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:54:10,379 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:54:11,284 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:54:11,284 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:54:11,380 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:54:14,000 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:54:14,000 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:54:14,002 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:54:15,229 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:54:15,229 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:54:15,229 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:54:15,236 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:54:15,287 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:54:17,269 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:54:17,290 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:54:21,985 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:54:21,986 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:54:21,986 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:54:21,991 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:54:22,294 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:54:22,992 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:54:23,298 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:54:26,490 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:54:26,491 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:54:26,491 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:54:26,497 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:54:27,303 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:54:27,304 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:54:28,497 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:54:29,000 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:54:29,000 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:54:29,002 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:54:30,978 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:54:30,978 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:54:30,978 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:54:30,984 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:54:31,166 WARNING FileStreamThread:2491270 [file_stream.py:request_with_retry():668] requests_with_retry encountered retryable exception: 502 Server Error: Bad Gateway for url: https://api.wandb.ai/files/lilei_stones/hf-vlms/yrs4wcl6/file_stream. func: functools.partial(>, timeout=180.0), args: ('https://api.wandb.ai/files/lilei_stones/hf-vlms/yrs4wcl6/file_stream',), kwargs: {'json': {'files': {'wandb-summary.json': {'offset': 0, 'content': ['{"Finetune/Step": 4893, "Finetune/Loss": 0.8488271832466125, "Finetune/Loss (Raw)": 1.0013339519500732, "Finetune/Learning Rate": 1.7996355070443727e-07, "Finetune/Step Time": 5.442965777590871, "_timestamp": 1710280455.2282858, "_runtime": 26711.591413736343, "_step": 4893}']}, 'wandb-history.jsonl': {'offset': 4887, 'content': ['{"Finetune/Step": 4888, "Finetune/Loss": 0.8460896611213684, "Finetune/Loss (Raw)": 0.9415021538734436, "Finetune/Learning Rate": 1.8589386071896975e-07, "Finetune/Step Time": 5.447112096473575, "_timestamp": 1710280428.4794598, "_runtime": 26684.842587709427, "_step": 4888}', '{"Finetune/Step": 4889, "Finetune/Loss": 0.8469873666763306, "Finetune/Loss (Raw)": 0.8779363036155701, "Finetune/Learning Rate": 1.8470017954154262e-07, "Finetune/Step Time": 5.43055328913033, "_timestamp": 1710280433.5201728, "_runtime": 26689.88330078125, "_step": 4889}', '{"Finetune/Step": 4890, "Finetune/Loss": 0.8475115299224854, "Finetune/Loss (Raw)": 0.8610472083091736, "Finetune/Learning Rate": 1.8351030749034992e-07, "Finetune/Step Time": 5.449922828003764, "_timestamp": 1710280440.5315814, "_runtime": 26696.89470934868, "_step": 4890}', '{"Finetune/Step": 4891, "Finetune/Loss": 0.8467695713043213, "Finetune/Loss (Raw)": 0.7841209769248962, "Finetune/Learning Rate": 1.8232424502715672e-07, "Finetune/Step Time": 5.447011042386293, "_timestamp": 1710280445.7236629, "_runtime": 26702.086790800095, "_step": 4891}', '{"Finetune/Step": 4892, "Finetune/Loss": 0.8470671772956848, "Finetune/Loss (Raw)": 0.8847424983978271, "Finetune/Learning Rate": 1.811419926122493e-07, "Finetune/Step Time": 5.442794909700751, "_timestamp": 1710280450.372564, "_runtime": 26706.73569202423, "_step": 4892}', '{"Finetune/Step": 4893, "Finetune/Loss": 0.8488271832466125, "Finetune/Loss (Raw)": 1.0013339519500732, "Finetune/Learning Rate": 1.7996355070443727e-07, "Finetune/Step Time": 5.442965777590871, "_timestamp": 1710280455.2282858, "_runtime": 26711.591413736343, "_step": 4893}']}, 'wandb-events.jsonl': {'offset': 888, 'content': ['{"system.proc.memory.availableMB": 1783659.71, "system.memory": 13.6, "system.proc.memory.rssMB": 8293.02, "system.proc.memory.percent": 0.4, "system.cpu": 0.39, "system.cpu.0.cpu_percent": 18.31, "system.cpu.1.cpu_percent": 32.33, "system.cpu.2.cpu_percent": 0.3, "system.cpu.3.cpu_percent": 0.23, "system.cpu.4.cpu_percent": 1.07, "system.cpu.5.cpu_percent": 0.13, "system.cpu.6.cpu_percent": 0.03, "system.cpu.7.cpu_percent": 53.69, "system.cpu.8.cpu_percent": 2.33, "system.cpu.9.cpu_percent": 0.1, "system.cpu.10.cpu_percent": 0.0, "system.cpu.11.cpu_percent": 1.37, "system.cpu.12.cpu_percent": 0.03, "system.cpu.13.cpu_percent": 31.67, "system.cpu.14.cpu_percent": 1.33, "system.cpu.15.cpu_percent": 32.53, "system.cpu.16.cpu_percent": 17.61, "system.cpu.17.cpu_percent": 67.55, "system.cpu.18.cpu_percent": 0.1, "system.cpu.19.cpu_percent": 0.0, "system.cpu.20.cpu_percent": 0.03, "system.cpu.21.cpu_percent": 1.07, "system.cpu.22.cpu_percent": 0.0, "system.cpu.23.cpu_percent": 0.0, "system.cpu.24.cpu_percent": 14.87, "system.cpu.25.cpu_percent": 0.0, "system.cpu.26.cpu_percent": 0.63, "system.cpu.27.cpu_percent": 0.03, "system.cpu.28.cpu_percent": 0.53, "system.cpu.29.cpu_percent": 0.6, "system.cpu.30.cpu_percent": 0.0, "system.cpu.31.cpu_percent": 0.69, "system.cpu.32.cpu_percent": 6.03, "system.cpu.33.cpu_percent": 68.97, "system.cpu.34.cpu_percent": 0.03, "system.cpu.35.cpu_percent": 0.0, "system.cpu.36.cpu_percent": 0.0, "system.cpu.37.cpu_percent": 0.07, "system.cpu.38.cpu_percent": 0.03, "system.cpu.39.cpu_percent": 1.13, "system.cpu.40.cpu_percent": 24.97, "system.cpu.41.cpu_percent": 0.1, "system.cpu.42.cpu_percent": 0.17, "system.cpu.43.cpu_percent": 0.1, "system.cpu.44.cpu_percent": 0.1, "system.cpu.45.cpu_percent": 0.0, "system.cpu.46.cpu_percent": 0.07, "system.cpu.47.cpu_percent": 0.0, "system.cpu.48.cpu_percent": 0.03, "system.cpu.49.cpu_percent": 1.2, "system.cpu.50.cpu_percent": 0.0, "system.cpu.51.cpu_percent": 0.0, "system.cpu.52.cpu_percent": 0.0, "system.cpu.53.cpu_percent": 0.0, "system.cpu.54.cpu_percent": 0.0, "system.cpu.55.cpu_percent": 0.03, "system.cpu.56.cpu_percent": 0.13, "system.cpu.57.cpu_percent": 0.13, "system.cpu.58.cpu_percent": 0.2, "system.cpu.59.cpu_percent": 0.1, "system.cpu.60.cpu_percent": 0.07, "system.cpu.61.cpu_percent": 0.07, "system.cpu.62.cpu_percent": 0.03, "system.cpu.63.cpu_percent": 0.6, "system.cpu.64.cpu_percent": 0.0, "system.cpu.65.cpu_percent": 0.0, "system.cpu.66.cpu_percent": 0.0, "system.cpu.67.cpu_percent": 0.07, "system.cpu.68.cpu_percent": 0.03, "system.cpu.69.cpu_percent": 0.0, "system.cpu.70.cpu_percent": 0.0, "system.cpu.71.cpu_percent": 0.13, "system.cpu.72.cpu_percent": 0.0, "system.cpu.73.cpu_percent": 0.0, "system.cpu.74.cpu_percent": 0.0, "system.cpu.75.cpu_percent": 0.0, "system.cpu.76.cpu_percent": 0.0, "system.cpu.77.cpu_percent": 0.0, "system.cpu.78.cpu_percent": 0.1, "system.cpu.79.cpu_percent": 0.0, "system.cpu.80.cpu_percent": 0.0, "system.cpu.81.cpu_percent": 0.0, "system.cpu.82.cpu_percent": 0.0, "system.cpu.83.cpu_percent": 0.0, "system.cpu.84.cpu_percent": 0.67, "system.cpu.85.cpu_percent": 0.0, "system.cpu.86.cpu_percent": 0.33, "system.cpu.87.cpu_percent": 0.03, "system.cpu.88.cpu_percent": 0.03, "system.cpu.89.cpu_percent": 0.0, "system.cpu.90.cpu_percent": 0.03, "system.cpu.91.cpu_percent": 0.07, "system.cpu.92.cpu_percent": 0.0, "system.cpu.93.cpu_percent": 0.03, "system.cpu.94.cpu_percent": 0.07, "system.cpu.95.cpu_percent": 0.03, "system.cpu.96.cpu_percent": 0.0, "system.cpu.97.cpu_percent": 0.0, "system.cpu.98.cpu_percent": 0.0, "system.cpu.99.cpu_percent": 0.03, "system.cpu.100.cpu_percent": 0.0, "system.cpu.101.cpu_percent": 0.03, "system.cpu.102.cpu_percent": 0.0, "system.cpu.103.cpu_percent": 0.03, "system.cpu.104.cpu_percent": 0.0, "system.cpu.105.cpu_percent": 0.0, "system.cpu.106.cpu_percent": 0.0, "system.cpu.107.cpu_percent": 0.0, "system.cpu.108.cpu_percent": 0.0, "system.cpu.109.cpu_percent": 0.0, "system.cpu.110.cpu_percent": 0.6, "system.cpu.111.cpu_percent": 0.0, "system.cpu.112.cpu_percent": 0.0, "system.cpu.113.cpu_percent": 0.0, "system.cpu.114.cpu_percent": 0.0, "system.cpu.115.cpu_percent": 0.0, "system.cpu.116.cpu_percent": 0.0, "system.cpu.117.cpu_percent": 0.0, "system.cpu.118.cpu_percent": 0.0, "system.cpu.119.cpu_percent": 0.0, "system.cpu.120.cpu_percent": 0.03, "system.cpu.121.cpu_percent": 0.03, "system.cpu.122.cpu_percent": 0.03, "system.cpu.123.cpu_percent": 0.0, "system.cpu.124.cpu_percent": 0.0, "system.cpu.125.cpu_percent": 0.03, "system.cpu.126.cpu_percent": 0.0, "system.cpu.127.cpu_percent": 0.0, "system.cpu.128.cpu_percent": 14.41, "system.cpu.129.cpu_percent": 0.0, "system.cpu.130.cpu_percent": 14.1, "system.cpu.131.cpu_percent": 32.33, "system.cpu.132.cpu_percent": 1.27, "system.cpu.133.cpu_percent": 0.1, "system.cpu.134.cpu_percent": 68.31, "system.cpu.135.cpu_percent": 0.67, "system.cpu.136.cpu_percent": 0.1, "system.cpu.137.cpu_percent": 67.43, "system.cpu.138.cpu_percent": 0.8, "system.cpu.139.cpu_percent": 68.13, "system.cpu.140.cpu_percent": 1.97, "system.cpu.141.cpu_percent": 0.03, "system.cpu.142.cpu_percent": 0.63, "system.cpu.143.cpu_percent": 0.13, "system.cpu.144.cpu_percent": 0.1, "system.cpu.145.cpu_percent": 0.57, "system.cpu.146.cpu_percent": 0.03, "system.cpu.147.cpu_percent": 0.0, "system.cpu.148.cpu_percent": 0.0, "system.cpu.149.cpu_percent": 0.03, "system.cpu.150.cpu_percent": 0.03, "system.cpu.151.cpu_percent": 1.2, "system.cpu.152.cpu_percent": 0.07, "system.cpu.153.cpu_percent": 0.0, "system.cpu.154.cpu_percent": 0.0, "system.cpu.155.cpu_percent": 0.03, "system.cpu.156.cpu_percent": 0.03, "system.cpu.157.cpu_percent": 0.03, "system.cpu.158.cpu_percent": 0.0, "system.cpu.159.cpu_percent": 0.07, "system.cpu.160.cpu_percent": 0.0, "system.cpu.161.cpu_percent": 0.03, "system.cpu.162.cpu_percent": 0.03, "system.cpu.163.cpu_percent": 0.13, "system.cpu.164.cpu_percent": 0.13, "system.cpu.165.cpu_percent": 0.07, "system.cpu.166.cpu_percent": 0.0, "system.cpu.167.cpu_percent": 1.13, "system.cpu.168.cpu_percent": 0.0, "system.cpu.169.cpu_percent": 0.0, "system.cpu.170.cpu_percent": 0.0, "system.cpu.171.cpu_percent": 67.64, "system.cpu.172.cpu_percent": 0.67, "system.cpu.173.cpu_percent": 0.03, "system.cpu.174.cpu_percent": 0.63, "system.cpu.175.cpu_percent": 8.3, "system.cpu.176.cpu_percent": 0.0, "system.cpu.177.cpu_percent": 1.1, "system.cpu.178.cpu_percent": 0.17, "system.cpu.179.cpu_percent": 0.07, "system.cpu.180.cpu_percent": 0.03, "system.cpu.181.cpu_percent": 1.1, "system.cpu.182.cpu_percent": 0.0, "system.cpu.183.cpu_percent": 32.61, "system.cpu.184.cpu_percent": 0.6, "system.cpu.185.cpu_percent": 0.13, "system.cpu.186.cpu_percent": 2.49, "system.cpu.187.cpu_percent": 0.0, "system.cpu.188.cpu_percent": 0.07, "system.cpu.189.cpu_percent": 1.17, "system.cpu.190.cpu_percent": 67.41, "system.cpu.191.cpu_percent": 0.03, "system.cpu.192.cpu_percent": 0.0, "system.cpu.193.cpu_percent": 0.0, "system.cpu.194.cpu_percent": 0.0, "system.cpu.195.cpu_percent": 0.03, "system.cpu.196.cpu_percent": 0.0, "system.cpu.197.cpu_percent": 0.07, "system.cpu.198.cpu_percent": 0.0, "system.cpu.199.cpu_percent": 0.0, "system.cpu.200.cpu_percent": 0.0, "system.cpu.201.cpu_percent": 0.0, "system.cpu.202.cpu_percent": 0.0, "system.cpu.203.cpu_percent": 0.0, "system.cpu.204.cpu_percent": 0.0, "system.cpu.205.cpu_percent": 0.0, "system.cpu.206.cpu_percent": 0.0, "system.cpu.207.cpu_percent": 0.0, "system.cpu.208.cpu_percent": 0.0, "system.cpu.209.cpu_percent": 0.0, "system.cpu.210.cpu_percent": 0.0, "system.cpu.211.cpu_percent": 0.0, "system.cpu.212.cpu_percent": 0.0, "system.cpu.213.cpu_percent": 0.0, "system.cpu.214.cpu_percent": 0.0, "system.cpu.215.cpu_percent": 0.0, "system.cpu.216.cpu_percent": 0.0, "system.cpu.217.cpu_percent": 0.0, "system.cpu.218.cpu_percent": 0.43, "system.cpu.219.cpu_percent": 0.0, "system.cpu.220.cpu_percent": 0.03, "system.cpu.221.cpu_percent": 0.0, "system.cpu.222.cpu_percent": 0.0, "system.cpu.223.cpu_percent": 0.07, "system.cpu.224.cpu_percent": 0.0, "system.cpu.225.cpu_percent": 0.0, "system.cpu.226.cpu_percent": 0.0, "system.cpu.227.cpu_percent": 0.0, "system.cpu.228.cpu_percent": 0.0, "system.cpu.229.cpu_percent": 0.0, "system.cpu.230.cpu_percent": 0.0, "system.cpu.231.cpu_percent": 0.0, "system.cpu.232.cpu_percent": 0.0, "system.cpu.233.cpu_percent": 0.03, "system.cpu.234.cpu_percent": 0.53, "system.cpu.235.cpu_percent": 0.0, "system.cpu.236.cpu_percent": 0.03, "system.cpu.237.cpu_percent": 0.0, "system.cpu.238.cpu_percent": 0.0, "system.cpu.239.cpu_percent": 0.03, "system.cpu.240.cpu_percent": 0.0, "system.cpu.241.cpu_percent": 0.0, "system.cpu.242.cpu_percent": 0.0, "system.cpu.243.cpu_percent": 0.0, "system.cpu.244.cpu_percent": 0.0, "system.cpu.245.cpu_percent": 0.1, "system.cpu.246.cpu_percent": 0.0, "system.cpu.247.cpu_percent": 0.0, "system.cpu.248.cpu_percent": 0.0, "system.cpu.249.cpu_percent": 0.0, "system.cpu.250.cpu_percent": 0.0, "system.cpu.251.cpu_percent": 0.0, "system.cpu.252.cpu_percent": 0.03, "system.cpu.253.cpu_percent": 0.0, "system.cpu.254.cpu_percent": 0.0, "system.proc.cpu.threads": 45, "system.disk.\\\\.usagePercent": 20.9, "system.disk.\\\\.usageGB": 50.52, "system.disk.in": 6.73, "system.disk.out": 1017.14, "system.network.sent": 15543496764.73, "system.network.recv": 58448851231.4, "system.gpu.0.memoryAllocated": 98.15, "system.gpu.process.0.memoryAllocated": 98.15, "system.gpu.1.memoryAllocated": 82.61, "system.gpu.2.memoryAllocated": 93.26, "system.gpu.3.memoryAllocated": 92.26, "system.gpu.4.memoryAllocated": 97.2, "system.gpu.5.memoryAllocated": 77.66, "system.gpu.6.memoryAllocated": 92.48, "system.gpu.7.memoryAllocated": 71.01, "system.gpu.0.memoryAllocatedBytes": 84309114880.0, "system.gpu.process.0.memoryAllocatedBytes": 84309114880.0, "system.gpu.1.memoryAllocatedBytes": 70964936704.0, "system.gpu.2.memoryAllocatedBytes": 80112713728.0, "system.gpu.3.memoryAllocatedBytes": 79248687104.0, "system.gpu.4.memoryAllocatedBytes": 83493322752.0, "system.gpu.5.memoryAllocatedBytes": 66707718144.0, "system.gpu.6.memoryAllocatedBytes": 79439527936.0, "system.gpu.7.memoryAllocatedBytes": 60995076096.0, "system.gpu.0.memory": 39.73, "system.gpu.process.0.memory": 39.73, "system.gpu.1.memory": 35.6, "system.gpu.2.memory": 32.6, "system.gpu.3.memory": 33.93, "system.gpu.4.memory": 32.73, "system.gpu.5.memory": 33.33, "system.gpu.6.memory": 31.67, "system.gpu.7.memory": 30.47, "system.gpu.0.gpu": 99.8, "system.gpu.process.0.gpu": 99.8, "system.gpu.1.gpu": 99.93, "system.gpu.2.gpu": 99.93, "system.gpu.3.gpu": 99.93, "system.gpu.4.gpu": 99.87, "system.gpu.5.gpu": 99.93, "system.gpu.6.gpu": 99.93, "system.gpu.7.gpu": 99.93, "system.gpu.0.temp": 75.13, "system.gpu.process.0.temp": 75.13, "system.gpu.1.temp": 64.73, "system.gpu.2.temp": 64.0, "system.gpu.3.temp": 73.73, "system.gpu.4.temp": 74.73, "system.gpu.5.temp": 64.33, "system.gpu.6.temp": 64.0, "system.gpu.7.temp": 72.07, "system.gpu.0.powerWatts": 379.86, "system.gpu.process.0.powerWatts": 379.86, "system.gpu.1.powerWatts": 371.56, "system.gpu.2.powerWatts": 349.11, "system.gpu.3.powerWatts": 342.07, "system.gpu.4.powerWatts": 365.87, "system.gpu.5.powerWatts": 353.54, "system.gpu.6.powerWatts": 339.52, "system.gpu.7.powerWatts": 363.3, "system.gpu.0.powerPercent": 96.78, "system.gpu.process.0.powerPercent": 96.78, "system.gpu.1.powerPercent": 93.13, "system.gpu.2.powerPercent": 90.64, "system.gpu.3.powerPercent": 88.17, "system.gpu.4.powerPercent": 87.55, "system.gpu.5.powerPercent": 90.27, "system.gpu.6.powerPercent": 87.66, "system.gpu.7.powerPercent": 90.63, "_wandb": true, "_timestamp": 1710280449.900746, "_runtime": 26706.263874}']}, 'output.log': {'offset': 1, 'content': ['ERROR 2024-03-12T21:54:17.222823 =>> [Global Step] 004893 =>> LR :: 0.000000 -- Loss :: 0.8488: 94%|██████████████████████████████████████████▎ | 4893/5198 [7:24:58<26:26, 5.20s/it]\n']}}, 'dropped': 0}} +2024-03-12 21:54:31,306 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:54:33,312 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:54:34,021 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:54:36,179 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:54:36,180 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:54:36,181 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:54:36,186 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:54:36,314 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:54:37,317 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:54:39,187 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:54:39,905 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 21:54:42,321 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:54:42,322 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:54:42,322 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:54:42,328 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:54:43,324 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:54:43,324 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:54:44,000 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:54:44,001 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:54:44,002 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:54:44,295 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:54:47,264 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:54:47,265 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:54:47,265 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:54:47,271 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:54:47,326 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:54:49,305 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:54:49,330 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:54:51,714 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:54:51,715 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:54:51,715 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:54:51,720 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:54:52,332 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:54:53,335 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:54:54,721 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:54:55,977 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:54:55,978 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:54:55,978 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:54:55,983 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:54:56,337 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:54:57,339 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:54:59,000 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:54:59,001 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:54:59,002 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:55:00,260 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:55:00,807 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:55:00,808 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:55:00,808 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:55:00,814 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:55:01,344 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:55:01,345 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:55:05,277 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:55:05,278 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:55:05,278 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:55:05,278 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:55:05,283 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:55:05,347 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:55:07,351 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:55:09,907 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 21:55:10,798 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:55:10,805 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:55:10,805 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:55:10,805 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:55:10,810 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:55:11,356 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:55:11,356 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:55:14,000 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:55:14,001 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:55:14,002 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:55:15,967 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:55:15,967 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:55:15,967 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:55:15,968 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:55:15,973 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:55:16,360 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:55:17,362 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:55:20,747 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:55:20,748 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:55:20,748 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:55:20,754 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:55:21,367 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:55:21,368 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:55:21,754 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:55:26,124 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:55:26,125 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:55:26,125 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:55:26,131 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:55:26,371 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:55:27,131 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:55:27,373 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:55:29,000 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:55:29,001 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:55:29,002 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:55:32,299 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:55:32,832 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:55:32,833 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:55:32,833 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:55:32,839 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:55:33,381 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:55:33,381 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:55:37,874 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:55:37,977 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:55:37,978 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:55:37,978 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:55:37,984 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:55:38,385 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:55:39,388 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:55:39,909 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 21:55:42,236 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:55:42,236 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:55:42,237 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:55:42,243 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:55:42,390 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:55:43,243 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:55:43,393 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:55:44,001 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:55:44,001 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:55:44,002 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:55:46,510 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:55:46,511 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:55:46,511 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:55:46,516 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:55:47,399 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:55:47,399 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:55:48,553 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:55:51,816 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:55:51,817 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:55:51,818 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:55:51,823 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:55:52,402 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:55:53,405 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:55:53,824 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:55:57,766 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:55:57,767 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:55:57,767 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:55:57,773 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:55:58,408 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:55:59,001 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:55:59,001 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:55:59,002 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:55:59,255 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:55:59,411 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:56:04,255 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:56:06,083 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:56:06,084 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:56:06,084 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:56:06,090 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:56:06,417 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:56:07,419 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:56:09,911 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 21:56:09,913 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:56:11,346 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:56:11,346 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:56:11,346 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:56:11,353 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:56:11,423 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:56:13,427 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:56:14,001 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:56:14,001 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:56:14,003 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:56:15,213 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:56:16,163 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:56:16,164 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:56:16,164 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:56:16,170 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:56:16,430 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:56:17,433 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:56:21,205 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:56:24,684 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:56:24,685 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:56:24,685 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:56:24,691 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:56:25,441 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:56:25,441 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:56:26,692 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:56:29,001 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:56:29,001 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:56:29,002 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:56:29,127 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:56:29,248 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:56:29,248 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:56:29,255 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:56:29,444 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:56:31,447 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:56:32,256 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:56:33,640 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:56:33,641 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:56:33,641 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:56:33,647 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:56:34,451 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:56:35,455 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:56:37,647 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:56:37,945 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:56:37,945 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:56:37,946 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:56:37,951 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:56:38,457 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:56:39,460 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:56:39,913 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 21:56:42,915 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:56:43,892 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:56:43,893 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:56:43,893 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:56:43,899 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:56:44,001 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:56:44,001 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:56:44,002 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:56:44,464 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:56:45,467 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:56:48,296 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:56:48,520 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:56:48,521 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:56:48,521 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:56:48,527 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:56:49,472 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:56:49,473 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:56:53,097 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:56:53,098 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:56:53,099 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:56:53,104 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:56:53,475 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:56:54,105 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:56:55,479 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:56:58,261 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:56:58,262 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:56:58,262 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:56:58,268 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:56:58,481 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:56:59,001 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:56:59,001 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:56:59,003 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:56:59,268 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:56:59,484 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:57:02,908 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:57:02,908 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:57:02,908 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:57:02,914 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:57:03,490 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:57:03,490 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:57:04,949 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:57:07,217 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:57:07,217 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:57:07,218 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:57:07,223 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:57:07,493 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:57:09,497 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:57:09,916 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 21:57:10,917 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:57:12,316 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:57:12,317 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:57:12,317 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:57:12,322 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:57:12,499 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:57:13,502 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:57:14,001 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:57:14,001 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:57:14,003 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:57:16,269 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:57:20,082 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:57:20,082 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:57:20,083 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:57:20,089 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:57:20,507 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:57:21,511 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:57:22,125 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:57:24,585 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:57:24,586 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:57:24,586 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:57:24,591 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:57:25,515 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:57:25,516 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:57:27,592 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:57:28,880 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:57:28,881 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:57:28,881 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:57:28,887 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:57:29,001 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:57:29,001 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:57:29,003 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:57:29,520 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:57:29,520 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:57:33,212 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:57:33,745 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:57:33,746 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:57:33,746 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:57:33,751 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:57:34,525 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:57:35,529 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:57:38,189 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:57:38,190 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:57:38,190 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:57:38,196 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:57:38,531 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:57:39,196 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:57:39,534 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:57:39,921 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 21:57:44,001 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:57:44,001 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:57:44,003 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:57:44,268 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:57:44,728 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:57:44,728 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:57:44,729 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:57:44,734 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:57:45,540 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:57:45,541 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:57:49,433 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:57:49,434 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:57:49,434 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:57:49,434 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:57:49,439 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:57:49,544 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:57:51,547 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:57:54,286 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:57:54,287 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:57:54,287 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:57:54,293 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:57:54,549 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:57:55,293 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:57:55,552 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:57:59,001 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:57:59,002 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:57:59,003 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:58:00,238 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:58:00,239 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:58:00,239 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:58:00,245 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:58:00,555 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:58:01,281 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:58:01,558 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:58:04,339 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:58:04,340 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:58:04,341 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:58:04,346 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:58:04,560 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:58:05,563 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:58:06,347 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:58:09,920 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 21:58:10,774 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:58:10,774 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:58:10,774 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:58:10,780 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:58:11,570 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:58:11,570 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:58:11,780 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:58:14,001 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:58:14,002 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:58:14,003 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:58:17,286 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:58:19,093 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:58:19,094 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:58:19,094 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:58:19,100 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:58:19,578 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:58:19,578 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:58:23,137 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:58:23,702 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:58:23,703 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:58:23,703 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:58:23,709 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:58:24,582 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:58:25,586 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:58:28,410 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:58:28,410 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:58:28,411 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:58:28,411 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:58:28,416 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:58:28,588 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:58:29,001 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:58:29,002 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:58:29,003 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:58:29,592 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:58:34,196 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:58:34,197 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:58:34,197 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:58:34,198 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:58:34,204 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:58:34,597 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:58:35,600 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:58:39,237 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:58:39,823 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:58:39,823 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:58:39,824 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:58:39,829 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:58:39,925 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 21:58:40,604 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:58:41,608 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:58:44,001 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:58:44,002 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:58:44,003 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:58:44,308 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:58:44,457 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:58:44,458 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:58:44,458 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:58:44,463 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:58:44,611 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:58:45,614 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:58:49,089 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:58:49,090 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:58:49,090 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:58:49,099 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:58:49,621 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:58:49,621 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:58:50,099 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:58:54,482 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:58:54,482 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:58:54,483 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:58:54,488 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:58:54,625 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:58:55,488 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:58:55,630 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:58:59,002 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:58:59,002 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:58:59,003 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:58:59,469 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:58:59,470 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:58:59,470 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:58:59,475 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:58:59,633 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:59:00,513 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:59:01,638 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:59:04,021 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:59:04,022 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:59:04,022 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:59:04,028 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:59:04,640 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:59:05,644 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:59:06,028 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:59:08,402 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:59:08,403 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:59:08,403 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:59:08,409 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:59:08,646 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:59:09,650 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:59:09,924 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 21:59:11,926 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:59:14,002 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:59:14,002 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:59:14,003 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:59:14,543 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:59:14,543 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:59:14,543 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:59:14,550 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:59:14,654 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:59:15,658 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:59:17,585 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:59:19,238 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:59:19,239 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:59:19,239 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:59:19,245 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:59:19,664 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:59:19,664 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:59:23,246 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:59:24,553 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:59:24,554 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:59:24,554 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:59:24,560 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:59:24,668 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:59:25,673 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:59:28,561 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:59:29,002 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:59:29,002 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:59:29,003 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:59:29,438 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:59:29,439 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:59:29,439 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:59:29,445 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:59:29,676 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:59:31,681 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:59:34,481 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:59:39,482 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:59:39,929 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 21:59:41,035 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:59:41,036 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:59:41,037 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:59:41,042 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:59:41,694 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:59:41,694 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:59:44,002 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:59:44,002 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:59:44,004 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:59:45,264 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:59:46,061 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:59:46,061 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:59:46,062 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:59:46,067 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:59:46,698 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:59:47,703 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:59:50,007 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:59:50,008 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:59:50,008 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:59:50,013 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:59:50,705 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:59:51,014 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:59:51,709 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:59:55,287 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:59:55,288 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 21:59:55,288 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:59:55,293 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:59:55,715 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 21:59:55,715 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 21:59:56,294 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:59:59,002 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:59:59,002 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:59:59,004 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 22:00:01,706 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:00:01,706 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:00:01,707 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:00:01,707 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:00:01,713 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:00:01,720 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:00:03,726 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:00:06,749 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:00:09,929 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 22:00:10,896 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:00:10,896 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:00:10,897 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:00:10,902 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:00:11,737 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:00:11,737 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:00:11,903 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:00:14,002 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 22:00:14,002 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 22:00:14,004 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 22:00:15,449 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:00:15,449 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:00:15,449 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:00:15,455 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:00:15,740 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:00:17,493 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:00:17,745 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:00:20,507 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:00:20,508 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:00:20,508 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:00:20,513 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:00:20,747 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:00:21,751 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:00:22,514 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:00:24,932 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:00:24,933 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:00:24,933 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:00:24,939 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:00:25,757 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:00:25,758 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:00:27,939 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:00:29,002 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 22:00:29,002 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 22:00:29,004 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 22:00:29,883 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:00:29,884 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:00:29,884 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:00:29,889 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:00:30,761 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:00:31,765 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:00:33,925 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:00:34,575 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:00:34,576 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:00:34,576 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:00:34,582 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:00:34,768 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:00:35,772 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:00:39,086 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:00:39,086 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:00:39,087 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:00:39,087 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:00:39,092 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:00:39,777 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:00:39,778 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:00:39,931 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 22:00:43,790 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:00:43,790 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:00:43,791 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:00:43,797 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:00:44,003 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 22:00:44,003 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 22:00:44,004 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 22:00:44,241 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:00:44,781 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:00:45,785 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:00:49,241 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:00:49,840 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:00:49,840 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:00:49,840 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:00:49,846 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:00:50,789 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:00:51,794 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:00:54,671 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:00:54,676 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:00:54,676 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:00:54,676 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:00:54,682 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:00:54,796 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:00:55,799 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:00:59,002 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 22:00:59,003 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 22:00:59,004 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 22:01:00,245 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:01:00,912 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:01:00,913 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:01:00,913 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:01:00,918 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:01:01,808 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:01:01,808 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:01:05,953 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:01:06,132 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:01:06,133 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:01:06,134 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:01:06,139 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:01:06,811 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:01:07,816 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:01:09,934 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 22:01:11,935 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:01:12,186 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:01:12,186 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:01:12,186 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:01:12,192 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:01:12,820 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:01:13,824 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:01:14,002 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 22:01:14,003 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 22:01:14,004 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 22:01:17,298 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:01:18,550 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:01:18,550 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:01:18,550 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:01:18,556 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:01:18,829 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:01:19,834 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:01:22,594 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:01:23,025 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:01:23,026 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:01:23,026 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:01:23,032 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:01:23,839 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:01:23,840 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:01:28,032 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:01:29,002 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 22:01:29,003 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 22:01:29,004 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 22:01:31,304 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:01:31,304 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:01:31,304 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:01:31,310 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:01:31,850 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:01:31,851 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:01:33,345 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:01:36,093 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:01:36,094 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:01:36,094 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:01:36,100 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:01:36,857 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:01:37,860 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:01:39,101 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:01:39,936 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 22:01:40,729 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:01:40,729 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:01:40,730 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:01:40,735 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:01:40,862 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:01:41,865 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:01:43,972 WARNING FileStreamThread:2491270 [file_stream.py:request_with_retry():668] requests_with_retry encountered retryable exception: 502 Server Error: Bad Gateway for url: https://api.wandb.ai/files/lilei_stones/hf-vlms/yrs4wcl6/file_stream. func: functools.partial(>, timeout=180.0), args: ('https://api.wandb.ai/files/lilei_stones/hf-vlms/yrs4wcl6/file_stream',), kwargs: {'json': {'files': {'wandb-summary.json': {'offset': 0, 'content': ['{"Finetune/Step": 4965, "Finetune/Loss": 0.8460050821304321, "Finetune/Loss (Raw)": 0.975489616394043, "Finetune/Learning Rate": 1.0515768749432942e-07, "Finetune/Step Time": 5.372652670368552, "_timestamp": 1710280839.0857449, "_runtime": 27095.44887280464, "_step": 4965}']}, 'wandb-history.jsonl': {'offset': 4958, 'content': ['{"Finetune/Step": 4959, "Finetune/Loss": 0.846284031867981, "Finetune/Loss (Raw)": 0.4407346248626709, "Finetune/Learning Rate": 1.1063312931768877e-07, "Finetune/Step Time": 5.411009738221765, "_timestamp": 1710280810.8957832, "_runtime": 27067.258911132812, "_step": 4959}', '{"Finetune/Step": 4960, "Finetune/Loss": 0.8468049764633179, "Finetune/Loss (Raw)": 0.8288559913635254, "Finetune/Learning Rate": 1.0971095892601702e-07, "Finetune/Step Time": 5.378879863768816, "_timestamp": 1710280815.4488149, "_runtime": 27071.81194281578, "_step": 4960}', '{"Finetune/Step": 4961, "Finetune/Loss": 0.8465644717216492, "Finetune/Loss (Raw)": 0.8159416317939758, "Finetune/Learning Rate": 1.0879262676243174e-07, "Finetune/Step Time": 5.377058727666736, "_timestamp": 1710280820.507151, "_runtime": 27076.870278835297, "_step": 4961}', '{"Finetune/Step": 4962, "Finetune/Loss": 0.8459833860397339, "Finetune/Loss (Raw)": 0.9207056164741516, "Finetune/Learning Rate": 1.0787813318331786e-07, "Finetune/Step Time": 5.375954560935497, "_timestamp": 1710280824.9324932, "_runtime": 27081.295621156693, "_step": 4962}', '{"Finetune/Step": 4963, "Finetune/Loss": 0.8452917337417603, "Finetune/Loss (Raw)": 0.8503878116607666, "Finetune/Learning Rate": 1.0696747854357259e-07, "Finetune/Step Time": 5.380342086777091, "_timestamp": 1710280829.883471, "_runtime": 27086.24659895897, "_step": 4963}', '{"Finetune/Step": 4964, "Finetune/Loss": 0.8451402187347412, "Finetune/Loss (Raw)": 0.947278618812561, "Finetune/Learning Rate": 1.0606066319660435e-07, "Finetune/Step Time": 5.372084315866232, "_timestamp": 1710280834.5755053, "_runtime": 27090.938633203506, "_step": 4964}', '{"Finetune/Step": 4965, "Finetune/Loss": 0.8460050821304321, "Finetune/Loss (Raw)": 0.975489616394043, "Finetune/Learning Rate": 1.0515768749432942e-07, "Finetune/Step Time": 5.372652670368552, "_timestamp": 1710280839.0857449, "_runtime": 27095.44887280464, "_step": 4965}']}, 'wandb-events.jsonl': {'offset': 901, 'content': ['{"system.proc.memory.availableMB": 1783514.09, "system.memory": 13.6, "system.proc.memory.rssMB": 8293.18, "system.proc.memory.percent": 0.4, "system.cpu": 0.39, "system.cpu.0.cpu_percent": 31.41, "system.cpu.1.cpu_percent": 31.41, "system.cpu.2.cpu_percent": 0.83, "system.cpu.3.cpu_percent": 0.27, "system.cpu.4.cpu_percent": 1.8, "system.cpu.5.cpu_percent": 31.02, "system.cpu.6.cpu_percent": 69.07, "system.cpu.7.cpu_percent": 68.63, "system.cpu.8.cpu_percent": 0.0, "system.cpu.9.cpu_percent": 0.0, "system.cpu.10.cpu_percent": 1.3, "system.cpu.11.cpu_percent": 0.63, "system.cpu.12.cpu_percent": 0.07, "system.cpu.13.cpu_percent": 30.93, "system.cpu.14.cpu_percent": 1.96, "system.cpu.15.cpu_percent": 31.57, "system.cpu.16.cpu_percent": 7.54, "system.cpu.17.cpu_percent": 68.59, "system.cpu.18.cpu_percent": 0.13, "system.cpu.19.cpu_percent": 0.03, "system.cpu.20.cpu_percent": 0.07, "system.cpu.21.cpu_percent": 0.6, "system.cpu.22.cpu_percent": 0.0, "system.cpu.23.cpu_percent": 0.0, "system.cpu.24.cpu_percent": 23.81, "system.cpu.25.cpu_percent": 0.0, "system.cpu.26.cpu_percent": 0.07, "system.cpu.27.cpu_percent": 0.07, "system.cpu.28.cpu_percent": 1.63, "system.cpu.29.cpu_percent": 1.13, "system.cpu.30.cpu_percent": 0.03, "system.cpu.31.cpu_percent": 0.6, "system.cpu.32.cpu_percent": 29.73, "system.cpu.33.cpu_percent": 70.33, "system.cpu.34.cpu_percent": 0.07, "system.cpu.35.cpu_percent": 0.0, "system.cpu.36.cpu_percent": 0.0, "system.cpu.37.cpu_percent": 0.1, "system.cpu.38.cpu_percent": 1.0, "system.cpu.39.cpu_percent": 0.23, "system.cpu.40.cpu_percent": 0.0, "system.cpu.41.cpu_percent": 0.77, "system.cpu.42.cpu_percent": 0.1, "system.cpu.43.cpu_percent": 0.1, "system.cpu.44.cpu_percent": 0.27, "system.cpu.45.cpu_percent": 0.0, "system.cpu.46.cpu_percent": 0.07, "system.cpu.47.cpu_percent": 7.34, "system.cpu.48.cpu_percent": 0.1, "system.cpu.49.cpu_percent": 1.33, "system.cpu.50.cpu_percent": 0.07, "system.cpu.51.cpu_percent": 0.07, "system.cpu.52.cpu_percent": 0.03, "system.cpu.53.cpu_percent": 0.07, "system.cpu.54.cpu_percent": 0.03, "system.cpu.55.cpu_percent": 0.07, "system.cpu.56.cpu_percent": 0.13, "system.cpu.57.cpu_percent": 0.07, "system.cpu.58.cpu_percent": 0.2, "system.cpu.59.cpu_percent": 0.03, "system.cpu.60.cpu_percent": 0.07, "system.cpu.61.cpu_percent": 0.03, "system.cpu.62.cpu_percent": 0.0, "system.cpu.63.cpu_percent": 0.57, "system.cpu.64.cpu_percent": 0.0, "system.cpu.65.cpu_percent": 0.0, "system.cpu.66.cpu_percent": 0.0, "system.cpu.67.cpu_percent": 0.13, "system.cpu.68.cpu_percent": 0.1, "system.cpu.69.cpu_percent": 0.0, "system.cpu.70.cpu_percent": 0.0, "system.cpu.71.cpu_percent": 0.03, "system.cpu.72.cpu_percent": 0.07, "system.cpu.73.cpu_percent": 0.07, "system.cpu.74.cpu_percent": 0.0, "system.cpu.75.cpu_percent": 0.07, "system.cpu.76.cpu_percent": 0.03, "system.cpu.77.cpu_percent": 0.03, "system.cpu.78.cpu_percent": 0.03, "system.cpu.79.cpu_percent": 0.0, "system.cpu.80.cpu_percent": 0.0, "system.cpu.81.cpu_percent": 0.03, "system.cpu.82.cpu_percent": 0.03, "system.cpu.83.cpu_percent": 0.07, "system.cpu.84.cpu_percent": 0.2, "system.cpu.85.cpu_percent": 0.03, "system.cpu.86.cpu_percent": 0.03, "system.cpu.87.cpu_percent": 0.03, "system.cpu.88.cpu_percent": 0.0, "system.cpu.89.cpu_percent": 0.0, "system.cpu.90.cpu_percent": 0.0, "system.cpu.91.cpu_percent": 0.07, "system.cpu.92.cpu_percent": 0.0, "system.cpu.93.cpu_percent": 0.03, "system.cpu.94.cpu_percent": 0.0, "system.cpu.95.cpu_percent": 0.03, "system.cpu.96.cpu_percent": 0.03, "system.cpu.97.cpu_percent": 0.03, "system.cpu.98.cpu_percent": 0.0, "system.cpu.99.cpu_percent": 0.03, "system.cpu.100.cpu_percent": 0.0, "system.cpu.101.cpu_percent": 0.0, "system.cpu.102.cpu_percent": 0.0, "system.cpu.103.cpu_percent": 0.0, "system.cpu.104.cpu_percent": 0.0, "system.cpu.105.cpu_percent": 0.03, "system.cpu.106.cpu_percent": 0.03, "system.cpu.107.cpu_percent": 0.0, "system.cpu.108.cpu_percent": 0.0, "system.cpu.109.cpu_percent": 0.03, "system.cpu.110.cpu_percent": 0.63, "system.cpu.111.cpu_percent": 0.0, "system.cpu.112.cpu_percent": 0.0, "system.cpu.113.cpu_percent": 0.07, "system.cpu.114.cpu_percent": 0.0, "system.cpu.115.cpu_percent": 0.0, "system.cpu.116.cpu_percent": 0.03, "system.cpu.117.cpu_percent": 0.03, "system.cpu.118.cpu_percent": 0.03, "system.cpu.119.cpu_percent": 0.0, "system.cpu.120.cpu_percent": 0.0, "system.cpu.121.cpu_percent": 0.0, "system.cpu.122.cpu_percent": 0.03, "system.cpu.123.cpu_percent": 0.03, "system.cpu.124.cpu_percent": 0.0, "system.cpu.125.cpu_percent": 0.0, "system.cpu.126.cpu_percent": 0.03, "system.cpu.127.cpu_percent": 0.0, "system.cpu.128.cpu_percent": 0.0, "system.cpu.129.cpu_percent": 0.03, "system.cpu.130.cpu_percent": 0.2, "system.cpu.131.cpu_percent": 0.0, "system.cpu.132.cpu_percent": 0.67, "system.cpu.133.cpu_percent": 0.2, "system.cpu.134.cpu_percent": 0.57, "system.cpu.135.cpu_percent": 0.17, "system.cpu.136.cpu_percent": 0.17, "system.cpu.137.cpu_percent": 68.46, "system.cpu.138.cpu_percent": 2.43, "system.cpu.139.cpu_percent": 68.56, "system.cpu.140.cpu_percent": 0.7, "system.cpu.141.cpu_percent": 0.07, "system.cpu.142.cpu_percent": 1.17, "system.cpu.143.cpu_percent": 0.13, "system.cpu.144.cpu_percent": 0.0, "system.cpu.145.cpu_percent": 1.17, "system.cpu.146.cpu_percent": 0.07, "system.cpu.147.cpu_percent": 0.0, "system.cpu.148.cpu_percent": 0.0, "system.cpu.149.cpu_percent": 0.0, "system.cpu.150.cpu_percent": 0.07, "system.cpu.151.cpu_percent": 1.09, "system.cpu.152.cpu_percent": 0.1, "system.cpu.153.cpu_percent": 0.03, "system.cpu.154.cpu_percent": 0.0, "system.cpu.155.cpu_percent": 0.03, "system.cpu.156.cpu_percent": 0.03, "system.cpu.157.cpu_percent": 0.07, "system.cpu.158.cpu_percent": 0.0, "system.cpu.159.cpu_percent": 0.1, "system.cpu.160.cpu_percent": 0.0, "system.cpu.161.cpu_percent": 0.1, "system.cpu.162.cpu_percent": 0.0, "system.cpu.163.cpu_percent": 0.17, "system.cpu.164.cpu_percent": 0.13, "system.cpu.165.cpu_percent": 0.03, "system.cpu.166.cpu_percent": 0.0, "system.cpu.167.cpu_percent": 0.6, "system.cpu.168.cpu_percent": 0.0, "system.cpu.169.cpu_percent": 0.0, "system.cpu.170.cpu_percent": 0.03, "system.cpu.171.cpu_percent": 69.02, "system.cpu.172.cpu_percent": 1.27, "system.cpu.173.cpu_percent": 0.03, "system.cpu.174.cpu_percent": 0.63, "system.cpu.175.cpu_percent": 0.0, "system.cpu.176.cpu_percent": 0.0, "system.cpu.177.cpu_percent": 0.63, "system.cpu.178.cpu_percent": 0.0, "system.cpu.179.cpu_percent": 0.07, "system.cpu.180.cpu_percent": 0.03, "system.cpu.181.cpu_percent": 1.1, "system.cpu.182.cpu_percent": 0.0, "system.cpu.183.cpu_percent": 33.51, "system.cpu.184.cpu_percent": 1.17, "system.cpu.185.cpu_percent": 0.1, "system.cpu.186.cpu_percent": 0.67, "system.cpu.187.cpu_percent": 1.23, "system.cpu.188.cpu_percent": 0.03, "system.cpu.189.cpu_percent": 1.3, "system.cpu.190.cpu_percent": 66.51, "system.cpu.191.cpu_percent": 0.03, "system.cpu.192.cpu_percent": 0.07, "system.cpu.193.cpu_percent": 0.0, "system.cpu.194.cpu_percent": 0.0, "system.cpu.195.cpu_percent": 0.03, "system.cpu.196.cpu_percent": 0.03, "system.cpu.197.cpu_percent": 0.0, "system.cpu.198.cpu_percent": 0.0, "system.cpu.199.cpu_percent": 0.0, "system.cpu.200.cpu_percent": 0.0, "system.cpu.201.cpu_percent": 0.0, "system.cpu.202.cpu_percent": 0.0, "system.cpu.203.cpu_percent": 0.0, "system.cpu.204.cpu_percent": 0.0, "system.cpu.205.cpu_percent": 0.0, "system.cpu.206.cpu_percent": 0.0, "system.cpu.207.cpu_percent": 0.0, "system.cpu.208.cpu_percent": 0.0, "system.cpu.209.cpu_percent": 0.0, "system.cpu.210.cpu_percent": 0.0, "system.cpu.211.cpu_percent": 0.0, "system.cpu.212.cpu_percent": 0.0, "system.cpu.213.cpu_percent": 0.0, "system.cpu.214.cpu_percent": 0.0, "system.cpu.215.cpu_percent": 0.0, "system.cpu.216.cpu_percent": 0.0, "system.cpu.217.cpu_percent": 0.0, "system.cpu.218.cpu_percent": 0.47, "system.cpu.219.cpu_percent": 0.0, "system.cpu.220.cpu_percent": 0.1, "system.cpu.221.cpu_percent": 0.0, "system.cpu.222.cpu_percent": 0.0, "system.cpu.223.cpu_percent": 0.1, "system.cpu.224.cpu_percent": 0.0, "system.cpu.225.cpu_percent": 0.0, "system.cpu.226.cpu_percent": 0.0, "system.cpu.227.cpu_percent": 0.0, "system.cpu.228.cpu_percent": 0.0, "system.cpu.229.cpu_percent": 0.0, "system.cpu.230.cpu_percent": 0.0, "system.cpu.231.cpu_percent": 0.0, "system.cpu.232.cpu_percent": 0.0, "system.cpu.233.cpu_percent": 0.03, "system.cpu.234.cpu_percent": 0.0, "system.cpu.235.cpu_percent": 0.0, "system.cpu.236.cpu_percent": 0.0, "system.cpu.237.cpu_percent": 0.0, "system.cpu.238.cpu_percent": 0.0, "system.cpu.239.cpu_percent": 0.0, "system.cpu.240.cpu_percent": 0.0, "system.cpu.241.cpu_percent": 0.0, "system.cpu.242.cpu_percent": 0.0, "system.cpu.243.cpu_percent": 0.0, "system.cpu.244.cpu_percent": 0.0, "system.cpu.245.cpu_percent": 0.0, "system.cpu.246.cpu_percent": 0.0, "system.cpu.247.cpu_percent": 0.0, "system.cpu.248.cpu_percent": 0.0, "system.cpu.249.cpu_percent": 0.0, "system.cpu.250.cpu_percent": 0.0, "system.cpu.251.cpu_percent": 0.0, "system.cpu.252.cpu_percent": 0.0, "system.cpu.253.cpu_percent": 0.0, "system.cpu.254.cpu_percent": 0.0, "system.proc.cpu.threads": 45, "system.disk.\\\\.usagePercent": 20.9, "system.disk.\\\\.usageGB": 50.52, "system.disk.in": 6.73, "system.disk.out": 1030.31, "system.network.sent": 15769502352.8, "system.network.recv": 59135337396.8, "system.gpu.0.memoryAllocated": 98.15, "system.gpu.process.0.memoryAllocated": 98.15, "system.gpu.1.memoryAllocated": 82.61, "system.gpu.2.memoryAllocated": 93.26, "system.gpu.3.memoryAllocated": 92.26, "system.gpu.4.memoryAllocated": 97.2, "system.gpu.5.memoryAllocated": 77.66, "system.gpu.6.memoryAllocated": 92.48, "system.gpu.7.memoryAllocated": 71.01, "system.gpu.0.memoryAllocatedBytes": 84309114880.0, "system.gpu.process.0.memoryAllocatedBytes": 84309114880.0, "system.gpu.1.memoryAllocatedBytes": 70964936704.0, "system.gpu.2.memoryAllocatedBytes": 80112713728.0, "system.gpu.3.memoryAllocatedBytes": 79248687104.0, "system.gpu.4.memoryAllocatedBytes": 83493322752.0, "system.gpu.5.memoryAllocatedBytes": 66707718144.0, "system.gpu.6.memoryAllocatedBytes": 79439527936.0, "system.gpu.7.memoryAllocatedBytes": 60995076096.0, "system.gpu.0.memory": 36.47, "system.gpu.process.0.memory": 36.47, "system.gpu.1.memory": 34.13, "system.gpu.2.memory": 32.2, "system.gpu.3.memory": 32.4, "system.gpu.4.memory": 30.87, "system.gpu.5.memory": 32.67, "system.gpu.6.memory": 30.47, "system.gpu.7.memory": 33.13, "system.gpu.0.gpu": 99.93, "system.gpu.process.0.gpu": 99.93, "system.gpu.1.gpu": 99.93, "system.gpu.2.gpu": 99.87, "system.gpu.3.gpu": 99.93, "system.gpu.4.gpu": 99.87, "system.gpu.5.gpu": 99.93, "system.gpu.6.gpu": 99.93, "system.gpu.7.gpu": 93.27, "system.gpu.0.temp": 74.53, "system.gpu.process.0.temp": 74.53, "system.gpu.1.temp": 63.73, "system.gpu.2.temp": 63.27, "system.gpu.3.temp": 72.13, "system.gpu.4.temp": 74.13, "system.gpu.5.temp": 63.6, "system.gpu.6.temp": 63.2, "system.gpu.7.temp": 72.27, "system.gpu.0.powerWatts": 375.92, "system.gpu.process.0.powerWatts": 375.92, "system.gpu.1.powerWatts": 354.62, "system.gpu.2.powerWatts": 375.65, "system.gpu.3.powerWatts": 355.77, "system.gpu.4.powerWatts": 365.66, "system.gpu.5.powerWatts": 338.48, "system.gpu.6.powerWatts": 355.63, "system.gpu.7.powerWatts": 368.22, "system.gpu.0.powerPercent": 93.57, "system.gpu.process.0.powerPercent": 93.57, "system.gpu.1.powerPercent": 92.76, "system.gpu.2.powerPercent": 93.74, "system.gpu.3.powerPercent": 89.83, "system.gpu.4.powerPercent": 91.57, "system.gpu.5.powerPercent": 84.36, "system.gpu.6.powerPercent": 89.18, "system.gpu.7.powerPercent": 92.47, "_wandb": true, "_timestamp": 1710280839.929379, "_runtime": 27096.292507}']}, 'output.log': {'offset': 1, 'content': ['ERROR 2024-03-12T22:00:39.608796 =>> [Global Step] 004965 =>> LR :: 0.000000 -- Loss :: 0.8460: 96%|██████████████████████████████████████████▉ | 4965/5198 [7:31:22<19:06, 4.92s/it]\n']}}, 'dropped': 0}} +2024-03-12 22:01:44,003 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 22:01:44,003 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 22:01:44,004 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 22:01:44,284 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:01:48,151 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:01:48,152 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:01:48,152 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:01:48,159 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:01:48,873 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:01:49,877 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:01:50,193 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:01:53,297 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:01:53,298 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:01:53,298 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:01:53,304 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:01:53,883 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:01:53,883 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:01:55,305 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:01:58,579 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:01:58,580 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:01:58,580 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:01:58,586 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:01:58,887 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:01:59,003 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 22:01:59,003 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 22:01:59,004 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 22:01:59,891 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:02:01,276 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:02:04,999 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:02:05,000 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:02:05,000 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:02:05,007 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:02:05,899 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:02:05,900 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:02:07,041 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:02:09,942 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 22:02:10,014 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:02:10,015 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:02:10,015 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:02:10,021 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:02:10,904 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:02:11,908 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:02:13,022 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:02:14,003 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 22:02:14,003 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 22:02:14,004 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 22:02:14,276 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:02:14,276 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:02:14,276 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:02:14,282 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:02:14,911 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:02:15,914 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:02:18,317 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:02:22,561 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:02:22,561 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:02:22,562 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:02:22,567 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:02:22,921 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:02:23,568 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:02:23,926 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:02:28,256 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:02:28,257 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:02:28,257 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:02:28,263 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:02:28,930 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:02:29,003 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 22:02:29,003 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 22:02:29,004 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 22:02:29,288 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:02:29,934 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:02:32,937 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:02:32,937 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:02:32,937 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:02:32,943 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:02:33,943 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:02:33,943 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:02:34,981 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:02:37,400 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:02:37,401 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:02:37,401 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:02:37,407 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:02:37,948 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:02:37,949 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:02:39,941 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 22:02:40,942 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:02:42,312 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:02:42,313 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:02:42,313 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:02:42,319 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:02:42,953 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:02:43,956 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:02:44,003 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 22:02:44,003 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 22:02:44,005 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 22:02:46,307 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:02:47,547 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:02:47,548 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:02:47,548 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:02:47,554 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:02:47,960 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:02:49,964 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:02:51,590 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:02:52,245 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:02:52,246 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:02:52,246 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:02:52,252 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:02:52,966 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:02:53,970 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:02:56,960 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:02:56,960 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:02:56,961 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:02:56,961 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:02:56,966 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:02:56,972 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:02:57,975 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:02:59,003 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 22:02:59,003 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 22:02:59,005 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 22:03:01,302 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:03:01,302 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:03:01,303 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:03:01,309 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:03:01,981 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:03:01,982 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:03:02,345 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:03:07,346 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:03:07,813 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:03:07,814 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:03:07,814 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:03:07,820 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:03:07,987 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:03:09,946 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 22:03:09,993 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:03:12,948 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:03:14,003 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 22:03:14,003 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 22:03:14,005 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 22:03:14,557 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:03:14,558 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:03:14,558 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:03:14,564 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:03:14,997 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:03:16,000 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:03:18,601 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:03:20,023 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:03:20,024 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:03:20,025 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:03:20,031 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:03:21,005 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:03:22,009 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:03:24,032 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:03:24,938 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:03:24,938 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:03:24,938 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:03:24,944 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:03:25,011 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:03:26,015 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:03:29,003 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 22:03:29,003 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 22:03:29,005 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 22:03:29,253 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:03:29,807 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:03:29,808 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:03:29,808 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:03:29,814 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:03:30,018 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:03:32,023 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:03:33,931 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:03:33,932 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:03:33,932 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:03:33,938 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:03:34,027 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:03:34,938 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:03:36,032 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:03:38,494 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:03:38,495 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:03:38,495 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:03:38,501 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:03:39,034 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:03:39,945 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 22:03:39,946 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:03:40,038 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:03:43,159 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:03:43,160 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:03:43,160 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:03:43,166 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:03:44,003 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 22:03:44,004 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 22:03:44,005 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 22:03:44,043 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:03:44,044 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:03:45,204 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:03:50,205 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:03:52,448 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:03:52,449 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:03:52,449 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:03:52,455 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:03:53,050 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:03:54,054 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:03:55,489 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:03:56,778 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:03:56,779 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:03:56,779 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:03:56,786 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:03:57,056 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:03:58,060 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:03:59,003 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 22:03:59,004 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 22:03:59,005 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 22:04:01,230 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:04:02,913 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:04:02,913 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:04:02,914 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:04:02,920 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:04:03,064 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:04:04,069 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:04:06,957 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:04:07,318 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:04:07,319 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:04:07,319 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:04:07,325 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:04:08,074 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:04:08,075 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:04:09,950 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 22:04:12,952 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:04:13,026 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:04:13,026 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:04:13,026 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:04:13,032 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:04:13,078 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:04:14,003 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 22:04:14,004 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 22:04:14,005 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 22:04:14,083 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:04:17,535 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:04:17,535 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:04:17,536 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:04:17,542 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:04:18,089 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:04:18,090 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:04:18,577 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:04:21,581 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:04:21,582 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:04:21,582 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:04:21,588 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:04:22,094 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:04:22,095 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:04:23,589 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:04:27,187 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:04:27,188 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:04:27,188 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:04:27,194 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:04:28,103 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:04:28,103 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:04:29,003 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 22:04:29,004 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 22:04:29,005 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 22:04:29,218 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:04:32,453 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:04:32,453 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:04:32,454 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:04:32,459 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:04:33,107 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:04:34,114 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:04:34,497 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:04:36,869 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:04:36,870 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:04:36,870 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:04:36,876 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:04:37,116 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:04:38,119 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:04:39,876 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:04:39,949 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 22:04:41,295 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:04:41,296 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:04:41,296 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:04:41,302 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:04:42,125 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:04:42,126 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:04:44,003 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 22:04:44,004 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 22:04:44,005 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 22:04:45,241 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:04:45,868 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:04:45,869 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:04:45,869 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:04:45,875 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:04:46,129 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:04:48,133 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:04:50,910 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:04:50,967 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:04:50,968 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:04:50,968 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:04:50,975 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:04:51,135 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:04:52,138 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:04:55,976 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:04:56,022 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:04:56,022 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:04:56,022 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:04:56,029 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:04:56,142 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:04:58,146 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:04:59,004 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 22:04:59,004 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 22:04:59,005 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 22:05:00,474 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:05:00,475 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:05:00,475 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:05:00,481 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:05:01,148 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:05:01,517 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:05:02,152 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:05:04,803 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:05:04,804 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:05:04,804 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:05:04,810 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:05:05,154 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:05:06,157 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:05:06,811 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:05:09,951 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 22:05:11,953 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:05:12,498 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:05:12,499 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:05:12,499 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:05:12,505 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:05:13,163 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:05:14,004 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 22:05:14,004 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 22:05:14,005 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 22:05:14,167 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:05:17,298 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:05:17,442 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:05:17,443 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:05:17,443 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:05:17,449 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:05:18,174 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:05:18,175 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:05:22,489 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:05:23,040 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:05:23,040 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:05:23,041 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:05:23,046 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:05:23,180 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:05:24,183 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:05:27,353 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:05:27,353 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:05:27,354 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:05:27,360 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:05:28,188 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:05:28,189 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:05:28,361 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:05:29,004 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 22:05:29,004 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 22:05:29,005 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 22:05:34,308 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:05:35,745 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:05:35,745 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:05:35,746 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:05:35,752 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:05:36,199 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:05:38,203 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:05:39,790 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:05:39,953 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 22:05:40,307 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:05:40,308 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:05:40,308 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:05:40,315 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:05:41,206 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:05:42,208 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:05:44,004 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 22:05:44,004 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 22:05:44,005 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 22:05:44,182 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:05:44,232 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:05:44,232 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:05:44,239 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:05:45,210 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:05:45,239 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:05:46,214 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:05:49,505 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:05:49,505 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:05:49,506 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:05:49,512 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:05:50,221 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:05:50,221 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:05:50,549 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:05:55,138 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:05:55,139 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:05:55,139 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:05:55,145 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:05:55,225 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:05:56,146 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:05:56,229 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:05:59,004 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 22:05:59,004 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 22:05:59,006 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 22:06:00,937 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:06:00,938 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:06:00,938 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:06:00,944 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:06:01,233 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:06:01,981 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:06:02,237 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:06:06,009 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:06:06,010 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:06:06,010 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:06:06,018 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:06:06,241 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:06:07,018 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:06:08,245 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:06:09,956 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 22:06:10,703 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:06:10,704 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:06:10,704 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:06:10,710 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:06:11,248 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:06:12,251 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:06:12,711 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:06:14,004 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 22:06:14,004 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 22:06:14,006 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 22:06:15,433 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:06:15,434 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:06:15,434 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:06:15,440 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:06:16,257 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:06:16,257 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:06:18,477 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:06:19,892 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:06:19,893 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:06:19,893 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:06:19,899 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:06:20,261 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:06:22,266 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:06:23,900 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:06:25,827 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:06:25,828 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:06:25,828 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:06:25,834 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:06:26,270 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:06:28,275 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:06:29,004 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 22:06:29,004 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 22:06:29,006 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 22:06:29,315 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:06:30,513 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:06:30,514 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:06:30,514 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:06:30,520 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:06:31,277 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:06:32,281 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:06:34,557 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:06:35,039 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:06:35,044 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:06:35,044 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:06:35,050 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:06:35,284 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:06:36,288 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:06:39,958 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 22:06:39,959 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:06:41,618 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:06:41,619 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:06:41,619 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:06:41,625 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:06:42,293 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:06:42,293 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:06:44,004 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 22:06:44,004 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 22:06:44,006 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 22:06:45,242 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:06:45,883 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:06:45,884 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:06:45,884 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:06:45,890 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:06:46,298 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:06:48,304 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:06:50,926 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:06:50,989 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:06:50,990 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:06:50,990 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:06:50,996 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:06:51,306 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:06:52,309 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:06:55,075 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:06:55,076 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:06:55,076 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:06:55,082 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:06:55,311 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:06:56,083 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:06:56,316 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:06:59,004 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 22:06:59,005 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 22:06:59,006 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 22:06:59,588 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:06:59,589 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:06:59,589 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:06:59,595 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:07:00,322 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:07:00,322 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:07:01,629 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:07:04,990 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:07:04,991 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:07:04,992 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:07:04,997 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:07:05,326 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:07:06,330 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:07:06,998 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:07:09,929 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:07:09,930 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:07:09,930 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:07:09,936 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:07:09,960 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 22:07:10,334 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:07:12,339 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:07:12,961 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:07:14,004 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 22:07:14,005 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 22:07:14,006 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 22:07:15,691 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:07:15,692 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:07:15,692 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:07:15,698 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:07:16,344 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:07:16,345 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:07:18,733 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:07:20,119 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:07:20,120 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:07:20,120 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:07:20,126 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:07:20,348 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:07:22,354 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:07:24,126 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:07:26,124 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:07:26,125 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:07:26,125 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:07:26,131 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:07:26,357 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:07:28,363 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:07:29,004 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 22:07:29,005 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 22:07:29,006 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 22:07:29,298 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:07:31,017 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:07:31,018 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:07:31,018 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:07:31,024 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:07:31,365 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:07:32,368 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:07:35,061 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:07:36,250 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:07:36,251 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:07:36,252 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:07:36,257 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:07:36,373 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:07:38,377 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:07:39,965 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 22:07:40,967 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:07:44,004 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 22:07:44,005 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 22:07:44,006 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 22:07:44,531 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:07:44,532 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:07:44,532 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:07:44,538 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:07:45,383 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:07:46,388 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:07:46,573 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:07:51,574 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:07:53,477 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:07:53,478 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:07:53,478 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:07:53,484 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:07:54,399 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:07:54,400 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:07:57,485 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:07:58,009 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:07:58,010 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:07:58,010 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:07:58,016 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:07:58,403 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:07:59,005 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 22:07:59,005 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 22:07:59,007 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 22:08:00,408 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:08:02,437 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:08:02,437 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:08:02,438 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:08:02,444 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:08:03,410 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:08:03,481 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:08:04,414 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:08:08,094 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:08:08,095 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:08:08,095 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:08:08,101 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:08:08,417 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:08:09,101 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:08:09,964 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 22:08:10,422 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:08:14,005 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 22:08:14,005 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 22:08:14,006 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 22:08:14,097 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:08:14,219 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:08:14,219 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:08:14,219 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:08:14,225 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:08:14,426 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:08:16,431 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:08:19,226 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:08:20,575 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:08:20,575 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:08:20,576 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:08:20,582 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:08:21,435 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:08:22,439 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:08:24,583 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:08:28,912 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:08:28,913 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:08:28,913 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:08:28,920 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:08:29,005 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 22:08:29,005 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 22:08:29,006 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 22:08:29,446 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:08:30,301 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:08:30,450 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:08:34,611 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:08:34,612 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:08:34,612 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:08:34,618 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:08:35,457 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:08:35,653 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:08:36,461 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:08:39,882 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:08:39,883 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:08:39,883 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:08:39,889 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:08:39,969 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 22:08:40,466 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:08:40,466 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:08:40,970 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:08:44,005 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 22:08:44,005 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 22:08:44,006 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 22:08:44,209 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:08:44,265 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:08:44,265 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:08:44,271 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:08:44,468 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:08:46,271 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:08:46,473 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:08:48,700 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:08:48,701 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:08:48,701 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:08:48,707 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:08:49,476 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:08:50,479 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:08:51,741 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:08:53,700 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:08:53,701 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:08:53,702 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:08:53,707 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:08:54,485 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:08:54,486 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:08:57,708 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:08:58,506 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:08:58,507 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:08:58,507 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:08:58,512 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:08:59,005 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 22:08:59,005 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 22:08:59,007 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 22:08:59,489 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:09:00,491 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:09:02,991 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:09:02,991 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:09:02,992 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:09:02,992 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:09:02,997 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:09:03,495 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:09:04,499 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:09:07,522 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:09:07,523 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:09:07,523 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:09:07,529 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:09:08,505 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:09:08,505 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:09:08,529 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:09:09,969 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 22:09:11,759 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:09:11,760 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:09:11,760 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:09:11,766 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:09:12,510 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:09:12,511 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:09:13,766 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:09:14,005 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 22:09:14,005 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 22:09:14,007 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 22:09:17,034 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:09:17,035 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:09:17,035 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:09:17,041 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:09:17,515 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:09:18,520 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:09:19,077 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:09:21,513 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:09:21,514 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:09:21,514 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:09:21,520 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:09:21,522 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:09:22,525 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:09:24,521 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:09:26,824 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:09:26,825 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:09:26,825 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:09:26,831 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:09:27,529 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:09:28,533 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:09:29,005 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 22:09:29,005 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 22:09:29,007 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 22:09:30,299 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:09:31,906 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:09:31,907 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:09:31,907 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:09:31,913 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:09:32,540 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:09:32,540 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:09:35,949 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:09:36,612 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:09:36,612 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:09:36,613 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:09:36,618 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:09:37,546 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:09:38,548 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:09:39,974 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 22:09:40,975 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:09:41,994 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:09:41,995 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:09:41,995 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:09:42,001 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:09:42,552 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:09:44,005 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 22:09:44,005 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 22:09:44,007 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 22:09:44,557 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:09:45,832 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:09:45,832 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:09:45,832 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:09:45,838 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:09:46,561 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:09:46,561 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:09:46,873 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:09:50,829 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:09:50,830 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:09:50,830 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:09:50,836 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:09:51,565 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:09:52,569 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:09:52,836 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:09:56,284 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:09:56,285 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:09:56,285 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:09:56,291 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:09:56,572 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:09:58,291 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:09:58,577 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:09:59,005 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 22:09:59,005 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 22:09:59,007 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 22:10:02,797 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:10:02,797 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:10:02,798 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:10:02,804 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:10:03,581 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:10:03,841 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:10:04,586 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:10:07,831 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:10:07,832 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:10:07,832 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:10:07,838 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:10:08,592 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:10:08,592 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:10:09,839 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:10:09,973 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 22:10:13,000 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:10:13,001 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:10:13,002 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:10:13,007 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:10:13,597 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:10:14,005 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 22:10:14,006 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 22:10:14,007 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 22:10:14,602 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:10:15,253 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:10:17,865 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:10:17,866 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:10:17,866 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:10:17,872 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:10:18,607 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:10:18,608 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:10:20,909 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:10:25,910 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:10:26,417 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:10:26,418 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:10:26,418 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:10:26,424 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:10:26,616 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:10:28,622 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:10:29,005 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 22:10:29,006 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 22:10:29,007 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 22:10:30,909 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:10:30,910 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:10:30,910 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:10:30,915 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:10:30,916 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:10:31,624 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:10:32,627 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:10:35,954 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:10:38,034 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:10:38,035 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:10:38,036 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:10:38,042 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:10:38,633 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:10:39,975 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 22:10:40,639 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:10:40,976 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:10:42,081 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:10:42,082 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:10:42,082 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:10:42,088 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:10:42,640 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:10:44,005 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 22:10:44,006 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 22:10:44,007 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 22:10:44,644 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:10:46,313 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:10:46,817 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:10:46,818 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:10:46,818 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:10:46,824 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:10:47,646 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:10:48,650 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:10:51,121 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:10:51,122 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:10:51,122 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:10:51,128 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:10:51,652 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:10:52,129 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:10:52,656 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:10:55,566 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:10:55,567 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:10:55,567 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:10:55,573 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:10:55,658 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:10:56,660 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:10:57,574 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:10:59,005 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 22:10:59,006 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 22:10:59,007 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 22:11:03,249 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:11:03,943 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:11:03,944 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:11:03,944 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:11:03,951 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:11:04,673 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:11:04,674 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:11:08,986 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:11:09,547 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:11:09,548 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:11:09,548 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:11:09,554 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:11:09,677 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:11:09,977 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 22:11:10,681 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:11:14,006 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 22:11:14,006 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 22:11:14,008 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 22:11:14,289 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:11:14,836 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:11:14,837 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:11:14,837 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:11:14,843 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:11:15,686 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:11:16,690 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:11:19,791 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:11:19,791 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:11:19,792 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:11:19,792 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:11:19,798 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:11:20,697 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:11:20,697 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:11:24,641 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:11:24,642 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:11:24,642 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:11:24,648 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:11:24,700 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:11:25,648 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:11:26,706 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:11:29,006 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 22:11:29,006 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 22:11:29,007 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 22:11:31,274 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:11:32,071 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:11:32,072 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:11:32,072 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:11:32,078 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:11:32,712 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:11:34,718 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:11:36,571 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:11:36,573 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:11:36,573 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:11:36,573 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:11:36,579 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:11:36,720 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:11:38,724 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:11:39,980 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 22:11:41,981 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:11:42,172 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:11:42,173 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:11:42,173 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:11:42,180 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:11:42,727 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:11:44,006 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 22:11:44,006 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 22:11:44,007 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 22:11:44,731 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:11:46,865 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:11:46,866 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:11:46,866 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:11:46,872 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:11:47,734 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:11:47,909 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:11:48,739 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:11:52,793 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:11:52,794 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:11:52,795 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:11:52,800 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:11:53,743 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:11:53,801 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:11:54,747 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:11:58,801 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:11:59,006 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 22:11:59,006 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 22:11:59,007 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 22:12:00,973 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:12:00,974 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:12:00,974 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:12:00,980 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:12:01,753 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:12:02,758 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:12:04,017 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:12:06,400 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:12:06,401 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:12:06,401 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:12:06,407 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:12:06,761 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:12:08,766 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:12:09,408 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:12:09,985 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 22:12:12,318 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:12:12,319 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:12:12,319 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:12:12,325 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:12:12,770 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:12:14,006 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 22:12:14,006 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 22:12:14,008 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 22:12:14,775 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:12:15,290 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:12:17,856 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:12:17,857 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:12:17,857 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:12:17,863 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:12:18,782 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:12:18,782 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:12:20,897 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:12:25,898 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:12:26,864 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:12:26,865 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:12:26,865 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:12:26,871 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:12:27,791 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:12:28,796 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:12:29,006 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 22:12:29,006 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 22:12:29,008 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 22:12:31,223 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:12:31,917 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:12:31,918 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:12:31,918 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:12:31,924 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:12:32,801 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:12:32,802 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:12:36,649 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:12:36,650 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:12:36,651 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:12:36,651 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:12:36,656 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:12:36,806 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:12:38,810 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:12:39,984 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 22:12:41,807 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:12:41,808 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:12:41,808 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:12:41,808 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:12:41,814 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:12:41,814 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:12:42,817 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:12:44,006 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 22:12:44,006 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 22:12:44,008 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 22:12:46,440 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:12:46,441 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:12:46,441 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:12:46,447 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:12:46,820 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:12:47,481 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:12:48,825 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:12:51,783 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:12:51,784 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:12:51,785 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:12:51,790 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:12:51,827 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:12:52,791 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:12:52,829 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:12:56,459 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:12:56,460 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:12:56,460 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:12:56,466 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:12:56,835 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:12:58,467 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:12:58,840 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:12:59,006 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 22:12:59,006 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 22:12:59,008 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 22:13:00,584 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:13:00,585 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:13:00,585 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:13:00,591 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:13:00,841 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:13:02,846 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:13:03,625 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:13:05,442 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:13:05,443 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:13:05,443 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:13:05,449 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:13:05,848 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:13:06,852 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:13:09,450 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:13:09,989 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 22:13:10,182 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:13:10,183 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:13:10,183 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:13:10,189 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:13:10,858 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:13:10,858 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:13:14,006 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 22:13:14,006 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 22:13:14,008 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 22:13:15,279 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:13:15,512 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:13:15,513 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:13:15,513 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:13:15,519 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:13:15,862 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:13:16,866 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:13:20,554 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:13:20,577 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:13:20,578 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:13:20,579 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:13:20,585 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:13:20,870 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:13:22,876 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:13:24,933 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:13:24,933 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:13:24,934 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:13:24,940 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:13:25,878 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:13:25,941 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:13:26,882 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:13:29,006 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 22:13:29,007 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 22:13:29,008 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 22:13:30,305 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:13:30,306 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:13:30,306 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:13:30,313 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:13:30,885 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:13:31,349 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:13:32,891 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:13:36,083 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:13:36,084 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:13:36,084 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:13:36,090 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:13:36,901 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:13:36,901 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:13:37,091 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:13:39,989 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 22:13:42,990 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:13:43,711 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:13:43,712 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:13:43,713 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:13:43,719 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:13:43,906 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:13:44,006 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 22:13:44,007 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 22:13:44,008 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 22:13:44,910 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:13:48,300 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:13:52,006 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:13:52,007 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:13:52,007 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:13:52,013 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:13:52,920 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:13:52,921 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:13:54,049 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:13:59,006 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 22:13:59,007 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 22:13:59,008 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 22:13:59,299 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:14:01,201 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:14:01,202 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:14:01,202 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:14:01,209 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:14:01,930 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:14:02,935 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:14:05,245 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:14:05,781 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:14:05,782 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:14:05,782 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:14:05,789 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:14:05,938 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:14:06,942 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:14:09,995 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 22:14:10,378 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:14:10,379 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:14:10,379 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:14:10,379 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:14:10,387 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:14:10,947 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:14:12,953 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:14:14,006 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 22:14:14,007 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 22:14:14,008 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 22:14:16,236 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:14:18,700 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:14:18,701 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:14:18,701 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:14:18,708 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:14:18,959 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:14:20,964 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:14:21,741 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:14:23,598 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:14:23,599 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:14:23,599 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:14:23,607 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:14:23,967 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:14:24,971 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:14:27,608 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:14:28,603 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:14:28,603 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:14:28,604 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:14:28,611 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:14:28,975 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:14:29,007 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 22:14:29,007 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 22:14:29,008 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 22:14:30,979 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:14:33,064 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:14:33,065 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:14:33,065 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:14:33,065 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:14:33,072 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:14:33,983 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:14:34,990 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:14:38,106 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:14:39,993 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 22:14:41,473 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:14:41,475 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:14:41,475 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:14:41,481 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:14:41,997 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:14:43,000 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:14:43,481 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:14:44,007 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 22:14:44,007 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 22:14:44,008 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 22:14:46,434 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:14:46,435 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:14:46,435 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:14:46,441 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:14:47,003 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:14:49,007 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:14:49,478 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:14:50,698 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:14:50,699 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:14:50,699 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:14:50,705 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:14:51,008 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:14:53,011 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:14:54,706 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:14:55,999 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:14:56,000 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:14:56,000 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:14:56,007 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:14:56,013 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:14:57,016 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:14:59,007 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 22:14:59,007 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 22:14:59,008 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 22:15:00,288 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:15:01,091 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:15:01,092 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:15:01,092 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:15:01,100 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:15:02,021 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:15:03,026 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:15:05,809 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:15:05,810 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:15:05,810 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:15:05,810 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:15:05,820 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:15:06,030 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:15:07,039 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:15:09,995 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 22:15:10,996 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:15:14,007 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 22:15:14,007 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 22:15:14,009 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 22:15:14,104 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:15:14,266 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:15:14,267 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:15:14,275 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:15:15,058 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:15:15,058 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:15:16,276 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:15:20,467 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:15:20,468 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:15:20,468 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:15:20,479 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:15:21,063 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:15:21,479 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:15:23,069 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:15:25,572 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:15:25,573 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:15:25,573 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:15:25,581 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:15:26,072 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:15:26,581 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:15:27,074 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:15:29,007 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 22:15:29,007 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 22:15:29,008 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 22:15:32,309 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:15:33,996 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:15:33,996 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:15:33,997 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:15:34,007 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:15:34,082 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:15:35,089 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:15:38,038 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:15:39,473 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:15:39,474 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:15:39,474 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:15:39,481 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:15:39,997 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 22:15:40,096 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:15:41,099 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:15:43,999 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:15:44,007 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 22:15:44,007 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 22:15:44,008 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 22:15:44,516 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:15:44,516 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:15:44,517 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:15:44,525 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:15:45,104 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:15:47,109 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:15:49,491 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:15:49,492 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:15:49,492 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:15:49,492 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:15:49,503 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:15:50,113 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:15:51,118 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:15:54,053 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:15:54,054 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:15:54,054 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:15:54,062 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:15:54,121 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:15:55,062 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:15:55,128 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:15:58,573 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:15:58,574 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:15:58,574 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:15:58,584 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:15:59,007 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 22:15:59,007 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 22:15:59,009 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 22:15:59,133 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:16:00,241 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:16:01,139 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:16:03,433 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:16:03,434 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:16:03,434 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:16:03,445 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:16:04,141 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:16:05,148 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:16:05,477 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:16:09,964 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:16:09,965 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:16:09,965 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:16:09,974 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:16:09,999 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 22:16:10,153 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:16:11,000 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:16:11,159 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:16:14,007 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 22:16:14,008 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 22:16:14,009 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 22:16:14,838 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:16:14,839 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:16:14,839 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:16:14,847 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:16:15,165 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:16:16,881 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:16:17,173 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:16:20,965 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:16:20,965 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:16:20,966 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:16:20,976 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:16:21,176 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:16:21,976 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:16:23,182 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:16:26,977 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:16:29,007 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 22:16:29,007 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 22:16:29,009 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 22:16:29,280 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:16:29,281 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:16:29,281 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:16:29,290 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:16:30,190 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:16:31,194 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:16:32,321 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:16:33,749 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:16:33,750 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:16:33,750 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:16:33,758 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:16:34,198 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:16:35,205 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:16:37,759 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:16:38,503 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:16:38,504 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:16:38,504 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:16:38,512 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:16:39,210 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:16:40,007 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 22:16:41,217 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:16:43,008 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:16:43,447 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:16:43,448 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:16:43,448 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:16:43,455 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:16:44,007 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 22:16:44,008 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 22:16:44,009 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 22:16:44,221 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:16:45,226 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:16:48,150 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:16:48,150 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:16:48,151 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:16:48,151 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:16:48,159 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:16:48,229 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:16:49,234 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:16:53,194 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:16:56,578 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:16:56,579 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:16:56,579 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:16:56,587 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:16:57,243 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:16:58,587 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:16:59,007 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 22:16:59,008 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 22:16:59,009 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 22:16:59,248 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:17:01,650 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:17:01,651 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:17:01,651 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:17:01,663 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:17:02,251 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:17:03,260 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:17:03,693 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:17:06,607 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:17:06,608 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:17:06,608 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:17:06,617 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:17:07,265 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:17:09,271 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:17:09,617 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:17:10,003 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 22:17:11,213 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:17:11,214 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:17:11,214 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:17:11,223 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:17:11,273 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:17:13,280 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:17:14,007 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 22:17:14,008 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 22:17:14,009 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 22:17:15,205 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:17:16,375 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:17:16,375 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:17:16,376 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:17:16,382 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:17:17,290 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:17:17,290 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:17:20,417 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:17:21,848 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:17:21,849 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:17:21,849 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:17:21,857 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:17:22,295 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:17:23,302 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:17:25,858 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:17:26,591 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:17:26,591 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:17:26,592 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:17:26,601 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:17:27,307 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:17:29,007 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 22:17:29,008 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 22:17:29,009 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 22:17:29,314 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:17:31,260 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:17:32,252 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:17:32,253 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:17:32,253 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:17:32,259 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:17:32,318 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:17:33,322 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:17:36,294 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:17:37,388 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:17:37,389 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:17:37,389 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:17:37,399 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:17:38,332 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:17:39,337 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:17:40,012 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 22:17:42,013 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:17:44,008 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 22:17:44,008 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 22:17:44,009 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 22:17:45,556 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:17:45,556 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:17:45,557 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:17:45,566 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:17:46,345 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:17:47,350 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:17:47,597 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:17:52,598 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:17:53,904 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:17:53,905 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:17:53,905 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:17:53,915 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:17:54,360 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:17:55,366 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:17:57,916 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:17:58,651 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:17:58,652 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:17:58,652 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:17:58,662 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:17:59,008 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 22:17:59,008 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 22:17:59,009 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 22:17:59,371 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:18:01,380 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:18:03,241 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:18:05,010 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:18:05,011 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:18:05,011 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:18:05,020 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:18:05,384 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:18:07,391 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:18:09,053 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:18:09,445 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:18:09,446 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:18:09,446 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:18:09,453 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:18:10,008 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 22:18:10,394 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:18:11,400 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:18:14,008 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 22:18:14,008 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 22:18:14,010 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 22:18:14,200 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:18:14,286 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:18:14,286 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:18:14,286 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:18:14,295 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:18:14,403 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:18:15,408 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:18:19,296 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:18:20,000 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:18:20,001 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:18:20,001 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:18:20,014 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:18:20,414 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:18:21,418 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:18:24,437 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:18:24,439 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:18:24,439 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:18:24,439 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:18:24,447 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:18:25,426 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:18:25,427 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:18:29,008 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 22:18:29,008 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 22:18:29,009 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 22:18:30,305 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:18:30,961 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:18:30,962 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:18:30,962 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:18:30,972 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:18:31,433 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:18:33,438 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:18:36,006 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:18:36,642 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:18:36,643 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:18:36,643 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:18:36,652 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:18:37,444 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:18:39,450 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:18:40,010 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 22:18:41,012 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:18:41,896 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:18:41,897 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:18:41,897 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:18:41,908 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:18:42,454 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:18:43,459 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:18:44,008 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 22:18:44,008 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 22:18:44,010 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 22:18:46,218 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:18:47,784 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:18:47,785 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:18:47,785 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:18:47,795 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:18:48,464 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:18:49,472 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:18:51,825 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:18:53,577 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:18:53,578 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:18:53,578 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:18:53,587 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:18:54,478 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:18:55,484 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:18:57,588 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:18:59,008 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 22:18:59,008 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 22:18:59,010 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 22:18:59,770 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:18:59,770 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:18:59,771 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:18:59,783 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:19:00,490 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:19:01,494 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:19:02,813 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:19:03,813 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:19:03,814 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:19:03,814 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:19:03,822 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:19:04,497 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:19:05,502 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:19:07,823 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:19:08,472 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:19:08,473 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:19:08,473 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:19:08,481 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:19:08,505 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:19:09,509 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:19:10,013 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 22:19:13,014 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:19:13,715 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:19:13,716 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:19:13,716 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:19:13,725 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:19:14,008 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 22:19:14,008 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 22:19:14,010 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 22:19:14,515 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:19:15,517 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:19:18,321 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:19:19,322 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:19:19,322 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:19:19,323 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:19:19,329 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:19:19,523 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:19:21,532 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:19:23,366 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:19:23,663 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:19:23,664 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:19:23,664 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:19:23,671 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:19:24,535 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:19:25,542 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:19:28,369 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:19:28,369 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:19:28,370 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:19:28,370 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:19:28,377 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:19:28,545 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:19:29,008 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 22:19:29,009 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 22:19:29,010 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 22:19:29,551 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:19:32,818 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:19:32,819 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:19:32,819 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:19:32,827 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:19:33,554 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:19:33,861 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:19:35,564 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:19:37,214 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:19:37,214 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:19:37,215 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:19:37,222 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:19:37,566 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:19:39,222 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:19:39,571 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:19:40,015 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 22:19:42,567 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:19:42,568 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:19:42,568 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:19:42,575 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:19:42,575 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:19:43,579 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:19:44,008 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 22:19:44,009 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 22:19:44,010 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 22:19:45,209 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:19:50,209 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:19:51,427 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:19:51,428 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:19:51,428 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:19:51,438 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:19:51,589 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:19:53,595 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:19:55,470 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:19:55,561 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:19:55,562 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:19:55,562 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:19:55,571 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:19:55,597 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:19:57,603 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:19:59,008 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 22:19:59,009 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 22:19:59,010 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 22:19:59,645 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:19:59,646 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:19:59,646 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:19:59,655 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:20:00,606 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:20:00,689 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:20:01,609 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:20:05,518 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:20:05,519 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:20:05,519 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:20:05,528 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:20:05,614 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:20:06,529 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:20:07,623 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:20:10,017 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 22:20:11,108 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:20:11,108 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:20:11,109 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:20:11,115 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:20:11,628 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:20:12,116 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:20:13,634 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:20:14,008 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 22:20:14,009 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 22:20:14,010 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 22:20:16,176 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:20:16,177 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:20:16,177 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:20:16,185 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:20:16,637 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:20:17,217 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:20:17,641 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:20:21,761 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:20:21,762 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:20:21,762 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:20:21,771 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:20:22,645 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:20:22,771 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:20:23,649 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:20:26,759 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:20:26,760 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:20:26,760 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:20:26,768 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:20:27,652 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:20:28,769 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:20:29,008 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 22:20:29,009 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 22:20:29,010 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 22:20:29,657 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:20:32,549 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:20:32,549 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:20:32,550 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:20:32,557 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:20:32,659 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:20:33,664 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:20:34,593 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:20:39,594 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:20:39,628 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:20:39,628 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:20:39,629 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:20:39,635 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:20:39,671 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:20:40,019 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 22:20:41,677 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:20:44,008 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 22:20:44,009 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 22:20:44,010 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 22:20:45,255 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:20:48,035 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:20:48,036 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:20:48,036 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:20:48,046 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:20:48,683 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:20:49,688 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:20:51,077 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:20:54,953 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:20:54,954 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:20:54,954 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:20:54,960 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:20:55,693 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:20:56,960 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:20:57,698 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:20:59,009 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 22:20:59,009 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 22:20:59,011 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 22:20:59,658 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:20:59,659 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:20:59,659 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:20:59,666 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:20:59,699 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:21:01,703 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:21:02,701 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:21:05,554 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:21:05,555 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:21:05,555 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:21:05,562 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:21:05,706 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:21:07,710 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:21:08,562 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:21:10,025 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 22:21:10,626 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:21:10,627 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:21:10,627 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:21:10,633 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:21:10,712 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:21:11,716 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:21:13,634 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:21:14,009 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 22:21:14,009 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 22:21:14,010 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 22:21:16,716 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:21:16,716 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:21:16,717 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:21:16,723 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:21:16,723 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:21:18,757 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:21:19,727 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:21:21,526 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:21:21,527 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:21:21,527 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:21:21,533 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:21:21,728 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:21:23,732 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:21:24,533 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:21:28,038 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:21:28,039 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:21:28,039 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:21:28,045 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:21:28,735 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:21:29,009 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 22:21:29,009 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 22:21:29,010 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 22:21:29,736 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:21:30,268 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:21:35,269 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:21:36,500 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:21:36,501 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:21:36,501 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:21:36,507 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:21:36,742 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:21:37,746 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:21:40,024 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 22:21:41,025 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:21:41,046 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:21:41,047 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:21:41,047 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:21:41,052 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:21:41,749 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:21:43,752 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:21:44,009 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 22:21:44,009 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 22:21:44,010 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 22:21:45,686 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:21:45,686 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:21:45,686 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:21:45,692 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:21:45,754 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:21:46,729 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:21:47,758 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:21:49,618 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:21:49,619 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:21:49,620 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:21:49,625 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:21:49,759 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:21:51,763 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:21:52,626 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:21:54,637 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 22:21:54,638 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:21:54,638 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:21:54,644 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:21:54,765 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:21:55,768 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:21:57,645 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:21:59,009 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 22:21:59,009 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 22:21:59,011 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 22:22:03,308 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:22:08,308 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:22:10,030 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 22:22:14,009 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 22:22:14,009 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 22:22:14,054 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 22:22:14,235 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:22:19,235 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:22:24,236 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:22:29,236 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:22:34,237 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:22:39,237 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:22:40,028 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 22:22:45,030 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:22:48,590 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 22:22:48,591 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 22:22:48,664 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 22:22:50,925 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:22:55,926 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:23:00,926 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:23:04,818 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 22:23:04,818 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 22:23:04,819 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 22:23:06,113 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:23:10,030 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 22:23:12,032 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:23:17,032 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:23:19,818 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 22:23:19,818 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 22:23:19,818 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 22:23:22,079 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:23:27,079 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:23:32,080 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:23:34,818 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 22:23:34,818 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 22:23:34,818 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 22:23:37,084 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:23:40,033 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 22:23:43,034 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:23:48,035 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:23:49,818 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 22:23:49,818 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 22:23:49,818 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 22:23:53,094 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:23:58,094 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:24:03,095 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:24:04,818 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 22:24:04,818 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 22:24:04,818 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 22:24:09,087 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:24:10,040 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 22:24:15,041 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:24:19,818 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 22:24:19,818 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 22:24:19,818 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 22:24:20,050 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:24:25,050 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:24:30,051 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:24:34,818 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 22:24:34,818 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 22:24:34,818 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 22:24:35,052 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:24:40,037 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 22:24:41,039 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:24:46,039 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:24:49,818 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 22:24:49,818 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 22:24:49,819 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 22:24:52,038 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:24:57,038 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:25:02,038 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:25:04,818 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 22:25:04,818 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 22:25:04,819 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 22:25:07,956 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:25:10,040 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 22:25:13,042 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:25:18,042 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:25:19,818 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 22:25:19,819 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 22:25:19,819 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 22:25:23,057 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:25:28,057 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:25:33,057 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:25:34,818 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 22:25:34,819 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 22:25:34,819 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 22:25:39,056 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:25:40,045 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 22:25:45,046 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:25:49,819 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 22:25:49,819 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 22:25:49,819 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 22:25:50,079 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:25:55,079 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:26:00,080 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:26:04,819 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 22:26:04,819 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 22:26:04,819 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 22:26:05,082 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:26:10,044 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 22:26:11,045 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:26:16,045 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:26:19,819 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 22:26:19,819 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 22:26:19,819 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: stop_status +2024-03-12 22:26:21,105 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:26:21,914 DEBUG SenderThread:2491270 [sender.py:send():382] send: telemetry +2024-03-12 22:26:21,914 DEBUG SenderThread:2491270 [sender.py:send():382] send: exit +2024-03-12 22:26:21,914 INFO SenderThread:2491270 [sender.py:send_exit():589] handling exit code: 0 +2024-03-12 22:26:21,914 INFO SenderThread:2491270 [sender.py:send_exit():591] handling runtime: 28633 +2024-03-12 22:26:21,920 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:26:21,921 INFO SenderThread:2491270 [sender.py:send_exit():597] send defer +2024-03-12 22:26:21,921 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: defer +2024-03-12 22:26:21,921 INFO HandlerThread:2491270 [handler.py:handle_request_defer():172] handle defer: 0 +2024-03-12 22:26:21,921 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: defer +2024-03-12 22:26:21,921 INFO SenderThread:2491270 [sender.py:send_request_defer():613] handle sender defer: 0 +2024-03-12 22:26:21,921 INFO SenderThread:2491270 [sender.py:transition_state():617] send defer: 1 +2024-03-12 22:26:21,921 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: defer +2024-03-12 22:26:21,921 INFO HandlerThread:2491270 [handler.py:handle_request_defer():172] handle defer: 1 +2024-03-12 22:26:21,921 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: defer +2024-03-12 22:26:21,921 INFO SenderThread:2491270 [sender.py:send_request_defer():613] handle sender defer: 1 +2024-03-12 22:26:21,922 INFO SenderThread:2491270 [sender.py:transition_state():617] send defer: 2 +2024-03-12 22:26:21,922 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: defer +2024-03-12 22:26:21,922 INFO HandlerThread:2491270 [handler.py:handle_request_defer():172] handle defer: 2 +2024-03-12 22:26:21,922 INFO HandlerThread:2491270 [system_monitor.py:finish():203] Stopping system monitor +2024-03-12 22:26:21,922 DEBUG SystemMonitor:2491270 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2024-03-12 22:26:21,922 INFO HandlerThread:2491270 [interfaces.py:finish():202] Joined cpu monitor +2024-03-12 22:26:21,922 DEBUG SystemMonitor:2491270 [system_monitor.py:_start():183] Publishing last batch of metrics +2024-03-12 22:26:21,923 INFO HandlerThread:2491270 [interfaces.py:finish():202] Joined disk monitor +2024-03-12 22:26:22,402 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:26:23,405 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:26:23,535 INFO HandlerThread:2491270 [interfaces.py:finish():202] Joined gpu monitor +2024-03-12 22:26:23,536 INFO HandlerThread:2491270 [interfaces.py:finish():202] Joined memory monitor +2024-03-12 22:26:23,536 INFO HandlerThread:2491270 [interfaces.py:finish():202] Joined network monitor +2024-03-12 22:26:23,536 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: poll_exit +2024-03-12 22:26:23,536 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: defer +2024-03-12 22:26:23,536 INFO SenderThread:2491270 [sender.py:send_request_defer():613] handle sender defer: 2 +2024-03-12 22:26:23,536 INFO SenderThread:2491270 [sender.py:transition_state():617] send defer: 3 +2024-03-12 22:26:23,536 DEBUG SenderThread:2491270 [sender.py:send():382] send: stats +2024-03-12 22:26:23,537 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: poll_exit +2024-03-12 22:26:23,537 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: defer +2024-03-12 22:26:23,537 INFO HandlerThread:2491270 [handler.py:handle_request_defer():172] handle defer: 3 +2024-03-12 22:26:23,538 DEBUG SenderThread:2491270 [sender.py:send():382] send: history +2024-03-12 22:26:23,538 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: summary_record +2024-03-12 22:26:23,544 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:26:23,545 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: defer +2024-03-12 22:26:23,545 INFO SenderThread:2491270 [sender.py:send_request_defer():613] handle sender defer: 3 +2024-03-12 22:26:23,545 INFO SenderThread:2491270 [sender.py:transition_state():617] send defer: 4 +2024-03-12 22:26:23,545 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: defer +2024-03-12 22:26:23,545 INFO HandlerThread:2491270 [handler.py:handle_request_defer():172] handle defer: 4 +2024-03-12 22:26:23,545 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: defer +2024-03-12 22:26:23,545 INFO SenderThread:2491270 [sender.py:send_request_defer():613] handle sender defer: 4 +2024-03-12 22:26:23,545 INFO SenderThread:2491270 [sender.py:transition_state():617] send defer: 5 +2024-03-12 22:26:23,545 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: defer +2024-03-12 22:26:23,545 INFO HandlerThread:2491270 [handler.py:handle_request_defer():172] handle defer: 5 +2024-03-12 22:26:23,546 DEBUG SenderThread:2491270 [sender.py:send():382] send: summary +2024-03-12 22:26:23,551 INFO SenderThread:2491270 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 22:26:23,551 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: defer +2024-03-12 22:26:23,551 INFO SenderThread:2491270 [sender.py:send_request_defer():613] handle sender defer: 5 +2024-03-12 22:26:23,551 INFO SenderThread:2491270 [sender.py:transition_state():617] send defer: 6 +2024-03-12 22:26:23,551 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: defer +2024-03-12 22:26:23,551 INFO HandlerThread:2491270 [handler.py:handle_request_defer():172] handle defer: 6 +2024-03-12 22:26:23,551 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: defer +2024-03-12 22:26:23,551 INFO SenderThread:2491270 [sender.py:send_request_defer():613] handle sender defer: 6 +2024-03-12 22:26:23,554 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 22:26:23,774 INFO SenderThread:2491270 [sender.py:transition_state():617] send defer: 7 +2024-03-12 22:26:23,774 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: defer +2024-03-12 22:26:23,774 INFO HandlerThread:2491270 [handler.py:handle_request_defer():172] handle defer: 7 +2024-03-12 22:26:23,774 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: defer +2024-03-12 22:26:23,774 INFO SenderThread:2491270 [sender.py:send_request_defer():613] handle sender defer: 7 +2024-03-12 22:26:23,887 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: poll_exit +2024-03-12 22:26:24,408 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/config.yaml +2024-03-12 22:26:24,408 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:26:24,408 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:26:25,147 INFO SenderThread:2491270 [sender.py:transition_state():617] send defer: 8 +2024-03-12 22:26:25,147 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: poll_exit +2024-03-12 22:26:25,147 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: defer +2024-03-12 22:26:25,147 INFO HandlerThread:2491270 [handler.py:handle_request_defer():172] handle defer: 8 +2024-03-12 22:26:25,147 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: defer +2024-03-12 22:26:25,147 INFO SenderThread:2491270 [sender.py:send_request_defer():613] handle sender defer: 8 +2024-03-12 22:26:25,147 INFO SenderThread:2491270 [job_builder.py:build():296] Attempting to build job artifact +2024-03-12 22:26:25,148 INFO SenderThread:2491270 [job_builder.py:_get_source_type():426] is repo sourced job +2024-03-12 22:26:25,237 INFO SenderThread:2491270 [job_builder.py:build():402] adding wandb-job metadata file +2024-03-12 22:26:25,268 INFO SenderThread:2491270 [sender.py:transition_state():617] send defer: 9 +2024-03-12 22:26:25,268 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: defer +2024-03-12 22:26:25,268 INFO HandlerThread:2491270 [handler.py:handle_request_defer():172] handle defer: 9 +2024-03-12 22:26:25,268 DEBUG SenderThread:2491270 [sender.py:send():382] send: artifact +2024-03-12 22:26:25,408 INFO Thread-12 :2491270 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:26:25,887 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: poll_exit +2024-03-12 22:26:26,312 INFO SenderThread:2491270 [sender.py:send_artifact():1494] sent artifact job-https___github.com_TRI-ML_prismatic-vlms.git_scripts_pretrain.py - {'id': 'QXJ0aWZhY3Q6NzUzNTUzNzEx', 'state': 'COMMITTED', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjE0MzcwMzk1OA==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6NzUzNTUzNzEx', 'versionIndex': 7}}} +2024-03-12 22:26:26,312 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: defer +2024-03-12 22:26:26,312 INFO SenderThread:2491270 [sender.py:send_request_defer():613] handle sender defer: 9 +2024-03-12 22:26:26,313 INFO SenderThread:2491270 [dir_watcher.py:finish():358] shutting down directory watcher +2024-03-12 22:26:26,410 INFO SenderThread:2491270 [dir_watcher.py:finish():388] scan: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files +2024-03-12 22:26:26,410 INFO SenderThread:2491270 [dir_watcher.py:finish():402] scan save: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/config.yaml config.yaml +2024-03-12 22:26:26,410 INFO SenderThread:2491270 [dir_watcher.py:finish():402] scan save: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/conda-environment.yaml conda-environment.yaml +2024-03-12 22:26:26,410 INFO SenderThread:2491270 [dir_watcher.py:finish():402] scan save: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-metadata.json wandb-metadata.json +2024-03-12 22:26:26,411 INFO SenderThread:2491270 [dir_watcher.py:finish():402] scan save: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/requirements.txt requirements.txt +2024-03-12 22:26:26,411 INFO SenderThread:2491270 [dir_watcher.py:finish():402] scan save: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log output.log +2024-03-12 22:26:26,411 INFO SenderThread:2491270 [dir_watcher.py:finish():402] scan save: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json wandb-summary.json +2024-03-12 22:26:26,411 INFO SenderThread:2491270 [sender.py:transition_state():617] send defer: 10 +2024-03-12 22:26:26,411 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: poll_exit +2024-03-12 22:26:26,411 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: defer +2024-03-12 22:26:26,411 INFO HandlerThread:2491270 [handler.py:handle_request_defer():172] handle defer: 10 +2024-03-12 22:26:26,420 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: defer +2024-03-12 22:26:26,420 INFO SenderThread:2491270 [sender.py:send_request_defer():613] handle sender defer: 10 +2024-03-12 22:26:26,420 INFO SenderThread:2491270 [file_pusher.py:finish():172] shutting down file pusher +2024-03-12 22:26:26,888 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: poll_exit +2024-03-12 22:26:26,888 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: poll_exit +2024-03-12 22:26:27,146 INFO wandb-upload_1:2491270 [upload_job.py:push():131] Uploaded file runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/config.yaml +2024-03-12 22:26:27,341 INFO wandb-upload_3:2491270 [upload_job.py:push():131] Uploaded file runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/output.log +2024-03-12 22:26:27,347 INFO wandb-upload_2:2491270 [upload_job.py:push():131] Uploaded file runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/requirements.txt +2024-03-12 22:26:27,354 INFO wandb-upload_4:2491270 [upload_job.py:push():131] Uploaded file runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/wandb-summary.json +2024-03-12 22:26:27,371 INFO wandb-upload_0:2491270 [upload_job.py:push():131] Uploaded file runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/files/conda-environment.yaml +2024-03-12 22:26:27,571 INFO Thread-11 (_thread_body):2491270 [sender.py:transition_state():617] send defer: 11 +2024-03-12 22:26:27,572 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: defer +2024-03-12 22:26:27,572 INFO HandlerThread:2491270 [handler.py:handle_request_defer():172] handle defer: 11 +2024-03-12 22:26:27,572 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: defer +2024-03-12 22:26:27,572 INFO SenderThread:2491270 [sender.py:send_request_defer():613] handle sender defer: 11 +2024-03-12 22:26:27,572 INFO SenderThread:2491270 [file_pusher.py:join():178] waiting for file pusher +2024-03-12 22:26:27,573 INFO SenderThread:2491270 [sender.py:transition_state():617] send defer: 12 +2024-03-12 22:26:27,573 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: defer +2024-03-12 22:26:27,573 INFO HandlerThread:2491270 [handler.py:handle_request_defer():172] handle defer: 12 +2024-03-12 22:26:27,573 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: defer +2024-03-12 22:26:27,573 INFO SenderThread:2491270 [sender.py:send_request_defer():613] handle sender defer: 12 +2024-03-12 22:26:27,573 INFO SenderThread:2491270 [file_stream.py:finish():595] file stream finish called +2024-03-12 22:26:27,888 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: poll_exit +2024-03-12 22:26:28,207 INFO SenderThread:2491270 [file_stream.py:finish():599] file stream finish is done +2024-03-12 22:26:28,207 INFO SenderThread:2491270 [sender.py:transition_state():617] send defer: 13 +2024-03-12 22:26:28,207 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: poll_exit +2024-03-12 22:26:28,208 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: defer +2024-03-12 22:26:28,208 INFO HandlerThread:2491270 [handler.py:handle_request_defer():172] handle defer: 13 +2024-03-12 22:26:28,208 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: defer +2024-03-12 22:26:28,208 INFO SenderThread:2491270 [sender.py:send_request_defer():613] handle sender defer: 13 +2024-03-12 22:26:28,208 INFO SenderThread:2491270 [sender.py:transition_state():617] send defer: 14 +2024-03-12 22:26:28,208 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: defer +2024-03-12 22:26:28,208 INFO HandlerThread:2491270 [handler.py:handle_request_defer():172] handle defer: 14 +2024-03-12 22:26:28,209 DEBUG SenderThread:2491270 [sender.py:send():382] send: final +2024-03-12 22:26:28,209 DEBUG SenderThread:2491270 [sender.py:send():382] send: footer +2024-03-12 22:26:28,209 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: defer +2024-03-12 22:26:28,209 INFO SenderThread:2491270 [sender.py:send_request_defer():613] handle sender defer: 14 +2024-03-12 22:26:28,209 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: poll_exit +2024-03-12 22:26:28,209 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: poll_exit +2024-03-12 22:26:28,210 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 22:26:28,210 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: server_info +2024-03-12 22:26:28,210 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: server_info +2024-03-12 22:26:28,253 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: get_summary +2024-03-12 22:26:28,253 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: sampled_history +2024-03-12 22:26:28,254 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: job_info +2024-03-12 22:26:28,401 DEBUG SenderThread:2491270 [sender.py:send_request():409] send_request: job_info +2024-03-12 22:26:28,401 DEBUG HandlerThread:2491270 [handler.py:handle_request():146] handle_request: shutdown +2024-03-12 22:26:28,402 INFO HandlerThread:2491270 [handler.py:finish():869] shutting down handler +2024-03-12 22:26:29,255 INFO WriterThread:2491270 [datastore.py:close():296] close: runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/run-yrs4wcl6.wandb +2024-03-12 22:26:29,401 INFO SenderThread:2491270 [sender.py:finish():1572] shutting down sender +2024-03-12 22:26:29,401 INFO SenderThread:2491270 [file_pusher.py:finish():172] shutting down file pusher +2024-03-12 22:26:29,401 INFO SenderThread:2491270 [file_pusher.py:join():178] waiting for file pusher diff --git a/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/logs/debug.log b/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..81198ce718643157275aa0f3baa37edc2174e091 --- /dev/null +++ b/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/logs/debug.log @@ -0,0 +1,34 @@ +2024-03-12 14:29:03,625 INFO MainThread:2490955 [wandb_setup.py:_flush():76] Current SDK version is 0.16.3 +2024-03-12 14:29:03,626 INFO MainThread:2490955 [wandb_setup.py:_flush():76] Configure stats pid to 2490955 +2024-03-12 14:29:03,626 INFO MainThread:2490955 [wandb_setup.py:_flush():76] Loading settings from /home/lilei/.config/wandb/settings +2024-03-12 14:29:03,626 INFO MainThread:2490955 [wandb_setup.py:_flush():76] Loading settings from /home/lilei/prismatic-vlms/wandb/settings +2024-03-12 14:29:03,626 INFO MainThread:2490955 [wandb_setup.py:_flush():76] Loading settings from environment variables: {} +2024-03-12 14:29:03,626 INFO MainThread:2490955 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False} +2024-03-12 14:29:03,626 INFO MainThread:2490955 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'scripts/pretrain.py', 'program_abspath': '/home/lilei/prismatic-vlms/scripts/pretrain.py', 'program': '/home/lilei/prismatic-vlms/scripts/pretrain.py'} +2024-03-12 14:29:03,626 INFO MainThread:2490955 [wandb_init.py:_log_setup():526] Logging user logs to runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/logs/debug.log +2024-03-12 14:29:03,626 INFO MainThread:2490955 [wandb_init.py:_log_setup():527] Logging internal logs to runs/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/logs/debug-internal.log +2024-03-12 14:29:03,626 INFO MainThread:2490955 [wandb_init.py:init():566] calling init triggers +2024-03-12 14:29:03,626 INFO MainThread:2490955 [wandb_init.py:init():573] wandb.init called with sweep_config: {} +config: {'model': {'type': 'one-stage+7b', 'model_id': 'one-stage+7b_qformer2_256', 'arch_specifier': 'qformer2_256', 'vision_backbone_id': 'clip-vit-l-336px', 'llm_backbone_id': 'vicuna-v15-7b', 'image_resize_strategy': 'letterbox', 'llm_max_length': 2048, 'align_epochs': 1, 'align_max_steps': None, 'align_global_batch_size': 256, 'align_per_device_batch_size': 16, 'align_learning_rate': 0.001, 'align_weight_decay': 0.0, 'align_max_grad_norm': 1.0, 'align_lr_scheduler_type': 'linear-warmup+cosine-decay', 'align_warmup_ratio': 0.03, 'align_train_strategy': 'fsdp-shard-grad-op', 'finetune_epochs': 1, 'finetune_max_steps': None, 'finetune_global_batch_size': 128, 'finetune_per_device_batch_size': 16, 'finetune_learning_rate': 2e-05, 'finetune_weight_decay': 0.1, 'finetune_max_grad_norm': 1.0, 'finetune_lr_scheduler_type': 'linear-warmup+cosine-decay', 'finetune_warmup_ratio': 0.03, 'finetune_train_strategy': 'fsdp-full-shard', 'enable_gradient_checkpointing': True, 'enable_mixed_precision_training': True, 'reduce_in_full_precision': False}, 'dataset': {'type': 'llava-v15', 'dataset_id': 'llava-v15', 'align_stage_components': ['download/llava-laion-cc-sbu-558k/chat.json', 'download/llava-laion-cc-sbu-558k'], 'finetune_stage_components': ['download/llava-v1.5-instruct/llava_v1_5_mix665k.json', 'download/llava-v1.5-instruct'], 'dataset_root_dir': 'data'}, 'stage': 'finetune', 'pretrained_checkpoint': None, 'run_id': 'qformer2_256', 'run_root_dir': 'runs', 'seed': 7, 'hf_token': '.hf_token', 'trackers': ['jsonl', 'wandb'], 'wandb_project': 'hf-vlms', 'wandb_entity': 'lilei_stones', 'max_length': 4096} +2024-03-12 14:29:03,626 INFO MainThread:2490955 [wandb_init.py:init():616] starting backend +2024-03-12 14:29:03,626 INFO MainThread:2490955 [wandb_init.py:init():620] setting up manager +2024-03-12 14:29:03,633 INFO MainThread:2490955 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2024-03-12 14:29:03,636 INFO MainThread:2490955 [wandb_init.py:init():628] backend started and connected +2024-03-12 14:29:03,649 INFO MainThread:2490955 [wandb_init.py:init():720] updated telemetry +2024-03-12 14:29:03,675 INFO MainThread:2490955 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout +2024-03-12 14:29:07,875 INFO MainThread:2490955 [wandb_run.py:_on_init():2262] communicating current version +2024-03-12 14:29:07,923 INFO MainThread:2490955 [wandb_run.py:_on_init():2271] got version response upgrade_message: "wandb version 0.16.4 is available! To upgrade, please run:\n $ pip install wandb --upgrade" + +2024-03-12 14:29:07,923 INFO MainThread:2490955 [wandb_init.py:init():804] starting run threads in backend +2024-03-12 14:29:13,120 INFO MainThread:2490955 [wandb_run.py:_console_start():2241] atexit reg +2024-03-12 14:29:13,120 INFO MainThread:2490955 [wandb_run.py:_redirect():2096] redirect: wrap_raw +2024-03-12 14:29:13,120 INFO MainThread:2490955 [wandb_run.py:_redirect():2161] Wrapping output streams. +2024-03-12 14:29:13,120 INFO MainThread:2490955 [wandb_run.py:_redirect():2186] Redirects installed. +2024-03-12 14:29:13,121 INFO MainThread:2490955 [wandb_init.py:init():847] run started, returning control to user process +2024-03-12 22:26:21,885 INFO MainThread:2490955 [wandb_run.py:_finish():1970] finishing run lilei_stones/hf-vlms/yrs4wcl6 +2024-03-12 22:26:21,886 INFO MainThread:2490955 [wandb_run.py:_atexit_cleanup():2210] got exitcode: 0 +2024-03-12 22:26:21,886 INFO MainThread:2490955 [wandb_run.py:_restore():2193] restore +2024-03-12 22:26:21,886 INFO MainThread:2490955 [wandb_run.py:_restore():2199] restore done +2024-03-12 22:26:29,403 INFO MainThread:2490955 [wandb_run.py:_footer_history_summary_info():3866] rendering history +2024-03-12 22:26:29,403 INFO MainThread:2490955 [wandb_run.py:_footer_history_summary_info():3898] rendering summary +2024-03-12 22:26:29,414 INFO MainThread:2490955 [wandb_run.py:_footer_sync_info():3825] logging synced files diff --git a/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/run-yrs4wcl6.wandb b/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/run-yrs4wcl6.wandb new file mode 100644 index 0000000000000000000000000000000000000000..62bcfd9007bc70d1a8067976d6c254f53be0d63d --- /dev/null +++ b/qformer2_256/wandb/run-20240312_142903-yrs4wcl6/run-yrs4wcl6.wandb @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b0119eb5a720af0addd03c34727d5e8d071915c8dcf6bff6f6ffaa10069713df +size 13457018 diff --git a/qformer2_32/checkpoints/latest-checkpoint.pt b/qformer2_32/checkpoints/latest-checkpoint.pt new file mode 100644 index 0000000000000000000000000000000000000000..0bf57feb33d44456bd51f808be30c95e5c6e3b57 --- /dev/null +++ b/qformer2_32/checkpoints/latest-checkpoint.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:40092396c797c1e9981c568a92376a5dca6be7de66b54500b5de9310d226e0cd +size 28820836534 diff --git a/qformer2_32/config.json b/qformer2_32/config.json new file mode 100644 index 0000000000000000000000000000000000000000..d7eca57bf3bb069c093e7bda432f0fd2bff8a7fa --- /dev/null +++ b/qformer2_32/config.json @@ -0,0 +1,60 @@ +{ + "dataset": { + "align_stage_components": [ + "download/llava-laion-cc-sbu-558k/chat.json", + "download/llava-laion-cc-sbu-558k" + ], + "dataset_id": "llava-v15", + "dataset_root_dir": "data", + "finetune_stage_components": [ + "download/llava-v1.5-instruct/llava_v1_5_mix665k.json", + "download/llava-v1.5-instruct" + ], + "type": "llava-v15" + }, + "hf_token": ".hf_token", + "max_length": 4096, + "model": { + "align_epochs": 1, + "align_global_batch_size": 256, + "align_learning_rate": 0.001, + "align_lr_scheduler_type": "linear-warmup+cosine-decay", + "align_max_grad_norm": 1.0, + "align_max_steps": null, + "align_per_device_batch_size": 16, + "align_train_strategy": "fsdp-shard-grad-op", + "align_warmup_ratio": 0.03, + "align_weight_decay": 0.0, + "arch_specifier": "qformer2_32", + "enable_gradient_checkpointing": true, + "enable_mixed_precision_training": true, + "finetune_epochs": 1, + "finetune_global_batch_size": 128, + "finetune_learning_rate": 2e-05, + "finetune_lr_scheduler_type": "linear-warmup+cosine-decay", + "finetune_max_grad_norm": 1.0, + "finetune_max_steps": null, + "finetune_per_device_batch_size": 16, + "finetune_train_strategy": "fsdp-full-shard", + "finetune_warmup_ratio": 0.03, + "finetune_weight_decay": 0.1, + "image_resize_strategy": "letterbox", + "llm_backbone_id": "vicuna-v15-7b", + "llm_max_length": 2048, + "model_id": "one-stage+7b_qformer2_32", + "reduce_in_full_precision": false, + "type": "one-stage+7b", + "vision_backbone_id": "clip-vit-l-336px" + }, + "pretrained_checkpoint": null, + "run_id": "qformer2_32", + "run_root_dir": "runs", + "seed": 7, + "stage": "finetune", + "trackers": [ + "jsonl", + "wandb" + ], + "wandb_entity": "lilei_stones", + "wandb_project": "hf-vlms" +} \ No newline at end of file diff --git a/qformer2_32/config.yaml b/qformer2_32/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..7d65aed2cf5145ff0d900ea040b237c07647e7bd --- /dev/null +++ b/qformer2_32/config.yaml @@ -0,0 +1,53 @@ +dataset: + align_stage_components: + - download/llava-laion-cc-sbu-558k/chat.json + - download/llava-laion-cc-sbu-558k + dataset_id: llava-v15 + dataset_root_dir: data + finetune_stage_components: + - download/llava-v1.5-instruct/llava_v1_5_mix665k.json + - download/llava-v1.5-instruct + type: llava-v15 +hf_token: .hf_token +max_length: 4096 +model: + align_epochs: 1 + align_global_batch_size: 256 + align_learning_rate: 0.001 + align_lr_scheduler_type: linear-warmup+cosine-decay + align_max_grad_norm: 1.0 + align_max_steps: null + align_per_device_batch_size: 16 + align_train_strategy: fsdp-shard-grad-op + align_warmup_ratio: 0.03 + align_weight_decay: 0.0 + arch_specifier: qformer2_32 + enable_gradient_checkpointing: true + enable_mixed_precision_training: true + finetune_epochs: 1 + finetune_global_batch_size: 128 + finetune_learning_rate: 2.0e-05 + finetune_lr_scheduler_type: linear-warmup+cosine-decay + finetune_max_grad_norm: 1.0 + finetune_max_steps: null + finetune_per_device_batch_size: 16 + finetune_train_strategy: fsdp-full-shard + finetune_warmup_ratio: 0.03 + finetune_weight_decay: 0.1 + image_resize_strategy: letterbox + llm_backbone_id: vicuna-v15-7b + llm_max_length: 2048 + model_id: one-stage+7b_qformer2_32 + reduce_in_full_precision: false + type: one-stage+7b + vision_backbone_id: clip-vit-l-336px +pretrained_checkpoint: null +run_id: qformer2_32 +run_root_dir: runs +seed: 7 +stage: finetune +trackers: +- jsonl +- wandb +wandb_entity: lilei_stones +wandb_project: hf-vlms diff --git a/qformer2_32/qformer2_32.jsonl b/qformer2_32/qformer2_32.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..ad50a3443ab5da3dab417f1289be8ab3b79d9f13 --- /dev/null +++ b/qformer2_32/qformer2_32.jsonl @@ -0,0 +1,5198 @@ +{"Finetune/Learning Rate": 1.2903225806451614e-07, "Finetune/Loss": 6.756393909454346, "Finetune/Loss (Raw)": 6.756393909454346, "Finetune/Step": 1, "Finetune/Step Time": 25.559718132019043} +{"Finetune/Learning Rate": 2.580645161290323e-07, "Finetune/Loss": 6.988838195800781, "Finetune/Loss (Raw)": 7.221282005310059, "Finetune/Step": 2, "Finetune/Step Time": 16.157243371009827} +{"Finetune/Learning Rate": 3.870967741935484e-07, "Finetune/Loss": 5.525903701782227, "Finetune/Loss (Raw)": 2.6000354290008545, "Finetune/Step": 3, "Finetune/Step Time": 11.846802552541098} +{"Finetune/Learning Rate": 5.161290322580646e-07, "Finetune/Loss": 4.512779235839844, "Finetune/Loss (Raw)": 1.4734050035476685, "Finetune/Step": 4, "Finetune/Step Time": 9.73515635728836} +{"Finetune/Learning Rate": 6.451612903225807e-07, "Finetune/Loss": 3.8737285137176514, "Finetune/Loss (Raw)": 1.3175264596939087, "Finetune/Step": 5, "Finetune/Step Time": 8.65389895439148} +{"Finetune/Learning Rate": 7.741935483870968e-07, "Finetune/Loss": 3.4516749382019043, "Finetune/Loss (Raw)": 1.3414068222045898, "Finetune/Step": 6, "Finetune/Step Time": 7.877288579940796} +{"Finetune/Learning Rate": 9.032258064516129e-07, "Finetune/Loss": 3.1318299770355225, "Finetune/Loss (Raw)": 1.2127622365951538, "Finetune/Step": 7, "Finetune/Step Time": 7.300677537918091} +{"Finetune/Learning Rate": 1.0322580645161291e-06, "Finetune/Loss": 2.9180374145507812, "Finetune/Loss (Raw)": 1.4214866161346436, "Finetune/Step": 8, "Finetune/Step Time": 7.043136656284332} +{"Finetune/Learning Rate": 1.1612903225806454e-06, "Finetune/Loss": 2.7176034450531006, "Finetune/Loss (Raw)": 1.1141334772109985, "Finetune/Step": 9, "Finetune/Step Time": 6.698729885949029} +{"Finetune/Learning Rate": 1.2903225806451614e-06, "Finetune/Loss": 2.5936317443847656, "Finetune/Loss (Raw)": 1.4778844118118286, "Finetune/Step": 10, "Finetune/Step Time": 6.680484795570374} +{"Finetune/Learning Rate": 1.4193548387096776e-06, "Finetune/Loss": 2.4811394214630127, "Finetune/Loss (Raw)": 1.3562171459197998, "Finetune/Step": 11, "Finetune/Step Time": 6.400553074750033} +{"Finetune/Learning Rate": 1.5483870967741937e-06, "Finetune/Loss": 2.378415584564209, "Finetune/Loss (Raw)": 1.248451590538025, "Finetune/Step": 12, "Finetune/Step Time": 6.181921422481537} +{"Finetune/Learning Rate": 1.67741935483871e-06, "Finetune/Loss": 2.2837798595428467, "Finetune/Loss (Raw)": 1.1481510400772095, "Finetune/Step": 13, "Finetune/Step Time": 5.9927407044630785} +{"Finetune/Learning Rate": 1.8064516129032258e-06, "Finetune/Loss": 2.218845844268799, "Finetune/Loss (Raw)": 1.3747031688690186, "Finetune/Step": 14, "Finetune/Step Time": 5.812684399741037} +{"Finetune/Learning Rate": 1.935483870967742e-06, "Finetune/Loss": 2.1631531715393066, "Finetune/Loss (Raw)": 1.3834552764892578, "Finetune/Step": 15, "Finetune/Step Time": 5.749755382537842} +{"Finetune/Learning Rate": 2.0645161290322582e-06, "Finetune/Loss": 2.0888895988464355, "Finetune/Loss (Raw)": 0.9749372601509094, "Finetune/Step": 16, "Finetune/Step Time": 5.703050807118416} +{"Finetune/Learning Rate": 2.1935483870967745e-06, "Finetune/Loss": 2.034822702407837, "Finetune/Loss (Raw)": 1.1697529554367065, "Finetune/Step": 17, "Finetune/Step Time": 5.598249968360452} +{"Finetune/Learning Rate": 2.3225806451612907e-06, "Finetune/Loss": 1.9775469303131104, "Finetune/Loss (Raw)": 1.0038601160049438, "Finetune/Step": 18, "Finetune/Step Time": 5.533714877234565} +{"Finetune/Learning Rate": 2.4516129032258066e-06, "Finetune/Loss": 1.9345588684082031, "Finetune/Loss (Raw)": 1.160771369934082, "Finetune/Step": 19, "Finetune/Step Time": 5.42417405780993} +{"Finetune/Learning Rate": 2.580645161290323e-06, "Finetune/Loss": 1.8943769931793213, "Finetune/Loss (Raw)": 1.130920171737671, "Finetune/Step": 20, "Finetune/Step Time": 5.311732745170593} +{"Finetune/Learning Rate": 2.709677419354839e-06, "Finetune/Loss": 1.859256625175476, "Finetune/Loss (Raw)": 1.1568485498428345, "Finetune/Step": 21, "Finetune/Step Time": 5.299148366564796} +{"Finetune/Learning Rate": 2.8387096774193553e-06, "Finetune/Loss": 1.8231977224349976, "Finetune/Loss (Raw)": 1.06596040725708, "Finetune/Step": 22, "Finetune/Step Time": 5.235131274570119} +{"Finetune/Learning Rate": 2.967741935483871e-06, "Finetune/Loss": 1.7539821863174438, "Finetune/Loss (Raw)": 0.2312442809343338, "Finetune/Step": 23, "Finetune/Step Time": 5.367493992266447} +{"Finetune/Learning Rate": 3.0967741935483874e-06, "Finetune/Loss": 1.7309036254882812, "Finetune/Loss (Raw)": 1.200096607208252, "Finetune/Step": 24, "Finetune/Step Time": 5.279127438863118} +{"Finetune/Learning Rate": 3.225806451612903e-06, "Finetune/Loss": 1.7041571140289307, "Finetune/Loss (Raw)": 1.0622406005859375, "Finetune/Step": 25, "Finetune/Step Time": 5.224675521850586} +{"Finetune/Learning Rate": 3.35483870967742e-06, "Finetune/Loss": 1.6489447355270386, "Finetune/Loss (Raw)": 0.2686329483985901, "Finetune/Step": 26, "Finetune/Step Time": 5.355510546610906} +{"Finetune/Learning Rate": 3.4838709677419357e-06, "Finetune/Loss": 1.6346688270568848, "Finetune/Loss (Raw)": 1.2634986639022827, "Finetune/Step": 27, "Finetune/Step Time": 5.26979234483507} +{"Finetune/Learning Rate": 3.6129032258064515e-06, "Finetune/Loss": 1.6160169839859009, "Finetune/Loss (Raw)": 1.1124154329299927, "Finetune/Step": 28, "Finetune/Step Time": 5.1988926359585355} +{"Finetune/Learning Rate": 3.741935483870968e-06, "Finetune/Loss": 1.6033827066421509, "Finetune/Loss (Raw)": 1.2496246099472046, "Finetune/Step": 29, "Finetune/Step Time": 5.184215685416913} +{"Finetune/Learning Rate": 3.870967741935484e-06, "Finetune/Loss": 1.5848758220672607, "Finetune/Loss (Raw)": 1.0481692552566528, "Finetune/Step": 30, "Finetune/Step Time": 5.173705514272054} +{"Finetune/Learning Rate": 4.000000000000001e-06, "Finetune/Loss": 1.5725643634796143, "Finetune/Loss (Raw)": 1.2032291889190674, "Finetune/Step": 31, "Finetune/Step Time": 5.137937561158211} +{"Finetune/Learning Rate": 4.1290322580645165e-06, "Finetune/Loss": 1.5571048259735107, "Finetune/Loss (Raw)": 1.0778563022613525, "Finetune/Step": 32, "Finetune/Step Time": 5.159327268600464} +{"Finetune/Learning Rate": 4.258064516129032e-06, "Finetune/Loss": 1.5386760234832764, "Finetune/Loss (Raw)": 0.9489543437957764, "Finetune/Step": 33, "Finetune/Step Time": 5.1322808482430196} +{"Finetune/Learning Rate": 4.387096774193549e-06, "Finetune/Loss": 1.523219347000122, "Finetune/Loss (Raw)": 1.0131523609161377, "Finetune/Step": 34, "Finetune/Step Time": 5.090349449830897} +{"Finetune/Learning Rate": 4.516129032258065e-06, "Finetune/Loss": 1.5169976949691772, "Finetune/Loss (Raw)": 1.305463433265686, "Finetune/Step": 35, "Finetune/Step Time": 5.062282453264509} +{"Finetune/Learning Rate": 4.6451612903225815e-06, "Finetune/Loss": 1.5062142610549927, "Finetune/Loss (Raw)": 1.1287897825241089, "Finetune/Step": 36, "Finetune/Step Time": 5.069472763273451} +{"Finetune/Learning Rate": 4.774193548387097e-06, "Finetune/Loss": 1.4997373819351196, "Finetune/Loss (Raw)": 1.2665672302246094, "Finetune/Step": 37, "Finetune/Step Time": 5.116732848657144} +{"Finetune/Learning Rate": 4.903225806451613e-06, "Finetune/Loss": 1.4898908138275146, "Finetune/Loss (Raw)": 1.125571370124817, "Finetune/Step": 38, "Finetune/Step Time": 5.068706706950539} +{"Finetune/Learning Rate": 5.032258064516129e-06, "Finetune/Loss": 1.4813194274902344, "Finetune/Loss (Raw)": 1.1556059122085571, "Finetune/Step": 39, "Finetune/Step Time": 5.0214407321734305} +{"Finetune/Learning Rate": 5.161290322580646e-06, "Finetune/Loss": 1.4729048013687134, "Finetune/Loss (Raw)": 1.1447352170944214, "Finetune/Step": 40, "Finetune/Step Time": 5.026522082090378} +{"Finetune/Learning Rate": 5.290322580645162e-06, "Finetune/Loss": 1.4677108526229858, "Finetune/Loss (Raw)": 1.2599557638168335, "Finetune/Step": 41, "Finetune/Step Time": 4.971475595381202} +{"Finetune/Learning Rate": 5.419354838709678e-06, "Finetune/Loss": 1.4619437456130981, "Finetune/Loss (Raw)": 1.2254929542541504, "Finetune/Step": 42, "Finetune/Step Time": 4.931206305821736} +{"Finetune/Learning Rate": 5.548387096774194e-06, "Finetune/Loss": 1.454825758934021, "Finetune/Loss (Raw)": 1.1558690071105957, "Finetune/Step": 43, "Finetune/Step Time": 4.92685509836951} +{"Finetune/Learning Rate": 5.677419354838711e-06, "Finetune/Loss": 1.4484580755233765, "Finetune/Loss (Raw)": 1.174645185470581, "Finetune/Step": 44, "Finetune/Step Time": 4.915334495631131} +{"Finetune/Learning Rate": 5.806451612903226e-06, "Finetune/Loss": 1.4395774602890015, "Finetune/Loss (Raw)": 1.0488303899765015, "Finetune/Step": 45, "Finetune/Step Time": 4.9736610465579565} +{"Finetune/Learning Rate": 5.935483870967742e-06, "Finetune/Loss": 1.414703369140625, "Finetune/Loss (Raw)": 0.29537495970726013, "Finetune/Step": 46, "Finetune/Step Time": 5.072945190512615} +{"Finetune/Learning Rate": 6.064516129032259e-06, "Finetune/Loss": 1.4088406562805176, "Finetune/Loss (Raw)": 1.1391515731811523, "Finetune/Step": 47, "Finetune/Step Time": 5.059368721982266} +{"Finetune/Learning Rate": 6.193548387096775e-06, "Finetune/Loss": 1.4028522968292236, "Finetune/Loss (Raw)": 1.1213957071304321, "Finetune/Step": 48, "Finetune/Step Time": 5.0643997093041735} +{"Finetune/Learning Rate": 6.3225806451612906e-06, "Finetune/Loss": 1.3979815244674683, "Finetune/Loss (Raw)": 1.164191722869873, "Finetune/Step": 49, "Finetune/Step Time": 5.037692546844482} +{"Finetune/Learning Rate": 6.451612903225806e-06, "Finetune/Loss": 1.390769600868225, "Finetune/Loss (Raw)": 1.0373854637145996, "Finetune/Step": 50, "Finetune/Step Time": 5.00078046798706} +{"Finetune/Learning Rate": 6.580645161290323e-06, "Finetune/Loss": 1.3859739303588867, "Finetune/Loss (Raw)": 1.1461825370788574, "Finetune/Step": 51, "Finetune/Step Time": 4.987367816999847} +{"Finetune/Learning Rate": 6.70967741935484e-06, "Finetune/Loss": 1.3826881647109985, "Finetune/Loss (Raw)": 1.2151175737380981, "Finetune/Step": 52, "Finetune/Step Time": 4.976546058287988} +{"Finetune/Learning Rate": 6.838709677419355e-06, "Finetune/Loss": 1.374049425125122, "Finetune/Loss (Raw)": 0.9248284101486206, "Finetune/Step": 53, "Finetune/Step Time": 4.988106628633895} +{"Finetune/Learning Rate": 6.967741935483871e-06, "Finetune/Loss": 1.3659292459487915, "Finetune/Loss (Raw)": 0.9355676770210266, "Finetune/Step": 54, "Finetune/Step Time": 4.980701018262793} +{"Finetune/Learning Rate": 7.096774193548388e-06, "Finetune/Loss": 1.3622725009918213, "Finetune/Loss (Raw)": 1.1648024320602417, "Finetune/Step": 55, "Finetune/Step Time": 4.981637998060747} +{"Finetune/Learning Rate": 7.225806451612903e-06, "Finetune/Loss": 1.3579027652740479, "Finetune/Loss (Raw)": 1.1175669431686401, "Finetune/Step": 56, "Finetune/Step Time": 4.990835330315998} +{"Finetune/Learning Rate": 7.35483870967742e-06, "Finetune/Loss": 1.3520245552062988, "Finetune/Loss (Raw)": 1.0228521823883057, "Finetune/Step": 57, "Finetune/Step Time": 4.967195724186144} +{"Finetune/Learning Rate": 7.483870967741936e-06, "Finetune/Loss": 1.347023367881775, "Finetune/Loss (Raw)": 1.0619544982910156, "Finetune/Step": 58, "Finetune/Step Time": 4.992353258461788} +{"Finetune/Learning Rate": 7.612903225806451e-06, "Finetune/Loss": 1.3427077531814575, "Finetune/Loss (Raw)": 1.0924029350280762, "Finetune/Step": 59, "Finetune/Step Time": 4.998422036736699} +{"Finetune/Learning Rate": 7.741935483870968e-06, "Finetune/Loss": 1.3386160135269165, "Finetune/Loss (Raw)": 1.0971958637237549, "Finetune/Step": 60, "Finetune/Step Time": 4.972740395863851} +{"Finetune/Learning Rate": 7.870967741935484e-06, "Finetune/Loss": 1.3355371952056885, "Finetune/Loss (Raw)": 1.1508193016052246, "Finetune/Step": 61, "Finetune/Step Time": 4.956511212176964} +{"Finetune/Learning Rate": 8.000000000000001e-06, "Finetune/Loss": 1.3334004878997803, "Finetune/Loss (Raw)": 1.2030550241470337, "Finetune/Step": 62, "Finetune/Step Time": 4.93185886644548} +{"Finetune/Learning Rate": 8.129032258064517e-06, "Finetune/Loss": 1.3315191268920898, "Finetune/Loss (Raw)": 1.2148648500442505, "Finetune/Step": 63, "Finetune/Step Time": 4.910368510654995} +{"Finetune/Learning Rate": 8.258064516129033e-06, "Finetune/Loss": 1.3273473978042603, "Finetune/Loss (Raw)": 1.064539909362793, "Finetune/Step": 64, "Finetune/Step Time": 4.9449238404631615} +{"Finetune/Learning Rate": 8.387096774193549e-06, "Finetune/Loss": 1.323078989982605, "Finetune/Loss (Raw)": 1.0498994588851929, "Finetune/Step": 65, "Finetune/Step Time": 4.937969405834491} +{"Finetune/Learning Rate": 8.516129032258065e-06, "Finetune/Loss": 1.3183441162109375, "Finetune/Loss (Raw)": 1.010581135749817, "Finetune/Step": 66, "Finetune/Step Time": 4.922883221597383} +{"Finetune/Learning Rate": 8.64516129032258e-06, "Finetune/Loss": 1.3121943473815918, "Finetune/Loss (Raw)": 0.9063062071800232, "Finetune/Step": 67, "Finetune/Step Time": 4.897652807520397} +{"Finetune/Learning Rate": 8.774193548387098e-06, "Finetune/Loss": 1.3096356391906738, "Finetune/Loss (Raw)": 1.1381975412368774, "Finetune/Step": 68, "Finetune/Step Time": 4.894338996971355} +{"Finetune/Learning Rate": 8.903225806451614e-06, "Finetune/Loss": 1.307422161102295, "Finetune/Loss (Raw)": 1.1569066047668457, "Finetune/Step": 69, "Finetune/Step Time": 4.870148403057153} +{"Finetune/Learning Rate": 9.03225806451613e-06, "Finetune/Loss": 1.305782437324524, "Finetune/Loss (Raw)": 1.1926445960998535, "Finetune/Step": 70, "Finetune/Step Time": 4.849939390591213} +{"Finetune/Learning Rate": 9.161290322580645e-06, "Finetune/Loss": 1.3004591464996338, "Finetune/Loss (Raw)": 0.9278355836868286, "Finetune/Step": 71, "Finetune/Step Time": 4.831880448569714} +{"Finetune/Learning Rate": 9.290322580645163e-06, "Finetune/Loss": 1.296641230583191, "Finetune/Loss (Raw)": 1.0255675315856934, "Finetune/Step": 72, "Finetune/Step Time": 4.808054125971264} +{"Finetune/Learning Rate": 9.419354838709677e-06, "Finetune/Loss": 1.291560173034668, "Finetune/Loss (Raw)": 0.9257193803787231, "Finetune/Step": 73, "Finetune/Step Time": 4.79008472455691} +{"Finetune/Learning Rate": 9.548387096774195e-06, "Finetune/Loss": 1.2867850065231323, "Finetune/Loss (Raw)": 0.9381915926933289, "Finetune/Step": 74, "Finetune/Step Time": 4.772224461710131} +{"Finetune/Learning Rate": 9.67741935483871e-06, "Finetune/Loss": 1.2806801795959473, "Finetune/Loss (Raw)": 0.8289226293563843, "Finetune/Step": 75, "Finetune/Step Time": 4.76683669090271} +{"Finetune/Learning Rate": 9.806451612903226e-06, "Finetune/Loss": 1.2766222953796387, "Finetune/Loss (Raw)": 0.9722849726676941, "Finetune/Step": 76, "Finetune/Step Time": 4.759102102957274} +{"Finetune/Learning Rate": 9.935483870967742e-06, "Finetune/Loss": 1.2745240926742554, "Finetune/Loss (Raw)": 1.115057110786438, "Finetune/Step": 77, "Finetune/Step Time": 4.738301809732016} +{"Finetune/Learning Rate": 1.0064516129032258e-05, "Finetune/Loss": 1.2738271951675415, "Finetune/Loss (Raw)": 1.2201757431030273, "Finetune/Step": 78, "Finetune/Step Time": 4.718592821023403} +{"Finetune/Learning Rate": 1.0193548387096774e-05, "Finetune/Loss": 1.272646188735962, "Finetune/Loss (Raw)": 1.1805206537246704, "Finetune/Step": 79, "Finetune/Step Time": 4.703135590010051} +{"Finetune/Learning Rate": 1.0322580645161291e-05, "Finetune/Loss": 1.2689924240112305, "Finetune/Loss (Raw)": 0.9803488850593567, "Finetune/Step": 80, "Finetune/Step Time": 4.708546394109726} +{"Finetune/Learning Rate": 1.0451612903225807e-05, "Finetune/Loss": 1.2674801349639893, "Finetune/Loss (Raw)": 1.146500825881958, "Finetune/Step": 81, "Finetune/Step Time": 4.7057679523656395} +{"Finetune/Learning Rate": 1.0580645161290325e-05, "Finetune/Loss": 1.2655059099197388, "Finetune/Loss (Raw)": 1.1055951118469238, "Finetune/Step": 82, "Finetune/Step Time": 4.6889290024594565} +{"Finetune/Learning Rate": 1.070967741935484e-05, "Finetune/Loss": 1.261765718460083, "Finetune/Loss (Raw)": 0.955069899559021, "Finetune/Step": 83, "Finetune/Step Time": 4.696989909712091} +{"Finetune/Learning Rate": 1.0838709677419356e-05, "Finetune/Loss": 1.260933518409729, "Finetune/Loss (Raw)": 1.1918506622314453, "Finetune/Step": 84, "Finetune/Step Time": 4.698683636529105} +{"Finetune/Learning Rate": 1.096774193548387e-05, "Finetune/Loss": 1.2585428953170776, "Finetune/Loss (Raw)": 1.057733178138733, "Finetune/Step": 85, "Finetune/Step Time": 4.684595826092888} +{"Finetune/Learning Rate": 1.1096774193548388e-05, "Finetune/Loss": 1.2480708360671997, "Finetune/Loss (Raw)": 0.35795098543167114, "Finetune/Step": 86, "Finetune/Step Time": 4.736994036408358} +{"Finetune/Learning Rate": 1.1225806451612904e-05, "Finetune/Loss": 1.2456079721450806, "Finetune/Loss (Raw)": 1.033797025680542, "Finetune/Step": 87, "Finetune/Step Time": 4.723083844130066} +{"Finetune/Learning Rate": 1.1354838709677421e-05, "Finetune/Loss": 1.245484471321106, "Finetune/Loss (Raw)": 1.2347346544265747, "Finetune/Step": 88, "Finetune/Step Time": 4.730379250916568} +{"Finetune/Learning Rate": 1.1483870967741937e-05, "Finetune/Loss": 1.2435888051986694, "Finetune/Loss (Raw)": 1.0767796039581299, "Finetune/Step": 89, "Finetune/Step Time": 4.716632031322865} +{"Finetune/Learning Rate": 1.1612903225806453e-05, "Finetune/Loss": 1.2424994707107544, "Finetune/Loss (Raw)": 1.145547866821289, "Finetune/Step": 90, "Finetune/Step Time": 4.724131017261081} +{"Finetune/Learning Rate": 1.1741935483870967e-05, "Finetune/Loss": 1.2417619228363037, "Finetune/Loss (Raw)": 1.1753736734390259, "Finetune/Step": 91, "Finetune/Step Time": 4.726539546316797} +{"Finetune/Learning Rate": 1.1870967741935484e-05, "Finetune/Loss": 1.2413893938064575, "Finetune/Loss (Raw)": 1.207489252090454, "Finetune/Step": 92, "Finetune/Step Time": 4.714933493862981} +{"Finetune/Learning Rate": 1.2e-05, "Finetune/Loss": 1.238023042678833, "Finetune/Loss (Raw)": 0.9283209443092346, "Finetune/Step": 93, "Finetune/Step Time": 4.705873420161586} +{"Finetune/Learning Rate": 1.2129032258064518e-05, "Finetune/Loss": 1.2371960878372192, "Finetune/Loss (Raw)": 1.1602962017059326, "Finetune/Step": 94, "Finetune/Step Time": 4.689267881373142} +{"Finetune/Learning Rate": 1.2258064516129034e-05, "Finetune/Loss": 1.2358161211013794, "Finetune/Loss (Raw)": 1.1060932874679565, "Finetune/Step": 95, "Finetune/Step Time": 4.67698231496309} +{"Finetune/Learning Rate": 1.238709677419355e-05, "Finetune/Loss": 1.2342784404754639, "Finetune/Loss (Raw)": 1.0881974697113037, "Finetune/Step": 96, "Finetune/Step Time": 4.679396485288938} +{"Finetune/Learning Rate": 1.2516129032258067e-05, "Finetune/Loss": 1.2322012186050415, "Finetune/Loss (Raw)": 1.0327972173690796, "Finetune/Step": 97, "Finetune/Step Time": 4.674191238953895} +{"Finetune/Learning Rate": 1.2645161290322581e-05, "Finetune/Loss": 1.2311837673187256, "Finetune/Loss (Raw)": 1.1324915885925293, "Finetune/Step": 98, "Finetune/Step Time": 4.66241480866257} +{"Finetune/Learning Rate": 1.2774193548387097e-05, "Finetune/Loss": 1.2210414409637451, "Finetune/Loss (Raw)": 0.2270854264497757, "Finetune/Step": 99, "Finetune/Step Time": 4.699242815826878} +{"Finetune/Learning Rate": 1.2903225806451613e-05, "Finetune/Loss": 1.221058964729309, "Finetune/Loss (Raw)": 1.2227994203567505, "Finetune/Step": 100, "Finetune/Step Time": 4.691341950893402} +{"Finetune/Learning Rate": 1.303225806451613e-05, "Finetune/Loss": 1.2194232940673828, "Finetune/Loss (Raw)": 1.0558552742004395, "Finetune/Step": 101, "Finetune/Step Time": 4.6927257339553075} +{"Finetune/Learning Rate": 1.3161290322580646e-05, "Finetune/Loss": 1.2193794250488281, "Finetune/Loss (Raw)": 1.2149442434310913, "Finetune/Step": 102, "Finetune/Step Time": 4.675601201898911} +{"Finetune/Learning Rate": 1.3290322580645164e-05, "Finetune/Loss": 1.2104191780090332, "Finetune/Loss (Raw)": 0.2964695394039154, "Finetune/Step": 103, "Finetune/Step Time": 4.712035380520867} +{"Finetune/Learning Rate": 1.341935483870968e-05, "Finetune/Loss": 1.2099422216415405, "Finetune/Loss (Raw)": 1.160814881324768, "Finetune/Step": 104, "Finetune/Step Time": 4.698244330974726} +{"Finetune/Learning Rate": 1.3548387096774194e-05, "Finetune/Loss": 1.2086013555526733, "Finetune/Loss (Raw)": 1.0691615343093872, "Finetune/Step": 105, "Finetune/Step Time": 4.680929208937146} +{"Finetune/Learning Rate": 1.367741935483871e-05, "Finetune/Loss": 1.20730721950531, "Finetune/Loss (Raw)": 1.0714179277420044, "Finetune/Step": 106, "Finetune/Step Time": 4.668170465613311} +{"Finetune/Learning Rate": 1.3806451612903227e-05, "Finetune/Loss": 1.2066324949264526, "Finetune/Loss (Raw)": 1.1351284980773926, "Finetune/Step": 107, "Finetune/Step Time": 4.656716041475813} +{"Finetune/Learning Rate": 1.3935483870967743e-05, "Finetune/Loss": 1.2048996686935425, "Finetune/Loss (Raw)": 1.0194734334945679, "Finetune/Step": 108, "Finetune/Step Time": 4.648261906924071} +{"Finetune/Learning Rate": 1.406451612903226e-05, "Finetune/Loss": 1.204376459121704, "Finetune/Loss (Raw)": 1.1478744745254517, "Finetune/Step": 109, "Finetune/Step Time": 4.637304673501111} +{"Finetune/Learning Rate": 1.4193548387096776e-05, "Finetune/Loss": 1.203081488609314, "Finetune/Loss (Raw)": 1.0619415044784546, "Finetune/Step": 110, "Finetune/Step Time": 4.62483023296703} +{"Finetune/Learning Rate": 1.4322580645161292e-05, "Finetune/Loss": 1.2024459838867188, "Finetune/Loss (Raw)": 1.1325092315673828, "Finetune/Step": 111, "Finetune/Step Time": 4.629901505805351} +{"Finetune/Learning Rate": 1.4451612903225806e-05, "Finetune/Loss": 1.2014755010604858, "Finetune/Loss (Raw)": 1.093754768371582, "Finetune/Step": 112, "Finetune/Step Time": 4.630905941128731} +{"Finetune/Learning Rate": 1.4580645161290324e-05, "Finetune/Loss": 1.2009072303771973, "Finetune/Loss (Raw)": 1.1372706890106201, "Finetune/Step": 113, "Finetune/Step Time": 4.6338455845824384} +{"Finetune/Learning Rate": 1.470967741935484e-05, "Finetune/Loss": 1.1925736665725708, "Finetune/Loss (Raw)": 0.2508881688117981, "Finetune/Step": 114, "Finetune/Step Time": 4.675132128230312} +{"Finetune/Learning Rate": 1.4838709677419357e-05, "Finetune/Loss": 1.191778302192688, "Finetune/Loss (Raw)": 1.1010936498641968, "Finetune/Step": 115, "Finetune/Step Time": 4.664840289820796} +{"Finetune/Learning Rate": 1.4967741935483873e-05, "Finetune/Loss": 1.189552664756775, "Finetune/Loss (Raw)": 0.9336130023002625, "Finetune/Step": 116, "Finetune/Step Time": 4.675447260511333} +{"Finetune/Learning Rate": 1.5096774193548389e-05, "Finetune/Loss": 1.1889071464538574, "Finetune/Loss (Raw)": 1.1140103340148926, "Finetune/Step": 117, "Finetune/Step Time": 4.670095164551694} +{"Finetune/Learning Rate": 1.5225806451612903e-05, "Finetune/Loss": 1.1885173320770264, "Finetune/Loss (Raw)": 1.1429160833358765, "Finetune/Step": 118, "Finetune/Step Time": 4.659015661579067} +{"Finetune/Learning Rate": 1.535483870967742e-05, "Finetune/Loss": 1.187376856803894, "Finetune/Loss (Raw)": 1.0527968406677246, "Finetune/Step": 119, "Finetune/Step Time": 4.649793963472383} +{"Finetune/Learning Rate": 1.5483870967741936e-05, "Finetune/Loss": 1.186582326889038, "Finetune/Loss (Raw)": 1.0920393466949463, "Finetune/Step": 120, "Finetune/Step Time": 4.645192716519038} +{"Finetune/Learning Rate": 1.5612903225806454e-05, "Finetune/Loss": 1.1852737665176392, "Finetune/Loss (Raw)": 1.0282652378082275, "Finetune/Step": 121, "Finetune/Step Time": 4.659623835697647} +{"Finetune/Learning Rate": 1.5741935483870968e-05, "Finetune/Loss": 1.1835074424743652, "Finetune/Loss (Raw)": 0.9697718024253845, "Finetune/Step": 122, "Finetune/Step Time": 4.6500771827385075} +{"Finetune/Learning Rate": 1.5870967741935485e-05, "Finetune/Loss": 1.18220055103302, "Finetune/Loss (Raw)": 1.0227702856063843, "Finetune/Step": 123, "Finetune/Step Time": 4.6402222393004875} +{"Finetune/Learning Rate": 1.6000000000000003e-05, "Finetune/Loss": 1.1817837953567505, "Finetune/Loss (Raw)": 1.1305245161056519, "Finetune/Step": 124, "Finetune/Step Time": 4.632923137757086} +{"Finetune/Learning Rate": 1.6129032258064517e-05, "Finetune/Loss": 1.1804676055908203, "Finetune/Loss (Raw)": 1.017223834991455, "Finetune/Step": 125, "Finetune/Step Time": 4.625945554733276} +{"Finetune/Learning Rate": 1.6258064516129034e-05, "Finetune/Loss": 1.1798903942108154, "Finetune/Loss (Raw)": 1.1077531576156616, "Finetune/Step": 126, "Finetune/Step Time": 4.637451523826236} +{"Finetune/Learning Rate": 1.638709677419355e-05, "Finetune/Loss": 1.178405523300171, "Finetune/Loss (Raw)": 0.9913174510002136, "Finetune/Step": 127, "Finetune/Step Time": 4.627047411100132} +{"Finetune/Learning Rate": 1.6516129032258066e-05, "Finetune/Loss": 1.1780672073364258, "Finetune/Loss (Raw)": 1.1351022720336914, "Finetune/Step": 128, "Finetune/Step Time": 4.61973056383431} +{"Finetune/Learning Rate": 1.6645161290322583e-05, "Finetune/Loss": 1.1343185901641846, "Finetune/Loss (Raw)": 1.156572699546814, "Finetune/Step": 129, "Finetune/Step Time": 4.447232672944665} +{"Finetune/Learning Rate": 1.6774193548387098e-05, "Finetune/Loss": 1.0868645906448364, "Finetune/Loss (Raw)": 1.1471772193908691, "Finetune/Step": 130, "Finetune/Step Time": 4.430007420480251} +{"Finetune/Learning Rate": 1.6903225806451615e-05, "Finetune/Loss": 1.0751985311508179, "Finetune/Loss (Raw)": 1.1067695617675781, "Finetune/Step": 131, "Finetune/Step Time": 4.429597590118647} +{"Finetune/Learning Rate": 1.703225806451613e-05, "Finetune/Loss": 1.072368860244751, "Finetune/Loss (Raw)": 1.1112104654312134, "Finetune/Step": 132, "Finetune/Step Time": 4.429737892001867} +{"Finetune/Learning Rate": 1.7161290322580647e-05, "Finetune/Loss": 1.06911301612854, "Finetune/Loss (Raw)": 0.9007814526557922, "Finetune/Step": 133, "Finetune/Step Time": 4.439595429226756} +{"Finetune/Learning Rate": 1.729032258064516e-05, "Finetune/Loss": 1.0683670043945312, "Finetune/Loss (Raw)": 1.245910406112671, "Finetune/Step": 134, "Finetune/Step Time": 4.435443660244346} +{"Finetune/Learning Rate": 1.741935483870968e-05, "Finetune/Loss": 1.0666955709457397, "Finetune/Loss (Raw)": 0.9988220930099487, "Finetune/Step": 135, "Finetune/Step Time": 4.437543205916882} +{"Finetune/Learning Rate": 1.7548387096774196e-05, "Finetune/Loss": 1.064096450805664, "Finetune/Loss (Raw)": 1.0887969732284546, "Finetune/Step": 136, "Finetune/Step Time": 4.4226576536893845} +{"Finetune/Learning Rate": 1.7677419354838713e-05, "Finetune/Loss": 1.0630344152450562, "Finetune/Loss (Raw)": 0.9781990647315979, "Finetune/Step": 137, "Finetune/Step Time": 4.425190346315503} +{"Finetune/Learning Rate": 1.7806451612903228e-05, "Finetune/Loss": 1.060280203819275, "Finetune/Loss (Raw)": 1.1253432035446167, "Finetune/Step": 138, "Finetune/Step Time": 4.3963794726878405} +{"Finetune/Learning Rate": 1.7935483870967742e-05, "Finetune/Loss": 1.0568846464157104, "Finetune/Loss (Raw)": 0.921586275100708, "Finetune/Step": 139, "Finetune/Step Time": 4.397781303152442} +{"Finetune/Learning Rate": 1.806451612903226e-05, "Finetune/Loss": 1.0572054386138916, "Finetune/Loss (Raw)": 1.2895088195800781, "Finetune/Step": 140, "Finetune/Step Time": 4.391804343089461} +{"Finetune/Learning Rate": 1.8193548387096777e-05, "Finetune/Loss": 1.0563323497772217, "Finetune/Loss (Raw)": 1.0363866090774536, "Finetune/Step": 141, "Finetune/Step Time": 4.3855719566345215} +{"Finetune/Learning Rate": 1.832258064516129e-05, "Finetune/Loss": 1.0536818504333496, "Finetune/Loss (Raw)": 1.0354537963867188, "Finetune/Step": 142, "Finetune/Step Time": 4.425460318103433} +{"Finetune/Learning Rate": 1.845161290322581e-05, "Finetune/Loss": 1.0504602193832397, "Finetune/Loss (Raw)": 0.9710733294487, "Finetune/Step": 143, "Finetune/Step Time": 4.439600044861436} +{"Finetune/Learning Rate": 1.8580645161290326e-05, "Finetune/Loss": 1.0458672046661377, "Finetune/Loss (Raw)": 0.3870387077331543, "Finetune/Step": 144, "Finetune/Step Time": 4.470251027494669} +{"Finetune/Learning Rate": 1.870967741935484e-05, "Finetune/Loss": 1.0394948720932007, "Finetune/Loss (Raw)": 0.3540947139263153, "Finetune/Step": 145, "Finetune/Step Time": 4.505656864494085} +{"Finetune/Learning Rate": 1.8838709677419354e-05, "Finetune/Loss": 1.039281964302063, "Finetune/Loss (Raw)": 0.9766167402267456, "Finetune/Step": 146, "Finetune/Step Time": 4.495890229940414} +{"Finetune/Learning Rate": 1.896774193548387e-05, "Finetune/Loss": 1.0372899770736694, "Finetune/Loss (Raw)": 0.905788242816925, "Finetune/Step": 147, "Finetune/Step Time": 4.502027478069067} +{"Finetune/Learning Rate": 1.909677419354839e-05, "Finetune/Loss": 1.0369361639022827, "Finetune/Loss (Raw)": 1.0856295824050903, "Finetune/Step": 148, "Finetune/Step Time": 4.504585854709148} +{"Finetune/Learning Rate": 1.9225806451612907e-05, "Finetune/Loss": 1.0367443561553955, "Finetune/Loss (Raw)": 1.1322875022888184, "Finetune/Step": 149, "Finetune/Step Time": 4.494022365659475} +{"Finetune/Learning Rate": 1.935483870967742e-05, "Finetune/Loss": 1.0373106002807617, "Finetune/Loss (Raw)": 1.1384612321853638, "Finetune/Step": 150, "Finetune/Step Time": 4.497750177979469} +{"Finetune/Learning Rate": 1.948387096774194e-05, "Finetune/Loss": 1.0441124439239502, "Finetune/Loss (Raw)": 1.1018630266189575, "Finetune/Step": 151, "Finetune/Step Time": 4.464414218440652} +{"Finetune/Learning Rate": 1.9612903225806452e-05, "Finetune/Loss": 1.0430681705474854, "Finetune/Loss (Raw)": 1.0664399862289429, "Finetune/Step": 152, "Finetune/Step Time": 4.470320239663124} +{"Finetune/Learning Rate": 1.974193548387097e-05, "Finetune/Loss": 1.0438697338104248, "Finetune/Loss (Raw)": 1.164848804473877, "Finetune/Step": 153, "Finetune/Step Time": 4.47963454015553} +{"Finetune/Learning Rate": 1.9870967741935484e-05, "Finetune/Loss": 1.050683856010437, "Finetune/Loss (Raw)": 1.1408374309539795, "Finetune/Step": 154, "Finetune/Step Time": 4.440315891057253} +{"Finetune/Learning Rate": 2e-05, "Finetune/Loss": 1.050298810005188, "Finetune/Loss (Raw)": 1.214214563369751, "Finetune/Step": 155, "Finetune/Step Time": 4.447756744921207} +{"Finetune/Learning Rate": 1.9999998059597617e-05, "Finetune/Loss": 1.0439434051513672, "Finetune/Loss (Raw)": 0.29892054200172424, "Finetune/Step": 156, "Finetune/Step Time": 4.48940428905189} +{"Finetune/Learning Rate": 1.9999992238391222e-05, "Finetune/Loss": 1.042668104171753, "Finetune/Loss (Raw)": 1.086381435394287, "Finetune/Step": 157, "Finetune/Step Time": 4.479449667036533} +{"Finetune/Learning Rate": 1.999998253638307e-05, "Finetune/Loss": 1.0431710481643677, "Finetune/Loss (Raw)": 1.1125478744506836, "Finetune/Step": 158, "Finetune/Step Time": 4.4699313919991255} +{"Finetune/Learning Rate": 1.999996895357693e-05, "Finetune/Loss": 1.0422217845916748, "Finetune/Loss (Raw)": 1.081709384918213, "Finetune/Step": 159, "Finetune/Step Time": 4.46255212649703} +{"Finetune/Learning Rate": 1.9999951489978074e-05, "Finetune/Loss": 1.0417673587799072, "Finetune/Loss (Raw)": 1.019707202911377, "Finetune/Step": 160, "Finetune/Step Time": 4.451987283304334} +{"Finetune/Learning Rate": 1.9999930145593275e-05, "Finetune/Loss": 1.0424548387527466, "Finetune/Loss (Raw)": 1.0369490385055542, "Finetune/Step": 161, "Finetune/Step Time": 4.448323262855411} +{"Finetune/Learning Rate": 1.9999904920430818e-05, "Finetune/Loss": 1.0444759130477905, "Finetune/Loss (Raw)": 1.271852970123291, "Finetune/Step": 162, "Finetune/Step Time": 4.447047721594572} +{"Finetune/Learning Rate": 1.9999875814500492e-05, "Finetune/Loss": 1.0427747964859009, "Finetune/Loss (Raw)": 1.0877106189727783, "Finetune/Step": 163, "Finetune/Step Time": 4.447164572775364} +{"Finetune/Learning Rate": 1.99998428278136e-05, "Finetune/Loss": 1.0415102243423462, "Finetune/Loss (Raw)": 0.9669220447540283, "Finetune/Step": 164, "Finetune/Step Time": 4.4448920004069805} +{"Finetune/Learning Rate": 1.9999805960382928e-05, "Finetune/Loss": 1.0398719310760498, "Finetune/Loss (Raw)": 1.0568677186965942, "Finetune/Step": 165, "Finetune/Step Time": 4.422018192708492} +{"Finetune/Learning Rate": 1.99997652122228e-05, "Finetune/Loss": 1.0388587713241577, "Finetune/Loss (Raw)": 0.9958874583244324, "Finetune/Step": 166, "Finetune/Step Time": 4.42397534288466} +{"Finetune/Learning Rate": 1.9999720583349015e-05, "Finetune/Loss": 1.038878083229065, "Finetune/Loss (Raw)": 1.158083438873291, "Finetune/Step": 167, "Finetune/Step Time": 4.426246473565698} +{"Finetune/Learning Rate": 1.9999672073778905e-05, "Finetune/Loss": 1.0376842021942139, "Finetune/Loss (Raw)": 0.9919185042381287, "Finetune/Step": 168, "Finetune/Step Time": 4.414645092561841} +{"Finetune/Learning Rate": 1.999961968353129e-05, "Finetune/Loss": 1.0354454517364502, "Finetune/Loss (Raw)": 0.9733970761299133, "Finetune/Step": 169, "Finetune/Step Time": 4.419678432866931} +{"Finetune/Learning Rate": 1.9999563412626503e-05, "Finetune/Loss": 1.0287078619003296, "Finetune/Loss (Raw)": 0.3630765676498413, "Finetune/Step": 170, "Finetune/Step Time": 4.457724509760737} +{"Finetune/Learning Rate": 1.9999503261086378e-05, "Finetune/Loss": 1.0275607109069824, "Finetune/Loss (Raw)": 1.009029746055603, "Finetune/Step": 171, "Finetune/Step Time": 4.455643851310015} +{"Finetune/Learning Rate": 1.999943922893426e-05, "Finetune/Loss": 1.0261743068695068, "Finetune/Loss (Raw)": 0.9971907734870911, "Finetune/Step": 172, "Finetune/Step Time": 4.458527425304055} +{"Finetune/Learning Rate": 1.9999371316195005e-05, "Finetune/Loss": 1.0274271965026855, "Finetune/Loss (Raw)": 1.209212064743042, "Finetune/Step": 173, "Finetune/Step Time": 4.4262816440314054} +{"Finetune/Learning Rate": 1.9999299522894957e-05, "Finetune/Loss": 1.0321757793426514, "Finetune/Loss (Raw)": 0.9031805992126465, "Finetune/Step": 174, "Finetune/Step Time": 4.384265469387174} +{"Finetune/Learning Rate": 1.9999223849061987e-05, "Finetune/Loss": 1.0326306819915771, "Finetune/Loss (Raw)": 1.197380542755127, "Finetune/Step": 175, "Finetune/Step Time": 4.376027435064316} +{"Finetune/Learning Rate": 1.999914429472546e-05, "Finetune/Loss": 1.0266832113265991, "Finetune/Loss (Raw)": 0.36011195182800293, "Finetune/Step": 176, "Finetune/Step Time": 4.400413991883397} +{"Finetune/Learning Rate": 1.9999060859916247e-05, "Finetune/Loss": 1.0252747535705566, "Finetune/Loss (Raw)": 0.9839119911193848, "Finetune/Step": 177, "Finetune/Step Time": 4.403093969449401} +{"Finetune/Learning Rate": 1.9998973544666734e-05, "Finetune/Loss": 1.0266557931900024, "Finetune/Loss (Raw)": 1.2141615152359009, "Finetune/Step": 178, "Finetune/Step Time": 4.410608785226941} +{"Finetune/Learning Rate": 1.9998882349010798e-05, "Finetune/Loss": 1.0248031616210938, "Finetune/Loss (Raw)": 0.9090489745140076, "Finetune/Step": 179, "Finetune/Step Time": 4.40856346487999} +{"Finetune/Learning Rate": 1.999878727298384e-05, "Finetune/Loss": 1.023693561553955, "Finetune/Loss (Raw)": 1.0730780363082886, "Finetune/Step": 180, "Finetune/Step Time": 4.41678211838007} +{"Finetune/Learning Rate": 1.999868831662274e-05, "Finetune/Loss": 1.0245040655136108, "Finetune/Loss (Raw)": 1.0285857915878296, "Finetune/Step": 181, "Finetune/Step Time": 4.39705009944737} +{"Finetune/Learning Rate": 1.9998585479965918e-05, "Finetune/Loss": 1.0255229473114014, "Finetune/Loss (Raw)": 1.0659849643707275, "Finetune/Step": 182, "Finetune/Step Time": 4.4016495775431395} +{"Finetune/Learning Rate": 1.999847876305328e-05, "Finetune/Loss": 1.0244115591049194, "Finetune/Loss (Raw)": 1.0225374698638916, "Finetune/Step": 183, "Finetune/Step Time": 4.390705091878772} +{"Finetune/Learning Rate": 1.9998368165926237e-05, "Finetune/Loss": 1.0247828960418701, "Finetune/Loss (Raw)": 1.1651052236557007, "Finetune/Step": 184, "Finetune/Step Time": 4.37717667594552} +{"Finetune/Learning Rate": 1.9998253688627706e-05, "Finetune/Loss": 1.0241012573242188, "Finetune/Loss (Raw)": 0.9356112480163574, "Finetune/Step": 185, "Finetune/Step Time": 4.380655923858285} +{"Finetune/Learning Rate": 1.9998135331202124e-05, "Finetune/Loss": 1.0237770080566406, "Finetune/Loss (Raw)": 1.0204331874847412, "Finetune/Step": 186, "Finetune/Step Time": 4.371033865958452} +{"Finetune/Learning Rate": 1.9998013093695413e-05, "Finetune/Loss": 1.0231657028198242, "Finetune/Loss (Raw)": 1.0141575336456299, "Finetune/Step": 187, "Finetune/Step Time": 4.356723239645362} +{"Finetune/Learning Rate": 1.9997886976155018e-05, "Finetune/Loss": 1.0235378742218018, "Finetune/Loss (Raw)": 1.1448315382003784, "Finetune/Step": 188, "Finetune/Step Time": 4.355249751359224} +{"Finetune/Learning Rate": 1.9997756978629876e-05, "Finetune/Loss": 1.022457480430603, "Finetune/Loss (Raw)": 1.0125417709350586, "Finetune/Step": 189, "Finetune/Step Time": 4.366017855703831} +{"Finetune/Learning Rate": 1.999762310117044e-05, "Finetune/Loss": 1.0214701890945435, "Finetune/Loss (Raw)": 1.0766781568527222, "Finetune/Step": 190, "Finetune/Step Time": 4.3707071505486965} +{"Finetune/Learning Rate": 1.999748534382867e-05, "Finetune/Loss": 1.020503044128418, "Finetune/Loss (Raw)": 1.091062068939209, "Finetune/Step": 191, "Finetune/Step Time": 4.376392249017954} +{"Finetune/Learning Rate": 1.9997343706658023e-05, "Finetune/Loss": 1.0205907821655273, "Finetune/Loss (Raw)": 1.0757659673690796, "Finetune/Step": 192, "Finetune/Step Time": 4.356488764286041} +{"Finetune/Learning Rate": 1.9997198189713462e-05, "Finetune/Loss": 1.0198426246643066, "Finetune/Loss (Raw)": 0.9541568756103516, "Finetune/Step": 193, "Finetune/Step Time": 4.349392540752888} +{"Finetune/Learning Rate": 1.9997048793051463e-05, "Finetune/Loss": 1.020256519317627, "Finetune/Loss (Raw)": 1.063537836074829, "Finetune/Step": 194, "Finetune/Step Time": 4.348482228815556} +{"Finetune/Learning Rate": 1.9996895516730005e-05, "Finetune/Loss": 1.0211225748062134, "Finetune/Loss (Raw)": 1.0171698331832886, "Finetune/Step": 195, "Finetune/Step Time": 4.355502866208553} +{"Finetune/Learning Rate": 1.9996738360808566e-05, "Finetune/Loss": 1.0198744535446167, "Finetune/Loss (Raw)": 0.9784379005432129, "Finetune/Step": 196, "Finetune/Step Time": 4.351671742275357} +{"Finetune/Learning Rate": 1.999657732534814e-05, "Finetune/Loss": 1.0191129446029663, "Finetune/Loss (Raw)": 1.0594276189804077, "Finetune/Step": 197, "Finetune/Step Time": 4.358436835929751} +{"Finetune/Learning Rate": 1.9996412410411224e-05, "Finetune/Loss": 1.017790675163269, "Finetune/Loss (Raw)": 1.0234007835388184, "Finetune/Step": 198, "Finetune/Step Time": 4.3580890242010355} +{"Finetune/Learning Rate": 1.9996243616061812e-05, "Finetune/Loss": 1.0184069871902466, "Finetune/Loss (Raw)": 1.0067161321640015, "Finetune/Step": 199, "Finetune/Step Time": 4.362547030672431} +{"Finetune/Learning Rate": 1.9996070942365416e-05, "Finetune/Loss": 1.0185188055038452, "Finetune/Loss (Raw)": 1.039892315864563, "Finetune/Step": 200, "Finetune/Step Time": 4.367122836410999} +{"Finetune/Learning Rate": 1.9995894389389038e-05, "Finetune/Loss": 1.020193099975586, "Finetune/Loss (Raw)": 1.1400262117385864, "Finetune/Step": 201, "Finetune/Step Time": 4.371223429217935} +{"Finetune/Learning Rate": 1.9995713957201208e-05, "Finetune/Loss": 1.0217058658599854, "Finetune/Loss (Raw)": 1.131822943687439, "Finetune/Step": 202, "Finetune/Step Time": 4.374686306342483} +{"Finetune/Learning Rate": 1.9995529645871938e-05, "Finetune/Loss": 1.023655652999878, "Finetune/Loss (Raw)": 1.0785094499588013, "Finetune/Step": 203, "Finetune/Step Time": 4.369254551827908} +{"Finetune/Learning Rate": 1.999534145547276e-05, "Finetune/Loss": 1.024444580078125, "Finetune/Loss (Raw)": 1.0732556581497192, "Finetune/Step": 204, "Finetune/Step Time": 4.36413548886776} +{"Finetune/Learning Rate": 1.99951493860767e-05, "Finetune/Loss": 1.0182839632034302, "Finetune/Loss (Raw)": 0.32650065422058105, "Finetune/Step": 205, "Finetune/Step Time": 4.412211965769529} +{"Finetune/Learning Rate": 1.9994953437758305e-05, "Finetune/Loss": 1.0167368650436401, "Finetune/Loss (Raw)": 1.0221470594406128, "Finetune/Step": 206, "Finetune/Step Time": 4.420636709779501} +{"Finetune/Learning Rate": 1.999475361059362e-05, "Finetune/Loss": 1.0168465375900269, "Finetune/Loss (Raw)": 1.1945505142211914, "Finetune/Step": 207, "Finetune/Step Time": 4.443421022966504} +{"Finetune/Learning Rate": 1.9994549904660187e-05, "Finetune/Loss": 1.0175058841705322, "Finetune/Loss (Raw)": 1.0647574663162231, "Finetune/Step": 208, "Finetune/Step Time": 4.428468260914087} +{"Finetune/Learning Rate": 1.9994342320037065e-05, "Finetune/Loss": 1.016495943069458, "Finetune/Loss (Raw)": 1.0172367095947266, "Finetune/Step": 209, "Finetune/Step Time": 4.42241926304996} +{"Finetune/Learning Rate": 1.9994130856804808e-05, "Finetune/Loss": 1.0104008913040161, "Finetune/Loss (Raw)": 0.3254067599773407, "Finetune/Step": 210, "Finetune/Step Time": 4.460828876122832} +{"Finetune/Learning Rate": 1.9993915515045485e-05, "Finetune/Loss": 1.0107260942459106, "Finetune/Loss (Raw)": 0.9967005252838135, "Finetune/Step": 211, "Finetune/Step Time": 4.453577559441328} +{"Finetune/Learning Rate": 1.999369629484267e-05, "Finetune/Loss": 1.009859561920166, "Finetune/Loss (Raw)": 1.0809401273727417, "Finetune/Step": 212, "Finetune/Step Time": 4.45113804936409} +{"Finetune/Learning Rate": 1.999347319628143e-05, "Finetune/Loss": 1.010969638824463, "Finetune/Loss (Raw)": 1.1998220682144165, "Finetune/Step": 213, "Finetune/Step Time": 4.454058764502406} +{"Finetune/Learning Rate": 1.9993246219448347e-05, "Finetune/Loss": 1.0159289836883545, "Finetune/Loss (Raw)": 0.9927390813827515, "Finetune/Step": 214, "Finetune/Step Time": 4.4278259221464396} +{"Finetune/Learning Rate": 1.999301536443151e-05, "Finetune/Loss": 1.0153332948684692, "Finetune/Loss (Raw)": 0.9575512409210205, "Finetune/Step": 215, "Finetune/Step Time": 4.42742052488029} +{"Finetune/Learning Rate": 1.9992780631320508e-05, "Finetune/Loss": 1.0130542516708374, "Finetune/Loss (Raw)": 0.9430172443389893, "Finetune/Step": 216, "Finetune/Step Time": 4.426172675564885} +{"Finetune/Learning Rate": 1.9992542020206432e-05, "Finetune/Loss": 1.0134307146072388, "Finetune/Loss (Raw)": 1.124964952468872, "Finetune/Step": 217, "Finetune/Step Time": 4.424382487311959} +{"Finetune/Learning Rate": 1.9992299531181892e-05, "Finetune/Loss": 1.0129003524780273, "Finetune/Loss (Raw)": 1.0776664018630981, "Finetune/Step": 218, "Finetune/Step Time": 4.41407048329711} +{"Finetune/Learning Rate": 1.999205316434098e-05, "Finetune/Loss": 1.0132498741149902, "Finetune/Loss (Raw)": 1.2201114892959595, "Finetune/Step": 219, "Finetune/Step Time": 4.402727073058486} +{"Finetune/Learning Rate": 1.9991802919779318e-05, "Finetune/Loss": 1.0114681720733643, "Finetune/Loss (Raw)": 0.9794334173202515, "Finetune/Step": 220, "Finetune/Step Time": 4.400120299309492} +{"Finetune/Learning Rate": 1.9991548797594017e-05, "Finetune/Loss": 1.0129473209381104, "Finetune/Loss (Raw)": 1.1176456212997437, "Finetune/Step": 221, "Finetune/Step Time": 4.403449388220906} +{"Finetune/Learning Rate": 1.9991290797883693e-05, "Finetune/Loss": 1.0114080905914307, "Finetune/Loss (Raw)": 0.9632884860038757, "Finetune/Step": 222, "Finetune/Step Time": 4.409839576110244} +{"Finetune/Learning Rate": 1.9991028920748475e-05, "Finetune/Loss": 1.0105786323547363, "Finetune/Loss (Raw)": 0.9998953342437744, "Finetune/Step": 223, "Finetune/Step Time": 4.4111727476119995} +{"Finetune/Learning Rate": 1.9990763166289993e-05, "Finetune/Loss": 1.0049030780792236, "Finetune/Loss (Raw)": 0.3617391884326935, "Finetune/Step": 224, "Finetune/Step Time": 4.439201395958662} +{"Finetune/Learning Rate": 1.9990493534611376e-05, "Finetune/Loss": 1.0051229000091553, "Finetune/Loss (Raw)": 1.0609524250030518, "Finetune/Step": 225, "Finetune/Step Time": 4.439713317900896} +{"Finetune/Learning Rate": 1.9990220025817268e-05, "Finetune/Loss": 1.0055923461914062, "Finetune/Loss (Raw)": 1.192564606666565, "Finetune/Step": 226, "Finetune/Step Time": 4.436886254698038} +{"Finetune/Learning Rate": 1.998994264001381e-05, "Finetune/Loss": 1.0131845474243164, "Finetune/Loss (Raw)": 1.1988970041275024, "Finetune/Step": 227, "Finetune/Step Time": 4.39773677662015} +{"Finetune/Learning Rate": 1.9989661377308654e-05, "Finetune/Loss": 1.0127472877502441, "Finetune/Loss (Raw)": 1.1668298244476318, "Finetune/Step": 228, "Finetune/Step Time": 4.39282120577991} +{"Finetune/Learning Rate": 1.9989376237810944e-05, "Finetune/Loss": 1.012563705444336, "Finetune/Loss (Raw)": 1.0323550701141357, "Finetune/Step": 229, "Finetune/Step Time": 4.386040285229683} +{"Finetune/Learning Rate": 1.9989087221631344e-05, "Finetune/Loss": 1.0121748447418213, "Finetune/Loss (Raw)": 1.1651579141616821, "Finetune/Step": 230, "Finetune/Step Time": 4.388495333492756} +{"Finetune/Learning Rate": 1.9988794328882017e-05, "Finetune/Loss": 1.0178667306900024, "Finetune/Loss (Raw)": 1.0250309705734253, "Finetune/Step": 231, "Finetune/Step Time": 4.350813593715429} +{"Finetune/Learning Rate": 1.9988497559676622e-05, "Finetune/Loss": 1.0171551704406738, "Finetune/Loss (Raw)": 1.0697427988052368, "Finetune/Step": 232, "Finetune/Step Time": 4.354619035497308} +{"Finetune/Learning Rate": 1.9988196914130332e-05, "Finetune/Loss": 1.0112512111663818, "Finetune/Loss (Raw)": 0.31345000863075256, "Finetune/Step": 233, "Finetune/Step Time": 4.398183124139905} +{"Finetune/Learning Rate": 1.9987892392359827e-05, "Finetune/Loss": 1.0120327472686768, "Finetune/Loss (Raw)": 1.1714624166488647, "Finetune/Step": 234, "Finetune/Step Time": 4.4173957742750645} +{"Finetune/Learning Rate": 1.998758399448328e-05, "Finetune/Loss": 1.0126652717590332, "Finetune/Loss (Raw)": 1.2160836458206177, "Finetune/Step": 235, "Finetune/Step Time": 4.457807455211878} +{"Finetune/Learning Rate": 1.9987271720620376e-05, "Finetune/Loss": 1.0127387046813965, "Finetune/Loss (Raw)": 1.028883457183838, "Finetune/Step": 236, "Finetune/Step Time": 4.457576626911759} +{"Finetune/Learning Rate": 1.9986955570892302e-05, "Finetune/Loss": 1.0123423337936401, "Finetune/Loss (Raw)": 1.097130298614502, "Finetune/Step": 237, "Finetune/Step Time": 4.490736631676555} +{"Finetune/Learning Rate": 1.998663554542175e-05, "Finetune/Loss": 1.0120055675506592, "Finetune/Loss (Raw)": 1.01882803440094, "Finetune/Step": 238, "Finetune/Step Time": 4.4944068025797606} +{"Finetune/Learning Rate": 1.9986311644332913e-05, "Finetune/Loss": 1.0101977586746216, "Finetune/Loss (Raw)": 0.9011191725730896, "Finetune/Step": 239, "Finetune/Step Time": 4.478504566475749} +{"Finetune/Learning Rate": 1.9985983867751497e-05, "Finetune/Loss": 1.0092519521713257, "Finetune/Loss (Raw)": 0.9726955890655518, "Finetune/Step": 240, "Finetune/Step Time": 4.470950508490205} +{"Finetune/Learning Rate": 1.99856522158047e-05, "Finetune/Loss": 1.0090479850769043, "Finetune/Loss (Raw)": 1.1111713647842407, "Finetune/Step": 241, "Finetune/Step Time": 4.458339359611273} +{"Finetune/Learning Rate": 1.998531668862123e-05, "Finetune/Loss": 1.0154540538787842, "Finetune/Loss (Raw)": 1.070852279663086, "Finetune/Step": 242, "Finetune/Step Time": 4.420210462063551} +{"Finetune/Learning Rate": 1.9984977286331302e-05, "Finetune/Loss": 1.0153247117996216, "Finetune/Loss (Raw)": 1.0845372676849365, "Finetune/Step": 243, "Finetune/Step Time": 4.440894328057766} +{"Finetune/Learning Rate": 1.998463400906663e-05, "Finetune/Loss": 1.015288233757019, "Finetune/Loss (Raw)": 0.9289486408233643, "Finetune/Step": 244, "Finetune/Step Time": 4.42042993567884} +{"Finetune/Learning Rate": 1.998428685696043e-05, "Finetune/Loss": 1.0161068439483643, "Finetune/Loss (Raw)": 1.2187883853912354, "Finetune/Step": 245, "Finetune/Step Time": 4.412933606654406} +{"Finetune/Learning Rate": 1.998393583014743e-05, "Finetune/Loss": 1.0163835287094116, "Finetune/Loss (Raw)": 1.1783268451690674, "Finetune/Step": 246, "Finetune/Step Time": 4.439882501959801} +{"Finetune/Learning Rate": 1.9983580928763852e-05, "Finetune/Loss": 1.0164958238601685, "Finetune/Loss (Raw)": 1.0671745538711548, "Finetune/Step": 247, "Finetune/Step Time": 4.439966980367899} +{"Finetune/Learning Rate": 1.998322215294743e-05, "Finetune/Loss": 1.0158610343933105, "Finetune/Loss (Raw)": 1.0107874870300293, "Finetune/Step": 248, "Finetune/Step Time": 4.443364528939128} +{"Finetune/Learning Rate": 1.9982859502837395e-05, "Finetune/Loss": 1.015864372253418, "Finetune/Loss (Raw)": 1.0286840200424194, "Finetune/Step": 249, "Finetune/Step Time": 4.4244621358811855} +{"Finetune/Learning Rate": 1.998249297857449e-05, "Finetune/Loss": 1.0167878866195679, "Finetune/Loss (Raw)": 1.0879884958267212, "Finetune/Step": 250, "Finetune/Step Time": 4.4235581159591675} +{"Finetune/Learning Rate": 1.9982122580300947e-05, "Finetune/Loss": 1.0178089141845703, "Finetune/Loss (Raw)": 1.1534700393676758, "Finetune/Step": 251, "Finetune/Step Time": 4.438379466533661} +{"Finetune/Learning Rate": 1.9981748308160517e-05, "Finetune/Loss": 1.0171606540679932, "Finetune/Loss (Raw)": 1.0475488901138306, "Finetune/Step": 252, "Finetune/Step Time": 4.43737162463367} +{"Finetune/Learning Rate": 1.9981370162298444e-05, "Finetune/Loss": 1.0177515745162964, "Finetune/Loss (Raw)": 1.0928531885147095, "Finetune/Step": 253, "Finetune/Step Time": 4.4323111567646265} +{"Finetune/Learning Rate": 1.9980988142861486e-05, "Finetune/Loss": 1.016808271408081, "Finetune/Loss (Raw)": 0.9870041608810425, "Finetune/Step": 254, "Finetune/Step Time": 4.412659540772438} +{"Finetune/Learning Rate": 1.998060224999789e-05, "Finetune/Loss": 1.0115966796875, "Finetune/Loss (Raw)": 0.3242402970790863, "Finetune/Step": 255, "Finetune/Step Time": 4.4519907385110855} +{"Finetune/Learning Rate": 1.9980212483857412e-05, "Finetune/Loss": 1.0102097988128662, "Finetune/Loss (Raw)": 0.9575772881507874, "Finetune/Step": 256, "Finetune/Step Time": 4.452771082520485} +{"Finetune/Learning Rate": 1.9979818844591318e-05, "Finetune/Loss": 1.0103625059127808, "Finetune/Loss (Raw)": 1.1761258840560913, "Finetune/Step": 257, "Finetune/Step Time": 4.452514873817563} +{"Finetune/Learning Rate": 1.9979421332352372e-05, "Finetune/Loss": 1.0102200508117676, "Finetune/Loss (Raw)": 1.1289492845535278, "Finetune/Step": 258, "Finetune/Step Time": 4.452464111149311} +{"Finetune/Learning Rate": 1.9979019947294836e-05, "Finetune/Loss": 1.010123372077942, "Finetune/Loss (Raw)": 1.094387173652649, "Finetune/Step": 259, "Finetune/Step Time": 4.454731462523341} +{"Finetune/Learning Rate": 1.9978614689574486e-05, "Finetune/Loss": 1.0101712942123413, "Finetune/Loss (Raw)": 1.1173450946807861, "Finetune/Step": 260, "Finetune/Step Time": 4.474506717175245} +{"Finetune/Learning Rate": 1.9978205559348586e-05, "Finetune/Loss": 1.0100293159484863, "Finetune/Loss (Raw)": 0.8826173543930054, "Finetune/Step": 261, "Finetune/Step Time": 4.457686439156532} +{"Finetune/Learning Rate": 1.997779255677592e-05, "Finetune/Loss": 1.0069626569747925, "Finetune/Loss (Raw)": 0.8533703684806824, "Finetune/Step": 262, "Finetune/Step Time": 4.46869096532464} +{"Finetune/Learning Rate": 1.997737568201676e-05, "Finetune/Loss": 1.0072283744812012, "Finetune/Loss (Raw)": 1.0328282117843628, "Finetune/Step": 263, "Finetune/Step Time": 4.4664799608290195} +{"Finetune/Learning Rate": 1.9976954935232892e-05, "Finetune/Loss": 1.006495714187622, "Finetune/Loss (Raw)": 0.9950127601623535, "Finetune/Step": 264, "Finetune/Step Time": 4.480819515883923} +{"Finetune/Learning Rate": 1.99765303165876e-05, "Finetune/Loss": 1.0066485404968262, "Finetune/Loss (Raw)": 0.9977825284004211, "Finetune/Step": 265, "Finetune/Step Time": 4.476950891315937} +{"Finetune/Learning Rate": 1.997610182624566e-05, "Finetune/Loss": 1.0074877738952637, "Finetune/Loss (Raw)": 1.2327498197555542, "Finetune/Step": 266, "Finetune/Step Time": 4.488129893317819} +{"Finetune/Learning Rate": 1.9975669464373377e-05, "Finetune/Loss": 1.0073832273483276, "Finetune/Loss (Raw)": 0.9082025289535522, "Finetune/Step": 267, "Finetune/Step Time": 4.485936224460602} +{"Finetune/Learning Rate": 1.9975233231138525e-05, "Finetune/Loss": 1.0060670375823975, "Finetune/Loss (Raw)": 1.1210408210754395, "Finetune/Step": 268, "Finetune/Step Time": 4.4895349238067865} +{"Finetune/Learning Rate": 1.997479312671041e-05, "Finetune/Loss": 1.0061436891555786, "Finetune/Loss (Raw)": 1.0461961030960083, "Finetune/Step": 269, "Finetune/Step Time": 4.495459822937846} +{"Finetune/Learning Rate": 1.9974349151259824e-05, "Finetune/Loss": 1.0063416957855225, "Finetune/Loss (Raw)": 1.060789942741394, "Finetune/Step": 270, "Finetune/Step Time": 4.460524229332805} +{"Finetune/Learning Rate": 1.9973901304959066e-05, "Finetune/Loss": 1.0067846775054932, "Finetune/Loss (Raw)": 1.0277677774429321, "Finetune/Step": 271, "Finetune/Step Time": 4.451803833246231} +{"Finetune/Learning Rate": 1.9973449587981935e-05, "Finetune/Loss": 1.0125315189361572, "Finetune/Loss (Raw)": 1.1226593255996704, "Finetune/Step": 272, "Finetune/Step Time": 4.412725912407041} +{"Finetune/Learning Rate": 1.9972994000503734e-05, "Finetune/Loss": 1.0180270671844482, "Finetune/Loss (Raw)": 1.0575106143951416, "Finetune/Step": 273, "Finetune/Step Time": 4.376871386542916} +{"Finetune/Learning Rate": 1.997253454270127e-05, "Finetune/Loss": 1.0190283060073853, "Finetune/Loss (Raw)": 1.1047779321670532, "Finetune/Step": 274, "Finetune/Step Time": 4.383422018960118} +{"Finetune/Learning Rate": 1.9972071214752843e-05, "Finetune/Loss": 1.0199726819992065, "Finetune/Loss (Raw)": 1.0266785621643066, "Finetune/Step": 275, "Finetune/Step Time": 4.380401419475675} +{"Finetune/Learning Rate": 1.9971604016838264e-05, "Finetune/Loss": 1.0180271863937378, "Finetune/Loss (Raw)": 0.8366029858589172, "Finetune/Step": 276, "Finetune/Step Time": 4.413280511274934} +{"Finetune/Learning Rate": 1.997113294913885e-05, "Finetune/Loss": 1.017228603363037, "Finetune/Loss (Raw)": 1.03006112575531, "Finetune/Step": 277, "Finetune/Step Time": 4.412118304520845} +{"Finetune/Learning Rate": 1.9970658011837404e-05, "Finetune/Loss": 1.0164908170700073, "Finetune/Loss (Raw)": 1.044028401374817, "Finetune/Step": 278, "Finetune/Step Time": 4.405400440096855} +{"Finetune/Learning Rate": 1.997017920511825e-05, "Finetune/Loss": 1.01741623878479, "Finetune/Loss (Raw)": 1.2203059196472168, "Finetune/Step": 279, "Finetune/Step Time": 4.41138356924057} +{"Finetune/Learning Rate": 1.9969696529167197e-05, "Finetune/Loss": 1.0168521404266357, "Finetune/Loss (Raw)": 0.9942509531974792, "Finetune/Step": 280, "Finetune/Step Time": 4.409911021590233} +{"Finetune/Learning Rate": 1.996920998417156e-05, "Finetune/Loss": 1.0162564516067505, "Finetune/Loss (Raw)": 1.0886051654815674, "Finetune/Step": 281, "Finetune/Step Time": 4.395671151578426} +{"Finetune/Learning Rate": 1.996871957032016e-05, "Finetune/Loss": 1.015343427658081, "Finetune/Loss (Raw)": 1.0239583253860474, "Finetune/Step": 282, "Finetune/Step Time": 4.394849257543683} +{"Finetune/Learning Rate": 1.996822528780332e-05, "Finetune/Loss": 1.0147849321365356, "Finetune/Loss (Raw)": 1.1427240371704102, "Finetune/Step": 283, "Finetune/Step Time": 4.392863415181637} +{"Finetune/Learning Rate": 1.996772713681286e-05, "Finetune/Loss": 1.0214804410934448, "Finetune/Loss (Raw)": 1.155942440032959, "Finetune/Step": 284, "Finetune/Step Time": 4.351143280044198} +{"Finetune/Learning Rate": 1.9967225117542097e-05, "Finetune/Loss": 1.0209107398986816, "Finetune/Loss (Raw)": 1.0134669542312622, "Finetune/Step": 285, "Finetune/Step Time": 4.368825912475586} +{"Finetune/Learning Rate": 1.9966719230185864e-05, "Finetune/Loss": 1.020702600479126, "Finetune/Loss (Raw)": 1.0859049558639526, "Finetune/Step": 286, "Finetune/Step Time": 4.383966693654656} +{"Finetune/Learning Rate": 1.9966209474940477e-05, "Finetune/Loss": 1.020751953125, "Finetune/Loss (Raw)": 1.0880264043807983, "Finetune/Step": 287, "Finetune/Step Time": 4.391366355121136} +{"Finetune/Learning Rate": 1.996569585200377e-05, "Finetune/Loss": 1.0199177265167236, "Finetune/Loss (Raw)": 0.9129184484481812, "Finetune/Step": 288, "Finetune/Step Time": 4.399216368794441} +{"Finetune/Learning Rate": 1.9965178361575068e-05, "Finetune/Loss": 1.0206453800201416, "Finetune/Loss (Raw)": 1.1300982236862183, "Finetune/Step": 289, "Finetune/Step Time": 4.403476487845182} +{"Finetune/Learning Rate": 1.9964657003855197e-05, "Finetune/Loss": 1.018256425857544, "Finetune/Loss (Raw)": 0.9660729765892029, "Finetune/Step": 290, "Finetune/Step Time": 4.414548730477691} +{"Finetune/Learning Rate": 1.9964131779046484e-05, "Finetune/Loss": 1.018563985824585, "Finetune/Loss (Raw)": 1.127076506614685, "Finetune/Step": 291, "Finetune/Step Time": 4.408476620912552} +{"Finetune/Learning Rate": 1.996360268735276e-05, "Finetune/Loss": 1.0197057723999023, "Finetune/Loss (Raw)": 1.113065481185913, "Finetune/Step": 292, "Finetune/Step Time": 4.396942609921098} +{"Finetune/Learning Rate": 1.9963069728979357e-05, "Finetune/Loss": 1.0204658508300781, "Finetune/Loss (Raw)": 1.1541693210601807, "Finetune/Step": 293, "Finetune/Step Time": 4.392625542357564} +{"Finetune/Learning Rate": 1.9962532904133107e-05, "Finetune/Loss": 1.0186558961868286, "Finetune/Loss (Raw)": 0.7642061114311218, "Finetune/Step": 294, "Finetune/Step Time": 4.399033544585109} +{"Finetune/Learning Rate": 1.996199221302234e-05, "Finetune/Loss": 1.0181877613067627, "Finetune/Loss (Raw)": 1.0981500148773193, "Finetune/Step": 295, "Finetune/Step Time": 4.40511305257678} +{"Finetune/Learning Rate": 1.9961447655856883e-05, "Finetune/Loss": 1.0184088945388794, "Finetune/Loss (Raw)": 1.0202345848083496, "Finetune/Step": 296, "Finetune/Step Time": 4.41011012904346} +{"Finetune/Learning Rate": 1.9960899232848074e-05, "Finetune/Loss": 1.019684076309204, "Finetune/Loss (Raw)": 1.136615514755249, "Finetune/Step": 297, "Finetune/Step Time": 4.412860183045268} +{"Finetune/Learning Rate": 1.9960346944208743e-05, "Finetune/Loss": 1.0239808559417725, "Finetune/Loss (Raw)": 0.9130536913871765, "Finetune/Step": 298, "Finetune/Step Time": 4.40001499094069} +{"Finetune/Learning Rate": 1.995979079015322e-05, "Finetune/Loss": 1.0185542106628418, "Finetune/Loss (Raw)": 0.3144294321537018, "Finetune/Step": 299, "Finetune/Step Time": 4.4358694814145565} +{"Finetune/Learning Rate": 1.995923077089734e-05, "Finetune/Loss": 1.0136528015136719, "Finetune/Loss (Raw)": 0.369809627532959, "Finetune/Step": 300, "Finetune/Step Time": 4.4638505931943655} +{"Finetune/Learning Rate": 1.9958666886658436e-05, "Finetune/Loss": 1.0073415040969849, "Finetune/Loss (Raw)": 0.4013614058494568, "Finetune/Step": 301, "Finetune/Step Time": 4.503452921286225} +{"Finetune/Learning Rate": 1.995809913765534e-05, "Finetune/Loss": 1.0072695016860962, "Finetune/Loss (Raw)": 0.8939750790596008, "Finetune/Step": 302, "Finetune/Step Time": 4.512006584554911} +{"Finetune/Learning Rate": 1.9957527524108386e-05, "Finetune/Loss": 1.0004675388336182, "Finetune/Loss (Raw)": 0.32673153281211853, "Finetune/Step": 303, "Finetune/Step Time": 4.551334489136934} +{"Finetune/Learning Rate": 1.99569520462394e-05, "Finetune/Loss": 1.006249189376831, "Finetune/Loss (Raw)": 1.1001546382904053, "Finetune/Step": 304, "Finetune/Step Time": 4.513949161395431} +{"Finetune/Learning Rate": 1.995637270427172e-05, "Finetune/Loss": 1.0068076848983765, "Finetune/Loss (Raw)": 1.0553966760635376, "Finetune/Step": 305, "Finetune/Step Time": 4.506455551832914} +{"Finetune/Learning Rate": 1.9955789498430178e-05, "Finetune/Loss": 1.0047149658203125, "Finetune/Loss (Raw)": 0.9462962746620178, "Finetune/Step": 306, "Finetune/Step Time": 4.50762739777565} +{"Finetune/Learning Rate": 1.99552024289411e-05, "Finetune/Loss": 1.0067334175109863, "Finetune/Loss (Raw)": 1.167414665222168, "Finetune/Step": 307, "Finetune/Step Time": 4.515820683911443} +{"Finetune/Learning Rate": 1.995461149603232e-05, "Finetune/Loss": 1.0065624713897705, "Finetune/Loss (Raw)": 1.0511912107467651, "Finetune/Step": 308, "Finetune/Step Time": 4.505172302946448} +{"Finetune/Learning Rate": 1.9954016699933163e-05, "Finetune/Loss": 1.0075428485870361, "Finetune/Loss (Raw)": 1.1540745496749878, "Finetune/Step": 309, "Finetune/Step Time": 4.530077261850238} +{"Finetune/Learning Rate": 1.995341804087446e-05, "Finetune/Loss": 1.0071629285812378, "Finetune/Loss (Raw)": 1.0173674821853638, "Finetune/Step": 310, "Finetune/Step Time": 4.51948362775147} +{"Finetune/Learning Rate": 1.9952815519088544e-05, "Finetune/Loss": 1.0073100328445435, "Finetune/Loss (Raw)": 1.0413583517074585, "Finetune/Step": 311, "Finetune/Step Time": 4.531289653852582} +{"Finetune/Learning Rate": 1.9952209134809234e-05, "Finetune/Loss": 1.0059127807617188, "Finetune/Loss (Raw)": 0.9862590432167053, "Finetune/Step": 312, "Finetune/Step Time": 4.53632203117013} +{"Finetune/Learning Rate": 1.995159888827186e-05, "Finetune/Loss": 1.0073270797729492, "Finetune/Loss (Raw)": 1.1166375875473022, "Finetune/Step": 313, "Finetune/Step Time": 4.539007602259517} +{"Finetune/Learning Rate": 1.9950984779713244e-05, "Finetune/Loss": 1.0086156129837036, "Finetune/Loss (Raw)": 1.1853727102279663, "Finetune/Step": 314, "Finetune/Step Time": 4.523047685623169} +{"Finetune/Learning Rate": 1.9950366809371713e-05, "Finetune/Loss": 1.0084609985351562, "Finetune/Loss (Raw)": 0.9943679571151733, "Finetune/Step": 315, "Finetune/Step Time": 4.529214268550277} +{"Finetune/Learning Rate": 1.9949744977487087e-05, "Finetune/Loss": 1.002507209777832, "Finetune/Loss (Raw)": 0.38274046778678894, "Finetune/Step": 316, "Finetune/Step Time": 4.568866992369294} +{"Finetune/Learning Rate": 1.994911928430069e-05, "Finetune/Loss": 1.0022096633911133, "Finetune/Loss (Raw)": 0.9744617342948914, "Finetune/Step": 317, "Finetune/Step Time": 4.559046246111393} +{"Finetune/Learning Rate": 1.994848973005534e-05, "Finetune/Loss": 1.0021004676818848, "Finetune/Loss (Raw)": 1.0626956224441528, "Finetune/Step": 318, "Finetune/Step Time": 4.55851243622601} +{"Finetune/Learning Rate": 1.994785631499535e-05, "Finetune/Loss": 1.001373052597046, "Finetune/Loss (Raw)": 0.9979491829872131, "Finetune/Step": 319, "Finetune/Step Time": 4.565543407574296} +{"Finetune/Learning Rate": 1.994721903936654e-05, "Finetune/Loss": 1.0003927946090698, "Finetune/Loss (Raw)": 0.9502983689308167, "Finetune/Step": 320, "Finetune/Step Time": 4.561228010803461} +{"Finetune/Learning Rate": 1.9946577903416224e-05, "Finetune/Loss": 0.9955390095710754, "Finetune/Loss (Raw)": 0.33287277817726135, "Finetune/Step": 321, "Finetune/Step Time": 4.599885789677501} +{"Finetune/Learning Rate": 1.994593290739322e-05, "Finetune/Loss": 0.9950309991836548, "Finetune/Loss (Raw)": 0.998512864112854, "Finetune/Step": 322, "Finetune/Step Time": 4.600284248590469} +{"Finetune/Learning Rate": 1.9945284051547823e-05, "Finetune/Loss": 0.9954437017440796, "Finetune/Loss (Raw)": 1.0699890851974487, "Finetune/Step": 323, "Finetune/Step Time": 4.594381278380752} +{"Finetune/Learning Rate": 1.9944631336131856e-05, "Finetune/Loss": 0.9964612722396851, "Finetune/Loss (Raw)": 1.1086945533752441, "Finetune/Step": 324, "Finetune/Step Time": 4.593984209001064} +{"Finetune/Learning Rate": 1.994397476139862e-05, "Finetune/Loss": 0.9963905811309814, "Finetune/Loss (Raw)": 1.0503710508346558, "Finetune/Step": 325, "Finetune/Step Time": 4.591196160763502} +{"Finetune/Learning Rate": 1.994331432760292e-05, "Finetune/Loss": 0.9979238510131836, "Finetune/Loss (Raw)": 1.2196743488311768, "Finetune/Step": 326, "Finetune/Step Time": 4.591501684859395} +{"Finetune/Learning Rate": 1.9942650035001053e-05, "Finetune/Loss": 0.9980301856994629, "Finetune/Loss (Raw)": 1.0203213691711426, "Finetune/Step": 327, "Finetune/Step Time": 4.58558614552021} +{"Finetune/Learning Rate": 1.994198188385082e-05, "Finetune/Loss": 0.998356282711029, "Finetune/Loss (Raw)": 1.0816316604614258, "Finetune/Step": 328, "Finetune/Step Time": 4.5926464684307575} +{"Finetune/Learning Rate": 1.9941309874411525e-05, "Finetune/Loss": 0.9968553781509399, "Finetune/Loss (Raw)": 0.9479139447212219, "Finetune/Step": 329, "Finetune/Step Time": 4.608969369903207} +{"Finetune/Learning Rate": 1.9940634006943952e-05, "Finetune/Loss": 0.9948809146881104, "Finetune/Loss (Raw)": 0.8790929913520813, "Finetune/Step": 330, "Finetune/Step Time": 4.613304926082492} +{"Finetune/Learning Rate": 1.9939954281710395e-05, "Finetune/Loss": 0.994750440120697, "Finetune/Loss (Raw)": 1.0617990493774414, "Finetune/Step": 331, "Finetune/Step Time": 4.617619534954429} +{"Finetune/Learning Rate": 1.9939270698974643e-05, "Finetune/Loss": 0.9928625226020813, "Finetune/Loss (Raw)": 0.831608235836029, "Finetune/Step": 332, "Finetune/Step Time": 4.63354423828423} +{"Finetune/Learning Rate": 1.9938583259001983e-05, "Finetune/Loss": 0.9984480738639832, "Finetune/Loss (Raw)": 1.0414515733718872, "Finetune/Step": 333, "Finetune/Step Time": 4.593229053542018} +{"Finetune/Learning Rate": 1.993789196205919e-05, "Finetune/Loss": 0.9982953667640686, "Finetune/Loss (Raw)": 1.0026029348373413, "Finetune/Step": 334, "Finetune/Step Time": 4.588216222822666} +{"Finetune/Learning Rate": 1.9937196808414552e-05, "Finetune/Loss": 0.9968751668930054, "Finetune/Loss (Raw)": 1.0127519369125366, "Finetune/Step": 335, "Finetune/Step Time": 4.57572971843183} +{"Finetune/Learning Rate": 1.993649779833784e-05, "Finetune/Loss": 0.9972726106643677, "Finetune/Loss (Raw)": 1.1156377792358398, "Finetune/Step": 336, "Finetune/Step Time": 4.579299613833427} +{"Finetune/Learning Rate": 1.993579493210032e-05, "Finetune/Loss": 0.9989805221557617, "Finetune/Loss (Raw)": 1.2358587980270386, "Finetune/Step": 337, "Finetune/Step Time": 4.574940534308553} +{"Finetune/Learning Rate": 1.9935088209974774e-05, "Finetune/Loss": 1.0042471885681152, "Finetune/Loss (Raw)": 0.9995244741439819, "Finetune/Step": 338, "Finetune/Step Time": 4.5455808490514755} +{"Finetune/Learning Rate": 1.9934377632235455e-05, "Finetune/Loss": 1.004040002822876, "Finetune/Loss (Raw)": 0.9701859354972839, "Finetune/Step": 339, "Finetune/Step Time": 4.548375625163317} +{"Finetune/Learning Rate": 1.9933663199158135e-05, "Finetune/Loss": 1.0027672052383423, "Finetune/Loss (Raw)": 0.9180252552032471, "Finetune/Step": 340, "Finetune/Step Time": 4.546720227226615} +{"Finetune/Learning Rate": 1.993294491102006e-05, "Finetune/Loss": 1.0014671087265015, "Finetune/Loss (Raw)": 1.0334079265594482, "Finetune/Step": 341, "Finetune/Step Time": 4.54799928329885} +{"Finetune/Learning Rate": 1.9932222768099992e-05, "Finetune/Loss": 1.00223708152771, "Finetune/Loss (Raw)": 1.0912986993789673, "Finetune/Step": 342, "Finetune/Step Time": 4.529144000262022} +{"Finetune/Learning Rate": 1.9931496770678178e-05, "Finetune/Loss": 1.0027587413787842, "Finetune/Loss (Raw)": 1.0243163108825684, "Finetune/Step": 343, "Finetune/Step Time": 4.527310237288475} +{"Finetune/Learning Rate": 1.993076691903636e-05, "Finetune/Loss": 1.0039138793945312, "Finetune/Loss (Raw)": 1.0908681154251099, "Finetune/Step": 344, "Finetune/Step Time": 4.5124129969626665} +{"Finetune/Learning Rate": 1.993003321345779e-05, "Finetune/Loss": 1.0039538145065308, "Finetune/Loss (Raw)": 1.130096197128296, "Finetune/Step": 345, "Finetune/Step Time": 4.515062937512994} +{"Finetune/Learning Rate": 1.9929295654227193e-05, "Finetune/Loss": 1.0043845176696777, "Finetune/Loss (Raw)": 1.1327952146530151, "Finetune/Step": 346, "Finetune/Step Time": 4.521631745621562} +{"Finetune/Learning Rate": 1.9928554241630804e-05, "Finetune/Loss": 1.0038971900939941, "Finetune/Loss (Raw)": 1.157729148864746, "Finetune/Step": 347, "Finetune/Step Time": 4.525152903050184} +{"Finetune/Learning Rate": 1.9927808975956354e-05, "Finetune/Loss": 1.0047025680541992, "Finetune/Loss (Raw)": 1.0825272798538208, "Finetune/Step": 348, "Finetune/Step Time": 4.545012084767222} +{"Finetune/Learning Rate": 1.9927059857493068e-05, "Finetune/Loss": 1.0050230026245117, "Finetune/Loss (Raw)": 1.158665657043457, "Finetune/Step": 349, "Finetune/Step Time": 4.553512694314122} +{"Finetune/Learning Rate": 1.9926306886531658e-05, "Finetune/Loss": 1.005255937576294, "Finetune/Loss (Raw)": 0.9930864572525024, "Finetune/Step": 350, "Finetune/Step Time": 4.567979026585817} +{"Finetune/Learning Rate": 1.9925550063364337e-05, "Finetune/Loss": 1.0052666664123535, "Finetune/Loss (Raw)": 1.0012754201889038, "Finetune/Step": 351, "Finetune/Step Time": 4.581301724538207} +{"Finetune/Learning Rate": 1.992478938828482e-05, "Finetune/Loss": 1.010284423828125, "Finetune/Loss (Raw)": 1.004012107849121, "Finetune/Step": 352, "Finetune/Step Time": 4.551991591230035} +{"Finetune/Learning Rate": 1.9924024861588307e-05, "Finetune/Loss": 1.0096670389175415, "Finetune/Loss (Raw)": 0.9819274544715881, "Finetune/Step": 353, "Finetune/Step Time": 4.5507365223020315} +{"Finetune/Learning Rate": 1.9923256483571495e-05, "Finetune/Loss": 1.0074706077575684, "Finetune/Loss (Raw)": 0.911411464214325, "Finetune/Step": 354, "Finetune/Step Time": 4.557951010763645} +{"Finetune/Learning Rate": 1.9922484254532578e-05, "Finetune/Loss": 1.0065102577209473, "Finetune/Loss (Raw)": 1.0759694576263428, "Finetune/Step": 355, "Finetune/Step Time": 4.567658135667443} +{"Finetune/Learning Rate": 1.9921708174771237e-05, "Finetune/Loss": 1.004575252532959, "Finetune/Loss (Raw)": 0.9191581606864929, "Finetune/Step": 356, "Finetune/Step Time": 4.575480384752154} +{"Finetune/Learning Rate": 1.9920928244588662e-05, "Finetune/Loss": 1.003798484802246, "Finetune/Loss (Raw)": 0.9329369068145752, "Finetune/Step": 357, "Finetune/Step Time": 4.589775059372187} +{"Finetune/Learning Rate": 1.9920144464287527e-05, "Finetune/Loss": 0.9976269006729126, "Finetune/Loss (Raw)": 0.3751925230026245, "Finetune/Step": 358, "Finetune/Step Time": 4.628779157996178} +{"Finetune/Learning Rate": 1.9919356834171996e-05, "Finetune/Loss": 0.9963082075119019, "Finetune/Loss (Raw)": 0.8562285304069519, "Finetune/Step": 359, "Finetune/Step Time": 4.629730936139822} +{"Finetune/Learning Rate": 1.9918565354547738e-05, "Finetune/Loss": 0.9958709478378296, "Finetune/Loss (Raw)": 1.013776421546936, "Finetune/Step": 360, "Finetune/Step Time": 4.62915956787765} +{"Finetune/Learning Rate": 1.991777002572191e-05, "Finetune/Loss": 1.0004076957702637, "Finetune/Loss (Raw)": 0.8941576480865479, "Finetune/Step": 361, "Finetune/Step Time": 4.589471036568284} +{"Finetune/Learning Rate": 1.9916970848003166e-05, "Finetune/Loss": 0.9989314079284668, "Finetune/Loss (Raw)": 0.9824913144111633, "Finetune/Step": 362, "Finetune/Step Time": 4.581611681729555} +{"Finetune/Learning Rate": 1.9916167821701645e-05, "Finetune/Loss": 0.9978795051574707, "Finetune/Loss (Raw)": 1.081437349319458, "Finetune/Step": 363, "Finetune/Step Time": 4.540154905989766} +{"Finetune/Learning Rate": 1.991536094712899e-05, "Finetune/Loss": 0.9984908103942871, "Finetune/Loss (Raw)": 1.1071405410766602, "Finetune/Step": 364, "Finetune/Step Time": 4.538733683526516} +{"Finetune/Learning Rate": 1.991455022459833e-05, "Finetune/Loss": 0.9987154603004456, "Finetune/Loss (Raw)": 1.12588632106781, "Finetune/Step": 365, "Finetune/Step Time": 4.5079276617616415} +{"Finetune/Learning Rate": 1.9913735654424297e-05, "Finetune/Loss": 0.9978227019309998, "Finetune/Loss (Raw)": 0.9045533537864685, "Finetune/Step": 366, "Finetune/Step Time": 4.512917220592499} +{"Finetune/Learning Rate": 1.9912917236923003e-05, "Finetune/Loss": 0.9986642599105835, "Finetune/Loss (Raw)": 1.008834958076477, "Finetune/Step": 367, "Finetune/Step Time": 4.518546583130956} +{"Finetune/Learning Rate": 1.9912094972412067e-05, "Finetune/Loss": 0.9991090893745422, "Finetune/Loss (Raw)": 1.029633641242981, "Finetune/Step": 368, "Finetune/Step Time": 4.518578028306365} +{"Finetune/Learning Rate": 1.9911268861210584e-05, "Finetune/Loss": 0.998173177242279, "Finetune/Loss (Raw)": 0.9913713335990906, "Finetune/Step": 369, "Finetune/Step Time": 4.518395137041807} +{"Finetune/Learning Rate": 1.991043890363916e-05, "Finetune/Loss": 0.9989367127418518, "Finetune/Loss (Raw)": 1.1685903072357178, "Finetune/Step": 370, "Finetune/Step Time": 4.5126843973994255} +{"Finetune/Learning Rate": 1.9909605100019883e-05, "Finetune/Loss": 0.9995850324630737, "Finetune/Loss (Raw)": 1.1675245761871338, "Finetune/Step": 371, "Finetune/Step Time": 4.4928529020398855} +{"Finetune/Learning Rate": 1.9908767450676336e-05, "Finetune/Loss": 1.000535011291504, "Finetune/Loss (Raw)": 1.0505419969558716, "Finetune/Step": 372, "Finetune/Step Time": 4.507498845458031} +{"Finetune/Learning Rate": 1.9907925955933593e-05, "Finetune/Loss": 0.9945108294487, "Finetune/Loss (Raw)": 0.4476885497570038, "Finetune/Step": 373, "Finetune/Step Time": 4.549969330430031} +{"Finetune/Learning Rate": 1.9907080616118223e-05, "Finetune/Loss": 0.9881826043128967, "Finetune/Loss (Raw)": 0.36832302808761597, "Finetune/Step": 374, "Finetune/Step Time": 4.567246422171593} +{"Finetune/Learning Rate": 1.9906231431558283e-05, "Finetune/Loss": 0.9885472655296326, "Finetune/Loss (Raw)": 1.1138405799865723, "Finetune/Step": 375, "Finetune/Step Time": 4.564700296148658} +{"Finetune/Learning Rate": 1.990537840258333e-05, "Finetune/Loss": 0.9877041578292847, "Finetune/Loss (Raw)": 0.9028702974319458, "Finetune/Step": 376, "Finetune/Step Time": 4.56368643976748} +{"Finetune/Learning Rate": 1.9904521529524402e-05, "Finetune/Loss": 0.9872822761535645, "Finetune/Loss (Raw)": 0.974683403968811, "Finetune/Step": 377, "Finetune/Step Time": 4.558077860623598} +{"Finetune/Learning Rate": 1.990366081271404e-05, "Finetune/Loss": 0.9883382320404053, "Finetune/Loss (Raw)": 1.223157286643982, "Finetune/Step": 378, "Finetune/Step Time": 4.556922445073724} +{"Finetune/Learning Rate": 1.990279625248627e-05, "Finetune/Loss": 0.9883915185928345, "Finetune/Loss (Raw)": 1.1602827310562134, "Finetune/Step": 379, "Finetune/Step Time": 4.554929478093982} +{"Finetune/Learning Rate": 1.990192784917661e-05, "Finetune/Loss": 0.9883604645729065, "Finetune/Loss (Raw)": 1.0435818433761597, "Finetune/Step": 380, "Finetune/Step Time": 4.559239938855171} +{"Finetune/Learning Rate": 1.9901055603122067e-05, "Finetune/Loss": 0.9881429672241211, "Finetune/Loss (Raw)": 1.065011739730835, "Finetune/Step": 381, "Finetune/Step Time": 4.5725279320031404} +{"Finetune/Learning Rate": 1.9900179514661148e-05, "Finetune/Loss": 0.9876707792282104, "Finetune/Loss (Raw)": 0.9265612363815308, "Finetune/Step": 382, "Finetune/Step Time": 4.6171468291431665} +{"Finetune/Learning Rate": 1.9899299584133848e-05, "Finetune/Loss": 0.9925206303596497, "Finetune/Loss (Raw)": 0.9450308680534363, "Finetune/Step": 383, "Finetune/Step Time": 4.616026394069195} +{"Finetune/Learning Rate": 1.9898415811881643e-05, "Finetune/Loss": 0.9889174699783325, "Finetune/Loss (Raw)": 0.4963683485984802, "Finetune/Step": 384, "Finetune/Step Time": 4.650915363803506} +{"Finetune/Learning Rate": 1.989752819824751e-05, "Finetune/Loss": 0.9876630902290344, "Finetune/Loss (Raw)": 1.015562653541565, "Finetune/Step": 385, "Finetune/Step Time": 4.648118443787098} +{"Finetune/Learning Rate": 1.989663674357592e-05, "Finetune/Loss": 0.9868046045303345, "Finetune/Loss (Raw)": 1.0190613269805908, "Finetune/Step": 386, "Finetune/Step Time": 4.647645289078355} +{"Finetune/Learning Rate": 1.9895741448212822e-05, "Finetune/Loss": 0.9851269125938416, "Finetune/Loss (Raw)": 0.8796448707580566, "Finetune/Step": 387, "Finetune/Step Time": 4.649480754509568} +{"Finetune/Learning Rate": 1.9894842312505673e-05, "Finetune/Loss": 0.9838564395904541, "Finetune/Loss (Raw)": 0.9547267556190491, "Finetune/Step": 388, "Finetune/Step Time": 4.627629816532135} +{"Finetune/Learning Rate": 1.9893939336803396e-05, "Finetune/Loss": 0.9843359589576721, "Finetune/Loss (Raw)": 0.9439851641654968, "Finetune/Step": 389, "Finetune/Step Time": 4.626902459189296} +{"Finetune/Learning Rate": 1.989303252145643e-05, "Finetune/Loss": 0.9842574596405029, "Finetune/Loss (Raw)": 0.8433318734169006, "Finetune/Step": 390, "Finetune/Step Time": 4.6132861990481615} +{"Finetune/Learning Rate": 1.9892121866816682e-05, "Finetune/Loss": 0.9840725660324097, "Finetune/Loss (Raw)": 1.0091601610183716, "Finetune/Step": 391, "Finetune/Step Time": 4.609083550050855} +{"Finetune/Learning Rate": 1.989120737323757e-05, "Finetune/Loss": 0.9839696884155273, "Finetune/Loss (Raw)": 0.9818430542945862, "Finetune/Step": 392, "Finetune/Step Time": 4.605562197044492} +{"Finetune/Learning Rate": 1.9890289041073984e-05, "Finetune/Loss": 0.9828639030456543, "Finetune/Loss (Raw)": 0.8562423586845398, "Finetune/Step": 393, "Finetune/Step Time": 4.617997420951724} +{"Finetune/Learning Rate": 1.988936687068231e-05, "Finetune/Loss": 0.9795519113540649, "Finetune/Loss (Raw)": 0.8088210821151733, "Finetune/Step": 394, "Finetune/Step Time": 4.633229179307818} +{"Finetune/Learning Rate": 1.9888440862420433e-05, "Finetune/Loss": 0.9819064736366272, "Finetune/Loss (Raw)": 1.209580898284912, "Finetune/Step": 395, "Finetune/Step Time": 4.632226545363665} +{"Finetune/Learning Rate": 1.988751101664771e-05, "Finetune/Loss": 0.9818335771560669, "Finetune/Loss (Raw)": 1.1117109060287476, "Finetune/Step": 396, "Finetune/Step Time": 4.633328840136528} +{"Finetune/Learning Rate": 1.9886577333724995e-05, "Finetune/Loss": 0.9803071618080139, "Finetune/Loss (Raw)": 0.8508127331733704, "Finetune/Step": 397, "Finetune/Step Time": 4.649327116087079} +{"Finetune/Learning Rate": 1.9885639814014642e-05, "Finetune/Loss": 0.9812614917755127, "Finetune/Loss (Raw)": 1.1829403638839722, "Finetune/Step": 398, "Finetune/Step Time": 4.642562661319971} +{"Finetune/Learning Rate": 1.9884698457880476e-05, "Finetune/Loss": 0.9805010557174683, "Finetune/Loss (Raw)": 0.9304415583610535, "Finetune/Step": 399, "Finetune/Step Time": 4.629958849400282} +{"Finetune/Learning Rate": 1.988375326568782e-05, "Finetune/Loss": 0.9791675806045532, "Finetune/Loss (Raw)": 0.9519716501235962, "Finetune/Step": 400, "Finetune/Step Time": 4.6275400798767805} +{"Finetune/Learning Rate": 1.9882804237803487e-05, "Finetune/Loss": 0.9796498417854309, "Finetune/Loss (Raw)": 1.119245171546936, "Finetune/Step": 401, "Finetune/Step Time": 4.6306536588817835} +{"Finetune/Learning Rate": 1.9881851374595774e-05, "Finetune/Loss": 0.9789236783981323, "Finetune/Loss (Raw)": 1.0118207931518555, "Finetune/Step": 402, "Finetune/Step Time": 4.642273282632232} +{"Finetune/Learning Rate": 1.988089467643447e-05, "Finetune/Loss": 0.9785380363464355, "Finetune/Loss (Raw)": 0.977319061756134, "Finetune/Step": 403, "Finetune/Step Time": 4.641667794436216} +{"Finetune/Learning Rate": 1.9879934143690848e-05, "Finetune/Loss": 0.9804472327232361, "Finetune/Loss (Raw)": 1.0809850692749023, "Finetune/Step": 404, "Finetune/Step Time": 4.609040342271328} +{"Finetune/Learning Rate": 1.9878969776737675e-05, "Finetune/Loss": 0.9781437516212463, "Finetune/Loss (Raw)": 0.735207200050354, "Finetune/Step": 405, "Finetune/Step Time": 4.621538171544671} +{"Finetune/Learning Rate": 1.9878001575949205e-05, "Finetune/Loss": 0.9737305045127869, "Finetune/Loss (Raw)": 0.4791395664215088, "Finetune/Step": 406, "Finetune/Step Time": 4.660413997247815} +{"Finetune/Learning Rate": 1.9877029541701174e-05, "Finetune/Loss": 0.9674854874610901, "Finetune/Loss (Raw)": 0.4209394156932831, "Finetune/Step": 407, "Finetune/Step Time": 4.687969019636512} +{"Finetune/Learning Rate": 1.9876053674370806e-05, "Finetune/Loss": 0.9674390554428101, "Finetune/Loss (Raw)": 0.9883085489273071, "Finetune/Step": 408, "Finetune/Step Time": 4.688125539571047} +{"Finetune/Learning Rate": 1.9875073974336827e-05, "Finetune/Loss": 0.9682750701904297, "Finetune/Loss (Raw)": 1.1956162452697754, "Finetune/Step": 409, "Finetune/Step Time": 4.688758742064238} +{"Finetune/Learning Rate": 1.9874090441979433e-05, "Finetune/Loss": 0.9699072241783142, "Finetune/Loss (Raw)": 1.2328745126724243, "Finetune/Step": 410, "Finetune/Step Time": 4.687077697366476} +{"Finetune/Learning Rate": 1.987310307768031e-05, "Finetune/Loss": 0.96880042552948, "Finetune/Loss (Raw)": 1.0010499954223633, "Finetune/Step": 411, "Finetune/Step Time": 4.689974108710885} +{"Finetune/Learning Rate": 1.987211188182264e-05, "Finetune/Loss": 0.9627154469490051, "Finetune/Loss (Raw)": 0.3770710229873657, "Finetune/Step": 412, "Finetune/Step Time": 4.737039113417268} +{"Finetune/Learning Rate": 1.9871116854791085e-05, "Finetune/Loss": 0.9635941982269287, "Finetune/Loss (Raw)": 1.125938057899475, "Finetune/Step": 413, "Finetune/Step Time": 4.714843392372131} +{"Finetune/Learning Rate": 1.9870117996971797e-05, "Finetune/Loss": 0.9625241756439209, "Finetune/Loss (Raw)": 0.9489456415176392, "Finetune/Step": 414, "Finetune/Step Time": 4.699912901967764} +{"Finetune/Learning Rate": 1.986911530875241e-05, "Finetune/Loss": 0.9623361825942993, "Finetune/Loss (Raw)": 1.0639623403549194, "Finetune/Step": 415, "Finetune/Step Time": 4.696568392217159} +{"Finetune/Learning Rate": 1.9868108790522052e-05, "Finetune/Loss": 0.9640243649482727, "Finetune/Loss (Raw)": 1.1290100812911987, "Finetune/Step": 416, "Finetune/Step Time": 4.682031624019146} +{"Finetune/Learning Rate": 1.986709844267133e-05, "Finetune/Loss": 0.9634197950363159, "Finetune/Loss (Raw)": 1.0527207851409912, "Finetune/Step": 417, "Finetune/Step Time": 4.676824262365699} +{"Finetune/Learning Rate": 1.9866084265592343e-05, "Finetune/Loss": 0.9644862413406372, "Finetune/Loss (Raw)": 1.1025758981704712, "Finetune/Step": 418, "Finetune/Step Time": 4.664571408182383} +{"Finetune/Learning Rate": 1.986506625967867e-05, "Finetune/Loss": 0.9633024334907532, "Finetune/Loss (Raw)": 0.9755446314811707, "Finetune/Step": 419, "Finetune/Step Time": 4.682197708636522} +{"Finetune/Learning Rate": 1.9864044425325382e-05, "Finetune/Loss": 0.9574563503265381, "Finetune/Loss (Raw)": 0.3647710680961609, "Finetune/Step": 420, "Finetune/Step Time": 4.720194788649678} +{"Finetune/Learning Rate": 1.986301876292903e-05, "Finetune/Loss": 0.9565415382385254, "Finetune/Loss (Raw)": 1.0370686054229736, "Finetune/Step": 421, "Finetune/Step Time": 4.720634218305349} +{"Finetune/Learning Rate": 1.986198927288766e-05, "Finetune/Loss": 0.9584706425666809, "Finetune/Loss (Raw)": 1.011133074760437, "Finetune/Step": 422, "Finetune/Step Time": 4.726503862068057} +{"Finetune/Learning Rate": 1.9860955955600784e-05, "Finetune/Loss": 0.9587507843971252, "Finetune/Loss (Raw)": 1.1339994668960571, "Finetune/Step": 423, "Finetune/Step Time": 4.720172189176083} +{"Finetune/Learning Rate": 1.9859918811469424e-05, "Finetune/Loss": 0.9589368104934692, "Finetune/Loss (Raw)": 1.0440471172332764, "Finetune/Step": 424, "Finetune/Step Time": 4.719356935471296} +{"Finetune/Learning Rate": 1.9858877840896076e-05, "Finetune/Loss": 0.9559987783432007, "Finetune/Loss (Raw)": 0.7605565190315247, "Finetune/Step": 425, "Finetune/Step Time": 4.7549921832978725} +{"Finetune/Learning Rate": 1.9857833044284714e-05, "Finetune/Loss": 0.9567911624908447, "Finetune/Loss (Raw)": 1.0144802331924438, "Finetune/Step": 426, "Finetune/Step Time": 4.734101826325059} +{"Finetune/Learning Rate": 1.9856784422040802e-05, "Finetune/Loss": 0.9620341062545776, "Finetune/Loss (Raw)": 0.9855169653892517, "Finetune/Step": 427, "Finetune/Step Time": 4.6983955185860395} +{"Finetune/Learning Rate": 1.98557319745713e-05, "Finetune/Loss": 0.9679394364356995, "Finetune/Loss (Raw)": 1.1256979703903198, "Finetune/Step": 428, "Finetune/Step Time": 4.659275475889444} +{"Finetune/Learning Rate": 1.985467570228463e-05, "Finetune/Loss": 0.9732378721237183, "Finetune/Loss (Raw)": 1.0795639753341675, "Finetune/Step": 429, "Finetune/Step Time": 4.623939868062735} +{"Finetune/Learning Rate": 1.985361560559072e-05, "Finetune/Loss": 0.9733276963233948, "Finetune/Loss (Raw)": 0.9054695963859558, "Finetune/Step": 430, "Finetune/Step Time": 4.617338174954057} +{"Finetune/Learning Rate": 1.9852551684900964e-05, "Finetune/Loss": 0.9792773723602295, "Finetune/Loss (Raw)": 1.0882877111434937, "Finetune/Step": 431, "Finetune/Step Time": 4.5872228648513556} +{"Finetune/Learning Rate": 1.9851483940628256e-05, "Finetune/Loss": 0.9780625104904175, "Finetune/Loss (Raw)": 0.9446555376052856, "Finetune/Step": 432, "Finetune/Step Time": 4.593749990686774} +{"Finetune/Learning Rate": 1.9850412373186967e-05, "Finetune/Loss": 0.9769881963729858, "Finetune/Loss (Raw)": 0.9178889989852905, "Finetune/Step": 433, "Finetune/Step Time": 4.597005404531956} +{"Finetune/Learning Rate": 1.9849336982992948e-05, "Finetune/Loss": 0.9773147106170654, "Finetune/Loss (Raw)": 0.9880813956260681, "Finetune/Step": 434, "Finetune/Step Time": 4.597529647871852} +{"Finetune/Learning Rate": 1.984825777046354e-05, "Finetune/Loss": 0.9747656583786011, "Finetune/Loss (Raw)": 0.8411334753036499, "Finetune/Step": 435, "Finetune/Step Time": 4.582664655521512} +{"Finetune/Learning Rate": 1.9847174736017557e-05, "Finetune/Loss": 0.9738011956214905, "Finetune/Loss (Raw)": 0.9277476072311401, "Finetune/Step": 436, "Finetune/Step Time": 4.579553332179785} +{"Finetune/Learning Rate": 1.9846087880075314e-05, "Finetune/Loss": 0.9728522896766663, "Finetune/Loss (Raw)": 1.032610297203064, "Finetune/Step": 437, "Finetune/Step Time": 4.560103055089712} +{"Finetune/Learning Rate": 1.984499720305859e-05, "Finetune/Loss": 0.9724702835083008, "Finetune/Loss (Raw)": 0.9684743285179138, "Finetune/Step": 438, "Finetune/Step Time": 4.557675147429109} +{"Finetune/Learning Rate": 1.984390270539066e-05, "Finetune/Loss": 0.9720149040222168, "Finetune/Loss (Raw)": 0.9830663204193115, "Finetune/Step": 439, "Finetune/Step Time": 4.555784970521927} +{"Finetune/Learning Rate": 1.9842804387496276e-05, "Finetune/Loss": 0.9727792143821716, "Finetune/Loss (Raw)": 1.0840891599655151, "Finetune/Step": 440, "Finetune/Step Time": 4.550895111635327} +{"Finetune/Learning Rate": 1.9841702249801677e-05, "Finetune/Loss": 0.9723092317581177, "Finetune/Loss (Raw)": 1.0564868450164795, "Finetune/Step": 441, "Finetune/Step Time": 4.5600400641560555} +{"Finetune/Learning Rate": 1.9840596292734573e-05, "Finetune/Loss": 0.9665089845657349, "Finetune/Loss (Raw)": 0.44294100999832153, "Finetune/Step": 442, "Finetune/Step Time": 4.60068285651505} +{"Finetune/Learning Rate": 1.9839486516724173e-05, "Finetune/Loss": 0.967167317867279, "Finetune/Loss (Raw)": 1.078629493713379, "Finetune/Step": 443, "Finetune/Step Time": 4.610548434779048} +{"Finetune/Learning Rate": 1.983837292220115e-05, "Finetune/Loss": 0.9731034636497498, "Finetune/Loss (Raw)": 1.1425650119781494, "Finetune/Step": 444, "Finetune/Step Time": 4.571896342560649} +{"Finetune/Learning Rate": 1.983725550959768e-05, "Finetune/Loss": 0.9734233617782593, "Finetune/Loss (Raw)": 1.0154081583023071, "Finetune/Step": 445, "Finetune/Step Time": 4.578681971877813} +{"Finetune/Learning Rate": 1.98361342793474e-05, "Finetune/Loss": 0.9738765358924866, "Finetune/Loss (Raw)": 1.1207064390182495, "Finetune/Step": 446, "Finetune/Step Time": 4.5808045566082} +{"Finetune/Learning Rate": 1.9835009231885444e-05, "Finetune/Loss": 0.9741915464401245, "Finetune/Loss (Raw)": 1.0382617712020874, "Finetune/Step": 447, "Finetune/Step Time": 4.570873200893402} +{"Finetune/Learning Rate": 1.9833880367648412e-05, "Finetune/Loss": 0.9742965698242188, "Finetune/Loss (Raw)": 0.9637437462806702, "Finetune/Step": 448, "Finetune/Step Time": 4.570580754429102} +{"Finetune/Learning Rate": 1.9832747687074403e-05, "Finetune/Loss": 0.9806140065193176, "Finetune/Loss (Raw)": 1.141511082649231, "Finetune/Step": 449, "Finetune/Step Time": 4.5351560562849045} +{"Finetune/Learning Rate": 1.9831611190602984e-05, "Finetune/Loss": 0.9802160263061523, "Finetune/Loss (Raw)": 0.947565495967865, "Finetune/Step": 450, "Finetune/Step Time": 4.534454423934221} +{"Finetune/Learning Rate": 1.983047087867521e-05, "Finetune/Loss": 0.9795000553131104, "Finetune/Loss (Raw)": 0.9783509373664856, "Finetune/Step": 451, "Finetune/Step Time": 4.544923743233085} +{"Finetune/Learning Rate": 1.9829326751733607e-05, "Finetune/Loss": 0.9787195324897766, "Finetune/Loss (Raw)": 1.0087863206863403, "Finetune/Step": 452, "Finetune/Step Time": 4.5375409089028835} +{"Finetune/Learning Rate": 1.9828178810222197e-05, "Finetune/Loss": 0.9801069498062134, "Finetune/Loss (Raw)": 1.2279623746871948, "Finetune/Step": 453, "Finetune/Step Time": 4.532364085316658} +{"Finetune/Learning Rate": 1.9827027054586468e-05, "Finetune/Loss": 0.9783655405044556, "Finetune/Loss (Raw)": 0.9967743158340454, "Finetune/Step": 454, "Finetune/Step Time": 4.547200122848153} +{"Finetune/Learning Rate": 1.9825871485273395e-05, "Finetune/Loss": 0.978949785232544, "Finetune/Loss (Raw)": 1.0951029062271118, "Finetune/Step": 455, "Finetune/Step Time": 4.545813350006938} +{"Finetune/Learning Rate": 1.9824712102731435e-05, "Finetune/Loss": 0.9732252359390259, "Finetune/Loss (Raw)": 0.3488866090774536, "Finetune/Step": 456, "Finetune/Step Time": 4.57265642657876} +{"Finetune/Learning Rate": 1.982354890741052e-05, "Finetune/Loss": 0.9729876518249512, "Finetune/Loss (Raw)": 0.917508065700531, "Finetune/Step": 457, "Finetune/Step Time": 4.5535353403538465} +{"Finetune/Learning Rate": 1.982238189976206e-05, "Finetune/Loss": 0.9756500720977783, "Finetune/Loss (Raw)": 1.219878911972046, "Finetune/Step": 458, "Finetune/Step Time": 4.550319366157055} +{"Finetune/Learning Rate": 1.9821211080238953e-05, "Finetune/Loss": 0.975162923336029, "Finetune/Loss (Raw)": 0.9994450807571411, "Finetune/Step": 459, "Finetune/Step Time": 4.55258509889245} +{"Finetune/Learning Rate": 1.9820036449295564e-05, "Finetune/Loss": 0.975317656993866, "Finetune/Loss (Raw)": 0.8514099717140198, "Finetune/Step": 460, "Finetune/Step Time": 4.551716247573495} +{"Finetune/Learning Rate": 1.9818858007387753e-05, "Finetune/Loss": 0.9748753309249878, "Finetune/Loss (Raw)": 0.984836995601654, "Finetune/Step": 461, "Finetune/Step Time": 4.550248108804226} +{"Finetune/Learning Rate": 1.9817675754972846e-05, "Finetune/Loss": 0.9748315215110779, "Finetune/Loss (Raw)": 0.9969937205314636, "Finetune/Step": 462, "Finetune/Step Time": 4.549587346613407} +{"Finetune/Learning Rate": 1.9816489692509654e-05, "Finetune/Loss": 0.9762677550315857, "Finetune/Loss (Raw)": 1.1965906620025635, "Finetune/Step": 463, "Finetune/Step Time": 4.543550970032811} +{"Finetune/Learning Rate": 1.9815299820458458e-05, "Finetune/Loss": 0.9698506593704224, "Finetune/Loss (Raw)": 0.2942434549331665, "Finetune/Step": 464, "Finetune/Step Time": 4.582210831344128} +{"Finetune/Learning Rate": 1.9814106139281033e-05, "Finetune/Loss": 0.9683672189712524, "Finetune/Loss (Raw)": 1.0459893941879272, "Finetune/Step": 465, "Finetune/Step Time": 4.590029999613762} +{"Finetune/Learning Rate": 1.9812908649440618e-05, "Finetune/Loss": 0.9679428339004517, "Finetune/Loss (Raw)": 0.9452037215232849, "Finetune/Step": 466, "Finetune/Step Time": 4.594531126320362} +{"Finetune/Learning Rate": 1.981170735140194e-05, "Finetune/Loss": 0.9688721895217896, "Finetune/Loss (Raw)": 1.089141845703125, "Finetune/Step": 467, "Finetune/Step Time": 4.5944565217942} +{"Finetune/Learning Rate": 1.9810502245631194e-05, "Finetune/Loss": 0.9682401418685913, "Finetune/Loss (Raw)": 0.837121307849884, "Finetune/Step": 468, "Finetune/Step Time": 4.606221022084355} +{"Finetune/Learning Rate": 1.980929333259606e-05, "Finetune/Loss": 0.9629511833190918, "Finetune/Loss (Raw)": 0.35641565918922424, "Finetune/Step": 469, "Finetune/Step Time": 4.640486389398575} +{"Finetune/Learning Rate": 1.9808080612765694e-05, "Finetune/Loss": 0.9621667861938477, "Finetune/Loss (Raw)": 0.9908986687660217, "Finetune/Step": 470, "Finetune/Step Time": 4.640276957303286} +{"Finetune/Learning Rate": 1.980686408661073e-05, "Finetune/Loss": 0.9621031880378723, "Finetune/Loss (Raw)": 1.0161789655685425, "Finetune/Step": 471, "Finetune/Step Time": 4.655646776780486} +{"Finetune/Learning Rate": 1.9805643754603275e-05, "Finetune/Loss": 0.9624841213226318, "Finetune/Loss (Raw)": 1.1396225690841675, "Finetune/Step": 472, "Finetune/Step Time": 4.657093998044729} +{"Finetune/Learning Rate": 1.9804419617216922e-05, "Finetune/Loss": 0.9608657360076904, "Finetune/Loss (Raw)": 0.9229401350021362, "Finetune/Step": 473, "Finetune/Step Time": 4.655090356245637} +{"Finetune/Learning Rate": 1.9803191674926727e-05, "Finetune/Loss": 0.9602899551391602, "Finetune/Loss (Raw)": 1.0591013431549072, "Finetune/Step": 474, "Finetune/Step Time": 4.650920914486051} +{"Finetune/Learning Rate": 1.9801959928209236e-05, "Finetune/Loss": 0.9596060514450073, "Finetune/Loss (Raw)": 1.0701912641525269, "Finetune/Step": 475, "Finetune/Step Time": 4.649945966899395} +{"Finetune/Learning Rate": 1.9800724377542463e-05, "Finetune/Loss": 0.9592212438583374, "Finetune/Loss (Raw)": 1.0332679748535156, "Finetune/Step": 476, "Finetune/Step Time": 4.634424911811948} +{"Finetune/Learning Rate": 1.9799485023405904e-05, "Finetune/Loss": 0.9588576555252075, "Finetune/Loss (Raw)": 1.1121342182159424, "Finetune/Step": 477, "Finetune/Step Time": 4.640209263190627} +{"Finetune/Learning Rate": 1.9798241866280523e-05, "Finetune/Loss": 0.958712100982666, "Finetune/Loss (Raw)": 0.9744465947151184, "Finetune/Step": 478, "Finetune/Step Time": 4.616724440827966} +{"Finetune/Learning Rate": 1.979699490664877e-05, "Finetune/Loss": 0.958954393863678, "Finetune/Loss (Raw)": 1.032288908958435, "Finetune/Step": 479, "Finetune/Step Time": 4.601703807711601} +{"Finetune/Learning Rate": 1.9795744144994562e-05, "Finetune/Loss": 0.9596725702285767, "Finetune/Loss (Raw)": 1.0959439277648926, "Finetune/Step": 480, "Finetune/Step Time": 4.59373708255589} +{"Finetune/Learning Rate": 1.97944895818033e-05, "Finetune/Loss": 0.9600366950035095, "Finetune/Loss (Raw)": 1.028533697128296, "Finetune/Step": 481, "Finetune/Step Time": 4.598198637366295} +{"Finetune/Learning Rate": 1.979323121756185e-05, "Finetune/Loss": 0.961119532585144, "Finetune/Loss (Raw)": 1.0500144958496094, "Finetune/Step": 482, "Finetune/Step Time": 4.6068490874022245} +{"Finetune/Learning Rate": 1.9791969052758563e-05, "Finetune/Loss": 0.9608334302902222, "Finetune/Loss (Raw)": 1.0393526554107666, "Finetune/Step": 483, "Finetune/Step Time": 4.617514545097947} +{"Finetune/Learning Rate": 1.9790703087883258e-05, "Finetune/Loss": 0.9612863063812256, "Finetune/Loss (Raw)": 0.9771184325218201, "Finetune/Step": 484, "Finetune/Step Time": 4.621036596596241} +{"Finetune/Learning Rate": 1.9789433323427233e-05, "Finetune/Loss": 0.9606845378875732, "Finetune/Loss (Raw)": 0.8559169173240662, "Finetune/Step": 485, "Finetune/Step Time": 4.624664982780814} +{"Finetune/Learning Rate": 1.9788159759883258e-05, "Finetune/Loss": 0.9654976725578308, "Finetune/Loss (Raw)": 0.991270899772644, "Finetune/Step": 486, "Finetune/Step Time": 4.598574748262763} +{"Finetune/Learning Rate": 1.9786882397745573e-05, "Finetune/Loss": 0.9669666886329651, "Finetune/Loss (Raw)": 1.0442649126052856, "Finetune/Step": 487, "Finetune/Step Time": 4.597555119544268} +{"Finetune/Learning Rate": 1.9785601237509907e-05, "Finetune/Loss": 0.9672175645828247, "Finetune/Loss (Raw)": 1.0458884239196777, "Finetune/Step": 488, "Finetune/Step Time": 4.594913316890597} +{"Finetune/Learning Rate": 1.978431627967345e-05, "Finetune/Loss": 0.9676685333251953, "Finetune/Loss (Raw)": 0.9518780708312988, "Finetune/Step": 489, "Finetune/Step Time": 4.599339170381427} +{"Finetune/Learning Rate": 1.9783027524734865e-05, "Finetune/Loss": 0.9673435091972351, "Finetune/Loss (Raw)": 0.9408881068229675, "Finetune/Step": 490, "Finetune/Step Time": 4.605625027790666} +{"Finetune/Learning Rate": 1.9781734973194294e-05, "Finetune/Loss": 0.964223325252533, "Finetune/Loss (Raw)": 0.6820563673973083, "Finetune/Step": 491, "Finetune/Step Time": 4.633943719789386} +{"Finetune/Learning Rate": 1.9780438625553352e-05, "Finetune/Loss": 0.9631409645080566, "Finetune/Loss (Raw)": 0.9686011075973511, "Finetune/Step": 492, "Finetune/Step Time": 4.639752460643649} +{"Finetune/Learning Rate": 1.9779138482315128e-05, "Finetune/Loss": 0.9628167152404785, "Finetune/Loss (Raw)": 1.0843805074691772, "Finetune/Step": 493, "Finetune/Step Time": 4.642506698146462} +{"Finetune/Learning Rate": 1.977783454398418e-05, "Finetune/Loss": 0.9633768796920776, "Finetune/Loss (Raw)": 0.9762558341026306, "Finetune/Step": 494, "Finetune/Step Time": 4.653776939958334} +{"Finetune/Learning Rate": 1.9776526811066543e-05, "Finetune/Loss": 0.963448703289032, "Finetune/Loss (Raw)": 1.0180225372314453, "Finetune/Step": 495, "Finetune/Step Time": 4.653854671865702} +{"Finetune/Learning Rate": 1.977521528406972e-05, "Finetune/Loss": 0.9635535478591919, "Finetune/Loss (Raw)": 1.0430556535720825, "Finetune/Step": 496, "Finetune/Step Time": 4.6675807274878025} +{"Finetune/Learning Rate": 1.9773899963502693e-05, "Finetune/Loss": 0.9634220600128174, "Finetune/Loss (Raw)": 0.9745445251464844, "Finetune/Step": 497, "Finetune/Step Time": 4.66969926841557} +{"Finetune/Learning Rate": 1.9772580849875905e-05, "Finetune/Loss": 0.962510347366333, "Finetune/Loss (Raw)": 1.0518816709518433, "Finetune/Step": 498, "Finetune/Step Time": 4.6681589391082525} +{"Finetune/Learning Rate": 1.9771257943701284e-05, "Finetune/Loss": 0.9603633880615234, "Finetune/Loss (Raw)": 0.8927162289619446, "Finetune/Step": 499, "Finetune/Step Time": 4.67027978040278} +{"Finetune/Learning Rate": 1.9769931245492225e-05, "Finetune/Loss": 0.9599588513374329, "Finetune/Loss (Raw)": 0.9987647533416748, "Finetune/Step": 500, "Finetune/Step Time": 4.669204156845808} +{"Finetune/Learning Rate": 1.976860075576359e-05, "Finetune/Loss": 0.9627038836479187, "Finetune/Loss (Raw)": 0.7990559339523315, "Finetune/Step": 501, "Finetune/Step Time": 4.645992448553443} +{"Finetune/Learning Rate": 1.9767266475031716e-05, "Finetune/Loss": 0.9667930603027344, "Finetune/Loss (Raw)": 0.8917385935783386, "Finetune/Step": 502, "Finetune/Step Time": 4.620353052392602} +{"Finetune/Learning Rate": 1.9765928403814416e-05, "Finetune/Loss": 0.9662085175514221, "Finetune/Loss (Raw)": 1.0390141010284424, "Finetune/Step": 503, "Finetune/Step Time": 4.636741561815143} +{"Finetune/Learning Rate": 1.9764586542630965e-05, "Finetune/Loss": 0.9674954414367676, "Finetune/Loss (Raw)": 1.0675983428955078, "Finetune/Step": 504, "Finetune/Step Time": 4.6327219028025866} +{"Finetune/Learning Rate": 1.9763240892002113e-05, "Finetune/Loss": 0.9678102731704712, "Finetune/Loss (Raw)": 1.014976143836975, "Finetune/Step": 505, "Finetune/Step Time": 4.633880943059921} +{"Finetune/Learning Rate": 1.9761891452450082e-05, "Finetune/Loss": 0.9647411108016968, "Finetune/Loss (Raw)": 0.8303106427192688, "Finetune/Step": 506, "Finetune/Step Time": 4.651111101731658} +{"Finetune/Learning Rate": 1.9760538224498563e-05, "Finetune/Loss": 0.963649570941925, "Finetune/Loss (Raw)": 1.0205618143081665, "Finetune/Step": 507, "Finetune/Step Time": 4.647329192608595} +{"Finetune/Learning Rate": 1.975918120867272e-05, "Finetune/Loss": 0.9633055925369263, "Finetune/Loss (Raw)": 0.9995550513267517, "Finetune/Step": 508, "Finetune/Step Time": 4.640660334378481} +{"Finetune/Learning Rate": 1.975782040549918e-05, "Finetune/Loss": 0.9626173973083496, "Finetune/Loss (Raw)": 0.9769263863563538, "Finetune/Step": 509, "Finetune/Step Time": 4.639151129871607} +{"Finetune/Learning Rate": 1.9756455815506045e-05, "Finetune/Loss": 0.9626999497413635, "Finetune/Loss (Raw)": 0.9371254444122314, "Finetune/Step": 510, "Finetune/Step Time": 4.595296397805214} +{"Finetune/Learning Rate": 1.9755087439222888e-05, "Finetune/Loss": 0.9639474153518677, "Finetune/Loss (Raw)": 1.1047056913375854, "Finetune/Step": 511, "Finetune/Step Time": 4.561387633904815} +{"Finetune/Learning Rate": 1.975371527718075e-05, "Finetune/Loss": 0.9681299328804016, "Finetune/Loss (Raw)": 1.0317314863204956, "Finetune/Step": 512, "Finetune/Step Time": 4.530330963432789} +{"Finetune/Learning Rate": 1.9752339329912137e-05, "Finetune/Loss": 0.9685317277908325, "Finetune/Loss (Raw)": 1.0669876337051392, "Finetune/Step": 513, "Finetune/Step Time": 4.540542280301452} +{"Finetune/Learning Rate": 1.9750959597951026e-05, "Finetune/Loss": 0.9691781401634216, "Finetune/Loss (Raw)": 1.1018046140670776, "Finetune/Step": 514, "Finetune/Step Time": 4.553946575149894} +{"Finetune/Learning Rate": 1.9749576081832865e-05, "Finetune/Loss": 0.9654761552810669, "Finetune/Loss (Raw)": 0.40579280257225037, "Finetune/Step": 515, "Finetune/Step Time": 4.591951701790094} +{"Finetune/Learning Rate": 1.9748188782094576e-05, "Finetune/Loss": 0.9665426015853882, "Finetune/Loss (Raw)": 1.0912292003631592, "Finetune/Step": 516, "Finetune/Step Time": 4.606454545632005} +{"Finetune/Learning Rate": 1.9746797699274535e-05, "Finetune/Loss": 0.9673036336898804, "Finetune/Loss (Raw)": 1.0413960218429565, "Finetune/Step": 517, "Finetune/Step Time": 4.605551462620497} +{"Finetune/Learning Rate": 1.9745402833912598e-05, "Finetune/Loss": 0.9689947366714478, "Finetune/Loss (Raw)": 1.05979323387146, "Finetune/Step": 518, "Finetune/Step Time": 4.608297297731042} +{"Finetune/Learning Rate": 1.9744004186550086e-05, "Finetune/Loss": 0.9694651365280151, "Finetune/Loss (Raw)": 1.0693696737289429, "Finetune/Step": 519, "Finetune/Step Time": 4.622974867001176} +{"Finetune/Learning Rate": 1.974260175772978e-05, "Finetune/Loss": 0.9697339534759521, "Finetune/Loss (Raw)": 1.0162513256072998, "Finetune/Step": 520, "Finetune/Step Time": 4.622707640752196} +{"Finetune/Learning Rate": 1.9741195547995944e-05, "Finetune/Loss": 0.9709497690200806, "Finetune/Loss (Raw)": 1.0118718147277832, "Finetune/Step": 521, "Finetune/Step Time": 4.614207725971937} +{"Finetune/Learning Rate": 1.9739785557894296e-05, "Finetune/Loss": 0.9724652171134949, "Finetune/Loss (Raw)": 1.0027990341186523, "Finetune/Step": 522, "Finetune/Step Time": 4.595914704725146} +{"Finetune/Learning Rate": 1.9738371787972025e-05, "Finetune/Loss": 0.9701381325721741, "Finetune/Loss (Raw)": 0.9117085933685303, "Finetune/Step": 523, "Finetune/Step Time": 4.602878592908382} +{"Finetune/Learning Rate": 1.9736954238777793e-05, "Finetune/Loss": 0.9694243669509888, "Finetune/Loss (Raw)": 1.0203484296798706, "Finetune/Step": 524, "Finetune/Step Time": 4.598559083417058} +{"Finetune/Learning Rate": 1.9735532910861713e-05, "Finetune/Loss": 0.9707977771759033, "Finetune/Loss (Raw)": 1.0266191959381104, "Finetune/Step": 525, "Finetune/Step Time": 4.581542165949941} +{"Finetune/Learning Rate": 1.9734107804775385e-05, "Finetune/Loss": 0.9688554406166077, "Finetune/Loss (Raw)": 0.9343103766441345, "Finetune/Step": 526, "Finetune/Step Time": 4.5943477135151625} +{"Finetune/Learning Rate": 1.9732678921071858e-05, "Finetune/Loss": 0.9691683053970337, "Finetune/Loss (Raw)": 0.9704932570457458, "Finetune/Step": 527, "Finetune/Step Time": 4.58858123421669} +{"Finetune/Learning Rate": 1.9731246260305657e-05, "Finetune/Loss": 0.9699132442474365, "Finetune/Loss (Raw)": 1.0473253726959229, "Finetune/Step": 528, "Finetune/Step Time": 4.583616869524121} +{"Finetune/Learning Rate": 1.9729809823032767e-05, "Finetune/Loss": 0.9697762131690979, "Finetune/Loss (Raw)": 1.1017014980316162, "Finetune/Step": 529, "Finetune/Step Time": 4.582719396799803} +{"Finetune/Learning Rate": 1.9728369609810648e-05, "Finetune/Loss": 0.9704877138137817, "Finetune/Loss (Raw)": 1.1028921604156494, "Finetune/Step": 530, "Finetune/Step Time": 4.567397782579064} +{"Finetune/Learning Rate": 1.972692562119821e-05, "Finetune/Loss": 0.9719125628471375, "Finetune/Loss (Raw)": 1.1597012281417847, "Finetune/Step": 531, "Finetune/Step Time": 4.56415132433176} +{"Finetune/Learning Rate": 1.9725477857755842e-05, "Finetune/Loss": 0.9711881875991821, "Finetune/Loss (Raw)": 0.9882606267929077, "Finetune/Step": 532, "Finetune/Step Time": 4.562162550166249} +{"Finetune/Learning Rate": 1.9724026320045393e-05, "Finetune/Loss": 0.9728446006774902, "Finetune/Loss (Raw)": 0.9472377300262451, "Finetune/Step": 533, "Finetune/Step Time": 4.558397151529789} +{"Finetune/Learning Rate": 1.9722571008630172e-05, "Finetune/Loss": 0.9764419794082642, "Finetune/Loss (Raw)": 0.9395925402641296, "Finetune/Step": 534, "Finetune/Step Time": 4.5271206349134445} +{"Finetune/Learning Rate": 1.9721111924074958e-05, "Finetune/Loss": 0.9804308414459229, "Finetune/Loss (Raw)": 0.9315131306648254, "Finetune/Step": 535, "Finetune/Step Time": 4.499853849411011} +{"Finetune/Learning Rate": 1.9719649066946e-05, "Finetune/Loss": 0.9791193008422852, "Finetune/Loss (Raw)": 0.8204357028007507, "Finetune/Step": 536, "Finetune/Step Time": 4.521873973309994} +{"Finetune/Learning Rate": 1.9718182437810996e-05, "Finetune/Loss": 0.9780983924865723, "Finetune/Loss (Raw)": 1.0649504661560059, "Finetune/Step": 537, "Finetune/Step Time": 4.523210601881146} +{"Finetune/Learning Rate": 1.971671203723912e-05, "Finetune/Loss": 0.9764517545700073, "Finetune/Loss (Raw)": 1.022094488143921, "Finetune/Step": 538, "Finetune/Step Time": 4.532201459631324} +{"Finetune/Learning Rate": 1.9715237865801007e-05, "Finetune/Loss": 0.9772117137908936, "Finetune/Loss (Raw)": 1.0983222723007202, "Finetune/Step": 539, "Finetune/Step Time": 4.526227377355099} +{"Finetune/Learning Rate": 1.971375992406875e-05, "Finetune/Loss": 0.9818850755691528, "Finetune/Loss (Raw)": 0.9752693176269531, "Finetune/Step": 540, "Finetune/Step Time": 4.4790981486439705} +{"Finetune/Learning Rate": 1.9712278212615913e-05, "Finetune/Loss": 0.9809199571609497, "Finetune/Loss (Raw)": 1.002402901649475, "Finetune/Step": 541, "Finetune/Step Time": 4.486562082543969} +{"Finetune/Learning Rate": 1.9710792732017517e-05, "Finetune/Loss": 0.9815078377723694, "Finetune/Loss (Raw)": 1.024190902709961, "Finetune/Step": 542, "Finetune/Step Time": 4.485674053430557} +{"Finetune/Learning Rate": 1.9709303482850047e-05, "Finetune/Loss": 0.9819895625114441, "Finetune/Loss (Raw)": 1.125623106956482, "Finetune/Step": 543, "Finetune/Step Time": 4.485666116699576} +{"Finetune/Learning Rate": 1.9707810465691456e-05, "Finetune/Loss": 0.9816279411315918, "Finetune/Loss (Raw)": 1.0827245712280273, "Finetune/Step": 544, "Finetune/Step Time": 4.48644839040935} +{"Finetune/Learning Rate": 1.970631368112115e-05, "Finetune/Loss": 0.9811412692070007, "Finetune/Loss (Raw)": 0.9904292821884155, "Finetune/Step": 545, "Finetune/Step Time": 4.483908785507083} +{"Finetune/Learning Rate": 1.970481312972001e-05, "Finetune/Loss": 0.9798922538757324, "Finetune/Loss (Raw)": 0.9426963925361633, "Finetune/Step": 546, "Finetune/Step Time": 4.485291089862585} +{"Finetune/Learning Rate": 1.9703308812070358e-05, "Finetune/Loss": 0.9807150363922119, "Finetune/Loss (Raw)": 1.0808658599853516, "Finetune/Step": 547, "Finetune/Step Time": 4.476215492933989} +{"Finetune/Learning Rate": 1.9701800728756e-05, "Finetune/Loss": 0.9853804111480713, "Finetune/Loss (Raw)": 0.9619383215904236, "Finetune/Step": 548, "Finetune/Step Time": 4.452565720304847} +{"Finetune/Learning Rate": 1.970028888036219e-05, "Finetune/Loss": 0.9839152097702026, "Finetune/Loss (Raw)": 0.8495209813117981, "Finetune/Step": 549, "Finetune/Step Time": 4.461111709475517} +{"Finetune/Learning Rate": 1.9698773267475652e-05, "Finetune/Loss": 0.9838290214538574, "Finetune/Loss (Raw)": 1.0000944137573242, "Finetune/Step": 550, "Finetune/Step Time": 4.443302413448691} +{"Finetune/Learning Rate": 1.9697253890684556e-05, "Finetune/Loss": 0.9826482534408569, "Finetune/Loss (Raw)": 0.9828664660453796, "Finetune/Step": 551, "Finetune/Step Time": 4.444582661613822} +{"Finetune/Learning Rate": 1.969573075057855e-05, "Finetune/Loss": 0.9830585718154907, "Finetune/Loss (Raw)": 1.096571445465088, "Finetune/Step": 552, "Finetune/Step Time": 4.4402673449367285} +{"Finetune/Learning Rate": 1.9694203847748734e-05, "Finetune/Loss": 0.9830854535102844, "Finetune/Loss (Raw)": 0.7640037536621094, "Finetune/Step": 553, "Finetune/Step Time": 4.404032688587904} +{"Finetune/Learning Rate": 1.969267318278767e-05, "Finetune/Loss": 0.9840611815452576, "Finetune/Loss (Raw)": 1.1393636465072632, "Finetune/Step": 554, "Finetune/Step Time": 4.401593187823892} +{"Finetune/Learning Rate": 1.969113875628937e-05, "Finetune/Loss": 0.9837973117828369, "Finetune/Loss (Raw)": 0.9517378807067871, "Finetune/Step": 555, "Finetune/Step Time": 4.41521224938333} +{"Finetune/Learning Rate": 1.9689600568849328e-05, "Finetune/Loss": 0.9829339981079102, "Finetune/Loss (Raw)": 1.0151996612548828, "Finetune/Step": 556, "Finetune/Step Time": 4.42726956307888} +{"Finetune/Learning Rate": 1.968805862106448e-05, "Finetune/Loss": 0.9819692373275757, "Finetune/Loss (Raw)": 0.956080436706543, "Finetune/Step": 557, "Finetune/Step Time": 4.4236771538853645} +{"Finetune/Learning Rate": 1.968651291353322e-05, "Finetune/Loss": 0.9768807888031006, "Finetune/Loss (Raw)": 0.2541472911834717, "Finetune/Step": 558, "Finetune/Step Time": 4.456300074234605} +{"Finetune/Learning Rate": 1.968496344685541e-05, "Finetune/Loss": 0.9760308265686035, "Finetune/Loss (Raw)": 0.9794825911521912, "Finetune/Step": 559, "Finetune/Step Time": 4.450433544814587} +{"Finetune/Learning Rate": 1.9683410221632374e-05, "Finetune/Loss": 0.9752801656723022, "Finetune/Loss (Raw)": 0.8485738635063171, "Finetune/Step": 560, "Finetune/Step Time": 4.445128278806806} +{"Finetune/Learning Rate": 1.9681853238466883e-05, "Finetune/Loss": 0.9750363230705261, "Finetune/Loss (Raw)": 0.8866850733757019, "Finetune/Step": 561, "Finetune/Step Time": 4.446177922189236} +{"Finetune/Learning Rate": 1.9680292497963172e-05, "Finetune/Loss": 0.9746842384338379, "Finetune/Loss (Raw)": 0.9430090188980103, "Finetune/Step": 562, "Finetune/Step Time": 4.444118211045861} +{"Finetune/Learning Rate": 1.9678728000726936e-05, "Finetune/Loss": 0.9763240218162537, "Finetune/Loss (Raw)": 1.0510215759277344, "Finetune/Step": 563, "Finetune/Step Time": 4.444766920059919} +{"Finetune/Learning Rate": 1.967715974736532e-05, "Finetune/Loss": 0.9771701097488403, "Finetune/Loss (Raw)": 1.0360524654388428, "Finetune/Step": 564, "Finetune/Step Time": 4.454715333878994} +{"Finetune/Learning Rate": 1.9675587738486935e-05, "Finetune/Loss": 0.9765684008598328, "Finetune/Loss (Raw)": 0.9555912613868713, "Finetune/Step": 565, "Finetune/Step Time": 4.470460524782538} +{"Finetune/Learning Rate": 1.9674011974701855e-05, "Finetune/Loss": 0.9774143695831299, "Finetune/Loss (Raw)": 1.0767529010772705, "Finetune/Step": 566, "Finetune/Step Time": 4.472010042518377} +{"Finetune/Learning Rate": 1.9672432456621597e-05, "Finetune/Loss": 0.9775633215904236, "Finetune/Loss (Raw)": 1.0021284818649292, "Finetune/Step": 567, "Finetune/Step Time": 4.470192287117243} +{"Finetune/Learning Rate": 1.9670849184859136e-05, "Finetune/Loss": 0.9713770747184753, "Finetune/Loss (Raw)": 0.2922639846801758, "Finetune/Step": 568, "Finetune/Step Time": 4.504663128405809} +{"Finetune/Learning Rate": 1.9669262160028916e-05, "Finetune/Loss": 0.9703670144081116, "Finetune/Loss (Raw)": 0.9271964430809021, "Finetune/Step": 569, "Finetune/Step Time": 4.503577519208193} +{"Finetune/Learning Rate": 1.966767138274683e-05, "Finetune/Loss": 0.9748517274856567, "Finetune/Loss (Raw)": 1.0169758796691895, "Finetune/Step": 570, "Finetune/Step Time": 4.46962015889585} +{"Finetune/Learning Rate": 1.966607685363022e-05, "Finetune/Loss": 0.9738430380821228, "Finetune/Loss (Raw)": 0.949519693851471, "Finetune/Step": 571, "Finetune/Step Time": 4.469944771379232} +{"Finetune/Learning Rate": 1.96644785732979e-05, "Finetune/Loss": 0.9730644822120667, "Finetune/Loss (Raw)": 1.0429120063781738, "Finetune/Step": 572, "Finetune/Step Time": 4.469234922900796} +{"Finetune/Learning Rate": 1.966287654237013e-05, "Finetune/Loss": 0.9734989404678345, "Finetune/Loss (Raw)": 1.0710275173187256, "Finetune/Step": 573, "Finetune/Step Time": 4.468662332743406} +{"Finetune/Learning Rate": 1.9661270761468625e-05, "Finetune/Loss": 0.9733166694641113, "Finetune/Loss (Raw)": 1.0973681211471558, "Finetune/Step": 574, "Finetune/Step Time": 4.461109193041921} +{"Finetune/Learning Rate": 1.965966123121656e-05, "Finetune/Loss": 0.974094033241272, "Finetune/Loss (Raw)": 1.137759804725647, "Finetune/Step": 575, "Finetune/Step Time": 4.483585277572274} +{"Finetune/Learning Rate": 1.965804795223856e-05, "Finetune/Loss": 0.9742499589920044, "Finetune/Loss (Raw)": 0.9837092161178589, "Finetune/Step": 576, "Finetune/Step Time": 4.494986599311233} +{"Finetune/Learning Rate": 1.9656430925160704e-05, "Finetune/Loss": 0.9730734825134277, "Finetune/Loss (Raw)": 0.9909250140190125, "Finetune/Step": 577, "Finetune/Step Time": 4.490912465378642} +{"Finetune/Learning Rate": 1.9654810150610536e-05, "Finetune/Loss": 0.9731408357620239, "Finetune/Loss (Raw)": 0.9561811685562134, "Finetune/Step": 578, "Finetune/Step Time": 4.486791778355837} +{"Finetune/Learning Rate": 1.9653185629217038e-05, "Finetune/Loss": 0.9721708297729492, "Finetune/Loss (Raw)": 0.8541842103004456, "Finetune/Step": 579, "Finetune/Step Time": 4.504947729408741} +{"Finetune/Learning Rate": 1.9651557361610663e-05, "Finetune/Loss": 0.9724392890930176, "Finetune/Loss (Raw)": 1.0431543588638306, "Finetune/Step": 580, "Finetune/Step Time": 4.507237762212753} +{"Finetune/Learning Rate": 1.9649925348423308e-05, "Finetune/Loss": 0.9717311859130859, "Finetune/Loss (Raw)": 1.1373281478881836, "Finetune/Step": 581, "Finetune/Step Time": 4.511718450114131} +{"Finetune/Learning Rate": 1.964828959028832e-05, "Finetune/Loss": 0.9721356630325317, "Finetune/Loss (Raw)": 1.0485432147979736, "Finetune/Step": 582, "Finetune/Step Time": 4.50725656375289} +{"Finetune/Learning Rate": 1.964665008784051e-05, "Finetune/Loss": 0.9706072807312012, "Finetune/Loss (Raw)": 0.899471640586853, "Finetune/Step": 583, "Finetune/Step Time": 4.513363093137741} +{"Finetune/Learning Rate": 1.9645006841716137e-05, "Finetune/Loss": 0.9763460755348206, "Finetune/Loss (Raw)": 1.0834503173828125, "Finetune/Step": 584, "Finetune/Step Time": 4.4844178725034} +{"Finetune/Learning Rate": 1.9643359852552912e-05, "Finetune/Loss": 0.9776195287704468, "Finetune/Loss (Raw)": 1.0805103778839111, "Finetune/Step": 585, "Finetune/Step Time": 4.4860161282122135} +{"Finetune/Learning Rate": 1.9641709120989996e-05, "Finetune/Loss": 0.9707973003387451, "Finetune/Loss (Raw)": 0.34663528203964233, "Finetune/Step": 586, "Finetune/Step Time": 4.52047729678452} +{"Finetune/Learning Rate": 1.9640054647668015e-05, "Finetune/Loss": 0.9722958207130432, "Finetune/Loss (Raw)": 1.1912542581558228, "Finetune/Step": 587, "Finetune/Step Time": 4.5118700582534075} +{"Finetune/Learning Rate": 1.9638396433229025e-05, "Finetune/Loss": 0.9734170436859131, "Finetune/Loss (Raw)": 0.994929850101471, "Finetune/Step": 588, "Finetune/Step Time": 4.520499683916569} +{"Finetune/Learning Rate": 1.963673447831656e-05, "Finetune/Loss": 0.9742242097854614, "Finetune/Loss (Raw)": 1.088159203529358, "Finetune/Step": 589, "Finetune/Step Time": 4.515578348189592} +{"Finetune/Learning Rate": 1.9635068783575577e-05, "Finetune/Loss": 0.974972128868103, "Finetune/Loss (Raw)": 1.0927186012268066, "Finetune/Step": 590, "Finetune/Step Time": 4.516735251992941} +{"Finetune/Learning Rate": 1.9633399349652512e-05, "Finetune/Loss": 0.9730089902877808, "Finetune/Loss (Raw)": 0.9453123211860657, "Finetune/Step": 591, "Finetune/Step Time": 4.518957886844873} +{"Finetune/Learning Rate": 1.9631726177195235e-05, "Finetune/Loss": 0.9783391952514648, "Finetune/Loss (Raw)": 0.9765037298202515, "Finetune/Step": 592, "Finetune/Step Time": 4.477927044034004} +{"Finetune/Learning Rate": 1.9630049266853076e-05, "Finetune/Loss": 0.9779587984085083, "Finetune/Loss (Raw)": 0.99729984998703, "Finetune/Step": 593, "Finetune/Step Time": 4.492307910695672} +{"Finetune/Learning Rate": 1.96283686192768e-05, "Finetune/Loss": 0.9784849882125854, "Finetune/Loss (Raw)": 1.0125577449798584, "Finetune/Step": 594, "Finetune/Step Time": 4.480285469442606} +{"Finetune/Learning Rate": 1.9626684235118646e-05, "Finetune/Loss": 0.9778710007667542, "Finetune/Loss (Raw)": 1.0105537176132202, "Finetune/Step": 595, "Finetune/Step Time": 4.470367370173335} +{"Finetune/Learning Rate": 1.9624996115032287e-05, "Finetune/Loss": 0.9786580801010132, "Finetune/Loss (Raw)": 0.9378678798675537, "Finetune/Step": 596, "Finetune/Step Time": 4.452940342947841} +{"Finetune/Learning Rate": 1.9623304259672844e-05, "Finetune/Loss": 0.9827559590339661, "Finetune/Loss (Raw)": 0.880946934223175, "Finetune/Step": 597, "Finetune/Step Time": 4.417210778221488} +{"Finetune/Learning Rate": 1.96216086696969e-05, "Finetune/Loss": 0.9828659296035767, "Finetune/Loss (Raw)": 1.0049769878387451, "Finetune/Step": 598, "Finetune/Step Time": 4.439438164234161} +{"Finetune/Learning Rate": 1.9619909345762474e-05, "Finetune/Loss": 0.9829283356666565, "Finetune/Loss (Raw)": 1.0241597890853882, "Finetune/Step": 599, "Finetune/Step Time": 4.421132387593389} +{"Finetune/Learning Rate": 1.961820628852905e-05, "Finetune/Loss": 0.9826830625534058, "Finetune/Loss (Raw)": 1.1082264184951782, "Finetune/Step": 600, "Finetune/Step Time": 4.425710313022137} +{"Finetune/Learning Rate": 1.9616499498657537e-05, "Finetune/Loss": 0.9831290245056152, "Finetune/Loss (Raw)": 0.9800249934196472, "Finetune/Step": 601, "Finetune/Step Time": 4.449192447587848} +{"Finetune/Learning Rate": 1.961478897681032e-05, "Finetune/Loss": 0.9832153916358948, "Finetune/Loss (Raw)": 1.0701597929000854, "Finetune/Step": 602, "Finetune/Step Time": 4.442312851548195} +{"Finetune/Learning Rate": 1.9613074723651214e-05, "Finetune/Loss": 0.9821567535400391, "Finetune/Loss (Raw)": 0.934678852558136, "Finetune/Step": 603, "Finetune/Step Time": 4.446820585057139} +{"Finetune/Learning Rate": 1.9611356739845484e-05, "Finetune/Loss": 0.9824275970458984, "Finetune/Loss (Raw)": 1.0679367780685425, "Finetune/Step": 604, "Finetune/Step Time": 4.44572377204895} +{"Finetune/Learning Rate": 1.9609635026059855e-05, "Finetune/Loss": 0.9811443090438843, "Finetune/Loss (Raw)": 0.9478790760040283, "Finetune/Step": 605, "Finetune/Step Time": 4.440722484141588} +{"Finetune/Learning Rate": 1.9607909582962478e-05, "Finetune/Loss": 0.9807791709899902, "Finetune/Loss (Raw)": 0.9277095198631287, "Finetune/Step": 606, "Finetune/Step Time": 4.457815997302532} +{"Finetune/Learning Rate": 1.9606180411222974e-05, "Finetune/Loss": 0.9791899919509888, "Finetune/Loss (Raw)": 0.8288699984550476, "Finetune/Step": 607, "Finetune/Step Time": 4.487493149936199} +{"Finetune/Learning Rate": 1.9604447511512396e-05, "Finetune/Loss": 0.9791877269744873, "Finetune/Loss (Raw)": 1.0956569910049438, "Finetune/Step": 608, "Finetune/Step Time": 4.484625121578574} +{"Finetune/Learning Rate": 1.960271088450325e-05, "Finetune/Loss": 0.9783225655555725, "Finetune/Loss (Raw)": 0.9177966713905334, "Finetune/Step": 609, "Finetune/Step Time": 4.479923600330949} +{"Finetune/Learning Rate": 1.960097053086949e-05, "Finetune/Loss": 0.9777170419692993, "Finetune/Loss (Raw)": 0.9725046157836914, "Finetune/Step": 610, "Finetune/Step Time": 4.470503017306328} +{"Finetune/Learning Rate": 1.9599226451286504e-05, "Finetune/Loss": 0.9777076244354248, "Finetune/Loss (Raw)": 1.038141131401062, "Finetune/Step": 611, "Finetune/Step Time": 4.448357664048672} +{"Finetune/Learning Rate": 1.9597478646431147e-05, "Finetune/Loss": 0.9795323610305786, "Finetune/Loss (Raw)": 1.210689663887024, "Finetune/Step": 612, "Finetune/Step Time": 4.445631939917803} +{"Finetune/Learning Rate": 1.95957271169817e-05, "Finetune/Loss": 0.9799044132232666, "Finetune/Loss (Raw)": 0.9035392999649048, "Finetune/Step": 613, "Finetune/Step Time": 4.437971293926239} +{"Finetune/Learning Rate": 1.95939718636179e-05, "Finetune/Loss": 0.9798377156257629, "Finetune/Loss (Raw)": 0.9827317595481873, "Finetune/Step": 614, "Finetune/Step Time": 4.423502245917916} +{"Finetune/Learning Rate": 1.9592212887020923e-05, "Finetune/Loss": 0.9797720909118652, "Finetune/Loss (Raw)": 1.035869836807251, "Finetune/Step": 615, "Finetune/Step Time": 4.430108061060309} +{"Finetune/Learning Rate": 1.9590450187873403e-05, "Finetune/Loss": 0.9794299602508545, "Finetune/Loss (Raw)": 1.0020874738693237, "Finetune/Step": 616, "Finetune/Step Time": 4.432361191138625} +{"Finetune/Learning Rate": 1.95886837668594e-05, "Finetune/Loss": 0.9798091053962708, "Finetune/Loss (Raw)": 1.0004116296768188, "Finetune/Step": 617, "Finetune/Step Time": 4.424480525776744} +{"Finetune/Learning Rate": 1.958691362466443e-05, "Finetune/Loss": 0.980605959892273, "Finetune/Loss (Raw)": 1.042880892753601, "Finetune/Step": 618, "Finetune/Step Time": 4.418696848675609} +{"Finetune/Learning Rate": 1.958513976197545e-05, "Finetune/Loss": 0.9824929237365723, "Finetune/Loss (Raw)": 0.9235922694206238, "Finetune/Step": 619, "Finetune/Step Time": 4.413414794951677} +{"Finetune/Learning Rate": 1.9583362179480866e-05, "Finetune/Loss": 0.9830489158630371, "Finetune/Loss (Raw)": 1.0397676229476929, "Finetune/Step": 620, "Finetune/Step Time": 4.403942156583071} +{"Finetune/Learning Rate": 1.9581580877870514e-05, "Finetune/Loss": 0.9825637340545654, "Finetune/Loss (Raw)": 1.022274136543274, "Finetune/Step": 621, "Finetune/Step Time": 4.422943515703082} +{"Finetune/Learning Rate": 1.9579795857835692e-05, "Finetune/Loss": 0.9825292825698853, "Finetune/Loss (Raw)": 0.9718537330627441, "Finetune/Step": 622, "Finetune/Step Time": 4.41071074269712} +{"Finetune/Learning Rate": 1.9578007120069123e-05, "Finetune/Loss": 0.9829810857772827, "Finetune/Loss (Raw)": 1.0758527517318726, "Finetune/Step": 623, "Finetune/Step Time": 4.412256868556142} +{"Finetune/Learning Rate": 1.957621466526499e-05, "Finetune/Loss": 0.9826370477676392, "Finetune/Loss (Raw)": 0.9990223050117493, "Finetune/Step": 624, "Finetune/Step Time": 4.398155519738793} +{"Finetune/Learning Rate": 1.9574418494118905e-05, "Finetune/Loss": 0.983264684677124, "Finetune/Loss (Raw)": 1.054876446723938, "Finetune/Step": 625, "Finetune/Step Time": 4.39930540509522} +{"Finetune/Learning Rate": 1.9572618607327928e-05, "Finetune/Loss": 0.983392059803009, "Finetune/Loss (Raw)": 1.068185567855835, "Finetune/Step": 626, "Finetune/Step Time": 4.396950911730528} +{"Finetune/Learning Rate": 1.9570815005590556e-05, "Finetune/Loss": 0.9840537309646606, "Finetune/Loss (Raw)": 0.977415919303894, "Finetune/Step": 627, "Finetune/Step Time": 4.398768996819854} +{"Finetune/Learning Rate": 1.9569007689606735e-05, "Finetune/Loss": 0.9840328097343445, "Finetune/Loss (Raw)": 0.996080756187439, "Finetune/Step": 628, "Finetune/Step Time": 4.394116222858429} +{"Finetune/Learning Rate": 1.9567196660077848e-05, "Finetune/Loss": 0.9858593940734863, "Finetune/Loss (Raw)": 1.0328530073165894, "Finetune/Step": 629, "Finetune/Step Time": 4.3788927718997} +{"Finetune/Learning Rate": 1.9565381917706726e-05, "Finetune/Loss": 0.9877941608428955, "Finetune/Loss (Raw)": 1.1393948793411255, "Finetune/Step": 630, "Finetune/Step Time": 4.36049397289753} +{"Finetune/Learning Rate": 1.9563563463197625e-05, "Finetune/Loss": 0.9868106842041016, "Finetune/Loss (Raw)": 0.9131379723548889, "Finetune/Step": 631, "Finetune/Step Time": 4.346186030656099} +{"Finetune/Learning Rate": 1.9561741297256257e-05, "Finetune/Loss": 0.9876259565353394, "Finetune/Loss (Raw)": 1.1719456911087036, "Finetune/Step": 632, "Finetune/Step Time": 4.344798790290952} +{"Finetune/Learning Rate": 1.955991542058977e-05, "Finetune/Loss": 0.9895893335342407, "Finetune/Loss (Raw)": 1.2662841081619263, "Finetune/Step": 633, "Finetune/Step Time": 4.352760825306177} +{"Finetune/Learning Rate": 1.9558085833906753e-05, "Finetune/Loss": 0.9904841184616089, "Finetune/Loss (Raw)": 0.9448456168174744, "Finetune/Step": 634, "Finetune/Step Time": 4.33828698284924} +{"Finetune/Learning Rate": 1.9556252537917228e-05, "Finetune/Loss": 0.990371823310852, "Finetune/Loss (Raw)": 1.0061904191970825, "Finetune/Step": 635, "Finetune/Step Time": 4.335512163117528} +{"Finetune/Learning Rate": 1.955441553333266e-05, "Finetune/Loss": 0.9910407066345215, "Finetune/Loss (Raw)": 1.0851696729660034, "Finetune/Step": 636, "Finetune/Step Time": 4.360400324687362} +{"Finetune/Learning Rate": 1.955257482086596e-05, "Finetune/Loss": 0.9920088648796082, "Finetune/Loss (Raw)": 1.100846767425537, "Finetune/Step": 637, "Finetune/Step Time": 4.349641485139728} +{"Finetune/Learning Rate": 1.955073040123147e-05, "Finetune/Loss": 0.9932916164398193, "Finetune/Loss (Raw)": 1.1013193130493164, "Finetune/Step": 638, "Finetune/Step Time": 4.3552129324525595} +{"Finetune/Learning Rate": 1.9548882275144974e-05, "Finetune/Loss": 0.9926313161849976, "Finetune/Loss (Raw)": 1.0201992988586426, "Finetune/Step": 639, "Finetune/Step Time": 4.352331027388573} +{"Finetune/Learning Rate": 1.9547030443323695e-05, "Finetune/Loss": 0.9924306869506836, "Finetune/Loss (Raw)": 1.0060385465621948, "Finetune/Step": 640, "Finetune/Step Time": 4.346690287813544} +{"Finetune/Learning Rate": 1.9545174906486287e-05, "Finetune/Loss": 0.9926730394363403, "Finetune/Loss (Raw)": 1.09800386428833, "Finetune/Step": 641, "Finetune/Step Time": 4.3478794246912} +{"Finetune/Learning Rate": 1.9543315665352855e-05, "Finetune/Loss": 0.9867151975631714, "Finetune/Loss (Raw)": 0.3392100930213928, "Finetune/Step": 642, "Finetune/Step Time": 4.365501360967755} +{"Finetune/Learning Rate": 1.954145272064493e-05, "Finetune/Loss": 0.991377592086792, "Finetune/Loss (Raw)": 1.0025835037231445, "Finetune/Step": 643, "Finetune/Step Time": 4.3262685518711805} +{"Finetune/Learning Rate": 1.9539586073085482e-05, "Finetune/Loss": 0.9910480976104736, "Finetune/Loss (Raw)": 1.049043893814087, "Finetune/Step": 644, "Finetune/Step Time": 4.319817954674363} +{"Finetune/Learning Rate": 1.9537715723398926e-05, "Finetune/Loss": 0.9913774728775024, "Finetune/Loss (Raw)": 1.0835548639297485, "Finetune/Step": 645, "Finetune/Step Time": 4.318586936220527} +{"Finetune/Learning Rate": 1.9535841672311106e-05, "Finetune/Loss": 0.9910765886306763, "Finetune/Loss (Raw)": 1.0212873220443726, "Finetune/Step": 646, "Finetune/Step Time": 4.334487887099385} +{"Finetune/Learning Rate": 1.9533963920549307e-05, "Finetune/Loss": 0.9901198148727417, "Finetune/Loss (Raw)": 0.9468994140625, "Finetune/Step": 647, "Finetune/Step Time": 4.334922401234508} +{"Finetune/Learning Rate": 1.9532082468842243e-05, "Finetune/Loss": 0.9902710914611816, "Finetune/Loss (Raw)": 1.0356123447418213, "Finetune/Step": 648, "Finetune/Step Time": 4.338539332151413} +{"Finetune/Learning Rate": 1.953019731792007e-05, "Finetune/Loss": 0.9904520511627197, "Finetune/Loss (Raw)": 1.0350333452224731, "Finetune/Step": 649, "Finetune/Step Time": 4.339235993102193} +{"Finetune/Learning Rate": 1.9528308468514383e-05, "Finetune/Loss": 0.9903831481933594, "Finetune/Loss (Raw)": 0.993985116481781, "Finetune/Step": 650, "Finetune/Step Time": 4.352540206164122} +{"Finetune/Learning Rate": 1.9526415921358202e-05, "Finetune/Loss": 0.9913235902786255, "Finetune/Loss (Raw)": 1.0320872068405151, "Finetune/Step": 651, "Finetune/Step Time": 4.3607422560453415} +{"Finetune/Learning Rate": 1.952451967718599e-05, "Finetune/Loss": 0.9911960363388062, "Finetune/Loss (Raw)": 1.004014253616333, "Finetune/Step": 652, "Finetune/Step Time": 4.365019395947456} +{"Finetune/Learning Rate": 1.952261973673364e-05, "Finetune/Loss": 0.9903497099876404, "Finetune/Loss (Raw)": 0.9182906150817871, "Finetune/Step": 653, "Finetune/Step Time": 4.373407427221537} +{"Finetune/Learning Rate": 1.9520716100738487e-05, "Finetune/Loss": 0.9925035238265991, "Finetune/Loss (Raw)": 1.2100043296813965, "Finetune/Step": 654, "Finetune/Step Time": 4.364825641736388} +{"Finetune/Learning Rate": 1.9518808769939287e-05, "Finetune/Loss": 0.9926548004150391, "Finetune/Loss (Raw)": 0.9898555278778076, "Finetune/Step": 655, "Finetune/Step Time": 4.374264447018504} +{"Finetune/Learning Rate": 1.9516897745076243e-05, "Finetune/Loss": 0.9920952320098877, "Finetune/Loss (Raw)": 0.9756882190704346, "Finetune/Step": 656, "Finetune/Step Time": 4.377923384308815} +{"Finetune/Learning Rate": 1.9514983026890988e-05, "Finetune/Loss": 0.9910121560096741, "Finetune/Loss (Raw)": 0.9630751609802246, "Finetune/Step": 657, "Finetune/Step Time": 4.398676589131355} +{"Finetune/Learning Rate": 1.9513064616126588e-05, "Finetune/Loss": 0.9900919198989868, "Finetune/Loss (Raw)": 0.9851048588752747, "Finetune/Step": 658, "Finetune/Step Time": 4.40043518319726} +{"Finetune/Learning Rate": 1.951114251352753e-05, "Finetune/Loss": 0.9872936010360718, "Finetune/Loss (Raw)": 0.8015201687812805, "Finetune/Step": 659, "Finetune/Step Time": 4.422233922407031} +{"Finetune/Learning Rate": 1.950921671983976e-05, "Finetune/Loss": 0.9884145259857178, "Finetune/Loss (Raw)": 1.1317344903945923, "Finetune/Step": 660, "Finetune/Step Time": 4.427054863423109} +{"Finetune/Learning Rate": 1.9507287235810625e-05, "Finetune/Loss": 0.9889596700668335, "Finetune/Loss (Raw)": 1.0170183181762695, "Finetune/Step": 661, "Finetune/Step Time": 4.41943796724081} +{"Finetune/Learning Rate": 1.9505354062188934e-05, "Finetune/Loss": 0.9899135828018188, "Finetune/Loss (Raw)": 1.0616979598999023, "Finetune/Step": 662, "Finetune/Step Time": 4.4088369738310575} +{"Finetune/Learning Rate": 1.9503417199724904e-05, "Finetune/Loss": 0.991344690322876, "Finetune/Loss (Raw)": 1.114690899848938, "Finetune/Step": 663, "Finetune/Step Time": 4.3978933077305555} +{"Finetune/Learning Rate": 1.95014766491702e-05, "Finetune/Loss": 0.9929375648498535, "Finetune/Loss (Raw)": 1.0243239402770996, "Finetune/Step": 664, "Finetune/Step Time": 4.385529236868024} +{"Finetune/Learning Rate": 1.9499532411277907e-05, "Finetune/Loss": 0.9936341047286987, "Finetune/Loss (Raw)": 1.1541008949279785, "Finetune/Step": 665, "Finetune/Step Time": 4.3936513569206} +{"Finetune/Learning Rate": 1.9497584486802548e-05, "Finetune/Loss": 0.9948755502700806, "Finetune/Loss (Raw)": 1.1810020208358765, "Finetune/Step": 666, "Finetune/Step Time": 4.385763216763735} +{"Finetune/Learning Rate": 1.9495632876500076e-05, "Finetune/Loss": 0.9950782060623169, "Finetune/Loss (Raw)": 1.1242647171020508, "Finetune/Step": 667, "Finetune/Step Time": 4.3992765713483095} +{"Finetune/Learning Rate": 1.949367758112787e-05, "Finetune/Loss": 0.9942352771759033, "Finetune/Loss (Raw)": 0.8673723936080933, "Finetune/Step": 668, "Finetune/Step Time": 4.404407182708383} +{"Finetune/Learning Rate": 1.949171860144474e-05, "Finetune/Loss": 0.9945129156112671, "Finetune/Loss (Raw)": 1.0379400253295898, "Finetune/Step": 669, "Finetune/Step Time": 4.398208945989609} +{"Finetune/Learning Rate": 1.9489755938210936e-05, "Finetune/Loss": 0.9936809539794922, "Finetune/Loss (Raw)": 0.9177114367485046, "Finetune/Step": 670, "Finetune/Step Time": 4.40036828443408} +{"Finetune/Learning Rate": 1.9487789592188123e-05, "Finetune/Loss": 0.9926314949989319, "Finetune/Loss (Raw)": 0.9912809729576111, "Finetune/Step": 671, "Finetune/Step Time": 4.399174930527806} +{"Finetune/Learning Rate": 1.9485819564139404e-05, "Finetune/Loss": 0.9924527406692505, "Finetune/Loss (Raw)": 1.0598491430282593, "Finetune/Step": 672, "Finetune/Step Time": 4.395939061418176} +{"Finetune/Learning Rate": 1.9483845854829304e-05, "Finetune/Loss": 0.9934714436531067, "Finetune/Loss (Raw)": 1.120818853378296, "Finetune/Step": 673, "Finetune/Step Time": 4.397464046254754} +{"Finetune/Learning Rate": 1.9481868465023788e-05, "Finetune/Loss": 0.9948680996894836, "Finetune/Loss (Raw)": 1.1214656829833984, "Finetune/Step": 674, "Finetune/Step Time": 4.397662304341793} +{"Finetune/Learning Rate": 1.9479887395490234e-05, "Finetune/Loss": 0.9947230815887451, "Finetune/Loss (Raw)": 1.0623019933700562, "Finetune/Step": 675, "Finetune/Step Time": 4.3999898955225945} +{"Finetune/Learning Rate": 1.9477902646997464e-05, "Finetune/Loss": 0.9953173398971558, "Finetune/Loss (Raw)": 1.0380116701126099, "Finetune/Step": 676, "Finetune/Step Time": 4.3944473043084145} +{"Finetune/Learning Rate": 1.947591422031572e-05, "Finetune/Loss": 0.9959168434143066, "Finetune/Loss (Raw)": 0.926249086856842, "Finetune/Step": 677, "Finetune/Step Time": 4.42676124162972} +{"Finetune/Learning Rate": 1.9473922116216664e-05, "Finetune/Loss": 0.9969786405563354, "Finetune/Loss (Raw)": 1.136008620262146, "Finetune/Step": 678, "Finetune/Step Time": 4.438916016370058} +{"Finetune/Learning Rate": 1.9471926335473396e-05, "Finetune/Loss": 0.9982626438140869, "Finetune/Loss (Raw)": 1.1472196578979492, "Finetune/Step": 679, "Finetune/Step Time": 4.444097265601158} +{"Finetune/Learning Rate": 1.9469926878860447e-05, "Finetune/Loss": 0.9972586631774902, "Finetune/Loss (Raw)": 0.9680576324462891, "Finetune/Step": 680, "Finetune/Step Time": 4.442138055339456} +{"Finetune/Learning Rate": 1.9467923747153756e-05, "Finetune/Loss": 0.9996966123580933, "Finetune/Loss (Raw)": 1.0760709047317505, "Finetune/Step": 681, "Finetune/Step Time": 4.440641740337014} +{"Finetune/Learning Rate": 1.9465916941130705e-05, "Finetune/Loss": 0.9992483258247375, "Finetune/Loss (Raw)": 1.0819761753082275, "Finetune/Step": 682, "Finetune/Step Time": 4.437891302630305} +{"Finetune/Learning Rate": 1.9463906461570098e-05, "Finetune/Loss": 1.0004912614822388, "Finetune/Loss (Raw)": 1.1108348369598389, "Finetune/Step": 683, "Finetune/Step Time": 4.419253142550588} +{"Finetune/Learning Rate": 1.9461892309252158e-05, "Finetune/Loss": 1.0000925064086914, "Finetune/Loss (Raw)": 0.9641566872596741, "Finetune/Step": 684, "Finetune/Step Time": 4.408599017187953} +{"Finetune/Learning Rate": 1.945987448495854e-05, "Finetune/Loss": 0.9985933303833008, "Finetune/Loss (Raw)": 0.7641931176185608, "Finetune/Step": 685, "Finetune/Step Time": 4.424109412357211} +{"Finetune/Learning Rate": 1.945785298947233e-05, "Finetune/Loss": 1.003936529159546, "Finetune/Loss (Raw)": 0.938077986240387, "Finetune/Step": 686, "Finetune/Step Time": 4.397914165630937} +{"Finetune/Learning Rate": 1.9455827823578017e-05, "Finetune/Loss": 1.0038855075836182, "Finetune/Loss (Raw)": 0.9729384183883667, "Finetune/Step": 687, "Finetune/Step Time": 4.394144540652633} +{"Finetune/Learning Rate": 1.9453798988061535e-05, "Finetune/Loss": 1.0052287578582764, "Finetune/Loss (Raw)": 1.0205202102661133, "Finetune/Step": 688, "Finetune/Step Time": 4.393116457387805} +{"Finetune/Learning Rate": 1.9451766483710236e-05, "Finetune/Loss": 1.0040459632873535, "Finetune/Loss (Raw)": 0.7352901697158813, "Finetune/Step": 689, "Finetune/Step Time": 4.406109856441617} +{"Finetune/Learning Rate": 1.94497303113129e-05, "Finetune/Loss": 1.0032581090927124, "Finetune/Loss (Raw)": 0.842150866985321, "Finetune/Step": 690, "Finetune/Step Time": 4.451886856928468} +{"Finetune/Learning Rate": 1.9447690471659715e-05, "Finetune/Loss": 1.0033349990844727, "Finetune/Loss (Raw)": 1.0608593225479126, "Finetune/Step": 691, "Finetune/Step Time": 4.453385295346379} +{"Finetune/Learning Rate": 1.944564696554231e-05, "Finetune/Loss": 1.0025581121444702, "Finetune/Loss (Raw)": 0.936629056930542, "Finetune/Step": 692, "Finetune/Step Time": 4.470855003222823} +{"Finetune/Learning Rate": 1.944359979375373e-05, "Finetune/Loss": 1.0027936697006226, "Finetune/Loss (Raw)": 0.9857311248779297, "Finetune/Step": 693, "Finetune/Step Time": 4.464118577539921} +{"Finetune/Learning Rate": 1.9441548957088438e-05, "Finetune/Loss": 0.9975452423095703, "Finetune/Loss (Raw)": 0.404956191778183, "Finetune/Step": 694, "Finetune/Step Time": 4.501078367233276} +{"Finetune/Learning Rate": 1.9439494456342327e-05, "Finetune/Loss": 0.9965176582336426, "Finetune/Loss (Raw)": 0.8705959320068359, "Finetune/Step": 695, "Finetune/Step Time": 4.507131267338991} +{"Finetune/Learning Rate": 1.943743629231271e-05, "Finetune/Loss": 1.0029385089874268, "Finetune/Loss (Raw)": 1.114133596420288, "Finetune/Step": 696, "Finetune/Step Time": 4.479405287653208} +{"Finetune/Learning Rate": 1.9435374465798318e-05, "Finetune/Loss": 1.0030794143676758, "Finetune/Loss (Raw)": 0.9452455639839172, "Finetune/Step": 697, "Finetune/Step Time": 4.480706784874201} +{"Finetune/Learning Rate": 1.9433308977599306e-05, "Finetune/Loss": 1.0044519901275635, "Finetune/Loss (Raw)": 1.192657709121704, "Finetune/Step": 698, "Finetune/Step Time": 4.473729815334082} +{"Finetune/Learning Rate": 1.943123982851725e-05, "Finetune/Loss": 1.0064817667007446, "Finetune/Loss (Raw)": 1.2093206644058228, "Finetune/Step": 699, "Finetune/Step Time": 4.476330226287246} +{"Finetune/Learning Rate": 1.9429167019355143e-05, "Finetune/Loss": 1.0060949325561523, "Finetune/Loss (Raw)": 0.9934167861938477, "Finetune/Step": 700, "Finetune/Step Time": 4.496045785024762} +{"Finetune/Learning Rate": 1.9427090550917407e-05, "Finetune/Loss": 1.0001952648162842, "Finetune/Loss (Raw)": 0.3158552050590515, "Finetune/Step": 701, "Finetune/Step Time": 4.529729159548879} +{"Finetune/Learning Rate": 1.9425010424009876e-05, "Finetune/Loss": 0.9992319941520691, "Finetune/Loss (Raw)": 0.9740740060806274, "Finetune/Step": 702, "Finetune/Step Time": 4.536775277927518} +{"Finetune/Learning Rate": 1.9422926639439806e-05, "Finetune/Loss": 0.9971978664398193, "Finetune/Loss (Raw)": 0.8773877024650574, "Finetune/Step": 703, "Finetune/Step Time": 4.522489992901683} +{"Finetune/Learning Rate": 1.9420839198015878e-05, "Finetune/Loss": 0.9972862005233765, "Finetune/Loss (Raw)": 0.9950199723243713, "Finetune/Step": 704, "Finetune/Step Time": 4.529771761968732} +{"Finetune/Learning Rate": 1.941874810054818e-05, "Finetune/Loss": 0.9975076913833618, "Finetune/Loss (Raw)": 1.0192749500274658, "Finetune/Step": 705, "Finetune/Step Time": 4.528981424868107} +{"Finetune/Learning Rate": 1.941665334784823e-05, "Finetune/Loss": 0.9961949586868286, "Finetune/Loss (Raw)": 0.7881529331207275, "Finetune/Step": 706, "Finetune/Step Time": 4.537094524130225} +{"Finetune/Learning Rate": 1.9414554940728963e-05, "Finetune/Loss": 0.9965914487838745, "Finetune/Loss (Raw)": 0.9049350619316101, "Finetune/Step": 707, "Finetune/Step Time": 4.5108382273465395} +{"Finetune/Learning Rate": 1.9412452880004728e-05, "Finetune/Loss": 0.997536301612854, "Finetune/Loss (Raw)": 1.1641020774841309, "Finetune/Step": 708, "Finetune/Step Time": 4.515385681763291} +{"Finetune/Learning Rate": 1.941034716649129e-05, "Finetune/Loss": 0.9962669610977173, "Finetune/Loss (Raw)": 0.9748410582542419, "Finetune/Step": 709, "Finetune/Step Time": 4.516551969572902} +{"Finetune/Learning Rate": 1.9408237801005837e-05, "Finetune/Loss": 0.9949765205383301, "Finetune/Loss (Raw)": 0.8833694458007812, "Finetune/Step": 710, "Finetune/Step Time": 4.505505030974746} +{"Finetune/Learning Rate": 1.9406124784366976e-05, "Finetune/Loss": 0.9963972568511963, "Finetune/Loss (Raw)": 1.0813261270523071, "Finetune/Step": 711, "Finetune/Step Time": 4.518601732328534} +{"Finetune/Learning Rate": 1.9404008117394726e-05, "Finetune/Loss": 0.9935510158538818, "Finetune/Loss (Raw)": 0.7191334366798401, "Finetune/Step": 712, "Finetune/Step Time": 4.5287393145263195} +{"Finetune/Learning Rate": 1.9401887800910525e-05, "Finetune/Loss": 0.992734432220459, "Finetune/Loss (Raw)": 0.9759920835494995, "Finetune/Step": 713, "Finetune/Step Time": 4.529131457209587} +{"Finetune/Learning Rate": 1.939976383573722e-05, "Finetune/Loss": 0.9973586797714233, "Finetune/Loss (Raw)": 0.938531756401062, "Finetune/Step": 714, "Finetune/Step Time": 4.501337107270956} +{"Finetune/Learning Rate": 1.9397636222699088e-05, "Finetune/Loss": 0.9968205094337463, "Finetune/Loss (Raw)": 1.1223634481430054, "Finetune/Step": 715, "Finetune/Step Time": 4.51158794015646} +{"Finetune/Learning Rate": 1.939550496262181e-05, "Finetune/Loss": 0.9958218932151794, "Finetune/Loss (Raw)": 0.867122232913971, "Finetune/Step": 716, "Finetune/Step Time": 4.485983315855265} +{"Finetune/Learning Rate": 1.939337005633249e-05, "Finetune/Loss": 0.9962103366851807, "Finetune/Loss (Raw)": 1.137872338294983, "Finetune/Step": 717, "Finetune/Step Time": 4.498397000133991} +{"Finetune/Learning Rate": 1.939123150465964e-05, "Finetune/Loss": 0.9959526062011719, "Finetune/Loss (Raw)": 1.0597285032272339, "Finetune/Step": 718, "Finetune/Step Time": 4.502409202978015} +{"Finetune/Learning Rate": 1.938908930843319e-05, "Finetune/Loss": 0.9967141151428223, "Finetune/Loss (Raw)": 1.042786717414856, "Finetune/Step": 719, "Finetune/Step Time": 4.502862308174372} +{"Finetune/Learning Rate": 1.9386943468484487e-05, "Finetune/Loss": 0.9985814690589905, "Finetune/Loss (Raw)": 1.2155296802520752, "Finetune/Step": 720, "Finetune/Step Time": 4.504994807764888} +{"Finetune/Learning Rate": 1.938479398564629e-05, "Finetune/Loss": 0.9980573654174805, "Finetune/Loss (Raw)": 0.9302040934562683, "Finetune/Step": 721, "Finetune/Step Time": 4.508707964792848} +{"Finetune/Learning Rate": 1.938264086075277e-05, "Finetune/Loss": 0.9979188442230225, "Finetune/Loss (Raw)": 0.9948296546936035, "Finetune/Step": 722, "Finetune/Step Time": 4.534884063526988} +{"Finetune/Learning Rate": 1.938048409463951e-05, "Finetune/Loss": 0.9987617135047913, "Finetune/Loss (Raw)": 1.1184438467025757, "Finetune/Step": 723, "Finetune/Step Time": 4.531886486336589} +{"Finetune/Learning Rate": 1.9378323688143515e-05, "Finetune/Loss": 0.9999024271965027, "Finetune/Loss (Raw)": 1.0838861465454102, "Finetune/Step": 724, "Finetune/Step Time": 4.547386171296239} +{"Finetune/Learning Rate": 1.937615964210319e-05, "Finetune/Loss": 1.0019733905792236, "Finetune/Loss (Raw)": 1.1460219621658325, "Finetune/Step": 725, "Finetune/Step Time": 4.546030657365918} +{"Finetune/Learning Rate": 1.9373991957358362e-05, "Finetune/Loss": 1.0001007318496704, "Finetune/Loss (Raw)": 0.7652730941772461, "Finetune/Step": 726, "Finetune/Step Time": 4.533899454399943} +{"Finetune/Learning Rate": 1.9371820634750266e-05, "Finetune/Loss": 1.0001533031463623, "Finetune/Loss (Raw)": 1.0308904647827148, "Finetune/Step": 727, "Finetune/Step Time": 4.540721969678998} +{"Finetune/Learning Rate": 1.9369645675121554e-05, "Finetune/Loss": 1.0008893013000488, "Finetune/Loss (Raw)": 1.2024372816085815, "Finetune/Step": 728, "Finetune/Step Time": 4.539993602782488} +{"Finetune/Learning Rate": 1.936746707931628e-05, "Finetune/Loss": 0.9983295202255249, "Finetune/Loss (Raw)": 0.6523666977882385, "Finetune/Step": 729, "Finetune/Step Time": 4.542974920943379} +{"Finetune/Learning Rate": 1.9365284848179917e-05, "Finetune/Loss": 0.9978392124176025, "Finetune/Loss (Raw)": 1.0074074268341064, "Finetune/Step": 730, "Finetune/Step Time": 4.546612340956926} +{"Finetune/Learning Rate": 1.9363098982559346e-05, "Finetune/Loss": 0.9990639090538025, "Finetune/Loss (Raw)": 1.091434359550476, "Finetune/Step": 731, "Finetune/Step Time": 4.542493345215917} +{"Finetune/Learning Rate": 1.9360909483302862e-05, "Finetune/Loss": 0.9978345632553101, "Finetune/Loss (Raw)": 0.9105885624885559, "Finetune/Step": 732, "Finetune/Step Time": 4.574559550732374} +{"Finetune/Learning Rate": 1.9358716351260164e-05, "Finetune/Loss": 0.9979242086410522, "Finetune/Loss (Raw)": 0.9593400359153748, "Finetune/Step": 733, "Finetune/Step Time": 4.5725321639329195} +{"Finetune/Learning Rate": 1.935651958728236e-05, "Finetune/Loss": 0.9974939227104187, "Finetune/Loss (Raw)": 0.8726421594619751, "Finetune/Step": 734, "Finetune/Step Time": 4.560536693781614} +{"Finetune/Learning Rate": 1.9354319192221975e-05, "Finetune/Loss": 0.9980779886245728, "Finetune/Loss (Raw)": 0.9036354422569275, "Finetune/Step": 735, "Finetune/Step Time": 4.5380418840795755} +{"Finetune/Learning Rate": 1.9352115166932943e-05, "Finetune/Loss": 0.9967997670173645, "Finetune/Loss (Raw)": 0.9320428371429443, "Finetune/Step": 736, "Finetune/Step Time": 4.574213724583387} +{"Finetune/Learning Rate": 1.9349907512270597e-05, "Finetune/Loss": 0.9983845949172974, "Finetune/Loss (Raw)": 1.120646595954895, "Finetune/Step": 737, "Finetune/Step Time": 4.570038363337517} +{"Finetune/Learning Rate": 1.9347696229091685e-05, "Finetune/Loss": 0.9987646341323853, "Finetune/Loss (Raw)": 1.021155595779419, "Finetune/Step": 738, "Finetune/Step Time": 4.566950326785445} +{"Finetune/Learning Rate": 1.934548131825437e-05, "Finetune/Loss": 0.9942655563354492, "Finetune/Loss (Raw)": 0.46226295828819275, "Finetune/Step": 739, "Finetune/Step Time": 4.611022993922234} +{"Finetune/Learning Rate": 1.9343262780618206e-05, "Finetune/Loss": 0.9937760829925537, "Finetune/Loss (Raw)": 1.148041844367981, "Finetune/Step": 740, "Finetune/Step Time": 4.613491173833609} +{"Finetune/Learning Rate": 1.934104061704417e-05, "Finetune/Loss": 0.9930931925773621, "Finetune/Loss (Raw)": 0.8161176443099976, "Finetune/Step": 741, "Finetune/Step Time": 4.631830545142293} +{"Finetune/Learning Rate": 1.9338814828394638e-05, "Finetune/Loss": 0.9931066036224365, "Finetune/Loss (Raw)": 0.984449028968811, "Finetune/Step": 742, "Finetune/Step Time": 4.637230241671205} +{"Finetune/Learning Rate": 1.9336585415533398e-05, "Finetune/Loss": 0.993391752243042, "Finetune/Loss (Raw)": 1.0723797082901, "Finetune/Step": 743, "Finetune/Step Time": 4.631017494946718} +{"Finetune/Learning Rate": 1.9334352379325638e-05, "Finetune/Loss": 0.9933702945709229, "Finetune/Loss (Raw)": 0.9993323683738708, "Finetune/Step": 744, "Finetune/Step Time": 4.632523477077484} +{"Finetune/Learning Rate": 1.9332115720637957e-05, "Finetune/Loss": 0.9925379157066345, "Finetune/Loss (Raw)": 0.8938645720481873, "Finetune/Step": 745, "Finetune/Step Time": 4.650762900710106} +{"Finetune/Learning Rate": 1.9329875440338363e-05, "Finetune/Loss": 0.9929981231689453, "Finetune/Loss (Raw)": 1.1017922163009644, "Finetune/Step": 746, "Finetune/Step Time": 4.641184985637665} +{"Finetune/Learning Rate": 1.9327631539296258e-05, "Finetune/Loss": 0.9945694208145142, "Finetune/Loss (Raw)": 1.1247100830078125, "Finetune/Step": 747, "Finetune/Step Time": 4.620938515290618} +{"Finetune/Learning Rate": 1.932538401838246e-05, "Finetune/Loss": 0.9950343370437622, "Finetune/Loss (Raw)": 1.099280595779419, "Finetune/Step": 748, "Finetune/Step Time": 4.622789673507214} +{"Finetune/Learning Rate": 1.9323132878469185e-05, "Finetune/Loss": 0.994914174079895, "Finetune/Loss (Raw)": 1.0068899393081665, "Finetune/Step": 749, "Finetune/Step Time": 4.611900120973587} +{"Finetune/Learning Rate": 1.9320878120430063e-05, "Finetune/Loss": 0.9953582286834717, "Finetune/Loss (Raw)": 1.0286924839019775, "Finetune/Step": 750, "Finetune/Step Time": 4.617474535480142} +{"Finetune/Learning Rate": 1.9318619745140115e-05, "Finetune/Loss": 0.993928074836731, "Finetune/Loss (Raw)": 0.8927977085113525, "Finetune/Step": 751, "Finetune/Step Time": 4.619943380355835} +{"Finetune/Learning Rate": 1.9316357753475776e-05, "Finetune/Loss": 0.9945614337921143, "Finetune/Loss (Raw)": 1.0800931453704834, "Finetune/Step": 752, "Finetune/Step Time": 4.621038632467389} +{"Finetune/Learning Rate": 1.9314092146314872e-05, "Finetune/Loss": 0.9894330501556396, "Finetune/Loss (Raw)": 0.39844173192977905, "Finetune/Step": 753, "Finetune/Step Time": 4.657275274395943} +{"Finetune/Learning Rate": 1.9311822924536656e-05, "Finetune/Loss": 0.9879484176635742, "Finetune/Loss (Raw)": 0.8781494498252869, "Finetune/Step": 754, "Finetune/Step Time": 4.65757642686367} +{"Finetune/Learning Rate": 1.930955008902176e-05, "Finetune/Loss": 0.9876279830932617, "Finetune/Loss (Raw)": 0.9364017248153687, "Finetune/Step": 755, "Finetune/Step Time": 4.679290579631925} +{"Finetune/Learning Rate": 1.9307273640652228e-05, "Finetune/Loss": 0.9872761964797974, "Finetune/Loss (Raw)": 0.9510564804077148, "Finetune/Step": 756, "Finetune/Step Time": 4.670618524774909} +{"Finetune/Learning Rate": 1.9304993580311505e-05, "Finetune/Loss": 0.9869489669799805, "Finetune/Loss (Raw)": 0.9909654855728149, "Finetune/Step": 757, "Finetune/Step Time": 4.682444518432021} +{"Finetune/Learning Rate": 1.9302709908884437e-05, "Finetune/Loss": 0.9870806336402893, "Finetune/Loss (Raw)": 1.1562451124191284, "Finetune/Step": 758, "Finetune/Step Time": 4.6908442210406065} +{"Finetune/Learning Rate": 1.9300422627257275e-05, "Finetune/Loss": 0.9882909655570984, "Finetune/Loss (Raw)": 1.068065881729126, "Finetune/Step": 759, "Finetune/Step Time": 4.690368579700589} +{"Finetune/Learning Rate": 1.9298131736317666e-05, "Finetune/Loss": 0.9871480464935303, "Finetune/Loss (Raw)": 1.0256569385528564, "Finetune/Step": 760, "Finetune/Step Time": 4.682742917910218} +{"Finetune/Learning Rate": 1.929583723695466e-05, "Finetune/Loss": 0.9845754504203796, "Finetune/Loss (Raw)": 0.9369845986366272, "Finetune/Step": 761, "Finetune/Step Time": 4.679667359218001} +{"Finetune/Learning Rate": 1.9293539130058708e-05, "Finetune/Loss": 0.9801914691925049, "Finetune/Loss (Raw)": 0.3836979269981384, "Finetune/Step": 762, "Finetune/Step Time": 4.728534048423171} +{"Finetune/Learning Rate": 1.9291237416521662e-05, "Finetune/Loss": 0.9793509244918823, "Finetune/Loss (Raw)": 0.8985919952392578, "Finetune/Step": 763, "Finetune/Step Time": 4.7321263533085585} +{"Finetune/Learning Rate": 1.9288932097236772e-05, "Finetune/Loss": 0.9787258505821228, "Finetune/Loss (Raw)": 1.0051689147949219, "Finetune/Step": 764, "Finetune/Step Time": 4.709406735375524} +{"Finetune/Learning Rate": 1.9286623173098683e-05, "Finetune/Loss": 0.9777960181236267, "Finetune/Loss (Raw)": 0.9818262457847595, "Finetune/Step": 765, "Finetune/Step Time": 4.713397866114974} +{"Finetune/Learning Rate": 1.928431064500345e-05, "Finetune/Loss": 0.9777891039848328, "Finetune/Loss (Raw)": 1.1004356145858765, "Finetune/Step": 766, "Finetune/Step Time": 4.72587595321238} +{"Finetune/Learning Rate": 1.9281994513848515e-05, "Finetune/Loss": 0.9770338535308838, "Finetune/Loss (Raw)": 0.9235313534736633, "Finetune/Step": 767, "Finetune/Step Time": 4.734062395989895} +{"Finetune/Learning Rate": 1.9279674780532726e-05, "Finetune/Loss": 0.9764557480812073, "Finetune/Loss (Raw)": 0.9320335984230042, "Finetune/Step": 768, "Finetune/Step Time": 4.743575796484947} +{"Finetune/Learning Rate": 1.9277351445956323e-05, "Finetune/Loss": 0.9743435382843018, "Finetune/Loss (Raw)": 0.8276388645172119, "Finetune/Step": 769, "Finetune/Step Time": 4.755873614922166} +{"Finetune/Learning Rate": 1.927502451102095e-05, "Finetune/Loss": 0.9797594547271729, "Finetune/Loss (Raw)": 1.0324532985687256, "Finetune/Step": 770, "Finetune/Step Time": 4.735765540972352} +{"Finetune/Learning Rate": 1.9272693976629642e-05, "Finetune/Loss": 0.9803591966629028, "Finetune/Loss (Raw)": 1.0793424844741821, "Finetune/Step": 771, "Finetune/Step Time": 4.740178050473332} +{"Finetune/Learning Rate": 1.9270359843686837e-05, "Finetune/Loss": 0.9799912571907043, "Finetune/Loss (Raw)": 1.0019580125808716, "Finetune/Step": 772, "Finetune/Step Time": 4.733480226248503} +{"Finetune/Learning Rate": 1.9268022113098366e-05, "Finetune/Loss": 0.9798572063446045, "Finetune/Loss (Raw)": 1.0663889646530151, "Finetune/Step": 773, "Finetune/Step Time": 4.732340935617685} +{"Finetune/Learning Rate": 1.9265680785771455e-05, "Finetune/Loss": 0.9787178039550781, "Finetune/Loss (Raw)": 0.8754473924636841, "Finetune/Step": 774, "Finetune/Step Time": 4.737786129117012} +{"Finetune/Learning Rate": 1.926333586261473e-05, "Finetune/Loss": 0.979417085647583, "Finetune/Loss (Raw)": 1.036415934562683, "Finetune/Step": 775, "Finetune/Step Time": 4.74009745195508} +{"Finetune/Learning Rate": 1.9260987344538205e-05, "Finetune/Loss": 0.9775831699371338, "Finetune/Loss (Raw)": 0.8008619546890259, "Finetune/Step": 776, "Finetune/Step Time": 4.736136168241501} +{"Finetune/Learning Rate": 1.9258635232453296e-05, "Finetune/Loss": 0.9784258604049683, "Finetune/Loss (Raw)": 1.1428966522216797, "Finetune/Step": 777, "Finetune/Step Time": 4.752709768712521} +{"Finetune/Learning Rate": 1.9256279527272815e-05, "Finetune/Loss": 0.9792157411575317, "Finetune/Loss (Raw)": 1.0950942039489746, "Finetune/Step": 778, "Finetune/Step Time": 4.737661210820079} +{"Finetune/Learning Rate": 1.9253920229910966e-05, "Finetune/Loss": 0.9784713983535767, "Finetune/Loss (Raw)": 0.9368070363998413, "Finetune/Step": 779, "Finetune/Step Time": 4.734608395025134} +{"Finetune/Learning Rate": 1.925155734128334e-05, "Finetune/Loss": 0.9732849597930908, "Finetune/Loss (Raw)": 0.34014692902565, "Finetune/Step": 780, "Finetune/Step Time": 4.772966753691435} +{"Finetune/Learning Rate": 1.9249190862306935e-05, "Finetune/Loss": 0.9734328985214233, "Finetune/Loss (Raw)": 0.9372276663780212, "Finetune/Step": 781, "Finetune/Step Time": 4.767503350973129} +{"Finetune/Learning Rate": 1.9246820793900127e-05, "Finetune/Loss": 0.9697519540786743, "Finetune/Loss (Raw)": 0.7388471961021423, "Finetune/Step": 782, "Finetune/Step Time": 4.797806803137064} +{"Finetune/Learning Rate": 1.92444471369827e-05, "Finetune/Loss": 0.9692652225494385, "Finetune/Loss (Raw)": 0.9275618195533752, "Finetune/Step": 783, "Finetune/Step Time": 4.841664040461183} +{"Finetune/Learning Rate": 1.924206989247582e-05, "Finetune/Loss": 0.9675306081771851, "Finetune/Loss (Raw)": 0.7536432147026062, "Finetune/Step": 784, "Finetune/Step Time": 4.858523063361645} +{"Finetune/Learning Rate": 1.9239689061302053e-05, "Finetune/Loss": 0.9684462547302246, "Finetune/Loss (Raw)": 1.080287218093872, "Finetune/Step": 785, "Finetune/Step Time": 4.853595122694969} +{"Finetune/Learning Rate": 1.923730464438535e-05, "Finetune/Loss": 0.9696660041809082, "Finetune/Loss (Raw)": 1.141226053237915, "Finetune/Step": 786, "Finetune/Step Time": 4.856528364121914} +{"Finetune/Learning Rate": 1.923491664265106e-05, "Finetune/Loss": 0.9721421003341675, "Finetune/Loss (Raw)": 1.1184661388397217, "Finetune/Step": 787, "Finetune/Step Time": 4.865241980180144} +{"Finetune/Learning Rate": 1.9232525057025915e-05, "Finetune/Loss": 0.9721455574035645, "Finetune/Loss (Raw)": 1.1321805715560913, "Finetune/Step": 788, "Finetune/Step Time": 4.861873347312212} +{"Finetune/Learning Rate": 1.9230129888438046e-05, "Finetune/Loss": 0.9720734357833862, "Finetune/Loss (Raw)": 1.0077826976776123, "Finetune/Step": 789, "Finetune/Step Time": 4.8605641555041075} +{"Finetune/Learning Rate": 1.922773113781697e-05, "Finetune/Loss": 0.9697917699813843, "Finetune/Loss (Raw)": 0.7696465849876404, "Finetune/Step": 790, "Finetune/Step Time": 4.890470603480935} +{"Finetune/Learning Rate": 1.9225328806093596e-05, "Finetune/Loss": 0.9684288501739502, "Finetune/Loss (Raw)": 0.9402334094047546, "Finetune/Step": 791, "Finetune/Step Time": 4.891767514869571} +{"Finetune/Learning Rate": 1.922292289420022e-05, "Finetune/Loss": 0.9670596122741699, "Finetune/Loss (Raw)": 0.849062442779541, "Finetune/Step": 792, "Finetune/Step Time": 4.882408240810037} +{"Finetune/Learning Rate": 1.9220513403070534e-05, "Finetune/Loss": 0.9665302038192749, "Finetune/Loss (Raw)": 1.0863348245620728, "Finetune/Step": 793, "Finetune/Step Time": 4.885041758418083} +{"Finetune/Learning Rate": 1.9218100333639608e-05, "Finetune/Loss": 0.9646738767623901, "Finetune/Loss (Raw)": 0.9433975219726562, "Finetune/Step": 794, "Finetune/Step Time": 4.887604543939233} +{"Finetune/Learning Rate": 1.9215683686843912e-05, "Finetune/Loss": 0.963585615158081, "Finetune/Loss (Raw)": 0.9849685430526733, "Finetune/Step": 795, "Finetune/Step Time": 4.890480035915971} +{"Finetune/Learning Rate": 1.92132634636213e-05, "Finetune/Loss": 0.9642713665962219, "Finetune/Loss (Raw)": 0.9551401734352112, "Finetune/Step": 796, "Finetune/Step Time": 4.886850468814373} +{"Finetune/Learning Rate": 1.9210839664911013e-05, "Finetune/Loss": 0.9638272523880005, "Finetune/Loss (Raw)": 0.9810922741889954, "Finetune/Step": 797, "Finetune/Step Time": 4.893553838133812} +{"Finetune/Learning Rate": 1.9208412291653674e-05, "Finetune/Loss": 0.9646108150482178, "Finetune/Loss (Raw)": 1.0180083513259888, "Finetune/Step": 798, "Finetune/Step Time": 4.895978659391403} +{"Finetune/Learning Rate": 1.9205981344791307e-05, "Finetune/Loss": 0.9639672636985779, "Finetune/Loss (Raw)": 0.9089106321334839, "Finetune/Step": 799, "Finetune/Step Time": 4.903132066130638} +{"Finetune/Learning Rate": 1.9203546825267313e-05, "Finetune/Loss": 0.9639638662338257, "Finetune/Loss (Raw)": 1.059409737586975, "Finetune/Step": 800, "Finetune/Step Time": 4.909055618569255} +{"Finetune/Learning Rate": 1.9201108734026477e-05, "Finetune/Loss": 0.96361243724823, "Finetune/Loss (Raw)": 1.075844407081604, "Finetune/Step": 801, "Finetune/Step Time": 4.93362326361239} +{"Finetune/Learning Rate": 1.9198667072014978e-05, "Finetune/Loss": 0.9626064300537109, "Finetune/Loss (Raw)": 0.9926957488059998, "Finetune/Step": 802, "Finetune/Step Time": 4.9373122099787} +{"Finetune/Learning Rate": 1.9196221840180378e-05, "Finetune/Loss": 0.9606053233146667, "Finetune/Loss (Raw)": 0.8061581254005432, "Finetune/Step": 803, "Finetune/Step Time": 4.942902859300375} +{"Finetune/Learning Rate": 1.9193773039471622e-05, "Finetune/Loss": 0.9602346420288086, "Finetune/Loss (Raw)": 0.9905654191970825, "Finetune/Step": 804, "Finetune/Step Time": 4.933704296126962} +{"Finetune/Learning Rate": 1.9191320670839047e-05, "Finetune/Loss": 0.9611274003982544, "Finetune/Loss (Raw)": 1.0405211448669434, "Finetune/Step": 805, "Finetune/Step Time": 4.903907582163811} +{"Finetune/Learning Rate": 1.918886473523436e-05, "Finetune/Loss": 0.958833634853363, "Finetune/Loss (Raw)": 0.8424050807952881, "Finetune/Step": 806, "Finetune/Step Time": 4.917920170351863} +{"Finetune/Learning Rate": 1.918640523361067e-05, "Finetune/Loss": 0.9576135873794556, "Finetune/Loss (Raw)": 0.9910557866096497, "Finetune/Step": 807, "Finetune/Step Time": 4.909569639712572} +{"Finetune/Learning Rate": 1.9183942166922455e-05, "Finetune/Loss": 0.9582130908966064, "Finetune/Loss (Raw)": 1.0447944402694702, "Finetune/Step": 808, "Finetune/Step Time": 4.9292375314980745} +{"Finetune/Learning Rate": 1.9181475536125588e-05, "Finetune/Loss": 0.9572716355323792, "Finetune/Loss (Raw)": 0.955565333366394, "Finetune/Step": 809, "Finetune/Step Time": 4.933067407459021} +{"Finetune/Learning Rate": 1.917900534217732e-05, "Finetune/Loss": 0.9573779106140137, "Finetune/Loss (Raw)": 1.0955760478973389, "Finetune/Step": 810, "Finetune/Step Time": 4.936132576316595} +{"Finetune/Learning Rate": 1.9176531586036282e-05, "Finetune/Loss": 0.9553297758102417, "Finetune/Loss (Raw)": 0.8486757278442383, "Finetune/Step": 811, "Finetune/Step Time": 4.9333116970956326} +{"Finetune/Learning Rate": 1.917405426866249e-05, "Finetune/Loss": 0.957383394241333, "Finetune/Loss (Raw)": 1.2270132303237915, "Finetune/Step": 812, "Finetune/Step Time": 4.9300885535776615} +{"Finetune/Learning Rate": 1.9171573391017348e-05, "Finetune/Loss": 0.959128737449646, "Finetune/Loss (Raw)": 0.9876052737236023, "Finetune/Step": 813, "Finetune/Step Time": 4.916951386258006} +{"Finetune/Learning Rate": 1.9169088954063634e-05, "Finetune/Loss": 0.9593390226364136, "Finetune/Loss (Raw)": 0.9649907350540161, "Finetune/Step": 814, "Finetune/Step Time": 4.906298719346523} +{"Finetune/Learning Rate": 1.9166600958765508e-05, "Finetune/Loss": 0.9593961238861084, "Finetune/Loss (Raw)": 0.9802477955818176, "Finetune/Step": 815, "Finetune/Step Time": 4.910700149834156} +{"Finetune/Learning Rate": 1.916410940608851e-05, "Finetune/Loss": 0.9597911238670349, "Finetune/Loss (Raw)": 1.0710790157318115, "Finetune/Step": 816, "Finetune/Step Time": 4.910430325195193} +{"Finetune/Learning Rate": 1.9161614296999565e-05, "Finetune/Loss": 0.9624894857406616, "Finetune/Loss (Raw)": 1.0806832313537598, "Finetune/Step": 817, "Finetune/Step Time": 4.899884028360248} +{"Finetune/Learning Rate": 1.9159115632466978e-05, "Finetune/Loss": 0.9585083723068237, "Finetune/Loss (Raw)": 0.332567036151886, "Finetune/Step": 818, "Finetune/Step Time": 4.888437131419778} +{"Finetune/Learning Rate": 1.9156613413460432e-05, "Finetune/Loss": 0.9566764831542969, "Finetune/Loss (Raw)": 0.8263747692108154, "Finetune/Step": 819, "Finetune/Step Time": 4.888710146769881} +{"Finetune/Learning Rate": 1.9154107640950986e-05, "Finetune/Loss": 0.9551507830619812, "Finetune/Loss (Raw)": 0.7413424253463745, "Finetune/Step": 820, "Finetune/Step Time": 4.869401564821601} +{"Finetune/Learning Rate": 1.9151598315911088e-05, "Finetune/Loss": 0.9552675485610962, "Finetune/Loss (Raw)": 1.0006771087646484, "Finetune/Step": 821, "Finetune/Step Time": 4.87527104280889} +{"Finetune/Learning Rate": 1.9149085439314548e-05, "Finetune/Loss": 0.9594155550003052, "Finetune/Loss (Raw)": 0.9359019994735718, "Finetune/Step": 822, "Finetune/Step Time": 4.841137206181884} +{"Finetune/Learning Rate": 1.9146569012136572e-05, "Finetune/Loss": 0.961185872554779, "Finetune/Loss (Raw)": 1.0971934795379639, "Finetune/Step": 823, "Finetune/Step Time": 4.827193694189191} +{"Finetune/Learning Rate": 1.914404903535373e-05, "Finetune/Loss": 0.960931658744812, "Finetune/Loss (Raw)": 1.0815991163253784, "Finetune/Step": 824, "Finetune/Step Time": 4.82038339972496} +{"Finetune/Learning Rate": 1.9141525509943987e-05, "Finetune/Loss": 0.9607001543045044, "Finetune/Loss (Raw)": 0.915614664554596, "Finetune/Step": 825, "Finetune/Step Time": 4.810917412862182} +{"Finetune/Learning Rate": 1.9138998436886664e-05, "Finetune/Loss": 0.9580754041671753, "Finetune/Loss (Raw)": 0.8566825985908508, "Finetune/Step": 826, "Finetune/Step Time": 4.816911099478602} +{"Finetune/Learning Rate": 1.913646781716247e-05, "Finetune/Loss": 0.9564666748046875, "Finetune/Loss (Raw)": 1.003404140472412, "Finetune/Step": 827, "Finetune/Step Time": 4.803340181708336} +{"Finetune/Learning Rate": 1.9133933651753493e-05, "Finetune/Loss": 0.9517014026641846, "Finetune/Loss (Raw)": 0.3834601938724518, "Finetune/Step": 828, "Finetune/Step Time": 4.822219470515847} +{"Finetune/Learning Rate": 1.913139594164319e-05, "Finetune/Loss": 0.9576435685157776, "Finetune/Loss (Raw)": 1.0764601230621338, "Finetune/Step": 829, "Finetune/Step Time": 4.791158355772495} +{"Finetune/Learning Rate": 1.9128854687816394e-05, "Finetune/Loss": 0.9578217267990112, "Finetune/Loss (Raw)": 0.9968844056129456, "Finetune/Step": 830, "Finetune/Step Time": 4.788216350600123} +{"Finetune/Learning Rate": 1.9126309891259324e-05, "Finetune/Loss": 0.9591432213783264, "Finetune/Loss (Raw)": 1.0465328693389893, "Finetune/Step": 831, "Finetune/Step Time": 4.799113590270281} +{"Finetune/Learning Rate": 1.9123761552959562e-05, "Finetune/Loss": 0.9598888158798218, "Finetune/Loss (Raw)": 1.090450644493103, "Finetune/Step": 832, "Finetune/Step Time": 4.775095783174038} +{"Finetune/Learning Rate": 1.9121209673906064e-05, "Finetune/Loss": 0.9552720785140991, "Finetune/Loss (Raw)": 0.4283379912376404, "Finetune/Step": 833, "Finetune/Step Time": 4.814905496314168} +{"Finetune/Learning Rate": 1.911865425508917e-05, "Finetune/Loss": 0.9568943977355957, "Finetune/Loss (Raw)": 0.9958074688911438, "Finetune/Step": 834, "Finetune/Step Time": 4.817379608750343} +{"Finetune/Learning Rate": 1.9116095297500588e-05, "Finetune/Loss": 0.957543671131134, "Finetune/Loss (Raw)": 0.9880418181419373, "Finetune/Step": 835, "Finetune/Step Time": 4.8226519878953695} +{"Finetune/Learning Rate": 1.9113532802133394e-05, "Finetune/Loss": 0.9563702940940857, "Finetune/Loss (Raw)": 1.013907551765442, "Finetune/Step": 836, "Finetune/Step Time": 4.823449995368719} +{"Finetune/Learning Rate": 1.911096676998205e-05, "Finetune/Loss": 0.9560462236404419, "Finetune/Loss (Raw)": 0.9333606362342834, "Finetune/Step": 837, "Finetune/Step Time": 4.832246020436287} +{"Finetune/Learning Rate": 1.9108397202042376e-05, "Finetune/Loss": 0.9574346542358398, "Finetune/Loss (Raw)": 1.0610922574996948, "Finetune/Step": 838, "Finetune/Step Time": 4.841866670176387} +{"Finetune/Learning Rate": 1.910582409931158e-05, "Finetune/Loss": 0.9562352895736694, "Finetune/Loss (Raw)": 0.9278024435043335, "Finetune/Step": 839, "Finetune/Step Time": 4.833657359704375} +{"Finetune/Learning Rate": 1.910324746278822e-05, "Finetune/Loss": 0.9596916437149048, "Finetune/Loss (Raw)": 1.161543369293213, "Finetune/Step": 840, "Finetune/Step Time": 4.822391981258988} +{"Finetune/Learning Rate": 1.9100667293472252e-05, "Finetune/Loss": 0.9590566158294678, "Finetune/Loss (Raw)": 0.894720196723938, "Finetune/Step": 841, "Finetune/Step Time": 4.826084986329079} +{"Finetune/Learning Rate": 1.9098083592364976e-05, "Finetune/Loss": 0.9600616693496704, "Finetune/Loss (Raw)": 1.0671725273132324, "Finetune/Step": 842, "Finetune/Step Time": 4.820233143866062} +{"Finetune/Learning Rate": 1.9095496360469087e-05, "Finetune/Loss": 0.9594643115997314, "Finetune/Loss (Raw)": 1.0458972454071045, "Finetune/Step": 843, "Finetune/Step Time": 4.823072161525488} +{"Finetune/Learning Rate": 1.9092905598788633e-05, "Finetune/Loss": 0.9604860544204712, "Finetune/Loss (Raw)": 0.9979097247123718, "Finetune/Step": 844, "Finetune/Step Time": 4.829971907660365} +{"Finetune/Learning Rate": 1.9090311308329045e-05, "Finetune/Loss": 0.960033655166626, "Finetune/Loss (Raw)": 1.0799612998962402, "Finetune/Step": 845, "Finetune/Step Time": 4.816140968352556} +{"Finetune/Learning Rate": 1.9087713490097108e-05, "Finetune/Loss": 0.9598835706710815, "Finetune/Loss (Raw)": 1.0405244827270508, "Finetune/Step": 846, "Finetune/Step Time": 4.808533266186714} +{"Finetune/Learning Rate": 1.9085112145100987e-05, "Finetune/Loss": 0.9553825259208679, "Finetune/Loss (Raw)": 0.4666517674922943, "Finetune/Step": 847, "Finetune/Step Time": 4.841849325224757} +{"Finetune/Learning Rate": 1.9082507274350216e-05, "Finetune/Loss": 0.9525281190872192, "Finetune/Loss (Raw)": 0.8501614332199097, "Finetune/Step": 848, "Finetune/Step Time": 4.850308595225215} +{"Finetune/Learning Rate": 1.9079898878855695e-05, "Finetune/Loss": 0.9531341791152954, "Finetune/Loss (Raw)": 1.0077855587005615, "Finetune/Step": 849, "Finetune/Step Time": 4.833585409447551} +{"Finetune/Learning Rate": 1.9077286959629684e-05, "Finetune/Loss": 0.9523288607597351, "Finetune/Loss (Raw)": 0.8917445540428162, "Finetune/Step": 850, "Finetune/Step Time": 4.8181140050292015} +{"Finetune/Learning Rate": 1.907467151768583e-05, "Finetune/Loss": 0.9515952467918396, "Finetune/Loss (Raw)": 1.0245434045791626, "Finetune/Step": 851, "Finetune/Step Time": 4.824721114709973} +{"Finetune/Learning Rate": 1.9072052554039123e-05, "Finetune/Loss": 0.9500239491462708, "Finetune/Loss (Raw)": 0.8827608227729797, "Finetune/Step": 852, "Finetune/Step Time": 4.809289904311299} +{"Finetune/Learning Rate": 1.9069430069705936e-05, "Finetune/Loss": 0.9485273361206055, "Finetune/Loss (Raw)": 0.954454243183136, "Finetune/Step": 853, "Finetune/Step Time": 4.805998304858804} +{"Finetune/Learning Rate": 1.9066804065704006e-05, "Finetune/Loss": 0.9498997330665588, "Finetune/Loss (Raw)": 0.9409425258636475, "Finetune/Step": 854, "Finetune/Step Time": 4.808488242328167} +{"Finetune/Learning Rate": 1.9064174543052435e-05, "Finetune/Loss": 0.9481400847434998, "Finetune/Loss (Raw)": 0.8056511282920837, "Finetune/Step": 855, "Finetune/Step Time": 4.844404833391309} +{"Finetune/Learning Rate": 1.9061541502771682e-05, "Finetune/Loss": 0.9464834332466125, "Finetune/Loss (Raw)": 0.9903839826583862, "Finetune/Step": 856, "Finetune/Step Time": 4.8417848609387875} +{"Finetune/Learning Rate": 1.905890494588359e-05, "Finetune/Loss": 0.9488092660903931, "Finetune/Loss (Raw)": 0.9500810503959656, "Finetune/Step": 857, "Finetune/Step Time": 4.818493647500873} +{"Finetune/Learning Rate": 1.9056264873411343e-05, "Finetune/Loss": 0.9502230286598206, "Finetune/Loss (Raw)": 1.1883645057678223, "Finetune/Step": 858, "Finetune/Step Time": 4.816173166036606} +{"Finetune/Learning Rate": 1.905362128637951e-05, "Finetune/Loss": 0.9503161311149597, "Finetune/Loss (Raw)": 1.1033507585525513, "Finetune/Step": 859, "Finetune/Step Time": 4.8311113733798265} +{"Finetune/Learning Rate": 1.905097418581401e-05, "Finetune/Loss": 0.9506291151046753, "Finetune/Loss (Raw)": 0.9506466388702393, "Finetune/Step": 860, "Finetune/Step Time": 4.799599787220359} +{"Finetune/Learning Rate": 1.904832357274214e-05, "Finetune/Loss": 0.948796272277832, "Finetune/Loss (Raw)": 0.7247394919395447, "Finetune/Step": 861, "Finetune/Step Time": 4.789630673825741} +{"Finetune/Learning Rate": 1.9045669448192537e-05, "Finetune/Loss": 0.9499187469482422, "Finetune/Loss (Raw)": 1.016322374343872, "Finetune/Step": 862, "Finetune/Step Time": 4.792047282680869} +{"Finetune/Learning Rate": 1.904301181319523e-05, "Finetune/Loss": 0.9494199752807617, "Finetune/Loss (Raw)": 0.8397931456565857, "Finetune/Step": 863, "Finetune/Step Time": 4.797638250514865} +{"Finetune/Learning Rate": 1.9040350668781584e-05, "Finetune/Loss": 0.9502969980239868, "Finetune/Loss (Raw)": 1.0443024635314941, "Finetune/Step": 864, "Finetune/Step Time": 4.764588920399547} +{"Finetune/Learning Rate": 1.9037686015984343e-05, "Finetune/Loss": 0.9490423202514648, "Finetune/Loss (Raw)": 0.9600521326065063, "Finetune/Step": 865, "Finetune/Step Time": 4.78614517301321} +{"Finetune/Learning Rate": 1.9035017855837604e-05, "Finetune/Loss": 0.9498238563537598, "Finetune/Loss (Raw)": 1.1211884021759033, "Finetune/Step": 866, "Finetune/Step Time": 4.787315830588341} +{"Finetune/Learning Rate": 1.9032346189376828e-05, "Finetune/Loss": 0.9545639157295227, "Finetune/Loss (Raw)": 1.0689876079559326, "Finetune/Step": 867, "Finetune/Step Time": 4.746946232393384} +{"Finetune/Learning Rate": 1.902967101763884e-05, "Finetune/Loss": 0.9529720544815063, "Finetune/Loss (Raw)": 0.9442869424819946, "Finetune/Step": 868, "Finetune/Step Time": 4.739945713430643} +{"Finetune/Learning Rate": 1.902699234166182e-05, "Finetune/Loss": 0.9551218748092651, "Finetune/Loss (Raw)": 1.0912891626358032, "Finetune/Step": 869, "Finetune/Step Time": 4.730607025325298} +{"Finetune/Learning Rate": 1.9024310162485308e-05, "Finetune/Loss": 0.954470157623291, "Finetune/Loss (Raw)": 0.9010312557220459, "Finetune/Step": 870, "Finetune/Step Time": 4.733719687908888} +{"Finetune/Learning Rate": 1.9021624481150208e-05, "Finetune/Loss": 0.952756404876709, "Finetune/Loss (Raw)": 0.8530209064483643, "Finetune/Step": 871, "Finetune/Step Time": 4.7369988206774} +{"Finetune/Learning Rate": 1.9018935298698777e-05, "Finetune/Loss": 0.9527533054351807, "Finetune/Loss (Raw)": 0.9989365935325623, "Finetune/Step": 872, "Finetune/Step Time": 4.7368155140429735} +{"Finetune/Learning Rate": 1.9016242616174637e-05, "Finetune/Loss": 0.9539264440536499, "Finetune/Loss (Raw)": 1.0440198183059692, "Finetune/Step": 873, "Finetune/Step Time": 4.720273280516267} +{"Finetune/Learning Rate": 1.9013546434622766e-05, "Finetune/Loss": 0.9522280693054199, "Finetune/Loss (Raw)": 0.8844053149223328, "Finetune/Step": 874, "Finetune/Step Time": 4.725592909380794} +{"Finetune/Learning Rate": 1.90108467550895e-05, "Finetune/Loss": 0.9508768320083618, "Finetune/Loss (Raw)": 0.9517536163330078, "Finetune/Step": 875, "Finetune/Step Time": 4.737043192610145} +{"Finetune/Learning Rate": 1.9008143578622527e-05, "Finetune/Loss": 0.949569582939148, "Finetune/Loss (Raw)": 0.9319429993629456, "Finetune/Step": 876, "Finetune/Step Time": 4.738526085391641} +{"Finetune/Learning Rate": 1.9005436906270902e-05, "Finetune/Loss": 0.9497645497322083, "Finetune/Loss (Raw)": 1.0318536758422852, "Finetune/Step": 877, "Finetune/Step Time": 4.7370556108653545} +{"Finetune/Learning Rate": 1.9002726739085027e-05, "Finetune/Loss": 0.9495370388031006, "Finetune/Loss (Raw)": 0.99957674741745, "Finetune/Step": 878, "Finetune/Step Time": 4.729996247217059} +{"Finetune/Learning Rate": 1.900001307811667e-05, "Finetune/Loss": 0.949523389339447, "Finetune/Loss (Raw)": 0.8910492062568665, "Finetune/Step": 879, "Finetune/Step Time": 4.750967951491475} +{"Finetune/Learning Rate": 1.899729592441895e-05, "Finetune/Loss": 0.9490718245506287, "Finetune/Loss (Raw)": 1.0222841501235962, "Finetune/Step": 880, "Finetune/Step Time": 4.754767892882228} +{"Finetune/Learning Rate": 1.899457527904634e-05, "Finetune/Loss": 0.9529584646224976, "Finetune/Loss (Raw)": 0.8959343433380127, "Finetune/Step": 881, "Finetune/Step Time": 4.729861916974187} +{"Finetune/Learning Rate": 1.8991851143054662e-05, "Finetune/Loss": 0.954423189163208, "Finetune/Loss (Raw)": 1.0656332969665527, "Finetune/Step": 882, "Finetune/Step Time": 4.729313096031547} +{"Finetune/Learning Rate": 1.8989123517501112e-05, "Finetune/Loss": 0.9553831219673157, "Finetune/Loss (Raw)": 1.0592764616012573, "Finetune/Step": 883, "Finetune/Step Time": 4.700677329674363} +{"Finetune/Learning Rate": 1.8986392403444222e-05, "Finetune/Loss": 0.956336498260498, "Finetune/Loss (Raw)": 1.0730793476104736, "Finetune/Step": 884, "Finetune/Step Time": 4.701347919180989} +{"Finetune/Learning Rate": 1.8983657801943884e-05, "Finetune/Loss": 0.9503979086875916, "Finetune/Loss (Raw)": 0.23083622753620148, "Finetune/Step": 885, "Finetune/Step Time": 4.726926423609257} +{"Finetune/Learning Rate": 1.8980919714061343e-05, "Finetune/Loss": 0.9493376016616821, "Finetune/Loss (Raw)": 1.0205258131027222, "Finetune/Step": 886, "Finetune/Step Time": 4.7304855436086655} +{"Finetune/Learning Rate": 1.89781781408592e-05, "Finetune/Loss": 0.9441883563995361, "Finetune/Loss (Raw)": 0.40895962715148926, "Finetune/Step": 887, "Finetune/Step Time": 4.768927875906229} +{"Finetune/Learning Rate": 1.8975433083401403e-05, "Finetune/Loss": 0.9446573257446289, "Finetune/Loss (Raw)": 1.0856853723526, "Finetune/Step": 888, "Finetune/Step Time": 4.775263346731663} +{"Finetune/Learning Rate": 1.897268454275326e-05, "Finetune/Loss": 0.945490837097168, "Finetune/Loss (Raw)": 1.0436779260635376, "Finetune/Step": 889, "Finetune/Step Time": 4.785605860874057} +{"Finetune/Learning Rate": 1.896993251998142e-05, "Finetune/Loss": 0.9501121044158936, "Finetune/Loss (Raw)": 0.9752085208892822, "Finetune/Step": 890, "Finetune/Step Time": 4.737918233498931} +{"Finetune/Learning Rate": 1.8967177016153895e-05, "Finetune/Loss": 0.9508407115936279, "Finetune/Loss (Raw)": 0.9918549060821533, "Finetune/Step": 891, "Finetune/Step Time": 4.736420094966888} +{"Finetune/Learning Rate": 1.8964418032340037e-05, "Finetune/Loss": 0.9499307870864868, "Finetune/Loss (Raw)": 0.8887065052986145, "Finetune/Step": 892, "Finetune/Step Time": 4.741087954491377} +{"Finetune/Learning Rate": 1.8961655569610557e-05, "Finetune/Loss": 0.9510995149612427, "Finetune/Loss (Raw)": 1.1314235925674438, "Finetune/Step": 893, "Finetune/Step Time": 4.737603781744838} +{"Finetune/Learning Rate": 1.8958889629037514e-05, "Finetune/Loss": 0.9500946998596191, "Finetune/Loss (Raw)": 0.971811056137085, "Finetune/Step": 894, "Finetune/Step Time": 4.728188326582313} +{"Finetune/Learning Rate": 1.895612021169431e-05, "Finetune/Loss": 0.9505397081375122, "Finetune/Loss (Raw)": 0.9804992079734802, "Finetune/Step": 895, "Finetune/Step Time": 4.735205415636301} +{"Finetune/Learning Rate": 1.8953347318655704e-05, "Finetune/Loss": 0.9515976309776306, "Finetune/Loss (Raw)": 1.0674477815628052, "Finetune/Step": 896, "Finetune/Step Time": 4.727335624396801} +{"Finetune/Learning Rate": 1.8950570950997808e-05, "Finetune/Loss": 0.9517678022384644, "Finetune/Loss (Raw)": 0.8494159579277039, "Finetune/Step": 897, "Finetune/Step Time": 4.726247830316424} +{"Finetune/Learning Rate": 1.894779110979807e-05, "Finetune/Loss": 0.9465439319610596, "Finetune/Loss (Raw)": 0.36380207538604736, "Finetune/Step": 898, "Finetune/Step Time": 4.745622957125306} +{"Finetune/Learning Rate": 1.894500779613529e-05, "Finetune/Loss": 0.9445560574531555, "Finetune/Loss (Raw)": 0.8248898983001709, "Finetune/Step": 899, "Finetune/Step Time": 4.820753451436758} +{"Finetune/Learning Rate": 1.8942221011089623e-05, "Finetune/Loss": 0.9454404711723328, "Finetune/Loss (Raw)": 1.1151658296585083, "Finetune/Step": 900, "Finetune/Step Time": 4.826657436788082} +{"Finetune/Learning Rate": 1.8939430755742564e-05, "Finetune/Loss": 0.9457225799560547, "Finetune/Loss (Raw)": 1.1024971008300781, "Finetune/Step": 901, "Finetune/Step Time": 4.8323190081864595} +{"Finetune/Learning Rate": 1.8936637031176957e-05, "Finetune/Loss": 0.9472172260284424, "Finetune/Loss (Raw)": 1.066765546798706, "Finetune/Step": 902, "Finetune/Step Time": 4.814953053370118} +{"Finetune/Learning Rate": 1.893383983847699e-05, "Finetune/Loss": 0.9475172162055969, "Finetune/Loss (Raw)": 1.0748131275177002, "Finetune/Step": 903, "Finetune/Step Time": 4.829634228721261} +{"Finetune/Learning Rate": 1.89310391787282e-05, "Finetune/Loss": 0.9487663507461548, "Finetune/Loss (Raw)": 0.9607507586479187, "Finetune/Step": 904, "Finetune/Step Time": 4.831366430968046} +{"Finetune/Learning Rate": 1.892823505301747e-05, "Finetune/Loss": 0.9481188654899597, "Finetune/Loss (Raw)": 1.0600191354751587, "Finetune/Step": 905, "Finetune/Step Time": 4.8059276714921} +{"Finetune/Learning Rate": 1.8925427462433027e-05, "Finetune/Loss": 0.9469085931777954, "Finetune/Loss (Raw)": 0.9401785731315613, "Finetune/Step": 906, "Finetune/Step Time": 4.808989800512791} +{"Finetune/Learning Rate": 1.8922616408064436e-05, "Finetune/Loss": 0.9477523565292358, "Finetune/Loss (Raw)": 1.044801950454712, "Finetune/Step": 907, "Finetune/Step Time": 4.796277927234769} +{"Finetune/Learning Rate": 1.8919801891002615e-05, "Finetune/Loss": 0.9526945352554321, "Finetune/Loss (Raw)": 0.9727542400360107, "Finetune/Step": 908, "Finetune/Step Time": 4.757561519742012} +{"Finetune/Learning Rate": 1.891698391233983e-05, "Finetune/Loss": 0.9531256556510925, "Finetune/Loss (Raw)": 0.9924105405807495, "Finetune/Step": 909, "Finetune/Step Time": 4.761825433000922} +{"Finetune/Learning Rate": 1.8914162473169676e-05, "Finetune/Loss": 0.9549106955528259, "Finetune/Loss (Raw)": 0.9673279523849487, "Finetune/Step": 910, "Finetune/Step Time": 4.738505654036999} +{"Finetune/Learning Rate": 1.8911337574587098e-05, "Finetune/Loss": 0.9550045728683472, "Finetune/Loss (Raw)": 0.9395772218704224, "Finetune/Step": 911, "Finetune/Step Time": 4.689271381124854} +{"Finetune/Learning Rate": 1.8908509217688387e-05, "Finetune/Loss": 0.9578913450241089, "Finetune/Loss (Raw)": 1.123155951499939, "Finetune/Step": 912, "Finetune/Step Time": 4.680272959172726} +{"Finetune/Learning Rate": 1.8905677403571177e-05, "Finetune/Loss": 0.955072283744812, "Finetune/Loss (Raw)": 0.7194488048553467, "Finetune/Step": 913, "Finetune/Step Time": 4.674976816400886} +{"Finetune/Learning Rate": 1.890284213333443e-05, "Finetune/Loss": 0.9549104571342468, "Finetune/Loss (Raw)": 1.1205111742019653, "Finetune/Step": 914, "Finetune/Step Time": 4.6734748017042875} +{"Finetune/Learning Rate": 1.8900003408078468e-05, "Finetune/Loss": 0.9540530443191528, "Finetune/Loss (Raw)": 1.0087099075317383, "Finetune/Step": 915, "Finetune/Step Time": 4.641647148877382} +{"Finetune/Learning Rate": 1.889716122890494e-05, "Finetune/Loss": 0.9532989263534546, "Finetune/Loss (Raw)": 1.035665512084961, "Finetune/Step": 916, "Finetune/Step Time": 4.6483299639075994} +{"Finetune/Learning Rate": 1.889431559691684e-05, "Finetune/Loss": 0.9547322392463684, "Finetune/Loss (Raw)": 1.1912423372268677, "Finetune/Step": 917, "Finetune/Step Time": 4.686702275648713} +{"Finetune/Learning Rate": 1.8891466513218507e-05, "Finetune/Loss": 0.956344723701477, "Finetune/Loss (Raw)": 0.9760355353355408, "Finetune/Step": 918, "Finetune/Step Time": 4.664131859317422} +{"Finetune/Learning Rate": 1.888861397891561e-05, "Finetune/Loss": 0.9549903869628906, "Finetune/Loss (Raw)": 0.7668859362602234, "Finetune/Step": 919, "Finetune/Step Time": 4.683795226737857} +{"Finetune/Learning Rate": 1.888575799511516e-05, "Finetune/Loss": 0.9549421072006226, "Finetune/Loss (Raw)": 0.8428837656974792, "Finetune/Step": 920, "Finetune/Step Time": 4.689946556463838} +{"Finetune/Learning Rate": 1.888289856292552e-05, "Finetune/Loss": 0.95415198802948, "Finetune/Loss (Raw)": 0.9851987361907959, "Finetune/Step": 921, "Finetune/Step Time": 4.67992396466434} +{"Finetune/Learning Rate": 1.8880035683456365e-05, "Finetune/Loss": 0.9539596438407898, "Finetune/Loss (Raw)": 0.918781578540802, "Finetune/Step": 922, "Finetune/Step Time": 4.679312629625201} +{"Finetune/Learning Rate": 1.887716935781873e-05, "Finetune/Loss": 0.9543395042419434, "Finetune/Loss (Raw)": 1.0335839986801147, "Finetune/Step": 923, "Finetune/Step Time": 4.671427519991994} +{"Finetune/Learning Rate": 1.887429958712498e-05, "Finetune/Loss": 0.9541997909545898, "Finetune/Loss (Raw)": 0.9372589588165283, "Finetune/Step": 924, "Finetune/Step Time": 4.679866898804903} +{"Finetune/Learning Rate": 1.8871426372488818e-05, "Finetune/Loss": 0.9528768062591553, "Finetune/Loss (Raw)": 0.8117526173591614, "Finetune/Step": 925, "Finetune/Step Time": 4.699118051677942} +{"Finetune/Learning Rate": 1.886854971502528e-05, "Finetune/Loss": 0.9522743225097656, "Finetune/Loss (Raw)": 0.9408861994743347, "Finetune/Step": 926, "Finetune/Step Time": 4.703056454658508} +{"Finetune/Learning Rate": 1.886566961585074e-05, "Finetune/Loss": 0.9533004760742188, "Finetune/Loss (Raw)": 1.0402625799179077, "Finetune/Step": 927, "Finetune/Step Time": 4.695616398006678} +{"Finetune/Learning Rate": 1.886278607608291e-05, "Finetune/Loss": 0.9528830051422119, "Finetune/Loss (Raw)": 1.005964994430542, "Finetune/Step": 928, "Finetune/Step Time": 4.687236582860351} +{"Finetune/Learning Rate": 1.8859899096840836e-05, "Finetune/Loss": 0.952522873878479, "Finetune/Loss (Raw)": 1.0297446250915527, "Finetune/Step": 929, "Finetune/Step Time": 4.662913190200925} +{"Finetune/Learning Rate": 1.8857008679244897e-05, "Finetune/Loss": 0.9518115520477295, "Finetune/Loss (Raw)": 0.9016625881195068, "Finetune/Step": 930, "Finetune/Step Time": 4.667319759726524} +{"Finetune/Learning Rate": 1.885411482441681e-05, "Finetune/Loss": 0.9535285234451294, "Finetune/Loss (Raw)": 1.025921106338501, "Finetune/Step": 931, "Finetune/Step Time": 4.648911450058222} +{"Finetune/Learning Rate": 1.885121753347962e-05, "Finetune/Loss": 0.954056441783905, "Finetune/Loss (Raw)": 1.0581425428390503, "Finetune/Step": 932, "Finetune/Step Time": 4.655137022957206} +{"Finetune/Learning Rate": 1.884831680755771e-05, "Finetune/Loss": 0.9540201425552368, "Finetune/Loss (Raw)": 1.0358774662017822, "Finetune/Step": 933, "Finetune/Step Time": 4.664475677534938} +{"Finetune/Learning Rate": 1.8845412647776795e-05, "Finetune/Loss": 0.9553533792495728, "Finetune/Loss (Raw)": 1.0130521059036255, "Finetune/Step": 934, "Finetune/Step Time": 4.651135176420212} +{"Finetune/Learning Rate": 1.8842505055263928e-05, "Finetune/Loss": 0.9542866349220276, "Finetune/Loss (Raw)": 0.8545140624046326, "Finetune/Step": 935, "Finetune/Step Time": 4.652128303423524} +{"Finetune/Learning Rate": 1.883959403114748e-05, "Finetune/Loss": 0.953466534614563, "Finetune/Loss (Raw)": 0.9398260712623596, "Finetune/Step": 936, "Finetune/Step Time": 4.644348733127117} +{"Finetune/Learning Rate": 1.8836679576557165e-05, "Finetune/Loss": 0.9548269510269165, "Finetune/Loss (Raw)": 1.1296870708465576, "Finetune/Step": 937, "Finetune/Step Time": 4.639179598540068} +{"Finetune/Learning Rate": 1.8833761692624034e-05, "Finetune/Loss": 0.9551348686218262, "Finetune/Loss (Raw)": 1.1349939107894897, "Finetune/Step": 938, "Finetune/Step Time": 4.642951246351004} +{"Finetune/Learning Rate": 1.883084038048045e-05, "Finetune/Loss": 0.9557122588157654, "Finetune/Loss (Raw)": 0.9225841760635376, "Finetune/Step": 939, "Finetune/Step Time": 4.643967308104038} +{"Finetune/Learning Rate": 1.8827915641260127e-05, "Finetune/Loss": 0.9541547298431396, "Finetune/Loss (Raw)": 1.0276525020599365, "Finetune/Step": 940, "Finetune/Step Time": 4.646791277453303} +{"Finetune/Learning Rate": 1.882498747609809e-05, "Finetune/Loss": 0.9540157318115234, "Finetune/Loss (Raw)": 0.9698099493980408, "Finetune/Step": 941, "Finetune/Step Time": 4.646031802520156} +{"Finetune/Learning Rate": 1.8822055886130708e-05, "Finetune/Loss": 0.9537804126739502, "Finetune/Loss (Raw)": 0.9348725080490112, "Finetune/Step": 942, "Finetune/Step Time": 4.652360176667571} +{"Finetune/Learning Rate": 1.8819120872495675e-05, "Finetune/Loss": 0.9535839557647705, "Finetune/Loss (Raw)": 0.9550997018814087, "Finetune/Step": 943, "Finetune/Step Time": 4.651561684906483} +{"Finetune/Learning Rate": 1.8816182436332006e-05, "Finetune/Loss": 0.9530011415481567, "Finetune/Loss (Raw)": 0.996482789516449, "Finetune/Step": 944, "Finetune/Step Time": 4.646692765876651} +{"Finetune/Learning Rate": 1.881324057878006e-05, "Finetune/Loss": 0.9512861967086792, "Finetune/Loss (Raw)": 0.8611628413200378, "Finetune/Step": 945, "Finetune/Step Time": 4.655937800183892} +{"Finetune/Learning Rate": 1.8810295300981504e-05, "Finetune/Loss": 0.9556846618652344, "Finetune/Loss (Raw)": 0.8955725431442261, "Finetune/Step": 946, "Finetune/Step Time": 4.63355852663517} +{"Finetune/Learning Rate": 1.8807346604079352e-05, "Finetune/Loss": 0.9567208290100098, "Finetune/Loss (Raw)": 0.9590030312538147, "Finetune/Step": 947, "Finetune/Step Time": 4.652747493237257} +{"Finetune/Learning Rate": 1.880439448921793e-05, "Finetune/Loss": 0.9539529085159302, "Finetune/Loss (Raw)": 0.3870539665222168, "Finetune/Step": 948, "Finetune/Step Time": 4.681959832087159} +{"Finetune/Learning Rate": 1.88014389575429e-05, "Finetune/Loss": 0.9550033807754517, "Finetune/Loss (Raw)": 1.1351410150527954, "Finetune/Step": 949, "Finetune/Step Time": 4.661908844485879} +{"Finetune/Learning Rate": 1.879848001020124e-05, "Finetune/Loss": 0.9547442197799683, "Finetune/Loss (Raw)": 0.902725875377655, "Finetune/Step": 950, "Finetune/Step Time": 4.659227164462209} +{"Finetune/Learning Rate": 1.8795517648341267e-05, "Finetune/Loss": 0.9554018974304199, "Finetune/Loss (Raw)": 1.1813703775405884, "Finetune/Step": 951, "Finetune/Step Time": 4.652521133422852} +{"Finetune/Learning Rate": 1.8792551873112614e-05, "Finetune/Loss": 0.9535760283470154, "Finetune/Loss (Raw)": 0.847895085811615, "Finetune/Step": 952, "Finetune/Step Time": 4.657217947766185} +{"Finetune/Learning Rate": 1.8789582685666237e-05, "Finetune/Loss": 0.9540944695472717, "Finetune/Loss (Raw)": 0.9819677472114563, "Finetune/Step": 953, "Finetune/Step Time": 4.6490923799574375} +{"Finetune/Learning Rate": 1.878661008715442e-05, "Finetune/Loss": 0.9545507431030273, "Finetune/Loss (Raw)": 0.9150857925415039, "Finetune/Step": 954, "Finetune/Step Time": 4.646038601174951} +{"Finetune/Learning Rate": 1.8783634078730774e-05, "Finetune/Loss": 0.9544185400009155, "Finetune/Loss (Raw)": 0.9864842295646667, "Finetune/Step": 955, "Finetune/Step Time": 4.64474212191999} +{"Finetune/Learning Rate": 1.878065466155023e-05, "Finetune/Loss": 0.95943284034729, "Finetune/Loss (Raw)": 1.025296926498413, "Finetune/Step": 956, "Finetune/Step Time": 4.607844544574618} +{"Finetune/Learning Rate": 1.877767183676904e-05, "Finetune/Loss": 0.9580672979354858, "Finetune/Loss (Raw)": 0.9016640186309814, "Finetune/Step": 957, "Finetune/Step Time": 4.604698741808534} +{"Finetune/Learning Rate": 1.8774685605544776e-05, "Finetune/Loss": 0.957954466342926, "Finetune/Loss (Raw)": 0.982450008392334, "Finetune/Step": 958, "Finetune/Step Time": 4.613769078627229} +{"Finetune/Learning Rate": 1.8771695969036342e-05, "Finetune/Loss": 0.957021951675415, "Finetune/Loss (Raw)": 0.9271653890609741, "Finetune/Step": 959, "Finetune/Step Time": 4.589887160807848} +{"Finetune/Learning Rate": 1.8768702928403954e-05, "Finetune/Loss": 0.9559295773506165, "Finetune/Loss (Raw)": 0.9506257176399231, "Finetune/Step": 960, "Finetune/Step Time": 4.59264425188303} +{"Finetune/Learning Rate": 1.8765706484809155e-05, "Finetune/Loss": 0.9594769477844238, "Finetune/Loss (Raw)": 0.8823962807655334, "Finetune/Step": 961, "Finetune/Step Time": 4.555060992017388} +{"Finetune/Learning Rate": 1.8762706639414806e-05, "Finetune/Loss": 0.9602952003479004, "Finetune/Loss (Raw)": 1.1005444526672363, "Finetune/Step": 962, "Finetune/Step Time": 4.551927253603935} +{"Finetune/Learning Rate": 1.8759703393385087e-05, "Finetune/Loss": 0.9596489071846008, "Finetune/Loss (Raw)": 0.9053155779838562, "Finetune/Step": 963, "Finetune/Step Time": 4.5448261462152} +{"Finetune/Learning Rate": 1.87566967478855e-05, "Finetune/Loss": 0.9589972496032715, "Finetune/Loss (Raw)": 0.9304983615875244, "Finetune/Step": 964, "Finetune/Step Time": 4.543777352198958} +{"Finetune/Learning Rate": 1.8753686704082868e-05, "Finetune/Loss": 0.9579728245735168, "Finetune/Loss (Raw)": 0.8022359609603882, "Finetune/Step": 965, "Finetune/Step Time": 4.538891833275557} +{"Finetune/Learning Rate": 1.8750673263145323e-05, "Finetune/Loss": 0.9570239782333374, "Finetune/Loss (Raw)": 0.9396446943283081, "Finetune/Step": 966, "Finetune/Step Time": 4.5463536735624075} +{"Finetune/Learning Rate": 1.874765642624233e-05, "Finetune/Loss": 0.9574868679046631, "Finetune/Loss (Raw)": 0.9870456457138062, "Finetune/Step": 967, "Finetune/Step Time": 4.551239186897874} +{"Finetune/Learning Rate": 1.874463619454466e-05, "Finetune/Loss": 0.9566646814346313, "Finetune/Loss (Raw)": 1.0563045740127563, "Finetune/Step": 968, "Finetune/Step Time": 4.5493946727365255} +{"Finetune/Learning Rate": 1.874161256922441e-05, "Finetune/Loss": 0.9568590521812439, "Finetune/Loss (Raw)": 0.9196010231971741, "Finetune/Step": 969, "Finetune/Step Time": 4.5678568705916405} +{"Finetune/Learning Rate": 1.8738585551454982e-05, "Finetune/Loss": 0.9539860486984253, "Finetune/Loss (Raw)": 0.6994293928146362, "Finetune/Step": 970, "Finetune/Step Time": 4.574835877865553} +{"Finetune/Learning Rate": 1.873555514241111e-05, "Finetune/Loss": 0.9536451697349548, "Finetune/Loss (Raw)": 1.0022635459899902, "Finetune/Step": 971, "Finetune/Step Time": 4.588353663682938} +{"Finetune/Learning Rate": 1.8732521343268838e-05, "Finetune/Loss": 0.9489045143127441, "Finetune/Loss (Raw)": 0.39110419154167175, "Finetune/Step": 972, "Finetune/Step Time": 4.620254911482334} +{"Finetune/Learning Rate": 1.8729484155205517e-05, "Finetune/Loss": 0.947400689125061, "Finetune/Loss (Raw)": 0.8874803781509399, "Finetune/Step": 973, "Finetune/Step Time": 4.625615896657109} +{"Finetune/Learning Rate": 1.8726443579399828e-05, "Finetune/Loss": 0.94682377576828, "Finetune/Loss (Raw)": 0.9666808247566223, "Finetune/Step": 974, "Finetune/Step Time": 4.646368311718106} +{"Finetune/Learning Rate": 1.8723399617031754e-05, "Finetune/Loss": 0.9517773389816284, "Finetune/Loss (Raw)": 1.1006968021392822, "Finetune/Step": 975, "Finetune/Step Time": 4.625202799215913} +{"Finetune/Learning Rate": 1.8720352269282592e-05, "Finetune/Loss": 0.9528269171714783, "Finetune/Loss (Raw)": 0.9845115542411804, "Finetune/Step": 976, "Finetune/Step Time": 4.6161487605422735} +{"Finetune/Learning Rate": 1.8717301537334975e-05, "Finetune/Loss": 0.953271746635437, "Finetune/Loss (Raw)": 1.0647293329238892, "Finetune/Step": 977, "Finetune/Step Time": 4.61659736558795} +{"Finetune/Learning Rate": 1.8714247422372816e-05, "Finetune/Loss": 0.9536957740783691, "Finetune/Loss (Raw)": 0.9460103511810303, "Finetune/Step": 978, "Finetune/Step Time": 4.601365523412824} +{"Finetune/Learning Rate": 1.8711189925581366e-05, "Finetune/Loss": 0.9525777101516724, "Finetune/Loss (Raw)": 0.8814314603805542, "Finetune/Step": 979, "Finetune/Step Time": 4.602463016286492} +{"Finetune/Learning Rate": 1.8708129048147176e-05, "Finetune/Loss": 0.9547511339187622, "Finetune/Loss (Raw)": 1.1609593629837036, "Finetune/Step": 980, "Finetune/Step Time": 4.60090197622776} +{"Finetune/Learning Rate": 1.8705064791258118e-05, "Finetune/Loss": 0.9550098776817322, "Finetune/Loss (Raw)": 0.9875797033309937, "Finetune/Step": 981, "Finetune/Step Time": 4.601383585482836} +{"Finetune/Learning Rate": 1.870199715610336e-05, "Finetune/Loss": 0.9557146430015564, "Finetune/Loss (Raw)": 1.0311496257781982, "Finetune/Step": 982, "Finetune/Step Time": 4.589334841817617} +{"Finetune/Learning Rate": 1.8698926143873404e-05, "Finetune/Loss": 0.9574360847473145, "Finetune/Loss (Raw)": 1.0259939432144165, "Finetune/Step": 983, "Finetune/Step Time": 4.5590332839637995} +{"Finetune/Learning Rate": 1.8695851755760042e-05, "Finetune/Loss": 0.9572778344154358, "Finetune/Loss (Raw)": 0.970132052898407, "Finetune/Step": 984, "Finetune/Step Time": 4.569809766486287} +{"Finetune/Learning Rate": 1.8692773992956383e-05, "Finetune/Loss": 0.9572665095329285, "Finetune/Loss (Raw)": 0.9486331939697266, "Finetune/Step": 985, "Finetune/Step Time": 4.568032497540116} +{"Finetune/Learning Rate": 1.8689692856656854e-05, "Finetune/Loss": 0.9548488855361938, "Finetune/Loss (Raw)": 0.8789107203483582, "Finetune/Step": 986, "Finetune/Step Time": 4.571324843913317} +{"Finetune/Learning Rate": 1.8686608348057178e-05, "Finetune/Loss": 0.95406174659729, "Finetune/Loss (Raw)": 1.0025917291641235, "Finetune/Step": 987, "Finetune/Step Time": 4.568448541685939} +{"Finetune/Learning Rate": 1.868352046835439e-05, "Finetune/Loss": 0.9533624053001404, "Finetune/Loss (Raw)": 0.8611279726028442, "Finetune/Step": 988, "Finetune/Step Time": 4.570699397474527} +{"Finetune/Learning Rate": 1.8680429218746842e-05, "Finetune/Loss": 0.9557743668556213, "Finetune/Loss (Raw)": 1.0334768295288086, "Finetune/Step": 989, "Finetune/Step Time": 4.574086204171181} +{"Finetune/Learning Rate": 1.867733460043419e-05, "Finetune/Loss": 0.9536892175674438, "Finetune/Loss (Raw)": 0.7494227886199951, "Finetune/Step": 990, "Finetune/Step Time": 4.579561606049538} +{"Finetune/Learning Rate": 1.8674236614617386e-05, "Finetune/Loss": 0.9552614688873291, "Finetune/Loss (Raw)": 1.0410338640213013, "Finetune/Step": 991, "Finetune/Step Time": 4.570235336199403} +{"Finetune/Learning Rate": 1.86711352624987e-05, "Finetune/Loss": 0.9547327756881714, "Finetune/Loss (Raw)": 0.9766295552253723, "Finetune/Step": 992, "Finetune/Step Time": 4.568081649020314} +{"Finetune/Learning Rate": 1.866803054528171e-05, "Finetune/Loss": 0.9549318552017212, "Finetune/Loss (Raw)": 0.9855425953865051, "Finetune/Step": 993, "Finetune/Step Time": 4.578786574304104} +{"Finetune/Learning Rate": 1.8664922464171295e-05, "Finetune/Loss": 0.9522033929824829, "Finetune/Loss (Raw)": 0.7719420194625854, "Finetune/Step": 994, "Finetune/Step Time": 4.586944403126836} +{"Finetune/Learning Rate": 1.866181102037364e-05, "Finetune/Loss": 0.9511966705322266, "Finetune/Loss (Raw)": 0.9401283860206604, "Finetune/Step": 995, "Finetune/Step Time": 4.585953684523702} +{"Finetune/Learning Rate": 1.8658696215096235e-05, "Finetune/Loss": 0.951350212097168, "Finetune/Loss (Raw)": 0.9639394879341125, "Finetune/Step": 996, "Finetune/Step Time": 4.590407930314541} +{"Finetune/Learning Rate": 1.8655578049547878e-05, "Finetune/Loss": 0.9501555562019348, "Finetune/Loss (Raw)": 0.9383737444877625, "Finetune/Step": 997, "Finetune/Step Time": 4.570009559392929} +{"Finetune/Learning Rate": 1.8652456524938665e-05, "Finetune/Loss": 0.9505249261856079, "Finetune/Loss (Raw)": 0.9483110904693604, "Finetune/Step": 998, "Finetune/Step Time": 4.583028998225927} +{"Finetune/Learning Rate": 1.864933164248e-05, "Finetune/Loss": 0.9525774717330933, "Finetune/Loss (Raw)": 1.1157416105270386, "Finetune/Step": 999, "Finetune/Step Time": 4.576598163694143} +{"Finetune/Learning Rate": 1.8646203403384583e-05, "Finetune/Loss": 0.953689694404602, "Finetune/Loss (Raw)": 1.1413071155548096, "Finetune/Step": 1000, "Finetune/Step Time": 4.5700024124234915} +{"Finetune/Learning Rate": 1.8643071808866434e-05, "Finetune/Loss": 0.948066234588623, "Finetune/Loss (Raw)": 0.32421571016311646, "Finetune/Step": 1001, "Finetune/Step Time": 4.611847767606378} +{"Finetune/Learning Rate": 1.8639936860140856e-05, "Finetune/Loss": 0.9488570690155029, "Finetune/Loss (Raw)": 0.9856332540512085, "Finetune/Step": 1002, "Finetune/Step Time": 4.612476473674178} +{"Finetune/Learning Rate": 1.8636798558424464e-05, "Finetune/Loss": 0.9496430158615112, "Finetune/Loss (Raw)": 1.0523535013198853, "Finetune/Step": 1003, "Finetune/Step Time": 4.615748705342412} +{"Finetune/Learning Rate": 1.8633656904935168e-05, "Finetune/Loss": 0.9506244659423828, "Finetune/Loss (Raw)": 1.057563304901123, "Finetune/Step": 1004, "Finetune/Step Time": 4.615388302132487} +{"Finetune/Learning Rate": 1.863051190089219e-05, "Finetune/Loss": 0.9505380392074585, "Finetune/Loss (Raw)": 1.0208015441894531, "Finetune/Step": 1005, "Finetune/Step Time": 4.6081973146647215} +{"Finetune/Learning Rate": 1.8627363547516032e-05, "Finetune/Loss": 0.9494913220405579, "Finetune/Loss (Raw)": 0.8655920028686523, "Finetune/Step": 1006, "Finetune/Step Time": 4.623860966414213} +{"Finetune/Learning Rate": 1.8624211846028522e-05, "Finetune/Loss": 0.9498173594474792, "Finetune/Loss (Raw)": 0.9327804446220398, "Finetune/Step": 1007, "Finetune/Step Time": 4.605117930099368} +{"Finetune/Learning Rate": 1.862105679765277e-05, "Finetune/Loss": 0.9494920969009399, "Finetune/Loss (Raw)": 0.9806554913520813, "Finetune/Step": 1008, "Finetune/Step Time": 4.604583527892828} +{"Finetune/Learning Rate": 1.8617898403613183e-05, "Finetune/Loss": 0.9458215236663818, "Finetune/Loss (Raw)": 0.4260966181755066, "Finetune/Step": 1009, "Finetune/Step Time": 4.629420595243573} +{"Finetune/Learning Rate": 1.8614736665135476e-05, "Finetune/Loss": 0.9445285201072693, "Finetune/Loss (Raw)": 0.9001314640045166, "Finetune/Step": 1010, "Finetune/Step Time": 4.631979374215007} +{"Finetune/Learning Rate": 1.861157158344666e-05, "Finetune/Loss": 0.9434000253677368, "Finetune/Loss (Raw)": 0.9148282408714294, "Finetune/Step": 1011, "Finetune/Step Time": 4.630933640524745} +{"Finetune/Learning Rate": 1.8608403159775037e-05, "Finetune/Loss": 0.9428693056106567, "Finetune/Loss (Raw)": 1.0051463842391968, "Finetune/Step": 1012, "Finetune/Step Time": 4.636610943824053} +{"Finetune/Learning Rate": 1.8605231395350213e-05, "Finetune/Loss": 0.9481691122055054, "Finetune/Loss (Raw)": 0.9092137217521667, "Finetune/Step": 1013, "Finetune/Step Time": 4.597619326785207} +{"Finetune/Learning Rate": 1.860205629140309e-05, "Finetune/Loss": 0.9483031034469604, "Finetune/Loss (Raw)": 1.037675380706787, "Finetune/Step": 1014, "Finetune/Step Time": 4.597642844542861} +{"Finetune/Learning Rate": 1.8598877849165857e-05, "Finetune/Loss": 0.9475745558738708, "Finetune/Loss (Raw)": 0.31570279598236084, "Finetune/Step": 1015, "Finetune/Step Time": 4.5999247264117} +{"Finetune/Learning Rate": 1.8595696069872013e-05, "Finetune/Loss": 0.9469848871231079, "Finetune/Loss (Raw)": 1.010209560394287, "Finetune/Step": 1016, "Finetune/Step Time": 4.598123637959361} +{"Finetune/Learning Rate": 1.859251095475634e-05, "Finetune/Loss": 0.9471696615219116, "Finetune/Loss (Raw)": 1.067327857017517, "Finetune/Step": 1017, "Finetune/Step Time": 4.584565535187721} +{"Finetune/Learning Rate": 1.858932250505492e-05, "Finetune/Loss": 0.9472710490226746, "Finetune/Loss (Raw)": 0.9881870150566101, "Finetune/Step": 1018, "Finetune/Step Time": 4.623685279861093} +{"Finetune/Learning Rate": 1.8586130722005128e-05, "Finetune/Loss": 0.9467207789421082, "Finetune/Loss (Raw)": 0.9214189648628235, "Finetune/Step": 1019, "Finetune/Step Time": 4.621070245280862} +{"Finetune/Learning Rate": 1.8582935606845636e-05, "Finetune/Loss": 0.9474855661392212, "Finetune/Loss (Raw)": 0.9866043329238892, "Finetune/Step": 1020, "Finetune/Step Time": 4.61604343354702} +{"Finetune/Learning Rate": 1.8579737160816397e-05, "Finetune/Loss": 0.9453179836273193, "Finetune/Loss (Raw)": 0.8539692759513855, "Finetune/Step": 1021, "Finetune/Step Time": 4.627393247559667} +{"Finetune/Learning Rate": 1.8576535385158675e-05, "Finetune/Loss": 0.9456130266189575, "Finetune/Loss (Raw)": 1.0095770359039307, "Finetune/Step": 1022, "Finetune/Step Time": 4.627464059740305} +{"Finetune/Learning Rate": 1.8573330281115008e-05, "Finetune/Loss": 0.9445314407348633, "Finetune/Loss (Raw)": 0.8420504331588745, "Finetune/Step": 1023, "Finetune/Step Time": 4.63088858127594} +{"Finetune/Learning Rate": 1.8570121849929245e-05, "Finetune/Loss": 0.9433270692825317, "Finetune/Loss (Raw)": 0.913296639919281, "Finetune/Step": 1024, "Finetune/Step Time": 4.632811613380909} +{"Finetune/Learning Rate": 1.8566910092846508e-05, "Finetune/Loss": 0.9435831308364868, "Finetune/Loss (Raw)": 0.8821830153465271, "Finetune/Step": 1025, "Finetune/Step Time": 4.618025751784444} +{"Finetune/Learning Rate": 1.8563695011113217e-05, "Finetune/Loss": 0.9486426115036011, "Finetune/Loss (Raw)": 1.0114145278930664, "Finetune/Step": 1026, "Finetune/Step Time": 4.58829072676599} +{"Finetune/Learning Rate": 1.8560476605977085e-05, "Finetune/Loss": 0.9451277852058411, "Finetune/Loss (Raw)": 0.3749949038028717, "Finetune/Step": 1027, "Finetune/Step Time": 4.548297356814146} +{"Finetune/Learning Rate": 1.8557254878687114e-05, "Finetune/Loss": 0.9445196390151978, "Finetune/Loss (Raw)": 1.0373307466506958, "Finetune/Step": 1028, "Finetune/Step Time": 4.551684066653252} +{"Finetune/Learning Rate": 1.8554029830493587e-05, "Finetune/Loss": 0.9427086114883423, "Finetune/Loss (Raw)": 0.8706824779510498, "Finetune/Step": 1029, "Finetune/Step Time": 4.559088150039315} +{"Finetune/Learning Rate": 1.855080146264809e-05, "Finetune/Loss": 0.9422235488891602, "Finetune/Loss (Raw)": 1.0046745538711548, "Finetune/Step": 1030, "Finetune/Step Time": 4.563251052051783} +{"Finetune/Learning Rate": 1.854756977640348e-05, "Finetune/Loss": 0.9414731860160828, "Finetune/Loss (Raw)": 0.9787694811820984, "Finetune/Step": 1031, "Finetune/Step Time": 4.538596237078309} +{"Finetune/Learning Rate": 1.8544334773013924e-05, "Finetune/Loss": 0.9367170333862305, "Finetune/Loss (Raw)": 0.3519669473171234, "Finetune/Step": 1032, "Finetune/Step Time": 4.574303813278675} +{"Finetune/Learning Rate": 1.8541096453734852e-05, "Finetune/Loss": 0.9362261295318604, "Finetune/Loss (Raw)": 0.9971792697906494, "Finetune/Step": 1033, "Finetune/Step Time": 4.6125007681548595} +{"Finetune/Learning Rate": 1.8537854819822998e-05, "Finetune/Loss": 0.935455858707428, "Finetune/Loss (Raw)": 0.8415801525115967, "Finetune/Step": 1034, "Finetune/Step Time": 4.614390110597014} +{"Finetune/Learning Rate": 1.853460987253638e-05, "Finetune/Loss": 0.9350757598876953, "Finetune/Loss (Raw)": 0.9961515665054321, "Finetune/Step": 1035, "Finetune/Step Time": 4.62667347304523} +{"Finetune/Learning Rate": 1.8531361613134286e-05, "Finetune/Loss": 0.9349618554115295, "Finetune/Loss (Raw)": 0.9581751823425293, "Finetune/Step": 1036, "Finetune/Step Time": 4.631552757695317} +{"Finetune/Learning Rate": 1.8528110042877316e-05, "Finetune/Loss": 0.9339368343353271, "Finetune/Loss (Raw)": 0.8612045049667358, "Finetune/Step": 1037, "Finetune/Step Time": 4.634557366371155} +{"Finetune/Learning Rate": 1.8524855163027337e-05, "Finetune/Loss": 0.9354629516601562, "Finetune/Loss (Raw)": 1.162674069404602, "Finetune/Step": 1038, "Finetune/Step Time": 4.622704282402992} +{"Finetune/Learning Rate": 1.85215969748475e-05, "Finetune/Loss": 0.9359761476516724, "Finetune/Loss (Raw)": 1.0052622556686401, "Finetune/Step": 1039, "Finetune/Step Time": 4.622243905439973} +{"Finetune/Learning Rate": 1.851833547960225e-05, "Finetune/Loss": 0.9344419240951538, "Finetune/Loss (Raw)": 0.926781415939331, "Finetune/Step": 1040, "Finetune/Step Time": 4.618269806727767} +{"Finetune/Learning Rate": 1.8515070678557304e-05, "Finetune/Loss": 0.9371949434280396, "Finetune/Loss (Raw)": 1.0718345642089844, "Finetune/Step": 1041, "Finetune/Step Time": 4.600477011874318} +{"Finetune/Learning Rate": 1.8511802572979672e-05, "Finetune/Loss": 0.9368788599967957, "Finetune/Loss (Raw)": 1.080051302909851, "Finetune/Step": 1042, "Finetune/Step Time": 4.601636646315455} +{"Finetune/Learning Rate": 1.8508531164137644e-05, "Finetune/Loss": 0.9319950342178345, "Finetune/Loss (Raw)": 0.3835834562778473, "Finetune/Step": 1043, "Finetune/Step Time": 4.649403277784586} +{"Finetune/Learning Rate": 1.8505256453300778e-05, "Finetune/Loss": 0.9305180311203003, "Finetune/Loss (Raw)": 0.8466031551361084, "Finetune/Step": 1044, "Finetune/Step Time": 4.649991920217872} +{"Finetune/Learning Rate": 1.850197844173994e-05, "Finetune/Loss": 0.9292829036712646, "Finetune/Loss (Raw)": 1.0331534147262573, "Finetune/Step": 1045, "Finetune/Step Time": 4.634330704808235} +{"Finetune/Learning Rate": 1.8498697130727255e-05, "Finetune/Loss": 0.9289925694465637, "Finetune/Loss (Raw)": 0.9388673305511475, "Finetune/Step": 1046, "Finetune/Step Time": 4.636477617546916} +{"Finetune/Learning Rate": 1.8495412521536135e-05, "Finetune/Loss": 0.9307739734649658, "Finetune/Loss (Raw)": 0.9949091672897339, "Finetune/Step": 1047, "Finetune/Step Time": 4.6176433600485325} +{"Finetune/Learning Rate": 1.8492124615441276e-05, "Finetune/Loss": 0.9332399368286133, "Finetune/Loss (Raw)": 1.1585235595703125, "Finetune/Step": 1048, "Finetune/Step Time": 4.615070976316929} +{"Finetune/Learning Rate": 1.8488833413718646e-05, "Finetune/Loss": 0.9322700500488281, "Finetune/Loss (Raw)": 0.8610507845878601, "Finetune/Step": 1049, "Finetune/Step Time": 4.612457508221269} +{"Finetune/Learning Rate": 1.84855389176455e-05, "Finetune/Loss": 0.9313814640045166, "Finetune/Loss (Raw)": 0.8050458431243896, "Finetune/Step": 1050, "Finetune/Step Time": 4.624026462435722} +{"Finetune/Learning Rate": 1.8482241128500366e-05, "Finetune/Loss": 0.9306384921073914, "Finetune/Loss (Raw)": 0.9384854435920715, "Finetune/Step": 1051, "Finetune/Step Time": 4.6279167998582125} +{"Finetune/Learning Rate": 1.847894004756305e-05, "Finetune/Loss": 0.9322812557220459, "Finetune/Loss (Raw)": 1.147533893585205, "Finetune/Step": 1052, "Finetune/Step Time": 4.626126650720835} +{"Finetune/Learning Rate": 1.8475635676114643e-05, "Finetune/Loss": 0.9335188269615173, "Finetune/Loss (Raw)": 0.9701568484306335, "Finetune/Step": 1053, "Finetune/Step Time": 4.613380579277873} +{"Finetune/Learning Rate": 1.8472328015437503e-05, "Finetune/Loss": 0.9348082542419434, "Finetune/Loss (Raw)": 1.1059350967407227, "Finetune/Step": 1054, "Finetune/Step Time": 4.60904286056757} +{"Finetune/Learning Rate": 1.8469017066815265e-05, "Finetune/Loss": 0.933671236038208, "Finetune/Loss (Raw)": 0.8947263956069946, "Finetune/Step": 1055, "Finetune/Step Time": 4.6098024901002645} +{"Finetune/Learning Rate": 1.8465702831532848e-05, "Finetune/Loss": 0.9342983961105347, "Finetune/Loss (Raw)": 1.0862393379211426, "Finetune/Step": 1056, "Finetune/Step Time": 4.61148801445961} +{"Finetune/Learning Rate": 1.8462385310876444e-05, "Finetune/Loss": 0.9333211779594421, "Finetune/Loss (Raw)": 0.9046586751937866, "Finetune/Step": 1057, "Finetune/Step Time": 4.6061560697853565} +{"Finetune/Learning Rate": 1.845906450613351e-05, "Finetune/Loss": 0.9334554672241211, "Finetune/Loss (Raw)": 0.9188476204872131, "Finetune/Step": 1058, "Finetune/Step Time": 4.599058378487825} +{"Finetune/Learning Rate": 1.8455740418592793e-05, "Finetune/Loss": 0.932740330696106, "Finetune/Loss (Raw)": 0.9343872666358948, "Finetune/Step": 1059, "Finetune/Step Time": 4.617384035140276} +{"Finetune/Learning Rate": 1.8452413049544304e-05, "Finetune/Loss": 0.9319760203361511, "Finetune/Loss (Raw)": 0.9603099226951599, "Finetune/Step": 1060, "Finetune/Step Time": 4.615509225055575} +{"Finetune/Learning Rate": 1.844908240027933e-05, "Finetune/Loss": 0.9303077459335327, "Finetune/Loss (Raw)": 0.8223460912704468, "Finetune/Step": 1061, "Finetune/Step Time": 4.596123240888119} +{"Finetune/Learning Rate": 1.8445748472090432e-05, "Finetune/Loss": 0.9301161766052246, "Finetune/Loss (Raw)": 0.988525390625, "Finetune/Step": 1062, "Finetune/Step Time": 4.591260325163603} +{"Finetune/Learning Rate": 1.844241126627144e-05, "Finetune/Loss": 0.931016206741333, "Finetune/Loss (Raw)": 0.9697251319885254, "Finetune/Step": 1063, "Finetune/Step Time": 4.598918665200472} +{"Finetune/Learning Rate": 1.843907078411746e-05, "Finetune/Loss": 0.9328873753547668, "Finetune/Loss (Raw)": 1.1793264150619507, "Finetune/Step": 1064, "Finetune/Step Time": 4.586466565728188} +{"Finetune/Learning Rate": 1.8435727026924864e-05, "Finetune/Loss": 0.932820200920105, "Finetune/Loss (Raw)": 1.1210888624191284, "Finetune/Step": 1065, "Finetune/Step Time": 4.587825486436486} +{"Finetune/Learning Rate": 1.8432379995991307e-05, "Finetune/Loss": 0.9309531450271606, "Finetune/Loss (Raw)": 0.8960142731666565, "Finetune/Step": 1066, "Finetune/Step Time": 4.581768313422799} +{"Finetune/Learning Rate": 1.8429029692615702e-05, "Finetune/Loss": 0.9323636889457703, "Finetune/Loss (Raw)": 1.1031333208084106, "Finetune/Step": 1067, "Finetune/Step Time": 4.588623933494091} +{"Finetune/Learning Rate": 1.8425676118098236e-05, "Finetune/Loss": 0.9313779473304749, "Finetune/Loss (Raw)": 0.9014807343482971, "Finetune/Step": 1068, "Finetune/Step Time": 4.592869291082025} +{"Finetune/Learning Rate": 1.8422319273740364e-05, "Finetune/Loss": 0.9312211275100708, "Finetune/Loss (Raw)": 0.9497352838516235, "Finetune/Step": 1069, "Finetune/Step Time": 4.618976349011064} +{"Finetune/Learning Rate": 1.8418959160844813e-05, "Finetune/Loss": 0.9319407939910889, "Finetune/Loss (Raw)": 1.0269864797592163, "Finetune/Step": 1070, "Finetune/Step Time": 4.607939060777426} +{"Finetune/Learning Rate": 1.8415595780715583e-05, "Finetune/Loss": 0.9325577020645142, "Finetune/Loss (Raw)": 1.034070372581482, "Finetune/Step": 1071, "Finetune/Step Time": 4.605444103479385} +{"Finetune/Learning Rate": 1.8412229134657926e-05, "Finetune/Loss": 0.9328455328941345, "Finetune/Loss (Raw)": 1.0333178043365479, "Finetune/Step": 1072, "Finetune/Step Time": 4.615265669301152} +{"Finetune/Learning Rate": 1.8408859223978378e-05, "Finetune/Loss": 0.9335535764694214, "Finetune/Loss (Raw)": 0.9517958164215088, "Finetune/Step": 1073, "Finetune/Step Time": 4.598358877003193} +{"Finetune/Learning Rate": 1.840548604998474e-05, "Finetune/Loss": 0.9346586465835571, "Finetune/Loss (Raw)": 1.037018895149231, "Finetune/Step": 1074, "Finetune/Step Time": 4.583608489483595} +{"Finetune/Learning Rate": 1.840210961398606e-05, "Finetune/Loss": 0.9351573586463928, "Finetune/Loss (Raw)": 1.0228434801101685, "Finetune/Step": 1075, "Finetune/Step Time": 4.578282928094268} +{"Finetune/Learning Rate": 1.8398729917292683e-05, "Finetune/Loss": 0.940475344657898, "Finetune/Loss (Raw)": 1.0677545070648193, "Finetune/Step": 1076, "Finetune/Step Time": 4.53948394022882} +{"Finetune/Learning Rate": 1.8395346961216193e-05, "Finetune/Loss": 0.9400308728218079, "Finetune/Loss (Raw)": 1.078249216079712, "Finetune/Step": 1077, "Finetune/Step Time": 4.54358197748661} +{"Finetune/Learning Rate": 1.8391960747069453e-05, "Finetune/Loss": 0.9412478804588318, "Finetune/Loss (Raw)": 1.0585051774978638, "Finetune/Step": 1078, "Finetune/Step Time": 4.551411075517535} +{"Finetune/Learning Rate": 1.8388571276166588e-05, "Finetune/Loss": 0.9386475086212158, "Finetune/Loss (Raw)": 0.848517894744873, "Finetune/Step": 1079, "Finetune/Step Time": 4.5555885918438435} +{"Finetune/Learning Rate": 1.8385178549822982e-05, "Finetune/Loss": 0.9394910335540771, "Finetune/Loss (Raw)": 0.9558610916137695, "Finetune/Step": 1080, "Finetune/Step Time": 4.555258354172111} +{"Finetune/Learning Rate": 1.8381782569355288e-05, "Finetune/Loss": 0.9402430057525635, "Finetune/Loss (Raw)": 1.0782276391983032, "Finetune/Step": 1081, "Finetune/Step Time": 4.558974338695407} +{"Finetune/Learning Rate": 1.837838333608142e-05, "Finetune/Loss": 0.9396643042564392, "Finetune/Loss (Raw)": 0.8410096764564514, "Finetune/Step": 1082, "Finetune/Step Time": 4.579552156850696} +{"Finetune/Learning Rate": 1.8374980851320555e-05, "Finetune/Loss": 0.9399269819259644, "Finetune/Loss (Raw)": 1.020101547241211, "Finetune/Step": 1083, "Finetune/Step Time": 4.573117656633258} +{"Finetune/Learning Rate": 1.8371575116393125e-05, "Finetune/Loss": 0.9401316046714783, "Finetune/Loss (Raw)": 1.0514940023422241, "Finetune/Step": 1084, "Finetune/Step Time": 4.587802125141025} +{"Finetune/Learning Rate": 1.8368166132620838e-05, "Finetune/Loss": 0.9409469962120056, "Finetune/Loss (Raw)": 1.0060362815856934, "Finetune/Step": 1085, "Finetune/Step Time": 4.577496709302068} +{"Finetune/Learning Rate": 1.8364753901326647e-05, "Finetune/Loss": 0.9413338899612427, "Finetune/Loss (Raw)": 1.0319793224334717, "Finetune/Step": 1086, "Finetune/Step Time": 4.564124848693609} +{"Finetune/Learning Rate": 1.8361338423834776e-05, "Finetune/Loss": 0.9419389367103577, "Finetune/Loss (Raw)": 1.0046026706695557, "Finetune/Step": 1087, "Finetune/Step Time": 4.566862361505628} +{"Finetune/Learning Rate": 1.8357919701470702e-05, "Finetune/Loss": 0.9409794807434082, "Finetune/Loss (Raw)": 0.8278135657310486, "Finetune/Step": 1088, "Finetune/Step Time": 4.574165606871247} +{"Finetune/Learning Rate": 1.835449773556117e-05, "Finetune/Loss": 0.9418901801109314, "Finetune/Loss (Raw)": 0.9989691376686096, "Finetune/Step": 1089, "Finetune/Step Time": 4.570249013602734} +{"Finetune/Learning Rate": 1.8351072527434173e-05, "Finetune/Loss": 0.941877007484436, "Finetune/Loss (Raw)": 1.098859429359436, "Finetune/Step": 1090, "Finetune/Step Time": 4.580260835587978} +{"Finetune/Learning Rate": 1.8347644078418968e-05, "Finetune/Loss": 0.941990852355957, "Finetune/Loss (Raw)": 0.9198853373527527, "Finetune/Step": 1091, "Finetune/Step Time": 4.57895078510046} +{"Finetune/Learning Rate": 1.834421238984607e-05, "Finetune/Loss": 0.9428496956825256, "Finetune/Loss (Raw)": 1.0404325723648071, "Finetune/Step": 1092, "Finetune/Step Time": 4.572455272078514} +{"Finetune/Learning Rate": 1.8340777463047253e-05, "Finetune/Loss": 0.9433985948562622, "Finetune/Loss (Raw)": 0.8724862933158875, "Finetune/Step": 1093, "Finetune/Step Time": 4.5698720663785934} +{"Finetune/Learning Rate": 1.833733929935554e-05, "Finetune/Loss": 0.9437240958213806, "Finetune/Loss (Raw)": 0.9813175201416016, "Finetune/Step": 1094, "Finetune/Step Time": 4.557864189147949} +{"Finetune/Learning Rate": 1.833389790010522e-05, "Finetune/Loss": 0.9440734386444092, "Finetune/Loss (Raw)": 1.0317652225494385, "Finetune/Step": 1095, "Finetune/Step Time": 4.5413813181221485} +{"Finetune/Learning Rate": 1.8330453266631828e-05, "Finetune/Loss": 0.9440600872039795, "Finetune/Loss (Raw)": 1.0545828342437744, "Finetune/Step": 1096, "Finetune/Step Time": 4.543521903455257} +{"Finetune/Learning Rate": 1.8327005400272165e-05, "Finetune/Loss": 0.9444576501846313, "Finetune/Loss (Raw)": 0.9704982042312622, "Finetune/Step": 1097, "Finetune/Step Time": 4.5332379546016455} +{"Finetune/Learning Rate": 1.8323554302364273e-05, "Finetune/Loss": 0.9466080069541931, "Finetune/Loss (Raw)": 0.9746704697608948, "Finetune/Step": 1098, "Finetune/Step Time": 4.546231212094426} +{"Finetune/Learning Rate": 1.8320099974247462e-05, "Finetune/Loss": 0.9465097188949585, "Finetune/Loss (Raw)": 0.9896877408027649, "Finetune/Step": 1099, "Finetune/Step Time": 4.518924444913864} +{"Finetune/Learning Rate": 1.831664241726229e-05, "Finetune/Loss": 0.9508984088897705, "Finetune/Loss (Raw)": 0.9528573751449585, "Finetune/Step": 1100, "Finetune/Step Time": 4.4911610670387745} +{"Finetune/Learning Rate": 1.831318163275056e-05, "Finetune/Loss": 0.9524534940719604, "Finetune/Loss (Raw)": 1.0865232944488525, "Finetune/Step": 1101, "Finetune/Step Time": 4.491330858319998} +{"Finetune/Learning Rate": 1.8309717622055343e-05, "Finetune/Loss": 0.9511421918869019, "Finetune/Loss (Raw)": 0.7988476753234863, "Finetune/Step": 1102, "Finetune/Step Time": 4.472992220893502} +{"Finetune/Learning Rate": 1.830625038652095e-05, "Finetune/Loss": 0.9513589143753052, "Finetune/Loss (Raw)": 1.1284277439117432, "Finetune/Step": 1103, "Finetune/Step Time": 4.4598720502108335} +{"Finetune/Learning Rate": 1.8302779927492945e-05, "Finetune/Loss": 0.9519641399383545, "Finetune/Loss (Raw)": 1.0619746446609497, "Finetune/Step": 1104, "Finetune/Step Time": 4.4697881899774075} +{"Finetune/Learning Rate": 1.829930624631815e-05, "Finetune/Loss": 0.9521717429161072, "Finetune/Loss (Raw)": 1.0913084745407104, "Finetune/Step": 1105, "Finetune/Step Time": 4.480084393173456} +{"Finetune/Learning Rate": 1.829582934434463e-05, "Finetune/Loss": 0.9523251056671143, "Finetune/Loss (Raw)": 0.9656454920768738, "Finetune/Step": 1106, "Finetune/Step Time": 4.49246877618134} +{"Finetune/Learning Rate": 1.829234922292171e-05, "Finetune/Loss": 0.9533860087394714, "Finetune/Loss (Raw)": 1.0172218084335327, "Finetune/Step": 1107, "Finetune/Step Time": 4.4870397839695215} +{"Finetune/Learning Rate": 1.8288865883399946e-05, "Finetune/Loss": 0.9527617692947388, "Finetune/Loss (Raw)": 1.0810617208480835, "Finetune/Step": 1108, "Finetune/Step Time": 4.487622300162911} +{"Finetune/Learning Rate": 1.8285379327131157e-05, "Finetune/Loss": 0.9513572454452515, "Finetune/Loss (Raw)": 0.8077906370162964, "Finetune/Step": 1109, "Finetune/Step Time": 4.49924541823566} +{"Finetune/Learning Rate": 1.828188955546841e-05, "Finetune/Loss": 0.9509704113006592, "Finetune/Loss (Raw)": 0.9816428422927856, "Finetune/Step": 1110, "Finetune/Step Time": 4.49821219407022} +{"Finetune/Learning Rate": 1.827839656976602e-05, "Finetune/Loss": 0.9516958594322205, "Finetune/Loss (Raw)": 1.1188517808914185, "Finetune/Step": 1111, "Finetune/Step Time": 4.487671975046396} +{"Finetune/Learning Rate": 1.8274900371379543e-05, "Finetune/Loss": 0.9512239694595337, "Finetune/Loss (Raw)": 0.9097208380699158, "Finetune/Step": 1112, "Finetune/Step Time": 4.47601156309247} +{"Finetune/Learning Rate": 1.8271400961665784e-05, "Finetune/Loss": 0.9511584043502808, "Finetune/Loss (Raw)": 0.9402462840080261, "Finetune/Step": 1113, "Finetune/Step Time": 4.472478978335857} +{"Finetune/Learning Rate": 1.8267898341982796e-05, "Finetune/Loss": 0.9520334601402283, "Finetune/Loss (Raw)": 0.9909204244613647, "Finetune/Step": 1114, "Finetune/Step Time": 4.500713644549251} +{"Finetune/Learning Rate": 1.8264392513689882e-05, "Finetune/Loss": 0.9520921111106873, "Finetune/Loss (Raw)": 1.0100953578948975, "Finetune/Step": 1115, "Finetune/Step Time": 4.497652970254421} +{"Finetune/Learning Rate": 1.8260883478147578e-05, "Finetune/Loss": 0.9537948369979858, "Finetune/Loss (Raw)": 1.0790835618972778, "Finetune/Step": 1116, "Finetune/Step Time": 4.492342848330736} +{"Finetune/Learning Rate": 1.8257371236717682e-05, "Finetune/Loss": 0.9525585174560547, "Finetune/Loss (Raw)": 0.875228226184845, "Finetune/Step": 1117, "Finetune/Step Time": 4.491657694801688} +{"Finetune/Learning Rate": 1.8253855790763215e-05, "Finetune/Loss": 0.9550789594650269, "Finetune/Loss (Raw)": 1.0720419883728027, "Finetune/Step": 1118, "Finetune/Step Time": 4.484229881316423} +{"Finetune/Learning Rate": 1.8250337141648464e-05, "Finetune/Loss": 0.9525061249732971, "Finetune/Loss (Raw)": 0.711705207824707, "Finetune/Step": 1119, "Finetune/Step Time": 4.494160741567612} +{"Finetune/Learning Rate": 1.8246815290738937e-05, "Finetune/Loss": 0.9523444175720215, "Finetune/Loss (Raw)": 0.9559281468391418, "Finetune/Step": 1120, "Finetune/Step Time": 4.49710214138031} +{"Finetune/Learning Rate": 1.8243290239401402e-05, "Finetune/Loss": 0.9535626173019409, "Finetune/Loss (Raw)": 1.1414742469787598, "Finetune/Step": 1121, "Finetune/Step Time": 4.466242667287588} +{"Finetune/Learning Rate": 1.8239761989003866e-05, "Finetune/Loss": 0.9555352926254272, "Finetune/Loss (Raw)": 1.0244503021240234, "Finetune/Step": 1122, "Finetune/Step Time": 4.47264339029789} +{"Finetune/Learning Rate": 1.8236230540915567e-05, "Finetune/Loss": 0.9553674459457397, "Finetune/Loss (Raw)": 0.9186370968818665, "Finetune/Step": 1123, "Finetune/Step Time": 4.490021919831634} +{"Finetune/Learning Rate": 1.8232695896506995e-05, "Finetune/Loss": 0.9573418498039246, "Finetune/Loss (Raw)": 1.2166643142700195, "Finetune/Step": 1124, "Finetune/Step Time": 4.486074544489384} +{"Finetune/Learning Rate": 1.8229158057149875e-05, "Finetune/Loss": 0.9572442770004272, "Finetune/Loss (Raw)": 0.9258862733840942, "Finetune/Step": 1125, "Finetune/Step Time": 4.482905050739646} +{"Finetune/Learning Rate": 1.8225617024217174e-05, "Finetune/Loss": 0.9573792815208435, "Finetune/Loss (Raw)": 0.9655903577804565, "Finetune/Step": 1126, "Finetune/Step Time": 4.465242998674512} +{"Finetune/Learning Rate": 1.8222072799083096e-05, "Finetune/Loss": 0.9565293788909912, "Finetune/Loss (Raw)": 1.0069541931152344, "Finetune/Step": 1127, "Finetune/Step Time": 4.464397311210632} +{"Finetune/Learning Rate": 1.821852538312309e-05, "Finetune/Loss": 0.9549578428268433, "Finetune/Loss (Raw)": 0.9401440620422363, "Finetune/Step": 1128, "Finetune/Step Time": 4.476427067071199} +{"Finetune/Learning Rate": 1.8214974777713834e-05, "Finetune/Loss": 0.9609357714653015, "Finetune/Loss (Raw)": 1.0893968343734741, "Finetune/Step": 1129, "Finetune/Step Time": 4.432116074487567} +{"Finetune/Learning Rate": 1.821142098423325e-05, "Finetune/Loss": 0.9593322277069092, "Finetune/Loss (Raw)": 0.7803831696510315, "Finetune/Step": 1130, "Finetune/Step Time": 4.440212696790695} +{"Finetune/Learning Rate": 1.8207864004060497e-05, "Finetune/Loss": 0.959457278251648, "Finetune/Loss (Raw)": 1.0683504343032837, "Finetune/Step": 1131, "Finetune/Step Time": 4.422586631029844} +{"Finetune/Learning Rate": 1.820430383857597e-05, "Finetune/Loss": 0.9591033458709717, "Finetune/Loss (Raw)": 1.012268304824829, "Finetune/Step": 1132, "Finetune/Step Time": 4.424547456204891} +{"Finetune/Learning Rate": 1.8200740489161296e-05, "Finetune/Loss": 0.958676815032959, "Finetune/Loss (Raw)": 0.9661984443664551, "Finetune/Step": 1133, "Finetune/Step Time": 4.419782334938645} +{"Finetune/Learning Rate": 1.819717395719935e-05, "Finetune/Loss": 0.9592933654785156, "Finetune/Loss (Raw)": 0.944516122341156, "Finetune/Step": 1134, "Finetune/Step Time": 4.401645787060261} +{"Finetune/Learning Rate": 1.819360424407422e-05, "Finetune/Loss": 0.960419774055481, "Finetune/Loss (Raw)": 1.076956033706665, "Finetune/Step": 1135, "Finetune/Step Time": 4.396812383085489} +{"Finetune/Learning Rate": 1.8190031351171256e-05, "Finetune/Loss": 0.960153341293335, "Finetune/Loss (Raw)": 0.9465481042861938, "Finetune/Step": 1136, "Finetune/Step Time": 4.3939262721687555} +{"Finetune/Learning Rate": 1.818645527987702e-05, "Finetune/Loss": 0.9649293422698975, "Finetune/Loss (Raw)": 1.0374352931976318, "Finetune/Step": 1137, "Finetune/Step Time": 4.355982897803187} +{"Finetune/Learning Rate": 1.8182876031579312e-05, "Finetune/Loss": 0.96567302942276, "Finetune/Loss (Raw)": 0.9953241348266602, "Finetune/Step": 1138, "Finetune/Step Time": 4.364825796335936} +{"Finetune/Learning Rate": 1.8179293607667177e-05, "Finetune/Loss": 0.9659314751625061, "Finetune/Loss (Raw)": 0.9479022026062012, "Finetune/Step": 1139, "Finetune/Step Time": 4.378314094617963} +{"Finetune/Learning Rate": 1.817570800953088e-05, "Finetune/Loss": 0.9659487009048462, "Finetune/Loss (Raw)": 1.0073533058166504, "Finetune/Step": 1140, "Finetune/Step Time": 4.370739948004484} +{"Finetune/Learning Rate": 1.8172119238561924e-05, "Finetune/Loss": 0.9661502838134766, "Finetune/Loss (Raw)": 0.9350208044052124, "Finetune/Step": 1141, "Finetune/Step Time": 4.376829018816352} +{"Finetune/Learning Rate": 1.8168527296153037e-05, "Finetune/Loss": 0.9604554772377014, "Finetune/Loss (Raw)": 0.3087318241596222, "Finetune/Step": 1142, "Finetune/Step Time": 4.4045517686754465} +{"Finetune/Learning Rate": 1.8164932183698184e-05, "Finetune/Loss": 0.9652681350708008, "Finetune/Loss (Raw)": 0.9317333102226257, "Finetune/Step": 1143, "Finetune/Step Time": 4.377798601984978} +{"Finetune/Learning Rate": 1.8161333902592554e-05, "Finetune/Loss": 0.9652180671691895, "Finetune/Loss (Raw)": 1.0037938356399536, "Finetune/Step": 1144, "Finetune/Step Time": 4.382139630615711} +{"Finetune/Learning Rate": 1.8157732454232577e-05, "Finetune/Loss": 0.9642552137374878, "Finetune/Loss (Raw)": 0.9440893530845642, "Finetune/Step": 1145, "Finetune/Step Time": 4.382659552618861} +{"Finetune/Learning Rate": 1.81541278400159e-05, "Finetune/Loss": 0.9646902084350586, "Finetune/Loss (Raw)": 1.0438623428344727, "Finetune/Step": 1146, "Finetune/Step Time": 4.352884773164988} +{"Finetune/Learning Rate": 1.81505200613414e-05, "Finetune/Loss": 0.9636874198913574, "Finetune/Loss (Raw)": 0.793057918548584, "Finetune/Step": 1147, "Finetune/Step Time": 4.355945525690913} +{"Finetune/Learning Rate": 1.8146909119609196e-05, "Finetune/Loss": 0.9587227702140808, "Finetune/Loss (Raw)": 0.35113346576690674, "Finetune/Step": 1148, "Finetune/Step Time": 4.402056340128183} +{"Finetune/Learning Rate": 1.8143295016220612e-05, "Finetune/Loss": 0.9606455564498901, "Finetune/Loss (Raw)": 1.1000791788101196, "Finetune/Step": 1149, "Finetune/Step Time": 4.402527766302228} +{"Finetune/Learning Rate": 1.813967775257822e-05, "Finetune/Loss": 0.9602952599525452, "Finetune/Loss (Raw)": 0.9647439122200012, "Finetune/Step": 1150, "Finetune/Step Time": 4.392914669588208} +{"Finetune/Learning Rate": 1.8136057330085805e-05, "Finetune/Loss": 0.9605056047439575, "Finetune/Loss (Raw)": 0.8689756989479065, "Finetune/Step": 1151, "Finetune/Step Time": 4.377486847341061} +{"Finetune/Learning Rate": 1.8132433750148383e-05, "Finetune/Loss": 0.9609977006912231, "Finetune/Loss (Raw)": 0.9762839078903198, "Finetune/Step": 1152, "Finetune/Step Time": 4.395020795986056} +{"Finetune/Learning Rate": 1.8128807014172197e-05, "Finetune/Loss": 0.962527871131897, "Finetune/Loss (Raw)": 1.0780494213104248, "Finetune/Step": 1153, "Finetune/Step Time": 4.40845013409853} +{"Finetune/Learning Rate": 1.812517712356471e-05, "Finetune/Loss": 0.9572493433952332, "Finetune/Loss (Raw)": 0.3357580602169037, "Finetune/Step": 1154, "Finetune/Step Time": 4.439021399244666} +{"Finetune/Learning Rate": 1.812154407973461e-05, "Finetune/Loss": 0.9570257663726807, "Finetune/Loss (Raw)": 0.34637534618377686, "Finetune/Step": 1155, "Finetune/Step Time": 4.436802322044969} +{"Finetune/Learning Rate": 1.8117907884091815e-05, "Finetune/Loss": 0.9577820897102356, "Finetune/Loss (Raw)": 1.1341438293457031, "Finetune/Step": 1156, "Finetune/Step Time": 4.427134266123176} +{"Finetune/Learning Rate": 1.8114268538047457e-05, "Finetune/Loss": 0.9589981436729431, "Finetune/Loss (Raw)": 1.0263395309448242, "Finetune/Step": 1157, "Finetune/Step Time": 4.41941093467176} +{"Finetune/Learning Rate": 1.81106260430139e-05, "Finetune/Loss": 0.9582825899124146, "Finetune/Loss (Raw)": 0.913078784942627, "Finetune/Step": 1158, "Finetune/Step Time": 4.416466146707535} +{"Finetune/Learning Rate": 1.810698040040472e-05, "Finetune/Loss": 0.9571975469589233, "Finetune/Loss (Raw)": 0.8398842811584473, "Finetune/Step": 1159, "Finetune/Step Time": 4.410592801868916} +{"Finetune/Learning Rate": 1.810333161163472e-05, "Finetune/Loss": 0.9618796110153198, "Finetune/Loss (Raw)": 0.9512770175933838, "Finetune/Step": 1160, "Finetune/Step Time": 4.387525068596005} +{"Finetune/Learning Rate": 1.809967967811993e-05, "Finetune/Loss": 0.9616411924362183, "Finetune/Loss (Raw)": 0.9666551351547241, "Finetune/Step": 1161, "Finetune/Step Time": 4.368657678365707} +{"Finetune/Learning Rate": 1.8096024601277586e-05, "Finetune/Loss": 0.9629492163658142, "Finetune/Loss (Raw)": 1.009006381034851, "Finetune/Step": 1162, "Finetune/Step Time": 4.371564136818051} +{"Finetune/Learning Rate": 1.8092366382526157e-05, "Finetune/Loss": 0.9611905813217163, "Finetune/Loss (Raw)": 0.7710506916046143, "Finetune/Step": 1163, "Finetune/Step Time": 4.385587129741907} +{"Finetune/Learning Rate": 1.8088705023285324e-05, "Finetune/Loss": 0.9629570245742798, "Finetune/Loss (Raw)": 1.1842774152755737, "Finetune/Step": 1164, "Finetune/Step Time": 4.383570173755288} +{"Finetune/Learning Rate": 1.8085040524975994e-05, "Finetune/Loss": 0.9591317176818848, "Finetune/Loss (Raw)": 0.3715651035308838, "Finetune/Step": 1165, "Finetune/Step Time": 4.411222351714969} +{"Finetune/Learning Rate": 1.8081372889020282e-05, "Finetune/Loss": 0.9568618535995483, "Finetune/Loss (Raw)": 0.8721286654472351, "Finetune/Step": 1166, "Finetune/Step Time": 4.425516145303845} +{"Finetune/Learning Rate": 1.8077702116841524e-05, "Finetune/Loss": 0.9554365277290344, "Finetune/Loss (Raw)": 0.8228216171264648, "Finetune/Step": 1167, "Finetune/Step Time": 4.4401952009648085} +{"Finetune/Learning Rate": 1.807402820986428e-05, "Finetune/Loss": 0.9554762840270996, "Finetune/Loss (Raw)": 0.9318683743476868, "Finetune/Step": 1168, "Finetune/Step Time": 4.4387799352407455} +{"Finetune/Learning Rate": 1.8070351169514324e-05, "Finetune/Loss": 0.9548941850662231, "Finetune/Loss (Raw)": 0.9973222017288208, "Finetune/Step": 1169, "Finetune/Step Time": 4.440609758719802} +{"Finetune/Learning Rate": 1.8066670997218633e-05, "Finetune/Loss": 0.9501678347587585, "Finetune/Loss (Raw)": 0.4750840663909912, "Finetune/Step": 1170, "Finetune/Step Time": 4.475321393460035} +{"Finetune/Learning Rate": 1.806298769440542e-05, "Finetune/Loss": 0.9558653235435486, "Finetune/Loss (Raw)": 1.112858533859253, "Finetune/Step": 1171, "Finetune/Step Time": 4.426632083952427} +{"Finetune/Learning Rate": 1.8059301262504097e-05, "Finetune/Loss": 0.957306981086731, "Finetune/Loss (Raw)": 1.0311458110809326, "Finetune/Step": 1172, "Finetune/Step Time": 4.42672948166728} +{"Finetune/Learning Rate": 1.80556117029453e-05, "Finetune/Loss": 0.9568591117858887, "Finetune/Loss (Raw)": 0.9758182764053345, "Finetune/Step": 1173, "Finetune/Step Time": 4.405594704672694} +{"Finetune/Learning Rate": 1.8051919017160875e-05, "Finetune/Loss": 0.9523990154266357, "Finetune/Loss (Raw)": 0.36797773838043213, "Finetune/Step": 1174, "Finetune/Step Time": 4.436163958162069} +{"Finetune/Learning Rate": 1.804822320658388e-05, "Finetune/Loss": 0.9519569873809814, "Finetune/Loss (Raw)": 0.938323974609375, "Finetune/Step": 1175, "Finetune/Step Time": 4.450057672336698} +{"Finetune/Learning Rate": 1.8044524272648588e-05, "Finetune/Loss": 0.9499939680099487, "Finetune/Loss (Raw)": 0.9072605967521667, "Finetune/Step": 1176, "Finetune/Step Time": 4.439843628555536} +{"Finetune/Learning Rate": 1.8040822216790475e-05, "Finetune/Loss": 0.9455633163452148, "Finetune/Loss (Raw)": 0.2939327359199524, "Finetune/Step": 1177, "Finetune/Step Time": 4.481153268367052} +{"Finetune/Learning Rate": 1.803711704044625e-05, "Finetune/Loss": 0.941837728023529, "Finetune/Loss (Raw)": 0.32816582918167114, "Finetune/Step": 1178, "Finetune/Step Time": 4.505636418238282} +{"Finetune/Learning Rate": 1.803340874505381e-05, "Finetune/Loss": 0.943432629108429, "Finetune/Loss (Raw)": 1.1426347494125366, "Finetune/Step": 1179, "Finetune/Step Time": 4.498704068362713} +{"Finetune/Learning Rate": 1.8029697332052277e-05, "Finetune/Loss": 0.941508412361145, "Finetune/Loss (Raw)": 0.9012312889099121, "Finetune/Step": 1180, "Finetune/Step Time": 4.515059122815728} +{"Finetune/Learning Rate": 1.8025982802881976e-05, "Finetune/Loss": 0.9423357248306274, "Finetune/Loss (Raw)": 1.0760531425476074, "Finetune/Step": 1181, "Finetune/Step Time": 4.5086233373731375} +{"Finetune/Learning Rate": 1.802226515898444e-05, "Finetune/Loss": 0.9391385316848755, "Finetune/Loss (Raw)": 0.6966975331306458, "Finetune/Step": 1182, "Finetune/Step Time": 4.510121911764145} +{"Finetune/Learning Rate": 1.8018544401802422e-05, "Finetune/Loss": 0.9401378035545349, "Finetune/Loss (Raw)": 1.0226317644119263, "Finetune/Step": 1183, "Finetune/Step Time": 4.511639516800642} +{"Finetune/Learning Rate": 1.801482053277987e-05, "Finetune/Loss": 0.9379984140396118, "Finetune/Loss (Raw)": 0.812393069267273, "Finetune/Step": 1184, "Finetune/Step Time": 4.540659697726369} +{"Finetune/Learning Rate": 1.8011093553361944e-05, "Finetune/Loss": 0.9387339353561401, "Finetune/Loss (Raw)": 0.99880450963974, "Finetune/Step": 1185, "Finetune/Step Time": 4.5435554794967175} +{"Finetune/Learning Rate": 1.8007363464995015e-05, "Finetune/Loss": 0.940162181854248, "Finetune/Loss (Raw)": 1.1016714572906494, "Finetune/Step": 1186, "Finetune/Step Time": 4.549204621464014} +{"Finetune/Learning Rate": 1.800363026912665e-05, "Finetune/Loss": 0.9356239438056946, "Finetune/Loss (Raw)": 0.3534848988056183, "Finetune/Step": 1187, "Finetune/Step Time": 4.573887702077627} +{"Finetune/Learning Rate": 1.7999893967205643e-05, "Finetune/Loss": 0.936301589012146, "Finetune/Loss (Raw)": 1.0470541715621948, "Finetune/Step": 1188, "Finetune/Step Time": 4.573055509477854} +{"Finetune/Learning Rate": 1.7996154560681963e-05, "Finetune/Loss": 0.9380970001220703, "Finetune/Loss (Raw)": 1.0521595478057861, "Finetune/Step": 1189, "Finetune/Step Time": 4.578242123126984} +{"Finetune/Learning Rate": 1.7992412051006817e-05, "Finetune/Loss": 0.9379104375839233, "Finetune/Loss (Raw)": 0.964643120765686, "Finetune/Step": 1190, "Finetune/Step Time": 4.5761743523180485} +{"Finetune/Learning Rate": 1.7988666439632593e-05, "Finetune/Loss": 0.9385008811950684, "Finetune/Loss (Raw)": 1.0453016757965088, "Finetune/Step": 1191, "Finetune/Step Time": 4.5695930570364} +{"Finetune/Learning Rate": 1.7984917728012886e-05, "Finetune/Loss": 0.9372345805168152, "Finetune/Loss (Raw)": 1.0172393321990967, "Finetune/Step": 1192, "Finetune/Step Time": 4.577194627374411} +{"Finetune/Learning Rate": 1.7981165917602502e-05, "Finetune/Loss": 0.9363724589347839, "Finetune/Loss (Raw)": 1.0107392072677612, "Finetune/Step": 1193, "Finetune/Step Time": 4.5762513764202595} +{"Finetune/Learning Rate": 1.7977411009857446e-05, "Finetune/Loss": 0.937437117099762, "Finetune/Loss (Raw)": 1.0322915315628052, "Finetune/Step": 1194, "Finetune/Step Time": 4.590079473331571} +{"Finetune/Learning Rate": 1.797365300623492e-05, "Finetune/Loss": 0.936455249786377, "Finetune/Loss (Raw)": 0.9774546027183533, "Finetune/Step": 1195, "Finetune/Step Time": 4.5800518952310085} +{"Finetune/Learning Rate": 1.7969891908193338e-05, "Finetune/Loss": 0.936988353729248, "Finetune/Loss (Raw)": 0.9697220325469971, "Finetune/Step": 1196, "Finetune/Step Time": 4.576917873695493} +{"Finetune/Learning Rate": 1.7966127717192305e-05, "Finetune/Loss": 0.9380173683166504, "Finetune/Loss (Raw)": 1.0814390182495117, "Finetune/Step": 1197, "Finetune/Step Time": 4.544191975146532} +{"Finetune/Learning Rate": 1.796236043469263e-05, "Finetune/Loss": 0.9384362697601318, "Finetune/Loss (Raw)": 1.080605149269104, "Finetune/Step": 1198, "Finetune/Step Time": 4.5497473385185} +{"Finetune/Learning Rate": 1.7958590062156318e-05, "Finetune/Loss": 0.9383819103240967, "Finetune/Loss (Raw)": 1.0271157026290894, "Finetune/Step": 1199, "Finetune/Step Time": 4.545539123937488} +{"Finetune/Learning Rate": 1.7954816601046587e-05, "Finetune/Loss": 0.9384081959724426, "Finetune/Loss (Raw)": 1.036679983139038, "Finetune/Step": 1200, "Finetune/Step Time": 4.5417814925313} +{"Finetune/Learning Rate": 1.7951040052827836e-05, "Finetune/Loss": 0.9390488862991333, "Finetune/Loss (Raw)": 1.0338085889816284, "Finetune/Step": 1201, "Finetune/Step Time": 4.555092863738537} +{"Finetune/Learning Rate": 1.794726041896567e-05, "Finetune/Loss": 0.9399081468582153, "Finetune/Loss (Raw)": 1.1470032930374146, "Finetune/Step": 1202, "Finetune/Step Time": 4.5592785608023405} +{"Finetune/Learning Rate": 1.7943477700926895e-05, "Finetune/Loss": 0.9406275749206543, "Finetune/Loss (Raw)": 1.1149264574050903, "Finetune/Step": 1203, "Finetune/Step Time": 4.55023742467165} +{"Finetune/Learning Rate": 1.7939691900179507e-05, "Finetune/Loss": 0.9406374096870422, "Finetune/Loss (Raw)": 1.0690205097198486, "Finetune/Step": 1204, "Finetune/Step Time": 4.558643260970712} +{"Finetune/Learning Rate": 1.79359030181927e-05, "Finetune/Loss": 0.9395837783813477, "Finetune/Loss (Raw)": 0.9433789849281311, "Finetune/Step": 1205, "Finetune/Step Time": 4.575819859281182} +{"Finetune/Learning Rate": 1.7932111056436868e-05, "Finetune/Loss": 0.9401352405548096, "Finetune/Loss (Raw)": 1.1290912628173828, "Finetune/Step": 1206, "Finetune/Step Time": 4.562752800062299} +{"Finetune/Learning Rate": 1.79283160163836e-05, "Finetune/Loss": 0.9408398866653442, "Finetune/Loss (Raw)": 0.9387114644050598, "Finetune/Step": 1207, "Finetune/Step Time": 4.570111691951752} +{"Finetune/Learning Rate": 1.792451789950567e-05, "Finetune/Loss": 0.9412143230438232, "Finetune/Loss (Raw)": 1.0037944316864014, "Finetune/Step": 1208, "Finetune/Step Time": 4.571057798340917} +{"Finetune/Learning Rate": 1.7920716707277054e-05, "Finetune/Loss": 0.9409251809120178, "Finetune/Loss (Raw)": 1.041214942932129, "Finetune/Step": 1209, "Finetune/Step Time": 4.570071192458272} +{"Finetune/Learning Rate": 1.7916912441172924e-05, "Finetune/Loss": 0.9419901371002197, "Finetune/Loss (Raw)": 0.9773246049880981, "Finetune/Step": 1210, "Finetune/Step Time": 4.548093335703015} +{"Finetune/Learning Rate": 1.7913105102669643e-05, "Finetune/Loss": 0.9413022398948669, "Finetune/Loss (Raw)": 0.9320549964904785, "Finetune/Step": 1211, "Finetune/Step Time": 4.55122240819037} +{"Finetune/Learning Rate": 1.790929469324476e-05, "Finetune/Loss": 0.9417015314102173, "Finetune/Loss (Raw)": 1.102601170539856, "Finetune/Step": 1212, "Finetune/Step Time": 4.53926819190383} +{"Finetune/Learning Rate": 1.7905481214377027e-05, "Finetune/Loss": 0.9411736130714417, "Finetune/Loss (Raw)": 0.9384645819664001, "Finetune/Step": 1213, "Finetune/Step Time": 4.538599614053965} +{"Finetune/Learning Rate": 1.790166466754637e-05, "Finetune/Loss": 0.939598560333252, "Finetune/Loss (Raw)": 0.8303723931312561, "Finetune/Step": 1214, "Finetune/Step Time": 4.5577953308820724} +{"Finetune/Learning Rate": 1.7897845054233927e-05, "Finetune/Loss": 0.9394015669822693, "Finetune/Loss (Raw)": 0.9793853759765625, "Finetune/Step": 1215, "Finetune/Step Time": 4.558238564059138} +{"Finetune/Learning Rate": 1.7894022375922006e-05, "Finetune/Loss": 0.9357025623321533, "Finetune/Loss (Raw)": 0.354342520236969, "Finetune/Step": 1216, "Finetune/Step Time": 4.58809969574213} +{"Finetune/Learning Rate": 1.7890196634094122e-05, "Finetune/Loss": 0.9364618062973022, "Finetune/Loss (Raw)": 1.096152424812317, "Finetune/Step": 1217, "Finetune/Step Time": 4.591489013284445} +{"Finetune/Learning Rate": 1.7886367830234968e-05, "Finetune/Loss": 0.935955286026001, "Finetune/Loss (Raw)": 1.0340231657028198, "Finetune/Step": 1218, "Finetune/Step Time": 4.576609723269939} +{"Finetune/Learning Rate": 1.788253596583043e-05, "Finetune/Loss": 0.9311796426773071, "Finetune/Loss (Raw)": 0.3086005747318268, "Finetune/Step": 1219, "Finetune/Step Time": 4.617049586027861} +{"Finetune/Learning Rate": 1.787870104236757e-05, "Finetune/Loss": 0.9305907487869263, "Finetune/Loss (Raw)": 0.9650603532791138, "Finetune/Step": 1220, "Finetune/Step Time": 4.616236066445708} +{"Finetune/Learning Rate": 1.7874863061334658e-05, "Finetune/Loss": 0.9316730499267578, "Finetune/Loss (Raw)": 1.0110235214233398, "Finetune/Step": 1221, "Finetune/Step Time": 4.626173313707113} +{"Finetune/Learning Rate": 1.7871022024221133e-05, "Finetune/Loss": 0.931686282157898, "Finetune/Loss (Raw)": 0.9829999804496765, "Finetune/Step": 1222, "Finetune/Step Time": 4.637021210044622} +{"Finetune/Learning Rate": 1.7867177932517634e-05, "Finetune/Loss": 0.9315272569656372, "Finetune/Loss (Raw)": 1.0114080905914307, "Finetune/Step": 1223, "Finetune/Step Time": 4.638087626546621} +{"Finetune/Learning Rate": 1.786333078771597e-05, "Finetune/Loss": 0.9311056137084961, "Finetune/Loss (Raw)": 1.0006213188171387, "Finetune/Step": 1224, "Finetune/Step Time": 4.63318282738328} +{"Finetune/Learning Rate": 1.785948059130915e-05, "Finetune/Loss": 0.9302986860275269, "Finetune/Loss (Raw)": 0.8672124743461609, "Finetune/Step": 1225, "Finetune/Step Time": 4.618487725034356} +{"Finetune/Learning Rate": 1.785562734479135e-05, "Finetune/Loss": 0.9295798540115356, "Finetune/Loss (Raw)": 0.8826553225517273, "Finetune/Step": 1226, "Finetune/Step Time": 4.608258340507746} +{"Finetune/Learning Rate": 1.785177104965795e-05, "Finetune/Loss": 0.9284387826919556, "Finetune/Loss (Raw)": 0.843632161617279, "Finetune/Step": 1227, "Finetune/Step Time": 4.6163877584040165} +{"Finetune/Learning Rate": 1.7847911707405494e-05, "Finetune/Loss": 0.9289239645004272, "Finetune/Loss (Raw)": 1.0149598121643066, "Finetune/Step": 1228, "Finetune/Step Time": 4.621912460774183} +{"Finetune/Learning Rate": 1.7844049319531725e-05, "Finetune/Loss": 0.9287253618240356, "Finetune/Loss (Raw)": 1.0611014366149902, "Finetune/Step": 1229, "Finetune/Step Time": 4.617609513923526} +{"Finetune/Learning Rate": 1.7840183887535558e-05, "Finetune/Loss": 0.9256571531295776, "Finetune/Loss (Raw)": 0.4061131179332733, "Finetune/Step": 1230, "Finetune/Step Time": 4.656123602762818} +{"Finetune/Learning Rate": 1.7836315412917088e-05, "Finetune/Loss": 0.9253897070884705, "Finetune/Loss (Raw)": 1.094199538230896, "Finetune/Step": 1231, "Finetune/Step Time": 4.649240158498287} +{"Finetune/Learning Rate": 1.78324438971776e-05, "Finetune/Loss": 0.9248705506324768, "Finetune/Loss (Raw)": 0.995523989200592, "Finetune/Step": 1232, "Finetune/Step Time": 4.6427890453487635} +{"Finetune/Learning Rate": 1.782856934181955e-05, "Finetune/Loss": 0.9242435097694397, "Finetune/Loss (Raw)": 1.0110490322113037, "Finetune/Step": 1233, "Finetune/Step Time": 4.622107001021504} +{"Finetune/Learning Rate": 1.782469174834658e-05, "Finetune/Loss": 0.9229748249053955, "Finetune/Loss (Raw)": 0.8032464981079102, "Finetune/Step": 1234, "Finetune/Step Time": 4.631110934540629} +{"Finetune/Learning Rate": 1.7820811118263503e-05, "Finetune/Loss": 0.9223570823669434, "Finetune/Loss (Raw)": 0.9381557106971741, "Finetune/Step": 1235, "Finetune/Step Time": 4.629722019657493} +{"Finetune/Learning Rate": 1.7816927453076324e-05, "Finetune/Loss": 0.9223833680152893, "Finetune/Loss (Raw)": 1.084425687789917, "Finetune/Step": 1236, "Finetune/Step Time": 4.6257832236588} +{"Finetune/Learning Rate": 1.781304075429221e-05, "Finetune/Loss": 0.9240220189094543, "Finetune/Loss (Raw)": 1.0175387859344482, "Finetune/Step": 1237, "Finetune/Step Time": 4.628409354016185} +{"Finetune/Learning Rate": 1.7809151023419517e-05, "Finetune/Loss": 0.925402820110321, "Finetune/Loss (Raw)": 1.1583845615386963, "Finetune/Step": 1238, "Finetune/Step Time": 4.630675110965967} +{"Finetune/Learning Rate": 1.7805258261967774e-05, "Finetune/Loss": 0.9241851568222046, "Finetune/Loss (Raw)": 0.9629940390586853, "Finetune/Step": 1239, "Finetune/Step Time": 4.644809540361166} +{"Finetune/Learning Rate": 1.780136247144768e-05, "Finetune/Loss": 0.9244797229766846, "Finetune/Loss (Raw)": 0.9474265575408936, "Finetune/Step": 1240, "Finetune/Step Time": 4.642711695283651} +{"Finetune/Learning Rate": 1.7797463653371125e-05, "Finetune/Loss": 0.9240372180938721, "Finetune/Loss (Raw)": 0.8836091756820679, "Finetune/Step": 1241, "Finetune/Step Time": 4.646886901929975} +{"Finetune/Learning Rate": 1.7793561809251153e-05, "Finetune/Loss": 0.923773467540741, "Finetune/Loss (Raw)": 0.9571540951728821, "Finetune/Step": 1242, "Finetune/Step Time": 4.615501319989562} +{"Finetune/Learning Rate": 1.7789656940602003e-05, "Finetune/Loss": 0.9240095615386963, "Finetune/Loss (Raw)": 1.0403162240982056, "Finetune/Step": 1243, "Finetune/Step Time": 4.610073611140251} +{"Finetune/Learning Rate": 1.7785749048939076e-05, "Finetune/Loss": 0.9230841994285583, "Finetune/Loss (Raw)": 0.9606345295906067, "Finetune/Step": 1244, "Finetune/Step Time": 4.613560298457742} +{"Finetune/Learning Rate": 1.7781838135778945e-05, "Finetune/Loss": 0.9236915111541748, "Finetune/Loss (Raw)": 0.9529695510864258, "Finetune/Step": 1245, "Finetune/Step Time": 4.6037713047116995} +{"Finetune/Learning Rate": 1.777792420263936e-05, "Finetune/Loss": 0.9223329424858093, "Finetune/Loss (Raw)": 0.8981417417526245, "Finetune/Step": 1246, "Finetune/Step Time": 4.625285014510155} +{"Finetune/Learning Rate": 1.7774007251039246e-05, "Finetune/Loss": 0.9249235987663269, "Finetune/Loss (Raw)": 1.0433090925216675, "Finetune/Step": 1247, "Finetune/Step Time": 4.6107980236411095} +{"Finetune/Learning Rate": 1.777008728249869e-05, "Finetune/Loss": 0.9236575365066528, "Finetune/Loss (Raw)": 0.7938724756240845, "Finetune/Step": 1248, "Finetune/Step Time": 4.621791604906321} +{"Finetune/Learning Rate": 1.776616429853896e-05, "Finetune/Loss": 0.9217981696128845, "Finetune/Loss (Raw)": 0.903476893901825, "Finetune/Step": 1249, "Finetune/Step Time": 4.622655130922794} +{"Finetune/Learning Rate": 1.7762238300682484e-05, "Finetune/Loss": 0.9217624664306641, "Finetune/Loss (Raw)": 1.0198801755905151, "Finetune/Step": 1250, "Finetune/Step Time": 4.603222718462348} +{"Finetune/Learning Rate": 1.775830929045287e-05, "Finetune/Loss": 0.9224547743797302, "Finetune/Loss (Raw)": 1.0072484016418457, "Finetune/Step": 1251, "Finetune/Step Time": 4.5895825903862715} +{"Finetune/Learning Rate": 1.7754377269374893e-05, "Finetune/Loss": 0.919846773147583, "Finetune/Loss (Raw)": 0.8828469514846802, "Finetune/Step": 1252, "Finetune/Step Time": 4.585037039592862} +{"Finetune/Learning Rate": 1.7750442238974483e-05, "Finetune/Loss": 0.9197808504104614, "Finetune/Loss (Raw)": 0.9174392223358154, "Finetune/Step": 1253, "Finetune/Step Time": 4.603548839688301} +{"Finetune/Learning Rate": 1.7746504200778756e-05, "Finetune/Loss": 0.9198213815689087, "Finetune/Loss (Raw)": 0.9707847833633423, "Finetune/Step": 1254, "Finetune/Step Time": 4.607047628611326} +{"Finetune/Learning Rate": 1.7742563156315987e-05, "Finetune/Loss": 0.9204838871955872, "Finetune/Loss (Raw)": 1.0917526483535767, "Finetune/Step": 1255, "Finetune/Step Time": 4.60920524597168} +{"Finetune/Learning Rate": 1.773861910711562e-05, "Finetune/Loss": 0.9209887385368347, "Finetune/Loss (Raw)": 1.0047718286514282, "Finetune/Step": 1256, "Finetune/Step Time": 4.605203865095973} +{"Finetune/Learning Rate": 1.773467205470826e-05, "Finetune/Loss": 0.9178293943405151, "Finetune/Loss (Raw)": 0.6849972009658813, "Finetune/Step": 1257, "Finetune/Step Time": 4.63157126866281} +{"Finetune/Learning Rate": 1.7730722000625683e-05, "Finetune/Loss": 0.9190635085105896, "Finetune/Loss (Raw)": 0.9383445382118225, "Finetune/Step": 1258, "Finetune/Step Time": 4.61993970349431} +{"Finetune/Learning Rate": 1.7726768946400825e-05, "Finetune/Loss": 0.9176602363586426, "Finetune/Loss (Raw)": 0.8887389898300171, "Finetune/Step": 1259, "Finetune/Step Time": 4.630968144163489} +{"Finetune/Learning Rate": 1.772281289356779e-05, "Finetune/Loss": 0.918797492980957, "Finetune/Loss (Raw)": 1.157835841178894, "Finetune/Step": 1260, "Finetune/Step Time": 4.632803237065673} +{"Finetune/Learning Rate": 1.771885384366185e-05, "Finetune/Loss": 0.9177760481834412, "Finetune/Loss (Raw)": 0.8354490399360657, "Finetune/Step": 1261, "Finetune/Step Time": 4.632062928751111} +{"Finetune/Learning Rate": 1.7714891798219432e-05, "Finetune/Loss": 0.9180725812911987, "Finetune/Loss (Raw)": 0.9824753999710083, "Finetune/Step": 1262, "Finetune/Step Time": 4.627297777682543} +{"Finetune/Learning Rate": 1.7710926758778123e-05, "Finetune/Loss": 0.9177881479263306, "Finetune/Loss (Raw)": 1.0405402183532715, "Finetune/Step": 1263, "Finetune/Step Time": 4.6239017229527235} +{"Finetune/Learning Rate": 1.7706958726876685e-05, "Finetune/Loss": 0.9180331826210022, "Finetune/Loss (Raw)": 0.9779227375984192, "Finetune/Step": 1264, "Finetune/Step Time": 4.622267112135887} +{"Finetune/Learning Rate": 1.770298770405503e-05, "Finetune/Loss": 0.9129056334495544, "Finetune/Loss (Raw)": 0.3811035454273224, "Finetune/Step": 1265, "Finetune/Step Time": 4.661207480356097} +{"Finetune/Learning Rate": 1.7699013691854238e-05, "Finetune/Loss": 0.9124414920806885, "Finetune/Loss (Raw)": 0.9359155297279358, "Finetune/Step": 1266, "Finetune/Step Time": 4.666862104088068} +{"Finetune/Learning Rate": 1.769503669181654e-05, "Finetune/Loss": 0.9131481051445007, "Finetune/Loss (Raw)": 1.038347840309143, "Finetune/Step": 1267, "Finetune/Step Time": 4.652251407504082} +{"Finetune/Learning Rate": 1.7691056705485333e-05, "Finetune/Loss": 0.9135000705718994, "Finetune/Loss (Raw)": 1.0524095296859741, "Finetune/Step": 1268, "Finetune/Step Time": 4.656108044087887} +{"Finetune/Learning Rate": 1.7687073734405177e-05, "Finetune/Loss": 0.9135260581970215, "Finetune/Loss (Raw)": 0.9383376240730286, "Finetune/Step": 1269, "Finetune/Step Time": 4.647451009601355} +{"Finetune/Learning Rate": 1.768308778012178e-05, "Finetune/Loss": 0.9196600914001465, "Finetune/Loss (Raw)": 1.0938917398452759, "Finetune/Step": 1270, "Finetune/Step Time": 4.617561474442482} +{"Finetune/Learning Rate": 1.767909884418202e-05, "Finetune/Loss": 0.9208720922470093, "Finetune/Loss (Raw)": 1.0868752002716064, "Finetune/Step": 1271, "Finetune/Step Time": 4.602429391816258} +{"Finetune/Learning Rate": 1.7675106928133915e-05, "Finetune/Loss": 0.9192276000976562, "Finetune/Loss (Raw)": 0.7932980060577393, "Finetune/Step": 1272, "Finetune/Step Time": 4.610843166708946} +{"Finetune/Learning Rate": 1.7671112033526655e-05, "Finetune/Loss": 0.9191101789474487, "Finetune/Loss (Raw)": 0.9290531277656555, "Finetune/Step": 1273, "Finetune/Step Time": 4.625624909996986} +{"Finetune/Learning Rate": 1.7667114161910586e-05, "Finetune/Loss": 0.9176735877990723, "Finetune/Loss (Raw)": 0.8599735498428345, "Finetune/Step": 1274, "Finetune/Step Time": 4.611520646139979} +{"Finetune/Learning Rate": 1.7663113314837192e-05, "Finetune/Loss": 0.9204497337341309, "Finetune/Loss (Raw)": 1.1484179496765137, "Finetune/Step": 1275, "Finetune/Step Time": 4.602883171290159} +{"Finetune/Learning Rate": 1.7659109493859134e-05, "Finetune/Loss": 0.9249619245529175, "Finetune/Loss (Raw)": 0.9286868572235107, "Finetune/Step": 1276, "Finetune/Step Time": 4.558719443157315} +{"Finetune/Learning Rate": 1.7655102700530212e-05, "Finetune/Loss": 0.9238079786300659, "Finetune/Loss (Raw)": 0.9523813128471375, "Finetune/Step": 1277, "Finetune/Step Time": 4.562094150111079} +{"Finetune/Learning Rate": 1.7651092936405385e-05, "Finetune/Loss": 0.9219498634338379, "Finetune/Loss (Raw)": 0.7269002795219421, "Finetune/Step": 1278, "Finetune/Step Time": 4.5685851983726025} +{"Finetune/Learning Rate": 1.7647080203040765e-05, "Finetune/Loss": 0.9228041172027588, "Finetune/Loss (Raw)": 0.9783213138580322, "Finetune/Step": 1279, "Finetune/Step Time": 4.570070758461952} +{"Finetune/Learning Rate": 1.7643064501993614e-05, "Finetune/Loss": 0.9239845871925354, "Finetune/Loss (Raw)": 1.1273850202560425, "Finetune/Step": 1280, "Finetune/Step Time": 4.549191135913134} +{"Finetune/Learning Rate": 1.763904583482235e-05, "Finetune/Loss": 0.9229506254196167, "Finetune/Loss (Raw)": 0.9456982016563416, "Finetune/Step": 1281, "Finetune/Step Time": 4.533769654110074} +{"Finetune/Learning Rate": 1.7635024203086532e-05, "Finetune/Loss": 0.9279659986495972, "Finetune/Loss (Raw)": 0.9777226448059082, "Finetune/Step": 1282, "Finetune/Step Time": 4.496168687939644} +{"Finetune/Learning Rate": 1.7630999608346888e-05, "Finetune/Loss": 0.9318508505821228, "Finetune/Loss (Raw)": 0.8436422944068909, "Finetune/Step": 1283, "Finetune/Step Time": 4.46942226588726} +{"Finetune/Learning Rate": 1.7626972052165275e-05, "Finetune/Loss": 0.9304285049438477, "Finetune/Loss (Raw)": 0.9520754814147949, "Finetune/Step": 1284, "Finetune/Step Time": 4.472499307245016} +{"Finetune/Learning Rate": 1.7622941536104715e-05, "Finetune/Loss": 0.9306296706199646, "Finetune/Loss (Raw)": 1.0520907640457153, "Finetune/Step": 1285, "Finetune/Step Time": 4.472554398700595} +{"Finetune/Learning Rate": 1.7618908061729373e-05, "Finetune/Loss": 0.9308526515960693, "Finetune/Loss (Raw)": 0.9416278004646301, "Finetune/Step": 1286, "Finetune/Step Time": 4.481784211471677} +{"Finetune/Learning Rate": 1.7614871630604554e-05, "Finetune/Loss": 0.9318349957466125, "Finetune/Loss (Raw)": 0.9656216502189636, "Finetune/Step": 1287, "Finetune/Step Time": 4.481377860531211} +{"Finetune/Learning Rate": 1.761083224429672e-05, "Finetune/Loss": 0.9314374923706055, "Finetune/Loss (Raw)": 0.9003929495811462, "Finetune/Step": 1288, "Finetune/Step Time": 4.462038170546293} +{"Finetune/Learning Rate": 1.760678990437349e-05, "Finetune/Loss": 0.9308546781539917, "Finetune/Loss (Raw)": 0.8920530080795288, "Finetune/Step": 1289, "Finetune/Step Time": 4.4469181299209595} +{"Finetune/Learning Rate": 1.76027446124036e-05, "Finetune/Loss": 0.9314954280853271, "Finetune/Loss (Raw)": 1.091018795967102, "Finetune/Step": 1290, "Finetune/Step Time": 4.441922556608915} +{"Finetune/Learning Rate": 1.759869636995696e-05, "Finetune/Loss": 0.933037281036377, "Finetune/Loss (Raw)": 0.9684123992919922, "Finetune/Step": 1291, "Finetune/Step Time": 4.433412650600076} +{"Finetune/Learning Rate": 1.7594645178604614e-05, "Finetune/Loss": 0.9311087131500244, "Finetune/Loss (Raw)": 0.9374253749847412, "Finetune/Step": 1292, "Finetune/Step Time": 4.4279562048614025} +{"Finetune/Learning Rate": 1.7590591039918744e-05, "Finetune/Loss": 0.9355229735374451, "Finetune/Loss (Raw)": 0.9365954995155334, "Finetune/Step": 1293, "Finetune/Step Time": 4.39467828348279} +{"Finetune/Learning Rate": 1.7586533955472686e-05, "Finetune/Loss": 0.936340868473053, "Finetune/Loss (Raw)": 0.9768127799034119, "Finetune/Step": 1294, "Finetune/Step Time": 4.391850292682648} +{"Finetune/Learning Rate": 1.7582473926840916e-05, "Finetune/Loss": 0.9370336532592773, "Finetune/Loss (Raw)": 0.9115003943443298, "Finetune/Step": 1295, "Finetune/Step Time": 4.390514396131039} +{"Finetune/Learning Rate": 1.7578410955599046e-05, "Finetune/Loss": 0.9370716214179993, "Finetune/Loss (Raw)": 0.9367283582687378, "Finetune/Step": 1296, "Finetune/Step Time": 4.394412325695157} +{"Finetune/Learning Rate": 1.7574345043323843e-05, "Finetune/Loss": 0.9368099570274353, "Finetune/Loss (Raw)": 0.9638258814811707, "Finetune/Step": 1297, "Finetune/Step Time": 4.408995261415839} +{"Finetune/Learning Rate": 1.7570276191593202e-05, "Finetune/Loss": 0.9395840764045715, "Finetune/Loss (Raw)": 0.8301737308502197, "Finetune/Step": 1298, "Finetune/Step Time": 4.365745713934302} +{"Finetune/Learning Rate": 1.7566204401986172e-05, "Finetune/Loss": 0.9365791082382202, "Finetune/Loss (Raw)": 0.7282246351242065, "Finetune/Step": 1299, "Finetune/Step Time": 4.375260349363089} +{"Finetune/Learning Rate": 1.7562129676082933e-05, "Finetune/Loss": 0.9354618787765503, "Finetune/Loss (Raw)": 0.888131320476532, "Finetune/Step": 1300, "Finetune/Step Time": 4.367191057652235} +{"Finetune/Learning Rate": 1.75580520154648e-05, "Finetune/Loss": 0.9356847405433655, "Finetune/Loss (Raw)": 1.0043483972549438, "Finetune/Step": 1301, "Finetune/Step Time": 4.385062716901302} +{"Finetune/Learning Rate": 1.755397142171424e-05, "Finetune/Loss": 0.9398953914642334, "Finetune/Loss (Raw)": 0.90694659948349, "Finetune/Step": 1302, "Finetune/Step Time": 4.372507782652974} +{"Finetune/Learning Rate": 1.7549887896414853e-05, "Finetune/Loss": 0.9398807287216187, "Finetune/Loss (Raw)": 0.936448872089386, "Finetune/Step": 1303, "Finetune/Step Time": 4.355832459405065} +{"Finetune/Learning Rate": 1.7545801441151368e-05, "Finetune/Loss": 0.9405431151390076, "Finetune/Loss (Raw)": 0.9920441508293152, "Finetune/Step": 1304, "Finetune/Step Time": 4.36162674985826} +{"Finetune/Learning Rate": 1.7541712057509666e-05, "Finetune/Loss": 0.9456546902656555, "Finetune/Loss (Raw)": 0.9482123255729675, "Finetune/Step": 1305, "Finetune/Step Time": 4.3256831634789705} +{"Finetune/Learning Rate": 1.7537619747076752e-05, "Finetune/Loss": 0.9507472515106201, "Finetune/Loss (Raw)": 0.9800108671188354, "Finetune/Step": 1306, "Finetune/Step Time": 4.314756598323584} +{"Finetune/Learning Rate": 1.7533524511440777e-05, "Finetune/Loss": 0.9501810073852539, "Finetune/Loss (Raw)": 1.0701624155044556, "Finetune/Step": 1307, "Finetune/Step Time": 4.308362847194076} +{"Finetune/Learning Rate": 1.7529426352191014e-05, "Finetune/Loss": 0.9514366388320923, "Finetune/Loss (Raw)": 1.0619515180587769, "Finetune/Step": 1308, "Finetune/Step Time": 4.290972406044602} +{"Finetune/Learning Rate": 1.752532527091789e-05, "Finetune/Loss": 0.9504243731498718, "Finetune/Loss (Raw)": 0.9464803338050842, "Finetune/Step": 1309, "Finetune/Step Time": 4.303913746029139} +{"Finetune/Learning Rate": 1.7521221269212944e-05, "Finetune/Loss": 0.9520549774169922, "Finetune/Loss (Raw)": 0.905412495136261, "Finetune/Step": 1310, "Finetune/Step Time": 4.302063819020987} +{"Finetune/Learning Rate": 1.751711434866886e-05, "Finetune/Loss": 0.9517782330513, "Finetune/Loss (Raw)": 0.9872138500213623, "Finetune/Step": 1311, "Finetune/Step Time": 4.304186604917049} +{"Finetune/Learning Rate": 1.751300451087946e-05, "Finetune/Loss": 0.9512536525726318, "Finetune/Loss (Raw)": 0.7452397346496582, "Finetune/Step": 1312, "Finetune/Step Time": 4.299002461135387} +{"Finetune/Learning Rate": 1.7508891757439687e-05, "Finetune/Loss": 0.9512826204299927, "Finetune/Loss (Raw)": 1.0025115013122559, "Finetune/Step": 1313, "Finetune/Step Time": 4.306017013266683} +{"Finetune/Learning Rate": 1.7504776089945626e-05, "Finetune/Loss": 0.9503645300865173, "Finetune/Loss (Raw)": 0.9841629862785339, "Finetune/Step": 1314, "Finetune/Step Time": 4.303359257057309} +{"Finetune/Learning Rate": 1.7500657509994478e-05, "Finetune/Loss": 0.9552383422851562, "Finetune/Loss (Raw)": 0.9773311018943787, "Finetune/Step": 1315, "Finetune/Step Time": 4.266335209831595} +{"Finetune/Learning Rate": 1.7496536019184588e-05, "Finetune/Loss": 0.9558296203613281, "Finetune/Loss (Raw)": 1.1227409839630127, "Finetune/Step": 1316, "Finetune/Step Time": 4.2713418286293745} +{"Finetune/Learning Rate": 1.749241161911543e-05, "Finetune/Loss": 0.9557330012321472, "Finetune/Loss (Raw)": 1.0397859811782837, "Finetune/Step": 1317, "Finetune/Step Time": 4.264019567519426} +{"Finetune/Learning Rate": 1.7488284311387597e-05, "Finetune/Loss": 0.955577552318573, "Finetune/Loss (Raw)": 0.9447475671768188, "Finetune/Step": 1318, "Finetune/Step Time": 4.264224894344807} +{"Finetune/Learning Rate": 1.7484154097602822e-05, "Finetune/Loss": 0.9550926685333252, "Finetune/Loss (Raw)": 0.983234703540802, "Finetune/Step": 1319, "Finetune/Step Time": 4.262137284502387} +{"Finetune/Learning Rate": 1.7480020979363956e-05, "Finetune/Loss": 0.9545077085494995, "Finetune/Loss (Raw)": 0.9423626661300659, "Finetune/Step": 1320, "Finetune/Step Time": 4.263147560879588} +{"Finetune/Learning Rate": 1.747588495827498e-05, "Finetune/Loss": 0.9542394280433655, "Finetune/Loss (Raw)": 0.9764023423194885, "Finetune/Step": 1321, "Finetune/Step Time": 4.265831930562854} +{"Finetune/Learning Rate": 1.7471746035941007e-05, "Finetune/Loss": 0.9536305665969849, "Finetune/Loss (Raw)": 0.9543563723564148, "Finetune/Step": 1322, "Finetune/Step Time": 4.2551615964621305} +{"Finetune/Learning Rate": 1.7467604213968274e-05, "Finetune/Loss": 0.954105794429779, "Finetune/Loss (Raw)": 1.0382894277572632, "Finetune/Step": 1323, "Finetune/Step Time": 4.255628980696201} +{"Finetune/Learning Rate": 1.746345949396414e-05, "Finetune/Loss": 0.9544367790222168, "Finetune/Loss (Raw)": 1.0120891332626343, "Finetune/Step": 1324, "Finetune/Step Time": 4.257553357630968} +{"Finetune/Learning Rate": 1.7459311877537085e-05, "Finetune/Loss": 0.9540919661521912, "Finetune/Loss (Raw)": 1.0372916460037231, "Finetune/Step": 1325, "Finetune/Step Time": 4.275201132521033} +{"Finetune/Learning Rate": 1.745516136629672e-05, "Finetune/Loss": 0.9536595344543457, "Finetune/Loss (Raw)": 1.0252631902694702, "Finetune/Step": 1326, "Finetune/Step Time": 4.267953470349312} +{"Finetune/Learning Rate": 1.7451007961853778e-05, "Finetune/Loss": 0.9527127146720886, "Finetune/Loss (Raw)": 0.9059184789657593, "Finetune/Step": 1327, "Finetune/Step Time": 4.3057004902511835} +{"Finetune/Learning Rate": 1.7446851665820118e-05, "Finetune/Loss": 0.9534436464309692, "Finetune/Loss (Raw)": 1.130240797996521, "Finetune/Step": 1328, "Finetune/Step Time": 4.301484640687704} +{"Finetune/Learning Rate": 1.744269247980871e-05, "Finetune/Loss": 0.9545001983642578, "Finetune/Loss (Raw)": 1.1690441370010376, "Finetune/Step": 1329, "Finetune/Step Time": 4.293425116688013} +{"Finetune/Learning Rate": 1.7438530405433663e-05, "Finetune/Loss": 0.9514939785003662, "Finetune/Loss (Raw)": 0.7622098326683044, "Finetune/Step": 1330, "Finetune/Step Time": 4.286508020013571} +{"Finetune/Learning Rate": 1.7434365444310185e-05, "Finetune/Loss": 0.9509210586547852, "Finetune/Loss (Raw)": 1.0415934324264526, "Finetune/Step": 1331, "Finetune/Step Time": 4.279671320691705} +{"Finetune/Learning Rate": 1.7430197598054622e-05, "Finetune/Loss": 0.9491971731185913, "Finetune/Loss (Raw)": 0.8483652472496033, "Finetune/Step": 1332, "Finetune/Step Time": 4.282587755471468} +{"Finetune/Learning Rate": 1.7426026868284438e-05, "Finetune/Loss": 0.9492189884185791, "Finetune/Loss (Raw)": 0.9461647272109985, "Finetune/Step": 1333, "Finetune/Step Time": 4.269127726554871} +{"Finetune/Learning Rate": 1.7421853256618204e-05, "Finetune/Loss": 0.9481643438339233, "Finetune/Loss (Raw)": 0.994093120098114, "Finetune/Step": 1334, "Finetune/Step Time": 4.2728376388549805} +{"Finetune/Learning Rate": 1.741767676467562e-05, "Finetune/Loss": 0.9487456679344177, "Finetune/Loss (Raw)": 1.0131303071975708, "Finetune/Step": 1335, "Finetune/Step Time": 4.264682006090879} +{"Finetune/Learning Rate": 1.7413497394077502e-05, "Finetune/Loss": 0.9488385915756226, "Finetune/Loss (Raw)": 1.0156804323196411, "Finetune/Step": 1336, "Finetune/Step Time": 4.259886592626572} +{"Finetune/Learning Rate": 1.7409315146445783e-05, "Finetune/Loss": 0.9492032527923584, "Finetune/Loss (Raw)": 1.0878938436508179, "Finetune/Step": 1337, "Finetune/Step Time": 4.26730278506875} +{"Finetune/Learning Rate": 1.7405130023403513e-05, "Finetune/Loss": 0.9440715909004211, "Finetune/Loss (Raw)": 0.3204737901687622, "Finetune/Step": 1338, "Finetune/Step Time": 4.307785922661424} +{"Finetune/Learning Rate": 1.740094202657485e-05, "Finetune/Loss": 0.9436914920806885, "Finetune/Loss (Raw)": 0.8834078907966614, "Finetune/Step": 1339, "Finetune/Step Time": 4.313379622995853} +{"Finetune/Learning Rate": 1.7396751157585078e-05, "Finetune/Loss": 0.9427071213722229, "Finetune/Loss (Raw)": 0.9765962958335876, "Finetune/Step": 1340, "Finetune/Step Time": 4.319769510999322} +{"Finetune/Learning Rate": 1.739255741806059e-05, "Finetune/Loss": 0.9426040649414062, "Finetune/Loss (Raw)": 0.9252752661705017, "Finetune/Step": 1341, "Finetune/Step Time": 4.32105902209878} +{"Finetune/Learning Rate": 1.73883608096289e-05, "Finetune/Loss": 0.9445102214813232, "Finetune/Loss (Raw)": 1.074363112449646, "Finetune/Step": 1342, "Finetune/Step Time": 4.313136892393231} +{"Finetune/Learning Rate": 1.738416133391862e-05, "Finetune/Loss": 0.9437946677207947, "Finetune/Loss (Raw)": 0.8877893090248108, "Finetune/Step": 1343, "Finetune/Step Time": 4.335408514365554} +{"Finetune/Learning Rate": 1.7379958992559494e-05, "Finetune/Loss": 0.9483028650283813, "Finetune/Loss (Raw)": 0.9313896298408508, "Finetune/Step": 1344, "Finetune/Step Time": 4.302044529467821} +{"Finetune/Learning Rate": 1.7375753787182364e-05, "Finetune/Loss": 0.9478679895401001, "Finetune/Loss (Raw)": 1.0404927730560303, "Finetune/Step": 1345, "Finetune/Step Time": 4.301719583570957} +{"Finetune/Learning Rate": 1.7371545719419188e-05, "Finetune/Loss": 0.9471123814582825, "Finetune/Loss (Raw)": 0.9373010993003845, "Finetune/Step": 1346, "Finetune/Step Time": 4.3033874072134495} +{"Finetune/Learning Rate": 1.7367334790903034e-05, "Finetune/Loss": 0.9535248279571533, "Finetune/Loss (Raw)": 1.1294010877609253, "Finetune/Step": 1347, "Finetune/Step Time": 4.282090425491333} +{"Finetune/Learning Rate": 1.736312100326808e-05, "Finetune/Loss": 0.9493951797485352, "Finetune/Loss (Raw)": 0.4364677965641022, "Finetune/Step": 1348, "Finetune/Step Time": 4.323970811441541} +{"Finetune/Learning Rate": 1.7358904358149625e-05, "Finetune/Loss": 0.9498082995414734, "Finetune/Loss (Raw)": 1.0638936758041382, "Finetune/Step": 1349, "Finetune/Step Time": 4.304429749026895} +{"Finetune/Learning Rate": 1.7354684857184056e-05, "Finetune/Loss": 0.9502787590026855, "Finetune/Loss (Raw)": 1.043214201927185, "Finetune/Step": 1350, "Finetune/Step Time": 4.296651398763061} +{"Finetune/Learning Rate": 1.735046250200888e-05, "Finetune/Loss": 0.9504298567771912, "Finetune/Loss (Raw)": 1.0307599306106567, "Finetune/Step": 1351, "Finetune/Step Time": 4.300168735906482} +{"Finetune/Learning Rate": 1.7346237294262717e-05, "Finetune/Loss": 0.9459445476531982, "Finetune/Loss (Raw)": 0.4265022873878479, "Finetune/Step": 1352, "Finetune/Step Time": 4.338856445625424} +{"Finetune/Learning Rate": 1.734200923558528e-05, "Finetune/Loss": 0.9464320540428162, "Finetune/Loss (Raw)": 0.9296098947525024, "Finetune/Step": 1353, "Finetune/Step Time": 4.3397243563085794} +{"Finetune/Learning Rate": 1.7337778327617397e-05, "Finetune/Loss": 0.9471766948699951, "Finetune/Loss (Raw)": 0.977972149848938, "Finetune/Step": 1354, "Finetune/Step Time": 4.331620840355754} +{"Finetune/Learning Rate": 1.733354457200101e-05, "Finetune/Loss": 0.9490335583686829, "Finetune/Loss (Raw)": 1.0813050270080566, "Finetune/Step": 1355, "Finetune/Step Time": 4.324987860396504} +{"Finetune/Learning Rate": 1.7329307970379145e-05, "Finetune/Loss": 0.9493433237075806, "Finetune/Loss (Raw)": 1.0546045303344727, "Finetune/Step": 1356, "Finetune/Step Time": 4.318897761404514} +{"Finetune/Learning Rate": 1.732506852439595e-05, "Finetune/Loss": 0.9492812156677246, "Finetune/Loss (Raw)": 1.053160309791565, "Finetune/Step": 1357, "Finetune/Step Time": 4.331544993445277} +{"Finetune/Learning Rate": 1.7320826235696672e-05, "Finetune/Loss": 0.9536126852035522, "Finetune/Loss (Raw)": 0.9605391025543213, "Finetune/Step": 1358, "Finetune/Step Time": 4.298678191378713} +{"Finetune/Learning Rate": 1.7316581105927658e-05, "Finetune/Loss": 0.9517033696174622, "Finetune/Loss (Raw)": 0.849809467792511, "Finetune/Step": 1359, "Finetune/Step Time": 4.312986904755235} +{"Finetune/Learning Rate": 1.731233313673636e-05, "Finetune/Loss": 0.9516429901123047, "Finetune/Loss (Raw)": 0.9877983331680298, "Finetune/Step": 1360, "Finetune/Step Time": 4.310289055109024} +{"Finetune/Learning Rate": 1.7308082329771335e-05, "Finetune/Loss": 0.9521751403808594, "Finetune/Loss (Raw)": 1.0791646242141724, "Finetune/Step": 1361, "Finetune/Step Time": 4.311616376042366} +{"Finetune/Learning Rate": 1.7303828686682233e-05, "Finetune/Loss": 0.9533534646034241, "Finetune/Loss (Raw)": 0.9540652632713318, "Finetune/Step": 1362, "Finetune/Step Time": 4.2943745497614145} +{"Finetune/Learning Rate": 1.7299572209119813e-05, "Finetune/Loss": 0.9535737633705139, "Finetune/Loss (Raw)": 0.9663574695587158, "Finetune/Step": 1363, "Finetune/Step Time": 4.29489104822278} +{"Finetune/Learning Rate": 1.7295312898735933e-05, "Finetune/Loss": 0.9524797201156616, "Finetune/Loss (Raw)": 0.9443914294242859, "Finetune/Step": 1364, "Finetune/Step Time": 4.3237238228321075} +{"Finetune/Learning Rate": 1.7291050757183546e-05, "Finetune/Loss": 0.95142662525177, "Finetune/Loss (Raw)": 0.8827359080314636, "Finetune/Step": 1365, "Finetune/Step Time": 4.314466644078493} +{"Finetune/Learning Rate": 1.7286785786116707e-05, "Finetune/Loss": 0.9497450590133667, "Finetune/Loss (Raw)": 0.9431391954421997, "Finetune/Step": 1366, "Finetune/Step Time": 4.314730295911431} +{"Finetune/Learning Rate": 1.7282517987190567e-05, "Finetune/Loss": 0.9492161870002747, "Finetune/Loss (Raw)": 0.8953066468238831, "Finetune/Step": 1367, "Finetune/Step Time": 4.327675215899944} +{"Finetune/Learning Rate": 1.7278247362061372e-05, "Finetune/Loss": 0.9497978091239929, "Finetune/Loss (Raw)": 1.0218746662139893, "Finetune/Step": 1368, "Finetune/Step Time": 4.3334730714559555} +{"Finetune/Learning Rate": 1.7273973912386473e-05, "Finetune/Loss": 0.9510605335235596, "Finetune/Loss (Raw)": 1.0452386140823364, "Finetune/Step": 1369, "Finetune/Step Time": 4.343359040096402} +{"Finetune/Learning Rate": 1.726969763982431e-05, "Finetune/Loss": 0.9501053094863892, "Finetune/Loss (Raw)": 0.8348800539970398, "Finetune/Step": 1370, "Finetune/Step Time": 4.345961716026068} +{"Finetune/Learning Rate": 1.7265418546034424e-05, "Finetune/Loss": 0.9494800567626953, "Finetune/Loss (Raw)": 0.9602896571159363, "Finetune/Step": 1371, "Finetune/Step Time": 4.346411753445864} +{"Finetune/Learning Rate": 1.7261136632677445e-05, "Finetune/Loss": 0.9449983835220337, "Finetune/Loss (Raw)": 0.3869782090187073, "Finetune/Step": 1372, "Finetune/Step Time": 4.388357058167458} +{"Finetune/Learning Rate": 1.7256851901415102e-05, "Finetune/Loss": 0.94617760181427, "Finetune/Loss (Raw)": 1.1039049625396729, "Finetune/Step": 1373, "Finetune/Step Time": 4.394786538556218} +{"Finetune/Learning Rate": 1.725256435391021e-05, "Finetune/Loss": 0.9463666677474976, "Finetune/Loss (Raw)": 0.9223455190658569, "Finetune/Step": 1374, "Finetune/Step Time": 4.3812833763659} +{"Finetune/Learning Rate": 1.7248273991826692e-05, "Finetune/Loss": 0.9462056159973145, "Finetune/Loss (Raw)": 1.022693157196045, "Finetune/Step": 1375, "Finetune/Step Time": 4.37915650755167} +{"Finetune/Learning Rate": 1.7243980816829543e-05, "Finetune/Loss": 0.9425145387649536, "Finetune/Loss (Raw)": 0.3214130103588104, "Finetune/Step": 1376, "Finetune/Step Time": 4.404283693060279} +{"Finetune/Learning Rate": 1.723968483058487e-05, "Finetune/Loss": 0.94345623254776, "Finetune/Loss (Raw)": 1.0240172147750854, "Finetune/Step": 1377, "Finetune/Step Time": 4.412591744214296} +{"Finetune/Learning Rate": 1.7235386034759853e-05, "Finetune/Loss": 0.9444008469581604, "Finetune/Loss (Raw)": 1.140790343284607, "Finetune/Step": 1378, "Finetune/Step Time": 4.430604387074709} +{"Finetune/Learning Rate": 1.723108443102278e-05, "Finetune/Loss": 0.9420292377471924, "Finetune/Loss (Raw)": 0.7036797404289246, "Finetune/Step": 1379, "Finetune/Step Time": 4.446655910462141} +{"Finetune/Learning Rate": 1.722678002104301e-05, "Finetune/Loss": 0.9422489404678345, "Finetune/Loss (Raw)": 0.9109728932380676, "Finetune/Step": 1380, "Finetune/Step Time": 4.446533566340804} +{"Finetune/Learning Rate": 1.722247280649101e-05, "Finetune/Loss": 0.9431713223457336, "Finetune/Loss (Raw)": 1.0355002880096436, "Finetune/Step": 1381, "Finetune/Step Time": 4.431625196710229} +{"Finetune/Learning Rate": 1.7218162789038312e-05, "Finetune/Loss": 0.9431359767913818, "Finetune/Loss (Raw)": 0.9662594795227051, "Finetune/Step": 1382, "Finetune/Step Time": 4.43607410416007} +{"Finetune/Learning Rate": 1.7213849970357565e-05, "Finetune/Loss": 0.9415226578712463, "Finetune/Loss (Raw)": 0.8852509260177612, "Finetune/Step": 1383, "Finetune/Step Time": 4.4409344755113125} +{"Finetune/Learning Rate": 1.7209534352122484e-05, "Finetune/Loss": 0.9410604238510132, "Finetune/Loss (Raw)": 0.9456034898757935, "Finetune/Step": 1384, "Finetune/Step Time": 4.443415340036154} +{"Finetune/Learning Rate": 1.720521593600787e-05, "Finetune/Loss": 0.9435521364212036, "Finetune/Loss (Raw)": 1.0039416551589966, "Finetune/Step": 1385, "Finetune/Step Time": 4.4195347763597965} +{"Finetune/Learning Rate": 1.7200894723689626e-05, "Finetune/Loss": 0.943354606628418, "Finetune/Loss (Raw)": 0.9130643010139465, "Finetune/Step": 1386, "Finetune/Step Time": 4.420588191598654} +{"Finetune/Learning Rate": 1.7196570716844724e-05, "Finetune/Loss": 0.943011999130249, "Finetune/Loss (Raw)": 0.8448742032051086, "Finetune/Step": 1387, "Finetune/Step Time": 4.41483261436224} +{"Finetune/Learning Rate": 1.7192243917151228e-05, "Finetune/Loss": 0.9418221116065979, "Finetune/Loss (Raw)": 1.0055325031280518, "Finetune/Step": 1388, "Finetune/Step Time": 4.410090431571007} +{"Finetune/Learning Rate": 1.7187914326288286e-05, "Finetune/Loss": 0.9429682493209839, "Finetune/Loss (Raw)": 0.9821550846099854, "Finetune/Step": 1389, "Finetune/Step Time": 4.410995492711663} +{"Finetune/Learning Rate": 1.718358194593612e-05, "Finetune/Loss": 0.9442207217216492, "Finetune/Loss (Raw)": 1.1427946090698242, "Finetune/Step": 1390, "Finetune/Step Time": 4.419579055160284} +{"Finetune/Learning Rate": 1.7179246777776053e-05, "Finetune/Loss": 0.9429207444190979, "Finetune/Loss (Raw)": 0.8741411566734314, "Finetune/Step": 1391, "Finetune/Step Time": 4.435708612203598} +{"Finetune/Learning Rate": 1.7174908823490475e-05, "Finetune/Loss": 0.9421459436416626, "Finetune/Loss (Raw)": 0.8787544965744019, "Finetune/Step": 1392, "Finetune/Step Time": 4.468997189775109} +{"Finetune/Learning Rate": 1.717056808476286e-05, "Finetune/Loss": 0.9466472864151001, "Finetune/Loss (Raw)": 0.9572667479515076, "Finetune/Step": 1393, "Finetune/Step Time": 4.467679435387254} +{"Finetune/Learning Rate": 1.716622456327776e-05, "Finetune/Loss": 0.9472531080245972, "Finetune/Loss (Raw)": 1.013466715812683, "Finetune/Step": 1394, "Finetune/Step Time": 4.454153081402183} +{"Finetune/Learning Rate": 1.7161878260720826e-05, "Finetune/Loss": 0.9465160965919495, "Finetune/Loss (Raw)": 0.94400554895401, "Finetune/Step": 1395, "Finetune/Step Time": 4.456829130649567} +{"Finetune/Learning Rate": 1.7157529178778757e-05, "Finetune/Loss": 0.943834662437439, "Finetune/Loss (Raw)": 0.7091842889785767, "Finetune/Step": 1396, "Finetune/Step Time": 4.464333388954401} +{"Finetune/Learning Rate": 1.7153177319139352e-05, "Finetune/Loss": 0.9444524049758911, "Finetune/Loss (Raw)": 1.017413854598999, "Finetune/Step": 1397, "Finetune/Step Time": 4.487747315317392} +{"Finetune/Learning Rate": 1.7148822683491483e-05, "Finetune/Loss": 0.9436044096946716, "Finetune/Loss (Raw)": 0.9853475689888, "Finetune/Step": 1398, "Finetune/Step Time": 4.477394741028547} +{"Finetune/Learning Rate": 1.7144465273525105e-05, "Finetune/Loss": 0.9412820339202881, "Finetune/Loss (Raw)": 0.7896058559417725, "Finetune/Step": 1399, "Finetune/Step Time": 4.483676640316844} +{"Finetune/Learning Rate": 1.7140105090931232e-05, "Finetune/Loss": 0.9430800676345825, "Finetune/Loss (Raw)": 1.0234509706497192, "Finetune/Step": 1400, "Finetune/Step Time": 4.478620624169707} +{"Finetune/Learning Rate": 1.7135742137401976e-05, "Finetune/Loss": 0.9427945613861084, "Finetune/Loss (Raw)": 0.8925108313560486, "Finetune/Step": 1401, "Finetune/Step Time": 4.471671963110566} +{"Finetune/Learning Rate": 1.7131376414630506e-05, "Finetune/Loss": 0.939172089099884, "Finetune/Loss (Raw)": 0.3962946832180023, "Finetune/Step": 1402, "Finetune/Step Time": 4.511929273605347} +{"Finetune/Learning Rate": 1.712700792431108e-05, "Finetune/Loss": 0.9373431205749512, "Finetune/Loss (Raw)": 0.9143123030662537, "Finetune/Step": 1403, "Finetune/Step Time": 4.512675112113357} +{"Finetune/Learning Rate": 1.7122636668139022e-05, "Finetune/Loss": 0.9375699162483215, "Finetune/Loss (Raw)": 0.9577138423919678, "Finetune/Step": 1404, "Finetune/Step Time": 4.508822124451399} +{"Finetune/Learning Rate": 1.711826264781073e-05, "Finetune/Loss": 0.9364817142486572, "Finetune/Loss (Raw)": 0.8130937814712524, "Finetune/Step": 1405, "Finetune/Step Time": 4.5067936684936285} +{"Finetune/Learning Rate": 1.711388586502368e-05, "Finetune/Loss": 0.9382491111755371, "Finetune/Loss (Raw)": 0.9531218409538269, "Finetune/Step": 1406, "Finetune/Step Time": 4.505406571552157} +{"Finetune/Learning Rate": 1.710950632147641e-05, "Finetune/Loss": 0.9335396885871887, "Finetune/Loss (Raw)": 0.37552058696746826, "Finetune/Step": 1407, "Finetune/Step Time": 4.538520714268088} +{"Finetune/Learning Rate": 1.710512401886854e-05, "Finetune/Loss": 0.9313215017318726, "Finetune/Loss (Raw)": 0.8434505462646484, "Finetune/Step": 1408, "Finetune/Step Time": 4.5413217563182116} +{"Finetune/Learning Rate": 1.7100738958900752e-05, "Finetune/Loss": 0.9316433072090149, "Finetune/Loss (Raw)": 0.9868963956832886, "Finetune/Step": 1409, "Finetune/Step Time": 4.538754420354962} +{"Finetune/Learning Rate": 1.7096351143274804e-05, "Finetune/Loss": 0.9309465885162354, "Finetune/Loss (Raw)": 0.8885359168052673, "Finetune/Step": 1410, "Finetune/Step Time": 4.56749532930553} +{"Finetune/Learning Rate": 1.7091960573693523e-05, "Finetune/Loss": 0.9322755336761475, "Finetune/Loss (Raw)": 1.013756513595581, "Finetune/Step": 1411, "Finetune/Step Time": 4.5563590582460165} +{"Finetune/Learning Rate": 1.70875672518608e-05, "Finetune/Loss": 0.9305605888366699, "Finetune/Loss (Raw)": 0.7325518727302551, "Finetune/Step": 1412, "Finetune/Step Time": 4.556426223367453} +{"Finetune/Learning Rate": 1.7083171179481603e-05, "Finetune/Loss": 0.9302816390991211, "Finetune/Loss (Raw)": 1.01638662815094, "Finetune/Step": 1413, "Finetune/Step Time": 4.560017483308911} +{"Finetune/Learning Rate": 1.7078772358261957e-05, "Finetune/Loss": 0.9313942193984985, "Finetune/Loss (Raw)": 1.0840498208999634, "Finetune/Step": 1414, "Finetune/Step Time": 4.549141611903906} +{"Finetune/Learning Rate": 1.707437078990896e-05, "Finetune/Loss": 0.931159257888794, "Finetune/Loss (Raw)": 0.9355344176292419, "Finetune/Step": 1415, "Finetune/Step Time": 4.553699858486652} +{"Finetune/Learning Rate": 1.7069966476130773e-05, "Finetune/Loss": 0.9324260950088501, "Finetune/Loss (Raw)": 1.0625554323196411, "Finetune/Step": 1416, "Finetune/Step Time": 4.5579848401248455} +{"Finetune/Learning Rate": 1.706555941863663e-05, "Finetune/Loss": 0.9324853420257568, "Finetune/Loss (Raw)": 0.8996374607086182, "Finetune/Step": 1417, "Finetune/Step Time": 4.5571392476558685} +{"Finetune/Learning Rate": 1.7061149619136818e-05, "Finetune/Loss": 0.9304571151733398, "Finetune/Loss (Raw)": 0.8314011096954346, "Finetune/Step": 1418, "Finetune/Step Time": 4.557916397228837} +{"Finetune/Learning Rate": 1.7056737079342694e-05, "Finetune/Loss": 0.9296115040779114, "Finetune/Loss (Raw)": 0.8601790070533752, "Finetune/Step": 1419, "Finetune/Step Time": 4.54090166836977} +{"Finetune/Learning Rate": 1.7052321800966684e-05, "Finetune/Loss": 0.929923951625824, "Finetune/Loss (Raw)": 0.9774134755134583, "Finetune/Step": 1420, "Finetune/Step Time": 4.547135915607214} +{"Finetune/Learning Rate": 1.7047903785722268e-05, "Finetune/Loss": 0.9296499490737915, "Finetune/Loss (Raw)": 0.9015276432037354, "Finetune/Step": 1421, "Finetune/Step Time": 4.542010400444269} +{"Finetune/Learning Rate": 1.704348303532399e-05, "Finetune/Loss": 0.9292066097259521, "Finetune/Loss (Raw)": 0.9200637340545654, "Finetune/Step": 1422, "Finetune/Step Time": 4.53906680084765} +{"Finetune/Learning Rate": 1.7039059551487457e-05, "Finetune/Loss": 0.9299903512001038, "Finetune/Loss (Raw)": 1.0118173360824585, "Finetune/Step": 1423, "Finetune/Step Time": 4.529508385807276} +{"Finetune/Learning Rate": 1.703463333592934e-05, "Finetune/Loss": 0.9292620420455933, "Finetune/Loss (Raw)": 0.8434990644454956, "Finetune/Step": 1424, "Finetune/Step Time": 4.524342631921172} +{"Finetune/Learning Rate": 1.7030204390367368e-05, "Finetune/Loss": 0.9293354749679565, "Finetune/Loss (Raw)": 0.9732345342636108, "Finetune/Step": 1425, "Finetune/Step Time": 4.52361018396914} +{"Finetune/Learning Rate": 1.7025772716520324e-05, "Finetune/Loss": 0.9296172857284546, "Finetune/Loss (Raw)": 0.8662412762641907, "Finetune/Step": 1426, "Finetune/Step Time": 4.536637339740992} +{"Finetune/Learning Rate": 1.702133831610805e-05, "Finetune/Loss": 0.9310852289199829, "Finetune/Loss (Raw)": 0.9161241054534912, "Finetune/Step": 1427, "Finetune/Step Time": 4.530632885172963} +{"Finetune/Learning Rate": 1.701690119085146e-05, "Finetune/Loss": 0.9313187599182129, "Finetune/Loss (Raw)": 0.9180136322975159, "Finetune/Step": 1428, "Finetune/Step Time": 4.542954372242093} +{"Finetune/Learning Rate": 1.701246134247251e-05, "Finetune/Loss": 0.9300267696380615, "Finetune/Loss (Raw)": 0.8389816284179688, "Finetune/Step": 1429, "Finetune/Step Time": 4.533556403592229} +{"Finetune/Learning Rate": 1.700801877269422e-05, "Finetune/Loss": 0.9302184581756592, "Finetune/Loss (Raw)": 0.9314823150634766, "Finetune/Step": 1430, "Finetune/Step Time": 4.519709009677172} +{"Finetune/Learning Rate": 1.7003573483240666e-05, "Finetune/Loss": 0.9296990633010864, "Finetune/Loss (Raw)": 0.8699628710746765, "Finetune/Step": 1431, "Finetune/Step Time": 4.5306920893490314} +{"Finetune/Learning Rate": 1.6999125475836973e-05, "Finetune/Loss": 0.9292019605636597, "Finetune/Loss (Raw)": 0.9284119009971619, "Finetune/Step": 1432, "Finetune/Step Time": 4.533436307683587} +{"Finetune/Learning Rate": 1.6994674752209334e-05, "Finetune/Loss": 0.9294641017913818, "Finetune/Loss (Raw)": 0.9817770719528198, "Finetune/Step": 1433, "Finetune/Step Time": 4.534775586798787} +{"Finetune/Learning Rate": 1.6990221314084976e-05, "Finetune/Loss": 0.9247795939445496, "Finetune/Loss (Raw)": 0.3803935647010803, "Finetune/Step": 1434, "Finetune/Step Time": 4.546056307852268} +{"Finetune/Learning Rate": 1.6985765163192204e-05, "Finetune/Loss": 0.9242218732833862, "Finetune/Loss (Raw)": 0.9987714886665344, "Finetune/Step": 1435, "Finetune/Step Time": 4.553518403321505} +{"Finetune/Learning Rate": 1.698130630126036e-05, "Finetune/Loss": 0.9220569133758545, "Finetune/Loss (Raw)": 0.7848342061042786, "Finetune/Step": 1436, "Finetune/Step Time": 4.564076783135533} +{"Finetune/Learning Rate": 1.697684473001983e-05, "Finetune/Loss": 0.9225876927375793, "Finetune/Loss (Raw)": 1.0144219398498535, "Finetune/Step": 1437, "Finetune/Step Time": 4.546591496095061} +{"Finetune/Learning Rate": 1.6972380451202083e-05, "Finetune/Loss": 0.9229997396469116, "Finetune/Loss (Raw)": 0.9581559300422668, "Finetune/Step": 1438, "Finetune/Step Time": 4.541906489059329} +{"Finetune/Learning Rate": 1.69679134665396e-05, "Finetune/Loss": 0.9233397245407104, "Finetune/Loss (Raw)": 1.030730962753296, "Finetune/Step": 1439, "Finetune/Step Time": 4.53849084675312} +{"Finetune/Learning Rate": 1.696344377776594e-05, "Finetune/Loss": 0.9228746891021729, "Finetune/Loss (Raw)": 0.6857175827026367, "Finetune/Step": 1440, "Finetune/Step Time": 4.543475620448589} +{"Finetune/Learning Rate": 1.69589713866157e-05, "Finetune/Loss": 0.922277569770813, "Finetune/Loss (Raw)": 0.9260750412940979, "Finetune/Step": 1441, "Finetune/Step Time": 4.539507178589702} +{"Finetune/Learning Rate": 1.6954496294824526e-05, "Finetune/Loss": 0.9219527244567871, "Finetune/Loss (Raw)": 0.942581057548523, "Finetune/Step": 1442, "Finetune/Step Time": 4.539826264604926} +{"Finetune/Learning Rate": 1.6950018504129112e-05, "Finetune/Loss": 0.9212452173233032, "Finetune/Loss (Raw)": 0.886781632900238, "Finetune/Step": 1443, "Finetune/Step Time": 4.543025698512793} +{"Finetune/Learning Rate": 1.6945538016267206e-05, "Finetune/Loss": 0.9194434285163879, "Finetune/Loss (Raw)": 0.8921048641204834, "Finetune/Step": 1444, "Finetune/Step Time": 4.546303026378155} +{"Finetune/Learning Rate": 1.6941054832977598e-05, "Finetune/Loss": 0.918409526348114, "Finetune/Loss (Raw)": 0.9074428081512451, "Finetune/Step": 1445, "Finetune/Step Time": 4.548680314794183} +{"Finetune/Learning Rate": 1.6936568956000123e-05, "Finetune/Loss": 0.9175970554351807, "Finetune/Loss (Raw)": 0.8407521843910217, "Finetune/Step": 1446, "Finetune/Step Time": 4.568891039118171} +{"Finetune/Learning Rate": 1.693208038707566e-05, "Finetune/Loss": 0.9173299074172974, "Finetune/Loss (Raw)": 0.9490459561347961, "Finetune/Step": 1447, "Finetune/Step Time": 4.573305891826749} +{"Finetune/Learning Rate": 1.6927589127946136e-05, "Finetune/Loss": 0.9180209636688232, "Finetune/Loss (Raw)": 1.030820608139038, "Finetune/Step": 1448, "Finetune/Step Time": 4.56436263024807} +{"Finetune/Learning Rate": 1.692309518035452e-05, "Finetune/Loss": 0.9169105291366577, "Finetune/Loss (Raw)": 0.8342586159706116, "Finetune/Step": 1449, "Finetune/Step Time": 4.566790975630283} +{"Finetune/Learning Rate": 1.691859854604483e-05, "Finetune/Loss": 0.9159795045852661, "Finetune/Loss (Raw)": 0.8351840376853943, "Finetune/Step": 1450, "Finetune/Step Time": 4.602930750697851} +{"Finetune/Learning Rate": 1.6914099226762116e-05, "Finetune/Loss": 0.9148045778274536, "Finetune/Loss (Raw)": 0.8879019021987915, "Finetune/Step": 1451, "Finetune/Step Time": 4.606058429926634} +{"Finetune/Learning Rate": 1.6909597224252478e-05, "Finetune/Loss": 0.9149783849716187, "Finetune/Loss (Raw)": 1.0343340635299683, "Finetune/Step": 1452, "Finetune/Step Time": 4.59821342676878} +{"Finetune/Learning Rate": 1.6905092540263055e-05, "Finetune/Loss": 0.9138429760932922, "Finetune/Loss (Raw)": 0.891960620880127, "Finetune/Step": 1453, "Finetune/Step Time": 4.592666206881404} +{"Finetune/Learning Rate": 1.690058517654203e-05, "Finetune/Loss": 0.9149592518806458, "Finetune/Loss (Raw)": 1.1681486368179321, "Finetune/Step": 1454, "Finetune/Step Time": 4.59542079269886} +{"Finetune/Learning Rate": 1.689607513483862e-05, "Finetune/Loss": 0.9156616926193237, "Finetune/Loss (Raw)": 0.995836615562439, "Finetune/Step": 1455, "Finetune/Step Time": 4.562651939690113} +{"Finetune/Learning Rate": 1.6891562416903083e-05, "Finetune/Loss": 0.9139426350593567, "Finetune/Loss (Raw)": 0.9101948738098145, "Finetune/Step": 1456, "Finetune/Step Time": 4.564694162458181} +{"Finetune/Learning Rate": 1.688704702448672e-05, "Finetune/Loss": 0.9077528715133667, "Finetune/Loss (Raw)": 0.376762330532074, "Finetune/Step": 1457, "Finetune/Step Time": 4.602530630305409} +{"Finetune/Learning Rate": 1.6882528959341865e-05, "Finetune/Loss": 0.9090698957443237, "Finetune/Loss (Raw)": 0.9307821989059448, "Finetune/Step": 1458, "Finetune/Step Time": 4.600046059116721} +{"Finetune/Learning Rate": 1.6878008223221895e-05, "Finetune/Loss": 0.909229576587677, "Finetune/Loss (Raw)": 1.0620381832122803, "Finetune/Step": 1459, "Finetune/Step Time": 4.608515584841371} +{"Finetune/Learning Rate": 1.687348481788121e-05, "Finetune/Loss": 0.9110636711120605, "Finetune/Loss (Raw)": 1.0831307172775269, "Finetune/Step": 1460, "Finetune/Step Time": 4.598450066521764} +{"Finetune/Learning Rate": 1.6868958745075262e-05, "Finetune/Loss": 0.9123232960700989, "Finetune/Loss (Raw)": 1.1073851585388184, "Finetune/Step": 1461, "Finetune/Step Time": 4.592089030891657} +{"Finetune/Learning Rate": 1.6864430006560527e-05, "Finetune/Loss": 0.9125836491584778, "Finetune/Loss (Raw)": 1.027422308921814, "Finetune/Step": 1462, "Finetune/Step Time": 4.594205006957054} +{"Finetune/Learning Rate": 1.685989860409453e-05, "Finetune/Loss": 0.9126815795898438, "Finetune/Loss (Raw)": 1.0256685018539429, "Finetune/Step": 1463, "Finetune/Step Time": 4.5948446579277515} +{"Finetune/Learning Rate": 1.6855364539435805e-05, "Finetune/Loss": 0.9115148186683655, "Finetune/Loss (Raw)": 0.8663368821144104, "Finetune/Step": 1464, "Finetune/Step Time": 4.599357724189758} +{"Finetune/Learning Rate": 1.685082781434395e-05, "Finetune/Loss": 0.9104443788528442, "Finetune/Loss (Raw)": 0.950874388217926, "Finetune/Step": 1465, "Finetune/Step Time": 4.592360373586416} +{"Finetune/Learning Rate": 1.6846288430579565e-05, "Finetune/Loss": 0.9152514934539795, "Finetune/Loss (Raw)": 0.9357842803001404, "Finetune/Step": 1466, "Finetune/Step Time": 4.558528173714876} +{"Finetune/Learning Rate": 1.6841746389904306e-05, "Finetune/Loss": 0.9156121015548706, "Finetune/Loss (Raw)": 0.929562509059906, "Finetune/Step": 1467, "Finetune/Step Time": 4.576105028390884} +{"Finetune/Learning Rate": 1.683720169408085e-05, "Finetune/Loss": 0.9158711433410645, "Finetune/Loss (Raw)": 1.0097527503967285, "Finetune/Step": 1468, "Finetune/Step Time": 4.568400057032704} +{"Finetune/Learning Rate": 1.6832654344872893e-05, "Finetune/Loss": 0.9116939306259155, "Finetune/Loss (Raw)": 0.3905859887599945, "Finetune/Step": 1469, "Finetune/Step Time": 4.614060351625085} +{"Finetune/Learning Rate": 1.6828104344045186e-05, "Finetune/Loss": 0.9108269214630127, "Finetune/Loss (Raw)": 0.9633947014808655, "Finetune/Step": 1470, "Finetune/Step Time": 4.601775178685784} +{"Finetune/Learning Rate": 1.682355169336349e-05, "Finetune/Loss": 0.9113678336143494, "Finetune/Loss (Raw)": 0.957028329372406, "Finetune/Step": 1471, "Finetune/Step Time": 4.575857853516936} +{"Finetune/Learning Rate": 1.6818996394594603e-05, "Finetune/Loss": 0.9107120633125305, "Finetune/Loss (Raw)": 0.8474500179290771, "Finetune/Step": 1472, "Finetune/Step Time": 4.569323921576142} +{"Finetune/Learning Rate": 1.681443844950634e-05, "Finetune/Loss": 0.9079325199127197, "Finetune/Loss (Raw)": 0.6847110986709595, "Finetune/Step": 1473, "Finetune/Step Time": 4.575611896812916} +{"Finetune/Learning Rate": 1.6809877859867557e-05, "Finetune/Loss": 0.9092072248458862, "Finetune/Loss (Raw)": 1.1004606485366821, "Finetune/Step": 1474, "Finetune/Step Time": 4.570659525692463} +{"Finetune/Learning Rate": 1.680531462744813e-05, "Finetune/Loss": 0.9062362909317017, "Finetune/Loss (Raw)": 0.749126136302948, "Finetune/Step": 1475, "Finetune/Step Time": 4.563056191429496} +{"Finetune/Learning Rate": 1.680074875401896e-05, "Finetune/Loss": 0.9104528427124023, "Finetune/Loss (Raw)": 0.9761835932731628, "Finetune/Step": 1476, "Finetune/Step Time": 4.526585916057229} +{"Finetune/Learning Rate": 1.679618024135197e-05, "Finetune/Loss": 0.9046034216880798, "Finetune/Loss (Raw)": 0.3151627779006958, "Finetune/Step": 1477, "Finetune/Step Time": 4.571097116917372} +{"Finetune/Learning Rate": 1.679160909122011e-05, "Finetune/Loss": 0.9046705961227417, "Finetune/Loss (Raw)": 1.0518147945404053, "Finetune/Step": 1478, "Finetune/Step Time": 4.563288623467088} +{"Finetune/Learning Rate": 1.6787035305397363e-05, "Finetune/Loss": 0.9034568071365356, "Finetune/Loss (Raw)": 0.8753968477249146, "Finetune/Step": 1479, "Finetune/Step Time": 4.582757335156202} +{"Finetune/Learning Rate": 1.6782458885658716e-05, "Finetune/Loss": 0.9075169563293457, "Finetune/Loss (Raw)": 0.9462035894393921, "Finetune/Step": 1480, "Finetune/Step Time": 4.54331967048347} +{"Finetune/Learning Rate": 1.6777879833780192e-05, "Finetune/Loss": 0.9078210592269897, "Finetune/Loss (Raw)": 0.9685332775115967, "Finetune/Step": 1481, "Finetune/Step Time": 4.555014401674271} +{"Finetune/Learning Rate": 1.677329815153883e-05, "Finetune/Loss": 0.907473087310791, "Finetune/Loss (Raw)": 0.933430552482605, "Finetune/Step": 1482, "Finetune/Step Time": 4.546760642901063} +{"Finetune/Learning Rate": 1.6768713840712692e-05, "Finetune/Loss": 0.9058394432067871, "Finetune/Loss (Raw)": 0.872194230556488, "Finetune/Step": 1483, "Finetune/Step Time": 4.571352526545525} +{"Finetune/Learning Rate": 1.6764126903080865e-05, "Finetune/Loss": 0.9053831100463867, "Finetune/Loss (Raw)": 0.9961978793144226, "Finetune/Step": 1484, "Finetune/Step Time": 4.562122689560056} +{"Finetune/Learning Rate": 1.675953734042344e-05, "Finetune/Loss": 0.9041414260864258, "Finetune/Loss (Raw)": 0.8942291736602783, "Finetune/Step": 1485, "Finetune/Step Time": 4.5509532783180475} +{"Finetune/Learning Rate": 1.6754945154521548e-05, "Finetune/Loss": 0.8990731239318848, "Finetune/Loss (Raw)": 0.31179478764533997, "Finetune/Step": 1486, "Finetune/Step Time": 4.583087742328644} +{"Finetune/Learning Rate": 1.6750350347157315e-05, "Finetune/Loss": 0.9008956551551819, "Finetune/Loss (Raw)": 1.0830936431884766, "Finetune/Step": 1487, "Finetune/Step Time": 4.578388661146164} +{"Finetune/Learning Rate": 1.6745752920113907e-05, "Finetune/Loss": 0.9023041725158691, "Finetune/Loss (Raw)": 1.1680928468704224, "Finetune/Step": 1488, "Finetune/Step Time": 4.57622398994863} +{"Finetune/Learning Rate": 1.6741152875175485e-05, "Finetune/Loss": 0.9000656604766846, "Finetune/Loss (Raw)": 0.7926272749900818, "Finetune/Step": 1489, "Finetune/Step Time": 4.590105382725596} +{"Finetune/Learning Rate": 1.6736550214127247e-05, "Finetune/Loss": 0.8997482061386108, "Finetune/Loss (Raw)": 0.9134365916252136, "Finetune/Step": 1490, "Finetune/Step Time": 4.61644689925015} +{"Finetune/Learning Rate": 1.673194493875539e-05, "Finetune/Loss": 0.8985247611999512, "Finetune/Loss (Raw)": 0.8097492456436157, "Finetune/Step": 1491, "Finetune/Step Time": 4.635221548378468} +{"Finetune/Learning Rate": 1.672733705084713e-05, "Finetune/Loss": 0.8994029760360718, "Finetune/Loss (Raw)": 1.0568044185638428, "Finetune/Step": 1492, "Finetune/Step Time": 4.606985658407211} +{"Finetune/Learning Rate": 1.67227265521907e-05, "Finetune/Loss": 0.8951412439346313, "Finetune/Loss (Raw)": 0.33723118901252747, "Finetune/Step": 1493, "Finetune/Step Time": 4.642218969762325} +{"Finetune/Learning Rate": 1.671811344457535e-05, "Finetune/Loss": 0.8949548006057739, "Finetune/Loss (Raw)": 0.9192754030227661, "Finetune/Step": 1494, "Finetune/Step Time": 4.6410568710416555} +{"Finetune/Learning Rate": 1.671349772979133e-05, "Finetune/Loss": 0.8956826329231262, "Finetune/Loss (Raw)": 0.9884753823280334, "Finetune/Step": 1495, "Finetune/Step Time": 4.626772623509169} +{"Finetune/Learning Rate": 1.6708879409629908e-05, "Finetune/Loss": 0.8952780365943909, "Finetune/Loss (Raw)": 0.9700879454612732, "Finetune/Step": 1496, "Finetune/Step Time": 4.6213233675807714} +{"Finetune/Learning Rate": 1.670425848588337e-05, "Finetune/Loss": 0.8936789035797119, "Finetune/Loss (Raw)": 0.8405416011810303, "Finetune/Step": 1497, "Finetune/Step Time": 4.615061864256859} +{"Finetune/Learning Rate": 1.6699634960345005e-05, "Finetune/Loss": 0.8953880667686462, "Finetune/Loss (Raw)": 1.0536589622497559, "Finetune/Step": 1498, "Finetune/Step Time": 4.617770209908485} +{"Finetune/Learning Rate": 1.6695008834809107e-05, "Finetune/Loss": 0.8912429809570312, "Finetune/Loss (Raw)": 0.42971545457839966, "Finetune/Step": 1499, "Finetune/Step Time": 4.650306385010481} +{"Finetune/Learning Rate": 1.669038011107099e-05, "Finetune/Loss": 0.8909966945648193, "Finetune/Loss (Raw)": 0.35545799136161804, "Finetune/Step": 1500, "Finetune/Step Time": 4.644272794947028} +{"Finetune/Learning Rate": 1.6685748790926975e-05, "Finetune/Loss": 0.8903211951255798, "Finetune/Loss (Raw)": 1.0174423456192017, "Finetune/Step": 1501, "Finetune/Step Time": 4.67085500061512} +{"Finetune/Learning Rate": 1.6681114876174377e-05, "Finetune/Loss": 0.8915061354637146, "Finetune/Loss (Raw)": 1.0740119218826294, "Finetune/Step": 1502, "Finetune/Step Time": 4.663209522143006} +{"Finetune/Learning Rate": 1.6676478368611536e-05, "Finetune/Loss": 0.8901143670082092, "Finetune/Loss (Raw)": 0.8445537686347961, "Finetune/Step": 1503, "Finetune/Step Time": 4.665748732164502} +{"Finetune/Learning Rate": 1.667183927003779e-05, "Finetune/Loss": 0.895193338394165, "Finetune/Loss (Raw)": 0.9715176820755005, "Finetune/Step": 1504, "Finetune/Step Time": 4.638412220403552} +{"Finetune/Learning Rate": 1.6667197582253474e-05, "Finetune/Loss": 0.8946720957756042, "Finetune/Loss (Raw)": 0.9572992324829102, "Finetune/Step": 1505, "Finetune/Step Time": 4.624184692278504} +{"Finetune/Learning Rate": 1.6662553307059947e-05, "Finetune/Loss": 0.8925148844718933, "Finetune/Loss (Raw)": 0.8646692633628845, "Finetune/Step": 1506, "Finetune/Step Time": 4.618962066248059} +{"Finetune/Learning Rate": 1.6657906446259554e-05, "Finetune/Loss": 0.8951512575149536, "Finetune/Loss (Raw)": 1.0411288738250732, "Finetune/Step": 1507, "Finetune/Step Time": 4.598047401756048} +{"Finetune/Learning Rate": 1.6653257001655652e-05, "Finetune/Loss": 0.8958274126052856, "Finetune/Loss (Raw)": 0.9975160360336304, "Finetune/Step": 1508, "Finetune/Step Time": 4.602090112864971} +{"Finetune/Learning Rate": 1.664860497505261e-05, "Finetune/Loss": 0.8955258727073669, "Finetune/Loss (Raw)": 0.9969097971916199, "Finetune/Step": 1509, "Finetune/Step Time": 4.596332602202892} +{"Finetune/Learning Rate": 1.664395036825577e-05, "Finetune/Loss": 0.8961871862411499, "Finetune/Loss (Raw)": 1.0509048700332642, "Finetune/Step": 1510, "Finetune/Step Time": 4.601315755397081} +{"Finetune/Learning Rate": 1.6639293183071514e-05, "Finetune/Loss": 0.8977819085121155, "Finetune/Loss (Raw)": 1.0893744230270386, "Finetune/Step": 1511, "Finetune/Step Time": 4.600223174318671} +{"Finetune/Learning Rate": 1.6634633421307187e-05, "Finetune/Loss": 0.8972774744033813, "Finetune/Loss (Raw)": 0.8810381293296814, "Finetune/Step": 1512, "Finetune/Step Time": 4.604077173396945} +{"Finetune/Learning Rate": 1.6629971084771165e-05, "Finetune/Loss": 0.8982603549957275, "Finetune/Loss (Raw)": 1.1297554969787598, "Finetune/Step": 1513, "Finetune/Step Time": 4.606231588870287} +{"Finetune/Learning Rate": 1.66253061752728e-05, "Finetune/Loss": 0.8983514904975891, "Finetune/Loss (Raw)": 0.924727737903595, "Finetune/Step": 1514, "Finetune/Step Time": 4.615898666903377} +{"Finetune/Learning Rate": 1.662063869462246e-05, "Finetune/Loss": 0.8991183042526245, "Finetune/Loss (Raw)": 0.9430240392684937, "Finetune/Step": 1515, "Finetune/Step Time": 4.61397241987288} +{"Finetune/Learning Rate": 1.6615968644631498e-05, "Finetune/Loss": 0.8985761404037476, "Finetune/Loss (Raw)": 0.9361301064491272, "Finetune/Step": 1516, "Finetune/Step Time": 4.616545017808676} +{"Finetune/Learning Rate": 1.661129602711227e-05, "Finetune/Loss": 0.8975778222084045, "Finetune/Loss (Raw)": 0.854376494884491, "Finetune/Step": 1517, "Finetune/Step Time": 4.618137549608946} +{"Finetune/Learning Rate": 1.6606620843878128e-05, "Finetune/Loss": 0.8969491124153137, "Finetune/Loss (Raw)": 1.062322735786438, "Finetune/Step": 1518, "Finetune/Step Time": 4.611797934398055} +{"Finetune/Learning Rate": 1.6601943096743426e-05, "Finetune/Loss": 0.8966183066368103, "Finetune/Loss (Raw)": 0.8317998051643372, "Finetune/Step": 1519, "Finetune/Step Time": 4.596930541098118} +{"Finetune/Learning Rate": 1.6597262787523493e-05, "Finetune/Loss": 0.8928442001342773, "Finetune/Loss (Raw)": 0.3956608176231384, "Finetune/Step": 1520, "Finetune/Step Time": 4.600103398784995} +{"Finetune/Learning Rate": 1.6592579918034678e-05, "Finetune/Loss": 0.8932787179946899, "Finetune/Loss (Raw)": 1.012884497642517, "Finetune/Step": 1521, "Finetune/Step Time": 4.562717452645302} +{"Finetune/Learning Rate": 1.65878944900943e-05, "Finetune/Loss": 0.8923074007034302, "Finetune/Loss (Raw)": 0.8891487717628479, "Finetune/Step": 1522, "Finetune/Step Time": 4.574379464611411} +{"Finetune/Learning Rate": 1.6583206505520695e-05, "Finetune/Loss": 0.891096293926239, "Finetune/Loss (Raw)": 0.7889775037765503, "Finetune/Step": 1523, "Finetune/Step Time": 4.588547909632325} +{"Finetune/Learning Rate": 1.6578515966133167e-05, "Finetune/Loss": 0.8922927379608154, "Finetune/Loss (Raw)": 0.8623218536376953, "Finetune/Step": 1524, "Finetune/Step Time": 4.606314348056912} +{"Finetune/Learning Rate": 1.6573822873752026e-05, "Finetune/Loss": 0.890986979007721, "Finetune/Loss (Raw)": 0.8502815365791321, "Finetune/Step": 1525, "Finetune/Step Time": 4.594730166718364} +{"Finetune/Learning Rate": 1.6569127230198575e-05, "Finetune/Loss": 0.8916088342666626, "Finetune/Loss (Raw)": 1.0649473667144775, "Finetune/Step": 1526, "Finetune/Step Time": 4.594119347631931} +{"Finetune/Learning Rate": 1.6564429037295097e-05, "Finetune/Loss": 0.892702043056488, "Finetune/Loss (Raw)": 0.929534912109375, "Finetune/Step": 1527, "Finetune/Step Time": 4.589169541373849} +{"Finetune/Learning Rate": 1.6559728296864868e-05, "Finetune/Loss": 0.8879887461662292, "Finetune/Loss (Raw)": 0.4201487898826599, "Finetune/Step": 1528, "Finetune/Step Time": 4.621940525248647} +{"Finetune/Learning Rate": 1.6555025010732156e-05, "Finetune/Loss": 0.8891041278839111, "Finetune/Loss (Raw)": 1.035274863243103, "Finetune/Step": 1529, "Finetune/Step Time": 4.6125781666487455} +{"Finetune/Learning Rate": 1.6550319180722213e-05, "Finetune/Loss": 0.8918117880821228, "Finetune/Loss (Raw)": 0.7428836226463318, "Finetune/Step": 1530, "Finetune/Step Time": 4.592773709446192} +{"Finetune/Learning Rate": 1.6545610808661278e-05, "Finetune/Loss": 0.892433762550354, "Finetune/Loss (Raw)": 0.993922770023346, "Finetune/Step": 1531, "Finetune/Step Time": 4.602204876020551} +{"Finetune/Learning Rate": 1.654089989637658e-05, "Finetune/Loss": 0.8922784328460693, "Finetune/Loss (Raw)": 0.937828540802002, "Finetune/Step": 1532, "Finetune/Step Time": 4.612115195021033} +{"Finetune/Learning Rate": 1.6536186445696337e-05, "Finetune/Loss": 0.8925266265869141, "Finetune/Loss (Raw)": 0.8448646664619446, "Finetune/Step": 1533, "Finetune/Step Time": 4.605725498870015} +{"Finetune/Learning Rate": 1.653147045844974e-05, "Finetune/Loss": 0.8923214673995972, "Finetune/Loss (Raw)": 0.9268675446510315, "Finetune/Step": 1534, "Finetune/Step Time": 4.60814986564219} +{"Finetune/Learning Rate": 1.6526751936466974e-05, "Finetune/Loss": 0.896255373954773, "Finetune/Loss (Raw)": 0.8790634870529175, "Finetune/Step": 1535, "Finetune/Step Time": 4.571666037663817} +{"Finetune/Learning Rate": 1.6522030881579207e-05, "Finetune/Loss": 0.896515965461731, "Finetune/Loss (Raw)": 0.8767945766448975, "Finetune/Step": 1536, "Finetune/Step Time": 4.577212492004037} +{"Finetune/Learning Rate": 1.6517307295618582e-05, "Finetune/Loss": 0.8957086801528931, "Finetune/Loss (Raw)": 0.8835686445236206, "Finetune/Step": 1537, "Finetune/Step Time": 4.577559286728501} +{"Finetune/Learning Rate": 1.6512581180418245e-05, "Finetune/Loss": 0.8922067880630493, "Finetune/Loss (Raw)": 0.44029250741004944, "Finetune/Step": 1538, "Finetune/Step Time": 4.58509511873126} +{"Finetune/Learning Rate": 1.650785253781229e-05, "Finetune/Loss": 0.8918790221214294, "Finetune/Loss (Raw)": 0.9718019962310791, "Finetune/Step": 1539, "Finetune/Step Time": 4.5861256793141365} +{"Finetune/Learning Rate": 1.6503121369635822e-05, "Finetune/Loss": 0.8936450481414795, "Finetune/Loss (Raw)": 0.9586017727851868, "Finetune/Step": 1540, "Finetune/Step Time": 4.58558246307075} +{"Finetune/Learning Rate": 1.6498387677724917e-05, "Finetune/Loss": 0.8939806222915649, "Finetune/Loss (Raw)": 1.0593422651290894, "Finetune/Step": 1541, "Finetune/Step Time": 4.5864340998232365} +{"Finetune/Learning Rate": 1.6493651463916622e-05, "Finetune/Loss": 0.8923074007034302, "Finetune/Loss (Raw)": 0.8698804378509521, "Finetune/Step": 1542, "Finetune/Step Time": 4.599653299897909} +{"Finetune/Learning Rate": 1.6488912730048973e-05, "Finetune/Loss": 0.8915122151374817, "Finetune/Loss (Raw)": 0.8337522149085999, "Finetune/Step": 1543, "Finetune/Step Time": 4.5955927558243275} +{"Finetune/Learning Rate": 1.6484171477960978e-05, "Finetune/Loss": 0.8907372951507568, "Finetune/Loss (Raw)": 0.9633622765541077, "Finetune/Step": 1544, "Finetune/Step Time": 4.587920198217034} +{"Finetune/Learning Rate": 1.6479427709492622e-05, "Finetune/Loss": 0.8906672596931458, "Finetune/Loss (Raw)": 0.8906683921813965, "Finetune/Step": 1545, "Finetune/Step Time": 4.5901507418602705} +{"Finetune/Learning Rate": 1.6474681426484878e-05, "Finetune/Loss": 0.8923156261444092, "Finetune/Loss (Raw)": 1.0423933267593384, "Finetune/Step": 1546, "Finetune/Step Time": 4.589205937460065} +{"Finetune/Learning Rate": 1.6469932630779677e-05, "Finetune/Loss": 0.892986536026001, "Finetune/Loss (Raw)": 0.9460562467575073, "Finetune/Step": 1547, "Finetune/Step Time": 4.595031440258026} +{"Finetune/Learning Rate": 1.6465181324219937e-05, "Finetune/Loss": 0.8931986689567566, "Finetune/Loss (Raw)": 1.0045686960220337, "Finetune/Step": 1548, "Finetune/Step Time": 4.599032299593091} +{"Finetune/Learning Rate": 1.6460427508649546e-05, "Finetune/Loss": 0.8892992734909058, "Finetune/Loss (Raw)": 0.4024036228656769, "Finetune/Step": 1549, "Finetune/Step Time": 4.644245121628046} +{"Finetune/Learning Rate": 1.6455671185913367e-05, "Finetune/Loss": 0.8897462487220764, "Finetune/Loss (Raw)": 0.9772783517837524, "Finetune/Step": 1550, "Finetune/Step Time": 4.641708668321371} +{"Finetune/Learning Rate": 1.645091235785724e-05, "Finetune/Loss": 0.8884471654891968, "Finetune/Loss (Raw)": 0.8455303907394409, "Finetune/Step": 1551, "Finetune/Step Time": 4.635576963424683} +{"Finetune/Learning Rate": 1.644615102632797e-05, "Finetune/Loss": 0.8895678520202637, "Finetune/Loss (Raw)": 0.9869574308395386, "Finetune/Step": 1552, "Finetune/Step Time": 4.636346882209182} +{"Finetune/Learning Rate": 1.6441387193173337e-05, "Finetune/Loss": 0.8887823820114136, "Finetune/Loss (Raw)": 0.8726900815963745, "Finetune/Step": 1553, "Finetune/Step Time": 4.642435954883695} +{"Finetune/Learning Rate": 1.6436620860242093e-05, "Finetune/Loss": 0.8886457085609436, "Finetune/Loss (Raw)": 0.8487468361854553, "Finetune/Step": 1554, "Finetune/Step Time": 4.632782327011228} +{"Finetune/Learning Rate": 1.6431852029383955e-05, "Finetune/Loss": 0.8894803524017334, "Finetune/Loss (Raw)": 1.0229583978652954, "Finetune/Step": 1555, "Finetune/Step Time": 4.6356781627982855} +{"Finetune/Learning Rate": 1.6427080702449616e-05, "Finetune/Loss": 0.8889411687850952, "Finetune/Loss (Raw)": 0.848992109298706, "Finetune/Step": 1556, "Finetune/Step Time": 4.640927156433463} +{"Finetune/Learning Rate": 1.6422306881290734e-05, "Finetune/Loss": 0.8889018297195435, "Finetune/Loss (Raw)": 0.833952009677887, "Finetune/Step": 1557, "Finetune/Step Time": 4.639836262911558} +{"Finetune/Learning Rate": 1.641753056775994e-05, "Finetune/Loss": 0.8887730836868286, "Finetune/Loss (Raw)": 0.9149984121322632, "Finetune/Step": 1558, "Finetune/Step Time": 4.62951635569334} +{"Finetune/Learning Rate": 1.641275176371082e-05, "Finetune/Loss": 0.8908806443214417, "Finetune/Loss (Raw)": 1.1397384405136108, "Finetune/Step": 1559, "Finetune/Step Time": 4.621049832552671} +{"Finetune/Learning Rate": 1.6407970470997943e-05, "Finetune/Loss": 0.8913729190826416, "Finetune/Loss (Raw)": 0.9914233088493347, "Finetune/Step": 1560, "Finetune/Step Time": 4.631770569831133} +{"Finetune/Learning Rate": 1.6403186691476828e-05, "Finetune/Loss": 0.8913446068763733, "Finetune/Loss (Raw)": 0.9781439304351807, "Finetune/Step": 1561, "Finetune/Step Time": 4.628742631524801} +{"Finetune/Learning Rate": 1.639840042700397e-05, "Finetune/Loss": 0.8959295749664307, "Finetune/Loss (Raw)": 0.9672693014144897, "Finetune/Step": 1562, "Finetune/Step Time": 4.599255762994289} +{"Finetune/Learning Rate": 1.6393611679436828e-05, "Finetune/Loss": 0.8951709866523743, "Finetune/Loss (Raw)": 0.9016776084899902, "Finetune/Step": 1563, "Finetune/Step Time": 4.6168511826545} +{"Finetune/Learning Rate": 1.6388820450633813e-05, "Finetune/Loss": 0.8959827423095703, "Finetune/Loss (Raw)": 0.8887360692024231, "Finetune/Step": 1564, "Finetune/Step Time": 4.602457229048014} +{"Finetune/Learning Rate": 1.6384026742454315e-05, "Finetune/Loss": 0.8946021199226379, "Finetune/Loss (Raw)": 0.8376984596252441, "Finetune/Step": 1565, "Finetune/Step Time": 4.601969040930271} +{"Finetune/Learning Rate": 1.6379230556758676e-05, "Finetune/Loss": 0.894339919090271, "Finetune/Loss (Raw)": 0.9245985150337219, "Finetune/Step": 1566, "Finetune/Step Time": 4.608850851655006} +{"Finetune/Learning Rate": 1.63744318954082e-05, "Finetune/Loss": 0.8937653303146362, "Finetune/Loss (Raw)": 0.9571806192398071, "Finetune/Step": 1567, "Finetune/Step Time": 4.615342751145363} +{"Finetune/Learning Rate": 1.636963076026516e-05, "Finetune/Loss": 0.895784854888916, "Finetune/Loss (Raw)": 0.9442227482795715, "Finetune/Step": 1568, "Finetune/Step Time": 4.587124299257994} +{"Finetune/Learning Rate": 1.6364827153192773e-05, "Finetune/Loss": 0.8948131799697876, "Finetune/Loss (Raw)": 0.8016965985298157, "Finetune/Step": 1569, "Finetune/Step Time": 4.60485952347517} +{"Finetune/Learning Rate": 1.6360021076055234e-05, "Finetune/Loss": 0.8953812122344971, "Finetune/Loss (Raw)": 1.0152918100357056, "Finetune/Step": 1570, "Finetune/Step Time": 4.601415429264307} +{"Finetune/Learning Rate": 1.6355212530717683e-05, "Finetune/Loss": 0.8958499431610107, "Finetune/Loss (Raw)": 0.9467818140983582, "Finetune/Step": 1571, "Finetune/Step Time": 4.597325902432203} +{"Finetune/Learning Rate": 1.6350401519046226e-05, "Finetune/Loss": 0.8966021537780762, "Finetune/Loss (Raw)": 0.988385796546936, "Finetune/Step": 1572, "Finetune/Step Time": 4.585240291431546} +{"Finetune/Learning Rate": 1.634558804290792e-05, "Finetune/Loss": 0.8970276713371277, "Finetune/Loss (Raw)": 0.9619036316871643, "Finetune/Step": 1573, "Finetune/Step Time": 4.601198401302099} +{"Finetune/Learning Rate": 1.634077210417078e-05, "Finetune/Loss": 0.89662104845047, "Finetune/Loss (Raw)": 0.7887105941772461, "Finetune/Step": 1574, "Finetune/Step Time": 4.579626293852925} +{"Finetune/Learning Rate": 1.633595370470378e-05, "Finetune/Loss": 0.8981295824050903, "Finetune/Loss (Raw)": 1.1421340703964233, "Finetune/Step": 1575, "Finetune/Step Time": 4.595383405685425} +{"Finetune/Learning Rate": 1.6331132846376843e-05, "Finetune/Loss": 0.8974176645278931, "Finetune/Loss (Raw)": 0.9396972060203552, "Finetune/Step": 1576, "Finetune/Step Time": 4.598426828160882} +{"Finetune/Learning Rate": 1.632630953106086e-05, "Finetune/Loss": 0.8981441259384155, "Finetune/Loss (Raw)": 0.9272396564483643, "Finetune/Step": 1577, "Finetune/Step Time": 4.595192177221179} +{"Finetune/Learning Rate": 1.6321483760627658e-05, "Finetune/Loss": 0.8992792963981628, "Finetune/Loss (Raw)": 0.9804909229278564, "Finetune/Step": 1578, "Finetune/Step Time": 4.552985234186053} +{"Finetune/Learning Rate": 1.6316655536950017e-05, "Finetune/Loss": 0.8996720314025879, "Finetune/Loss (Raw)": 0.9381674528121948, "Finetune/Step": 1579, "Finetune/Step Time": 4.5589583814144135} +{"Finetune/Learning Rate": 1.6311824861901693e-05, "Finetune/Loss": 0.8975765705108643, "Finetune/Loss (Raw)": 0.766128420829773, "Finetune/Step": 1580, "Finetune/Step Time": 4.593701692298055} +{"Finetune/Learning Rate": 1.6306991737357366e-05, "Finetune/Loss": 0.897106409072876, "Finetune/Loss (Raw)": 0.831770658493042, "Finetune/Step": 1581, "Finetune/Step Time": 4.589139584451914} +{"Finetune/Learning Rate": 1.630215616519268e-05, "Finetune/Loss": 0.8953537344932556, "Finetune/Loss (Raw)": 0.9438051581382751, "Finetune/Step": 1582, "Finetune/Step Time": 4.590637402608991} +{"Finetune/Learning Rate": 1.6297318147284223e-05, "Finetune/Loss": 0.8953540921211243, "Finetune/Loss (Raw)": 0.995884358882904, "Finetune/Step": 1583, "Finetune/Step Time": 4.5938339699059725} +{"Finetune/Learning Rate": 1.6292477685509538e-05, "Finetune/Loss": 0.8955322504043579, "Finetune/Loss (Raw)": 0.9329975247383118, "Finetune/Step": 1584, "Finetune/Step Time": 4.59526907093823} +{"Finetune/Learning Rate": 1.6287634781747113e-05, "Finetune/Loss": 0.900664210319519, "Finetune/Loss (Raw)": 1.0336577892303467, "Finetune/Step": 1585, "Finetune/Step Time": 4.556317763403058} +{"Finetune/Learning Rate": 1.628278943787639e-05, "Finetune/Loss": 0.9019500613212585, "Finetune/Loss (Raw)": 1.095367431640625, "Finetune/Step": 1586, "Finetune/Step Time": 4.609889963641763} +{"Finetune/Learning Rate": 1.6277941655777744e-05, "Finetune/Loss": 0.9017186164855957, "Finetune/Loss (Raw)": 1.0324139595031738, "Finetune/Step": 1587, "Finetune/Step Time": 4.60292580164969} +{"Finetune/Learning Rate": 1.6273091437332506e-05, "Finetune/Loss": 0.9005261063575745, "Finetune/Loss (Raw)": 0.930492639541626, "Finetune/Step": 1588, "Finetune/Step Time": 4.61347340233624} +{"Finetune/Learning Rate": 1.6268238784422954e-05, "Finetune/Loss": 0.8990581631660461, "Finetune/Loss (Raw)": 0.9194799065589905, "Finetune/Step": 1589, "Finetune/Step Time": 4.629137028008699} +{"Finetune/Learning Rate": 1.6263383698932307e-05, "Finetune/Loss": 0.898882269859314, "Finetune/Loss (Raw)": 1.0049179792404175, "Finetune/Step": 1590, "Finetune/Step Time": 4.634972864761949} +{"Finetune/Learning Rate": 1.6258526182744727e-05, "Finetune/Loss": 0.8970272541046143, "Finetune/Loss (Raw)": 0.788227915763855, "Finetune/Step": 1591, "Finetune/Step Time": 4.6508600283414125} +{"Finetune/Learning Rate": 1.6253666237745326e-05, "Finetune/Loss": 0.8969433307647705, "Finetune/Loss (Raw)": 0.8555902242660522, "Finetune/Step": 1592, "Finetune/Step Time": 4.652799090370536} +{"Finetune/Learning Rate": 1.6248803865820152e-05, "Finetune/Loss": 0.896499752998352, "Finetune/Loss (Raw)": 0.8940923810005188, "Finetune/Step": 1593, "Finetune/Step Time": 4.65045920945704} +{"Finetune/Learning Rate": 1.624393906885619e-05, "Finetune/Loss": 0.8963707685470581, "Finetune/Loss (Raw)": 0.9192771911621094, "Finetune/Step": 1594, "Finetune/Step Time": 4.655950548127294} +{"Finetune/Learning Rate": 1.6239071848741385e-05, "Finetune/Loss": 0.8955449461936951, "Finetune/Loss (Raw)": 0.8238531351089478, "Finetune/Step": 1595, "Finetune/Step Time": 4.633701169863343} +{"Finetune/Learning Rate": 1.6234202207364596e-05, "Finetune/Loss": 0.8958820700645447, "Finetune/Loss (Raw)": 1.0529117584228516, "Finetune/Step": 1596, "Finetune/Step Time": 4.638623792678118} +{"Finetune/Learning Rate": 1.6229330146615646e-05, "Finetune/Loss": 0.8992911577224731, "Finetune/Loss (Raw)": 0.8269479274749756, "Finetune/Step": 1597, "Finetune/Step Time": 4.598393730819225} +{"Finetune/Learning Rate": 1.6224455668385283e-05, "Finetune/Loss": 0.8992006778717041, "Finetune/Loss (Raw)": 0.9518123865127563, "Finetune/Step": 1598, "Finetune/Step Time": 4.611493634060025} +{"Finetune/Learning Rate": 1.62195787745652e-05, "Finetune/Loss": 0.8947014808654785, "Finetune/Loss (Raw)": 0.3811325132846832, "Finetune/Step": 1599, "Finetune/Step Time": 4.652090424671769} +{"Finetune/Learning Rate": 1.6214699467048017e-05, "Finetune/Loss": 0.895637571811676, "Finetune/Loss (Raw)": 0.9672686457633972, "Finetune/Step": 1600, "Finetune/Step Time": 4.661174949258566} +{"Finetune/Learning Rate": 1.6209817747727304e-05, "Finetune/Loss": 0.8933243751525879, "Finetune/Loss (Raw)": 0.388624906539917, "Finetune/Step": 1601, "Finetune/Step Time": 4.697251822799444} +{"Finetune/Learning Rate": 1.620493361849756e-05, "Finetune/Loss": 0.8929803371429443, "Finetune/Loss (Raw)": 1.056416630744934, "Finetune/Step": 1602, "Finetune/Step Time": 4.707016047090292} +{"Finetune/Learning Rate": 1.6200047081254223e-05, "Finetune/Loss": 0.8949546217918396, "Finetune/Loss (Raw)": 1.0018360614776611, "Finetune/Step": 1603, "Finetune/Step Time": 4.702441282570362} +{"Finetune/Learning Rate": 1.6195158137893656e-05, "Finetune/Loss": 0.8942549228668213, "Finetune/Loss (Raw)": 0.8866310119628906, "Finetune/Step": 1604, "Finetune/Step Time": 4.702027641236782} +{"Finetune/Learning Rate": 1.6190266790313165e-05, "Finetune/Loss": 0.8979829549789429, "Finetune/Loss (Raw)": 0.7923436164855957, "Finetune/Step": 1605, "Finetune/Step Time": 4.671867437660694} +{"Finetune/Learning Rate": 1.618537304041099e-05, "Finetune/Loss": 0.8956005573272705, "Finetune/Loss (Raw)": 0.7468698620796204, "Finetune/Step": 1606, "Finetune/Step Time": 4.7027938943356276} +{"Finetune/Learning Rate": 1.6180476890086297e-05, "Finetune/Loss": 0.8956910371780396, "Finetune/Loss (Raw)": 0.8869813084602356, "Finetune/Step": 1607, "Finetune/Step Time": 4.6847060564905405} +{"Finetune/Learning Rate": 1.6175578341239188e-05, "Finetune/Loss": 0.8913295269012451, "Finetune/Loss (Raw)": 0.38792866468429565, "Finetune/Step": 1608, "Finetune/Step Time": 4.723867991939187} +{"Finetune/Learning Rate": 1.617067739577069e-05, "Finetune/Loss": 0.891089677810669, "Finetune/Loss (Raw)": 0.9378323554992676, "Finetune/Step": 1609, "Finetune/Step Time": 4.711498253047466} +{"Finetune/Learning Rate": 1.616577405558277e-05, "Finetune/Loss": 0.8910597562789917, "Finetune/Loss (Raw)": 0.9295946359634399, "Finetune/Step": 1610, "Finetune/Step Time": 4.713492291048169} +{"Finetune/Learning Rate": 1.616086832257831e-05, "Finetune/Loss": 0.890882670879364, "Finetune/Loss (Raw)": 0.8495263457298279, "Finetune/Step": 1611, "Finetune/Step Time": 4.710585571825504} +{"Finetune/Learning Rate": 1.615596019866114e-05, "Finetune/Loss": 0.8901495933532715, "Finetune/Loss (Raw)": 0.902370035648346, "Finetune/Step": 1612, "Finetune/Step Time": 4.712896158918738} +{"Finetune/Learning Rate": 1.6151049685736e-05, "Finetune/Loss": 0.8895951509475708, "Finetune/Loss (Raw)": 0.8232592344284058, "Finetune/Step": 1613, "Finetune/Step Time": 4.724581059068441} +{"Finetune/Learning Rate": 1.6146136785708564e-05, "Finetune/Loss": 0.894699215888977, "Finetune/Loss (Raw)": 0.9651178121566772, "Finetune/Step": 1614, "Finetune/Step Time": 4.685034843161702} +{"Finetune/Learning Rate": 1.6141221500485437e-05, "Finetune/Loss": 0.8943428993225098, "Finetune/Loss (Raw)": 1.0374832153320312, "Finetune/Step": 1615, "Finetune/Step Time": 4.684614047408104} +{"Finetune/Learning Rate": 1.6136303831974146e-05, "Finetune/Loss": 0.8925111889839172, "Finetune/Loss (Raw)": 0.9336366057395935, "Finetune/Step": 1616, "Finetune/Step Time": 4.68954093568027} +{"Finetune/Learning Rate": 1.6131383782083135e-05, "Finetune/Loss": 0.8931248784065247, "Finetune/Loss (Raw)": 0.8711757063865662, "Finetune/Step": 1617, "Finetune/Step Time": 4.686295447871089} +{"Finetune/Learning Rate": 1.6126461352721784e-05, "Finetune/Loss": 0.8929240703582764, "Finetune/Loss (Raw)": 0.8877322673797607, "Finetune/Step": 1618, "Finetune/Step Time": 4.660178845748305} +{"Finetune/Learning Rate": 1.612153654580039e-05, "Finetune/Loss": 0.8928794264793396, "Finetune/Loss (Raw)": 0.8040381073951721, "Finetune/Step": 1619, "Finetune/Step Time": 4.64432636834681} +{"Finetune/Learning Rate": 1.6116609363230176e-05, "Finetune/Loss": 0.8934575915336609, "Finetune/Loss (Raw)": 1.1308108568191528, "Finetune/Step": 1620, "Finetune/Step Time": 4.646904645487666} +{"Finetune/Learning Rate": 1.6111679806923287e-05, "Finetune/Loss": 0.8973286151885986, "Finetune/Loss (Raw)": 0.8327195048332214, "Finetune/Step": 1621, "Finetune/Step Time": 4.612909995019436} +{"Finetune/Learning Rate": 1.6106747878792784e-05, "Finetune/Loss": 0.8982471227645874, "Finetune/Loss (Raw)": 1.0368428230285645, "Finetune/Step": 1622, "Finetune/Step Time": 4.610529087483883} +{"Finetune/Learning Rate": 1.610181358075265e-05, "Finetune/Loss": 0.8992464542388916, "Finetune/Loss (Raw)": 1.1163873672485352, "Finetune/Step": 1623, "Finetune/Step Time": 4.604812487959862} +{"Finetune/Learning Rate": 1.60968769147178e-05, "Finetune/Loss": 0.8987306356430054, "Finetune/Loss (Raw)": 0.9040670394897461, "Finetune/Step": 1624, "Finetune/Step Time": 4.6069212555885315} +{"Finetune/Learning Rate": 1.6091937882604047e-05, "Finetune/Loss": 0.899726152420044, "Finetune/Loss (Raw)": 0.9679701924324036, "Finetune/Step": 1625, "Finetune/Step Time": 4.60656869225204} +{"Finetune/Learning Rate": 1.608699648632814e-05, "Finetune/Loss": 0.8991483449935913, "Finetune/Loss (Raw)": 0.9797032475471497, "Finetune/Step": 1626, "Finetune/Step Time": 4.59728480130434} +{"Finetune/Learning Rate": 1.6082052727807732e-05, "Finetune/Loss": 0.9037163853645325, "Finetune/Loss (Raw)": 1.0144190788269043, "Finetune/Step": 1627, "Finetune/Step Time": 4.555547257885337} +{"Finetune/Learning Rate": 1.60771066089614e-05, "Finetune/Loss": 0.9084129929542542, "Finetune/Loss (Raw)": 0.9566237926483154, "Finetune/Step": 1628, "Finetune/Step Time": 4.515515549108386} +{"Finetune/Learning Rate": 1.6072158131708643e-05, "Finetune/Loss": 0.9076142311096191, "Finetune/Loss (Raw)": 0.915195107460022, "Finetune/Step": 1629, "Finetune/Step Time": 4.486099649220705} +{"Finetune/Learning Rate": 1.6067207297969864e-05, "Finetune/Loss": 0.9070993661880493, "Finetune/Loss (Raw)": 1.0081098079681396, "Finetune/Step": 1630, "Finetune/Step Time": 4.483729358762503} +{"Finetune/Learning Rate": 1.6062254109666383e-05, "Finetune/Loss": 0.9076956510543823, "Finetune/Loss (Raw)": 0.9208818674087524, "Finetune/Step": 1631, "Finetune/Step Time": 4.491997245699167} +{"Finetune/Learning Rate": 1.6057298568720436e-05, "Finetune/Loss": 0.9072155952453613, "Finetune/Loss (Raw)": 0.9100680351257324, "Finetune/Step": 1632, "Finetune/Step Time": 4.482321610674262} +{"Finetune/Learning Rate": 1.6052340677055175e-05, "Finetune/Loss": 0.906460702419281, "Finetune/Loss (Raw)": 0.8606763482093811, "Finetune/Step": 1633, "Finetune/Step Time": 4.482718480750918} +{"Finetune/Learning Rate": 1.6047380436594663e-05, "Finetune/Loss": 0.9072984457015991, "Finetune/Loss (Raw)": 0.9719014763832092, "Finetune/Step": 1634, "Finetune/Step Time": 4.4697315618395805} +{"Finetune/Learning Rate": 1.6042417849263863e-05, "Finetune/Loss": 0.9064465165138245, "Finetune/Loss (Raw)": 0.9320833683013916, "Finetune/Step": 1635, "Finetune/Step Time": 4.496159251779318} +{"Finetune/Learning Rate": 1.6037452916988665e-05, "Finetune/Loss": 0.906195342540741, "Finetune/Loss (Raw)": 0.9653620719909668, "Finetune/Step": 1636, "Finetune/Step Time": 4.493178455159068} +{"Finetune/Learning Rate": 1.6032485641695862e-05, "Finetune/Loss": 0.9056907892227173, "Finetune/Loss (Raw)": 0.9323246479034424, "Finetune/Step": 1637, "Finetune/Step Time": 4.499929901212454} +{"Finetune/Learning Rate": 1.602751602531316e-05, "Finetune/Loss": 0.904409646987915, "Finetune/Loss (Raw)": 0.8869206309318542, "Finetune/Step": 1638, "Finetune/Step Time": 4.490100119262934} +{"Finetune/Learning Rate": 1.602254406976916e-05, "Finetune/Loss": 0.9022396802902222, "Finetune/Loss (Raw)": 0.8116223812103271, "Finetune/Step": 1639, "Finetune/Step Time": 4.487068168818951} +{"Finetune/Learning Rate": 1.6017569776993388e-05, "Finetune/Loss": 0.9032187461853027, "Finetune/Loss (Raw)": 1.0063550472259521, "Finetune/Step": 1640, "Finetune/Step Time": 4.475445324555039} +{"Finetune/Learning Rate": 1.601259314891627e-05, "Finetune/Loss": 0.9022390842437744, "Finetune/Loss (Raw)": 1.0043578147888184, "Finetune/Step": 1641, "Finetune/Step Time": 4.472767297178507} +{"Finetune/Learning Rate": 1.6007614187469137e-05, "Finetune/Loss": 0.9023917317390442, "Finetune/Loss (Raw)": 0.944270670413971, "Finetune/Step": 1642, "Finetune/Step Time": 4.461930835619569} +{"Finetune/Learning Rate": 1.6002632894584227e-05, "Finetune/Loss": 0.9033151865005493, "Finetune/Loss (Raw)": 1.061227798461914, "Finetune/Step": 1643, "Finetune/Step Time": 4.469500571489334} +{"Finetune/Learning Rate": 1.599764927219468e-05, "Finetune/Loss": 0.9033100008964539, "Finetune/Loss (Raw)": 0.9354609251022339, "Finetune/Step": 1644, "Finetune/Step Time": 4.494665319100022} +{"Finetune/Learning Rate": 1.5992663322234548e-05, "Finetune/Loss": 0.9037036299705505, "Finetune/Loss (Raw)": 0.9047642946243286, "Finetune/Step": 1645, "Finetune/Step Time": 4.495313357561827} +{"Finetune/Learning Rate": 1.5987675046638777e-05, "Finetune/Loss": 0.9032245874404907, "Finetune/Loss (Raw)": 1.0010020732879639, "Finetune/Step": 1646, "Finetune/Step Time": 4.498901532962918} +{"Finetune/Learning Rate": 1.5982684447343214e-05, "Finetune/Loss": 0.9036000967025757, "Finetune/Loss (Raw)": 0.8798686861991882, "Finetune/Step": 1647, "Finetune/Step Time": 4.511626647785306} +{"Finetune/Learning Rate": 1.5977691526284623e-05, "Finetune/Loss": 0.9085020422935486, "Finetune/Loss (Raw)": 1.0231040716171265, "Finetune/Step": 1648, "Finetune/Step Time": 4.4771774765104055} +{"Finetune/Learning Rate": 1.5972696285400656e-05, "Finetune/Loss": 0.9077028036117554, "Finetune/Loss (Raw)": 0.910586953163147, "Finetune/Step": 1649, "Finetune/Step Time": 4.479587096720934} +{"Finetune/Learning Rate": 1.596769872662987e-05, "Finetune/Loss": 0.908126950263977, "Finetune/Loss (Raw)": 0.9434331655502319, "Finetune/Step": 1650, "Finetune/Step Time": 4.4678536877036095} +{"Finetune/Learning Rate": 1.5962698851911715e-05, "Finetune/Loss": 0.9086276292800903, "Finetune/Loss (Raw)": 0.8530697822570801, "Finetune/Step": 1651, "Finetune/Step Time": 4.469721004366875} +{"Finetune/Learning Rate": 1.5957696663186547e-05, "Finetune/Loss": 0.9087221622467041, "Finetune/Loss (Raw)": 0.8744144439697266, "Finetune/Step": 1652, "Finetune/Step Time": 4.452886147424579} +{"Finetune/Learning Rate": 1.595269216239562e-05, "Finetune/Loss": 0.9094111919403076, "Finetune/Loss (Raw)": 0.9384859204292297, "Finetune/Step": 1653, "Finetune/Step Time": 4.444892670959234} +{"Finetune/Learning Rate": 1.594768535148108e-05, "Finetune/Loss": 0.9087852239608765, "Finetune/Loss (Raw)": 0.9848189353942871, "Finetune/Step": 1654, "Finetune/Step Time": 4.458331333473325} +{"Finetune/Learning Rate": 1.5942676232385973e-05, "Finetune/Loss": 0.90451979637146, "Finetune/Loss (Raw)": 0.3835673928260803, "Finetune/Step": 1655, "Finetune/Step Time": 4.496006079018116} +{"Finetune/Learning Rate": 1.5937664807054242e-05, "Finetune/Loss": 0.9087026119232178, "Finetune/Loss (Raw)": 0.9555416703224182, "Finetune/Step": 1656, "Finetune/Step Time": 4.460103493183851} +{"Finetune/Learning Rate": 1.593265107743072e-05, "Finetune/Loss": 0.9073377847671509, "Finetune/Loss (Raw)": 0.8605839610099792, "Finetune/Step": 1657, "Finetune/Step Time": 4.458531990647316} +{"Finetune/Learning Rate": 1.5927635045461145e-05, "Finetune/Loss": 0.9089422821998596, "Finetune/Loss (Raw)": 0.9482587575912476, "Finetune/Step": 1658, "Finetune/Step Time": 4.443276733160019} +{"Finetune/Learning Rate": 1.5922616713092135e-05, "Finetune/Loss": 0.9092910289764404, "Finetune/Loss (Raw)": 1.0385559797286987, "Finetune/Step": 1659, "Finetune/Step Time": 4.437725763767958} +{"Finetune/Learning Rate": 1.5917596082271206e-05, "Finetune/Loss": 0.9093676209449768, "Finetune/Loss (Raw)": 0.9476361870765686, "Finetune/Step": 1660, "Finetune/Step Time": 4.433913731947541} +{"Finetune/Learning Rate": 1.5912573154946768e-05, "Finetune/Loss": 0.9097552299499512, "Finetune/Loss (Raw)": 0.8944752812385559, "Finetune/Step": 1661, "Finetune/Step Time": 4.429212341085076} +{"Finetune/Learning Rate": 1.5907547933068125e-05, "Finetune/Loss": 0.9105797410011292, "Finetune/Loss (Raw)": 1.0324121713638306, "Finetune/Step": 1662, "Finetune/Step Time": 4.433558313176036} +{"Finetune/Learning Rate": 1.590252041858546e-05, "Finetune/Loss": 0.9090421199798584, "Finetune/Loss (Raw)": 0.6822457909584045, "Finetune/Step": 1663, "Finetune/Step Time": 4.443319307640195} +{"Finetune/Learning Rate": 1.5897490613449864e-05, "Finetune/Loss": 0.9082839488983154, "Finetune/Loss (Raw)": 0.7797454595565796, "Finetune/Step": 1664, "Finetune/Step Time": 4.436088165268302} +{"Finetune/Learning Rate": 1.5892458519613296e-05, "Finetune/Loss": 0.9095982313156128, "Finetune/Loss (Raw)": 1.051792860031128, "Finetune/Step": 1665, "Finetune/Step Time": 4.436054468154907} +{"Finetune/Learning Rate": 1.588742413902862e-05, "Finetune/Loss": 0.9130958914756775, "Finetune/Loss (Raw)": 0.887998104095459, "Finetune/Step": 1666, "Finetune/Step Time": 4.392874348908663} +{"Finetune/Learning Rate": 1.5882387473649572e-05, "Finetune/Loss": 0.9130946397781372, "Finetune/Loss (Raw)": 0.9716356992721558, "Finetune/Step": 1667, "Finetune/Step Time": 4.390615779906511} +{"Finetune/Learning Rate": 1.5877348525430793e-05, "Finetune/Loss": 0.9136916399002075, "Finetune/Loss (Raw)": 1.0350172519683838, "Finetune/Step": 1668, "Finetune/Step Time": 4.395726554095745} +{"Finetune/Learning Rate": 1.5872307296327795e-05, "Finetune/Loss": 0.9127417206764221, "Finetune/Loss (Raw)": 0.9377551078796387, "Finetune/Step": 1669, "Finetune/Step Time": 4.391947258263826} +{"Finetune/Learning Rate": 1.5867263788296984e-05, "Finetune/Loss": 0.9136534333229065, "Finetune/Loss (Raw)": 0.9865792989730835, "Finetune/Step": 1670, "Finetune/Step Time": 4.37894481420517} +{"Finetune/Learning Rate": 1.5862218003295647e-05, "Finetune/Loss": 0.9139595031738281, "Finetune/Loss (Raw)": 0.8729295134544373, "Finetune/Step": 1671, "Finetune/Step Time": 4.386939447373152} +{"Finetune/Learning Rate": 1.5857169943281948e-05, "Finetune/Loss": 0.9142078161239624, "Finetune/Loss (Raw)": 0.9951478242874146, "Finetune/Step": 1672, "Finetune/Step Time": 4.4037652257829905} +{"Finetune/Learning Rate": 1.585211961021495e-05, "Finetune/Loss": 0.9135624170303345, "Finetune/Loss (Raw)": 0.8080576658248901, "Finetune/Step": 1673, "Finetune/Step Time": 4.408829180523753} +{"Finetune/Learning Rate": 1.5847067006054584e-05, "Finetune/Loss": 0.9113035202026367, "Finetune/Loss (Raw)": 0.7532626390457153, "Finetune/Step": 1674, "Finetune/Step Time": 4.4151542745530605} +{"Finetune/Learning Rate": 1.584201213276167e-05, "Finetune/Loss": 0.910689115524292, "Finetune/Loss (Raw)": 0.8674106001853943, "Finetune/Step": 1675, "Finetune/Step Time": 4.4188048001378775} +{"Finetune/Learning Rate": 1.5836954992297897e-05, "Finetune/Loss": 0.9109475612640381, "Finetune/Loss (Raw)": 1.0376487970352173, "Finetune/Step": 1676, "Finetune/Step Time": 4.413014380261302} +{"Finetune/Learning Rate": 1.583189558662585e-05, "Finetune/Loss": 0.9155658483505249, "Finetune/Loss (Raw)": 0.9935339689254761, "Finetune/Step": 1677, "Finetune/Step Time": 4.373967049643397} +{"Finetune/Learning Rate": 1.5826833917708983e-05, "Finetune/Loss": 0.9144537448883057, "Finetune/Loss (Raw)": 0.834937334060669, "Finetune/Step": 1678, "Finetune/Step Time": 4.37227133102715} +{"Finetune/Learning Rate": 1.5821769987511634e-05, "Finetune/Loss": 0.9109169840812683, "Finetune/Loss (Raw)": 0.39282429218292236, "Finetune/Step": 1679, "Finetune/Step Time": 4.413610886782408} +{"Finetune/Learning Rate": 1.5816703797999015e-05, "Finetune/Loss": 0.910439133644104, "Finetune/Loss (Raw)": 0.9257853031158447, "Finetune/Step": 1680, "Finetune/Step Time": 4.420368058606982} +{"Finetune/Learning Rate": 1.5811635351137212e-05, "Finetune/Loss": 0.9110466241836548, "Finetune/Loss (Raw)": 0.9504579305648804, "Finetune/Step": 1681, "Finetune/Step Time": 4.401871496811509} +{"Finetune/Learning Rate": 1.580656464889319e-05, "Finetune/Loss": 0.9129869937896729, "Finetune/Loss (Raw)": 1.0971142053604126, "Finetune/Step": 1682, "Finetune/Step Time": 4.404995849356055} +{"Finetune/Learning Rate": 1.5801491693234793e-05, "Finetune/Loss": 0.9115578532218933, "Finetune/Loss (Raw)": 0.840025782585144, "Finetune/Step": 1683, "Finetune/Step Time": 4.41769951954484} +{"Finetune/Learning Rate": 1.5796416486130735e-05, "Finetune/Loss": 0.9119594693183899, "Finetune/Loss (Raw)": 0.9003989696502686, "Finetune/Step": 1684, "Finetune/Step Time": 4.404088551178575} +{"Finetune/Learning Rate": 1.5791339029550604e-05, "Finetune/Loss": 0.9121062755584717, "Finetune/Loss (Raw)": 0.8527371287345886, "Finetune/Step": 1685, "Finetune/Step Time": 4.407955646514893} +{"Finetune/Learning Rate": 1.578625932546486e-05, "Finetune/Loss": 0.9110265374183655, "Finetune/Loss (Raw)": 0.7767972350120544, "Finetune/Step": 1686, "Finetune/Step Time": 4.419789297506213} +{"Finetune/Learning Rate": 1.578117737584484e-05, "Finetune/Loss": 0.9099188446998596, "Finetune/Loss (Raw)": 0.997952938079834, "Finetune/Step": 1687, "Finetune/Step Time": 4.422449957579374} +{"Finetune/Learning Rate": 1.577609318266275e-05, "Finetune/Loss": 0.9099240303039551, "Finetune/Loss (Raw)": 0.9920830130577087, "Finetune/Step": 1688, "Finetune/Step Time": 4.414073659107089} +{"Finetune/Learning Rate": 1.5771006747891663e-05, "Finetune/Loss": 0.9047110676765442, "Finetune/Loss (Raw)": 0.31089067459106445, "Finetune/Step": 1689, "Finetune/Step Time": 4.450319968163967} +{"Finetune/Learning Rate": 1.576591807350553e-05, "Finetune/Loss": 0.9054223299026489, "Finetune/Loss (Raw)": 1.0583131313323975, "Finetune/Step": 1690, "Finetune/Step Time": 4.441019162535667} +{"Finetune/Learning Rate": 1.5760827161479158e-05, "Finetune/Loss": 0.9056882262229919, "Finetune/Loss (Raw)": 0.9357130527496338, "Finetune/Step": 1691, "Finetune/Step Time": 4.423398584127426} +{"Finetune/Learning Rate": 1.5755734013788235e-05, "Finetune/Loss": 0.905439019203186, "Finetune/Loss (Raw)": 0.8568320870399475, "Finetune/Step": 1692, "Finetune/Step Time": 4.445516074076295} +{"Finetune/Learning Rate": 1.5750638632409315e-05, "Finetune/Loss": 0.9052224159240723, "Finetune/Loss (Raw)": 0.8099783062934875, "Finetune/Step": 1693, "Finetune/Step Time": 4.469448257237673} +{"Finetune/Learning Rate": 1.574554101931981e-05, "Finetune/Loss": 0.9035769701004028, "Finetune/Loss (Raw)": 0.7139735817909241, "Finetune/Step": 1694, "Finetune/Step Time": 4.471683822572231} +{"Finetune/Learning Rate": 1.5740441176498015e-05, "Finetune/Loss": 0.9029330015182495, "Finetune/Loss (Raw)": 0.8747571110725403, "Finetune/Step": 1695, "Finetune/Step Time": 4.477513926103711} +{"Finetune/Learning Rate": 1.5735339105923068e-05, "Finetune/Loss": 0.9030007123947144, "Finetune/Loss (Raw)": 0.9528856873512268, "Finetune/Step": 1696, "Finetune/Step Time": 4.475454028695822} +{"Finetune/Learning Rate": 1.5730234809574986e-05, "Finetune/Loss": 0.9043827056884766, "Finetune/Loss (Raw)": 0.9785977005958557, "Finetune/Step": 1697, "Finetune/Step Time": 4.4561973717063665} +{"Finetune/Learning Rate": 1.5725128289434647e-05, "Finetune/Loss": 0.9037095308303833, "Finetune/Loss (Raw)": 0.9291147589683533, "Finetune/Step": 1698, "Finetune/Step Time": 4.459247386083007} +{"Finetune/Learning Rate": 1.5720019547483795e-05, "Finetune/Loss": 0.9041462540626526, "Finetune/Loss (Raw)": 1.0026912689208984, "Finetune/Step": 1699, "Finetune/Step Time": 4.4575832933187485} +{"Finetune/Learning Rate": 1.571490858570503e-05, "Finetune/Loss": 0.9041351079940796, "Finetune/Loss (Raw)": 0.9869576692581177, "Finetune/Step": 1700, "Finetune/Step Time": 4.472641043365002} +{"Finetune/Learning Rate": 1.5709795406081818e-05, "Finetune/Loss": 0.9047493934631348, "Finetune/Loss (Raw)": 1.040530800819397, "Finetune/Step": 1701, "Finetune/Step Time": 4.457592472434044} +{"Finetune/Learning Rate": 1.570468001059848e-05, "Finetune/Loss": 0.9061978459358215, "Finetune/Loss (Raw)": 0.9741165041923523, "Finetune/Step": 1702, "Finetune/Step Time": 4.456907710060477} +{"Finetune/Learning Rate": 1.5699562401240206e-05, "Finetune/Loss": 0.9037418365478516, "Finetune/Loss (Raw)": 0.8277602195739746, "Finetune/Step": 1703, "Finetune/Step Time": 4.458154201507568} +{"Finetune/Learning Rate": 1.5694442579993042e-05, "Finetune/Loss": 0.8991289138793945, "Finetune/Loss (Raw)": 0.34924301505088806, "Finetune/Step": 1704, "Finetune/Step Time": 4.498763272538781} +{"Finetune/Learning Rate": 1.5689320548843882e-05, "Finetune/Loss": 0.8991706371307373, "Finetune/Loss (Raw)": 0.9325836896896362, "Finetune/Step": 1705, "Finetune/Step Time": 4.495847439393401} +{"Finetune/Learning Rate": 1.5684196309780495e-05, "Finetune/Loss": 0.8994803428649902, "Finetune/Loss (Raw)": 1.0201325416564941, "Finetune/Step": 1706, "Finetune/Step Time": 4.498114336282015} +{"Finetune/Learning Rate": 1.5679069864791493e-05, "Finetune/Loss": 0.8993950486183167, "Finetune/Loss (Raw)": 0.9272506833076477, "Finetune/Step": 1707, "Finetune/Step Time": 4.494882583618164} +{"Finetune/Learning Rate": 1.567394121586635e-05, "Finetune/Loss": 0.9006381630897522, "Finetune/Loss (Raw)": 0.9252434968948364, "Finetune/Step": 1708, "Finetune/Step Time": 4.475026486441493} +{"Finetune/Learning Rate": 1.56688103649954e-05, "Finetune/Loss": 0.9024698734283447, "Finetune/Loss (Raw)": 1.0662367343902588, "Finetune/Step": 1709, "Finetune/Step Time": 4.478378687053919} +{"Finetune/Learning Rate": 1.566367731416982e-05, "Finetune/Loss": 0.902695894241333, "Finetune/Loss (Raw)": 0.9727324843406677, "Finetune/Step": 1710, "Finetune/Step Time": 4.4872967805713415} +{"Finetune/Learning Rate": 1.5658542065381644e-05, "Finetune/Loss": 0.9026075005531311, "Finetune/Loss (Raw)": 0.9845682382583618, "Finetune/Step": 1711, "Finetune/Step Time": 4.490400129929185} +{"Finetune/Learning Rate": 1.565340462062377e-05, "Finetune/Loss": 0.9039973020553589, "Finetune/Loss (Raw)": 1.110896348953247, "Finetune/Step": 1712, "Finetune/Step Time": 4.49114010296762} +{"Finetune/Learning Rate": 1.5648264981889936e-05, "Finetune/Loss": 0.9035449028015137, "Finetune/Loss (Raw)": 0.9757490754127502, "Finetune/Step": 1713, "Finetune/Step Time": 4.491139372810721} +{"Finetune/Learning Rate": 1.5643123151174736e-05, "Finetune/Loss": 0.9026474952697754, "Finetune/Loss (Raw)": 0.9804977774620056, "Finetune/Step": 1714, "Finetune/Step Time": 4.445400185883045} +{"Finetune/Learning Rate": 1.5637979130473612e-05, "Finetune/Loss": 0.9007679224014282, "Finetune/Loss (Raw)": 0.7918322682380676, "Finetune/Step": 1715, "Finetune/Step Time": 4.454167153686285} +{"Finetune/Learning Rate": 1.563283292178286e-05, "Finetune/Loss": 0.9006257057189941, "Finetune/Loss (Raw)": 0.9122793078422546, "Finetune/Step": 1716, "Finetune/Step Time": 4.442090317606926} +{"Finetune/Learning Rate": 1.5627684527099624e-05, "Finetune/Loss": 0.9014074802398682, "Finetune/Loss (Raw)": 1.019560694694519, "Finetune/Step": 1717, "Finetune/Step Time": 4.44221319258213} +{"Finetune/Learning Rate": 1.5622533948421895e-05, "Finetune/Loss": 0.901076078414917, "Finetune/Loss (Raw)": 0.9624910950660706, "Finetune/Step": 1718, "Finetune/Step Time": 4.432357484474778} +{"Finetune/Learning Rate": 1.5617381187748507e-05, "Finetune/Loss": 0.9021849036216736, "Finetune/Loss (Raw)": 0.9301541447639465, "Finetune/Step": 1719, "Finetune/Step Time": 4.41844972409308} +{"Finetune/Learning Rate": 1.5612226247079155e-05, "Finetune/Loss": 0.9035710096359253, "Finetune/Loss (Raw)": 1.033018946647644, "Finetune/Step": 1720, "Finetune/Step Time": 4.41725267469883} +{"Finetune/Learning Rate": 1.5607069128414366e-05, "Finetune/Loss": 0.9041731357574463, "Finetune/Loss (Raw)": 0.9711633324623108, "Finetune/Step": 1721, "Finetune/Step Time": 4.418937873095274} +{"Finetune/Learning Rate": 1.5601909833755514e-05, "Finetune/Loss": 0.9045544266700745, "Finetune/Loss (Raw)": 0.9680785536766052, "Finetune/Step": 1722, "Finetune/Step Time": 4.410091686993837} +{"Finetune/Learning Rate": 1.559674836510482e-05, "Finetune/Loss": 0.9058800935745239, "Finetune/Loss (Raw)": 0.9935405850410461, "Finetune/Step": 1723, "Finetune/Step Time": 4.426414409652352} +{"Finetune/Learning Rate": 1.5591584724465363e-05, "Finetune/Loss": 0.9044778943061829, "Finetune/Loss (Raw)": 0.8734272718429565, "Finetune/Step": 1724, "Finetune/Step Time": 4.418832823634148} +{"Finetune/Learning Rate": 1.5586418913841034e-05, "Finetune/Loss": 0.9054827094078064, "Finetune/Loss (Raw)": 0.9555688500404358, "Finetune/Step": 1725, "Finetune/Step Time": 4.4144041277468204} +{"Finetune/Learning Rate": 1.558125093523659e-05, "Finetune/Loss": 0.904819905757904, "Finetune/Loss (Raw)": 0.8669713735580444, "Finetune/Step": 1726, "Finetune/Step Time": 4.406717706471682} +{"Finetune/Learning Rate": 1.5576080790657626e-05, "Finetune/Loss": 0.9093213081359863, "Finetune/Loss (Raw)": 0.9573094248771667, "Finetune/Step": 1727, "Finetune/Step Time": 4.383320704102516} +{"Finetune/Learning Rate": 1.5570908482110566e-05, "Finetune/Loss": 0.9099504947662354, "Finetune/Loss (Raw)": 1.0478094816207886, "Finetune/Step": 1728, "Finetune/Step Time": 4.374160047620535} +{"Finetune/Learning Rate": 1.5565734011602693e-05, "Finetune/Loss": 0.9147430658340454, "Finetune/Loss (Raw)": 1.0020751953125, "Finetune/Step": 1729, "Finetune/Step Time": 4.334057282656431} +{"Finetune/Learning Rate": 1.556055738114211e-05, "Finetune/Loss": 0.9128003120422363, "Finetune/Loss (Raw)": 0.8077358603477478, "Finetune/Step": 1730, "Finetune/Step Time": 4.3259763438254595} +{"Finetune/Learning Rate": 1.555537859273777e-05, "Finetune/Loss": 0.9134426116943359, "Finetune/Loss (Raw)": 1.0840524435043335, "Finetune/Step": 1731, "Finetune/Step Time": 4.322806376963854} +{"Finetune/Learning Rate": 1.5550197648399453e-05, "Finetune/Loss": 0.9148501753807068, "Finetune/Loss (Raw)": 1.0668007135391235, "Finetune/Step": 1732, "Finetune/Step Time": 4.337470455095172} +{"Finetune/Learning Rate": 1.554501455013779e-05, "Finetune/Loss": 0.9161635637283325, "Finetune/Loss (Raw)": 0.9604616761207581, "Finetune/Step": 1733, "Finetune/Step Time": 4.328729968518019} +{"Finetune/Learning Rate": 1.5539829299964236e-05, "Finetune/Loss": 0.9180762767791748, "Finetune/Loss (Raw)": 0.9916948080062866, "Finetune/Step": 1734, "Finetune/Step Time": 4.312712891027331} +{"Finetune/Learning Rate": 1.5534641899891084e-05, "Finetune/Loss": 0.9182199835777283, "Finetune/Loss (Raw)": 0.90537029504776, "Finetune/Step": 1735, "Finetune/Step Time": 4.312286471948028} +{"Finetune/Learning Rate": 1.5529452351931473e-05, "Finetune/Loss": 0.9226357936859131, "Finetune/Loss (Raw)": 0.9531517624855042, "Finetune/Step": 1736, "Finetune/Step Time": 4.272951886057854} +{"Finetune/Learning Rate": 1.5524260658099352e-05, "Finetune/Loss": 0.9221596717834473, "Finetune/Loss (Raw)": 0.8768929243087769, "Finetune/Step": 1737, "Finetune/Step Time": 4.2724152114242315} +{"Finetune/Learning Rate": 1.551906682040952e-05, "Finetune/Loss": 0.9177054166793823, "Finetune/Loss (Raw)": 0.35945209860801697, "Finetune/Step": 1738, "Finetune/Step Time": 4.308044811710715} +{"Finetune/Learning Rate": 1.5513870840877608e-05, "Finetune/Loss": 0.9181628227233887, "Finetune/Loss (Raw)": 0.9080702662467957, "Finetune/Step": 1739, "Finetune/Step Time": 4.281463269144297} +{"Finetune/Learning Rate": 1.5508672721520076e-05, "Finetune/Loss": 0.9177127480506897, "Finetune/Loss (Raw)": 0.8447585105895996, "Finetune/Step": 1740, "Finetune/Step Time": 4.2977749444544315} +{"Finetune/Learning Rate": 1.5503472464354207e-05, "Finetune/Loss": 0.919803261756897, "Finetune/Loss (Raw)": 1.0908454656600952, "Finetune/Step": 1741, "Finetune/Step Time": 4.293420743197203} +{"Finetune/Learning Rate": 1.5498270071398116e-05, "Finetune/Loss": 0.9188297986984253, "Finetune/Loss (Raw)": 0.8405225276947021, "Finetune/Step": 1742, "Finetune/Step Time": 4.301681948825717} +{"Finetune/Learning Rate": 1.549306554467076e-05, "Finetune/Loss": 0.9180415272712708, "Finetune/Loss (Raw)": 0.9365752935409546, "Finetune/Step": 1743, "Finetune/Step Time": 4.311078602448106} +{"Finetune/Learning Rate": 1.5487858886191903e-05, "Finetune/Loss": 0.9177077412605286, "Finetune/Loss (Raw)": 0.8909156322479248, "Finetune/Step": 1744, "Finetune/Step Time": 4.3066011648625135} +{"Finetune/Learning Rate": 1.548265009798216e-05, "Finetune/Loss": 0.9183522462844849, "Finetune/Loss (Raw)": 0.9536693692207336, "Finetune/Step": 1745, "Finetune/Step Time": 4.302044488489628} +{"Finetune/Learning Rate": 1.547743918206295e-05, "Finetune/Loss": 0.9193649888038635, "Finetune/Loss (Raw)": 1.017369031906128, "Finetune/Step": 1746, "Finetune/Step Time": 4.308588519692421} +{"Finetune/Learning Rate": 1.5472226140456537e-05, "Finetune/Loss": 0.9165071249008179, "Finetune/Loss (Raw)": 0.43822285532951355, "Finetune/Step": 1747, "Finetune/Step Time": 4.345903929322958} +{"Finetune/Learning Rate": 1.546701097518599e-05, "Finetune/Loss": 0.9140660166740417, "Finetune/Loss (Raw)": 0.8183590173721313, "Finetune/Step": 1748, "Finetune/Step Time": 4.356259360909462} +{"Finetune/Learning Rate": 1.546179368827522e-05, "Finetune/Loss": 0.9150155782699585, "Finetune/Loss (Raw)": 0.954260528087616, "Finetune/Step": 1749, "Finetune/Step Time": 4.355196235701442} +{"Finetune/Learning Rate": 1.545657428174895e-05, "Finetune/Loss": 0.915203332901001, "Finetune/Loss (Raw)": 1.060880422592163, "Finetune/Step": 1750, "Finetune/Step Time": 4.379327155649662} +{"Finetune/Learning Rate": 1.5451352757632734e-05, "Finetune/Loss": 0.914287805557251, "Finetune/Loss (Raw)": 0.9991897344589233, "Finetune/Step": 1751, "Finetune/Step Time": 4.382930362597108} +{"Finetune/Learning Rate": 1.544612911795294e-05, "Finetune/Loss": 0.9161875247955322, "Finetune/Loss (Raw)": 1.1472320556640625, "Finetune/Step": 1752, "Finetune/Step Time": 4.382368708029389} +{"Finetune/Learning Rate": 1.5440903364736763e-05, "Finetune/Loss": 0.9158273935317993, "Finetune/Loss (Raw)": 0.9218773245811462, "Finetune/Step": 1753, "Finetune/Step Time": 4.380426105111837} +{"Finetune/Learning Rate": 1.5435675500012212e-05, "Finetune/Loss": 0.9158058166503906, "Finetune/Loss (Raw)": 0.9769469499588013, "Finetune/Step": 1754, "Finetune/Step Time": 4.387715853750706} +{"Finetune/Learning Rate": 1.5430445525808123e-05, "Finetune/Loss": 0.9153797626495361, "Finetune/Loss (Raw)": 0.9598750472068787, "Finetune/Step": 1755, "Finetune/Step Time": 4.394584195688367} +{"Finetune/Learning Rate": 1.5425213444154152e-05, "Finetune/Loss": 0.9161967635154724, "Finetune/Loss (Raw)": 1.061200737953186, "Finetune/Step": 1756, "Finetune/Step Time": 4.398119196295738} +{"Finetune/Learning Rate": 1.5419979257080755e-05, "Finetune/Loss": 0.9118672609329224, "Finetune/Loss (Raw)": 0.3610129952430725, "Finetune/Step": 1757, "Finetune/Step Time": 4.44232114776969} +{"Finetune/Learning Rate": 1.541474296661923e-05, "Finetune/Loss": 0.910455584526062, "Finetune/Loss (Raw)": 0.8274264335632324, "Finetune/Step": 1758, "Finetune/Step Time": 4.44678701646626} +{"Finetune/Learning Rate": 1.5409504574801667e-05, "Finetune/Loss": 0.9111729264259338, "Finetune/Loss (Raw)": 1.012697458267212, "Finetune/Step": 1759, "Finetune/Step Time": 4.447246480733156} +{"Finetune/Learning Rate": 1.5404264083660993e-05, "Finetune/Loss": 0.9121310710906982, "Finetune/Loss (Raw)": 1.0327074527740479, "Finetune/Step": 1760, "Finetune/Step Time": 4.44542152993381} +{"Finetune/Learning Rate": 1.539902149523094e-05, "Finetune/Loss": 0.9120283126831055, "Finetune/Loss (Raw)": 0.8475275635719299, "Finetune/Step": 1761, "Finetune/Step Time": 4.450281111523509} +{"Finetune/Learning Rate": 1.5393776811546047e-05, "Finetune/Loss": 0.9109025001525879, "Finetune/Loss (Raw)": 0.8277981281280518, "Finetune/Step": 1762, "Finetune/Step Time": 4.459684122353792} +{"Finetune/Learning Rate": 1.538853003464168e-05, "Finetune/Loss": 0.9119871854782104, "Finetune/Loss (Raw)": 1.070925235748291, "Finetune/Step": 1763, "Finetune/Step Time": 4.449610745534301} +{"Finetune/Learning Rate": 1.538328116655401e-05, "Finetune/Loss": 0.9125295281410217, "Finetune/Loss (Raw)": 1.0347822904586792, "Finetune/Step": 1764, "Finetune/Step Time": 4.469530886039138} +{"Finetune/Learning Rate": 1.5378030209320018e-05, "Finetune/Loss": 0.9120211601257324, "Finetune/Loss (Raw)": 0.8672434091567993, "Finetune/Step": 1765, "Finetune/Step Time": 4.476789940148592} +{"Finetune/Learning Rate": 1.53727771649775e-05, "Finetune/Loss": 0.9127342700958252, "Finetune/Loss (Raw)": 0.9782059788703918, "Finetune/Step": 1766, "Finetune/Step Time": 4.479097468778491} +{"Finetune/Learning Rate": 1.5367522035565055e-05, "Finetune/Loss": 0.9127691984176636, "Finetune/Loss (Raw)": 0.8160946369171143, "Finetune/Step": 1767, "Finetune/Step Time": 4.480339473113418} +{"Finetune/Learning Rate": 1.5362264823122107e-05, "Finetune/Loss": 0.9117411375045776, "Finetune/Loss (Raw)": 0.8747583627700806, "Finetune/Step": 1768, "Finetune/Step Time": 4.481178374961019} +{"Finetune/Learning Rate": 1.5357005529688868e-05, "Finetune/Loss": 0.9100345373153687, "Finetune/Loss (Raw)": 0.7859112620353699, "Finetune/Step": 1769, "Finetune/Step Time": 4.496115695685148} +{"Finetune/Learning Rate": 1.5351744157306366e-05, "Finetune/Loss": 0.9104351997375488, "Finetune/Loss (Raw)": 0.9955627918243408, "Finetune/Step": 1770, "Finetune/Step Time": 4.497123872861266} +{"Finetune/Learning Rate": 1.5346480708016445e-05, "Finetune/Loss": 0.9090063571929932, "Finetune/Loss (Raw)": 0.87833172082901, "Finetune/Step": 1771, "Finetune/Step Time": 4.494916899129748} +{"Finetune/Learning Rate": 1.5341215183861738e-05, "Finetune/Loss": 0.9100644588470459, "Finetune/Loss (Raw)": 1.0709017515182495, "Finetune/Step": 1772, "Finetune/Step Time": 4.469504000619054} +{"Finetune/Learning Rate": 1.53359475868857e-05, "Finetune/Loss": 0.910246729850769, "Finetune/Loss (Raw)": 0.9280887842178345, "Finetune/Step": 1773, "Finetune/Step Time": 4.4900732189416885} +{"Finetune/Learning Rate": 1.5330677919132577e-05, "Finetune/Loss": 0.9097079038619995, "Finetune/Loss (Raw)": 0.9320419430732727, "Finetune/Step": 1774, "Finetune/Step Time": 4.503094535320997} +{"Finetune/Learning Rate": 1.5325406182647426e-05, "Finetune/Loss": 0.9108047485351562, "Finetune/Loss (Raw)": 1.0202579498291016, "Finetune/Step": 1775, "Finetune/Step Time": 4.4897124990820885} +{"Finetune/Learning Rate": 1.5320132379476107e-05, "Finetune/Loss": 0.9101218581199646, "Finetune/Loss (Raw)": 0.9356986880302429, "Finetune/Step": 1776, "Finetune/Step Time": 4.48808809928596} +{"Finetune/Learning Rate": 1.531485651166528e-05, "Finetune/Loss": 0.91065514087677, "Finetune/Loss (Raw)": 0.9788439869880676, "Finetune/Step": 1777, "Finetune/Step Time": 4.482211131602526} +{"Finetune/Learning Rate": 1.5309578581262404e-05, "Finetune/Loss": 0.9105509519577026, "Finetune/Loss (Raw)": 0.9300919771194458, "Finetune/Step": 1778, "Finetune/Step Time": 4.4841528087854385} +{"Finetune/Learning Rate": 1.530429859031574e-05, "Finetune/Loss": 0.9123886823654175, "Finetune/Loss (Raw)": 1.0882993936538696, "Finetune/Step": 1779, "Finetune/Step Time": 4.468279955908656} +{"Finetune/Learning Rate": 1.5299016540874346e-05, "Finetune/Loss": 0.912569522857666, "Finetune/Loss (Raw)": 0.8975656628608704, "Finetune/Step": 1780, "Finetune/Step Time": 4.460114497691393} +{"Finetune/Learning Rate": 1.5293732434988095e-05, "Finetune/Loss": 0.9115239381790161, "Finetune/Loss (Raw)": 0.8046526908874512, "Finetune/Step": 1781, "Finetune/Step Time": 4.461563492193818} +{"Finetune/Learning Rate": 1.5288446274707635e-05, "Finetune/Loss": 0.9109880328178406, "Finetune/Loss (Raw)": 0.9162184000015259, "Finetune/Step": 1782, "Finetune/Step Time": 4.448696816340089} +{"Finetune/Learning Rate": 1.5283158062084423e-05, "Finetune/Loss": 0.9145705699920654, "Finetune/Loss (Raw)": 0.8421376943588257, "Finetune/Step": 1783, "Finetune/Step Time": 4.41087201423943} +{"Finetune/Learning Rate": 1.5277867799170708e-05, "Finetune/Loss": 0.9153130054473877, "Finetune/Loss (Raw)": 1.0505679845809937, "Finetune/Step": 1784, "Finetune/Step Time": 4.415273433551192} +{"Finetune/Learning Rate": 1.527257548801955e-05, "Finetune/Loss": 0.9144545793533325, "Finetune/Loss (Raw)": 0.7507025599479675, "Finetune/Step": 1785, "Finetune/Step Time": 4.429647879675031} +{"Finetune/Learning Rate": 1.526728113068478e-05, "Finetune/Loss": 0.9140680432319641, "Finetune/Loss (Raw)": 0.898785412311554, "Finetune/Step": 1786, "Finetune/Step Time": 4.4271082896739244} +{"Finetune/Learning Rate": 1.526198472922104e-05, "Finetune/Loss": 0.9120614528656006, "Finetune/Loss (Raw)": 0.7817202210426331, "Finetune/Step": 1787, "Finetune/Step Time": 4.432463839650154} +{"Finetune/Learning Rate": 1.5256686285683753e-05, "Finetune/Loss": 0.9113688468933105, "Finetune/Loss (Raw)": 0.8589848279953003, "Finetune/Step": 1788, "Finetune/Step Time": 4.428150672465563} +{"Finetune/Learning Rate": 1.5251385802129149e-05, "Finetune/Loss": 0.9067096710205078, "Finetune/Loss (Raw)": 0.29809269309043884, "Finetune/Step": 1789, "Finetune/Step Time": 4.467425022274256} +{"Finetune/Learning Rate": 1.5246083280614241e-05, "Finetune/Loss": 0.9064135551452637, "Finetune/Loss (Raw)": 0.9945123195648193, "Finetune/Step": 1790, "Finetune/Step Time": 4.465224010869861} +{"Finetune/Learning Rate": 1.5240778723196833e-05, "Finetune/Loss": 0.9092617630958557, "Finetune/Loss (Raw)": 1.046815276145935, "Finetune/Step": 1791, "Finetune/Step Time": 4.457449207082391} +{"Finetune/Learning Rate": 1.523547213193552e-05, "Finetune/Loss": 0.9109002351760864, "Finetune/Loss (Raw)": 0.9894653558731079, "Finetune/Step": 1792, "Finetune/Step Time": 4.458111992105842} +{"Finetune/Learning Rate": 1.5230163508889686e-05, "Finetune/Loss": 0.9089678525924683, "Finetune/Loss (Raw)": 0.8044514656066895, "Finetune/Step": 1793, "Finetune/Step Time": 4.472873477265239} +{"Finetune/Learning Rate": 1.5224852856119503e-05, "Finetune/Loss": 0.9099712371826172, "Finetune/Loss (Raw)": 1.0164306163787842, "Finetune/Step": 1794, "Finetune/Step Time": 4.484258893877268} +{"Finetune/Learning Rate": 1.5219540175685938e-05, "Finetune/Loss": 0.9052956104278564, "Finetune/Loss (Raw)": 0.3731619715690613, "Finetune/Step": 1795, "Finetune/Step Time": 4.524649640545249} +{"Finetune/Learning Rate": 1.5214225469650727e-05, "Finetune/Loss": 0.9042292237281799, "Finetune/Loss (Raw)": 0.8985169529914856, "Finetune/Step": 1796, "Finetune/Step Time": 4.51742297410965} +{"Finetune/Learning Rate": 1.5208908740076416e-05, "Finetune/Loss": 0.9046400785446167, "Finetune/Loss (Raw)": 0.9903399348258972, "Finetune/Step": 1797, "Finetune/Step Time": 4.517051907256246} +{"Finetune/Learning Rate": 1.5203589989026318e-05, "Finetune/Loss": 0.9041826725006104, "Finetune/Loss (Raw)": 0.9280311465263367, "Finetune/Step": 1798, "Finetune/Step Time": 4.514986574649811} +{"Finetune/Learning Rate": 1.5198269218564535e-05, "Finetune/Loss": 0.904316782951355, "Finetune/Loss (Raw)": 0.8900923132896423, "Finetune/Step": 1799, "Finetune/Step Time": 4.506313268095255} +{"Finetune/Learning Rate": 1.5192946430755955e-05, "Finetune/Loss": 0.9036327600479126, "Finetune/Loss (Raw)": 0.90760338306427, "Finetune/Step": 1800, "Finetune/Step Time": 4.505550159141421} +{"Finetune/Learning Rate": 1.518762162766625e-05, "Finetune/Loss": 0.9054569005966187, "Finetune/Loss (Raw)": 1.0415456295013428, "Finetune/Step": 1801, "Finetune/Step Time": 4.5098396223038435} +{"Finetune/Learning Rate": 1.5182294811361871e-05, "Finetune/Loss": 0.9068938493728638, "Finetune/Loss (Raw)": 0.9371883869171143, "Finetune/Step": 1802, "Finetune/Step Time": 4.512431539595127} +{"Finetune/Learning Rate": 1.5176965983910053e-05, "Finetune/Loss": 0.9071875810623169, "Finetune/Loss (Raw)": 0.9050052165985107, "Finetune/Step": 1803, "Finetune/Step Time": 4.525124356150627} +{"Finetune/Learning Rate": 1.5171635147378804e-05, "Finetune/Loss": 0.9065183401107788, "Finetune/Loss (Raw)": 0.951988697052002, "Finetune/Step": 1804, "Finetune/Step Time": 4.528118710964918} +{"Finetune/Learning Rate": 1.5166302303836928e-05, "Finetune/Loss": 0.9037623405456543, "Finetune/Loss (Raw)": 0.6407643556594849, "Finetune/Step": 1805, "Finetune/Step Time": 4.52991234511137} +{"Finetune/Learning Rate": 1.5160967455353985e-05, "Finetune/Loss": 0.9039221405982971, "Finetune/Loss (Raw)": 0.8553950190544128, "Finetune/Step": 1806, "Finetune/Step Time": 4.547464989125729} +{"Finetune/Learning Rate": 1.5155630604000338e-05, "Finetune/Loss": 0.9081271886825562, "Finetune/Loss (Raw)": 0.9310751557350159, "Finetune/Step": 1807, "Finetune/Step Time": 4.507290817797184} +{"Finetune/Learning Rate": 1.5150291751847106e-05, "Finetune/Loss": 0.9081872701644897, "Finetune/Loss (Raw)": 0.9334710836410522, "Finetune/Step": 1808, "Finetune/Step Time": 4.49655163474381} +{"Finetune/Learning Rate": 1.5144950900966198e-05, "Finetune/Loss": 0.9080784320831299, "Finetune/Loss (Raw)": 0.9365311861038208, "Finetune/Step": 1809, "Finetune/Step Time": 4.496165161952376} +{"Finetune/Learning Rate": 1.5139608053430288e-05, "Finetune/Loss": 0.906829833984375, "Finetune/Loss (Raw)": 0.9372950792312622, "Finetune/Step": 1810, "Finetune/Step Time": 4.503316713497043} +{"Finetune/Learning Rate": 1.5134263211312837e-05, "Finetune/Loss": 0.9074876308441162, "Finetune/Loss (Raw)": 0.9242121577262878, "Finetune/Step": 1811, "Finetune/Step Time": 4.491555800661445} +{"Finetune/Learning Rate": 1.5128916376688071e-05, "Finetune/Loss": 0.9073660373687744, "Finetune/Loss (Raw)": 0.8848395943641663, "Finetune/Step": 1812, "Finetune/Step Time": 4.498485818505287} +{"Finetune/Learning Rate": 1.5123567551630993e-05, "Finetune/Loss": 0.9078187942504883, "Finetune/Loss (Raw)": 0.9106929898262024, "Finetune/Step": 1813, "Finetune/Step Time": 4.487598920240998} +{"Finetune/Learning Rate": 1.511821673821738e-05, "Finetune/Loss": 0.9093222618103027, "Finetune/Loss (Raw)": 0.9692407846450806, "Finetune/Step": 1814, "Finetune/Step Time": 4.475396566092968} +{"Finetune/Learning Rate": 1.5112863938523774e-05, "Finetune/Loss": 0.9083800911903381, "Finetune/Loss (Raw)": 0.8773481845855713, "Finetune/Step": 1815, "Finetune/Step Time": 4.473126318305731} +{"Finetune/Learning Rate": 1.5107509154627491e-05, "Finetune/Loss": 0.9086008071899414, "Finetune/Loss (Raw)": 1.0203357934951782, "Finetune/Step": 1816, "Finetune/Step Time": 4.46825996786356} +{"Finetune/Learning Rate": 1.5102152388606625e-05, "Finetune/Loss": 0.9132264256477356, "Finetune/Loss (Raw)": 0.9029730558395386, "Finetune/Step": 1817, "Finetune/Step Time": 4.431985950097442} +{"Finetune/Learning Rate": 1.5096793642540027e-05, "Finetune/Loss": 0.9119765758514404, "Finetune/Loss (Raw)": 0.898335337638855, "Finetune/Step": 1818, "Finetune/Step Time": 4.433154238387942} +{"Finetune/Learning Rate": 1.5091432918507322e-05, "Finetune/Loss": 0.9122204780578613, "Finetune/Loss (Raw)": 0.966924786567688, "Finetune/Step": 1819, "Finetune/Step Time": 4.430118864402175} +{"Finetune/Learning Rate": 1.5086070218588904e-05, "Finetune/Loss": 0.9120535850524902, "Finetune/Loss (Raw)": 0.8354761600494385, "Finetune/Step": 1820, "Finetune/Step Time": 4.408223757520318} +{"Finetune/Learning Rate": 1.5080705544865932e-05, "Finetune/Loss": 0.9131588339805603, "Finetune/Loss (Raw)": 0.9514517188072205, "Finetune/Step": 1821, "Finetune/Step Time": 4.388401510193944} +{"Finetune/Learning Rate": 1.5075338899420326e-05, "Finetune/Loss": 0.916778028011322, "Finetune/Loss (Raw)": 1.1772290468215942, "Finetune/Step": 1822, "Finetune/Step Time": 4.380327500402927} +{"Finetune/Learning Rate": 1.5069970284334785e-05, "Finetune/Loss": 0.9172626733779907, "Finetune/Loss (Raw)": 0.9367867708206177, "Finetune/Step": 1823, "Finetune/Step Time": 4.3715307135134935} +{"Finetune/Learning Rate": 1.506459970169276e-05, "Finetune/Loss": 0.9163246154785156, "Finetune/Loss (Raw)": 0.8328172564506531, "Finetune/Step": 1824, "Finetune/Step Time": 4.375962240621448} +{"Finetune/Learning Rate": 1.5059227153578466e-05, "Finetune/Loss": 0.9157162308692932, "Finetune/Loss (Raw)": 0.9007295966148376, "Finetune/Step": 1825, "Finetune/Step Time": 4.382877502590418} +{"Finetune/Learning Rate": 1.505385264207689e-05, "Finetune/Loss": 0.9163213968276978, "Finetune/Loss (Raw)": 1.0065721273422241, "Finetune/Step": 1826, "Finetune/Step Time": 4.383241074159741} +{"Finetune/Learning Rate": 1.5048476169273767e-05, "Finetune/Loss": 0.9157923460006714, "Finetune/Loss (Raw)": 0.9349716901779175, "Finetune/Step": 1827, "Finetune/Step Time": 4.415478380396962} +{"Finetune/Learning Rate": 1.5043097737255607e-05, "Finetune/Loss": 0.9164121150970459, "Finetune/Loss (Raw)": 1.0662894248962402, "Finetune/Step": 1828, "Finetune/Step Time": 4.4076515436172485} +{"Finetune/Learning Rate": 1.5037717348109677e-05, "Finetune/Loss": 0.9170029163360596, "Finetune/Loss (Raw)": 1.1161525249481201, "Finetune/Step": 1829, "Finetune/Step Time": 4.411898102611303} +{"Finetune/Learning Rate": 1.5032335003923994e-05, "Finetune/Loss": 0.9175020456314087, "Finetune/Loss (Raw)": 1.0380061864852905, "Finetune/Step": 1830, "Finetune/Step Time": 4.424331029877067} +{"Finetune/Learning Rate": 1.5026950706787346e-05, "Finetune/Loss": 0.9181538820266724, "Finetune/Loss (Raw)": 0.9111947417259216, "Finetune/Step": 1831, "Finetune/Step Time": 4.409036736935377} +{"Finetune/Learning Rate": 1.502156445878927e-05, "Finetune/Loss": 0.9234601259231567, "Finetune/Loss (Raw)": 1.0284395217895508, "Finetune/Step": 1832, "Finetune/Step Time": 4.368387974798679} +{"Finetune/Learning Rate": 1.5016176262020069e-05, "Finetune/Loss": 0.9227352142333984, "Finetune/Loss (Raw)": 0.8397973775863647, "Finetune/Step": 1833, "Finetune/Step Time": 4.387549998238683} +{"Finetune/Learning Rate": 1.5010786118570791e-05, "Finetune/Loss": 0.923261284828186, "Finetune/Loss (Raw)": 1.087468147277832, "Finetune/Step": 1834, "Finetune/Step Time": 4.386633018031716} +{"Finetune/Learning Rate": 1.5005394030533246e-05, "Finetune/Loss": 0.9233827590942383, "Finetune/Loss (Raw)": 0.9428014159202576, "Finetune/Step": 1835, "Finetune/Step Time": 4.395473008975387} +{"Finetune/Learning Rate": 1.5000000000000002e-05, "Finetune/Loss": 0.9237382411956787, "Finetune/Loss (Raw)": 0.9707412719726562, "Finetune/Step": 1836, "Finetune/Step Time": 4.390626227483153} +{"Finetune/Learning Rate": 1.4994604029064372e-05, "Finetune/Loss": 0.9226688742637634, "Finetune/Loss (Raw)": 0.9293602705001831, "Finetune/Step": 1837, "Finetune/Step Time": 4.391821736469865} +{"Finetune/Learning Rate": 1.4989206119820432e-05, "Finetune/Loss": 0.9226732850074768, "Finetune/Loss (Raw)": 0.9733005166053772, "Finetune/Step": 1838, "Finetune/Step Time": 4.390288043767214} +{"Finetune/Learning Rate": 1.4983806274363005e-05, "Finetune/Loss": 0.9225519895553589, "Finetune/Loss (Raw)": 0.9690355062484741, "Finetune/Step": 1839, "Finetune/Step Time": 4.398082150146365} +{"Finetune/Learning Rate": 1.497840449478766e-05, "Finetune/Loss": 0.9217071533203125, "Finetune/Loss (Raw)": 1.0027563571929932, "Finetune/Step": 1840, "Finetune/Step Time": 4.407646927982569} +{"Finetune/Learning Rate": 1.4973000783190728e-05, "Finetune/Loss": 0.9204992055892944, "Finetune/Loss (Raw)": 0.8211382031440735, "Finetune/Step": 1841, "Finetune/Step Time": 4.40953522734344} +{"Finetune/Learning Rate": 1.4967595141669278e-05, "Finetune/Loss": 0.9210731983184814, "Finetune/Loss (Raw)": 1.0539722442626953, "Finetune/Step": 1842, "Finetune/Step Time": 4.408440191298723} +{"Finetune/Learning Rate": 1.4962187572321143e-05, "Finetune/Loss": 0.922417402267456, "Finetune/Loss (Raw)": 0.963879406452179, "Finetune/Step": 1843, "Finetune/Step Time": 4.413923241198063} +{"Finetune/Learning Rate": 1.4956778077244884e-05, "Finetune/Loss": 0.9225382208824158, "Finetune/Loss (Raw)": 0.9277538061141968, "Finetune/Step": 1844, "Finetune/Step Time": 4.424971044063568} +{"Finetune/Learning Rate": 1.4951366658539832e-05, "Finetune/Loss": 0.921705961227417, "Finetune/Loss (Raw)": 0.9130316972732544, "Finetune/Step": 1845, "Finetune/Step Time": 4.418637620285153} +{"Finetune/Learning Rate": 1.4945953318306044e-05, "Finetune/Loss": 0.9223583936691284, "Finetune/Loss (Raw)": 1.0460000038146973, "Finetune/Step": 1846, "Finetune/Step Time": 4.420398944988847} +{"Finetune/Learning Rate": 1.4940538058644332e-05, "Finetune/Loss": 0.9228953123092651, "Finetune/Loss (Raw)": 0.9988837838172913, "Finetune/Step": 1847, "Finetune/Step Time": 4.429137736558914} +{"Finetune/Learning Rate": 1.4935120881656263e-05, "Finetune/Loss": 0.9222079515457153, "Finetune/Loss (Raw)": 0.9450337886810303, "Finetune/Step": 1848, "Finetune/Step Time": 4.421010406687856} +{"Finetune/Learning Rate": 1.4929701789444122e-05, "Finetune/Loss": 0.922717809677124, "Finetune/Loss (Raw)": 1.036425232887268, "Finetune/Step": 1849, "Finetune/Step Time": 4.414194654673338} +{"Finetune/Learning Rate": 1.4924280784110965e-05, "Finetune/Loss": 0.9232083559036255, "Finetune/Loss (Raw)": 1.0308665037155151, "Finetune/Step": 1850, "Finetune/Step Time": 4.41934534534812} +{"Finetune/Learning Rate": 1.4918857867760571e-05, "Finetune/Loss": 0.9219318628311157, "Finetune/Loss (Raw)": 0.8301544189453125, "Finetune/Step": 1851, "Finetune/Step Time": 4.422410840168595} +{"Finetune/Learning Rate": 1.4913433042497473e-05, "Finetune/Loss": 0.9228166937828064, "Finetune/Loss (Raw)": 0.9866852760314941, "Finetune/Step": 1852, "Finetune/Step Time": 4.423788944259286} +{"Finetune/Learning Rate": 1.4908006310426942e-05, "Finetune/Loss": 0.922895073890686, "Finetune/Loss (Raw)": 0.9655963778495789, "Finetune/Step": 1853, "Finetune/Step Time": 4.424713185057044} +{"Finetune/Learning Rate": 1.4902577673654977e-05, "Finetune/Loss": 0.9237010478973389, "Finetune/Loss (Raw)": 0.9701363444328308, "Finetune/Step": 1854, "Finetune/Step Time": 4.41876651532948} +{"Finetune/Learning Rate": 1.4897147134288334e-05, "Finetune/Loss": 0.9241583347320557, "Finetune/Loss (Raw)": 1.0158361196517944, "Finetune/Step": 1855, "Finetune/Step Time": 4.404009176418185} +{"Finetune/Learning Rate": 1.4891714694434495e-05, "Finetune/Loss": 0.9242798089981079, "Finetune/Loss (Raw)": 1.0633653402328491, "Finetune/Step": 1856, "Finetune/Step Time": 4.404178649187088} +{"Finetune/Learning Rate": 1.4886280356201688e-05, "Finetune/Loss": 0.9232954978942871, "Finetune/Loss (Raw)": 0.8760801553726196, "Finetune/Step": 1857, "Finetune/Step Time": 4.4410021267831326} +{"Finetune/Learning Rate": 1.4880844121698872e-05, "Finetune/Loss": 0.9246668815612793, "Finetune/Loss (Raw)": 0.983278214931488, "Finetune/Step": 1858, "Finetune/Step Time": 4.448709096759558} +{"Finetune/Learning Rate": 1.487540599303574e-05, "Finetune/Loss": 0.9244109392166138, "Finetune/Loss (Raw)": 1.051293969154358, "Finetune/Step": 1859, "Finetune/Step Time": 4.443243928253651} +{"Finetune/Learning Rate": 1.4869965972322728e-05, "Finetune/Loss": 0.9230496883392334, "Finetune/Loss (Raw)": 0.8925590515136719, "Finetune/Step": 1860, "Finetune/Step Time": 4.4316956251859665} +{"Finetune/Learning Rate": 1.4864524061670999e-05, "Finetune/Loss": 0.9221796989440918, "Finetune/Loss (Raw)": 0.8490963578224182, "Finetune/Step": 1861, "Finetune/Step Time": 4.431961182504892} +{"Finetune/Learning Rate": 1.4859080263192456e-05, "Finetune/Loss": 0.921119213104248, "Finetune/Loss (Raw)": 0.8559579849243164, "Finetune/Step": 1862, "Finetune/Step Time": 4.416154187172651} +{"Finetune/Learning Rate": 1.4853634578999728e-05, "Finetune/Loss": 0.9204199314117432, "Finetune/Loss (Raw)": 0.8158619403839111, "Finetune/Step": 1863, "Finetune/Step Time": 4.41602835804224} +{"Finetune/Learning Rate": 1.4848187011206177e-05, "Finetune/Loss": 0.9194722175598145, "Finetune/Loss (Raw)": 0.8318467736244202, "Finetune/Step": 1864, "Finetune/Step Time": 4.426485558971763} +{"Finetune/Learning Rate": 1.4842737561925901e-05, "Finetune/Loss": 0.9205952286720276, "Finetune/Loss (Raw)": 1.0206365585327148, "Finetune/Step": 1865, "Finetune/Step Time": 4.42873115837574} +{"Finetune/Learning Rate": 1.4837286233273722e-05, "Finetune/Loss": 0.9258790016174316, "Finetune/Loss (Raw)": 1.0357776880264282, "Finetune/Step": 1866, "Finetune/Step Time": 4.403015039861202} +{"Finetune/Learning Rate": 1.48318330273652e-05, "Finetune/Loss": 0.9271872043609619, "Finetune/Loss (Raw)": 1.0755184888839722, "Finetune/Step": 1867, "Finetune/Step Time": 4.4093344286084175} +{"Finetune/Learning Rate": 1.4826377946316611e-05, "Finetune/Loss": 0.9280202388763428, "Finetune/Loss (Raw)": 0.9513856172561646, "Finetune/Step": 1868, "Finetune/Step Time": 4.4109328761696815} +{"Finetune/Learning Rate": 1.4820920992244964e-05, "Finetune/Loss": 0.927579402923584, "Finetune/Loss (Raw)": 1.0344245433807373, "Finetune/Step": 1869, "Finetune/Step Time": 4.402740254998207} +{"Finetune/Learning Rate": 1.481546216726801e-05, "Finetune/Loss": 0.9278801679611206, "Finetune/Loss (Raw)": 0.8790128231048584, "Finetune/Step": 1870, "Finetune/Step Time": 4.395586019381881} +{"Finetune/Learning Rate": 1.4810001473504194e-05, "Finetune/Loss": 0.9277952909469604, "Finetune/Loss (Raw)": 0.9257071614265442, "Finetune/Step": 1871, "Finetune/Step Time": 4.409350927919149} +{"Finetune/Learning Rate": 1.4804538913072719e-05, "Finetune/Loss": 0.9272757768630981, "Finetune/Loss (Raw)": 0.8244193196296692, "Finetune/Step": 1872, "Finetune/Step Time": 4.41415748745203} +{"Finetune/Learning Rate": 1.479907448809349e-05, "Finetune/Loss": 0.9227006435394287, "Finetune/Loss (Raw)": 0.3680495321750641, "Finetune/Step": 1873, "Finetune/Step Time": 4.454055862501264} +{"Finetune/Learning Rate": 1.4793608200687146e-05, "Finetune/Loss": 0.9228968024253845, "Finetune/Loss (Raw)": 1.042485237121582, "Finetune/Step": 1874, "Finetune/Step Time": 4.454713724553585} +{"Finetune/Learning Rate": 1.478814005297505e-05, "Finetune/Loss": 0.9262760877609253, "Finetune/Loss (Raw)": 0.870772123336792, "Finetune/Step": 1875, "Finetune/Step Time": 4.427245527505875} +{"Finetune/Learning Rate": 1.4782670047079275e-05, "Finetune/Loss": 0.9272755980491638, "Finetune/Loss (Raw)": 0.9462900757789612, "Finetune/Step": 1876, "Finetune/Step Time": 4.413823824375868} +{"Finetune/Learning Rate": 1.477719818512263e-05, "Finetune/Loss": 0.9272189140319824, "Finetune/Loss (Raw)": 0.947015106678009, "Finetune/Step": 1877, "Finetune/Step Time": 4.418495357036591} +{"Finetune/Learning Rate": 1.4771724469228638e-05, "Finetune/Loss": 0.9250898361206055, "Finetune/Loss (Raw)": 0.7883468866348267, "Finetune/Step": 1878, "Finetune/Step Time": 4.405954979360104} +{"Finetune/Learning Rate": 1.4766248901521539e-05, "Finetune/Loss": 0.9255452156066895, "Finetune/Loss (Raw)": 1.057479977607727, "Finetune/Step": 1879, "Finetune/Step Time": 4.397444061934948} +{"Finetune/Learning Rate": 1.4760771484126296e-05, "Finetune/Loss": 0.9227806329727173, "Finetune/Loss (Raw)": 0.7933641076087952, "Finetune/Step": 1880, "Finetune/Step Time": 4.417984275147319} +{"Finetune/Learning Rate": 1.4755292219168581e-05, "Finetune/Loss": 0.9219890236854553, "Finetune/Loss (Raw)": 0.8205533027648926, "Finetune/Step": 1881, "Finetune/Step Time": 4.425639107823372} +{"Finetune/Learning Rate": 1.4749811108774798e-05, "Finetune/Loss": 0.9219495058059692, "Finetune/Loss (Raw)": 0.9718832969665527, "Finetune/Step": 1882, "Finetune/Step Time": 4.421939926221967} +{"Finetune/Learning Rate": 1.4744328155072058e-05, "Finetune/Loss": 0.921218991279602, "Finetune/Loss (Raw)": 0.8663787841796875, "Finetune/Step": 1883, "Finetune/Step Time": 4.42964930832386} +{"Finetune/Learning Rate": 1.4738843360188184e-05, "Finetune/Loss": 0.9157353639602661, "Finetune/Loss (Raw)": 0.3592901825904846, "Finetune/Step": 1884, "Finetune/Step Time": 4.466371651738882} +{"Finetune/Learning Rate": 1.4733356726251722e-05, "Finetune/Loss": 0.9204463958740234, "Finetune/Loss (Raw)": 0.9640230536460876, "Finetune/Step": 1885, "Finetune/Step Time": 4.424304297193885} +{"Finetune/Learning Rate": 1.4727868255391924e-05, "Finetune/Loss": 0.9223347902297974, "Finetune/Loss (Raw)": 1.069148302078247, "Finetune/Step": 1886, "Finetune/Step Time": 4.42802594229579} +{"Finetune/Learning Rate": 1.472237794973876e-05, "Finetune/Loss": 0.9207028150558472, "Finetune/Loss (Raw)": 0.8037990927696228, "Finetune/Step": 1887, "Finetune/Step Time": 4.422807900235057} +{"Finetune/Learning Rate": 1.471688581142291e-05, "Finetune/Loss": 0.918533205986023, "Finetune/Loss (Raw)": 0.7550019025802612, "Finetune/Step": 1888, "Finetune/Step Time": 4.427730243653059} +{"Finetune/Learning Rate": 1.4711391842575765e-05, "Finetune/Loss": 0.9191420078277588, "Finetune/Loss (Raw)": 0.9254512190818787, "Finetune/Step": 1889, "Finetune/Step Time": 4.433071671053767} +{"Finetune/Learning Rate": 1.4705896045329432e-05, "Finetune/Loss": 0.9197403192520142, "Finetune/Loss (Raw)": 0.9043863415718079, "Finetune/Step": 1890, "Finetune/Step Time": 4.426923971623182} +{"Finetune/Learning Rate": 1.4700398421816714e-05, "Finetune/Loss": 0.9191996455192566, "Finetune/Loss (Raw)": 1.0017147064208984, "Finetune/Step": 1891, "Finetune/Step Time": 4.41650241240859} +{"Finetune/Learning Rate": 1.469489897417114e-05, "Finetune/Loss": 0.919692873954773, "Finetune/Loss (Raw)": 1.0979163646697998, "Finetune/Step": 1892, "Finetune/Step Time": 4.3969012424349785} +{"Finetune/Learning Rate": 1.468939770452693e-05, "Finetune/Loss": 0.9201381206512451, "Finetune/Loss (Raw)": 0.9242358207702637, "Finetune/Step": 1893, "Finetune/Step Time": 4.398424744606018} +{"Finetune/Learning Rate": 1.4683894615019028e-05, "Finetune/Loss": 0.9188966751098633, "Finetune/Loss (Raw)": 0.8192968964576721, "Finetune/Step": 1894, "Finetune/Step Time": 4.389342648908496} +{"Finetune/Learning Rate": 1.4678389707783072e-05, "Finetune/Loss": 0.9194844961166382, "Finetune/Loss (Raw)": 0.8913419246673584, "Finetune/Step": 1895, "Finetune/Step Time": 4.388240830972791} +{"Finetune/Learning Rate": 1.4672882984955404e-05, "Finetune/Loss": 0.9200698733329773, "Finetune/Loss (Raw)": 0.9496853351593018, "Finetune/Step": 1896, "Finetune/Step Time": 4.390155650675297} +{"Finetune/Learning Rate": 1.4667374448673081e-05, "Finetune/Loss": 0.922427773475647, "Finetune/Loss (Raw)": 1.0877195596694946, "Finetune/Step": 1897, "Finetune/Step Time": 4.376821398735046} +{"Finetune/Learning Rate": 1.4661864101073857e-05, "Finetune/Loss": 0.9223174452781677, "Finetune/Loss (Raw)": 0.9814398288726807, "Finetune/Step": 1898, "Finetune/Step Time": 4.38179624453187} +{"Finetune/Learning Rate": 1.465635194429619e-05, "Finetune/Loss": 0.9218747615814209, "Finetune/Loss (Raw)": 0.8216700553894043, "Finetune/Step": 1899, "Finetune/Step Time": 4.395483203232288} +{"Finetune/Learning Rate": 1.4650837980479246e-05, "Finetune/Loss": 0.9204172492027283, "Finetune/Loss (Raw)": 0.8843438625335693, "Finetune/Step": 1900, "Finetune/Step Time": 4.408238839358091} +{"Finetune/Learning Rate": 1.4645322211762876e-05, "Finetune/Loss": 0.9202089905738831, "Finetune/Loss (Raw)": 0.9014247059822083, "Finetune/Step": 1901, "Finetune/Step Time": 4.3932090271264315} +{"Finetune/Learning Rate": 1.4639804640287652e-05, "Finetune/Loss": 0.9206646084785461, "Finetune/Loss (Raw)": 0.9903632998466492, "Finetune/Step": 1902, "Finetune/Step Time": 4.383683202788234} +{"Finetune/Learning Rate": 1.4634285268194828e-05, "Finetune/Loss": 0.9193518161773682, "Finetune/Loss (Raw)": 0.8522175550460815, "Finetune/Step": 1903, "Finetune/Step Time": 4.40369401499629} +{"Finetune/Learning Rate": 1.462876409762637e-05, "Finetune/Loss": 0.919262170791626, "Finetune/Loss (Raw)": 0.9242246150970459, "Finetune/Step": 1904, "Finetune/Step Time": 4.408734455704689} +{"Finetune/Learning Rate": 1.4623241130724937e-05, "Finetune/Loss": 0.9141594171524048, "Finetune/Loss (Raw)": 0.32569998502731323, "Finetune/Step": 1905, "Finetune/Step Time": 4.450357945635915} +{"Finetune/Learning Rate": 1.4617716369633876e-05, "Finetune/Loss": 0.9139125943183899, "Finetune/Loss (Raw)": 0.8984985947608948, "Finetune/Step": 1906, "Finetune/Step Time": 4.4681905545294285} +{"Finetune/Learning Rate": 1.461218981649725e-05, "Finetune/Loss": 0.9127548933029175, "Finetune/Loss (Raw)": 0.9401028156280518, "Finetune/Step": 1907, "Finetune/Step Time": 4.483910597860813} +{"Finetune/Learning Rate": 1.4606661473459801e-05, "Finetune/Loss": 0.912601113319397, "Finetune/Loss (Raw)": 0.877890408039093, "Finetune/Step": 1908, "Finetune/Step Time": 4.482076590880752} +{"Finetune/Learning Rate": 1.4601131342666973e-05, "Finetune/Loss": 0.9119417071342468, "Finetune/Loss (Raw)": 0.7202466726303101, "Finetune/Step": 1909, "Finetune/Step Time": 4.482593346387148} +{"Finetune/Learning Rate": 1.45955994262649e-05, "Finetune/Loss": 0.9119983911514282, "Finetune/Loss (Raw)": 0.923474133014679, "Finetune/Step": 1910, "Finetune/Step Time": 4.484604809433222} +{"Finetune/Learning Rate": 1.459006572640041e-05, "Finetune/Loss": 0.9130640029907227, "Finetune/Loss (Raw)": 0.9785338640213013, "Finetune/Step": 1911, "Finetune/Step Time": 4.490467805415392} +{"Finetune/Learning Rate": 1.4584530245221025e-05, "Finetune/Loss": 0.9124062061309814, "Finetune/Loss (Raw)": 0.9663681983947754, "Finetune/Step": 1912, "Finetune/Step Time": 4.483759252354503} +{"Finetune/Learning Rate": 1.4578992984874956e-05, "Finetune/Loss": 0.9139811992645264, "Finetune/Loss (Raw)": 0.9523072838783264, "Finetune/Step": 1913, "Finetune/Step Time": 4.46892530657351} +{"Finetune/Learning Rate": 1.457345394751111e-05, "Finetune/Loss": 0.9154120683670044, "Finetune/Loss (Raw)": 1.0819388628005981, "Finetune/Step": 1914, "Finetune/Step Time": 4.466510757803917} +{"Finetune/Learning Rate": 1.4567913135279075e-05, "Finetune/Loss": 0.9173200130462646, "Finetune/Loss (Raw)": 1.0259324312210083, "Finetune/Step": 1915, "Finetune/Step Time": 4.4888660833239555} +{"Finetune/Learning Rate": 1.456237055032913e-05, "Finetune/Loss": 0.9177196025848389, "Finetune/Loss (Raw)": 0.9101362228393555, "Finetune/Step": 1916, "Finetune/Step Time": 4.488718880340457} +{"Finetune/Learning Rate": 1.455682619481225e-05, "Finetune/Loss": 0.9218739867210388, "Finetune/Loss (Raw)": 0.8298535943031311, "Finetune/Step": 1917, "Finetune/Step Time": 4.449724491685629} +{"Finetune/Learning Rate": 1.4551280070880089e-05, "Finetune/Loss": 0.9209834337234497, "Finetune/Loss (Raw)": 0.880516767501831, "Finetune/Step": 1918, "Finetune/Step Time": 4.437896711751819} +{"Finetune/Learning Rate": 1.4545732180684989e-05, "Finetune/Loss": 0.920038104057312, "Finetune/Loss (Raw)": 0.9258114695549011, "Finetune/Step": 1919, "Finetune/Step Time": 4.4342246782034636} +{"Finetune/Learning Rate": 1.4540182526379976e-05, "Finetune/Loss": 0.9188768267631531, "Finetune/Loss (Raw)": 0.840822160243988, "Finetune/Step": 1920, "Finetune/Step Time": 4.443706475198269} +{"Finetune/Learning Rate": 1.4534631110118763e-05, "Finetune/Loss": 0.9204974174499512, "Finetune/Loss (Raw)": 1.0118942260742188, "Finetune/Step": 1921, "Finetune/Step Time": 4.427351133897901} +{"Finetune/Learning Rate": 1.4529077934055754e-05, "Finetune/Loss": 0.9149665236473083, "Finetune/Loss (Raw)": 0.30847254395484924, "Finetune/Step": 1922, "Finetune/Step Time": 4.464312182739377} +{"Finetune/Learning Rate": 1.4523523000346012e-05, "Finetune/Loss": 0.9195380210876465, "Finetune/Loss (Raw)": 0.9583126306533813, "Finetune/Step": 1923, "Finetune/Step Time": 4.425097346305847} +{"Finetune/Learning Rate": 1.4517966311145313e-05, "Finetune/Loss": 0.9206156730651855, "Finetune/Loss (Raw)": 1.036458969116211, "Finetune/Step": 1924, "Finetune/Step Time": 4.423299789428711} +{"Finetune/Learning Rate": 1.4512407868610094e-05, "Finetune/Loss": 0.9217032790184021, "Finetune/Loss (Raw)": 1.1295568943023682, "Finetune/Step": 1925, "Finetune/Step Time": 4.420583251863718} +{"Finetune/Learning Rate": 1.450684767489748e-05, "Finetune/Loss": 0.9171069860458374, "Finetune/Loss (Raw)": 0.3397045135498047, "Finetune/Step": 1926, "Finetune/Step Time": 4.455147789791226} +{"Finetune/Learning Rate": 1.4501285732165271e-05, "Finetune/Loss": 0.9132964611053467, "Finetune/Loss (Raw)": 0.40233680605888367, "Finetune/Step": 1927, "Finetune/Step Time": 4.495483545586467} +{"Finetune/Learning Rate": 1.4495722042571946e-05, "Finetune/Loss": 0.9141645431518555, "Finetune/Loss (Raw)": 1.0187186002731323, "Finetune/Step": 1928, "Finetune/Step Time": 4.4779251255095005} +{"Finetune/Learning Rate": 1.4490156608276674e-05, "Finetune/Loss": 0.9122874140739441, "Finetune/Loss (Raw)": 0.8012799620628357, "Finetune/Step": 1929, "Finetune/Step Time": 4.469614064320922} +{"Finetune/Learning Rate": 1.4484589431439283e-05, "Finetune/Loss": 0.9116944074630737, "Finetune/Loss (Raw)": 0.8612759709358215, "Finetune/Step": 1930, "Finetune/Step Time": 4.454901857301593} +{"Finetune/Learning Rate": 1.4479020514220285e-05, "Finetune/Loss": 0.9120857119560242, "Finetune/Loss (Raw)": 0.955100953578949, "Finetune/Step": 1931, "Finetune/Step Time": 4.438546860590577} +{"Finetune/Learning Rate": 1.4473449858780874e-05, "Finetune/Loss": 0.9111180305480957, "Finetune/Loss (Raw)": 0.8281240463256836, "Finetune/Step": 1932, "Finetune/Step Time": 4.452820789068937} +{"Finetune/Learning Rate": 1.4467877467282906e-05, "Finetune/Loss": 0.9135826826095581, "Finetune/Loss (Raw)": 0.9562386870384216, "Finetune/Step": 1933, "Finetune/Step Time": 4.446042966097593} +{"Finetune/Learning Rate": 1.4462303341888925e-05, "Finetune/Loss": 0.9146783947944641, "Finetune/Loss (Raw)": 0.995642900466919, "Finetune/Step": 1934, "Finetune/Step Time": 4.426427666097879} +{"Finetune/Learning Rate": 1.4456727484762137e-05, "Finetune/Loss": 0.9154146313667297, "Finetune/Loss (Raw)": 1.0253167152404785, "Finetune/Step": 1935, "Finetune/Step Time": 4.4333738554269075} +{"Finetune/Learning Rate": 1.4451149898066418e-05, "Finetune/Loss": 0.9146065711975098, "Finetune/Loss (Raw)": 0.8300409913063049, "Finetune/Step": 1936, "Finetune/Step Time": 4.44194271415472} +{"Finetune/Learning Rate": 1.4445570583966329e-05, "Finetune/Loss": 0.9148116111755371, "Finetune/Loss (Raw)": 0.9627730250358582, "Finetune/Step": 1937, "Finetune/Step Time": 4.441222909837961} +{"Finetune/Learning Rate": 1.4439989544627085e-05, "Finetune/Loss": 0.9106940031051636, "Finetune/Loss (Raw)": 0.4102371633052826, "Finetune/Step": 1938, "Finetune/Step Time": 4.469551729038358} +{"Finetune/Learning Rate": 1.4434406782214584e-05, "Finetune/Loss": 0.9106594324111938, "Finetune/Loss (Raw)": 0.9197888970375061, "Finetune/Step": 1939, "Finetune/Step Time": 4.465315368026495} +{"Finetune/Learning Rate": 1.4428822298895386e-05, "Finetune/Loss": 0.9082647562026978, "Finetune/Loss (Raw)": 0.5783272385597229, "Finetune/Step": 1940, "Finetune/Step Time": 4.472524458542466} +{"Finetune/Learning Rate": 1.442323609683672e-05, "Finetune/Loss": 0.9081162214279175, "Finetune/Loss (Raw)": 0.8916772603988647, "Finetune/Step": 1941, "Finetune/Step Time": 4.4716009721159935} +{"Finetune/Learning Rate": 1.4417648178206481e-05, "Finetune/Loss": 0.9091722965240479, "Finetune/Loss (Raw)": 1.1044211387634277, "Finetune/Step": 1942, "Finetune/Step Time": 4.473202716559172} +{"Finetune/Learning Rate": 1.4412058545173228e-05, "Finetune/Loss": 0.9103440642356873, "Finetune/Loss (Raw)": 1.0273329019546509, "Finetune/Step": 1943, "Finetune/Step Time": 4.472882254049182} +{"Finetune/Learning Rate": 1.4406467199906194e-05, "Finetune/Loss": 0.9098483324050903, "Finetune/Loss (Raw)": 0.9568736553192139, "Finetune/Step": 1944, "Finetune/Step Time": 4.482664877548814} +{"Finetune/Learning Rate": 1.440087414457527e-05, "Finetune/Loss": 0.905820369720459, "Finetune/Loss (Raw)": 0.3874048590660095, "Finetune/Step": 1945, "Finetune/Step Time": 4.521210074424744} +{"Finetune/Learning Rate": 1.4395279381351008e-05, "Finetune/Loss": 0.906056821346283, "Finetune/Loss (Raw)": 0.9285922646522522, "Finetune/Step": 1946, "Finetune/Step Time": 4.534507339820266} +{"Finetune/Learning Rate": 1.438968291240463e-05, "Finetune/Loss": 0.9071750640869141, "Finetune/Loss (Raw)": 1.1100682020187378, "Finetune/Step": 1947, "Finetune/Step Time": 4.5326606668531895} +{"Finetune/Learning Rate": 1.4384084739908016e-05, "Finetune/Loss": 0.9090930819511414, "Finetune/Loss (Raw)": 1.0809794664382935, "Finetune/Step": 1948, "Finetune/Step Time": 4.5325531382113695} +{"Finetune/Learning Rate": 1.4378484866033705e-05, "Finetune/Loss": 0.9098926186561584, "Finetune/Loss (Raw)": 1.053792119026184, "Finetune/Step": 1949, "Finetune/Step Time": 4.5305335111916065} +{"Finetune/Learning Rate": 1.43728832929549e-05, "Finetune/Loss": 0.9090923070907593, "Finetune/Loss (Raw)": 1.074788212776184, "Finetune/Step": 1950, "Finetune/Step Time": 4.526765046641231} +{"Finetune/Learning Rate": 1.4367280022845462e-05, "Finetune/Loss": 0.9100008010864258, "Finetune/Loss (Raw)": 1.0530719757080078, "Finetune/Step": 1951, "Finetune/Step Time": 4.53115776181221} +{"Finetune/Learning Rate": 1.4361675057879913e-05, "Finetune/Loss": 0.9117650985717773, "Finetune/Loss (Raw)": 1.0586532354354858, "Finetune/Step": 1952, "Finetune/Step Time": 4.52542675472796} +{"Finetune/Learning Rate": 1.4356068400233429e-05, "Finetune/Loss": 0.9102118015289307, "Finetune/Loss (Raw)": 0.7019010782241821, "Finetune/Step": 1953, "Finetune/Step Time": 4.531321130692959} +{"Finetune/Learning Rate": 1.435046005208184e-05, "Finetune/Loss": 0.9088649749755859, "Finetune/Loss (Raw)": 0.8341823816299438, "Finetune/Step": 1954, "Finetune/Step Time": 4.545199142768979} +{"Finetune/Learning Rate": 1.4344850015601644e-05, "Finetune/Loss": 0.9079697132110596, "Finetune/Loss (Raw)": 0.820377767086029, "Finetune/Step": 1955, "Finetune/Step Time": 4.529986580833793} +{"Finetune/Learning Rate": 1.4339238292969981e-05, "Finetune/Loss": 0.9062641859054565, "Finetune/Loss (Raw)": 0.8479759693145752, "Finetune/Step": 1956, "Finetune/Step Time": 4.5245461240410805} +{"Finetune/Learning Rate": 1.4333624886364655e-05, "Finetune/Loss": 0.9058483242988586, "Finetune/Loss (Raw)": 1.0629276037216187, "Finetune/Step": 1957, "Finetune/Step Time": 4.517541395500302} +{"Finetune/Learning Rate": 1.4328009797964115e-05, "Finetune/Loss": 0.9059110879898071, "Finetune/Loss (Raw)": 1.0460377931594849, "Finetune/Step": 1958, "Finetune/Step Time": 4.5026462264359} +{"Finetune/Learning Rate": 1.432239302994747e-05, "Finetune/Loss": 0.9052519798278809, "Finetune/Loss (Raw)": 0.8268312811851501, "Finetune/Step": 1959, "Finetune/Step Time": 4.5055854972451925} +{"Finetune/Learning Rate": 1.431677458449448e-05, "Finetune/Loss": 0.905778706073761, "Finetune/Loss (Raw)": 1.0958577394485474, "Finetune/Step": 1960, "Finetune/Step Time": 4.51087630726397} +{"Finetune/Learning Rate": 1.431115446378555e-05, "Finetune/Loss": 0.9055197834968567, "Finetune/Loss (Raw)": 0.8066558837890625, "Finetune/Step": 1961, "Finetune/Step Time": 4.498537927865982} +{"Finetune/Learning Rate": 1.4305532670001743e-05, "Finetune/Loss": 0.9002572298049927, "Finetune/Loss (Raw)": 0.4138638377189636, "Finetune/Step": 1962, "Finetune/Step Time": 4.539806364104152} +{"Finetune/Learning Rate": 1.4299909205324762e-05, "Finetune/Loss": 0.896011471748352, "Finetune/Loss (Raw)": 0.3993474245071411, "Finetune/Step": 1963, "Finetune/Step Time": 4.565190885215998} +{"Finetune/Learning Rate": 1.4294284071936968e-05, "Finetune/Loss": 0.8962112665176392, "Finetune/Loss (Raw)": 0.9963064193725586, "Finetune/Step": 1964, "Finetune/Step Time": 4.572019837796688} +{"Finetune/Learning Rate": 1.4288657272021367e-05, "Finetune/Loss": 0.8952206373214722, "Finetune/Loss (Raw)": 0.8025700449943542, "Finetune/Step": 1965, "Finetune/Step Time": 4.565804712474346} +{"Finetune/Learning Rate": 1.4283028807761603e-05, "Finetune/Loss": 0.8944146037101746, "Finetune/Loss (Raw)": 0.8701207637786865, "Finetune/Step": 1966, "Finetune/Step Time": 4.5551209188997746} +{"Finetune/Learning Rate": 1.4277398681341982e-05, "Finetune/Loss": 0.893354594707489, "Finetune/Loss (Raw)": 0.833354651927948, "Finetune/Step": 1967, "Finetune/Step Time": 4.537222350016236} +{"Finetune/Learning Rate": 1.4271766894947441e-05, "Finetune/Loss": 0.8934012055397034, "Finetune/Loss (Raw)": 1.0087244510650635, "Finetune/Step": 1968, "Finetune/Step Time": 4.5292115695774555} +{"Finetune/Learning Rate": 1.4266133450763564e-05, "Finetune/Loss": 0.8904919624328613, "Finetune/Loss (Raw)": 0.4487522542476654, "Finetune/Step": 1969, "Finetune/Step Time": 4.565662957727909} +{"Finetune/Learning Rate": 1.426049835097659e-05, "Finetune/Loss": 0.8892261385917664, "Finetune/Loss (Raw)": 0.8919495344161987, "Finetune/Step": 1970, "Finetune/Step Time": 4.560373790562153} +{"Finetune/Learning Rate": 1.425486159777338e-05, "Finetune/Loss": 0.8848831653594971, "Finetune/Loss (Raw)": 0.4079763889312744, "Finetune/Step": 1971, "Finetune/Step Time": 4.582948267459869} +{"Finetune/Learning Rate": 1.4249223193341456e-05, "Finetune/Loss": 0.8847342729568481, "Finetune/Loss (Raw)": 0.9087017774581909, "Finetune/Step": 1972, "Finetune/Step Time": 4.576744064688683} +{"Finetune/Learning Rate": 1.4243583139868969e-05, "Finetune/Loss": 0.8844025135040283, "Finetune/Loss (Raw)": 0.8705642819404602, "Finetune/Step": 1973, "Finetune/Step Time": 4.6084414422512054} +{"Finetune/Learning Rate": 1.423794143954471e-05, "Finetune/Loss": 0.8845490217208862, "Finetune/Loss (Raw)": 1.0647536516189575, "Finetune/Step": 1974, "Finetune/Step Time": 4.607298927381635} +{"Finetune/Learning Rate": 1.4232298094558125e-05, "Finetune/Loss": 0.8838866949081421, "Finetune/Loss (Raw)": 0.914106011390686, "Finetune/Step": 1975, "Finetune/Step Time": 4.596942165866494} +{"Finetune/Learning Rate": 1.4226653107099273e-05, "Finetune/Loss": 0.8838356733322144, "Finetune/Loss (Raw)": 0.9384995102882385, "Finetune/Step": 1976, "Finetune/Step Time": 4.59971709176898} +{"Finetune/Learning Rate": 1.422100647935887e-05, "Finetune/Loss": 0.8833791017532349, "Finetune/Loss (Raw)": 0.9779874086380005, "Finetune/Step": 1977, "Finetune/Step Time": 4.608088508248329} +{"Finetune/Learning Rate": 1.421535821352826e-05, "Finetune/Loss": 0.8834095001220703, "Finetune/Loss (Raw)": 1.0347508192062378, "Finetune/Step": 1978, "Finetune/Step Time": 4.601499689742923} +{"Finetune/Learning Rate": 1.4209708311799424e-05, "Finetune/Loss": 0.8798927068710327, "Finetune/Loss (Raw)": 0.3800123929977417, "Finetune/Step": 1979, "Finetune/Step Time": 4.617621835321188} +{"Finetune/Learning Rate": 1.4204056776364985e-05, "Finetune/Loss": 0.8784406781196594, "Finetune/Loss (Raw)": 0.8008216619491577, "Finetune/Step": 1980, "Finetune/Step Time": 4.631070405244827} +{"Finetune/Learning Rate": 1.4198403609418184e-05, "Finetune/Loss": 0.878705620765686, "Finetune/Loss (Raw)": 0.9995073676109314, "Finetune/Step": 1981, "Finetune/Step Time": 4.626063404604793} +{"Finetune/Learning Rate": 1.419274881315291e-05, "Finetune/Loss": 0.8786320686340332, "Finetune/Loss (Raw)": 0.9607216119766235, "Finetune/Step": 1982, "Finetune/Step Time": 4.634173942729831} +{"Finetune/Learning Rate": 1.418709238976368e-05, "Finetune/Loss": 0.8776686191558838, "Finetune/Loss (Raw)": 0.8925209045410156, "Finetune/Step": 1983, "Finetune/Step Time": 4.632487643510103} +{"Finetune/Learning Rate": 1.4181434341445634e-05, "Finetune/Loss": 0.8778532147407532, "Finetune/Loss (Raw)": 1.086990475654602, "Finetune/Step": 1984, "Finetune/Step Time": 4.632138326764107} +{"Finetune/Learning Rate": 1.4175774670394561e-05, "Finetune/Loss": 0.8788636922836304, "Finetune/Loss (Raw)": 1.005415678024292, "Finetune/Step": 1985, "Finetune/Step Time": 4.59247419424355} +{"Finetune/Learning Rate": 1.4170113378806862e-05, "Finetune/Loss": 0.8803185224533081, "Finetune/Loss (Raw)": 1.1694989204406738, "Finetune/Step": 1986, "Finetune/Step Time": 4.582983464002609} +{"Finetune/Learning Rate": 1.4164450468879578e-05, "Finetune/Loss": 0.8794260025024414, "Finetune/Loss (Raw)": 0.9370591044425964, "Finetune/Step": 1987, "Finetune/Step Time": 4.61448915861547} +{"Finetune/Learning Rate": 1.4158785942810367e-05, "Finetune/Loss": 0.8792569637298584, "Finetune/Loss (Raw)": 0.8709183931350708, "Finetune/Step": 1988, "Finetune/Step Time": 4.613263165578246} +{"Finetune/Learning Rate": 1.4153119802797526e-05, "Finetune/Loss": 0.8801857233047485, "Finetune/Loss (Raw)": 0.9679738283157349, "Finetune/Step": 1989, "Finetune/Step Time": 4.615158712491393} +{"Finetune/Learning Rate": 1.4147452051039974e-05, "Finetune/Loss": 0.8811991214752197, "Finetune/Loss (Raw)": 0.9856707453727722, "Finetune/Step": 1990, "Finetune/Step Time": 4.615750269964337} +{"Finetune/Learning Rate": 1.4141782689737252e-05, "Finetune/Loss": 0.8811227083206177, "Finetune/Loss (Raw)": 0.8060808181762695, "Finetune/Step": 1991, "Finetune/Step Time": 4.611371774226427} +{"Finetune/Learning Rate": 1.413611172108953e-05, "Finetune/Loss": 0.8821383714675903, "Finetune/Loss (Raw)": 0.9618552327156067, "Finetune/Step": 1992, "Finetune/Step Time": 4.600739859044552} +{"Finetune/Learning Rate": 1.41304391472976e-05, "Finetune/Loss": 0.8808560371398926, "Finetune/Loss (Raw)": 0.8564913868904114, "Finetune/Step": 1993, "Finetune/Step Time": 4.605116195976734} +{"Finetune/Learning Rate": 1.4124764970562874e-05, "Finetune/Loss": 0.8786091804504395, "Finetune/Loss (Raw)": 0.7481817603111267, "Finetune/Step": 1994, "Finetune/Step Time": 4.601153494790196} +{"Finetune/Learning Rate": 1.4119089193087396e-05, "Finetune/Loss": 0.8763751983642578, "Finetune/Loss (Raw)": 0.7895771861076355, "Finetune/Step": 1995, "Finetune/Step Time": 4.604367062449455} +{"Finetune/Learning Rate": 1.4113411817073818e-05, "Finetune/Loss": 0.8754384517669678, "Finetune/Loss (Raw)": 0.8314724564552307, "Finetune/Step": 1996, "Finetune/Step Time": 4.591464836150408} +{"Finetune/Learning Rate": 1.4107732844725424e-05, "Finetune/Loss": 0.8756647109985352, "Finetune/Loss (Raw)": 1.0633922815322876, "Finetune/Step": 1997, "Finetune/Step Time": 4.599673256278038} +{"Finetune/Learning Rate": 1.4102052278246107e-05, "Finetune/Loss": 0.875845730304718, "Finetune/Loss (Raw)": 0.9021812677383423, "Finetune/Step": 1998, "Finetune/Step Time": 4.600241217762232} +{"Finetune/Learning Rate": 1.4096370119840389e-05, "Finetune/Loss": 0.8718606233596802, "Finetune/Loss (Raw)": 0.415613055229187, "Finetune/Step": 1999, "Finetune/Step Time": 4.6074752304703} +{"Finetune/Learning Rate": 1.4090686371713403e-05, "Finetune/Loss": 0.8721956014633179, "Finetune/Loss (Raw)": 0.8672993183135986, "Finetune/Step": 2000, "Finetune/Step Time": 4.630682883784175} +{"Finetune/Learning Rate": 1.4085001036070896e-05, "Finetune/Loss": 0.8766691088676453, "Finetune/Loss (Raw)": 0.940656304359436, "Finetune/Step": 2001, "Finetune/Step Time": 4.589612912386656} +{"Finetune/Learning Rate": 1.4079314115119242e-05, "Finetune/Loss": 0.875760555267334, "Finetune/Loss (Raw)": 0.9261956214904785, "Finetune/Step": 2002, "Finetune/Step Time": 4.581579335033894} +{"Finetune/Learning Rate": 1.4073625611065422e-05, "Finetune/Loss": 0.8757544755935669, "Finetune/Loss (Raw)": 0.8699857592582703, "Finetune/Step": 2003, "Finetune/Step Time": 4.586639486253262} +{"Finetune/Learning Rate": 1.4067935526117033e-05, "Finetune/Loss": 0.8753079175949097, "Finetune/Loss (Raw)": 0.8891309499740601, "Finetune/Step": 2004, "Finetune/Step Time": 4.5912063885480165} +{"Finetune/Learning Rate": 1.4062243862482286e-05, "Finetune/Loss": 0.8749631643295288, "Finetune/Loss (Raw)": 0.9028854966163635, "Finetune/Step": 2005, "Finetune/Step Time": 4.584505805745721} +{"Finetune/Learning Rate": 1.4056550622370004e-05, "Finetune/Loss": 0.8760180473327637, "Finetune/Loss (Raw)": 0.9233777523040771, "Finetune/Step": 2006, "Finetune/Step Time": 4.573948126286268} +{"Finetune/Learning Rate": 1.4050855807989626e-05, "Finetune/Loss": 0.8765246272087097, "Finetune/Loss (Raw)": 1.1223235130310059, "Finetune/Step": 2007, "Finetune/Step Time": 4.573940450325608} +{"Finetune/Learning Rate": 1.4045159421551191e-05, "Finetune/Loss": 0.8779486417770386, "Finetune/Loss (Raw)": 0.9756348729133606, "Finetune/Step": 2008, "Finetune/Step Time": 4.55869354121387} +{"Finetune/Learning Rate": 1.4039461465265362e-05, "Finetune/Loss": 0.8784254789352417, "Finetune/Loss (Raw)": 0.881589412689209, "Finetune/Step": 2009, "Finetune/Step Time": 4.547572256997228} +{"Finetune/Learning Rate": 1.4033761941343403e-05, "Finetune/Loss": 0.8787788152694702, "Finetune/Loss (Raw)": 1.0171043872833252, "Finetune/Step": 2010, "Finetune/Step Time": 4.552591618150473} +{"Finetune/Learning Rate": 1.4028060851997183e-05, "Finetune/Loss": 0.8804752826690674, "Finetune/Loss (Raw)": 1.0835349559783936, "Finetune/Step": 2011, "Finetune/Step Time": 4.5463167652487755} +{"Finetune/Learning Rate": 1.4022358199439193e-05, "Finetune/Loss": 0.8848584890365601, "Finetune/Loss (Raw)": 0.9203377962112427, "Finetune/Step": 2012, "Finetune/Step Time": 4.506652196869254} +{"Finetune/Learning Rate": 1.4016653985882512e-05, "Finetune/Loss": 0.8851854205131531, "Finetune/Loss (Raw)": 1.0058696269989014, "Finetune/Step": 2013, "Finetune/Step Time": 4.522727055475116} +{"Finetune/Learning Rate": 1.4010948213540838e-05, "Finetune/Loss": 0.8829314708709717, "Finetune/Loss (Raw)": 0.7806486487388611, "Finetune/Step": 2014, "Finetune/Step Time": 4.531558662652969} +{"Finetune/Learning Rate": 1.4005240884628473e-05, "Finetune/Loss": 0.8797545433044434, "Finetune/Loss (Raw)": 0.3971501886844635, "Finetune/Step": 2015, "Finetune/Step Time": 4.567242367193103} +{"Finetune/Learning Rate": 1.3999532001360309e-05, "Finetune/Loss": 0.8770102262496948, "Finetune/Loss (Raw)": 0.4037289023399353, "Finetune/Step": 2016, "Finetune/Step Time": 4.602317007258534} +{"Finetune/Learning Rate": 1.3993821565951868e-05, "Finetune/Loss": 0.8762995004653931, "Finetune/Loss (Raw)": 0.8344810605049133, "Finetune/Step": 2017, "Finetune/Step Time": 4.6082110311836} +{"Finetune/Learning Rate": 1.3988109580619244e-05, "Finetune/Loss": 0.8766591548919678, "Finetune/Loss (Raw)": 0.9504199028015137, "Finetune/Step": 2018, "Finetune/Step Time": 4.609523167833686} +{"Finetune/Learning Rate": 1.3982396047579156e-05, "Finetune/Loss": 0.8755795359611511, "Finetune/Loss (Raw)": 0.8635178208351135, "Finetune/Step": 2019, "Finetune/Step Time": 4.610685355961323} +{"Finetune/Learning Rate": 1.3976680969048912e-05, "Finetune/Loss": 0.8745735287666321, "Finetune/Loss (Raw)": 0.9691467881202698, "Finetune/Step": 2020, "Finetune/Step Time": 4.614384148269892} +{"Finetune/Learning Rate": 1.3970964347246419e-05, "Finetune/Loss": 0.8747007250785828, "Finetune/Loss (Raw)": 0.940521776676178, "Finetune/Step": 2021, "Finetune/Step Time": 4.616217911243439} +{"Finetune/Learning Rate": 1.3965246184390193e-05, "Finetune/Loss": 0.8754183053970337, "Finetune/Loss (Raw)": 0.9111441373825073, "Finetune/Step": 2022, "Finetune/Step Time": 4.6231057941913605} +{"Finetune/Learning Rate": 1.3959526482699335e-05, "Finetune/Loss": 0.8743341565132141, "Finetune/Loss (Raw)": 0.752574622631073, "Finetune/Step": 2023, "Finetune/Step Time": 4.6480894554406404} +{"Finetune/Learning Rate": 1.3953805244393554e-05, "Finetune/Loss": 0.8740830421447754, "Finetune/Loss (Raw)": 0.9175423979759216, "Finetune/Step": 2024, "Finetune/Step Time": 4.647568490356207} +{"Finetune/Learning Rate": 1.394808247169315e-05, "Finetune/Loss": 0.8723375201225281, "Finetune/Loss (Raw)": 0.8642896413803101, "Finetune/Step": 2025, "Finetune/Step Time": 4.659794643521309} +{"Finetune/Learning Rate": 1.3942358166819016e-05, "Finetune/Loss": 0.8721087574958801, "Finetune/Loss (Raw)": 0.9521610140800476, "Finetune/Step": 2026, "Finetune/Step Time": 4.64622987434268} +{"Finetune/Learning Rate": 1.3936632331992648e-05, "Finetune/Loss": 0.8723458051681519, "Finetune/Loss (Raw)": 0.8520148992538452, "Finetune/Step": 2027, "Finetune/Step Time": 4.6257801577448845} +{"Finetune/Learning Rate": 1.3930904969436125e-05, "Finetune/Loss": 0.8728333711624146, "Finetune/Loss (Raw)": 0.946754515171051, "Finetune/Step": 2028, "Finetune/Step Time": 4.628015447407961} +{"Finetune/Learning Rate": 1.3925176081372129e-05, "Finetune/Loss": 0.8738610744476318, "Finetune/Loss (Raw)": 1.032963752746582, "Finetune/Step": 2029, "Finetune/Step Time": 4.621013671159744} +{"Finetune/Learning Rate": 1.3919445670023933e-05, "Finetune/Loss": 0.8688695430755615, "Finetune/Loss (Raw)": 0.35145485401153564, "Finetune/Step": 2030, "Finetune/Step Time": 4.653931885957718} +{"Finetune/Learning Rate": 1.3913713737615387e-05, "Finetune/Loss": 0.8691387176513672, "Finetune/Loss (Raw)": 0.8866680860519409, "Finetune/Step": 2031, "Finetune/Step Time": 4.63863331079483} +{"Finetune/Learning Rate": 1.3907980286370952e-05, "Finetune/Loss": 0.8699524402618408, "Finetune/Loss (Raw)": 1.028379201889038, "Finetune/Step": 2032, "Finetune/Step Time": 4.6391978822648525} +{"Finetune/Learning Rate": 1.3902245318515661e-05, "Finetune/Loss": 0.8751948475837708, "Finetune/Loss (Raw)": 0.9967288970947266, "Finetune/Step": 2033, "Finetune/Step Time": 4.603494236245751} +{"Finetune/Learning Rate": 1.389650883627515e-05, "Finetune/Loss": 0.8763582706451416, "Finetune/Loss (Raw)": 1.0474153757095337, "Finetune/Step": 2034, "Finetune/Step Time": 4.586184237152338} +{"Finetune/Learning Rate": 1.3890770841875632e-05, "Finetune/Loss": 0.8754358291625977, "Finetune/Loss (Raw)": 0.8220250010490417, "Finetune/Step": 2035, "Finetune/Step Time": 4.5955731719732285} +{"Finetune/Learning Rate": 1.388503133754391e-05, "Finetune/Loss": 0.8761847019195557, "Finetune/Loss (Raw)": 0.973753035068512, "Finetune/Step": 2036, "Finetune/Step Time": 4.608568435534835} +{"Finetune/Learning Rate": 1.3879290325507378e-05, "Finetune/Loss": 0.8778629302978516, "Finetune/Loss (Raw)": 0.9350597262382507, "Finetune/Step": 2037, "Finetune/Step Time": 4.629125049337745} +{"Finetune/Learning Rate": 1.3873547807994e-05, "Finetune/Loss": 0.8769338130950928, "Finetune/Loss (Raw)": 0.8045437932014465, "Finetune/Step": 2038, "Finetune/Step Time": 4.631465727463365} +{"Finetune/Learning Rate": 1.386780378723235e-05, "Finetune/Loss": 0.8774517774581909, "Finetune/Loss (Raw)": 1.044834852218628, "Finetune/Step": 2039, "Finetune/Step Time": 4.625622931867838} +{"Finetune/Learning Rate": 1.3862058265451561e-05, "Finetune/Loss": 0.8777697086334229, "Finetune/Loss (Raw)": 1.0070644617080688, "Finetune/Step": 2040, "Finetune/Step Time": 4.624565504491329} +{"Finetune/Learning Rate": 1.385631124488136e-05, "Finetune/Loss": 0.8779279589653015, "Finetune/Loss (Raw)": 0.9725608825683594, "Finetune/Step": 2041, "Finetune/Step Time": 4.667834358289838} +{"Finetune/Learning Rate": 1.385056272775205e-05, "Finetune/Loss": 0.8758438229560852, "Finetune/Loss (Raw)": 0.8151735067367554, "Finetune/Step": 2042, "Finetune/Step Time": 4.66910851188004} +{"Finetune/Learning Rate": 1.3844812716294523e-05, "Finetune/Loss": 0.8751163482666016, "Finetune/Loss (Raw)": 0.9328147172927856, "Finetune/Step": 2043, "Finetune/Step Time": 4.63932079449296} +{"Finetune/Learning Rate": 1.3839061212740244e-05, "Finetune/Loss": 0.8752776384353638, "Finetune/Loss (Raw)": 0.9307776093482971, "Finetune/Step": 2044, "Finetune/Step Time": 4.639325521886349} +{"Finetune/Learning Rate": 1.383330821932126e-05, "Finetune/Loss": 0.8766698241233826, "Finetune/Loss (Raw)": 1.0080554485321045, "Finetune/Step": 2045, "Finetune/Step Time": 4.650805301964283} +{"Finetune/Learning Rate": 1.3827553738270196e-05, "Finetune/Loss": 0.8778589963912964, "Finetune/Loss (Raw)": 1.0327333211898804, "Finetune/Step": 2046, "Finetune/Step Time": 4.650608625262976} +{"Finetune/Learning Rate": 1.382179777182025e-05, "Finetune/Loss": 0.8781523108482361, "Finetune/Loss (Raw)": 0.963356077671051, "Finetune/Step": 2047, "Finetune/Step Time": 4.653905464336276} +{"Finetune/Learning Rate": 1.3816040322205206e-05, "Finetune/Loss": 0.8779080510139465, "Finetune/Loss (Raw)": 0.8095571994781494, "Finetune/Step": 2048, "Finetune/Step Time": 4.643191639333963} +{"Finetune/Learning Rate": 1.3810281391659414e-05, "Finetune/Loss": 0.8754551410675049, "Finetune/Loss (Raw)": 0.6979159116744995, "Finetune/Step": 2049, "Finetune/Step Time": 4.663367420434952} +{"Finetune/Learning Rate": 1.3804520982417802e-05, "Finetune/Loss": 0.8794548511505127, "Finetune/Loss (Raw)": 0.8204393982887268, "Finetune/Step": 2050, "Finetune/Step Time": 4.632525412365794} +{"Finetune/Learning Rate": 1.3798759096715877e-05, "Finetune/Loss": 0.8803426027297974, "Finetune/Loss (Raw)": 1.071946382522583, "Finetune/Step": 2051, "Finetune/Step Time": 4.636518985033035} +{"Finetune/Learning Rate": 1.379299573678971e-05, "Finetune/Loss": 0.8798605799674988, "Finetune/Loss (Raw)": 0.9747650623321533, "Finetune/Step": 2052, "Finetune/Step Time": 4.648857079446316} +{"Finetune/Learning Rate": 1.3787230904875947e-05, "Finetune/Loss": 0.878328800201416, "Finetune/Loss (Raw)": 0.9334810972213745, "Finetune/Step": 2053, "Finetune/Step Time": 4.651277469471097} +{"Finetune/Learning Rate": 1.3781464603211811e-05, "Finetune/Loss": 0.8837982416152954, "Finetune/Loss (Raw)": 1.0397993326187134, "Finetune/Step": 2054, "Finetune/Step Time": 4.6143825482577085} +{"Finetune/Learning Rate": 1.3775696834035088e-05, "Finetune/Loss": 0.8884645104408264, "Finetune/Loss (Raw)": 0.9996132254600525, "Finetune/Step": 2055, "Finetune/Step Time": 4.580781554803252} +{"Finetune/Learning Rate": 1.3769927599584141e-05, "Finetune/Loss": 0.8883098363876343, "Finetune/Loss (Raw)": 0.99892258644104, "Finetune/Step": 2056, "Finetune/Step Time": 4.581599108874798} +{"Finetune/Learning Rate": 1.376415690209789e-05, "Finetune/Loss": 0.8889191150665283, "Finetune/Loss (Raw)": 0.8792689442634583, "Finetune/Step": 2057, "Finetune/Step Time": 4.590041195973754} +{"Finetune/Learning Rate": 1.3758384743815838e-05, "Finetune/Loss": 0.8892195224761963, "Finetune/Loss (Raw)": 0.8997243046760559, "Finetune/Step": 2058, "Finetune/Step Time": 4.606687590479851} +{"Finetune/Learning Rate": 1.3752611126978045e-05, "Finetune/Loss": 0.8882200121879578, "Finetune/Loss (Raw)": 0.8271621465682983, "Finetune/Step": 2059, "Finetune/Step Time": 4.602188987657428} +{"Finetune/Learning Rate": 1.3746836053825133e-05, "Finetune/Loss": 0.8889681100845337, "Finetune/Loss (Raw)": 0.9238845109939575, "Finetune/Step": 2060, "Finetune/Step Time": 4.582518849521875} +{"Finetune/Learning Rate": 1.3741059526598299e-05, "Finetune/Loss": 0.889474093914032, "Finetune/Loss (Raw)": 1.0210028886795044, "Finetune/Step": 2061, "Finetune/Step Time": 4.581440692767501} +{"Finetune/Learning Rate": 1.3735281547539304e-05, "Finetune/Loss": 0.8884193897247314, "Finetune/Loss (Raw)": 0.8606467247009277, "Finetune/Step": 2062, "Finetune/Step Time": 4.581418531015515} +{"Finetune/Learning Rate": 1.3729502118890463e-05, "Finetune/Loss": 0.8881851434707642, "Finetune/Loss (Raw)": 0.9953327178955078, "Finetune/Step": 2063, "Finetune/Step Time": 4.572733025997877} +{"Finetune/Learning Rate": 1.3723721242894663e-05, "Finetune/Loss": 0.8888393044471741, "Finetune/Loss (Raw)": 0.9137758016586304, "Finetune/Step": 2064, "Finetune/Step Time": 4.586523057892919} +{"Finetune/Learning Rate": 1.3717938921795347e-05, "Finetune/Loss": 0.8888298273086548, "Finetune/Loss (Raw)": 0.9615543484687805, "Finetune/Step": 2065, "Finetune/Step Time": 4.593656625598669} +{"Finetune/Learning Rate": 1.3712155157836522e-05, "Finetune/Loss": 0.8882820010185242, "Finetune/Loss (Raw)": 0.34011343121528625, "Finetune/Step": 2066, "Finetune/Step Time": 4.593453695997596} +{"Finetune/Learning Rate": 1.3706369953262754e-05, "Finetune/Loss": 0.8884438276290894, "Finetune/Loss (Raw)": 0.940502405166626, "Finetune/Step": 2067, "Finetune/Step Time": 4.594160735607147} +{"Finetune/Learning Rate": 1.3700583310319171e-05, "Finetune/Loss": 0.8910939693450928, "Finetune/Loss (Raw)": 0.9175461530685425, "Finetune/Step": 2068, "Finetune/Step Time": 4.57701295427978} +{"Finetune/Learning Rate": 1.3694795231251453e-05, "Finetune/Loss": 0.8912186622619629, "Finetune/Loss (Raw)": 0.907645046710968, "Finetune/Step": 2069, "Finetune/Step Time": 4.574012715369463} +{"Finetune/Learning Rate": 1.368900571830584e-05, "Finetune/Loss": 0.8879356384277344, "Finetune/Loss (Raw)": 0.684185802936554, "Finetune/Step": 2070, "Finetune/Step Time": 4.585839981213212} +{"Finetune/Learning Rate": 1.3683214773729123e-05, "Finetune/Loss": 0.8874572515487671, "Finetune/Loss (Raw)": 0.9661040902137756, "Finetune/Step": 2071, "Finetune/Step Time": 4.591903176158667} +{"Finetune/Learning Rate": 1.3677422399768667e-05, "Finetune/Loss": 0.888535737991333, "Finetune/Loss (Raw)": 1.0949149131774902, "Finetune/Step": 2072, "Finetune/Step Time": 4.584850227460265} +{"Finetune/Learning Rate": 1.3671628598672373e-05, "Finetune/Loss": 0.8926140069961548, "Finetune/Loss (Raw)": 0.9094234704971313, "Finetune/Step": 2073, "Finetune/Step Time": 4.55604100227356} +{"Finetune/Learning Rate": 1.3665833372688702e-05, "Finetune/Loss": 0.8933793306350708, "Finetune/Loss (Raw)": 1.0265576839447021, "Finetune/Step": 2074, "Finetune/Step Time": 4.55159754306078} +{"Finetune/Learning Rate": 1.366003672406667e-05, "Finetune/Loss": 0.8914803266525269, "Finetune/Loss (Raw)": 0.8669925332069397, "Finetune/Step": 2075, "Finetune/Step Time": 4.551086757332087} +{"Finetune/Learning Rate": 1.3654238655055836e-05, "Finetune/Loss": 0.8891004323959351, "Finetune/Loss (Raw)": 0.7763542532920837, "Finetune/Step": 2076, "Finetune/Step Time": 4.563441760838032} +{"Finetune/Learning Rate": 1.364843916790633e-05, "Finetune/Loss": 0.8876057267189026, "Finetune/Loss (Raw)": 0.8624701499938965, "Finetune/Step": 2077, "Finetune/Step Time": 4.563288453966379} +{"Finetune/Learning Rate": 1.3642638264868808e-05, "Finetune/Loss": 0.8860979080200195, "Finetune/Loss (Raw)": 0.8817867636680603, "Finetune/Step": 2078, "Finetune/Step Time": 4.563153011724353} +{"Finetune/Learning Rate": 1.3636835948194494e-05, "Finetune/Loss": 0.8850579857826233, "Finetune/Loss (Raw)": 0.9199610948562622, "Finetune/Step": 2079, "Finetune/Step Time": 4.558001911267638} +{"Finetune/Learning Rate": 1.3631032220135152e-05, "Finetune/Loss": 0.8828587532043457, "Finetune/Loss (Raw)": 0.7771596908569336, "Finetune/Step": 2080, "Finetune/Step Time": 4.566195540130138} +{"Finetune/Learning Rate": 1.3625227082943093e-05, "Finetune/Loss": 0.8844910264015198, "Finetune/Loss (Raw)": 0.910823404788971, "Finetune/Step": 2081, "Finetune/Step Time": 4.5659431498497725} +{"Finetune/Learning Rate": 1.361942053887118e-05, "Finetune/Loss": 0.8846171498298645, "Finetune/Loss (Raw)": 0.850326657295227, "Finetune/Step": 2082, "Finetune/Step Time": 4.547379691153765} +{"Finetune/Learning Rate": 1.361361259017282e-05, "Finetune/Loss": 0.8857877254486084, "Finetune/Loss (Raw)": 0.9702121615409851, "Finetune/Step": 2083, "Finetune/Step Time": 4.530115824192762} +{"Finetune/Learning Rate": 1.3607803239101965e-05, "Finetune/Loss": 0.8877620697021484, "Finetune/Loss (Raw)": 1.1006923913955688, "Finetune/Step": 2084, "Finetune/Step Time": 4.531529534608126} +{"Finetune/Learning Rate": 1.360199248791311e-05, "Finetune/Loss": 0.8819736242294312, "Finetune/Loss (Raw)": 0.3220067024230957, "Finetune/Step": 2085, "Finetune/Step Time": 4.569607684388757} +{"Finetune/Learning Rate": 1.359618033886129e-05, "Finetune/Loss": 0.8814544677734375, "Finetune/Loss (Raw)": 0.9795801639556885, "Finetune/Step": 2086, "Finetune/Step Time": 4.574453130364418} +{"Finetune/Learning Rate": 1.359036679420209e-05, "Finetune/Loss": 0.8832423686981201, "Finetune/Loss (Raw)": 1.0556832551956177, "Finetune/Step": 2087, "Finetune/Step Time": 4.563968127593398} +{"Finetune/Learning Rate": 1.3584551856191635e-05, "Finetune/Loss": 0.8828039765357971, "Finetune/Loss (Raw)": 1.039751648902893, "Finetune/Step": 2088, "Finetune/Step Time": 4.554047739133239} +{"Finetune/Learning Rate": 1.3578735527086585e-05, "Finetune/Loss": 0.8840625286102295, "Finetune/Loss (Raw)": 0.9677525162696838, "Finetune/Step": 2089, "Finetune/Step Time": 4.54888160340488} +{"Finetune/Learning Rate": 1.357291780914415e-05, "Finetune/Loss": 0.8829907178878784, "Finetune/Loss (Raw)": 0.2766678035259247, "Finetune/Step": 2090, "Finetune/Step Time": 4.5506092850118876} +{"Finetune/Learning Rate": 1.3567098704622064e-05, "Finetune/Loss": 0.8864403963088989, "Finetune/Loss (Raw)": 0.8409011960029602, "Finetune/Step": 2091, "Finetune/Step Time": 4.51513990573585} +{"Finetune/Learning Rate": 1.3561278215778618e-05, "Finetune/Loss": 0.8864370584487915, "Finetune/Loss (Raw)": 0.9958810806274414, "Finetune/Step": 2092, "Finetune/Step Time": 4.502499431371689} +{"Finetune/Learning Rate": 1.355545634487262e-05, "Finetune/Loss": 0.8876662850379944, "Finetune/Loss (Raw)": 0.9599151015281677, "Finetune/Step": 2093, "Finetune/Step Time": 4.50022722966969} +{"Finetune/Learning Rate": 1.354963309416343e-05, "Finetune/Loss": 0.8891197443008423, "Finetune/Loss (Raw)": 1.0561643838882446, "Finetune/Step": 2094, "Finetune/Step Time": 4.500689081847668} +{"Finetune/Learning Rate": 1.3543808465910938e-05, "Finetune/Loss": 0.8901183009147644, "Finetune/Loss (Raw)": 0.9611677527427673, "Finetune/Step": 2095, "Finetune/Step Time": 4.505298115313053} +{"Finetune/Learning Rate": 1.3537982462375566e-05, "Finetune/Loss": 0.8894633650779724, "Finetune/Loss (Raw)": 0.924893319606781, "Finetune/Step": 2096, "Finetune/Step Time": 4.506848879158497} +{"Finetune/Learning Rate": 1.3532155085818276e-05, "Finetune/Loss": 0.8941528797149658, "Finetune/Loss (Raw)": 1.0490072965621948, "Finetune/Step": 2097, "Finetune/Step Time": 4.466781809926033} +{"Finetune/Learning Rate": 1.3526326338500553e-05, "Finetune/Loss": 0.8942065834999084, "Finetune/Loss (Raw)": 0.8988244533538818, "Finetune/Step": 2098, "Finetune/Step Time": 4.486624035984278} +{"Finetune/Learning Rate": 1.3520496222684424e-05, "Finetune/Loss": 0.8938701152801514, "Finetune/Loss (Raw)": 0.36491233110427856, "Finetune/Step": 2099, "Finetune/Step Time": 4.489670433104038} +{"Finetune/Learning Rate": 1.3514664740632446e-05, "Finetune/Loss": 0.8928403854370117, "Finetune/Loss (Raw)": 0.7768986225128174, "Finetune/Step": 2100, "Finetune/Step Time": 4.490027900785208} +{"Finetune/Learning Rate": 1.3508831894607696e-05, "Finetune/Loss": 0.8938101530075073, "Finetune/Loss (Raw)": 0.994690477848053, "Finetune/Step": 2101, "Finetune/Step Time": 4.449567832052708} +{"Finetune/Learning Rate": 1.3502997686873797e-05, "Finetune/Loss": 0.8924980759620667, "Finetune/Loss (Raw)": 0.8968038558959961, "Finetune/Step": 2102, "Finetune/Step Time": 4.4526694398373365} +{"Finetune/Learning Rate": 1.3497162119694883e-05, "Finetune/Loss": 0.8882747888565063, "Finetune/Loss (Raw)": 0.3735318183898926, "Finetune/Step": 2103, "Finetune/Step Time": 4.4932373110204935} +{"Finetune/Learning Rate": 1.3491325195335627e-05, "Finetune/Loss": 0.8879292011260986, "Finetune/Loss (Raw)": 0.8942661285400391, "Finetune/Step": 2104, "Finetune/Step Time": 4.500535573810339} +{"Finetune/Learning Rate": 1.3485486916061228e-05, "Finetune/Loss": 0.8874524831771851, "Finetune/Loss (Raw)": 0.9169585108757019, "Finetune/Step": 2105, "Finetune/Step Time": 4.510643366724253} +{"Finetune/Learning Rate": 1.3479647284137398e-05, "Finetune/Loss": 0.8865905404090881, "Finetune/Loss (Raw)": 0.9244195222854614, "Finetune/Step": 2106, "Finetune/Step Time": 4.509846383705735} +{"Finetune/Learning Rate": 1.3473806301830398e-05, "Finetune/Loss": 0.8908816576004028, "Finetune/Loss (Raw)": 0.9292773008346558, "Finetune/Step": 2107, "Finetune/Step Time": 4.469971856102347} +{"Finetune/Learning Rate": 1.346796397140699e-05, "Finetune/Loss": 0.8915058970451355, "Finetune/Loss (Raw)": 0.8807281851768494, "Finetune/Step": 2108, "Finetune/Step Time": 4.457712672650814} +{"Finetune/Learning Rate": 1.3462120295134476e-05, "Finetune/Loss": 0.8910694718360901, "Finetune/Loss (Raw)": 0.9436460137367249, "Finetune/Step": 2109, "Finetune/Step Time": 4.460691541433334} +{"Finetune/Learning Rate": 1.3456275275280661e-05, "Finetune/Loss": 0.8910470604896545, "Finetune/Loss (Raw)": 0.9578485488891602, "Finetune/Step": 2110, "Finetune/Step Time": 4.4683626759797335} +{"Finetune/Learning Rate": 1.3450428914113892e-05, "Finetune/Loss": 0.8912643790245056, "Finetune/Loss (Raw)": 0.9203429818153381, "Finetune/Step": 2111, "Finetune/Step Time": 4.483219025656581} +{"Finetune/Learning Rate": 1.3444581213903026e-05, "Finetune/Loss": 0.8863606452941895, "Finetune/Loss (Raw)": 0.45931506156921387, "Finetune/Step": 2112, "Finetune/Step Time": 4.522150699049234} +{"Finetune/Learning Rate": 1.3438732176917437e-05, "Finetune/Loss": 0.8858224153518677, "Finetune/Loss (Raw)": 0.9365199208259583, "Finetune/Step": 2113, "Finetune/Step Time": 4.520131150260568} +{"Finetune/Learning Rate": 1.3432881805427032e-05, "Finetune/Loss": 0.8852756023406982, "Finetune/Loss (Raw)": 1.0994982719421387, "Finetune/Step": 2114, "Finetune/Step Time": 4.54323129914701} +{"Finetune/Learning Rate": 1.3427030101702215e-05, "Finetune/Loss": 0.8857735395431519, "Finetune/Loss (Raw)": 1.0008066892623901, "Finetune/Step": 2115, "Finetune/Step Time": 4.514575712382793} +{"Finetune/Learning Rate": 1.3421177068013925e-05, "Finetune/Loss": 0.8822762370109558, "Finetune/Loss (Raw)": 0.4232621192932129, "Finetune/Step": 2116, "Finetune/Step Time": 4.5495857279747725} +{"Finetune/Learning Rate": 1.3415322706633606e-05, "Finetune/Loss": 0.8806577920913696, "Finetune/Loss (Raw)": 0.7608141899108887, "Finetune/Step": 2117, "Finetune/Step Time": 4.565586993470788} +{"Finetune/Learning Rate": 1.3409467019833223e-05, "Finetune/Loss": 0.8796950578689575, "Finetune/Loss (Raw)": 0.862429678440094, "Finetune/Step": 2118, "Finetune/Step Time": 4.5612805634737015} +{"Finetune/Learning Rate": 1.3403610009885258e-05, "Finetune/Loss": 0.8801623582839966, "Finetune/Loss (Raw)": 0.8659000992774963, "Finetune/Step": 2119, "Finetune/Step Time": 4.574703590944409} +{"Finetune/Learning Rate": 1.3397751679062692e-05, "Finetune/Loss": 0.8752797842025757, "Finetune/Loss (Raw)": 0.3368914723396301, "Finetune/Step": 2120, "Finetune/Step Time": 4.614242546260357} +{"Finetune/Learning Rate": 1.3391892029639037e-05, "Finetune/Loss": 0.8759190440177917, "Finetune/Loss (Raw)": 0.9383113384246826, "Finetune/Step": 2121, "Finetune/Step Time": 4.606413250789046} +{"Finetune/Learning Rate": 1.3386031063888309e-05, "Finetune/Loss": 0.877862811088562, "Finetune/Loss (Raw)": 0.9969841837882996, "Finetune/Step": 2122, "Finetune/Step Time": 4.600320408120751} +{"Finetune/Learning Rate": 1.3380168784085028e-05, "Finetune/Loss": 0.879467785358429, "Finetune/Loss (Raw)": 0.9950143694877625, "Finetune/Step": 2123, "Finetune/Step Time": 4.591790586709976} +{"Finetune/Learning Rate": 1.337430519250424e-05, "Finetune/Loss": 0.8806829452514648, "Finetune/Loss (Raw)": 0.9870150089263916, "Finetune/Step": 2124, "Finetune/Step Time": 4.588906813412905} +{"Finetune/Learning Rate": 1.3368440291421475e-05, "Finetune/Loss": 0.8801043033599854, "Finetune/Loss (Raw)": 0.9893240928649902, "Finetune/Step": 2125, "Finetune/Step Time": 4.596684882417321} +{"Finetune/Learning Rate": 1.3362574083112804e-05, "Finetune/Loss": 0.8809429407119751, "Finetune/Loss (Raw)": 1.0095199346542358, "Finetune/Step": 2126, "Finetune/Step Time": 4.593278057873249} +{"Finetune/Learning Rate": 1.3356706569854778e-05, "Finetune/Loss": 0.8849397301673889, "Finetune/Loss (Raw)": 0.9272110462188721, "Finetune/Step": 2127, "Finetune/Step Time": 4.556720193475485} +{"Finetune/Learning Rate": 1.3350837753924466e-05, "Finetune/Loss": 0.8809438943862915, "Finetune/Loss (Raw)": 0.35582903027534485, "Finetune/Step": 2128, "Finetune/Step Time": 4.574570333585143} +{"Finetune/Learning Rate": 1.3344967637599445e-05, "Finetune/Loss": 0.8757867217063904, "Finetune/Loss (Raw)": 0.2805403769016266, "Finetune/Step": 2129, "Finetune/Step Time": 4.614150995388627} +{"Finetune/Learning Rate": 1.3339096223157782e-05, "Finetune/Loss": 0.8760541677474976, "Finetune/Loss (Raw)": 0.9604266285896301, "Finetune/Step": 2130, "Finetune/Step Time": 4.612675618380308} +{"Finetune/Learning Rate": 1.3333223512878068e-05, "Finetune/Loss": 0.8768858313560486, "Finetune/Loss (Raw)": 0.9764384627342224, "Finetune/Step": 2131, "Finetune/Step Time": 4.596570311114192} +{"Finetune/Learning Rate": 1.3327349509039387e-05, "Finetune/Loss": 0.8767470121383667, "Finetune/Loss (Raw)": 0.8713651895523071, "Finetune/Step": 2132, "Finetune/Step Time": 4.600828493013978} +{"Finetune/Learning Rate": 1.3321474213921317e-05, "Finetune/Loss": 0.8786349296569824, "Finetune/Loss (Raw)": 1.1445355415344238, "Finetune/Step": 2133, "Finetune/Step Time": 4.597709886729717} +{"Finetune/Learning Rate": 1.3315597629803959e-05, "Finetune/Loss": 0.8782258629798889, "Finetune/Loss (Raw)": 0.871017336845398, "Finetune/Step": 2134, "Finetune/Step Time": 4.608337605372071} +{"Finetune/Learning Rate": 1.3309719758967885e-05, "Finetune/Loss": 0.8762170076370239, "Finetune/Loss (Raw)": 0.8651930689811707, "Finetune/Step": 2135, "Finetune/Step Time": 4.6038213316351175} +{"Finetune/Learning Rate": 1.3303840603694192e-05, "Finetune/Loss": 0.8760616779327393, "Finetune/Loss (Raw)": 0.9557567834854126, "Finetune/Step": 2136, "Finetune/Step Time": 4.596148237586021} +{"Finetune/Learning Rate": 1.3297960166264462e-05, "Finetune/Loss": 0.8724914789199829, "Finetune/Loss (Raw)": 0.4245927333831787, "Finetune/Step": 2137, "Finetune/Step Time": 4.63697805814445} +{"Finetune/Learning Rate": 1.3292078448960778e-05, "Finetune/Loss": 0.8717175722122192, "Finetune/Loss (Raw)": 0.9180511832237244, "Finetune/Step": 2138, "Finetune/Step Time": 4.629842426627874} +{"Finetune/Learning Rate": 1.3286195454065726e-05, "Finetune/Loss": 0.8700231313705444, "Finetune/Loss (Raw)": 0.8666419982910156, "Finetune/Step": 2139, "Finetune/Step Time": 4.643735110759735} +{"Finetune/Learning Rate": 1.3280311183862371e-05, "Finetune/Loss": 0.8703718185424805, "Finetune/Loss (Raw)": 0.9649719595909119, "Finetune/Step": 2140, "Finetune/Step Time": 4.658607026562095} +{"Finetune/Learning Rate": 1.3274425640634287e-05, "Finetune/Loss": 0.8708540201187134, "Finetune/Loss (Raw)": 1.0675854682922363, "Finetune/Step": 2141, "Finetune/Step Time": 4.644705595448613} +{"Finetune/Learning Rate": 1.3268538826665544e-05, "Finetune/Loss": 0.8714520335197449, "Finetune/Loss (Raw)": 0.8572020530700684, "Finetune/Step": 2142, "Finetune/Step Time": 4.634091524407268} +{"Finetune/Learning Rate": 1.326265074424069e-05, "Finetune/Loss": 0.8755662441253662, "Finetune/Loss (Raw)": 0.9237726926803589, "Finetune/Step": 2143, "Finetune/Step Time": 4.611835733056068} +{"Finetune/Learning Rate": 1.3256761395644787e-05, "Finetune/Loss": 0.8793668746948242, "Finetune/Loss (Raw)": 0.890209972858429, "Finetune/Step": 2144, "Finetune/Step Time": 4.583178795874119} +{"Finetune/Learning Rate": 1.3250870783163363e-05, "Finetune/Loss": 0.878258228302002, "Finetune/Loss (Raw)": 0.6925681233406067, "Finetune/Step": 2145, "Finetune/Step Time": 4.583126028999686} +{"Finetune/Learning Rate": 1.3244978909082457e-05, "Finetune/Loss": 0.877997100353241, "Finetune/Loss (Raw)": 0.9169983267784119, "Finetune/Step": 2146, "Finetune/Step Time": 4.5982400216162205} +{"Finetune/Learning Rate": 1.3239085775688591e-05, "Finetune/Loss": 0.8785548806190491, "Finetune/Loss (Raw)": 0.9349108338356018, "Finetune/Step": 2147, "Finetune/Step Time": 4.599991584196687} +{"Finetune/Learning Rate": 1.3233191385268771e-05, "Finetune/Loss": 0.8792943954467773, "Finetune/Loss (Raw)": 1.0638008117675781, "Finetune/Step": 2148, "Finetune/Step Time": 4.602074800059199} +{"Finetune/Learning Rate": 1.3227295740110502e-05, "Finetune/Loss": 0.8796980381011963, "Finetune/Loss (Raw)": 0.9921879172325134, "Finetune/Step": 2149, "Finetune/Step Time": 4.5856922045350075} +{"Finetune/Learning Rate": 1.3221398842501758e-05, "Finetune/Loss": 0.8797527551651001, "Finetune/Loss (Raw)": 0.9181493520736694, "Finetune/Step": 2150, "Finetune/Step Time": 4.583379307761788} +{"Finetune/Learning Rate": 1.321550069473102e-05, "Finetune/Loss": 0.8827115297317505, "Finetune/Loss (Raw)": 1.1313018798828125, "Finetune/Step": 2151, "Finetune/Step Time": 4.567749090492725} +{"Finetune/Learning Rate": 1.3209601299087238e-05, "Finetune/Loss": 0.8818702101707458, "Finetune/Loss (Raw)": 0.8098577260971069, "Finetune/Step": 2152, "Finetune/Step Time": 4.573608769103885} +{"Finetune/Learning Rate": 1.3203700657859855e-05, "Finetune/Loss": 0.8818000555038452, "Finetune/Loss (Raw)": 0.8552999496459961, "Finetune/Step": 2153, "Finetune/Step Time": 4.5630388632416725} +{"Finetune/Learning Rate": 1.3197798773338793e-05, "Finetune/Loss": 0.8811516761779785, "Finetune/Loss (Raw)": 0.8691720366477966, "Finetune/Step": 2154, "Finetune/Step Time": 4.572583869099617} +{"Finetune/Learning Rate": 1.3191895647814457e-05, "Finetune/Loss": 0.8816342353820801, "Finetune/Loss (Raw)": 0.9137811660766602, "Finetune/Step": 2155, "Finetune/Step Time": 4.579337192699313} +{"Finetune/Learning Rate": 1.318599128357774e-05, "Finetune/Loss": 0.8802272081375122, "Finetune/Loss (Raw)": 0.7666594982147217, "Finetune/Step": 2156, "Finetune/Step Time": 4.574676705524325} +{"Finetune/Learning Rate": 1.3180085682920005e-05, "Finetune/Loss": 0.8790014982223511, "Finetune/Loss (Raw)": 0.8760692477226257, "Finetune/Step": 2157, "Finetune/Step Time": 4.584243629127741} +{"Finetune/Learning Rate": 1.31741788481331e-05, "Finetune/Loss": 0.8838398456573486, "Finetune/Loss (Raw)": 0.9707755446434021, "Finetune/Step": 2158, "Finetune/Step Time": 4.54454374499619} +{"Finetune/Learning Rate": 1.316827078150936e-05, "Finetune/Loss": 0.8845569491386414, "Finetune/Loss (Raw)": 0.9784470200538635, "Finetune/Step": 2159, "Finetune/Step Time": 4.555248711258173} +{"Finetune/Learning Rate": 1.316236148534158e-05, "Finetune/Loss": 0.8838143348693848, "Finetune/Loss (Raw)": 0.9333279728889465, "Finetune/Step": 2160, "Finetune/Step Time": 4.54540403932333} +{"Finetune/Learning Rate": 1.3156450961923051e-05, "Finetune/Loss": 0.8827613592147827, "Finetune/Loss (Raw)": 0.8619446158409119, "Finetune/Step": 2161, "Finetune/Step Time": 4.545773521065712} +{"Finetune/Learning Rate": 1.3150539213547528e-05, "Finetune/Loss": 0.8816388845443726, "Finetune/Loss (Raw)": 0.9037446975708008, "Finetune/Step": 2162, "Finetune/Step Time": 4.549998220056295} +{"Finetune/Learning Rate": 1.3144626242509243e-05, "Finetune/Loss": 0.8821160793304443, "Finetune/Loss (Raw)": 0.8831018805503845, "Finetune/Step": 2163, "Finetune/Step Time": 4.539712622761726} +{"Finetune/Learning Rate": 1.3138712051102908e-05, "Finetune/Loss": 0.882624626159668, "Finetune/Loss (Raw)": 1.0388458967208862, "Finetune/Step": 2164, "Finetune/Step Time": 4.525986472144723} +{"Finetune/Learning Rate": 1.3132796641623703e-05, "Finetune/Loss": 0.8825810551643372, "Finetune/Loss (Raw)": 0.9294825196266174, "Finetune/Step": 2165, "Finetune/Step Time": 4.529946703463793} +{"Finetune/Learning Rate": 1.3126880016367287e-05, "Finetune/Loss": 0.8837947249412537, "Finetune/Loss (Raw)": 0.95989590883255, "Finetune/Step": 2166, "Finetune/Step Time": 4.535503191873431} +{"Finetune/Learning Rate": 1.3120962177629783e-05, "Finetune/Loss": 0.8839168548583984, "Finetune/Loss (Raw)": 1.0604667663574219, "Finetune/Step": 2167, "Finetune/Step Time": 4.535281449556351} +{"Finetune/Learning Rate": 1.311504312770779e-05, "Finetune/Loss": 0.8832181096076965, "Finetune/Loss (Raw)": 0.9176238179206848, "Finetune/Step": 2168, "Finetune/Step Time": 4.5343425292521715} +{"Finetune/Learning Rate": 1.3109122868898374e-05, "Finetune/Loss": 0.8830889463424683, "Finetune/Loss (Raw)": 0.956028401851654, "Finetune/Step": 2169, "Finetune/Step Time": 4.491447437554598} +{"Finetune/Learning Rate": 1.310320140349907e-05, "Finetune/Loss": 0.8832228183746338, "Finetune/Loss (Raw)": 0.8323084712028503, "Finetune/Step": 2170, "Finetune/Step Time": 4.492891216650605} +{"Finetune/Learning Rate": 1.3097278733807892e-05, "Finetune/Loss": 0.8830732107162476, "Finetune/Loss (Raw)": 0.9136684536933899, "Finetune/Step": 2171, "Finetune/Step Time": 4.502262270078063} +{"Finetune/Learning Rate": 1.3091354862123304e-05, "Finetune/Loss": 0.8828234076499939, "Finetune/Loss (Raw)": 0.8987980484962463, "Finetune/Step": 2172, "Finetune/Step Time": 4.5020993538200855} +{"Finetune/Learning Rate": 1.3085429790744248e-05, "Finetune/Loss": 0.8820250034332275, "Finetune/Loss (Raw)": 0.905860424041748, "Finetune/Step": 2173, "Finetune/Step Time": 4.505892915651202} +{"Finetune/Learning Rate": 1.3079503521970127e-05, "Finetune/Loss": 0.8799263834953308, "Finetune/Loss (Raw)": 0.7641163468360901, "Finetune/Step": 2174, "Finetune/Step Time": 4.518736075609922} +{"Finetune/Learning Rate": 1.3073576058100815e-05, "Finetune/Loss": 0.8795753717422485, "Finetune/Loss (Raw)": 0.9184249639511108, "Finetune/Step": 2175, "Finetune/Step Time": 4.534571580588818} +{"Finetune/Learning Rate": 1.3067647401436636e-05, "Finetune/Loss": 0.8809397220611572, "Finetune/Loss (Raw)": 0.9841967225074768, "Finetune/Step": 2176, "Finetune/Step Time": 4.534295300021768} +{"Finetune/Learning Rate": 1.3061717554278393e-05, "Finetune/Loss": 0.8837841153144836, "Finetune/Loss (Raw)": 1.0619947910308838, "Finetune/Step": 2177, "Finetune/Step Time": 4.518316393718123} +{"Finetune/Learning Rate": 1.3055786518927342e-05, "Finetune/Loss": 0.8835691213607788, "Finetune/Loss (Raw)": 0.7929194569587708, "Finetune/Step": 2178, "Finetune/Step Time": 4.50610464066267} +{"Finetune/Learning Rate": 1.3049854297685206e-05, "Finetune/Loss": 0.8825688362121582, "Finetune/Loss (Raw)": 0.9439057111740112, "Finetune/Step": 2179, "Finetune/Step Time": 4.515859784558415} +{"Finetune/Learning Rate": 1.3043920892854155e-05, "Finetune/Loss": 0.8825148940086365, "Finetune/Loss (Raw)": 0.9678565263748169, "Finetune/Step": 2180, "Finetune/Step Time": 4.509356586262584} +{"Finetune/Learning Rate": 1.3037986306736839e-05, "Finetune/Loss": 0.8827660083770752, "Finetune/Loss (Raw)": 0.9656258821487427, "Finetune/Step": 2181, "Finetune/Step Time": 4.507146578282118} +{"Finetune/Learning Rate": 1.3032050541636343e-05, "Finetune/Loss": 0.8823590874671936, "Finetune/Loss (Raw)": 0.9877179861068726, "Finetune/Step": 2182, "Finetune/Step Time": 4.515256999060512} +{"Finetune/Learning Rate": 1.3026113599856229e-05, "Finetune/Loss": 0.8806753754615784, "Finetune/Loss (Raw)": 0.7840986847877502, "Finetune/Step": 2183, "Finetune/Step Time": 4.531157227233052} +{"Finetune/Learning Rate": 1.302017548370051e-05, "Finetune/Loss": 0.8791275024414062, "Finetune/Loss (Raw)": 0.8007972836494446, "Finetune/Step": 2184, "Finetune/Step Time": 4.54572581872344} +{"Finetune/Learning Rate": 1.3014236195473649e-05, "Finetune/Loss": 0.8795000910758972, "Finetune/Loss (Raw)": 0.9269618391990662, "Finetune/Step": 2185, "Finetune/Step Time": 4.532505430281162} +{"Finetune/Learning Rate": 1.3008295737480567e-05, "Finetune/Loss": 0.8794465065002441, "Finetune/Loss (Raw)": 0.8928539156913757, "Finetune/Step": 2186, "Finetune/Step Time": 4.512083863839507} +{"Finetune/Learning Rate": 1.300235411202664e-05, "Finetune/Loss": 0.8787924647331238, "Finetune/Loss (Raw)": 0.743450403213501, "Finetune/Step": 2187, "Finetune/Step Time": 4.5455176047980785} +{"Finetune/Learning Rate": 1.2996411321417699e-05, "Finetune/Loss": 0.8787063360214233, "Finetune/Loss (Raw)": 0.9128644466400146, "Finetune/Step": 2188, "Finetune/Step Time": 4.578323418274522} +{"Finetune/Learning Rate": 1.2990467367960027e-05, "Finetune/Loss": 0.877251386642456, "Finetune/Loss (Raw)": 0.8347675800323486, "Finetune/Step": 2189, "Finetune/Step Time": 4.585545131936669} +{"Finetune/Learning Rate": 1.298452225396035e-05, "Finetune/Loss": 0.8771267533302307, "Finetune/Loss (Raw)": 0.8446952104568481, "Finetune/Step": 2190, "Finetune/Step Time": 4.594699202105403} +{"Finetune/Learning Rate": 1.2978575981725858e-05, "Finetune/Loss": 0.8767964839935303, "Finetune/Loss (Raw)": 0.9530569314956665, "Finetune/Step": 2191, "Finetune/Step Time": 4.609756069257855} +{"Finetune/Learning Rate": 1.2972628553564178e-05, "Finetune/Loss": 0.875853419303894, "Finetune/Loss (Raw)": 0.793066143989563, "Finetune/Step": 2192, "Finetune/Step Time": 4.591309975832701} +{"Finetune/Learning Rate": 1.2966679971783387e-05, "Finetune/Loss": 0.8760620355606079, "Finetune/Loss (Raw)": 0.9882526397705078, "Finetune/Step": 2193, "Finetune/Step Time": 4.586757222190499} +{"Finetune/Learning Rate": 1.2960730238692023e-05, "Finetune/Loss": 0.8809373378753662, "Finetune/Loss (Raw)": 0.9641432166099548, "Finetune/Step": 2194, "Finetune/Step Time": 4.5512884221971035} +{"Finetune/Learning Rate": 1.2954779356599056e-05, "Finetune/Loss": 0.8813414573669434, "Finetune/Loss (Raw)": 0.9922346472740173, "Finetune/Step": 2195, "Finetune/Step Time": 4.550084264948964} +{"Finetune/Learning Rate": 1.2948827327813908e-05, "Finetune/Loss": 0.8808726072311401, "Finetune/Loss (Raw)": 0.8575295805931091, "Finetune/Step": 2196, "Finetune/Step Time": 4.56472142226994} +{"Finetune/Learning Rate": 1.2942874154646447e-05, "Finetune/Loss": 0.8816025257110596, "Finetune/Loss (Raw)": 1.0010850429534912, "Finetune/Step": 2197, "Finetune/Step Time": 4.565822901204228} +{"Finetune/Learning Rate": 1.2936919839406978e-05, "Finetune/Loss": 0.8831040859222412, "Finetune/Loss (Raw)": 0.8763822913169861, "Finetune/Step": 2198, "Finetune/Step Time": 4.559620397165418} +{"Finetune/Learning Rate": 1.2930964384406259e-05, "Finetune/Loss": 0.8821540474891663, "Finetune/Loss (Raw)": 0.8445018529891968, "Finetune/Step": 2199, "Finetune/Step Time": 4.550787780433893} +{"Finetune/Learning Rate": 1.2925007791955487e-05, "Finetune/Loss": 0.8804916143417358, "Finetune/Loss (Raw)": 0.882121205329895, "Finetune/Step": 2200, "Finetune/Step Time": 4.556733721867204} +{"Finetune/Learning Rate": 1.2919050064366294e-05, "Finetune/Loss": 0.8761819005012512, "Finetune/Loss (Raw)": 0.35778266191482544, "Finetune/Step": 2201, "Finetune/Step Time": 4.583478260785341} +{"Finetune/Learning Rate": 1.2913091203950764e-05, "Finetune/Loss": 0.87530517578125, "Finetune/Loss (Raw)": 0.914326012134552, "Finetune/Step": 2202, "Finetune/Step Time": 4.569722188636661} +{"Finetune/Learning Rate": 1.2907131213021404e-05, "Finetune/Loss": 0.8753436803817749, "Finetune/Loss (Raw)": 0.8719243407249451, "Finetune/Step": 2203, "Finetune/Step Time": 4.5705016031861305} +{"Finetune/Learning Rate": 1.2901170093891182e-05, "Finetune/Loss": 0.8758765459060669, "Finetune/Loss (Raw)": 0.8445692658424377, "Finetune/Step": 2204, "Finetune/Step Time": 4.567622676491737} +{"Finetune/Learning Rate": 1.2895207848873488e-05, "Finetune/Loss": 0.8724782466888428, "Finetune/Loss (Raw)": 0.4274846017360687, "Finetune/Step": 2205, "Finetune/Step Time": 4.606139525771141} +{"Finetune/Learning Rate": 1.288924448028215e-05, "Finetune/Loss": 0.8708392977714539, "Finetune/Loss (Raw)": 0.6720032691955566, "Finetune/Step": 2206, "Finetune/Step Time": 4.661368653178215} +{"Finetune/Learning Rate": 1.288327999043144e-05, "Finetune/Loss": 0.8711699843406677, "Finetune/Loss (Raw)": 0.9622881412506104, "Finetune/Step": 2207, "Finetune/Step Time": 4.656380822882056} +{"Finetune/Learning Rate": 1.2877314381636052e-05, "Finetune/Loss": 0.871848464012146, "Finetune/Loss (Raw)": 0.864000678062439, "Finetune/Step": 2208, "Finetune/Step Time": 4.671077229082584} +{"Finetune/Learning Rate": 1.287134765621113e-05, "Finetune/Loss": 0.8717036247253418, "Finetune/Loss (Raw)": 0.8922857046127319, "Finetune/Step": 2209, "Finetune/Step Time": 4.661124909296632} +{"Finetune/Learning Rate": 1.2865379816472244e-05, "Finetune/Loss": 0.8681397438049316, "Finetune/Loss (Raw)": 0.39414918422698975, "Finetune/Step": 2210, "Finetune/Step Time": 4.699233679100871} +{"Finetune/Learning Rate": 1.2859410864735389e-05, "Finetune/Loss": 0.8683872222900391, "Finetune/Loss (Raw)": 1.001889944076538, "Finetune/Step": 2211, "Finetune/Step Time": 4.7235611621290445} +{"Finetune/Learning Rate": 1.2853440803317003e-05, "Finetune/Loss": 0.8653754591941833, "Finetune/Loss (Raw)": 0.71518874168396, "Finetune/Step": 2212, "Finetune/Step Time": 4.740390386432409} +{"Finetune/Learning Rate": 1.2847469634533952e-05, "Finetune/Loss": 0.8692057132720947, "Finetune/Loss (Raw)": 0.8122828602790833, "Finetune/Step": 2213, "Finetune/Step Time": 4.708272319287062} +{"Finetune/Learning Rate": 1.2841497360703528e-05, "Finetune/Loss": 0.8671001195907593, "Finetune/Loss (Raw)": 0.710063636302948, "Finetune/Step": 2214, "Finetune/Step Time": 4.72052395530045} +{"Finetune/Learning Rate": 1.283552398414345e-05, "Finetune/Loss": 0.8664274215698242, "Finetune/Loss (Raw)": 0.9695766568183899, "Finetune/Step": 2215, "Finetune/Step Time": 4.7204816825687885} +{"Finetune/Learning Rate": 1.2829549507171877e-05, "Finetune/Loss": 0.865688681602478, "Finetune/Loss (Raw)": 0.9451923966407776, "Finetune/Step": 2216, "Finetune/Step Time": 4.723198847845197} +{"Finetune/Learning Rate": 1.2823573932107381e-05, "Finetune/Loss": 0.8660196661949158, "Finetune/Loss (Raw)": 1.0101163387298584, "Finetune/Step": 2217, "Finetune/Step Time": 4.724221287295222} +{"Finetune/Learning Rate": 1.2817597261268965e-05, "Finetune/Loss": 0.8696205615997314, "Finetune/Loss (Raw)": 0.7375757098197937, "Finetune/Step": 2218, "Finetune/Step Time": 4.689259357750416} +{"Finetune/Learning Rate": 1.2811619496976066e-05, "Finetune/Loss": 0.8697145581245422, "Finetune/Loss (Raw)": 0.8529371023178101, "Finetune/Step": 2219, "Finetune/Step Time": 4.7193631157279015} +{"Finetune/Learning Rate": 1.2805640641548529e-05, "Finetune/Loss": 0.8696582317352295, "Finetune/Loss (Raw)": 0.9886701703071594, "Finetune/Step": 2220, "Finetune/Step Time": 4.719532642513514} +{"Finetune/Learning Rate": 1.2799660697306636e-05, "Finetune/Loss": 0.8687170743942261, "Finetune/Loss (Raw)": 0.8394514322280884, "Finetune/Step": 2221, "Finetune/Step Time": 4.736242214217782} +{"Finetune/Learning Rate": 1.2793679666571084e-05, "Finetune/Loss": 0.8680120706558228, "Finetune/Loss (Raw)": 0.9659250974655151, "Finetune/Step": 2222, "Finetune/Step Time": 4.745216960087419} +{"Finetune/Learning Rate": 1.2787697551662994e-05, "Finetune/Loss": 0.8678405284881592, "Finetune/Loss (Raw)": 0.939204216003418, "Finetune/Step": 2223, "Finetune/Step Time": 4.749570801854134} +{"Finetune/Learning Rate": 1.2781714354903913e-05, "Finetune/Loss": 0.8677382469177246, "Finetune/Loss (Raw)": 0.9118037819862366, "Finetune/Step": 2224, "Finetune/Step Time": 4.745306394994259} +{"Finetune/Learning Rate": 1.2775730078615798e-05, "Finetune/Loss": 0.8664669990539551, "Finetune/Loss (Raw)": 0.8862848281860352, "Finetune/Step": 2225, "Finetune/Step Time": 4.756856435909867} +{"Finetune/Learning Rate": 1.276974472512103e-05, "Finetune/Loss": 0.8667293190956116, "Finetune/Loss (Raw)": 0.9324044585227966, "Finetune/Step": 2226, "Finetune/Step Time": 4.745246082544327} +{"Finetune/Learning Rate": 1.276375829674241e-05, "Finetune/Loss": 0.8715613484382629, "Finetune/Loss (Raw)": 0.98341304063797, "Finetune/Step": 2227, "Finetune/Step Time": 4.710211733356118} +{"Finetune/Learning Rate": 1.275777079580315e-05, "Finetune/Loss": 0.8725324869155884, "Finetune/Loss (Raw)": 0.9012114405632019, "Finetune/Step": 2228, "Finetune/Step Time": 4.7084319200366735} +{"Finetune/Learning Rate": 1.2751782224626891e-05, "Finetune/Loss": 0.8718916177749634, "Finetune/Loss (Raw)": 0.9126508831977844, "Finetune/Step": 2229, "Finetune/Step Time": 4.713559567928314} +{"Finetune/Learning Rate": 1.274579258553767e-05, "Finetune/Loss": 0.8716039061546326, "Finetune/Loss (Raw)": 0.8599770665168762, "Finetune/Step": 2230, "Finetune/Step Time": 4.714276662096381} +{"Finetune/Learning Rate": 1.2739801880859954e-05, "Finetune/Loss": 0.8763949871063232, "Finetune/Loss (Raw)": 0.9867910742759705, "Finetune/Step": 2231, "Finetune/Step Time": 4.675136184319854} +{"Finetune/Learning Rate": 1.2733810112918618e-05, "Finetune/Loss": 0.877463698387146, "Finetune/Loss (Raw)": 1.0310615301132202, "Finetune/Step": 2232, "Finetune/Step Time": 4.666771121323109} +{"Finetune/Learning Rate": 1.2727817284038953e-05, "Finetune/Loss": 0.8765443563461304, "Finetune/Loss (Raw)": 0.799278199672699, "Finetune/Step": 2233, "Finetune/Step Time": 4.670194169506431} +{"Finetune/Learning Rate": 1.2721823396546655e-05, "Finetune/Loss": 0.8769389390945435, "Finetune/Loss (Raw)": 0.9749277234077454, "Finetune/Step": 2234, "Finetune/Step Time": 4.673343690112233} +{"Finetune/Learning Rate": 1.2715828452767833e-05, "Finetune/Loss": 0.8777236938476562, "Finetune/Loss (Raw)": 1.0297249555587769, "Finetune/Step": 2235, "Finetune/Step Time": 4.689584337174892} +{"Finetune/Learning Rate": 1.2709832455029017e-05, "Finetune/Loss": 0.8786424398422241, "Finetune/Loss (Raw)": 0.9983319044113159, "Finetune/Step": 2236, "Finetune/Step Time": 4.6900106780231} +{"Finetune/Learning Rate": 1.2703835405657124e-05, "Finetune/Loss": 0.878234326839447, "Finetune/Loss (Raw)": 0.8914059996604919, "Finetune/Step": 2237, "Finetune/Step Time": 4.690859528258443} +{"Finetune/Learning Rate": 1.2697837306979499e-05, "Finetune/Loss": 0.8767139315605164, "Finetune/Loss (Raw)": 0.7632394433021545, "Finetune/Step": 2238, "Finetune/Step Time": 4.695154368877411} +{"Finetune/Learning Rate": 1.2691838161323888e-05, "Finetune/Loss": 0.8786027431488037, "Finetune/Loss (Raw)": 1.1621098518371582, "Finetune/Step": 2239, "Finetune/Step Time": 4.6810240391641855} +{"Finetune/Learning Rate": 1.2685837971018438e-05, "Finetune/Loss": 0.8833211660385132, "Finetune/Loss (Raw)": 1.063271403312683, "Finetune/Step": 2240, "Finetune/Step Time": 4.644468039274216} +{"Finetune/Learning Rate": 1.2679836738391714e-05, "Finetune/Loss": 0.883779764175415, "Finetune/Loss (Raw)": 0.9952255487442017, "Finetune/Step": 2241, "Finetune/Step Time": 4.652725948020816} +{"Finetune/Learning Rate": 1.2673834465772662e-05, "Finetune/Loss": 0.8821340203285217, "Finetune/Loss (Raw)": 0.8888370990753174, "Finetune/Step": 2242, "Finetune/Step Time": 4.630441566929221} +{"Finetune/Learning Rate": 1.2667831155490662e-05, "Finetune/Loss": 0.8812791705131531, "Finetune/Loss (Raw)": 0.8913879990577698, "Finetune/Step": 2243, "Finetune/Step Time": 4.632978409528732} +{"Finetune/Learning Rate": 1.2661826809875474e-05, "Finetune/Loss": 0.8870049715042114, "Finetune/Loss (Raw)": 1.156170129776001, "Finetune/Step": 2244, "Finetune/Step Time": 4.591949511319399} +{"Finetune/Learning Rate": 1.265582143125727e-05, "Finetune/Loss": 0.8886315822601318, "Finetune/Loss (Raw)": 0.9690102338790894, "Finetune/Step": 2245, "Finetune/Step Time": 4.5722961612045765} +{"Finetune/Learning Rate": 1.264981502196662e-05, "Finetune/Loss": 0.8903106451034546, "Finetune/Loss (Raw)": 1.0773519277572632, "Finetune/Step": 2246, "Finetune/Step Time": 4.575336018577218} +{"Finetune/Learning Rate": 1.2643807584334491e-05, "Finetune/Loss": 0.8902302384376526, "Finetune/Loss (Raw)": 0.8556094169616699, "Finetune/Step": 2247, "Finetune/Step Time": 4.579096302390099} +{"Finetune/Learning Rate": 1.2637799120692253e-05, "Finetune/Loss": 0.8936184644699097, "Finetune/Loss (Raw)": 0.7705823183059692, "Finetune/Step": 2248, "Finetune/Step Time": 4.536063468083739} +{"Finetune/Learning Rate": 1.2631789633371678e-05, "Finetune/Loss": 0.8929660320281982, "Finetune/Loss (Raw)": 0.8548082709312439, "Finetune/Step": 2249, "Finetune/Step Time": 4.535760700702667} +{"Finetune/Learning Rate": 1.2625779124704925e-05, "Finetune/Loss": 0.8925725221633911, "Finetune/Loss (Raw)": 0.9466143846511841, "Finetune/Step": 2250, "Finetune/Step Time": 4.556856770068407} +{"Finetune/Learning Rate": 1.2619767597024561e-05, "Finetune/Loss": 0.8921881318092346, "Finetune/Loss (Raw)": 0.9458056092262268, "Finetune/Step": 2251, "Finetune/Step Time": 4.558253787457943} +{"Finetune/Learning Rate": 1.2613755052663536e-05, "Finetune/Loss": 0.8869926929473877, "Finetune/Loss (Raw)": 0.32200026512145996, "Finetune/Step": 2252, "Finetune/Step Time": 4.593120696023107} +{"Finetune/Learning Rate": 1.2607741493955208e-05, "Finetune/Loss": 0.8873019218444824, "Finetune/Loss (Raw)": 1.0289040803909302, "Finetune/Step": 2253, "Finetune/Step Time": 4.574541164562106} +{"Finetune/Learning Rate": 1.2601726923233315e-05, "Finetune/Loss": 0.8875464797019958, "Finetune/Loss (Raw)": 1.0408281087875366, "Finetune/Step": 2254, "Finetune/Step Time": 4.580946557223797} +{"Finetune/Learning Rate": 1.2595711342831998e-05, "Finetune/Loss": 0.8877713084220886, "Finetune/Loss (Raw)": 0.9559865593910217, "Finetune/Step": 2255, "Finetune/Step Time": 4.57905494235456} +{"Finetune/Learning Rate": 1.2589694755085789e-05, "Finetune/Loss": 0.891951322555542, "Finetune/Loss (Raw)": 0.8908717632293701, "Finetune/Step": 2256, "Finetune/Step Time": 4.53576947376132} +{"Finetune/Learning Rate": 1.25836771623296e-05, "Finetune/Loss": 0.8973941802978516, "Finetune/Loss (Raw)": 0.9772282838821411, "Finetune/Step": 2257, "Finetune/Step Time": 4.4948344230651855} +{"Finetune/Learning Rate": 1.257765856689875e-05, "Finetune/Loss": 0.89571213722229, "Finetune/Loss (Raw)": 0.7451244592666626, "Finetune/Step": 2258, "Finetune/Step Time": 4.4951609540730715} +{"Finetune/Learning Rate": 1.2571638971128933e-05, "Finetune/Loss": 0.8946242332458496, "Finetune/Loss (Raw)": 0.8371888399124146, "Finetune/Step": 2259, "Finetune/Step Time": 4.492535624653101} +{"Finetune/Learning Rate": 1.2565618377356237e-05, "Finetune/Loss": 0.8944990038871765, "Finetune/Loss (Raw)": 0.8553299307823181, "Finetune/Step": 2260, "Finetune/Step Time": 4.503704918548465} +{"Finetune/Learning Rate": 1.2559596787917144e-05, "Finetune/Loss": 0.8930513858795166, "Finetune/Loss (Raw)": 0.9592376947402954, "Finetune/Step": 2261, "Finetune/Step Time": 4.506022663787007} +{"Finetune/Learning Rate": 1.2553574205148502e-05, "Finetune/Loss": 0.8944401741027832, "Finetune/Loss (Raw)": 1.0487912893295288, "Finetune/Step": 2262, "Finetune/Step Time": 4.49995044991374} +{"Finetune/Learning Rate": 1.2547550631387571e-05, "Finetune/Loss": 0.8942581415176392, "Finetune/Loss (Raw)": 0.8418851494789124, "Finetune/Step": 2263, "Finetune/Step Time": 4.504290236160159} +{"Finetune/Learning Rate": 1.2541526068971975e-05, "Finetune/Loss": 0.8942825794219971, "Finetune/Loss (Raw)": 0.9588944315910339, "Finetune/Step": 2264, "Finetune/Step Time": 4.507341763004661} +{"Finetune/Learning Rate": 1.2535500520239727e-05, "Finetune/Loss": 0.898017406463623, "Finetune/Loss (Raw)": 0.9026457071304321, "Finetune/Step": 2265, "Finetune/Step Time": 4.469890071079135} +{"Finetune/Learning Rate": 1.2529473987529235e-05, "Finetune/Loss": 0.8968308568000793, "Finetune/Loss (Raw)": 0.7661752104759216, "Finetune/Step": 2266, "Finetune/Step Time": 4.476652201265097} +{"Finetune/Learning Rate": 1.2523446473179267e-05, "Finetune/Loss": 0.8969351053237915, "Finetune/Loss (Raw)": 0.8799794912338257, "Finetune/Step": 2267, "Finetune/Step Time": 4.456931717693806} +{"Finetune/Learning Rate": 1.2517417979528991e-05, "Finetune/Loss": 0.8959506154060364, "Finetune/Loss (Raw)": 0.8389599919319153, "Finetune/Step": 2268, "Finetune/Step Time": 4.441931126639247} +{"Finetune/Learning Rate": 1.2511388508917946e-05, "Finetune/Loss": 0.8949402570724487, "Finetune/Loss (Raw)": 0.9382578134536743, "Finetune/Step": 2269, "Finetune/Step Time": 4.440878873690963} +{"Finetune/Learning Rate": 1.250535806368605e-05, "Finetune/Loss": 0.8943573236465454, "Finetune/Loss (Raw)": 0.7825865745544434, "Finetune/Step": 2270, "Finetune/Step Time": 4.449405701830983} +{"Finetune/Learning Rate": 1.2499326646173606e-05, "Finetune/Loss": 0.8946285247802734, "Finetune/Loss (Raw)": 0.9584802389144897, "Finetune/Step": 2271, "Finetune/Step Time": 4.429316647350788} +{"Finetune/Learning Rate": 1.2493294258721282e-05, "Finetune/Loss": 0.8952574729919434, "Finetune/Loss (Raw)": 0.9707221388816833, "Finetune/Step": 2272, "Finetune/Step Time": 4.423507647588849} +{"Finetune/Learning Rate": 1.2487260903670136e-05, "Finetune/Loss": 0.8970459699630737, "Finetune/Loss (Raw)": 0.9214991927146912, "Finetune/Step": 2273, "Finetune/Step Time": 4.408828351646662} +{"Finetune/Learning Rate": 1.2481226583361592e-05, "Finetune/Loss": 0.8963667154312134, "Finetune/Loss (Raw)": 0.8300523161888123, "Finetune/Step": 2274, "Finetune/Step Time": 4.419739615172148} +{"Finetune/Learning Rate": 1.2475191300137452e-05, "Finetune/Loss": 0.8970979452133179, "Finetune/Loss (Raw)": 1.0285037755966187, "Finetune/Step": 2275, "Finetune/Step Time": 4.412044752389193} +{"Finetune/Learning Rate": 1.2469155056339894e-05, "Finetune/Loss": 0.8960389494895935, "Finetune/Loss (Raw)": 0.9282520413398743, "Finetune/Step": 2276, "Finetune/Step Time": 4.408425554633141} +{"Finetune/Learning Rate": 1.2463117854311463e-05, "Finetune/Loss": 0.8941250443458557, "Finetune/Loss (Raw)": 0.7472086548805237, "Finetune/Step": 2277, "Finetune/Step Time": 4.413448525592685} +{"Finetune/Learning Rate": 1.2457079696395082e-05, "Finetune/Loss": 0.8945547342300415, "Finetune/Loss (Raw)": 0.9731506705284119, "Finetune/Step": 2278, "Finetune/Step Time": 4.417225170880556} +{"Finetune/Learning Rate": 1.2451040584934042e-05, "Finetune/Loss": 0.8937318921089172, "Finetune/Loss (Raw)": 1.0259723663330078, "Finetune/Step": 2279, "Finetune/Step Time": 4.406173629686236} +{"Finetune/Learning Rate": 1.2445000522272005e-05, "Finetune/Loss": 0.8943663835525513, "Finetune/Loss (Raw)": 0.891079843044281, "Finetune/Step": 2280, "Finetune/Step Time": 4.401112413033843} +{"Finetune/Learning Rate": 1.2438959510752997e-05, "Finetune/Loss": 0.89470374584198, "Finetune/Loss (Raw)": 0.8984756469726562, "Finetune/Step": 2281, "Finetune/Step Time": 4.402547599747777} +{"Finetune/Learning Rate": 1.243291755272142e-05, "Finetune/Loss": 0.8949248790740967, "Finetune/Loss (Raw)": 0.8974833488464355, "Finetune/Step": 2282, "Finetune/Step Time": 4.396189516410232} +{"Finetune/Learning Rate": 1.2426874650522039e-05, "Finetune/Loss": 0.8958337306976318, "Finetune/Loss (Raw)": 1.0301092863082886, "Finetune/Step": 2283, "Finetune/Step Time": 4.383739585056901} +{"Finetune/Learning Rate": 1.2420830806499988e-05, "Finetune/Loss": 0.8966725468635559, "Finetune/Loss (Raw)": 0.874032735824585, "Finetune/Step": 2284, "Finetune/Step Time": 4.403385894373059} +{"Finetune/Learning Rate": 1.2414786023000765e-05, "Finetune/Loss": 0.897231936454773, "Finetune/Loss (Raw)": 0.9476718306541443, "Finetune/Step": 2285, "Finetune/Step Time": 4.403190681710839} +{"Finetune/Learning Rate": 1.240874030237023e-05, "Finetune/Loss": 0.8971070051193237, "Finetune/Loss (Raw)": 0.9547805786132812, "Finetune/Step": 2286, "Finetune/Step Time": 4.413556035608053} +{"Finetune/Learning Rate": 1.2402693646954607e-05, "Finetune/Loss": 0.8954595327377319, "Finetune/Loss (Raw)": 0.7675698399543762, "Finetune/Step": 2287, "Finetune/Step Time": 4.398443648591638} +{"Finetune/Learning Rate": 1.2396646059100492e-05, "Finetune/Loss": 0.8955419063568115, "Finetune/Loss (Raw)": 0.9438729882240295, "Finetune/Step": 2288, "Finetune/Step Time": 4.402288343757391} +{"Finetune/Learning Rate": 1.2390597541154832e-05, "Finetune/Loss": 0.8945850133895874, "Finetune/Loss (Raw)": 0.7394604682922363, "Finetune/Step": 2289, "Finetune/Step Time": 4.409914640709758} +{"Finetune/Learning Rate": 1.238454809546494e-05, "Finetune/Loss": 0.8953347206115723, "Finetune/Loss (Raw)": 0.9997062087059021, "Finetune/Step": 2290, "Finetune/Step Time": 4.402383612468839} +{"Finetune/Learning Rate": 1.2378497724378484e-05, "Finetune/Loss": 0.8949427008628845, "Finetune/Loss (Raw)": 0.8329271078109741, "Finetune/Step": 2291, "Finetune/Step Time": 4.409740161150694} +{"Finetune/Learning Rate": 1.2372446430243497e-05, "Finetune/Loss": 0.8934819102287292, "Finetune/Loss (Raw)": 0.8518643975257874, "Finetune/Step": 2292, "Finetune/Step Time": 4.418347481638193} +{"Finetune/Learning Rate": 1.2366394215408369e-05, "Finetune/Loss": 0.8919587731361389, "Finetune/Loss (Raw)": 0.7345222234725952, "Finetune/Step": 2293, "Finetune/Step Time": 4.403019089251757} +{"Finetune/Learning Rate": 1.2360341082221847e-05, "Finetune/Loss": 0.8918458223342896, "Finetune/Loss (Raw)": 0.9454383850097656, "Finetune/Step": 2294, "Finetune/Step Time": 4.395652895793319} +{"Finetune/Learning Rate": 1.2354287033033032e-05, "Finetune/Loss": 0.8908085227012634, "Finetune/Loss (Raw)": 0.9276859164237976, "Finetune/Step": 2295, "Finetune/Step Time": 4.418872073292732} +{"Finetune/Learning Rate": 1.2348232070191383e-05, "Finetune/Loss": 0.8895803689956665, "Finetune/Loss (Raw)": 0.760421097278595, "Finetune/Step": 2296, "Finetune/Step Time": 4.41904266923666} +{"Finetune/Learning Rate": 1.2342176196046715e-05, "Finetune/Loss": 0.8893729448318481, "Finetune/Loss (Raw)": 0.9294801354408264, "Finetune/Step": 2297, "Finetune/Step Time": 4.427148129791021} +{"Finetune/Learning Rate": 1.2336119412949193e-05, "Finetune/Loss": 0.8896353244781494, "Finetune/Loss (Raw)": 0.8658952713012695, "Finetune/Step": 2298, "Finetune/Step Time": 4.428454035893083} +{"Finetune/Learning Rate": 1.2330061723249328e-05, "Finetune/Loss": 0.8900904059410095, "Finetune/Loss (Raw)": 0.9719187021255493, "Finetune/Step": 2299, "Finetune/Step Time": 4.426367154344916} +{"Finetune/Learning Rate": 1.2324003129298006e-05, "Finetune/Loss": 0.8906469345092773, "Finetune/Loss (Raw)": 0.9700312614440918, "Finetune/Step": 2300, "Finetune/Step Time": 4.432539023458958} +{"Finetune/Learning Rate": 1.2317943633446441e-05, "Finetune/Loss": 0.8900325894355774, "Finetune/Loss (Raw)": 0.8272297382354736, "Finetune/Step": 2301, "Finetune/Step Time": 4.424274448305368} +{"Finetune/Learning Rate": 1.2311883238046204e-05, "Finetune/Loss": 0.8903459310531616, "Finetune/Loss (Raw)": 0.8042263984680176, "Finetune/Step": 2302, "Finetune/Step Time": 4.416666222736239} +{"Finetune/Learning Rate": 1.2305821945449222e-05, "Finetune/Loss": 0.8903549313545227, "Finetune/Loss (Raw)": 0.9195760488510132, "Finetune/Step": 2303, "Finetune/Step Time": 4.398916635662317} +{"Finetune/Learning Rate": 1.2299759758007755e-05, "Finetune/Loss": 0.8882661461830139, "Finetune/Loss (Raw)": 0.7168306708335876, "Finetune/Step": 2304, "Finetune/Step Time": 4.407391099259257} +{"Finetune/Learning Rate": 1.2293696678074428e-05, "Finetune/Loss": 0.8862691521644592, "Finetune/Loss (Raw)": 0.8063746690750122, "Finetune/Step": 2305, "Finetune/Step Time": 4.402658734470606} +{"Finetune/Learning Rate": 1.22876327080022e-05, "Finetune/Loss": 0.8882421255111694, "Finetune/Loss (Raw)": 1.0454591512680054, "Finetune/Step": 2306, "Finetune/Step Time": 4.40488263592124} +{"Finetune/Learning Rate": 1.2281567850144383e-05, "Finetune/Loss": 0.8884304761886597, "Finetune/Loss (Raw)": 0.968014121055603, "Finetune/Step": 2307, "Finetune/Step Time": 4.389233296737075} +{"Finetune/Learning Rate": 1.2275502106854624e-05, "Finetune/Loss": 0.8856619596481323, "Finetune/Loss (Raw)": 0.6134926080703735, "Finetune/Step": 2308, "Finetune/Step Time": 4.391754347831011} +{"Finetune/Learning Rate": 1.2269435480486923e-05, "Finetune/Loss": 0.8853975534439087, "Finetune/Loss (Raw)": 0.9317768216133118, "Finetune/Step": 2309, "Finetune/Step Time": 4.391893554478884} +{"Finetune/Learning Rate": 1.226336797339562e-05, "Finetune/Loss": 0.8851116895675659, "Finetune/Loss (Raw)": 0.951133668422699, "Finetune/Step": 2310, "Finetune/Step Time": 4.3819171320647} +{"Finetune/Learning Rate": 1.2257299587935394e-05, "Finetune/Loss": 0.8870585560798645, "Finetune/Loss (Raw)": 1.0332889556884766, "Finetune/Step": 2311, "Finetune/Step Time": 4.362121777608991} +{"Finetune/Learning Rate": 1.2251230326461268e-05, "Finetune/Loss": 0.8875622153282166, "Finetune/Loss (Raw)": 0.8652682304382324, "Finetune/Step": 2312, "Finetune/Step Time": 4.37631137855351} +{"Finetune/Learning Rate": 1.2245160191328605e-05, "Finetune/Loss": 0.8876407146453857, "Finetune/Loss (Raw)": 0.937015950679779, "Finetune/Step": 2313, "Finetune/Step Time": 4.381338147446513} +{"Finetune/Learning Rate": 1.2239089184893102e-05, "Finetune/Loss": 0.8873833417892456, "Finetune/Loss (Raw)": 0.8599061965942383, "Finetune/Step": 2314, "Finetune/Step Time": 4.390100043267012} +{"Finetune/Learning Rate": 1.2233017309510804e-05, "Finetune/Loss": 0.8882595300674438, "Finetune/Loss (Raw)": 0.8556059002876282, "Finetune/Step": 2315, "Finetune/Step Time": 4.356400325894356} +{"Finetune/Learning Rate": 1.2226944567538084e-05, "Finetune/Loss": 0.8887314796447754, "Finetune/Loss (Raw)": 0.9732712507247925, "Finetune/Step": 2316, "Finetune/Step Time": 4.321521373465657} +{"Finetune/Learning Rate": 1.2220870961331654e-05, "Finetune/Loss": 0.8888989686965942, "Finetune/Loss (Raw)": 0.8562125563621521, "Finetune/Step": 2317, "Finetune/Step Time": 4.318123755976558} +{"Finetune/Learning Rate": 1.2214796493248564e-05, "Finetune/Loss": 0.889135479927063, "Finetune/Loss (Raw)": 0.8749666213989258, "Finetune/Step": 2318, "Finetune/Step Time": 4.306500798091292} +{"Finetune/Learning Rate": 1.2208721165646193e-05, "Finetune/Loss": 0.8901011347770691, "Finetune/Loss (Raw)": 1.0766605138778687, "Finetune/Step": 2319, "Finetune/Step Time": 4.297683294862509} +{"Finetune/Learning Rate": 1.2202644980882258e-05, "Finetune/Loss": 0.8912336230278015, "Finetune/Loss (Raw)": 0.9380203485488892, "Finetune/Step": 2320, "Finetune/Step Time": 4.296218030154705} +{"Finetune/Learning Rate": 1.2196567941314815e-05, "Finetune/Loss": 0.8907507658004761, "Finetune/Loss (Raw)": 0.9264432191848755, "Finetune/Step": 2321, "Finetune/Step Time": 4.303153570741415} +{"Finetune/Learning Rate": 1.2190490049302233e-05, "Finetune/Loss": 0.8908241391181946, "Finetune/Loss (Raw)": 0.9735398888587952, "Finetune/Step": 2322, "Finetune/Step Time": 4.303768187761307} +{"Finetune/Learning Rate": 1.218441130720323e-05, "Finetune/Loss": 0.8892865180969238, "Finetune/Loss (Raw)": 0.7954138517379761, "Finetune/Step": 2323, "Finetune/Step Time": 4.318022090941668} +{"Finetune/Learning Rate": 1.2178331717376848e-05, "Finetune/Loss": 0.8892393112182617, "Finetune/Loss (Raw)": 0.8514865040779114, "Finetune/Step": 2324, "Finetune/Step Time": 4.302982183173299} +{"Finetune/Learning Rate": 1.217225128218245e-05, "Finetune/Loss": 0.8893340229988098, "Finetune/Loss (Raw)": 1.0132118463516235, "Finetune/Step": 2325, "Finetune/Step Time": 4.310043280944228} +{"Finetune/Learning Rate": 1.2166170003979742e-05, "Finetune/Loss": 0.8859140872955322, "Finetune/Loss (Raw)": 0.4386306405067444, "Finetune/Step": 2326, "Finetune/Step Time": 4.342540051788092} +{"Finetune/Learning Rate": 1.2160087885128745e-05, "Finetune/Loss": 0.8864208459854126, "Finetune/Loss (Raw)": 0.9093640446662903, "Finetune/Step": 2327, "Finetune/Step Time": 4.363606503233314} +{"Finetune/Learning Rate": 1.2154004927989815e-05, "Finetune/Loss": 0.8861784934997559, "Finetune/Loss (Raw)": 0.8511031866073608, "Finetune/Step": 2328, "Finetune/Step Time": 4.357408074662089} +{"Finetune/Learning Rate": 1.2147921134923624e-05, "Finetune/Loss": 0.8910242319107056, "Finetune/Loss (Raw)": 0.9780390858650208, "Finetune/Step": 2329, "Finetune/Step Time": 4.320362294092774} +{"Finetune/Learning Rate": 1.2141836508291175e-05, "Finetune/Loss": 0.8901663422584534, "Finetune/Loss (Raw)": 0.8045121431350708, "Finetune/Step": 2330, "Finetune/Step Time": 4.332987552508712} +{"Finetune/Learning Rate": 1.2135751050453793e-05, "Finetune/Loss": 0.8906568288803101, "Finetune/Loss (Raw)": 0.9347082376480103, "Finetune/Step": 2331, "Finetune/Step Time": 4.340921217575669} +{"Finetune/Learning Rate": 1.2129664763773125e-05, "Finetune/Loss": 0.8927200436592102, "Finetune/Loss (Raw)": 1.1086584329605103, "Finetune/Step": 2332, "Finetune/Step Time": 4.33713074028492} +{"Finetune/Learning Rate": 1.212357765061114e-05, "Finetune/Loss": 0.897132396697998, "Finetune/Loss (Raw)": 0.9922702312469482, "Finetune/Step": 2333, "Finetune/Step Time": 4.315664675086737} +{"Finetune/Learning Rate": 1.211748971333013e-05, "Finetune/Loss": 0.899208664894104, "Finetune/Loss (Raw)": 0.9377709031105042, "Finetune/Step": 2334, "Finetune/Step Time": 4.274988146498799} +{"Finetune/Learning Rate": 1.21114009542927e-05, "Finetune/Loss": 0.8985079526901245, "Finetune/Loss (Raw)": 0.8725890517234802, "Finetune/Step": 2335, "Finetune/Step Time": 4.277420239523053} +{"Finetune/Learning Rate": 1.2105311375861786e-05, "Finetune/Loss": 0.8987149596214294, "Finetune/Loss (Raw)": 0.8904992938041687, "Finetune/Step": 2336, "Finetune/Step Time": 4.270930925384164} +{"Finetune/Learning Rate": 1.2099220980400625e-05, "Finetune/Loss": 0.8992952108383179, "Finetune/Loss (Raw)": 0.9665631651878357, "Finetune/Step": 2337, "Finetune/Step Time": 4.270184252411127} +{"Finetune/Learning Rate": 1.2093129770272787e-05, "Finetune/Loss": 0.9038323760032654, "Finetune/Loss (Raw)": 0.9749013781547546, "Finetune/Step": 2338, "Finetune/Step Time": 4.23175473511219} +{"Finetune/Learning Rate": 1.208703774784215e-05, "Finetune/Loss": 0.9024928212165833, "Finetune/Loss (Raw)": 0.8304280638694763, "Finetune/Step": 2339, "Finetune/Step Time": 4.226966839283705} +{"Finetune/Learning Rate": 1.2080944915472908e-05, "Finetune/Loss": 0.9039977788925171, "Finetune/Loss (Raw)": 0.9078218340873718, "Finetune/Step": 2340, "Finetune/Step Time": 4.219170931726694} +{"Finetune/Learning Rate": 1.2074851275529575e-05, "Finetune/Loss": 0.9057304859161377, "Finetune/Loss (Raw)": 1.0340739488601685, "Finetune/Step": 2341, "Finetune/Step Time": 4.246229067444801} +{"Finetune/Learning Rate": 1.2068756830376966e-05, "Finetune/Loss": 0.9029723405838013, "Finetune/Loss (Raw)": 0.3570152521133423, "Finetune/Step": 2342, "Finetune/Step Time": 4.270848723128438} +{"Finetune/Learning Rate": 1.2062661582380222e-05, "Finetune/Loss": 0.9031236171722412, "Finetune/Loss (Raw)": 0.9889431595802307, "Finetune/Step": 2343, "Finetune/Step Time": 4.2886198572814465} +{"Finetune/Learning Rate": 1.2056565533904787e-05, "Finetune/Loss": 0.902867317199707, "Finetune/Loss (Raw)": 0.9123846888542175, "Finetune/Step": 2344, "Finetune/Step Time": 4.316694255918264} +{"Finetune/Learning Rate": 1.2050468687316418e-05, "Finetune/Loss": 0.9039198160171509, "Finetune/Loss (Raw)": 1.1448408365249634, "Finetune/Step": 2345, "Finetune/Step Time": 4.316906979307532} +{"Finetune/Learning Rate": 1.2044371044981188e-05, "Finetune/Loss": 0.9052009582519531, "Finetune/Loss (Raw)": 0.9015517830848694, "Finetune/Step": 2346, "Finetune/Step Time": 4.312655022367835} +{"Finetune/Learning Rate": 1.2038272609265464e-05, "Finetune/Loss": 0.9055395126342773, "Finetune/Loss (Raw)": 0.8962706327438354, "Finetune/Step": 2347, "Finetune/Step Time": 4.281191734597087} +{"Finetune/Learning Rate": 1.2032173382535936e-05, "Finetune/Loss": 0.9055954813957214, "Finetune/Loss (Raw)": 0.9958376288414001, "Finetune/Step": 2348, "Finetune/Step Time": 4.283246491104364} +{"Finetune/Learning Rate": 1.2026073367159597e-05, "Finetune/Loss": 0.9062663316726685, "Finetune/Loss (Raw)": 0.925321102142334, "Finetune/Step": 2349, "Finetune/Step Time": 4.274630958214402} +{"Finetune/Learning Rate": 1.2019972565503733e-05, "Finetune/Loss": 0.9058776497840881, "Finetune/Loss (Raw)": 0.9161814451217651, "Finetune/Step": 2350, "Finetune/Step Time": 4.268206179141998} +{"Finetune/Learning Rate": 1.2013870979935957e-05, "Finetune/Loss": 0.9065585732460022, "Finetune/Loss (Raw)": 1.0263556241989136, "Finetune/Step": 2351, "Finetune/Step Time": 4.2650894112885} +{"Finetune/Learning Rate": 1.200776861282417e-05, "Finetune/Loss": 0.9077616333961487, "Finetune/Loss (Raw)": 1.0657997131347656, "Finetune/Step": 2352, "Finetune/Step Time": 4.273504225537181} +{"Finetune/Learning Rate": 1.2001665466536582e-05, "Finetune/Loss": 0.9079985022544861, "Finetune/Loss (Raw)": 0.9165998697280884, "Finetune/Step": 2353, "Finetune/Step Time": 4.267867896705866} +{"Finetune/Learning Rate": 1.199556154344171e-05, "Finetune/Loss": 0.9078718423843384, "Finetune/Loss (Raw)": 0.9161982536315918, "Finetune/Step": 2354, "Finetune/Step Time": 4.260174050927162} +{"Finetune/Learning Rate": 1.1989456845908356e-05, "Finetune/Loss": 0.9065850973129272, "Finetune/Loss (Raw)": 0.8187066316604614, "Finetune/Step": 2355, "Finetune/Step Time": 4.257094727829099} +{"Finetune/Learning Rate": 1.1983351376305648e-05, "Finetune/Loss": 0.9058185815811157, "Finetune/Loss (Raw)": 0.8030906319618225, "Finetune/Step": 2356, "Finetune/Step Time": 4.2573986276984215} +{"Finetune/Learning Rate": 1.197724513700299e-05, "Finetune/Loss": 0.9057390689849854, "Finetune/Loss (Raw)": 0.9024701118469238, "Finetune/Step": 2357, "Finetune/Step Time": 4.256278434768319} +{"Finetune/Learning Rate": 1.1971138130370097e-05, "Finetune/Loss": 0.9069803953170776, "Finetune/Loss (Raw)": 1.0188757181167603, "Finetune/Step": 2358, "Finetune/Step Time": 4.253344066441059} +{"Finetune/Learning Rate": 1.1965030358776975e-05, "Finetune/Loss": 0.9015653133392334, "Finetune/Loss (Raw)": 0.2936522364616394, "Finetune/Step": 2359, "Finetune/Step Time": 4.29112240485847} +{"Finetune/Learning Rate": 1.1958921824593936e-05, "Finetune/Loss": 0.9006617665290833, "Finetune/Loss (Raw)": 0.9154176115989685, "Finetune/Step": 2360, "Finetune/Step Time": 4.289776276797056} +{"Finetune/Learning Rate": 1.1952812530191586e-05, "Finetune/Loss": 0.9018237590789795, "Finetune/Loss (Raw)": 0.948015034198761, "Finetune/Step": 2361, "Finetune/Step Time": 4.2895355969667435} +{"Finetune/Learning Rate": 1.1946702477940815e-05, "Finetune/Loss": 0.9014990329742432, "Finetune/Loss (Raw)": 0.933362603187561, "Finetune/Step": 2362, "Finetune/Step Time": 4.282419003546238} +{"Finetune/Learning Rate": 1.194059167021282e-05, "Finetune/Loss": 0.8990233540534973, "Finetune/Loss (Raw)": 0.7128316760063171, "Finetune/Step": 2363, "Finetune/Step Time": 4.283083835616708} +{"Finetune/Learning Rate": 1.1934480109379083e-05, "Finetune/Loss": 0.8978062868118286, "Finetune/Loss (Raw)": 0.8425542712211609, "Finetune/Step": 2364, "Finetune/Step Time": 4.2806723564863205} +{"Finetune/Learning Rate": 1.1928367797811384e-05, "Finetune/Loss": 0.8988442420959473, "Finetune/Loss (Raw)": 1.024256706237793, "Finetune/Step": 2365, "Finetune/Step Time": 4.276274966076016} +{"Finetune/Learning Rate": 1.1922254737881792e-05, "Finetune/Loss": 0.8989209532737732, "Finetune/Loss (Raw)": 0.7730653882026672, "Finetune/Step": 2366, "Finetune/Step Time": 4.277909325435758} +{"Finetune/Learning Rate": 1.1916140931962661e-05, "Finetune/Loss": 0.8974111676216125, "Finetune/Loss (Raw)": 0.9688527584075928, "Finetune/Step": 2367, "Finetune/Step Time": 4.28359785117209} +{"Finetune/Learning Rate": 1.191002638242665e-05, "Finetune/Loss": 0.8960950970649719, "Finetune/Loss (Raw)": 0.8948146104812622, "Finetune/Step": 2368, "Finetune/Step Time": 4.279352996498346} +{"Finetune/Learning Rate": 1.1903911091646684e-05, "Finetune/Loss": 0.8961480855941772, "Finetune/Loss (Raw)": 1.0020126104354858, "Finetune/Step": 2369, "Finetune/Step Time": 4.276285383850336} +{"Finetune/Learning Rate": 1.1897795061995998e-05, "Finetune/Loss": 0.8963145613670349, "Finetune/Loss (Raw)": 0.9101414084434509, "Finetune/Step": 2370, "Finetune/Step Time": 4.278380736708641} +{"Finetune/Learning Rate": 1.18916782958481e-05, "Finetune/Loss": 0.8972038626670837, "Finetune/Loss (Raw)": 1.0052200555801392, "Finetune/Step": 2371, "Finetune/Step Time": 4.277541574090719} +{"Finetune/Learning Rate": 1.1885560795576782e-05, "Finetune/Loss": 0.8950866460800171, "Finetune/Loss (Raw)": 0.8851662874221802, "Finetune/Step": 2372, "Finetune/Step Time": 4.280107626691461} +{"Finetune/Learning Rate": 1.1879442563556141e-05, "Finetune/Loss": 0.8953743577003479, "Finetune/Loss (Raw)": 1.0058331489562988, "Finetune/Step": 2373, "Finetune/Step Time": 4.302143333479762} +{"Finetune/Learning Rate": 1.1873323602160526e-05, "Finetune/Loss": 0.8933417797088623, "Finetune/Loss (Raw)": 0.8171907067298889, "Finetune/Step": 2374, "Finetune/Step Time": 4.303506258875132} +{"Finetune/Learning Rate": 1.1867203913764599e-05, "Finetune/Loss": 0.8926340937614441, "Finetune/Loss (Raw)": 0.7650178670883179, "Finetune/Step": 2375, "Finetune/Step Time": 4.301603842526674} +{"Finetune/Learning Rate": 1.1861083500743284e-05, "Finetune/Loss": 0.8940732479095459, "Finetune/Loss (Raw)": 0.9547950625419617, "Finetune/Step": 2376, "Finetune/Step Time": 4.312013369053602} +{"Finetune/Learning Rate": 1.1854962365471798e-05, "Finetune/Loss": 0.8957289457321167, "Finetune/Loss (Raw)": 1.0667344331741333, "Finetune/Step": 2377, "Finetune/Step Time": 4.312559302896261} +{"Finetune/Learning Rate": 1.1848840510325635e-05, "Finetune/Loss": 0.8961742520332336, "Finetune/Loss (Raw)": 1.0036166906356812, "Finetune/Step": 2378, "Finetune/Step Time": 4.291020283475518} +{"Finetune/Learning Rate": 1.184271793768056e-05, "Finetune/Loss": 0.8961183428764343, "Finetune/Loss (Raw)": 0.9386528134346008, "Finetune/Step": 2379, "Finetune/Step Time": 4.308138590306044} +{"Finetune/Learning Rate": 1.1836594649912636e-05, "Finetune/Loss": 0.8997354507446289, "Finetune/Loss (Raw)": 0.7849856019020081, "Finetune/Step": 2380, "Finetune/Step Time": 4.267328867688775} +{"Finetune/Learning Rate": 1.1830470649398183e-05, "Finetune/Loss": 0.8988674879074097, "Finetune/Loss (Raw)": 0.9178020358085632, "Finetune/Step": 2381, "Finetune/Step Time": 4.270363328978419} +{"Finetune/Learning Rate": 1.1824345938513806e-05, "Finetune/Loss": 0.896805465221405, "Finetune/Loss (Raw)": 0.7768957614898682, "Finetune/Step": 2382, "Finetune/Step Time": 4.282163396477699} +{"Finetune/Learning Rate": 1.181822051963639e-05, "Finetune/Loss": 0.8946032524108887, "Finetune/Loss (Raw)": 0.6741032004356384, "Finetune/Step": 2383, "Finetune/Step Time": 4.295126684010029} +{"Finetune/Learning Rate": 1.1812094395143086e-05, "Finetune/Loss": 0.8961926698684692, "Finetune/Loss (Raw)": 1.0943107604980469, "Finetune/Step": 2384, "Finetune/Step Time": 4.2909097243100405} +{"Finetune/Learning Rate": 1.1805967567411325e-05, "Finetune/Loss": 0.8956480026245117, "Finetune/Loss (Raw)": 0.9075140953063965, "Finetune/Step": 2385, "Finetune/Step Time": 4.2960287015885115} +{"Finetune/Learning Rate": 1.1799840038818813e-05, "Finetune/Loss": 0.8978098034858704, "Finetune/Loss (Raw)": 1.0218340158462524, "Finetune/Step": 2386, "Finetune/Step Time": 4.302439454942942} +{"Finetune/Learning Rate": 1.1793711811743515e-05, "Finetune/Loss": 0.8980927467346191, "Finetune/Loss (Raw)": 0.8734058737754822, "Finetune/Step": 2387, "Finetune/Step Time": 4.316508941352367} +{"Finetune/Learning Rate": 1.1787582888563689e-05, "Finetune/Loss": 0.8990074396133423, "Finetune/Loss (Raw)": 0.9724076986312866, "Finetune/Step": 2388, "Finetune/Step Time": 4.320703264325857} +{"Finetune/Learning Rate": 1.1781453271657839e-05, "Finetune/Loss": 0.8995724320411682, "Finetune/Loss (Raw)": 1.0315643548965454, "Finetune/Step": 2389, "Finetune/Step Time": 4.3192657250911} +{"Finetune/Learning Rate": 1.1775322963404757e-05, "Finetune/Loss": 0.8994346857070923, "Finetune/Loss (Raw)": 1.031158447265625, "Finetune/Step": 2390, "Finetune/Step Time": 4.3283794317394495} +{"Finetune/Learning Rate": 1.176919196618349e-05, "Finetune/Loss": 0.9006006121635437, "Finetune/Loss (Raw)": 0.9911261796951294, "Finetune/Step": 2391, "Finetune/Step Time": 4.327621212229133} +{"Finetune/Learning Rate": 1.1763060282373364e-05, "Finetune/Loss": 0.9003793001174927, "Finetune/Loss (Raw)": 0.9305540919303894, "Finetune/Step": 2392, "Finetune/Step Time": 4.327860025689006} +{"Finetune/Learning Rate": 1.1756927914353963e-05, "Finetune/Loss": 0.9012829065322876, "Finetune/Loss (Raw)": 1.0183197259902954, "Finetune/Step": 2393, "Finetune/Step Time": 4.340754797682166} +{"Finetune/Learning Rate": 1.175079486450514e-05, "Finetune/Loss": 0.8984060287475586, "Finetune/Loss (Raw)": 0.3979245126247406, "Finetune/Step": 2394, "Finetune/Step Time": 4.374264383688569} +{"Finetune/Learning Rate": 1.1744661135207009e-05, "Finetune/Loss": 0.8988817930221558, "Finetune/Loss (Raw)": 0.9408844113349915, "Finetune/Step": 2395, "Finetune/Step Time": 4.378931263461709} +{"Finetune/Learning Rate": 1.1738526728839956e-05, "Finetune/Loss": 0.8991268873214722, "Finetune/Loss (Raw)": 0.8703311681747437, "Finetune/Step": 2396, "Finetune/Step Time": 4.376008868217468} +{"Finetune/Learning Rate": 1.1732391647784618e-05, "Finetune/Loss": 0.8991057872772217, "Finetune/Loss (Raw)": 0.9355603456497192, "Finetune/Step": 2397, "Finetune/Step Time": 4.3675707299262285} +{"Finetune/Learning Rate": 1.1726255894421908e-05, "Finetune/Loss": 0.90138179063797, "Finetune/Loss (Raw)": 1.0739116668701172, "Finetune/Step": 2398, "Finetune/Step Time": 4.359063632786274} +{"Finetune/Learning Rate": 1.1720119471132985e-05, "Finetune/Loss": 0.9015927314758301, "Finetune/Loss (Raw)": 0.9854779243469238, "Finetune/Step": 2399, "Finetune/Step Time": 4.365754123777151} +{"Finetune/Learning Rate": 1.1713982380299278e-05, "Finetune/Loss": 0.9004044532775879, "Finetune/Loss (Raw)": 0.8186240792274475, "Finetune/Step": 2400, "Finetune/Step Time": 4.357806988060474} +{"Finetune/Learning Rate": 1.1707844624302471e-05, "Finetune/Loss": 0.9005604386329651, "Finetune/Loss (Raw)": 0.9414680004119873, "Finetune/Step": 2401, "Finetune/Step Time": 4.361813819035888} +{"Finetune/Learning Rate": 1.170170620552451e-05, "Finetune/Loss": 0.9004733562469482, "Finetune/Loss (Raw)": 0.8189066052436829, "Finetune/Step": 2402, "Finetune/Step Time": 4.346515495330095} +{"Finetune/Learning Rate": 1.1695567126347593e-05, "Finetune/Loss": 0.899707555770874, "Finetune/Loss (Raw)": 0.9304757118225098, "Finetune/Step": 2403, "Finetune/Step Time": 4.355330355465412} +{"Finetune/Learning Rate": 1.1689427389154176e-05, "Finetune/Loss": 0.8997851610183716, "Finetune/Loss (Raw)": 0.9381898641586304, "Finetune/Step": 2404, "Finetune/Step Time": 4.3542244248092175} +{"Finetune/Learning Rate": 1.1683286996326975e-05, "Finetune/Loss": 0.9012559056282043, "Finetune/Loss (Raw)": 0.9354639053344727, "Finetune/Step": 2405, "Finetune/Step Time": 4.349721901118755} +{"Finetune/Learning Rate": 1.1677145950248954e-05, "Finetune/Loss": 0.9013657569885254, "Finetune/Loss (Raw)": 0.9872050285339355, "Finetune/Step": 2406, "Finetune/Step Time": 4.345335518941283} +{"Finetune/Learning Rate": 1.167100425330333e-05, "Finetune/Loss": 0.8987880349159241, "Finetune/Loss (Raw)": 0.6960299015045166, "Finetune/Step": 2407, "Finetune/Step Time": 4.366620298475027} +{"Finetune/Learning Rate": 1.1664861907873583e-05, "Finetune/Loss": 0.9004203081130981, "Finetune/Loss (Raw)": 1.100013256072998, "Finetune/Step": 2408, "Finetune/Step Time": 4.386224690824747} +{"Finetune/Learning Rate": 1.165871891634343e-05, "Finetune/Loss": 0.9003639817237854, "Finetune/Loss (Raw)": 0.8912615776062012, "Finetune/Step": 2409, "Finetune/Step Time": 4.386164575815201} +{"Finetune/Learning Rate": 1.165257528109685e-05, "Finetune/Loss": 0.9006386995315552, "Finetune/Loss (Raw)": 0.9326478838920593, "Finetune/Step": 2410, "Finetune/Step Time": 4.395483968779445} +{"Finetune/Learning Rate": 1.1646431004518068e-05, "Finetune/Loss": 0.8984894752502441, "Finetune/Loss (Raw)": 0.755013644695282, "Finetune/Step": 2411, "Finetune/Step Time": 4.402735894545913} +{"Finetune/Learning Rate": 1.1640286088991558e-05, "Finetune/Loss": 0.8985874056816101, "Finetune/Loss (Raw)": 0.8865643739700317, "Finetune/Step": 2412, "Finetune/Step Time": 4.375325841829181} +{"Finetune/Learning Rate": 1.1634140536902039e-05, "Finetune/Loss": 0.8975411653518677, "Finetune/Loss (Raw)": 0.8137591481208801, "Finetune/Step": 2413, "Finetune/Step Time": 4.36566355265677} +{"Finetune/Learning Rate": 1.1627994350634482e-05, "Finetune/Loss": 0.8975104093551636, "Finetune/Loss (Raw)": 0.9508326649665833, "Finetune/Step": 2414, "Finetune/Step Time": 4.351269630715251} +{"Finetune/Learning Rate": 1.1621847532574101e-05, "Finetune/Loss": 0.898439884185791, "Finetune/Loss (Raw)": 0.8865461349487305, "Finetune/Step": 2415, "Finetune/Step Time": 4.356569088995457} +{"Finetune/Learning Rate": 1.1615700085106357e-05, "Finetune/Loss": 0.8984485268592834, "Finetune/Loss (Raw)": 0.94498211145401, "Finetune/Step": 2416, "Finetune/Step Time": 4.354702774435282} +{"Finetune/Learning Rate": 1.1609552010616953e-05, "Finetune/Loss": 0.8996464014053345, "Finetune/Loss (Raw)": 0.8927891850471497, "Finetune/Step": 2417, "Finetune/Step Time": 4.352599108591676} +{"Finetune/Learning Rate": 1.1603403311491841e-05, "Finetune/Loss": 0.8952957391738892, "Finetune/Loss (Raw)": 0.4428146183490753, "Finetune/Step": 2418, "Finetune/Step Time": 4.39134980365634} +{"Finetune/Learning Rate": 1.1597253990117205e-05, "Finetune/Loss": 0.8956944346427917, "Finetune/Loss (Raw)": 0.8839641809463501, "Finetune/Step": 2419, "Finetune/Step Time": 4.398617306724191} +{"Finetune/Learning Rate": 1.1591104048879479e-05, "Finetune/Loss": 0.897404670715332, "Finetune/Loss (Raw)": 1.0707781314849854, "Finetune/Step": 2420, "Finetune/Step Time": 4.393802484497428} +{"Finetune/Learning Rate": 1.1584953490165334e-05, "Finetune/Loss": 0.8976567983627319, "Finetune/Loss (Raw)": 0.7667889595031738, "Finetune/Step": 2421, "Finetune/Step Time": 4.408284863457084} +{"Finetune/Learning Rate": 1.1578802316361685e-05, "Finetune/Loss": 0.8964008092880249, "Finetune/Loss (Raw)": 0.7846793532371521, "Finetune/Step": 2422, "Finetune/Step Time": 4.418186545372009} +{"Finetune/Learning Rate": 1.157265052985568e-05, "Finetune/Loss": 0.8948663473129272, "Finetune/Loss (Raw)": 0.7312700748443604, "Finetune/Step": 2423, "Finetune/Step Time": 4.405554054304957} +{"Finetune/Learning Rate": 1.156649813303471e-05, "Finetune/Loss": 0.8962283134460449, "Finetune/Loss (Raw)": 0.9347488880157471, "Finetune/Step": 2424, "Finetune/Step Time": 4.405271794646978} +{"Finetune/Learning Rate": 1.1560345128286396e-05, "Finetune/Loss": 0.8954500555992126, "Finetune/Loss (Raw)": 0.829863965511322, "Finetune/Step": 2425, "Finetune/Step Time": 4.409852581098676} +{"Finetune/Learning Rate": 1.1554191517998598e-05, "Finetune/Loss": 0.8915859460830688, "Finetune/Loss (Raw)": 0.37129566073417664, "Finetune/Step": 2426, "Finetune/Step Time": 4.448067374527454} +{"Finetune/Learning Rate": 1.1548037304559419e-05, "Finetune/Loss": 0.8915221691131592, "Finetune/Loss (Raw)": 0.9637559056282043, "Finetune/Step": 2427, "Finetune/Step Time": 4.439037036150694} +{"Finetune/Learning Rate": 1.1541882490357183e-05, "Finetune/Loss": 0.8902230858802795, "Finetune/Loss (Raw)": 0.803745448589325, "Finetune/Step": 2428, "Finetune/Step Time": 4.433048713952303} +{"Finetune/Learning Rate": 1.1535727077780456e-05, "Finetune/Loss": 0.8913089036941528, "Finetune/Loss (Raw)": 0.9662121534347534, "Finetune/Step": 2429, "Finetune/Step Time": 4.439739787951112} +{"Finetune/Learning Rate": 1.1529571069218033e-05, "Finetune/Loss": 0.8908786773681641, "Finetune/Loss (Raw)": 0.7491517663002014, "Finetune/Step": 2430, "Finetune/Step Time": 4.4642970487475395} +{"Finetune/Learning Rate": 1.152341446705894e-05, "Finetune/Loss": 0.8914645910263062, "Finetune/Loss (Raw)": 0.9945823550224304, "Finetune/Step": 2431, "Finetune/Step Time": 4.4680014699697495} +{"Finetune/Learning Rate": 1.1517257273692432e-05, "Finetune/Loss": 0.8929507732391357, "Finetune/Loss (Raw)": 0.9070563912391663, "Finetune/Step": 2432, "Finetune/Step Time": 4.470498664304614} +{"Finetune/Learning Rate": 1.1511099491508003e-05, "Finetune/Loss": 0.8933888673782349, "Finetune/Loss (Raw)": 0.8624529242515564, "Finetune/Step": 2433, "Finetune/Step Time": 4.470083108171821} +{"Finetune/Learning Rate": 1.1504941122895361e-05, "Finetune/Loss": 0.8924292922019958, "Finetune/Loss (Raw)": 0.9226307272911072, "Finetune/Step": 2434, "Finetune/Step Time": 4.469352159649134} +{"Finetune/Learning Rate": 1.1498782170244449e-05, "Finetune/Loss": 0.8917528986930847, "Finetune/Loss (Raw)": 0.8814367055892944, "Finetune/Step": 2435, "Finetune/Step Time": 4.478316083550453} +{"Finetune/Learning Rate": 1.1492622635945438e-05, "Finetune/Loss": 0.8945728540420532, "Finetune/Loss (Raw)": 0.9744449853897095, "Finetune/Step": 2436, "Finetune/Step Time": 4.4715633895248175} +{"Finetune/Learning Rate": 1.1486462522388724e-05, "Finetune/Loss": 0.894130289554596, "Finetune/Loss (Raw)": 0.8751318454742432, "Finetune/Step": 2437, "Finetune/Step Time": 4.477408932521939} +{"Finetune/Learning Rate": 1.1480301831964926e-05, "Finetune/Loss": 0.894572377204895, "Finetune/Loss (Raw)": 1.0077248811721802, "Finetune/Step": 2438, "Finetune/Step Time": 4.481330066919327} +{"Finetune/Learning Rate": 1.1474140567064889e-05, "Finetune/Loss": 0.8942952156066895, "Finetune/Loss (Raw)": 0.9978037476539612, "Finetune/Step": 2439, "Finetune/Step Time": 4.481151198968291} +{"Finetune/Learning Rate": 1.1467978730079677e-05, "Finetune/Loss": 0.8952993750572205, "Finetune/Loss (Raw)": 0.993802547454834, "Finetune/Step": 2440, "Finetune/Step Time": 4.455944698303938} +{"Finetune/Learning Rate": 1.1461816323400579e-05, "Finetune/Loss": 0.8954839110374451, "Finetune/Loss (Raw)": 0.9606384038925171, "Finetune/Step": 2441, "Finetune/Step Time": 4.452041337266564} +{"Finetune/Learning Rate": 1.1455653349419105e-05, "Finetune/Loss": 0.8959682583808899, "Finetune/Loss (Raw)": 0.9219039082527161, "Finetune/Step": 2442, "Finetune/Step Time": 4.447839895263314} +{"Finetune/Learning Rate": 1.1449489810526985e-05, "Finetune/Loss": 0.896931529045105, "Finetune/Loss (Raw)": 0.9789001941680908, "Finetune/Step": 2443, "Finetune/Step Time": 4.453395813703537} +{"Finetune/Learning Rate": 1.144332570911617e-05, "Finetune/Loss": 0.8968366980552673, "Finetune/Loss (Raw)": 0.9611367583274841, "Finetune/Step": 2444, "Finetune/Step Time": 4.460742732509971} +{"Finetune/Learning Rate": 1.1437161047578826e-05, "Finetune/Loss": 0.8931965827941895, "Finetune/Loss (Raw)": 0.39028337597846985, "Finetune/Step": 2445, "Finetune/Step Time": 4.495874596759677} +{"Finetune/Learning Rate": 1.1430995828307337e-05, "Finetune/Loss": 0.8935588598251343, "Finetune/Loss (Raw)": 0.9213322997093201, "Finetune/Step": 2446, "Finetune/Step Time": 4.502810498699546} +{"Finetune/Learning Rate": 1.1424830053694303e-05, "Finetune/Loss": 0.8922436237335205, "Finetune/Loss (Raw)": 0.9083122611045837, "Finetune/Step": 2447, "Finetune/Step Time": 4.508082825690508} +{"Finetune/Learning Rate": 1.1418663726132543e-05, "Finetune/Loss": 0.8926635980606079, "Finetune/Loss (Raw)": 0.9917773604393005, "Finetune/Step": 2448, "Finetune/Step Time": 4.509675268083811} +{"Finetune/Learning Rate": 1.141249684801509e-05, "Finetune/Loss": 0.8917542695999146, "Finetune/Loss (Raw)": 0.8100449442863464, "Finetune/Step": 2449, "Finetune/Step Time": 4.5017606765031815} +{"Finetune/Learning Rate": 1.140632942173519e-05, "Finetune/Loss": 0.8879660964012146, "Finetune/Loss (Raw)": 0.4886523485183716, "Finetune/Step": 2450, "Finetune/Step Time": 4.538078512996435} +{"Finetune/Learning Rate": 1.1400161449686293e-05, "Finetune/Loss": 0.884267270565033, "Finetune/Loss (Raw)": 0.32196488976478577, "Finetune/Step": 2451, "Finetune/Step Time": 4.564640061929822} +{"Finetune/Learning Rate": 1.1393992934262073e-05, "Finetune/Loss": 0.8850407600402832, "Finetune/Loss (Raw)": 0.950495183467865, "Finetune/Step": 2452, "Finetune/Step Time": 4.5654870849102736} +{"Finetune/Learning Rate": 1.1387823877856413e-05, "Finetune/Loss": 0.8843942880630493, "Finetune/Loss (Raw)": 0.9304622411727905, "Finetune/Step": 2453, "Finetune/Step Time": 4.582912867888808} +{"Finetune/Learning Rate": 1.1381654282863399e-05, "Finetune/Loss": 0.8881449699401855, "Finetune/Loss (Raw)": 0.9187151789665222, "Finetune/Step": 2454, "Finetune/Step Time": 4.554236119613051} +{"Finetune/Learning Rate": 1.1375484151677334e-05, "Finetune/Loss": 0.888831377029419, "Finetune/Loss (Raw)": 0.9972309470176697, "Finetune/Step": 2455, "Finetune/Step Time": 4.536038152873516} +{"Finetune/Learning Rate": 1.1369313486692723e-05, "Finetune/Loss": 0.8896266222000122, "Finetune/Loss (Raw)": 0.952888548374176, "Finetune/Step": 2456, "Finetune/Step Time": 4.526362044736743} +{"Finetune/Learning Rate": 1.136314229030428e-05, "Finetune/Loss": 0.8894091248512268, "Finetune/Loss (Raw)": 0.9502006769180298, "Finetune/Step": 2457, "Finetune/Step Time": 4.523557996377349} +{"Finetune/Learning Rate": 1.1356970564906928e-05, "Finetune/Loss": 0.8906346559524536, "Finetune/Loss (Raw)": 0.9613855481147766, "Finetune/Step": 2458, "Finetune/Step Time": 4.529592737555504} +{"Finetune/Learning Rate": 1.1350798312895795e-05, "Finetune/Loss": 0.8900958299636841, "Finetune/Loss (Raw)": 0.8657382726669312, "Finetune/Step": 2459, "Finetune/Step Time": 4.523598110303283} +{"Finetune/Learning Rate": 1.1344625536666208e-05, "Finetune/Loss": 0.8886854648590088, "Finetune/Loss (Raw)": 0.9281250834465027, "Finetune/Step": 2460, "Finetune/Step Time": 4.517575478181243} +{"Finetune/Learning Rate": 1.13384522386137e-05, "Finetune/Loss": 0.8891713619232178, "Finetune/Loss (Raw)": 1.054468035697937, "Finetune/Step": 2461, "Finetune/Step Time": 4.514305870980024} +{"Finetune/Learning Rate": 1.1332278421134007e-05, "Finetune/Loss": 0.8892554044723511, "Finetune/Loss (Raw)": 0.9485340118408203, "Finetune/Step": 2462, "Finetune/Step Time": 4.506865872070193} +{"Finetune/Learning Rate": 1.1326104086623075e-05, "Finetune/Loss": 0.8897786140441895, "Finetune/Loss (Raw)": 0.9395471215248108, "Finetune/Step": 2463, "Finetune/Step Time": 4.507455667480826} +{"Finetune/Learning Rate": 1.1319929237477033e-05, "Finetune/Loss": 0.8907300233840942, "Finetune/Loss (Raw)": 1.012284278869629, "Finetune/Step": 2464, "Finetune/Step Time": 4.492819651961327} +{"Finetune/Learning Rate": 1.1313753876092223e-05, "Finetune/Loss": 0.8901817202568054, "Finetune/Loss (Raw)": 0.8963847756385803, "Finetune/Step": 2465, "Finetune/Step Time": 4.49576660618186} +{"Finetune/Learning Rate": 1.1307578004865188e-05, "Finetune/Loss": 0.8888630867004395, "Finetune/Loss (Raw)": 0.8061186671257019, "Finetune/Step": 2466, "Finetune/Step Time": 4.497071264311671} +{"Finetune/Learning Rate": 1.130140162619265e-05, "Finetune/Loss": 0.88948655128479, "Finetune/Loss (Raw)": 0.9102237820625305, "Finetune/Step": 2467, "Finetune/Step Time": 4.477191077545285} +{"Finetune/Learning Rate": 1.1295224742471555e-05, "Finetune/Loss": 0.8910502195358276, "Finetune/Loss (Raw)": 1.1079736948013306, "Finetune/Step": 2468, "Finetune/Step Time": 4.475153228268027} +{"Finetune/Learning Rate": 1.128904735609902e-05, "Finetune/Loss": 0.8907285928726196, "Finetune/Loss (Raw)": 0.9929081797599792, "Finetune/Step": 2469, "Finetune/Step Time": 4.446279173716903} +{"Finetune/Learning Rate": 1.1282869469472375e-05, "Finetune/Loss": 0.8945239186286926, "Finetune/Loss (Raw)": 0.842818558216095, "Finetune/Step": 2470, "Finetune/Step Time": 4.413614626973867} +{"Finetune/Learning Rate": 1.1276691084989135e-05, "Finetune/Loss": 0.8944869041442871, "Finetune/Loss (Raw)": 0.9841994643211365, "Finetune/Step": 2471, "Finetune/Step Time": 4.396968603134155} +{"Finetune/Learning Rate": 1.1270512205047008e-05, "Finetune/Loss": 0.8947238922119141, "Finetune/Loss (Raw)": 0.9427292346954346, "Finetune/Step": 2472, "Finetune/Step Time": 4.366264306008816} +{"Finetune/Learning Rate": 1.1264332832043902e-05, "Finetune/Loss": 0.8931438326835632, "Finetune/Loss (Raw)": 0.9425841569900513, "Finetune/Step": 2473, "Finetune/Step Time": 4.3662793561816216} +{"Finetune/Learning Rate": 1.1258152968377907e-05, "Finetune/Loss": 0.8933089375495911, "Finetune/Loss (Raw)": 0.9226877093315125, "Finetune/Step": 2474, "Finetune/Step Time": 4.377061884850264} +{"Finetune/Learning Rate": 1.125197261644731e-05, "Finetune/Loss": 0.893153727054596, "Finetune/Loss (Raw)": 0.8764044642448425, "Finetune/Step": 2475, "Finetune/Step Time": 4.3819925505667925} +{"Finetune/Learning Rate": 1.1245791778650578e-05, "Finetune/Loss": 0.8926723003387451, "Finetune/Loss (Raw)": 0.9342185258865356, "Finetune/Step": 2476, "Finetune/Step Time": 4.379840135574341} +{"Finetune/Learning Rate": 1.1239610457386381e-05, "Finetune/Loss": 0.8926605582237244, "Finetune/Loss (Raw)": 0.9238095283508301, "Finetune/Step": 2477, "Finetune/Step Time": 4.377306651324034} +{"Finetune/Learning Rate": 1.1233428655053572e-05, "Finetune/Loss": 0.8925793766975403, "Finetune/Loss (Raw)": 0.905796229839325, "Finetune/Step": 2478, "Finetune/Step Time": 4.382312772795558} +{"Finetune/Learning Rate": 1.1227246374051175e-05, "Finetune/Loss": 0.8910781145095825, "Finetune/Loss (Raw)": 0.8341845870018005, "Finetune/Step": 2479, "Finetune/Step Time": 4.383014226332307} +{"Finetune/Learning Rate": 1.1221063616778425e-05, "Finetune/Loss": 0.8893287181854248, "Finetune/Loss (Raw)": 0.8418821692466736, "Finetune/Step": 2480, "Finetune/Step Time": 4.381751233711839} +{"Finetune/Learning Rate": 1.1214880385634723e-05, "Finetune/Loss": 0.8851180076599121, "Finetune/Loss (Raw)": 0.3776371479034424, "Finetune/Step": 2481, "Finetune/Step Time": 4.422601368278265} +{"Finetune/Learning Rate": 1.120869668301966e-05, "Finetune/Loss": 0.8865104913711548, "Finetune/Loss (Raw)": 1.0944347381591797, "Finetune/Step": 2482, "Finetune/Step Time": 4.419895542785525} +{"Finetune/Learning Rate": 1.1202512511333017e-05, "Finetune/Loss": 0.8871748447418213, "Finetune/Loss (Raw)": 0.9037441611289978, "Finetune/Step": 2483, "Finetune/Step Time": 4.425574043765664} +{"Finetune/Learning Rate": 1.119632787297474e-05, "Finetune/Loss": 0.8877158164978027, "Finetune/Loss (Raw)": 0.8723331689834595, "Finetune/Step": 2484, "Finetune/Step Time": 4.431128583848476} +{"Finetune/Learning Rate": 1.1190142770344976e-05, "Finetune/Loss": 0.8879266977310181, "Finetune/Loss (Raw)": 0.9294617176055908, "Finetune/Step": 2485, "Finetune/Step Time": 4.43771150149405} +{"Finetune/Learning Rate": 1.1183957205844037e-05, "Finetune/Loss": 0.8864225745201111, "Finetune/Loss (Raw)": 0.8263502717018127, "Finetune/Step": 2486, "Finetune/Step Time": 4.447943944483995} +{"Finetune/Learning Rate": 1.1177771181872416e-05, "Finetune/Loss": 0.8866048455238342, "Finetune/Loss (Raw)": 0.31697937846183777, "Finetune/Step": 2487, "Finetune/Step Time": 4.446787506341934} +{"Finetune/Learning Rate": 1.1171584700830803e-05, "Finetune/Loss": 0.8874751329421997, "Finetune/Loss (Raw)": 1.026809811592102, "Finetune/Step": 2488, "Finetune/Step Time": 4.451306393370032} +{"Finetune/Learning Rate": 1.1165397765120035e-05, "Finetune/Loss": 0.8875126838684082, "Finetune/Loss (Raw)": 0.9528325200080872, "Finetune/Step": 2489, "Finetune/Step Time": 4.458270628005266} +{"Finetune/Learning Rate": 1.115921037714115e-05, "Finetune/Loss": 0.8827273845672607, "Finetune/Loss (Raw)": 0.32083824276924133, "Finetune/Step": 2490, "Finetune/Step Time": 4.5009861793369055} +{"Finetune/Learning Rate": 1.1153022539295345e-05, "Finetune/Loss": 0.8840975165367126, "Finetune/Loss (Raw)": 0.8882067799568176, "Finetune/Step": 2491, "Finetune/Step Time": 4.493926705792546} +{"Finetune/Learning Rate": 1.1146834253984008e-05, "Finetune/Loss": 0.8837029933929443, "Finetune/Loss (Raw)": 0.7920609712600708, "Finetune/Step": 2492, "Finetune/Step Time": 4.4956045262515545} +{"Finetune/Learning Rate": 1.1140645523608684e-05, "Finetune/Loss": 0.8833282589912415, "Finetune/Loss (Raw)": 0.9762851595878601, "Finetune/Step": 2493, "Finetune/Step Time": 4.503553230315447} +{"Finetune/Learning Rate": 1.1134456350571099e-05, "Finetune/Loss": 0.88521409034729, "Finetune/Loss (Raw)": 1.0144494771957397, "Finetune/Step": 2494, "Finetune/Step Time": 4.486713318154216} +{"Finetune/Learning Rate": 1.112826673727316e-05, "Finetune/Loss": 0.8849339485168457, "Finetune/Loss (Raw)": 0.9329952001571655, "Finetune/Step": 2495, "Finetune/Step Time": 4.479697277769446} +{"Finetune/Learning Rate": 1.112207668611692e-05, "Finetune/Loss": 0.8850805759429932, "Finetune/Loss (Raw)": 0.9135944843292236, "Finetune/Step": 2496, "Finetune/Step Time": 4.501173248514533} +{"Finetune/Learning Rate": 1.1115886199504631e-05, "Finetune/Loss": 0.8835247755050659, "Finetune/Loss (Raw)": 0.8028628826141357, "Finetune/Step": 2497, "Finetune/Step Time": 4.499778214842081} +{"Finetune/Learning Rate": 1.110969527983869e-05, "Finetune/Loss": 0.8842798471450806, "Finetune/Loss (Raw)": 1.006789207458496, "Finetune/Step": 2498, "Finetune/Step Time": 4.5003584288060665} +{"Finetune/Learning Rate": 1.1103503929521677e-05, "Finetune/Loss": 0.8814975619316101, "Finetune/Loss (Raw)": 0.6490877270698547, "Finetune/Step": 2499, "Finetune/Step Time": 4.517747217789292} +{"Finetune/Learning Rate": 1.1097312150956336e-05, "Finetune/Loss": 0.8818889856338501, "Finetune/Loss (Raw)": 0.9352731108665466, "Finetune/Step": 2500, "Finetune/Step Time": 4.517652489244938} +{"Finetune/Learning Rate": 1.1091119946545568e-05, "Finetune/Loss": 0.8813347220420837, "Finetune/Loss (Raw)": 0.9348815083503723, "Finetune/Step": 2501, "Finetune/Step Time": 4.498620603233576} +{"Finetune/Learning Rate": 1.1084927318692453e-05, "Finetune/Loss": 0.8823449611663818, "Finetune/Loss (Raw)": 0.9465057253837585, "Finetune/Step": 2502, "Finetune/Step Time": 4.49879009462893} +{"Finetune/Learning Rate": 1.1078734269800227e-05, "Finetune/Loss": 0.8832838535308838, "Finetune/Loss (Raw)": 0.8851961493492126, "Finetune/Step": 2503, "Finetune/Step Time": 4.488012559711933} +{"Finetune/Learning Rate": 1.1072540802272291e-05, "Finetune/Loss": 0.8828226327896118, "Finetune/Loss (Raw)": 0.895753800868988, "Finetune/Step": 2504, "Finetune/Step Time": 4.487771742045879} +{"Finetune/Learning Rate": 1.1066346918512211e-05, "Finetune/Loss": 0.8814878463745117, "Finetune/Loss (Raw)": 0.8958854079246521, "Finetune/Step": 2505, "Finetune/Step Time": 4.486937511712313} +{"Finetune/Learning Rate": 1.1060152620923707e-05, "Finetune/Loss": 0.8803602457046509, "Finetune/Loss (Raw)": 0.8592897057533264, "Finetune/Step": 2506, "Finetune/Step Time": 4.490681679919362} +{"Finetune/Learning Rate": 1.105395791191067e-05, "Finetune/Loss": 0.8809382915496826, "Finetune/Loss (Raw)": 1.0126334428787231, "Finetune/Step": 2507, "Finetune/Step Time": 4.477984460070729} +{"Finetune/Learning Rate": 1.1047762793877146e-05, "Finetune/Loss": 0.8813482522964478, "Finetune/Loss (Raw)": 0.8374664187431335, "Finetune/Step": 2508, "Finetune/Step Time": 4.485475672408938} +{"Finetune/Learning Rate": 1.1041567269227336e-05, "Finetune/Loss": 0.8807151913642883, "Finetune/Loss (Raw)": 0.8367648124694824, "Finetune/Step": 2509, "Finetune/Step Time": 4.488498758524656} +{"Finetune/Learning Rate": 1.1035371340365606e-05, "Finetune/Loss": 0.882781982421875, "Finetune/Loss (Raw)": 1.0414481163024902, "Finetune/Step": 2510, "Finetune/Step Time": 4.477452678605914} +{"Finetune/Learning Rate": 1.1029175009696468e-05, "Finetune/Loss": 0.8851865530014038, "Finetune/Loss (Raw)": 0.9818835854530334, "Finetune/Step": 2511, "Finetune/Step Time": 4.474075695499778} +{"Finetune/Learning Rate": 1.1022978279624606e-05, "Finetune/Loss": 0.8829330205917358, "Finetune/Loss (Raw)": 0.8058629631996155, "Finetune/Step": 2512, "Finetune/Step Time": 4.483846046030521} +{"Finetune/Learning Rate": 1.1016781152554846e-05, "Finetune/Loss": 0.8824747800827026, "Finetune/Loss (Raw)": 0.8488621711730957, "Finetune/Step": 2513, "Finetune/Step Time": 4.482686473056674} +{"Finetune/Learning Rate": 1.1010583630892166e-05, "Finetune/Loss": 0.8824148178100586, "Finetune/Loss (Raw)": 1.0141494274139404, "Finetune/Step": 2514, "Finetune/Step Time": 4.489006228744984} +{"Finetune/Learning Rate": 1.1004385717041716e-05, "Finetune/Loss": 0.8815389275550842, "Finetune/Loss (Raw)": 0.7612971663475037, "Finetune/Step": 2515, "Finetune/Step Time": 4.493981916457415} +{"Finetune/Learning Rate": 1.0998187413408774e-05, "Finetune/Loss": 0.8811854124069214, "Finetune/Loss (Raw)": 0.927162230014801, "Finetune/Step": 2516, "Finetune/Step Time": 4.472141979262233} +{"Finetune/Learning Rate": 1.0991988722398786e-05, "Finetune/Loss": 0.8803992867469788, "Finetune/Loss (Raw)": 0.9309370517730713, "Finetune/Step": 2517, "Finetune/Step Time": 4.475592838600278} +{"Finetune/Learning Rate": 1.0985789646417341e-05, "Finetune/Loss": 0.8790155649185181, "Finetune/Loss (Raw)": 0.8540431261062622, "Finetune/Step": 2518, "Finetune/Step Time": 4.476190984249115} +{"Finetune/Learning Rate": 1.097959018787018e-05, "Finetune/Loss": 0.8780993223190308, "Finetune/Loss (Raw)": 0.8738546967506409, "Finetune/Step": 2519, "Finetune/Step Time": 4.4809442814439535} +{"Finetune/Learning Rate": 1.0973390349163195e-05, "Finetune/Loss": 0.8784694671630859, "Finetune/Loss (Raw)": 0.9779254794120789, "Finetune/Step": 2520, "Finetune/Step Time": 4.480538487434387} +{"Finetune/Learning Rate": 1.0967190132702414e-05, "Finetune/Loss": 0.8736734390258789, "Finetune/Loss (Raw)": 0.4044245779514313, "Finetune/Step": 2521, "Finetune/Step Time": 4.503261422738433} +{"Finetune/Learning Rate": 1.096098954089403e-05, "Finetune/Loss": 0.8779676556587219, "Finetune/Loss (Raw)": 0.9475864768028259, "Finetune/Step": 2522, "Finetune/Step Time": 4.481691358610988} +{"Finetune/Learning Rate": 1.0954788576144365e-05, "Finetune/Loss": 0.8761413097381592, "Finetune/Loss (Raw)": 0.7071113586425781, "Finetune/Step": 2523, "Finetune/Step Time": 4.48313781619072} +{"Finetune/Learning Rate": 1.0948587240859893e-05, "Finetune/Loss": 0.8762031197547913, "Finetune/Loss (Raw)": 0.8782438039779663, "Finetune/Step": 2524, "Finetune/Step Time": 4.487964732572436} +{"Finetune/Learning Rate": 1.0942385537447236e-05, "Finetune/Loss": 0.8755089044570923, "Finetune/Loss (Raw)": 0.8467007875442505, "Finetune/Step": 2525, "Finetune/Step Time": 4.500918623059988} +{"Finetune/Learning Rate": 1.0936183468313147e-05, "Finetune/Loss": 0.8744384050369263, "Finetune/Loss (Raw)": 0.9368948936462402, "Finetune/Step": 2526, "Finetune/Step Time": 4.50353530049324} +{"Finetune/Learning Rate": 1.0929981035864532e-05, "Finetune/Loss": 0.8743727207183838, "Finetune/Loss (Raw)": 0.9770691394805908, "Finetune/Step": 2527, "Finetune/Step Time": 4.513449441641569} +{"Finetune/Learning Rate": 1.0923778242508437e-05, "Finetune/Loss": 0.8757510185241699, "Finetune/Loss (Raw)": 0.9950369596481323, "Finetune/Step": 2528, "Finetune/Step Time": 4.519418528303504} +{"Finetune/Learning Rate": 1.0917575090652038e-05, "Finetune/Loss": 0.8705122470855713, "Finetune/Loss (Raw)": 0.27090492844581604, "Finetune/Step": 2529, "Finetune/Step Time": 4.5564924739301205} +{"Finetune/Learning Rate": 1.091137158270266e-05, "Finetune/Loss": 0.871340811252594, "Finetune/Loss (Raw)": 0.9249654412269592, "Finetune/Step": 2530, "Finetune/Step Time": 4.547268133610487} +{"Finetune/Learning Rate": 1.0905167721067762e-05, "Finetune/Loss": 0.870568037033081, "Finetune/Loss (Raw)": 0.8315589427947998, "Finetune/Step": 2531, "Finetune/Step Time": 4.5455785524100065} +{"Finetune/Learning Rate": 1.0898963508154947e-05, "Finetune/Loss": 0.8709263801574707, "Finetune/Loss (Raw)": 0.9840543866157532, "Finetune/Step": 2532, "Finetune/Step Time": 4.562655990943313} +{"Finetune/Learning Rate": 1.0892758946371943e-05, "Finetune/Loss": 0.8705650568008423, "Finetune/Loss (Raw)": 0.8892186284065247, "Finetune/Step": 2533, "Finetune/Step Time": 4.5650234669446945} +{"Finetune/Learning Rate": 1.0886554038126624e-05, "Finetune/Loss": 0.8657990097999573, "Finetune/Loss (Raw)": 0.3771559000015259, "Finetune/Step": 2534, "Finetune/Step Time": 4.602744344621897} +{"Finetune/Learning Rate": 1.0880348785826991e-05, "Finetune/Loss": 0.8682737350463867, "Finetune/Loss (Raw)": 1.0127952098846436, "Finetune/Step": 2535, "Finetune/Step Time": 4.587285801768303} +{"Finetune/Learning Rate": 1.087414319188118e-05, "Finetune/Loss": 0.8654313087463379, "Finetune/Loss (Raw)": 0.7361770868301392, "Finetune/Step": 2536, "Finetune/Step Time": 4.583869583904743} +{"Finetune/Learning Rate": 1.0867937258697463e-05, "Finetune/Loss": 0.8653882741928101, "Finetune/Loss (Raw)": 0.8857599496841431, "Finetune/Step": 2537, "Finetune/Step Time": 4.5910012517124414} +{"Finetune/Learning Rate": 1.086173098868424e-05, "Finetune/Loss": 0.8658510446548462, "Finetune/Loss (Raw)": 0.9918729662895203, "Finetune/Step": 2538, "Finetune/Step Time": 4.594569785520434} +{"Finetune/Learning Rate": 1.0855524384250048e-05, "Finetune/Loss": 0.8668173551559448, "Finetune/Loss (Raw)": 0.8787054419517517, "Finetune/Step": 2539, "Finetune/Step Time": 4.598898621276021} +{"Finetune/Learning Rate": 1.0849317447803542e-05, "Finetune/Loss": 0.8677078485488892, "Finetune/Loss (Raw)": 1.000543475151062, "Finetune/Step": 2540, "Finetune/Step Time": 4.605559032410383} +{"Finetune/Learning Rate": 1.0843110181753513e-05, "Finetune/Loss": 0.8679051399230957, "Finetune/Loss (Raw)": 0.8390246033668518, "Finetune/Step": 2541, "Finetune/Step Time": 4.622166762128472} +{"Finetune/Learning Rate": 1.0836902588508888e-05, "Finetune/Loss": 0.867613673210144, "Finetune/Loss (Raw)": 0.9135199785232544, "Finetune/Step": 2542, "Finetune/Step Time": 4.626501116901636} +{"Finetune/Learning Rate": 1.0830694670478706e-05, "Finetune/Loss": 0.8678826093673706, "Finetune/Loss (Raw)": 0.9209661483764648, "Finetune/Step": 2543, "Finetune/Step Time": 4.6160713993012905} +{"Finetune/Learning Rate": 1.082448643007214e-05, "Finetune/Loss": 0.8662168979644775, "Finetune/Loss (Raw)": 0.7317735552787781, "Finetune/Step": 2544, "Finetune/Step Time": 4.6259268913418055} +{"Finetune/Learning Rate": 1.0818277869698488e-05, "Finetune/Loss": 0.8660616874694824, "Finetune/Loss (Raw)": 0.8729165196418762, "Finetune/Step": 2545, "Finetune/Step Time": 4.625139059498906} +{"Finetune/Learning Rate": 1.0812068991767171e-05, "Finetune/Loss": 0.8647927045822144, "Finetune/Loss (Raw)": 0.28038671612739563, "Finetune/Step": 2546, "Finetune/Step Time": 4.6245670057833195} +{"Finetune/Learning Rate": 1.0805859798687727e-05, "Finetune/Loss": 0.8647355437278748, "Finetune/Loss (Raw)": 0.8766486048698425, "Finetune/Step": 2547, "Finetune/Step Time": 4.595642579719424} +{"Finetune/Learning Rate": 1.0799650292869835e-05, "Finetune/Loss": 0.86446213722229, "Finetune/Loss (Raw)": 1.0357850790023804, "Finetune/Step": 2548, "Finetune/Step Time": 4.592507779598236} +{"Finetune/Learning Rate": 1.0793440476723274e-05, "Finetune/Loss": 0.8610191345214844, "Finetune/Loss (Raw)": 0.3260858952999115, "Finetune/Step": 2549, "Finetune/Step Time": 4.605967758223414} +{"Finetune/Learning Rate": 1.0787230352657954e-05, "Finetune/Loss": 0.8621478080749512, "Finetune/Loss (Raw)": 0.9291490316390991, "Finetune/Step": 2550, "Finetune/Step Time": 4.601525055244565} +{"Finetune/Learning Rate": 1.0781019923083905e-05, "Finetune/Loss": 0.8638505339622498, "Finetune/Loss (Raw)": 0.9492123126983643, "Finetune/Step": 2551, "Finetune/Step Time": 4.597749721258879} +{"Finetune/Learning Rate": 1.077480919041127e-05, "Finetune/Loss": 0.8650605082511902, "Finetune/Loss (Raw)": 1.0896344184875488, "Finetune/Step": 2552, "Finetune/Step Time": 4.598628932610154} +{"Finetune/Learning Rate": 1.0768598157050313e-05, "Finetune/Loss": 0.8651659488677979, "Finetune/Loss (Raw)": 0.8433600068092346, "Finetune/Step": 2553, "Finetune/Step Time": 4.599846225231886} +{"Finetune/Learning Rate": 1.0762386825411422e-05, "Finetune/Loss": 0.869613766670227, "Finetune/Loss (Raw)": 0.9406110048294067, "Finetune/Step": 2554, "Finetune/Step Time": 4.588994482532144} +{"Finetune/Learning Rate": 1.0756175197905086e-05, "Finetune/Loss": 0.8646503686904907, "Finetune/Loss (Raw)": 0.3284425139427185, "Finetune/Step": 2555, "Finetune/Step Time": 4.626402247697115} +{"Finetune/Learning Rate": 1.0749963276941919e-05, "Finetune/Loss": 0.8653514385223389, "Finetune/Loss (Raw)": 0.8934845328330994, "Finetune/Step": 2556, "Finetune/Step Time": 4.626798452809453} +{"Finetune/Learning Rate": 1.0743751064932648e-05, "Finetune/Loss": 0.8637290000915527, "Finetune/Loss (Raw)": 0.758537232875824, "Finetune/Step": 2557, "Finetune/Step Time": 4.61045597307384} +{"Finetune/Learning Rate": 1.0737538564288104e-05, "Finetune/Loss": 0.8646609783172607, "Finetune/Loss (Raw)": 0.8684460520744324, "Finetune/Step": 2558, "Finetune/Step Time": 4.580497391521931} +{"Finetune/Learning Rate": 1.0731325777419246e-05, "Finetune/Loss": 0.8639422059059143, "Finetune/Loss (Raw)": 0.9025744199752808, "Finetune/Step": 2559, "Finetune/Step Time": 4.573829624801874} +{"Finetune/Learning Rate": 1.0725112706737133e-05, "Finetune/Loss": 0.863651692867279, "Finetune/Loss (Raw)": 0.8698751926422119, "Finetune/Step": 2560, "Finetune/Step Time": 4.58820596896112} +{"Finetune/Learning Rate": 1.0718899354652931e-05, "Finetune/Loss": 0.8638945817947388, "Finetune/Loss (Raw)": 0.8935443758964539, "Finetune/Step": 2561, "Finetune/Step Time": 4.59150180593133} +{"Finetune/Learning Rate": 1.071268572357793e-05, "Finetune/Loss": 0.8644888401031494, "Finetune/Loss (Raw)": 0.9986942410469055, "Finetune/Step": 2562, "Finetune/Step Time": 4.59051014855504} +{"Finetune/Learning Rate": 1.0706471815923511e-05, "Finetune/Loss": 0.8624446988105774, "Finetune/Loss (Raw)": 0.6197885870933533, "Finetune/Step": 2563, "Finetune/Step Time": 4.588631154969335} +{"Finetune/Learning Rate": 1.070025763410117e-05, "Finetune/Loss": 0.86203533411026, "Finetune/Loss (Raw)": 0.9220436811447144, "Finetune/Step": 2564, "Finetune/Step Time": 4.5914052631706} +{"Finetune/Learning Rate": 1.0694043180522515e-05, "Finetune/Loss": 0.8627859354019165, "Finetune/Loss (Raw)": 0.9712071418762207, "Finetune/Step": 2565, "Finetune/Step Time": 4.588620686903596} +{"Finetune/Learning Rate": 1.0687828457599254e-05, "Finetune/Loss": 0.8621448278427124, "Finetune/Loss (Raw)": 0.9256687164306641, "Finetune/Step": 2566, "Finetune/Step Time": 4.596980733796954} +{"Finetune/Learning Rate": 1.0681613467743195e-05, "Finetune/Loss": 0.8605853915214539, "Finetune/Loss (Raw)": 0.7981916666030884, "Finetune/Step": 2567, "Finetune/Step Time": 4.602419599890709} +{"Finetune/Learning Rate": 1.0675398213366257e-05, "Finetune/Loss": 0.8590537905693054, "Finetune/Loss (Raw)": 0.797761857509613, "Finetune/Step": 2568, "Finetune/Step Time": 4.606633063405752} +{"Finetune/Learning Rate": 1.0669182696880459e-05, "Finetune/Loss": 0.8602885007858276, "Finetune/Loss (Raw)": 1.1186809539794922, "Finetune/Step": 2569, "Finetune/Step Time": 4.604067049920559} +{"Finetune/Learning Rate": 1.0662966920697918e-05, "Finetune/Loss": 0.8602557182312012, "Finetune/Loss (Raw)": 0.9177073240280151, "Finetune/Step": 2570, "Finetune/Step Time": 4.608693411573768} +{"Finetune/Learning Rate": 1.0656750887230862e-05, "Finetune/Loss": 0.8609069585800171, "Finetune/Loss (Raw)": 1.062261939048767, "Finetune/Step": 2571, "Finetune/Step Time": 4.606572758406401} +{"Finetune/Learning Rate": 1.0650534598891608e-05, "Finetune/Loss": 0.8602532148361206, "Finetune/Loss (Raw)": 0.8774576783180237, "Finetune/Step": 2572, "Finetune/Step Time": 4.608876030892134} +{"Finetune/Learning Rate": 1.0644318058092577e-05, "Finetune/Loss": 0.8642655611038208, "Finetune/Loss (Raw)": 0.9038597345352173, "Finetune/Step": 2573, "Finetune/Step Time": 4.5773358922451735} +{"Finetune/Learning Rate": 1.0638101267246283e-05, "Finetune/Loss": 0.8639203310012817, "Finetune/Loss (Raw)": 0.8771448731422424, "Finetune/Step": 2574, "Finetune/Step Time": 4.5943635776638985} +{"Finetune/Learning Rate": 1.063188422876535e-05, "Finetune/Loss": 0.8632370233535767, "Finetune/Loss (Raw)": 0.8208472728729248, "Finetune/Step": 2575, "Finetune/Step Time": 4.585971945896745} +{"Finetune/Learning Rate": 1.0625666945062484e-05, "Finetune/Loss": 0.8627991080284119, "Finetune/Loss (Raw)": 0.9357205033302307, "Finetune/Step": 2576, "Finetune/Step Time": 4.5871450658887625} +{"Finetune/Learning Rate": 1.061944941855049e-05, "Finetune/Loss": 0.8641155958175659, "Finetune/Loss (Raw)": 0.9785591959953308, "Finetune/Step": 2577, "Finetune/Step Time": 4.5917561165988445} +{"Finetune/Learning Rate": 1.061323165164227e-05, "Finetune/Loss": 0.8682073354721069, "Finetune/Loss (Raw)": 1.012395977973938, "Finetune/Step": 2578, "Finetune/Step Time": 4.552842702716589} +{"Finetune/Learning Rate": 1.0607013646750818e-05, "Finetune/Loss": 0.8735164999961853, "Finetune/Loss (Raw)": 1.0015387535095215, "Finetune/Step": 2579, "Finetune/Step Time": 4.513475649058819} +{"Finetune/Learning Rate": 1.0600795406289221e-05, "Finetune/Loss": 0.8715349435806274, "Finetune/Loss (Raw)": 0.6968557834625244, "Finetune/Step": 2580, "Finetune/Step Time": 4.524206850677729} +{"Finetune/Learning Rate": 1.059457693267066e-05, "Finetune/Loss": 0.872377336025238, "Finetune/Loss (Raw)": 1.038286566734314, "Finetune/Step": 2581, "Finetune/Step Time": 4.500822205096483} +{"Finetune/Learning Rate": 1.0588358228308395e-05, "Finetune/Loss": 0.8720444440841675, "Finetune/Loss (Raw)": 0.8761053085327148, "Finetune/Step": 2582, "Finetune/Step Time": 4.490243187174201} +{"Finetune/Learning Rate": 1.0582139295615793e-05, "Finetune/Loss": 0.8706341981887817, "Finetune/Loss (Raw)": 0.8167164325714111, "Finetune/Step": 2583, "Finetune/Step Time": 4.498658625409007} +{"Finetune/Learning Rate": 1.0575920137006288e-05, "Finetune/Loss": 0.8712762594223022, "Finetune/Loss (Raw)": 1.0350697040557861, "Finetune/Step": 2584, "Finetune/Step Time": 4.498771632090211} +{"Finetune/Learning Rate": 1.056970075489343e-05, "Finetune/Loss": 0.8701242208480835, "Finetune/Loss (Raw)": 0.8027446269989014, "Finetune/Step": 2585, "Finetune/Step Time": 4.5009378381073475} +{"Finetune/Learning Rate": 1.0563481151690827e-05, "Finetune/Loss": 0.8701359033584595, "Finetune/Loss (Raw)": 0.9628793597221375, "Finetune/Step": 2586, "Finetune/Step Time": 4.4871094692498446} +{"Finetune/Learning Rate": 1.055726132981219e-05, "Finetune/Loss": 0.8707845211029053, "Finetune/Loss (Raw)": 0.9487593770027161, "Finetune/Step": 2587, "Finetune/Step Time": 4.492758305743337} +{"Finetune/Learning Rate": 1.0551041291671311e-05, "Finetune/Loss": 0.8702179193496704, "Finetune/Loss (Raw)": 0.8556027412414551, "Finetune/Step": 2588, "Finetune/Step Time": 4.501165935769677} +{"Finetune/Learning Rate": 1.0544821039682066e-05, "Finetune/Loss": 0.8696651458740234, "Finetune/Loss (Raw)": 0.9837155938148499, "Finetune/Step": 2589, "Finetune/Step Time": 4.505101483315229} +{"Finetune/Learning Rate": 1.0538600576258413e-05, "Finetune/Loss": 0.8694998025894165, "Finetune/Loss (Raw)": 0.9273653626441956, "Finetune/Step": 2590, "Finetune/Step Time": 4.502558281645179} +{"Finetune/Learning Rate": 1.0532379903814387e-05, "Finetune/Loss": 0.8685011863708496, "Finetune/Loss (Raw)": 0.8117303848266602, "Finetune/Step": 2591, "Finetune/Step Time": 4.5411689560860395} +{"Finetune/Learning Rate": 1.0526159024764118e-05, "Finetune/Loss": 0.8677219152450562, "Finetune/Loss (Raw)": 0.9125317335128784, "Finetune/Step": 2592, "Finetune/Step Time": 4.549668351188302} +{"Finetune/Learning Rate": 1.0519937941521805e-05, "Finetune/Loss": 0.8671886920928955, "Finetune/Loss (Raw)": 0.8281294107437134, "Finetune/Step": 2593, "Finetune/Step Time": 4.569678528234363} +{"Finetune/Learning Rate": 1.0513716656501723e-05, "Finetune/Loss": 0.8672595024108887, "Finetune/Loss (Raw)": 0.8151884078979492, "Finetune/Step": 2594, "Finetune/Step Time": 4.580945298075676} +{"Finetune/Learning Rate": 1.0507495172118242e-05, "Finetune/Loss": 0.8670755624771118, "Finetune/Loss (Raw)": 0.8866834044456482, "Finetune/Step": 2595, "Finetune/Step Time": 4.58633285202086} +{"Finetune/Learning Rate": 1.0501273490785789e-05, "Finetune/Loss": 0.8651562929153442, "Finetune/Loss (Raw)": 0.8623018264770508, "Finetune/Step": 2596, "Finetune/Step Time": 4.5793858747929335} +{"Finetune/Learning Rate": 1.049505161491888e-05, "Finetune/Loss": 0.8648959398269653, "Finetune/Loss (Raw)": 0.9595853686332703, "Finetune/Step": 2597, "Finetune/Step Time": 4.582983726635575} +{"Finetune/Learning Rate": 1.0488829546932106e-05, "Finetune/Loss": 0.863959550857544, "Finetune/Loss (Raw)": 0.7229596972465515, "Finetune/Step": 2598, "Finetune/Step Time": 4.594913007691503} +{"Finetune/Learning Rate": 1.0482607289240128e-05, "Finetune/Loss": 0.8637358546257019, "Finetune/Loss (Raw)": 0.9555637240409851, "Finetune/Step": 2599, "Finetune/Step Time": 4.599304614588618} +{"Finetune/Learning Rate": 1.0476384844257686e-05, "Finetune/Loss": 0.8632978200912476, "Finetune/Loss (Raw)": 0.8866677284240723, "Finetune/Step": 2600, "Finetune/Step Time": 4.603959146887064} +{"Finetune/Learning Rate": 1.0470162214399582e-05, "Finetune/Loss": 0.8630064725875854, "Finetune/Loss (Raw)": 0.9052826166152954, "Finetune/Step": 2601, "Finetune/Step Time": 4.610557276755571} +{"Finetune/Learning Rate": 1.0463939402080706e-05, "Finetune/Loss": 0.8637893795967102, "Finetune/Loss (Raw)": 1.022904872894287, "Finetune/Step": 2602, "Finetune/Step Time": 4.6016778610646725} +{"Finetune/Learning Rate": 1.0457716409716002e-05, "Finetune/Loss": 0.8645957708358765, "Finetune/Loss (Raw)": 0.9796239733695984, "Finetune/Step": 2603, "Finetune/Step Time": 4.595592994242907} +{"Finetune/Learning Rate": 1.0451493239720496e-05, "Finetune/Loss": 0.8641387820243835, "Finetune/Loss (Raw)": 0.8757168650627136, "Finetune/Step": 2604, "Finetune/Step Time": 4.601259021088481} +{"Finetune/Learning Rate": 1.0445269894509283e-05, "Finetune/Loss": 0.8630951642990112, "Finetune/Loss (Raw)": 0.790233314037323, "Finetune/Step": 2605, "Finetune/Step Time": 4.6048772018402815} +{"Finetune/Learning Rate": 1.043904637649751e-05, "Finetune/Loss": 0.862520694732666, "Finetune/Loss (Raw)": 0.8322587609291077, "Finetune/Step": 2606, "Finetune/Step Time": 4.6277030147612095} +{"Finetune/Learning Rate": 1.0432822688100414e-05, "Finetune/Loss": 0.863322377204895, "Finetune/Loss (Raw)": 0.936801016330719, "Finetune/Step": 2607, "Finetune/Step Time": 4.65249615162611} +{"Finetune/Learning Rate": 1.0426598831733283e-05, "Finetune/Loss": 0.8642024397850037, "Finetune/Loss (Raw)": 0.9545273184776306, "Finetune/Step": 2608, "Finetune/Step Time": 4.645242195576429} +{"Finetune/Learning Rate": 1.0420374809811472e-05, "Finetune/Loss": 0.8684980869293213, "Finetune/Loss (Raw)": 0.9274883270263672, "Finetune/Step": 2609, "Finetune/Step Time": 4.604146007448435} +{"Finetune/Learning Rate": 1.0414150624750408e-05, "Finetune/Loss": 0.8667623996734619, "Finetune/Loss (Raw)": 0.8722602725028992, "Finetune/Step": 2610, "Finetune/Step Time": 4.620650852099061} +{"Finetune/Learning Rate": 1.0407926278965568e-05, "Finetune/Loss": 0.8652645349502563, "Finetune/Loss (Raw)": 0.71202152967453, "Finetune/Step": 2611, "Finetune/Step Time": 4.621469482779503} +{"Finetune/Learning Rate": 1.0401701774872508e-05, "Finetune/Loss": 0.8653708100318909, "Finetune/Loss (Raw)": 0.8859316110610962, "Finetune/Step": 2612, "Finetune/Step Time": 4.609310977160931} +{"Finetune/Learning Rate": 1.0395477114886827e-05, "Finetune/Loss": 0.8660536408424377, "Finetune/Loss (Raw)": 1.0168664455413818, "Finetune/Step": 2613, "Finetune/Step Time": 4.6015173234045506} +{"Finetune/Learning Rate": 1.0389252301424197e-05, "Finetune/Loss": 0.8661510944366455, "Finetune/Loss (Raw)": 0.8388302326202393, "Finetune/Step": 2614, "Finetune/Step Time": 4.600096445530653} +{"Finetune/Learning Rate": 1.0383027336900356e-05, "Finetune/Loss": 0.8711491823196411, "Finetune/Loss (Raw)": 0.9567273855209351, "Finetune/Step": 2615, "Finetune/Step Time": 4.570875022560358} +{"Finetune/Learning Rate": 1.0376802223731077e-05, "Finetune/Loss": 0.8705298900604248, "Finetune/Loss (Raw)": 0.9475404620170593, "Finetune/Step": 2616, "Finetune/Step Time": 4.5682644713670015} +{"Finetune/Learning Rate": 1.0370576964332217e-05, "Finetune/Loss": 0.8661914467811584, "Finetune/Loss (Raw)": 0.3975134789943695, "Finetune/Step": 2617, "Finetune/Step Time": 4.583639323711395} +{"Finetune/Learning Rate": 1.0364351561119667e-05, "Finetune/Loss": 0.8699312806129456, "Finetune/Loss (Raw)": 0.7995352149009705, "Finetune/Step": 2618, "Finetune/Step Time": 4.551547855138779} +{"Finetune/Learning Rate": 1.0358126016509392e-05, "Finetune/Loss": 0.8704378604888916, "Finetune/Loss (Raw)": 0.953057050704956, "Finetune/Step": 2619, "Finetune/Step Time": 4.550928559154272} +{"Finetune/Learning Rate": 1.0351900332917401e-05, "Finetune/Loss": 0.8707499504089355, "Finetune/Loss (Raw)": 0.8319966197013855, "Finetune/Step": 2620, "Finetune/Step Time": 4.546371800825} +{"Finetune/Learning Rate": 1.034567451275976e-05, "Finetune/Loss": 0.8700153231620789, "Finetune/Loss (Raw)": 0.8822572231292725, "Finetune/Step": 2621, "Finetune/Step Time": 4.543669940903783} +{"Finetune/Learning Rate": 1.0339448558452594e-05, "Finetune/Loss": 0.8694002032279968, "Finetune/Loss (Raw)": 0.9357137680053711, "Finetune/Step": 2622, "Finetune/Step Time": 4.54487519711256} +{"Finetune/Learning Rate": 1.0333222472412064e-05, "Finetune/Loss": 0.869184672832489, "Finetune/Loss (Raw)": 0.9054095149040222, "Finetune/Step": 2623, "Finetune/Step Time": 4.565154319629073} +{"Finetune/Learning Rate": 1.03269962570544e-05, "Finetune/Loss": 0.8691157102584839, "Finetune/Loss (Raw)": 0.9047654867172241, "Finetune/Step": 2624, "Finetune/Step Time": 4.55438619479537} +{"Finetune/Learning Rate": 1.0320769914795874e-05, "Finetune/Loss": 0.8687705397605896, "Finetune/Loss (Raw)": 0.758681058883667, "Finetune/Step": 2625, "Finetune/Step Time": 4.568261185660958} +{"Finetune/Learning Rate": 1.0314543448052806e-05, "Finetune/Loss": 0.8633425235748291, "Finetune/Loss (Raw)": 0.3119998574256897, "Finetune/Step": 2626, "Finetune/Step Time": 4.60743897408247} +{"Finetune/Learning Rate": 1.0308316859241567e-05, "Finetune/Loss": 0.8651544451713562, "Finetune/Loss (Raw)": 0.8810145258903503, "Finetune/Step": 2627, "Finetune/Step Time": 4.5853584837168455} +{"Finetune/Learning Rate": 1.0302090150778572e-05, "Finetune/Loss": 0.86551433801651, "Finetune/Loss (Raw)": 0.9813397526741028, "Finetune/Step": 2628, "Finetune/Step Time": 4.5850924123078585} +{"Finetune/Learning Rate": 1.029586332508029e-05, "Finetune/Loss": 0.8632002472877502, "Finetune/Loss (Raw)": 0.6386762857437134, "Finetune/Step": 2629, "Finetune/Step Time": 4.605595737695694} +{"Finetune/Learning Rate": 1.0289636384563226e-05, "Finetune/Loss": 0.8621042966842651, "Finetune/Loss (Raw)": 0.8062319159507751, "Finetune/Step": 2630, "Finetune/Step Time": 4.605841111391783} +{"Finetune/Learning Rate": 1.0283409331643936e-05, "Finetune/Loss": 0.8632584810256958, "Finetune/Loss (Raw)": 1.032924771308899, "Finetune/Step": 2631, "Finetune/Step Time": 4.602701239287853} +{"Finetune/Learning Rate": 1.0277182168739019e-05, "Finetune/Loss": 0.8624995946884155, "Finetune/Loss (Raw)": 0.7986211180686951, "Finetune/Step": 2632, "Finetune/Step Time": 4.597935579717159} +{"Finetune/Learning Rate": 1.0270954898265113e-05, "Finetune/Loss": 0.8628072142601013, "Finetune/Loss (Raw)": 0.9352587461471558, "Finetune/Step": 2633, "Finetune/Step Time": 4.599858732894063} +{"Finetune/Learning Rate": 1.0264727522638899e-05, "Finetune/Loss": 0.8635247349739075, "Finetune/Loss (Raw)": 0.9511315226554871, "Finetune/Step": 2634, "Finetune/Step Time": 4.601328646764159} +{"Finetune/Learning Rate": 1.0258500044277106e-05, "Finetune/Loss": 0.862762451171875, "Finetune/Loss (Raw)": 0.9150640368461609, "Finetune/Step": 2635, "Finetune/Step Time": 4.601080449298024} +{"Finetune/Learning Rate": 1.025227246559649e-05, "Finetune/Loss": 0.8640934228897095, "Finetune/Loss (Raw)": 1.0078260898590088, "Finetune/Step": 2636, "Finetune/Step Time": 4.600889569148421} +{"Finetune/Learning Rate": 1.0246044789013855e-05, "Finetune/Loss": 0.8639729022979736, "Finetune/Loss (Raw)": 0.8213379383087158, "Finetune/Step": 2637, "Finetune/Step Time": 4.605479506775737} +{"Finetune/Learning Rate": 1.023981701694604e-05, "Finetune/Loss": 0.8634339570999146, "Finetune/Loss (Raw)": 0.9724660515785217, "Finetune/Step": 2638, "Finetune/Step Time": 4.620828213170171} +{"Finetune/Learning Rate": 1.0233589151809927e-05, "Finetune/Loss": 0.8622018098831177, "Finetune/Loss (Raw)": 0.8241647481918335, "Finetune/Step": 2639, "Finetune/Step Time": 4.611811254173517} +{"Finetune/Learning Rate": 1.0227361196022421e-05, "Finetune/Loss": 0.8586331009864807, "Finetune/Loss (Raw)": 0.3490767180919647, "Finetune/Step": 2640, "Finetune/Step Time": 4.641884436830878} +{"Finetune/Learning Rate": 1.0221133152000474e-05, "Finetune/Loss": 0.8590399026870728, "Finetune/Loss (Raw)": 0.9009299874305725, "Finetune/Step": 2641, "Finetune/Step Time": 4.633515911176801} +{"Finetune/Learning Rate": 1.021490502216107e-05, "Finetune/Loss": 0.85807204246521, "Finetune/Loss (Raw)": 0.8902542591094971, "Finetune/Step": 2642, "Finetune/Step Time": 4.622875453904271} +{"Finetune/Learning Rate": 1.020867680892122e-05, "Finetune/Loss": 0.8577877879142761, "Finetune/Loss (Raw)": 0.7249217629432678, "Finetune/Step": 2643, "Finetune/Step Time": 4.620414922013879} +{"Finetune/Learning Rate": 1.0202448514697975e-05, "Finetune/Loss": 0.8564988374710083, "Finetune/Loss (Raw)": 0.7621800899505615, "Finetune/Step": 2644, "Finetune/Step Time": 4.653410388156772} +{"Finetune/Learning Rate": 1.0196220141908417e-05, "Finetune/Loss": 0.8564776182174683, "Finetune/Loss (Raw)": 0.9282169342041016, "Finetune/Step": 2645, "Finetune/Step Time": 4.656594121828675} +{"Finetune/Learning Rate": 1.018999169296965e-05, "Finetune/Loss": 0.8571481108665466, "Finetune/Loss (Raw)": 0.9398676753044128, "Finetune/Step": 2646, "Finetune/Step Time": 4.642653774470091} +{"Finetune/Learning Rate": 1.0183763170298818e-05, "Finetune/Loss": 0.8567345142364502, "Finetune/Loss (Raw)": 0.8209087252616882, "Finetune/Step": 2647, "Finetune/Step Time": 4.648206863552332} +{"Finetune/Learning Rate": 1.0177534576313084e-05, "Finetune/Loss": 0.8564432263374329, "Finetune/Loss (Raw)": 0.9406437277793884, "Finetune/Step": 2648, "Finetune/Step Time": 4.659757563844323} +{"Finetune/Learning Rate": 1.0171305913429648e-05, "Finetune/Loss": 0.8598734140396118, "Finetune/Loss (Raw)": 0.8434838056564331, "Finetune/Step": 2649, "Finetune/Step Time": 4.634338419884443} +{"Finetune/Learning Rate": 1.0165077184065732e-05, "Finetune/Loss": 0.8587838411331177, "Finetune/Loss (Raw)": 0.8081220388412476, "Finetune/Step": 2650, "Finetune/Step Time": 4.61700894497335} +{"Finetune/Learning Rate": 1.0158848390638588e-05, "Finetune/Loss": 0.8598678112030029, "Finetune/Loss (Raw)": 0.845860481262207, "Finetune/Step": 2651, "Finetune/Step Time": 4.612336652353406} +{"Finetune/Learning Rate": 1.015261953556548e-05, "Finetune/Loss": 0.8608745336532593, "Finetune/Loss (Raw)": 1.0071033239364624, "Finetune/Step": 2652, "Finetune/Step Time": 4.613305388018489} +{"Finetune/Learning Rate": 1.0146390621263709e-05, "Finetune/Loss": 0.8611316680908203, "Finetune/Loss (Raw)": 0.8796155452728271, "Finetune/Step": 2653, "Finetune/Step Time": 4.624872151762247} +{"Finetune/Learning Rate": 1.0140161650150598e-05, "Finetune/Loss": 0.8564906120300293, "Finetune/Loss (Raw)": 0.3428448438644409, "Finetune/Step": 2654, "Finetune/Step Time": 4.656625125557184} +{"Finetune/Learning Rate": 1.0133932624643487e-05, "Finetune/Loss": 0.8560107946395874, "Finetune/Loss (Raw)": 0.9156498908996582, "Finetune/Step": 2655, "Finetune/Step Time": 4.655625190585852} +{"Finetune/Learning Rate": 1.012770354715974e-05, "Finetune/Loss": 0.8552029728889465, "Finetune/Loss (Raw)": 0.8916376233100891, "Finetune/Step": 2656, "Finetune/Step Time": 4.652693707495928} +{"Finetune/Learning Rate": 1.012147442011674e-05, "Finetune/Loss": 0.8585551381111145, "Finetune/Loss (Raw)": 0.6999800801277161, "Finetune/Step": 2657, "Finetune/Step Time": 4.628480834886432} +{"Finetune/Learning Rate": 1.0115245245931883e-05, "Finetune/Loss": 0.8567848205566406, "Finetune/Loss (Raw)": 0.6983585357666016, "Finetune/Step": 2658, "Finetune/Step Time": 4.652225257828832} +{"Finetune/Learning Rate": 1.0109016027022602e-05, "Finetune/Loss": 0.8580695390701294, "Finetune/Loss (Raw)": 0.9960078597068787, "Finetune/Step": 2659, "Finetune/Step Time": 4.6435974556952715} +{"Finetune/Learning Rate": 1.0102786765806329e-05, "Finetune/Loss": 0.8576306104660034, "Finetune/Loss (Raw)": 0.9278750419616699, "Finetune/Step": 2660, "Finetune/Step Time": 4.627501120790839} +{"Finetune/Learning Rate": 1.0096557464700518e-05, "Finetune/Loss": 0.8593018054962158, "Finetune/Loss (Raw)": 1.1031335592269897, "Finetune/Step": 2661, "Finetune/Step Time": 4.631926693022251} +{"Finetune/Learning Rate": 1.0090328126122642e-05, "Finetune/Loss": 0.8642607927322388, "Finetune/Loss (Raw)": 1.0118980407714844, "Finetune/Step": 2662, "Finetune/Step Time": 4.590447347611189} +{"Finetune/Learning Rate": 1.0084098752490179e-05, "Finetune/Loss": 0.8639723062515259, "Finetune/Loss (Raw)": 0.9758773446083069, "Finetune/Step": 2663, "Finetune/Step Time": 4.597314767539501} +{"Finetune/Learning Rate": 1.0077869346220635e-05, "Finetune/Loss": 0.8616654872894287, "Finetune/Loss (Raw)": 0.44090285897254944, "Finetune/Step": 2664, "Finetune/Step Time": 4.618333026766777} +{"Finetune/Learning Rate": 1.0071639909731519e-05, "Finetune/Loss": 0.8607873916625977, "Finetune/Loss (Raw)": 0.7733649611473083, "Finetune/Step": 2665, "Finetune/Step Time": 4.611189346760511} +{"Finetune/Learning Rate": 1.0065410445440352e-05, "Finetune/Loss": 0.8605543971061707, "Finetune/Loss (Raw)": 0.9620410799980164, "Finetune/Step": 2666, "Finetune/Step Time": 4.607772331684828} +{"Finetune/Learning Rate": 1.0059180955764666e-05, "Finetune/Loss": 0.8598775267601013, "Finetune/Loss (Raw)": 0.7920660972595215, "Finetune/Step": 2667, "Finetune/Step Time": 4.596789060160518} +{"Finetune/Learning Rate": 1.005295144312201e-05, "Finetune/Loss": 0.8587433099746704, "Finetune/Loss (Raw)": 0.8553668856620789, "Finetune/Step": 2668, "Finetune/Step Time": 4.586612094193697} +{"Finetune/Learning Rate": 1.0046721909929928e-05, "Finetune/Loss": 0.8600008487701416, "Finetune/Loss (Raw)": 0.9999887347221375, "Finetune/Step": 2669, "Finetune/Step Time": 4.569941045716405} +{"Finetune/Learning Rate": 1.0040492358605989e-05, "Finetune/Loss": 0.8600665330886841, "Finetune/Loss (Raw)": 0.9219326972961426, "Finetune/Step": 2670, "Finetune/Step Time": 4.5649554915726185} +{"Finetune/Learning Rate": 1.0034262791567755e-05, "Finetune/Loss": 0.858880877494812, "Finetune/Loss (Raw)": 0.7691991925239563, "Finetune/Step": 2671, "Finetune/Step Time": 4.571113675832748} +{"Finetune/Learning Rate": 1.0028033211232802e-05, "Finetune/Loss": 0.8585550785064697, "Finetune/Loss (Raw)": 0.6900689005851746, "Finetune/Step": 2672, "Finetune/Step Time": 4.57592304982245} +{"Finetune/Learning Rate": 1.0021803620018706e-05, "Finetune/Loss": 0.8584967255592346, "Finetune/Loss (Raw)": 0.865445077419281, "Finetune/Step": 2673, "Finetune/Step Time": 4.575474290177226} +{"Finetune/Learning Rate": 1.0015574020343048e-05, "Finetune/Loss": 0.8627696633338928, "Finetune/Loss (Raw)": 0.827325165271759, "Finetune/Step": 2674, "Finetune/Step Time": 4.547279121354222} +{"Finetune/Learning Rate": 1.0009344414623418e-05, "Finetune/Loss": 0.8626245260238647, "Finetune/Loss (Raw)": 0.8580741286277771, "Finetune/Step": 2675, "Finetune/Step Time": 4.5578874088823795} +{"Finetune/Learning Rate": 1.0003114805277405e-05, "Finetune/Loss": 0.8610888719558716, "Finetune/Loss (Raw)": 0.8392213582992554, "Finetune/Step": 2676, "Finetune/Step Time": 4.560886424034834} +{"Finetune/Learning Rate": 9.9968851947226e-06, "Finetune/Loss": 0.8661693930625916, "Finetune/Loss (Raw)": 0.9763917326927185, "Finetune/Step": 2677, "Finetune/Step Time": 4.5271518006920815} +{"Finetune/Learning Rate": 9.990655585376583e-06, "Finetune/Loss": 0.865753173828125, "Finetune/Loss (Raw)": 0.8758723735809326, "Finetune/Step": 2678, "Finetune/Step Time": 4.520811639726162} +{"Finetune/Learning Rate": 9.984425979656955e-06, "Finetune/Loss": 0.8662768006324768, "Finetune/Loss (Raw)": 1.016239047050476, "Finetune/Step": 2679, "Finetune/Step Time": 4.5182864386588335} +{"Finetune/Learning Rate": 9.978196379981297e-06, "Finetune/Loss": 0.8649573922157288, "Finetune/Loss (Raw)": 0.9207513332366943, "Finetune/Step": 2680, "Finetune/Step Time": 4.520984586328268} +{"Finetune/Learning Rate": 9.971966788767198e-06, "Finetune/Loss": 0.8610488176345825, "Finetune/Loss (Raw)": 0.34305405616760254, "Finetune/Step": 2681, "Finetune/Step Time": 4.547063034027815} +{"Finetune/Learning Rate": 9.96573720843225e-06, "Finetune/Loss": 0.8606877326965332, "Finetune/Loss (Raw)": 0.8943993449211121, "Finetune/Step": 2682, "Finetune/Step Time": 4.523068966343999} +{"Finetune/Learning Rate": 9.959507641394015e-06, "Finetune/Loss": 0.8652042150497437, "Finetune/Loss (Raw)": 0.9065472483634949, "Finetune/Step": 2683, "Finetune/Step Time": 4.482739228755236} +{"Finetune/Learning Rate": 9.953278090070075e-06, "Finetune/Loss": 0.865091860294342, "Finetune/Loss (Raw)": 0.8791108727455139, "Finetune/Step": 2684, "Finetune/Step Time": 4.482282808050513} +{"Finetune/Learning Rate": 9.947048556877992e-06, "Finetune/Loss": 0.8662219047546387, "Finetune/Loss (Raw)": 0.9031727313995361, "Finetune/Step": 2685, "Finetune/Step Time": 4.485888062044978} +{"Finetune/Learning Rate": 9.940819044235335e-06, "Finetune/Loss": 0.8656497597694397, "Finetune/Loss (Raw)": 0.7952159643173218, "Finetune/Step": 2686, "Finetune/Step Time": 4.493791835382581} +{"Finetune/Learning Rate": 9.934589554559653e-06, "Finetune/Loss": 0.8655439615249634, "Finetune/Loss (Raw)": 0.8890323638916016, "Finetune/Step": 2687, "Finetune/Step Time": 4.495305435732007} +{"Finetune/Learning Rate": 9.928360090268484e-06, "Finetune/Loss": 0.8659285306930542, "Finetune/Loss (Raw)": 0.9191028475761414, "Finetune/Step": 2688, "Finetune/Step Time": 4.470363978296518} +{"Finetune/Learning Rate": 9.922130653779368e-06, "Finetune/Loss": 0.8669081926345825, "Finetune/Loss (Raw)": 1.0189322233200073, "Finetune/Step": 2689, "Finetune/Step Time": 4.467882188037038} +{"Finetune/Learning Rate": 9.915901247509821e-06, "Finetune/Loss": 0.861998975276947, "Finetune/Loss (Raw)": 0.3703209161758423, "Finetune/Step": 2690, "Finetune/Step Time": 4.507234785705805} +{"Finetune/Learning Rate": 9.909671873877363e-06, "Finetune/Loss": 0.8653051257133484, "Finetune/Loss (Raw)": 1.042975664138794, "Finetune/Step": 2691, "Finetune/Step Time": 4.500050291419029} +{"Finetune/Learning Rate": 9.903442535299487e-06, "Finetune/Loss": 0.8647245168685913, "Finetune/Loss (Raw)": 0.8477296829223633, "Finetune/Step": 2692, "Finetune/Step Time": 4.499303411692381} +{"Finetune/Learning Rate": 9.897213234193673e-06, "Finetune/Loss": 0.8645851016044617, "Finetune/Loss (Raw)": 0.9533591866493225, "Finetune/Step": 2693, "Finetune/Step Time": 4.4970198180526495} +{"Finetune/Learning Rate": 9.890983972977401e-06, "Finetune/Loss": 0.8642905950546265, "Finetune/Loss (Raw)": 0.8879645466804504, "Finetune/Step": 2694, "Finetune/Step Time": 4.498681524768472} +{"Finetune/Learning Rate": 9.884754754068116e-06, "Finetune/Loss": 0.864681601524353, "Finetune/Loss (Raw)": 0.8482474088668823, "Finetune/Step": 2695, "Finetune/Step Time": 4.497189376503229} +{"Finetune/Learning Rate": 9.878525579883264e-06, "Finetune/Loss": 0.8659694790840149, "Finetune/Loss (Raw)": 0.9626121520996094, "Finetune/Step": 2696, "Finetune/Step Time": 4.508412444964051} +{"Finetune/Learning Rate": 9.872296452840266e-06, "Finetune/Loss": 0.8639717102050781, "Finetune/Loss (Raw)": 0.8629617691040039, "Finetune/Step": 2697, "Finetune/Step Time": 4.510902950540185} +{"Finetune/Learning Rate": 9.866067375356516e-06, "Finetune/Loss": 0.8632033467292786, "Finetune/Loss (Raw)": 0.8193585276603699, "Finetune/Step": 2698, "Finetune/Step Time": 4.505787266418338} +{"Finetune/Learning Rate": 9.859838349849404e-06, "Finetune/Loss": 0.8626439571380615, "Finetune/Loss (Raw)": 0.9906635880470276, "Finetune/Step": 2699, "Finetune/Step Time": 4.525053272023797} +{"Finetune/Learning Rate": 9.853609378736293e-06, "Finetune/Loss": 0.8624913096427917, "Finetune/Loss (Raw)": 0.8579174280166626, "Finetune/Step": 2700, "Finetune/Step Time": 4.518911590799689} +{"Finetune/Learning Rate": 9.847380464434525e-06, "Finetune/Loss": 0.8623592853546143, "Finetune/Loss (Raw)": 0.8869587182998657, "Finetune/Step": 2701, "Finetune/Step Time": 4.541245678439736} +{"Finetune/Learning Rate": 9.841151609361419e-06, "Finetune/Loss": 0.8626172542572021, "Finetune/Loss (Raw)": 0.9101639986038208, "Finetune/Step": 2702, "Finetune/Step Time": 4.535388147458434} +{"Finetune/Learning Rate": 9.83492281593427e-06, "Finetune/Loss": 0.8626617193222046, "Finetune/Loss (Raw)": 0.8265359997749329, "Finetune/Step": 2703, "Finetune/Step Time": 4.552463607862592} +{"Finetune/Learning Rate": 9.828694086570353e-06, "Finetune/Loss": 0.8615845441818237, "Finetune/Loss (Raw)": 0.7978506088256836, "Finetune/Step": 2704, "Finetune/Step Time": 4.564544301480055} +{"Finetune/Learning Rate": 9.822465423686917e-06, "Finetune/Loss": 0.860740065574646, "Finetune/Loss (Raw)": 0.8704642653465271, "Finetune/Step": 2705, "Finetune/Step Time": 4.564598696306348} +{"Finetune/Learning Rate": 9.816236829701187e-06, "Finetune/Loss": 0.8604689240455627, "Finetune/Loss (Raw)": 0.9776868224143982, "Finetune/Step": 2706, "Finetune/Step Time": 4.571009054780006} +{"Finetune/Learning Rate": 9.810008307030356e-06, "Finetune/Loss": 0.8604497909545898, "Finetune/Loss (Raw)": 0.9990919232368469, "Finetune/Step": 2707, "Finetune/Step Time": 4.567324250936508} +{"Finetune/Learning Rate": 9.803779858091586e-06, "Finetune/Loss": 0.8628227710723877, "Finetune/Loss (Raw)": 1.0005899667739868, "Finetune/Step": 2708, "Finetune/Step Time": 4.564493555575609} +{"Finetune/Learning Rate": 9.797551485302027e-06, "Finetune/Loss": 0.8614497184753418, "Finetune/Loss (Raw)": 0.8625427484512329, "Finetune/Step": 2709, "Finetune/Step Time": 4.574766894802451} +{"Finetune/Learning Rate": 9.791323191078781e-06, "Finetune/Loss": 0.8575853109359741, "Finetune/Loss (Raw)": 0.3814545273780823, "Finetune/Step": 2710, "Finetune/Step Time": 4.61511729285121} +{"Finetune/Learning Rate": 9.785094977838935e-06, "Finetune/Loss": 0.8586571216583252, "Finetune/Loss (Raw)": 0.9539110064506531, "Finetune/Step": 2711, "Finetune/Step Time": 4.605700712651014} +{"Finetune/Learning Rate": 9.77886684799953e-06, "Finetune/Loss": 0.8577948808670044, "Finetune/Loss (Raw)": 0.9247023463249207, "Finetune/Step": 2712, "Finetune/Step Time": 4.624373944476247} +{"Finetune/Learning Rate": 9.77263880397758e-06, "Finetune/Loss": 0.8583990335464478, "Finetune/Loss (Raw)": 0.8800721168518066, "Finetune/Step": 2713, "Finetune/Step Time": 4.631291942670941} +{"Finetune/Learning Rate": 9.766410848190076e-06, "Finetune/Loss": 0.858285665512085, "Finetune/Loss (Raw)": 0.9483751654624939, "Finetune/Step": 2714, "Finetune/Step Time": 4.63097021356225} +{"Finetune/Learning Rate": 9.76018298305396e-06, "Finetune/Loss": 0.8573188781738281, "Finetune/Loss (Raw)": 0.8250098824501038, "Finetune/Step": 2715, "Finetune/Step Time": 4.618860041722655} +{"Finetune/Learning Rate": 9.75395521098615e-06, "Finetune/Loss": 0.8572994470596313, "Finetune/Loss (Raw)": 0.8531107306480408, "Finetune/Step": 2716, "Finetune/Step Time": 4.61159279756248} +{"Finetune/Learning Rate": 9.747727534403515e-06, "Finetune/Loss": 0.8566141724586487, "Finetune/Loss (Raw)": 0.8960034847259521, "Finetune/Step": 2717, "Finetune/Step Time": 4.593622313812375} +{"Finetune/Learning Rate": 9.741499955722897e-06, "Finetune/Loss": 0.8561462163925171, "Finetune/Loss (Raw)": 0.8674646615982056, "Finetune/Step": 2718, "Finetune/Step Time": 4.593319756910205} +{"Finetune/Learning Rate": 9.735272477361103e-06, "Finetune/Loss": 0.8570693731307983, "Finetune/Loss (Raw)": 0.9298904538154602, "Finetune/Step": 2719, "Finetune/Step Time": 4.555527627468109} +{"Finetune/Learning Rate": 9.729045101734889e-06, "Finetune/Loss": 0.8572707176208496, "Finetune/Loss (Raw)": 0.9383140802383423, "Finetune/Step": 2720, "Finetune/Step Time": 4.551009280607104} +{"Finetune/Learning Rate": 9.722817831260985e-06, "Finetune/Loss": 0.8574810028076172, "Finetune/Loss (Raw)": 0.8550414443016052, "Finetune/Step": 2721, "Finetune/Step Time": 4.54151975736022} +{"Finetune/Learning Rate": 9.716590668356067e-06, "Finetune/Loss": 0.8579946756362915, "Finetune/Loss (Raw)": 0.8809404969215393, "Finetune/Step": 2722, "Finetune/Step Time": 4.531878592446446} +{"Finetune/Learning Rate": 9.710363615436777e-06, "Finetune/Loss": 0.8577008843421936, "Finetune/Loss (Raw)": 0.8490778207778931, "Finetune/Step": 2723, "Finetune/Step Time": 4.532517604529858} +{"Finetune/Learning Rate": 9.704136674919712e-06, "Finetune/Loss": 0.8581265211105347, "Finetune/Loss (Raw)": 0.916779637336731, "Finetune/Step": 2724, "Finetune/Step Time": 4.535459907725453} +{"Finetune/Learning Rate": 9.697909849221428e-06, "Finetune/Loss": 0.8577159643173218, "Finetune/Loss (Raw)": 0.907041609287262, "Finetune/Step": 2725, "Finetune/Step Time": 4.528490629047155} +{"Finetune/Learning Rate": 9.691683140758439e-06, "Finetune/Loss": 0.8591806888580322, "Finetune/Loss (Raw)": 0.9104393720626831, "Finetune/Step": 2726, "Finetune/Step Time": 4.525595998391509} +{"Finetune/Learning Rate": 9.685456551947199e-06, "Finetune/Loss": 0.8586015701293945, "Finetune/Loss (Raw)": 0.8814367651939392, "Finetune/Step": 2727, "Finetune/Step Time": 4.528314849361777} +{"Finetune/Learning Rate": 9.67923008520413e-06, "Finetune/Loss": 0.8577384948730469, "Finetune/Loss (Raw)": 0.7761942744255066, "Finetune/Step": 2728, "Finetune/Step Time": 4.5266395304352045} +{"Finetune/Learning Rate": 9.673003742945602e-06, "Finetune/Loss": 0.8539459705352783, "Finetune/Loss (Raw)": 0.4198355972766876, "Finetune/Step": 2729, "Finetune/Step Time": 4.5556593760848045} +{"Finetune/Learning Rate": 9.666777527587936e-06, "Finetune/Loss": 0.8519700765609741, "Finetune/Loss (Raw)": 0.7699921727180481, "Finetune/Step": 2730, "Finetune/Step Time": 4.557750776410103} +{"Finetune/Learning Rate": 9.660551441547411e-06, "Finetune/Loss": 0.8513028621673584, "Finetune/Loss (Raw)": 0.8942201733589172, "Finetune/Step": 2731, "Finetune/Step Time": 4.554620612412691} +{"Finetune/Learning Rate": 9.654325487240243e-06, "Finetune/Loss": 0.8517464399337769, "Finetune/Loss (Raw)": 0.9324937462806702, "Finetune/Step": 2732, "Finetune/Step Time": 4.549898220226169} +{"Finetune/Learning Rate": 9.6480996670826e-06, "Finetune/Loss": 0.8525098562240601, "Finetune/Loss (Raw)": 0.8879553079605103, "Finetune/Step": 2733, "Finetune/Step Time": 4.544601008296013} +{"Finetune/Learning Rate": 9.64187398349061e-06, "Finetune/Loss": 0.8534070253372192, "Finetune/Loss (Raw)": 0.9470990300178528, "Finetune/Step": 2734, "Finetune/Step Time": 4.521213386207819} +{"Finetune/Learning Rate": 9.635648438880334e-06, "Finetune/Loss": 0.8535537719726562, "Finetune/Loss (Raw)": 0.955578625202179, "Finetune/Step": 2735, "Finetune/Step Time": 4.493819320574403} +{"Finetune/Learning Rate": 9.629423035667788e-06, "Finetune/Loss": 0.8496834635734558, "Finetune/Loss (Raw)": 0.4591362774372101, "Finetune/Step": 2736, "Finetune/Step Time": 4.536599799990654} +{"Finetune/Learning Rate": 9.623197776268926e-06, "Finetune/Loss": 0.8470542430877686, "Finetune/Loss (Raw)": 0.5909445881843567, "Finetune/Step": 2737, "Finetune/Step Time": 4.541346862912178} +{"Finetune/Learning Rate": 9.616972663099648e-06, "Finetune/Loss": 0.8474075794219971, "Finetune/Loss (Raw)": 0.917490541934967, "Finetune/Step": 2738, "Finetune/Step Time": 4.530756928026676} +{"Finetune/Learning Rate": 9.610747698575801e-06, "Finetune/Loss": 0.8491756916046143, "Finetune/Loss (Raw)": 0.9383382797241211, "Finetune/Step": 2739, "Finetune/Step Time": 4.522395497187972} +{"Finetune/Learning Rate": 9.60452288511318e-06, "Finetune/Loss": 0.8487575650215149, "Finetune/Loss (Raw)": 0.8324052095413208, "Finetune/Step": 2740, "Finetune/Step Time": 4.531780008226633} +{"Finetune/Learning Rate": 9.598298225127499e-06, "Finetune/Loss": 0.848581075668335, "Finetune/Loss (Raw)": 0.994286060333252, "Finetune/Step": 2741, "Finetune/Step Time": 4.537503965198994} +{"Finetune/Learning Rate": 9.592073721034435e-06, "Finetune/Loss": 0.849098265171051, "Finetune/Loss (Raw)": 0.9050202369689941, "Finetune/Step": 2742, "Finetune/Step Time": 4.536784280091524} +{"Finetune/Learning Rate": 9.585849375249595e-06, "Finetune/Loss": 0.849065363407135, "Finetune/Loss (Raw)": 0.9525210857391357, "Finetune/Step": 2743, "Finetune/Step Time": 4.529248969629407} +{"Finetune/Learning Rate": 9.57962519018853e-06, "Finetune/Loss": 0.848858654499054, "Finetune/Loss (Raw)": 0.9210807085037231, "Finetune/Step": 2744, "Finetune/Step Time": 4.532066551968455} +{"Finetune/Learning Rate": 9.573401168266722e-06, "Finetune/Loss": 0.8534095287322998, "Finetune/Loss (Raw)": 0.9800304770469666, "Finetune/Step": 2745, "Finetune/Step Time": 4.493551438674331} +{"Finetune/Learning Rate": 9.567177311899589e-06, "Finetune/Loss": 0.8540750741958618, "Finetune/Loss (Raw)": 0.8847169876098633, "Finetune/Step": 2746, "Finetune/Step Time": 4.500247301533818} +{"Finetune/Learning Rate": 9.560953623502493e-06, "Finetune/Loss": 0.8534553050994873, "Finetune/Loss (Raw)": 0.8737272620201111, "Finetune/Step": 2747, "Finetune/Step Time": 4.511465139687061} +{"Finetune/Learning Rate": 9.554730105490722e-06, "Finetune/Loss": 0.854865312576294, "Finetune/Loss (Raw)": 1.0124750137329102, "Finetune/Step": 2748, "Finetune/Step Time": 4.513677749782801} +{"Finetune/Learning Rate": 9.548506760279506e-06, "Finetune/Loss": 0.8553334474563599, "Finetune/Loss (Raw)": 0.942182719707489, "Finetune/Step": 2749, "Finetune/Step Time": 4.509885726496577} +{"Finetune/Learning Rate": 9.542283590284003e-06, "Finetune/Loss": 0.8546960949897766, "Finetune/Loss (Raw)": 0.8541287779808044, "Finetune/Step": 2750, "Finetune/Step Time": 4.507833791896701} +{"Finetune/Learning Rate": 9.5360605979193e-06, "Finetune/Loss": 0.8548001050949097, "Finetune/Loss (Raw)": 0.9187292456626892, "Finetune/Step": 2751, "Finetune/Step Time": 4.496977621689439} +{"Finetune/Learning Rate": 9.52983778560042e-06, "Finetune/Loss": 0.8551241159439087, "Finetune/Loss (Raw)": 0.9462428689002991, "Finetune/Step": 2752, "Finetune/Step Time": 4.496190689504147} +{"Finetune/Learning Rate": 9.523615155742317e-06, "Finetune/Loss": 0.8568764328956604, "Finetune/Loss (Raw)": 0.9829725623130798, "Finetune/Step": 2753, "Finetune/Step Time": 4.482312232255936} +{"Finetune/Learning Rate": 9.517392710759872e-06, "Finetune/Loss": 0.8617562055587769, "Finetune/Loss (Raw)": 0.9366059899330139, "Finetune/Step": 2754, "Finetune/Step Time": 4.446127390488982} +{"Finetune/Learning Rate": 9.511170453067899e-06, "Finetune/Loss": 0.8576568365097046, "Finetune/Loss (Raw)": 0.3562934398651123, "Finetune/Step": 2755, "Finetune/Step Time": 4.488152764737606} +{"Finetune/Learning Rate": 9.504948385081121e-06, "Finetune/Loss": 0.8575405478477478, "Finetune/Loss (Raw)": 0.9664544463157654, "Finetune/Step": 2756, "Finetune/Step Time": 4.504576146602631} +{"Finetune/Learning Rate": 9.498726509214216e-06, "Finetune/Loss": 0.8586035966873169, "Finetune/Loss (Raw)": 0.7747482061386108, "Finetune/Step": 2757, "Finetune/Step Time": 4.515754196792841} +{"Finetune/Learning Rate": 9.492504827881761e-06, "Finetune/Loss": 0.8596837520599365, "Finetune/Loss (Raw)": 0.9444879293441772, "Finetune/Step": 2758, "Finetune/Step Time": 4.515788126736879} +{"Finetune/Learning Rate": 9.486283343498277e-06, "Finetune/Loss": 0.8580742478370667, "Finetune/Loss (Raw)": 0.8269204497337341, "Finetune/Step": 2759, "Finetune/Step Time": 4.516437001526356} +{"Finetune/Learning Rate": 9.480062058478201e-06, "Finetune/Loss": 0.8576292991638184, "Finetune/Loss (Raw)": 0.7416685819625854, "Finetune/Step": 2760, "Finetune/Step Time": 4.515749344602227} +{"Finetune/Learning Rate": 9.473840975235883e-06, "Finetune/Loss": 0.8536106944084167, "Finetune/Loss (Raw)": 0.4208718538284302, "Finetune/Step": 2761, "Finetune/Step Time": 4.555167580023408} +{"Finetune/Learning Rate": 9.467620096185616e-06, "Finetune/Loss": 0.8527748584747314, "Finetune/Loss (Raw)": 0.8441387414932251, "Finetune/Step": 2762, "Finetune/Step Time": 4.5639387387782335} +{"Finetune/Learning Rate": 9.46139942374159e-06, "Finetune/Loss": 0.8525387048721313, "Finetune/Loss (Raw)": 0.8848367929458618, "Finetune/Step": 2763, "Finetune/Step Time": 4.561467969790101} +{"Finetune/Learning Rate": 9.455178960317936e-06, "Finetune/Loss": 0.8514014482498169, "Finetune/Loss (Raw)": 0.8622593283653259, "Finetune/Step": 2764, "Finetune/Step Time": 4.561819970607758} +{"Finetune/Learning Rate": 9.448958708328694e-06, "Finetune/Loss": 0.8477449417114258, "Finetune/Loss (Raw)": 0.35331177711486816, "Finetune/Step": 2765, "Finetune/Step Time": 4.595209749415517} +{"Finetune/Learning Rate": 9.442738670187814e-06, "Finetune/Loss": 0.8468756675720215, "Finetune/Loss (Raw)": 0.8611932396888733, "Finetune/Step": 2766, "Finetune/Step Time": 4.585887875407934} +{"Finetune/Learning Rate": 9.436518848309178e-06, "Finetune/Loss": 0.8481292724609375, "Finetune/Loss (Raw)": 0.9846283793449402, "Finetune/Step": 2767, "Finetune/Step Time": 4.599045975133777} +{"Finetune/Learning Rate": 9.430299245106574e-06, "Finetune/Loss": 0.8528829216957092, "Finetune/Loss (Raw)": 0.9575445055961609, "Finetune/Step": 2768, "Finetune/Step Time": 4.560944713652134} +{"Finetune/Learning Rate": 9.424079862993712e-06, "Finetune/Loss": 0.8530874252319336, "Finetune/Loss (Raw)": 0.9271091818809509, "Finetune/Step": 2769, "Finetune/Step Time": 4.575272848829627} +{"Finetune/Learning Rate": 9.417860704384214e-06, "Finetune/Loss": 0.8513957262039185, "Finetune/Loss (Raw)": 0.6737163662910461, "Finetune/Step": 2770, "Finetune/Step Time": 4.592022841796279} +{"Finetune/Learning Rate": 9.411641771691608e-06, "Finetune/Loss": 0.852781355381012, "Finetune/Loss (Raw)": 0.9022790193557739, "Finetune/Step": 2771, "Finetune/Step Time": 4.584829531610012} +{"Finetune/Learning Rate": 9.405423067329344e-06, "Finetune/Loss": 0.8539736270904541, "Finetune/Loss (Raw)": 0.914795458316803, "Finetune/Step": 2772, "Finetune/Step Time": 4.554203974083066} +{"Finetune/Learning Rate": 9.399204593710777e-06, "Finetune/Loss": 0.8541765213012695, "Finetune/Loss (Raw)": 0.9541853070259094, "Finetune/Step": 2773, "Finetune/Step Time": 4.5470487251877785} +{"Finetune/Learning Rate": 9.392986353249183e-06, "Finetune/Loss": 0.8494200706481934, "Finetune/Loss (Raw)": 0.33103907108306885, "Finetune/Step": 2774, "Finetune/Step Time": 4.5861719492822886} +{"Finetune/Learning Rate": 9.386768348357735e-06, "Finetune/Loss": 0.850197434425354, "Finetune/Loss (Raw)": 0.9204172492027283, "Finetune/Step": 2775, "Finetune/Step Time": 4.576050762087107} +{"Finetune/Learning Rate": 9.380550581449512e-06, "Finetune/Loss": 0.8491187691688538, "Finetune/Loss (Raw)": 0.8025723099708557, "Finetune/Step": 2776, "Finetune/Step Time": 4.568104857578874} +{"Finetune/Learning Rate": 9.37433305493752e-06, "Finetune/Loss": 0.849186897277832, "Finetune/Loss (Raw)": 0.8522012233734131, "Finetune/Step": 2777, "Finetune/Step Time": 4.554692644625902} +{"Finetune/Learning Rate": 9.368115771234651e-06, "Finetune/Loss": 0.8498693704605103, "Finetune/Loss (Raw)": 0.8954834342002869, "Finetune/Step": 2778, "Finetune/Step Time": 4.555446436628699} +{"Finetune/Learning Rate": 9.361898732753715e-06, "Finetune/Loss": 0.8498269319534302, "Finetune/Loss (Raw)": 0.8404304385185242, "Finetune/Step": 2779, "Finetune/Step Time": 4.561336247250438} +{"Finetune/Learning Rate": 9.355681941907428e-06, "Finetune/Loss": 0.848895788192749, "Finetune/Loss (Raw)": 0.8879116773605347, "Finetune/Step": 2780, "Finetune/Step Time": 4.55996941588819} +{"Finetune/Learning Rate": 9.349465401108393e-06, "Finetune/Loss": 0.849786102771759, "Finetune/Loss (Raw)": 0.993571937084198, "Finetune/Step": 2781, "Finetune/Step Time": 4.5447283536195755} +{"Finetune/Learning Rate": 9.34324911276914e-06, "Finetune/Loss": 0.853835940361023, "Finetune/Loss (Raw)": 0.8612316250801086, "Finetune/Step": 2782, "Finetune/Step Time": 4.5076936315745115} +{"Finetune/Learning Rate": 9.337033079302082e-06, "Finetune/Loss": 0.8547789454460144, "Finetune/Loss (Raw)": 1.036347508430481, "Finetune/Step": 2783, "Finetune/Step Time": 4.511312177404761} +{"Finetune/Learning Rate": 9.330817303119545e-06, "Finetune/Loss": 0.8533075451850891, "Finetune/Loss (Raw)": 0.7033010125160217, "Finetune/Step": 2784, "Finetune/Step Time": 4.523610819131136} +{"Finetune/Learning Rate": 9.324601786633747e-06, "Finetune/Loss": 0.8545734882354736, "Finetune/Loss (Raw)": 0.8620201945304871, "Finetune/Step": 2785, "Finetune/Step Time": 4.518405046314001} +{"Finetune/Learning Rate": 9.318386532256807e-06, "Finetune/Loss": 0.8566226959228516, "Finetune/Loss (Raw)": 0.9606544375419617, "Finetune/Step": 2786, "Finetune/Step Time": 4.490616213530302} +{"Finetune/Learning Rate": 9.312171542400749e-06, "Finetune/Loss": 0.8553294539451599, "Finetune/Loss (Raw)": 0.8304742574691772, "Finetune/Step": 2787, "Finetune/Step Time": 4.490915365517139} +{"Finetune/Learning Rate": 9.305956819477485e-06, "Finetune/Loss": 0.8554049730300903, "Finetune/Loss (Raw)": 0.9375441074371338, "Finetune/Step": 2788, "Finetune/Step Time": 4.500829121097922} +{"Finetune/Learning Rate": 9.299742365898831e-06, "Finetune/Loss": 0.8534878492355347, "Finetune/Loss (Raw)": 0.8577390909194946, "Finetune/Step": 2789, "Finetune/Step Time": 4.513345573097467} +{"Finetune/Learning Rate": 9.293528184076495e-06, "Finetune/Loss": 0.8526623249053955, "Finetune/Loss (Raw)": 0.9062283039093018, "Finetune/Step": 2790, "Finetune/Step Time": 4.516777694225311} +{"Finetune/Learning Rate": 9.287314276422074e-06, "Finetune/Loss": 0.8507702350616455, "Finetune/Loss (Raw)": 0.7336923480033875, "Finetune/Step": 2791, "Finetune/Step Time": 4.526887767016888} +{"Finetune/Learning Rate": 9.28110064534707e-06, "Finetune/Loss": 0.8549515008926392, "Finetune/Loss (Raw)": 0.976108729839325, "Finetune/Step": 2792, "Finetune/Step Time": 4.502353131771088} +{"Finetune/Learning Rate": 9.274887293262869e-06, "Finetune/Loss": 0.8566150665283203, "Finetune/Loss (Raw)": 0.9863042235374451, "Finetune/Step": 2793, "Finetune/Step Time": 4.500528160482645} +{"Finetune/Learning Rate": 9.268674222580757e-06, "Finetune/Loss": 0.8555128574371338, "Finetune/Loss (Raw)": 0.8209578990936279, "Finetune/Step": 2794, "Finetune/Step Time": 4.508726347237825} +{"Finetune/Learning Rate": 9.262461435711898e-06, "Finetune/Loss": 0.8564119338989258, "Finetune/Loss (Raw)": 0.907151460647583, "Finetune/Step": 2795, "Finetune/Step Time": 4.5195945631712675} +{"Finetune/Learning Rate": 9.256248935067356e-06, "Finetune/Loss": 0.8564494848251343, "Finetune/Loss (Raw)": 0.8601700067520142, "Finetune/Step": 2796, "Finetune/Step Time": 4.536075035110116} +{"Finetune/Learning Rate": 9.250036723058083e-06, "Finetune/Loss": 0.8545647859573364, "Finetune/Loss (Raw)": 0.7587425112724304, "Finetune/Step": 2797, "Finetune/Step Time": 4.548643756657839} +{"Finetune/Learning Rate": 9.243824802094914e-06, "Finetune/Loss": 0.8503340482711792, "Finetune/Loss (Raw)": 0.3803982138633728, "Finetune/Step": 2798, "Finetune/Step Time": 4.592821417376399} +{"Finetune/Learning Rate": 9.237613174588583e-06, "Finetune/Loss": 0.8501132130622864, "Finetune/Loss (Raw)": 0.7409313321113586, "Finetune/Step": 2799, "Finetune/Step Time": 4.590828727930784} +{"Finetune/Learning Rate": 9.231401842949688e-06, "Finetune/Loss": 0.8504830598831177, "Finetune/Loss (Raw)": 0.7374105453491211, "Finetune/Step": 2800, "Finetune/Step Time": 4.585308773443103} +{"Finetune/Learning Rate": 9.225190809588731e-06, "Finetune/Loss": 0.8513456583023071, "Finetune/Loss (Raw)": 0.9758583307266235, "Finetune/Step": 2801, "Finetune/Step Time": 4.583960672840476} +{"Finetune/Learning Rate": 9.218980076916099e-06, "Finetune/Loss": 0.8524961471557617, "Finetune/Loss (Raw)": 0.9745839238166809, "Finetune/Step": 2802, "Finetune/Step Time": 4.589243058115244} +{"Finetune/Learning Rate": 9.212769647342046e-06, "Finetune/Loss": 0.8530217409133911, "Finetune/Loss (Raw)": 0.9253495335578918, "Finetune/Step": 2803, "Finetune/Step Time": 4.582538571208715} +{"Finetune/Learning Rate": 9.206559523276731e-06, "Finetune/Loss": 0.8536235094070435, "Finetune/Loss (Raw)": 0.9162447452545166, "Finetune/Step": 2804, "Finetune/Step Time": 4.5771037470549345} +{"Finetune/Learning Rate": 9.200349707130168e-06, "Finetune/Loss": 0.8540050983428955, "Finetune/Loss (Raw)": 1.025246500968933, "Finetune/Step": 2805, "Finetune/Step Time": 4.568929681554437} +{"Finetune/Learning Rate": 9.194140201312274e-06, "Finetune/Loss": 0.8536744713783264, "Finetune/Loss (Raw)": 0.8335472941398621, "Finetune/Step": 2806, "Finetune/Step Time": 4.575187167152762} +{"Finetune/Learning Rate": 9.187931008232832e-06, "Finetune/Loss": 0.8523108959197998, "Finetune/Loss (Raw)": 0.841708242893219, "Finetune/Step": 2807, "Finetune/Step Time": 4.573341703042388} +{"Finetune/Learning Rate": 9.181722130301512e-06, "Finetune/Loss": 0.8510141372680664, "Finetune/Loss (Raw)": 0.7547602653503418, "Finetune/Step": 2808, "Finetune/Step Time": 4.5838008020073175} +{"Finetune/Learning Rate": 9.175513569927865e-06, "Finetune/Loss": 0.8510851860046387, "Finetune/Loss (Raw)": 0.3521457314491272, "Finetune/Step": 2809, "Finetune/Step Time": 4.584133757278323} +{"Finetune/Learning Rate": 9.169305329521296e-06, "Finetune/Loss": 0.8500584363937378, "Finetune/Loss (Raw)": 0.7629772424697876, "Finetune/Step": 2810, "Finetune/Step Time": 4.588688302785158} +{"Finetune/Learning Rate": 9.163097411491115e-06, "Finetune/Loss": 0.8501128554344177, "Finetune/Loss (Raw)": 0.9135122895240784, "Finetune/Step": 2811, "Finetune/Step Time": 4.597919017076492} +{"Finetune/Learning Rate": 9.156889818246485e-06, "Finetune/Loss": 0.8500667810440063, "Finetune/Loss (Raw)": 0.8732160329818726, "Finetune/Step": 2812, "Finetune/Step Time": 4.597594788298011} +{"Finetune/Learning Rate": 9.150682552196461e-06, "Finetune/Loss": 0.8507722616195679, "Finetune/Loss (Raw)": 0.9934759736061096, "Finetune/Step": 2813, "Finetune/Step Time": 4.596735943108797} +{"Finetune/Learning Rate": 9.144475615749957e-06, "Finetune/Loss": 0.8473248481750488, "Finetune/Loss (Raw)": 0.3539383113384247, "Finetune/Step": 2814, "Finetune/Step Time": 4.629899840801954} +{"Finetune/Learning Rate": 9.138269011315761e-06, "Finetune/Loss": 0.8483166098594666, "Finetune/Loss (Raw)": 1.0159810781478882, "Finetune/Step": 2815, "Finetune/Step Time": 4.6322971396148205} +{"Finetune/Learning Rate": 9.13206274130254e-06, "Finetune/Loss": 0.8483561277389526, "Finetune/Loss (Raw)": 0.9241626262664795, "Finetune/Step": 2816, "Finetune/Step Time": 4.631009569391608} +{"Finetune/Learning Rate": 9.125856808118822e-06, "Finetune/Loss": 0.847079873085022, "Finetune/Loss (Raw)": 0.8555775880813599, "Finetune/Step": 2817, "Finetune/Step Time": 4.6380712650716305} +{"Finetune/Learning Rate": 9.119651214173012e-06, "Finetune/Loss": 0.8507075309753418, "Finetune/Loss (Raw)": 0.8346555233001709, "Finetune/Step": 2818, "Finetune/Step Time": 4.604613859206438} +{"Finetune/Learning Rate": 9.11344596187338e-06, "Finetune/Loss": 0.850659966468811, "Finetune/Loss (Raw)": 1.036885142326355, "Finetune/Step": 2819, "Finetune/Step Time": 4.613571785390377} +{"Finetune/Learning Rate": 9.107241053628058e-06, "Finetune/Loss": 0.8518615961074829, "Finetune/Loss (Raw)": 1.001537799835205, "Finetune/Step": 2820, "Finetune/Step Time": 4.613223360851407} +{"Finetune/Learning Rate": 9.101036491845056e-06, "Finetune/Loss": 0.8500317335128784, "Finetune/Loss (Raw)": 0.7191365957260132, "Finetune/Step": 2821, "Finetune/Step Time": 4.631420578807592} +{"Finetune/Learning Rate": 9.094832278932238e-06, "Finetune/Loss": 0.8501848578453064, "Finetune/Loss (Raw)": 0.9075688719749451, "Finetune/Step": 2822, "Finetune/Step Time": 4.621643194928765} +{"Finetune/Learning Rate": 9.088628417297346e-06, "Finetune/Loss": 0.846712589263916, "Finetune/Loss (Raw)": 0.40379947423934937, "Finetune/Step": 2823, "Finetune/Step Time": 4.655377270653844} +{"Finetune/Learning Rate": 9.082424909347969e-06, "Finetune/Loss": 0.8453966379165649, "Finetune/Loss (Raw)": 0.794165313243866, "Finetune/Step": 2824, "Finetune/Step Time": 4.641807001084089} +{"Finetune/Learning Rate": 9.076221757491568e-06, "Finetune/Loss": 0.8449004888534546, "Finetune/Loss (Raw)": 0.7994486689567566, "Finetune/Step": 2825, "Finetune/Step Time": 4.640837520360947} +{"Finetune/Learning Rate": 9.070018964135471e-06, "Finetune/Loss": 0.8445385694503784, "Finetune/Loss (Raw)": 0.7730394601821899, "Finetune/Step": 2826, "Finetune/Step Time": 4.6434616800397635} +{"Finetune/Learning Rate": 9.063816531686855e-06, "Finetune/Loss": 0.8441604375839233, "Finetune/Loss (Raw)": 0.942258894443512, "Finetune/Step": 2827, "Finetune/Step Time": 4.620930401608348} +{"Finetune/Learning Rate": 9.057614462552769e-06, "Finetune/Loss": 0.8456151485443115, "Finetune/Loss (Raw)": 1.0441244840621948, "Finetune/Step": 2828, "Finetune/Step Time": 4.620512681081891} +{"Finetune/Learning Rate": 9.05141275914011e-06, "Finetune/Loss": 0.8458864092826843, "Finetune/Loss (Raw)": 0.921682596206665, "Finetune/Step": 2829, "Finetune/Step Time": 4.603068057447672} +{"Finetune/Learning Rate": 9.045211423855637e-06, "Finetune/Loss": 0.8462854027748108, "Finetune/Loss (Raw)": 0.9612352848052979, "Finetune/Step": 2830, "Finetune/Step Time": 4.589662291109562} +{"Finetune/Learning Rate": 9.039010459105974e-06, "Finetune/Loss": 0.8480827212333679, "Finetune/Loss (Raw)": 1.0565900802612305, "Finetune/Step": 2831, "Finetune/Step Time": 4.573279991745949} +{"Finetune/Learning Rate": 9.032809867297586e-06, "Finetune/Loss": 0.8496543169021606, "Finetune/Loss (Raw)": 0.9990116953849792, "Finetune/Step": 2832, "Finetune/Step Time": 4.574060225859284} +{"Finetune/Learning Rate": 9.02660965083681e-06, "Finetune/Loss": 0.8498018980026245, "Finetune/Loss (Raw)": 0.8893572688102722, "Finetune/Step": 2833, "Finetune/Step Time": 4.581268789246678} +{"Finetune/Learning Rate": 9.020409812129824e-06, "Finetune/Loss": 0.8449456095695496, "Finetune/Loss (Raw)": 0.35608163475990295, "Finetune/Step": 2834, "Finetune/Step Time": 4.6142052710056305} +{"Finetune/Learning Rate": 9.014210353582662e-06, "Finetune/Loss": 0.8443800210952759, "Finetune/Loss (Raw)": 0.9267006516456604, "Finetune/Step": 2835, "Finetune/Step Time": 4.61681661196053} +{"Finetune/Learning Rate": 9.008011277601217e-06, "Finetune/Loss": 0.8422935009002686, "Finetune/Loss (Raw)": 0.7335147261619568, "Finetune/Step": 2836, "Finetune/Step Time": 4.6089987345039845} +{"Finetune/Learning Rate": 9.001812586591228e-06, "Finetune/Loss": 0.8430480360984802, "Finetune/Loss (Raw)": 0.9591212868690491, "Finetune/Step": 2837, "Finetune/Step Time": 4.597121808677912} +{"Finetune/Learning Rate": 8.995614282958289e-06, "Finetune/Loss": 0.8465695977210999, "Finetune/Loss (Raw)": 0.8322161436080933, "Finetune/Step": 2838, "Finetune/Step Time": 4.558704858645797} +{"Finetune/Learning Rate": 8.989416369107837e-06, "Finetune/Loss": 0.8455426692962646, "Finetune/Loss (Raw)": 0.822458803653717, "Finetune/Step": 2839, "Finetune/Step Time": 4.564103648066521} +{"Finetune/Learning Rate": 8.983218847445157e-06, "Finetune/Loss": 0.8456010222434998, "Finetune/Loss (Raw)": 0.9321724772453308, "Finetune/Step": 2840, "Finetune/Step Time": 4.547769974917173} +{"Finetune/Learning Rate": 8.977021720375395e-06, "Finetune/Loss": 0.8471043109893799, "Finetune/Loss (Raw)": 1.0724965333938599, "Finetune/Step": 2841, "Finetune/Step Time": 4.541872266680002} +{"Finetune/Learning Rate": 8.970824990303532e-06, "Finetune/Loss": 0.8466936349868774, "Finetune/Loss (Raw)": 0.8958138823509216, "Finetune/Step": 2842, "Finetune/Step Time": 4.544078171253204} +{"Finetune/Learning Rate": 8.9646286596344e-06, "Finetune/Loss": 0.846131443977356, "Finetune/Loss (Raw)": 0.7530440092086792, "Finetune/Step": 2843, "Finetune/Step Time": 4.5489795207977295} +{"Finetune/Learning Rate": 8.958432730772667e-06, "Finetune/Loss": 0.8465244174003601, "Finetune/Loss (Raw)": 0.9034085869789124, "Finetune/Step": 2844, "Finetune/Step Time": 4.548792153596878} +{"Finetune/Learning Rate": 8.952237206122855e-06, "Finetune/Loss": 0.8470090627670288, "Finetune/Loss (Raw)": 0.9580361843109131, "Finetune/Step": 2845, "Finetune/Step Time": 4.559588456526399} +{"Finetune/Learning Rate": 8.946042088089331e-06, "Finetune/Loss": 0.8468872904777527, "Finetune/Loss (Raw)": 0.8518842458724976, "Finetune/Step": 2846, "Finetune/Step Time": 4.557551944628358} +{"Finetune/Learning Rate": 8.939847379076293e-06, "Finetune/Loss": 0.8448420763015747, "Finetune/Loss (Raw)": 0.6680999994277954, "Finetune/Step": 2847, "Finetune/Step Time": 4.569773580878973} +{"Finetune/Learning Rate": 8.933653081487794e-06, "Finetune/Loss": 0.8450167179107666, "Finetune/Loss (Raw)": 0.9606644511222839, "Finetune/Step": 2848, "Finetune/Step Time": 4.5759967267513275} +{"Finetune/Learning Rate": 8.927459197727712e-06, "Finetune/Loss": 0.8450295925140381, "Finetune/Loss (Raw)": 0.8566941618919373, "Finetune/Step": 2849, "Finetune/Step Time": 4.572027489542961} +{"Finetune/Learning Rate": 8.921265730199775e-06, "Finetune/Loss": 0.8449978232383728, "Finetune/Loss (Raw)": 0.8768742680549622, "Finetune/Step": 2850, "Finetune/Step Time": 4.567258877679706} +{"Finetune/Learning Rate": 8.91507268130755e-06, "Finetune/Loss": 0.8456301689147949, "Finetune/Loss (Raw)": 0.9300165772438049, "Finetune/Step": 2851, "Finetune/Step Time": 4.563214931637049} +{"Finetune/Learning Rate": 8.908880053454432e-06, "Finetune/Loss": 0.8461638689041138, "Finetune/Loss (Raw)": 0.9850866198539734, "Finetune/Step": 2852, "Finetune/Step Time": 4.563381612300873} +{"Finetune/Learning Rate": 8.90268784904367e-06, "Finetune/Loss": 0.8458877801895142, "Finetune/Loss (Raw)": 0.8717150688171387, "Finetune/Step": 2853, "Finetune/Step Time": 4.564063653349876} +{"Finetune/Learning Rate": 8.896496070478326e-06, "Finetune/Loss": 0.8461356163024902, "Finetune/Loss (Raw)": 0.942157506942749, "Finetune/Step": 2854, "Finetune/Step Time": 4.557438300922513} +{"Finetune/Learning Rate": 8.890304720161311e-06, "Finetune/Loss": 0.8465726375579834, "Finetune/Loss (Raw)": 0.9373739361763, "Finetune/Step": 2855, "Finetune/Step Time": 4.562413075938821} +{"Finetune/Learning Rate": 8.884113800495372e-06, "Finetune/Loss": 0.8478267192840576, "Finetune/Loss (Raw)": 0.9367186427116394, "Finetune/Step": 2856, "Finetune/Step Time": 4.576210331171751} +{"Finetune/Learning Rate": 8.87792331388308e-06, "Finetune/Loss": 0.8521836996078491, "Finetune/Loss (Raw)": 0.9775265455245972, "Finetune/Step": 2857, "Finetune/Step Time": 4.540145145729184} +{"Finetune/Learning Rate": 8.871733262726846e-06, "Finetune/Loss": 0.8524678349494934, "Finetune/Loss (Raw)": 0.8063645362854004, "Finetune/Step": 2858, "Finetune/Step Time": 4.533690636977553} +{"Finetune/Learning Rate": 8.865543649428903e-06, "Finetune/Loss": 0.852514386177063, "Finetune/Loss (Raw)": 0.9001815319061279, "Finetune/Step": 2859, "Finetune/Step Time": 4.541146678850055} +{"Finetune/Learning Rate": 8.859354476391319e-06, "Finetune/Loss": 0.8513858318328857, "Finetune/Loss (Raw)": 0.788027822971344, "Finetune/Step": 2860, "Finetune/Step Time": 4.551487563177943} +{"Finetune/Learning Rate": 8.853165746015997e-06, "Finetune/Loss": 0.8524758815765381, "Finetune/Loss (Raw)": 1.0274931192398071, "Finetune/Step": 2861, "Finetune/Step Time": 4.5516837276518345} +{"Finetune/Learning Rate": 8.846977460704656e-06, "Finetune/Loss": 0.8514255285263062, "Finetune/Loss (Raw)": 0.8126570582389832, "Finetune/Step": 2862, "Finetune/Step Time": 4.568089675158262} +{"Finetune/Learning Rate": 8.840789622858856e-06, "Finetune/Loss": 0.851164698600769, "Finetune/Loss (Raw)": 0.9221872091293335, "Finetune/Step": 2863, "Finetune/Step Time": 4.5652549006044865} +{"Finetune/Learning Rate": 8.834602234879968e-06, "Finetune/Loss": 0.8542241454124451, "Finetune/Loss (Raw)": 0.8507411479949951, "Finetune/Step": 2864, "Finetune/Step Time": 4.519513625651598} +{"Finetune/Learning Rate": 8.8284152991692e-06, "Finetune/Loss": 0.8573623895645142, "Finetune/Loss (Raw)": 0.9926398396492004, "Finetune/Step": 2865, "Finetune/Step Time": 4.511552715674043} +{"Finetune/Learning Rate": 8.822228818127584e-06, "Finetune/Loss": 0.853270947933197, "Finetune/Loss (Raw)": 0.393785685300827, "Finetune/Step": 2866, "Finetune/Step Time": 4.547379037365317} +{"Finetune/Learning Rate": 8.81604279415597e-06, "Finetune/Loss": 0.8530170321464539, "Finetune/Loss (Raw)": 0.9058394432067871, "Finetune/Step": 2867, "Finetune/Step Time": 4.550285616889596} +{"Finetune/Learning Rate": 8.809857229655029e-06, "Finetune/Loss": 0.8544900417327881, "Finetune/Loss (Raw)": 1.0209513902664185, "Finetune/Step": 2868, "Finetune/Step Time": 4.545064719393849} +{"Finetune/Learning Rate": 8.803672127025264e-06, "Finetune/Loss": 0.8523138761520386, "Finetune/Loss (Raw)": 0.7157394886016846, "Finetune/Step": 2869, "Finetune/Step Time": 4.581028716638684} +{"Finetune/Learning Rate": 8.797487488666986e-06, "Finetune/Loss": 0.8482127785682678, "Finetune/Loss (Raw)": 0.38007354736328125, "Finetune/Step": 2870, "Finetune/Step Time": 4.6135219763964415} +{"Finetune/Learning Rate": 8.791303316980342e-06, "Finetune/Loss": 0.8487616181373596, "Finetune/Loss (Raw)": 1.02277410030365, "Finetune/Step": 2871, "Finetune/Step Time": 4.611696295440197} +{"Finetune/Learning Rate": 8.785119614365282e-06, "Finetune/Loss": 0.8491507172584534, "Finetune/Loss (Raw)": 0.9708833694458008, "Finetune/Step": 2872, "Finetune/Step Time": 4.611888037994504} +{"Finetune/Learning Rate": 8.77893638322158e-06, "Finetune/Loss": 0.8483932018280029, "Finetune/Loss (Raw)": 0.8830730319023132, "Finetune/Step": 2873, "Finetune/Step Time": 4.61321365647018} +{"Finetune/Learning Rate": 8.772753625948828e-06, "Finetune/Loss": 0.8442959785461426, "Finetune/Loss (Raw)": 0.3602679669857025, "Finetune/Step": 2874, "Finetune/Step Time": 4.638718407601118} +{"Finetune/Learning Rate": 8.766571344946433e-06, "Finetune/Loss": 0.8453012704849243, "Finetune/Loss (Raw)": 1.0024117231369019, "Finetune/Step": 2875, "Finetune/Step Time": 4.618595946580172} +{"Finetune/Learning Rate": 8.760389542613619e-06, "Finetune/Loss": 0.8442642688751221, "Finetune/Loss (Raw)": 0.8797273635864258, "Finetune/Step": 2876, "Finetune/Step Time": 4.628005696460605} +{"Finetune/Learning Rate": 8.754208221349427e-06, "Finetune/Loss": 0.8434296250343323, "Finetune/Loss (Raw)": 0.8353573083877563, "Finetune/Step": 2877, "Finetune/Step Time": 4.630647659301758} +{"Finetune/Learning Rate": 8.748027383552696e-06, "Finetune/Loss": 0.8439298272132874, "Finetune/Loss (Raw)": 0.9181534647941589, "Finetune/Step": 2878, "Finetune/Step Time": 4.636307802051306} +{"Finetune/Learning Rate": 8.741847031622096e-06, "Finetune/Loss": 0.8428767323493958, "Finetune/Loss (Raw)": 0.7839336395263672, "Finetune/Step": 2879, "Finetune/Step Time": 4.638150939717889} +{"Finetune/Learning Rate": 8.7356671679561e-06, "Finetune/Loss": 0.8431459665298462, "Finetune/Loss (Raw)": 0.9807074069976807, "Finetune/Step": 2880, "Finetune/Step Time": 4.635142423212528} +{"Finetune/Learning Rate": 8.729487794952992e-06, "Finetune/Loss": 0.8430913686752319, "Finetune/Loss (Raw)": 0.9759743809700012, "Finetune/Step": 2881, "Finetune/Step Time": 4.6401735432446} +{"Finetune/Learning Rate": 8.72330891501087e-06, "Finetune/Loss": 0.8434354066848755, "Finetune/Loss (Raw)": 0.9806426167488098, "Finetune/Step": 2882, "Finetune/Step Time": 4.636199003085494} +{"Finetune/Learning Rate": 8.717130530527626e-06, "Finetune/Loss": 0.8484638333320618, "Finetune/Loss (Raw)": 0.9999404549598694, "Finetune/Step": 2883, "Finetune/Step Time": 4.6004003789275885} +{"Finetune/Learning Rate": 8.710952643900983e-06, "Finetune/Loss": 0.84748375415802, "Finetune/Loss (Raw)": 0.8410019874572754, "Finetune/Step": 2884, "Finetune/Step Time": 4.606969986110926} +{"Finetune/Learning Rate": 8.704775257528446e-06, "Finetune/Loss": 0.8487343788146973, "Finetune/Loss (Raw)": 0.9348334670066833, "Finetune/Step": 2885, "Finetune/Step Time": 4.574848202988505} +{"Finetune/Learning Rate": 8.69859837380735e-06, "Finetune/Loss": 0.8467073440551758, "Finetune/Loss (Raw)": 0.6850244402885437, "Finetune/Step": 2886, "Finetune/Step Time": 4.598030339926481} +{"Finetune/Learning Rate": 8.692421995134819e-06, "Finetune/Loss": 0.846356213092804, "Finetune/Loss (Raw)": 0.7819758057594299, "Finetune/Step": 2887, "Finetune/Step Time": 4.608655879274011} +{"Finetune/Learning Rate": 8.686246123907778e-06, "Finetune/Loss": 0.8487503528594971, "Finetune/Loss (Raw)": 1.04811692237854, "Finetune/Step": 2888, "Finetune/Step Time": 4.6106737442314625} +{"Finetune/Learning Rate": 8.680070762522969e-06, "Finetune/Loss": 0.84804368019104, "Finetune/Loss (Raw)": 0.33042117953300476, "Finetune/Step": 2889, "Finetune/Step Time": 4.610407765954733} +{"Finetune/Learning Rate": 8.673895913376928e-06, "Finetune/Loss": 0.8477552533149719, "Finetune/Loss (Raw)": 0.8072145581245422, "Finetune/Step": 2890, "Finetune/Step Time": 4.5993930492550135} +{"Finetune/Learning Rate": 8.667721578865993e-06, "Finetune/Loss": 0.8478243350982666, "Finetune/Loss (Raw)": 0.8936807513237, "Finetune/Step": 2891, "Finetune/Step Time": 4.598285682499409} +{"Finetune/Learning Rate": 8.661547761386307e-06, "Finetune/Loss": 0.849563479423523, "Finetune/Loss (Raw)": 1.0848625898361206, "Finetune/Step": 2892, "Finetune/Step Time": 4.607262959703803} +{"Finetune/Learning Rate": 8.655374463333797e-06, "Finetune/Loss": 0.8536502122879028, "Finetune/Loss (Raw)": 0.8764239549636841, "Finetune/Step": 2893, "Finetune/Step Time": 4.565767550840974} +{"Finetune/Learning Rate": 8.649201687104209e-06, "Finetune/Loss": 0.8548148274421692, "Finetune/Loss (Raw)": 1.0102577209472656, "Finetune/Step": 2894, "Finetune/Step Time": 4.559290966019034} +{"Finetune/Learning Rate": 8.643029435093072e-06, "Finetune/Loss": 0.8540326356887817, "Finetune/Loss (Raw)": 0.8845071196556091, "Finetune/Step": 2895, "Finetune/Step Time": 4.551461014896631} +{"Finetune/Learning Rate": 8.63685770969572e-06, "Finetune/Loss": 0.8538265228271484, "Finetune/Loss (Raw)": 0.9311702251434326, "Finetune/Step": 2896, "Finetune/Step Time": 4.553741868585348} +{"Finetune/Learning Rate": 8.630686513307282e-06, "Finetune/Loss": 0.8542208671569824, "Finetune/Loss (Raw)": 0.9775804281234741, "Finetune/Step": 2897, "Finetune/Step Time": 4.542407421395183} +{"Finetune/Learning Rate": 8.62451584832267e-06, "Finetune/Loss": 0.8560630083084106, "Finetune/Loss (Raw)": 0.9095110297203064, "Finetune/Step": 2898, "Finetune/Step Time": 4.525297155603766} +{"Finetune/Learning Rate": 8.618345717136604e-06, "Finetune/Loss": 0.8557342290878296, "Finetune/Loss (Raw)": 0.8601923584938049, "Finetune/Step": 2899, "Finetune/Step Time": 4.5255233608186245} +{"Finetune/Learning Rate": 8.612176122143589e-06, "Finetune/Loss": 0.8546785116195679, "Finetune/Loss (Raw)": 0.7796629667282104, "Finetune/Step": 2900, "Finetune/Step Time": 4.528720522299409} +{"Finetune/Learning Rate": 8.606007065737928e-06, "Finetune/Loss": 0.8533546924591064, "Finetune/Loss (Raw)": 0.7847414016723633, "Finetune/Step": 2901, "Finetune/Step Time": 4.538066577166319} +{"Finetune/Learning Rate": 8.599838550313714e-06, "Finetune/Loss": 0.8579792380332947, "Finetune/Loss (Raw)": 0.922978401184082, "Finetune/Step": 2902, "Finetune/Step Time": 4.521732015535235} +{"Finetune/Learning Rate": 8.593670578264814e-06, "Finetune/Loss": 0.858222246170044, "Finetune/Loss (Raw)": 0.9515243768692017, "Finetune/Step": 2903, "Finetune/Step Time": 4.524701736867428} +{"Finetune/Learning Rate": 8.587503151984911e-06, "Finetune/Loss": 0.8586876392364502, "Finetune/Loss (Raw)": 0.8621413111686707, "Finetune/Step": 2904, "Finetune/Step Time": 4.522645374760032} +{"Finetune/Learning Rate": 8.581336273867455e-06, "Finetune/Loss": 0.8589853048324585, "Finetune/Loss (Raw)": 0.8903084397315979, "Finetune/Step": 2905, "Finetune/Step Time": 4.524366516619921} +{"Finetune/Learning Rate": 8.575169946305699e-06, "Finetune/Loss": 0.8592734336853027, "Finetune/Loss (Raw)": 0.9323590397834778, "Finetune/Step": 2906, "Finetune/Step Time": 4.531323105096817} +{"Finetune/Learning Rate": 8.569004171692667e-06, "Finetune/Loss": 0.8593753576278687, "Finetune/Loss (Raw)": 0.8534789681434631, "Finetune/Step": 2907, "Finetune/Step Time": 4.523742999881506} +{"Finetune/Learning Rate": 8.562838952421177e-06, "Finetune/Loss": 0.8597760200500488, "Finetune/Loss (Raw)": 0.9391953349113464, "Finetune/Step": 2908, "Finetune/Step Time": 4.527067529037595} +{"Finetune/Learning Rate": 8.556674290883833e-06, "Finetune/Loss": 0.8604476451873779, "Finetune/Loss (Raw)": 1.0795425176620483, "Finetune/Step": 2909, "Finetune/Step Time": 4.521468451246619} +{"Finetune/Learning Rate": 8.550510189473016e-06, "Finetune/Loss": 0.8614019155502319, "Finetune/Loss (Raw)": 0.9833678007125854, "Finetune/Step": 2910, "Finetune/Step Time": 4.522020295262337} +{"Finetune/Learning Rate": 8.544346650580899e-06, "Finetune/Loss": 0.8615026473999023, "Finetune/Loss (Raw)": 1.0492498874664307, "Finetune/Step": 2911, "Finetune/Step Time": 4.50759513489902} +{"Finetune/Learning Rate": 8.538183676599426e-06, "Finetune/Loss": 0.8633708953857422, "Finetune/Loss (Raw)": 0.9424350261688232, "Finetune/Step": 2912, "Finetune/Step Time": 4.499471854418516} +{"Finetune/Learning Rate": 8.532021269920326e-06, "Finetune/Loss": 0.8639392852783203, "Finetune/Loss (Raw)": 0.9347785711288452, "Finetune/Step": 2913, "Finetune/Step Time": 4.486069247126579} +{"Finetune/Learning Rate": 8.525859432935114e-06, "Finetune/Loss": 0.8618856072425842, "Finetune/Loss (Raw)": 0.6977757215499878, "Finetune/Step": 2914, "Finetune/Step Time": 4.51245092228055} +{"Finetune/Learning Rate": 8.519698168035074e-06, "Finetune/Loss": 0.8611385822296143, "Finetune/Loss (Raw)": 0.7348604202270508, "Finetune/Step": 2915, "Finetune/Step Time": 4.523917013779283} +{"Finetune/Learning Rate": 8.513537477611278e-06, "Finetune/Loss": 0.8610447645187378, "Finetune/Loss (Raw)": 0.9255316257476807, "Finetune/Step": 2916, "Finetune/Step Time": 4.510888762772083} +{"Finetune/Learning Rate": 8.507377364054566e-06, "Finetune/Loss": 0.861988365650177, "Finetune/Loss (Raw)": 0.9785197973251343, "Finetune/Step": 2917, "Finetune/Step Time": 4.500597281381488} +{"Finetune/Learning Rate": 8.501217829755554e-06, "Finetune/Loss": 0.8615216612815857, "Finetune/Loss (Raw)": 0.8464879393577576, "Finetune/Step": 2918, "Finetune/Step Time": 4.5054725632071495} +{"Finetune/Learning Rate": 8.495058877104642e-06, "Finetune/Loss": 0.8628978729248047, "Finetune/Loss (Raw)": 0.9098453521728516, "Finetune/Step": 2919, "Finetune/Step Time": 4.483603522181511} +{"Finetune/Learning Rate": 8.488900508491999e-06, "Finetune/Loss": 0.8579840064048767, "Finetune/Loss (Raw)": 0.34714096784591675, "Finetune/Step": 2920, "Finetune/Step Time": 4.509403409436345} +{"Finetune/Learning Rate": 8.48274272630757e-06, "Finetune/Loss": 0.8572472333908081, "Finetune/Loss (Raw)": 0.8919981122016907, "Finetune/Step": 2921, "Finetune/Step Time": 4.526723535731435} +{"Finetune/Learning Rate": 8.476585532941064e-06, "Finetune/Loss": 0.857499361038208, "Finetune/Loss (Raw)": 0.8532313704490662, "Finetune/Step": 2922, "Finetune/Step Time": 4.519629841670394} +{"Finetune/Learning Rate": 8.470428930781969e-06, "Finetune/Loss": 0.8572667837142944, "Finetune/Loss (Raw)": 0.8773795962333679, "Finetune/Step": 2923, "Finetune/Step Time": 4.54131406545639} +{"Finetune/Learning Rate": 8.464272922219547e-06, "Finetune/Loss": 0.8583074808120728, "Finetune/Loss (Raw)": 0.9933709502220154, "Finetune/Step": 2924, "Finetune/Step Time": 4.524671286344528} +{"Finetune/Learning Rate": 8.458117509642817e-06, "Finetune/Loss": 0.8602762222290039, "Finetune/Loss (Raw)": 1.0107475519180298, "Finetune/Step": 2925, "Finetune/Step Time": 4.514357691630721} +{"Finetune/Learning Rate": 8.451962695440586e-06, "Finetune/Loss": 0.8642352819442749, "Finetune/Loss (Raw)": 0.8871506452560425, "Finetune/Step": 2926, "Finetune/Step Time": 4.502056036144495} +{"Finetune/Learning Rate": 8.445808482001405e-06, "Finetune/Loss": 0.8638347387313843, "Finetune/Loss (Raw)": 0.6896635890007019, "Finetune/Step": 2927, "Finetune/Step Time": 4.511180009692907} +{"Finetune/Learning Rate": 8.439654871713608e-06, "Finetune/Loss": 0.8650786876678467, "Finetune/Loss (Raw)": 0.8966463208198547, "Finetune/Step": 2928, "Finetune/Step Time": 4.520209966227412} +{"Finetune/Learning Rate": 8.433501866965294e-06, "Finetune/Loss": 0.8655844926834106, "Finetune/Loss (Raw)": 1.0405970811843872, "Finetune/Step": 2929, "Finetune/Step Time": 4.510157883167267} +{"Finetune/Learning Rate": 8.42734947014432e-06, "Finetune/Loss": 0.8655346632003784, "Finetune/Loss (Raw)": 0.9682020545005798, "Finetune/Step": 2930, "Finetune/Step Time": 4.502272840589285} +{"Finetune/Learning Rate": 8.42119768363832e-06, "Finetune/Loss": 0.8656156659126282, "Finetune/Loss (Raw)": 0.9357237815856934, "Finetune/Step": 2931, "Finetune/Step Time": 4.505574280396104} +{"Finetune/Learning Rate": 8.415046509834668e-06, "Finetune/Loss": 0.8657265305519104, "Finetune/Loss (Raw)": 0.9304337501525879, "Finetune/Step": 2932, "Finetune/Step Time": 4.509291311725974} +{"Finetune/Learning Rate": 8.408895951120526e-06, "Finetune/Loss": 0.8647451996803284, "Finetune/Loss (Raw)": 0.8996329307556152, "Finetune/Step": 2933, "Finetune/Step Time": 4.520638346672058} +{"Finetune/Learning Rate": 8.402746009882798e-06, "Finetune/Loss": 0.8654398918151855, "Finetune/Loss (Raw)": 0.9224625825881958, "Finetune/Step": 2934, "Finetune/Step Time": 4.514056505635381} +{"Finetune/Learning Rate": 8.39659668850816e-06, "Finetune/Loss": 0.8659145832061768, "Finetune/Loss (Raw)": 0.9024736881256104, "Finetune/Step": 2935, "Finetune/Step Time": 4.524363238364458} +{"Finetune/Learning Rate": 8.39044798938305e-06, "Finetune/Loss": 0.8679667711257935, "Finetune/Loss (Raw)": 1.0174381732940674, "Finetune/Step": 2936, "Finetune/Step Time": 4.530230017378926} +{"Finetune/Learning Rate": 8.384299914893645e-06, "Finetune/Loss": 0.8734030723571777, "Finetune/Loss (Raw)": 1.0479987859725952, "Finetune/Step": 2937, "Finetune/Step Time": 4.498566061258316} +{"Finetune/Learning Rate": 8.378152467425902e-06, "Finetune/Loss": 0.8747986555099487, "Finetune/Loss (Raw)": 0.9416091442108154, "Finetune/Step": 2938, "Finetune/Step Time": 4.488466301932931} +{"Finetune/Learning Rate": 8.37200564936552e-06, "Finetune/Loss": 0.8746819496154785, "Finetune/Loss (Raw)": 0.8985750079154968, "Finetune/Step": 2939, "Finetune/Step Time": 4.481619110330939} +{"Finetune/Learning Rate": 8.365859463097963e-06, "Finetune/Loss": 0.8734889030456543, "Finetune/Loss (Raw)": 0.7204992175102234, "Finetune/Step": 2940, "Finetune/Step Time": 4.483446110039949} +{"Finetune/Learning Rate": 8.359713911008447e-06, "Finetune/Loss": 0.8709343075752258, "Finetune/Loss (Raw)": 0.6664927005767822, "Finetune/Step": 2941, "Finetune/Step Time": 4.500835610553622} +{"Finetune/Learning Rate": 8.353568995481934e-06, "Finetune/Loss": 0.8761451244354248, "Finetune/Loss (Raw)": 1.0209206342697144, "Finetune/Step": 2942, "Finetune/Step Time": 4.4660942777991295} +{"Finetune/Learning Rate": 8.347424718903152e-06, "Finetune/Loss": 0.8755468726158142, "Finetune/Loss (Raw)": 0.9394062161445618, "Finetune/Step": 2943, "Finetune/Step Time": 4.471142387017608} +{"Finetune/Learning Rate": 8.341281083656571e-06, "Finetune/Loss": 0.8754581809043884, "Finetune/Loss (Raw)": 0.9128119349479675, "Finetune/Step": 2944, "Finetune/Step Time": 4.478286048397422} +{"Finetune/Learning Rate": 8.33513809212642e-06, "Finetune/Loss": 0.8751938939094543, "Finetune/Loss (Raw)": 0.8217454552650452, "Finetune/Step": 2945, "Finetune/Step Time": 4.492307031527162} +{"Finetune/Learning Rate": 8.328995746696674e-06, "Finetune/Loss": 0.8757745027542114, "Finetune/Loss (Raw)": 0.9089812636375427, "Finetune/Step": 2946, "Finetune/Step Time": 4.489021783694625} +{"Finetune/Learning Rate": 8.32285404975105e-06, "Finetune/Loss": 0.8744447231292725, "Finetune/Loss (Raw)": 0.8666646480560303, "Finetune/Step": 2947, "Finetune/Step Time": 4.499775875359774} +{"Finetune/Learning Rate": 8.316713003673028e-06, "Finetune/Loss": 0.8740705847740173, "Finetune/Loss (Raw)": 0.9536561965942383, "Finetune/Step": 2948, "Finetune/Step Time": 4.510238701477647} +{"Finetune/Learning Rate": 8.310572610845824e-06, "Finetune/Loss": 0.8758031129837036, "Finetune/Loss (Raw)": 0.9408947825431824, "Finetune/Step": 2949, "Finetune/Step Time": 4.495278608053923} +{"Finetune/Learning Rate": 8.304432873652412e-06, "Finetune/Loss": 0.8757920265197754, "Finetune/Loss (Raw)": 0.9061483144760132, "Finetune/Step": 2950, "Finetune/Step Time": 4.495147289708257} +{"Finetune/Learning Rate": 8.298293794475495e-06, "Finetune/Loss": 0.8750173449516296, "Finetune/Loss (Raw)": 0.304645836353302, "Finetune/Step": 2951, "Finetune/Step Time": 4.495408216491342} +{"Finetune/Learning Rate": 8.29215537569753e-06, "Finetune/Loss": 0.8762189745903015, "Finetune/Loss (Raw)": 0.9479703903198242, "Finetune/Step": 2952, "Finetune/Step Time": 4.492957498878241} +{"Finetune/Learning Rate": 8.286017619700725e-06, "Finetune/Loss": 0.8776094317436218, "Finetune/Loss (Raw)": 0.9774280786514282, "Finetune/Step": 2953, "Finetune/Step Time": 4.491885473951697} +{"Finetune/Learning Rate": 8.279880528867017e-06, "Finetune/Loss": 0.8778172135353088, "Finetune/Loss (Raw)": 0.7996354699134827, "Finetune/Step": 2954, "Finetune/Step Time": 4.496968267485499} +{"Finetune/Learning Rate": 8.273744105578097e-06, "Finetune/Loss": 0.8786665201187134, "Finetune/Loss (Raw)": 1.0509693622589111, "Finetune/Step": 2955, "Finetune/Step Time": 4.504072707146406} +{"Finetune/Learning Rate": 8.267608352215385e-06, "Finetune/Loss": 0.8778365850448608, "Finetune/Loss (Raw)": 0.9378975629806519, "Finetune/Step": 2956, "Finetune/Step Time": 4.501143546774983} +{"Finetune/Learning Rate": 8.261473271160046e-06, "Finetune/Loss": 0.8776063323020935, "Finetune/Loss (Raw)": 0.892207145690918, "Finetune/Step": 2957, "Finetune/Step Time": 4.495155069977045} +{"Finetune/Learning Rate": 8.255338864792993e-06, "Finetune/Loss": 0.8772183656692505, "Finetune/Loss (Raw)": 0.9115754961967468, "Finetune/Step": 2958, "Finetune/Step Time": 4.49854402244091} +{"Finetune/Learning Rate": 8.249205135494862e-06, "Finetune/Loss": 0.8714664578437805, "Finetune/Loss (Raw)": 0.3203469514846802, "Finetune/Step": 2959, "Finetune/Step Time": 4.534205101430416} +{"Finetune/Learning Rate": 8.24307208564604e-06, "Finetune/Loss": 0.8706509470939636, "Finetune/Loss (Raw)": 0.8946239948272705, "Finetune/Step": 2960, "Finetune/Step Time": 4.526312788948417} +{"Finetune/Learning Rate": 8.236939717626639e-06, "Finetune/Loss": 0.8702468276023865, "Finetune/Loss (Raw)": 0.8376331925392151, "Finetune/Step": 2961, "Finetune/Step Time": 4.511914985254407} +{"Finetune/Learning Rate": 8.23080803381651e-06, "Finetune/Loss": 0.8758288025856018, "Finetune/Loss (Raw)": 1.0705732107162476, "Finetune/Step": 2962, "Finetune/Step Time": 4.478442331776023} +{"Finetune/Learning Rate": 8.224677036595245e-06, "Finetune/Loss": 0.8761705756187439, "Finetune/Loss (Raw)": 0.9704456925392151, "Finetune/Step": 2963, "Finetune/Step Time": 4.489536380395293} +{"Finetune/Learning Rate": 8.218546728342161e-06, "Finetune/Loss": 0.8781770467758179, "Finetune/Loss (Raw)": 0.9903456568717957, "Finetune/Step": 2964, "Finetune/Step Time": 4.492290521040559} +{"Finetune/Learning Rate": 8.212417111436315e-06, "Finetune/Loss": 0.878216564655304, "Finetune/Loss (Raw)": 0.9641793370246887, "Finetune/Step": 2965, "Finetune/Step Time": 4.488197539001703} +{"Finetune/Learning Rate": 8.206288188256486e-06, "Finetune/Loss": 0.8780837059020996, "Finetune/Loss (Raw)": 0.815207302570343, "Finetune/Step": 2966, "Finetune/Step Time": 4.490673230960965} +{"Finetune/Learning Rate": 8.20015996118119e-06, "Finetune/Loss": 0.8777487277984619, "Finetune/Loss (Raw)": 0.7795864343643188, "Finetune/Step": 2967, "Finetune/Step Time": 4.502784511074424} +{"Finetune/Learning Rate": 8.194032432588677e-06, "Finetune/Loss": 0.8783833980560303, "Finetune/Loss (Raw)": 1.0134038925170898, "Finetune/Step": 2968, "Finetune/Step Time": 4.5120612140744925} +{"Finetune/Learning Rate": 8.187905604856916e-06, "Finetune/Loss": 0.876787543296814, "Finetune/Loss (Raw)": 0.8682378530502319, "Finetune/Step": 2969, "Finetune/Step Time": 4.509490422904491} +{"Finetune/Learning Rate": 8.181779480363614e-06, "Finetune/Loss": 0.8768812417984009, "Finetune/Loss (Raw)": 0.9077975153923035, "Finetune/Step": 2970, "Finetune/Step Time": 4.52133658900857} +{"Finetune/Learning Rate": 8.175654061486197e-06, "Finetune/Loss": 0.878009557723999, "Finetune/Loss (Raw)": 0.8974736928939819, "Finetune/Step": 2971, "Finetune/Step Time": 4.528374716639519} +{"Finetune/Learning Rate": 8.16952935060182e-06, "Finetune/Loss": 0.878177285194397, "Finetune/Loss (Raw)": 0.9248771667480469, "Finetune/Step": 2972, "Finetune/Step Time": 4.524368746206164} +{"Finetune/Learning Rate": 8.163405350087366e-06, "Finetune/Loss": 0.876814603805542, "Finetune/Loss (Raw)": 0.7836146354675293, "Finetune/Step": 2973, "Finetune/Step Time": 4.532994754612446} +{"Finetune/Learning Rate": 8.157282062319437e-06, "Finetune/Loss": 0.8774170875549316, "Finetune/Loss (Raw)": 0.9289981126785278, "Finetune/Step": 2974, "Finetune/Step Time": 4.534162232652307} +{"Finetune/Learning Rate": 8.15115948967437e-06, "Finetune/Loss": 0.8752713203430176, "Finetune/Loss (Raw)": 0.3934430480003357, "Finetune/Step": 2975, "Finetune/Step Time": 4.560030560940504} +{"Finetune/Learning Rate": 8.145037634528205e-06, "Finetune/Loss": 0.8741985559463501, "Finetune/Loss (Raw)": 0.8233472108840942, "Finetune/Step": 2976, "Finetune/Step Time": 4.554427366703749} +{"Finetune/Learning Rate": 8.138916499256718e-06, "Finetune/Loss": 0.8746052980422974, "Finetune/Loss (Raw)": 0.908758282661438, "Finetune/Step": 2977, "Finetune/Step Time": 4.55180518515408} +{"Finetune/Learning Rate": 8.132796086235405e-06, "Finetune/Loss": 0.8754963874816895, "Finetune/Loss (Raw)": 0.9909361004829407, "Finetune/Step": 2978, "Finetune/Step Time": 4.558324437588453} +{"Finetune/Learning Rate": 8.126676397839475e-06, "Finetune/Loss": 0.8752011656761169, "Finetune/Loss (Raw)": 0.8922263383865356, "Finetune/Step": 2979, "Finetune/Step Time": 4.563250754028559} +{"Finetune/Learning Rate": 8.120557436443866e-06, "Finetune/Loss": 0.874082624912262, "Finetune/Loss (Raw)": 0.8419135212898254, "Finetune/Step": 2980, "Finetune/Step Time": 4.566291378811002} +{"Finetune/Learning Rate": 8.11443920442322e-06, "Finetune/Loss": 0.8734263777732849, "Finetune/Loss (Raw)": 0.7877182960510254, "Finetune/Step": 2981, "Finetune/Step Time": 4.592513781040907} +{"Finetune/Learning Rate": 8.108321704151903e-06, "Finetune/Loss": 0.8724795579910278, "Finetune/Loss (Raw)": 0.8209622502326965, "Finetune/Step": 2982, "Finetune/Step Time": 4.585754239931703} +{"Finetune/Learning Rate": 8.102204938004004e-06, "Finetune/Loss": 0.8730159401893616, "Finetune/Loss (Raw)": 1.0060313940048218, "Finetune/Step": 2983, "Finetune/Step Time": 4.5764820743352175} +{"Finetune/Learning Rate": 8.096088908353316e-06, "Finetune/Loss": 0.8734117746353149, "Finetune/Loss (Raw)": 0.9873860478401184, "Finetune/Step": 2984, "Finetune/Step Time": 4.576511405408382} +{"Finetune/Learning Rate": 8.089973617573355e-06, "Finetune/Loss": 0.8725988864898682, "Finetune/Loss (Raw)": 0.8734732270240784, "Finetune/Step": 2985, "Finetune/Step Time": 4.576065620407462} +{"Finetune/Learning Rate": 8.08385906803734e-06, "Finetune/Loss": 0.868880569934845, "Finetune/Loss (Raw)": 0.33042341470718384, "Finetune/Step": 2986, "Finetune/Step Time": 4.616961406543851} +{"Finetune/Learning Rate": 8.07774526211821e-06, "Finetune/Loss": 0.8683652877807617, "Finetune/Loss (Raw)": 0.8342244625091553, "Finetune/Step": 2987, "Finetune/Step Time": 4.629927067086101} +{"Finetune/Learning Rate": 8.071632202188618e-06, "Finetune/Loss": 0.869605541229248, "Finetune/Loss (Raw)": 0.9467726945877075, "Finetune/Step": 2988, "Finetune/Step Time": 4.627480715513229} +{"Finetune/Learning Rate": 8.065519890620922e-06, "Finetune/Loss": 0.8679372072219849, "Finetune/Loss (Raw)": 0.8139566779136658, "Finetune/Step": 2989, "Finetune/Step Time": 4.631907248869538} +{"Finetune/Learning Rate": 8.059408329787184e-06, "Finetune/Loss": 0.8694717288017273, "Finetune/Loss (Raw)": 1.0090715885162354, "Finetune/Step": 2990, "Finetune/Step Time": 4.607767885550857} +{"Finetune/Learning Rate": 8.053297522059189e-06, "Finetune/Loss": 0.8696812987327576, "Finetune/Loss (Raw)": 0.9490107893943787, "Finetune/Step": 2991, "Finetune/Step Time": 4.612060409039259} +{"Finetune/Learning Rate": 8.047187469808418e-06, "Finetune/Loss": 0.8691833019256592, "Finetune/Loss (Raw)": 0.7870035171508789, "Finetune/Step": 2992, "Finetune/Step Time": 4.622530873864889} +{"Finetune/Learning Rate": 8.041078175406064e-06, "Finetune/Loss": 0.8692304491996765, "Finetune/Loss (Raw)": 0.9986692667007446, "Finetune/Step": 2993, "Finetune/Step Time": 4.6186209209263325} +{"Finetune/Learning Rate": 8.03496964122303e-06, "Finetune/Loss": 0.8741090893745422, "Finetune/Loss (Raw)": 1.0182545185089111, "Finetune/Step": 2994, "Finetune/Step Time": 4.575932944193482} +{"Finetune/Learning Rate": 8.02886186962991e-06, "Finetune/Loss": 0.8741114139556885, "Finetune/Loss (Raw)": 0.9061395525932312, "Finetune/Step": 2995, "Finetune/Step Time": 4.579547137022018} +{"Finetune/Learning Rate": 8.022754862997016e-06, "Finetune/Loss": 0.872979998588562, "Finetune/Loss (Raw)": 0.8761312365531921, "Finetune/Step": 2996, "Finetune/Step Time": 4.594253275543451} +{"Finetune/Learning Rate": 8.016648623694354e-06, "Finetune/Loss": 0.8738753795623779, "Finetune/Loss (Raw)": 0.8303433656692505, "Finetune/Step": 2997, "Finetune/Step Time": 4.553972199559212} +{"Finetune/Learning Rate": 8.010543154091644e-06, "Finetune/Loss": 0.8776137828826904, "Finetune/Loss (Raw)": 0.8585930466651917, "Finetune/Step": 2998, "Finetune/Step Time": 4.523439880460501} +{"Finetune/Learning Rate": 8.004438456558297e-06, "Finetune/Loss": 0.8777354955673218, "Finetune/Loss (Raw)": 1.0383515357971191, "Finetune/Step": 2999, "Finetune/Step Time": 4.529536839574575} +{"Finetune/Learning Rate": 7.99833453346342e-06, "Finetune/Loss": 0.87712562084198, "Finetune/Loss (Raw)": 0.8928205966949463, "Finetune/Step": 3000, "Finetune/Step Time": 4.533064855262637} +{"Finetune/Learning Rate": 7.992231387175832e-06, "Finetune/Loss": 0.8769344091415405, "Finetune/Loss (Raw)": 0.8585940003395081, "Finetune/Step": 3001, "Finetune/Step Time": 4.536464810371399} +{"Finetune/Learning Rate": 7.986129020064044e-06, "Finetune/Loss": 0.8813945055007935, "Finetune/Loss (Raw)": 0.9311571717262268, "Finetune/Step": 3002, "Finetune/Step Time": 4.500383924692869} +{"Finetune/Learning Rate": 7.980027434496267e-06, "Finetune/Loss": 0.8805136680603027, "Finetune/Loss (Raw)": 0.889673113822937, "Finetune/Step": 3003, "Finetune/Step Time": 4.512386856600642} +{"Finetune/Learning Rate": 7.97392663284041e-06, "Finetune/Loss": 0.8809451460838318, "Finetune/Loss (Raw)": 0.9349531531333923, "Finetune/Step": 3004, "Finetune/Step Time": 4.504593726247549} +{"Finetune/Learning Rate": 7.967826617464065e-06, "Finetune/Loss": 0.8800762891769409, "Finetune/Loss (Raw)": 0.7241409420967102, "Finetune/Step": 3005, "Finetune/Step Time": 4.528886569663882} +{"Finetune/Learning Rate": 7.961727390734538e-06, "Finetune/Loss": 0.8802092671394348, "Finetune/Loss (Raw)": 0.9351746439933777, "Finetune/Step": 3006, "Finetune/Step Time": 4.520983222872019} +{"Finetune/Learning Rate": 7.955628955018813e-06, "Finetune/Loss": 0.880070686340332, "Finetune/Loss (Raw)": 0.7661974430084229, "Finetune/Step": 3007, "Finetune/Step Time": 4.511441960930824} +{"Finetune/Learning Rate": 7.949531312683582e-06, "Finetune/Loss": 0.8797276020050049, "Finetune/Loss (Raw)": 0.9367892742156982, "Finetune/Step": 3008, "Finetune/Step Time": 4.507025418803096} +{"Finetune/Learning Rate": 7.943434466095216e-06, "Finetune/Loss": 0.8797785639762878, "Finetune/Loss (Raw)": 0.982496440410614, "Finetune/Step": 3009, "Finetune/Step Time": 4.499397777020931} +{"Finetune/Learning Rate": 7.937338417619781e-06, "Finetune/Loss": 0.8788446187973022, "Finetune/Loss (Raw)": 0.8611050248146057, "Finetune/Step": 3010, "Finetune/Step Time": 4.499713905155659} +{"Finetune/Learning Rate": 7.931243169623037e-06, "Finetune/Loss": 0.8789653778076172, "Finetune/Loss (Raw)": 1.0153932571411133, "Finetune/Step": 3011, "Finetune/Step Time": 4.496211268007755} +{"Finetune/Learning Rate": 7.925148724470428e-06, "Finetune/Loss": 0.8797074556350708, "Finetune/Loss (Raw)": 0.9359906315803528, "Finetune/Step": 3012, "Finetune/Step Time": 4.47402623295784} +{"Finetune/Learning Rate": 7.919055084527094e-06, "Finetune/Loss": 0.8793554902076721, "Finetune/Loss (Raw)": 0.8897759318351746, "Finetune/Step": 3013, "Finetune/Step Time": 4.476100223138928} +{"Finetune/Learning Rate": 7.912962252157854e-06, "Finetune/Loss": 0.8805423974990845, "Finetune/Loss (Raw)": 0.8369458317756653, "Finetune/Step": 3014, "Finetune/Step Time": 4.4596395958215} +{"Finetune/Learning Rate": 7.906870229727216e-06, "Finetune/Loss": 0.880378007888794, "Finetune/Loss (Raw)": 0.7609415650367737, "Finetune/Step": 3015, "Finetune/Step Time": 4.463238123804331} +{"Finetune/Learning Rate": 7.900779019599378e-06, "Finetune/Loss": 0.8792711496353149, "Finetune/Loss (Raw)": 0.9064368605613708, "Finetune/Step": 3016, "Finetune/Step Time": 4.467172559350729} +{"Finetune/Learning Rate": 7.894688624138217e-06, "Finetune/Loss": 0.8827365040779114, "Finetune/Loss (Raw)": 0.7739819288253784, "Finetune/Step": 3017, "Finetune/Step Time": 4.4331362303346395} +{"Finetune/Learning Rate": 7.8885990457073e-06, "Finetune/Loss": 0.8830327987670898, "Finetune/Loss (Raw)": 0.8451415300369263, "Finetune/Step": 3018, "Finetune/Step Time": 4.428414765745401} +{"Finetune/Learning Rate": 7.882510286669875e-06, "Finetune/Loss": 0.8832460045814514, "Finetune/Loss (Raw)": 0.9209765195846558, "Finetune/Step": 3019, "Finetune/Step Time": 4.430182443931699} +{"Finetune/Learning Rate": 7.876422349388861e-06, "Finetune/Loss": 0.881561279296875, "Finetune/Loss (Raw)": 0.8692243099212646, "Finetune/Step": 3020, "Finetune/Step Time": 4.412015048786998} +{"Finetune/Learning Rate": 7.870335236226878e-06, "Finetune/Loss": 0.8816013336181641, "Finetune/Loss (Raw)": 0.881544291973114, "Finetune/Step": 3021, "Finetune/Step Time": 4.41536938585341} +{"Finetune/Learning Rate": 7.864248949546209e-06, "Finetune/Loss": 0.8812904357910156, "Finetune/Loss (Raw)": 0.9704615473747253, "Finetune/Step": 3022, "Finetune/Step Time": 4.414849825203419} +{"Finetune/Learning Rate": 7.858163491708827e-06, "Finetune/Loss": 0.8801275491714478, "Finetune/Loss (Raw)": 0.7356549501419067, "Finetune/Step": 3023, "Finetune/Step Time": 4.412929439917207} +{"Finetune/Learning Rate": 7.85207886507638e-06, "Finetune/Loss": 0.880013108253479, "Finetune/Loss (Raw)": 0.9165282845497131, "Finetune/Step": 3024, "Finetune/Step Time": 4.418165374547243} +{"Finetune/Learning Rate": 7.845995072010188e-06, "Finetune/Loss": 0.8796359300613403, "Finetune/Loss (Raw)": 0.9292919039726257, "Finetune/Step": 3025, "Finetune/Step Time": 4.41711724922061} +{"Finetune/Learning Rate": 7.839912114871256e-06, "Finetune/Loss": 0.8797677755355835, "Finetune/Loss (Raw)": 0.9263932704925537, "Finetune/Step": 3026, "Finetune/Step Time": 4.421762973070145} +{"Finetune/Learning Rate": 7.83382999602026e-06, "Finetune/Loss": 0.8807235956192017, "Finetune/Loss (Raw)": 0.9825340509414673, "Finetune/Step": 3027, "Finetune/Step Time": 4.425960339605808} +{"Finetune/Learning Rate": 7.827748717817552e-06, "Finetune/Loss": 0.8816320896148682, "Finetune/Loss (Raw)": 0.8959560394287109, "Finetune/Step": 3028, "Finetune/Step Time": 4.429119750857353} +{"Finetune/Learning Rate": 7.821668282623159e-06, "Finetune/Loss": 0.8837146759033203, "Finetune/Loss (Raw)": 1.0513079166412354, "Finetune/Step": 3029, "Finetune/Step Time": 4.419970652088523} +{"Finetune/Learning Rate": 7.815588692796771e-06, "Finetune/Loss": 0.8833800554275513, "Finetune/Loss (Raw)": 0.8801461458206177, "Finetune/Step": 3030, "Finetune/Step Time": 4.411527676507831} +{"Finetune/Learning Rate": 7.809509950697769e-06, "Finetune/Loss": 0.8830267786979675, "Finetune/Loss (Raw)": 0.9063094258308411, "Finetune/Step": 3031, "Finetune/Step Time": 4.40857226587832} +{"Finetune/Learning Rate": 7.803432058685187e-06, "Finetune/Loss": 0.8838717341423035, "Finetune/Loss (Raw)": 0.9702958464622498, "Finetune/Step": 3032, "Finetune/Step Time": 4.4131305031478405} +{"Finetune/Learning Rate": 7.797355019117743e-06, "Finetune/Loss": 0.8839749097824097, "Finetune/Loss (Raw)": 0.903511643409729, "Finetune/Step": 3033, "Finetune/Step Time": 4.41577772796154} +{"Finetune/Learning Rate": 7.79127883435381e-06, "Finetune/Loss": 0.8848071098327637, "Finetune/Loss (Raw)": 1.038881778717041, "Finetune/Step": 3034, "Finetune/Step Time": 4.405823718756437} +{"Finetune/Learning Rate": 7.78520350675144e-06, "Finetune/Loss": 0.8851211071014404, "Finetune/Loss (Raw)": 0.8936725854873657, "Finetune/Step": 3035, "Finetune/Step Time": 4.407427174970508} +{"Finetune/Learning Rate": 7.779129038668347e-06, "Finetune/Loss": 0.8846353888511658, "Finetune/Loss (Raw)": 0.8770241141319275, "Finetune/Step": 3036, "Finetune/Step Time": 4.4203112460672855} +{"Finetune/Learning Rate": 7.773055432461916e-06, "Finetune/Loss": 0.8841273784637451, "Finetune/Loss (Raw)": 1.0145045518875122, "Finetune/Step": 3037, "Finetune/Step Time": 4.422851674258709} +{"Finetune/Learning Rate": 7.766982690489198e-06, "Finetune/Loss": 0.8841228485107422, "Finetune/Loss (Raw)": 0.9827969074249268, "Finetune/Step": 3038, "Finetune/Step Time": 4.42191063798964} +{"Finetune/Learning Rate": 7.7609108151069e-06, "Finetune/Loss": 0.8824583292007446, "Finetune/Loss (Raw)": 0.8361908197402954, "Finetune/Step": 3039, "Finetune/Step Time": 4.422082653269172} +{"Finetune/Learning Rate": 7.754839808671397e-06, "Finetune/Loss": 0.8823805451393127, "Finetune/Loss (Raw)": 0.9324829578399658, "Finetune/Step": 3040, "Finetune/Step Time": 4.4151110500097275} +{"Finetune/Learning Rate": 7.748769673538734e-06, "Finetune/Loss": 0.8811112642288208, "Finetune/Loss (Raw)": 0.7723032832145691, "Finetune/Step": 3041, "Finetune/Step Time": 4.420106528326869} +{"Finetune/Learning Rate": 7.742700412064608e-06, "Finetune/Loss": 0.8828328251838684, "Finetune/Loss (Raw)": 0.9181399941444397, "Finetune/Step": 3042, "Finetune/Step Time": 4.4014705792069435} +{"Finetune/Learning Rate": 7.736632026604386e-06, "Finetune/Loss": 0.8795236349105835, "Finetune/Loss (Raw)": 0.3112785220146179, "Finetune/Step": 3043, "Finetune/Step Time": 4.430773822590709} +{"Finetune/Learning Rate": 7.730564519513082e-06, "Finetune/Loss": 0.8779000043869019, "Finetune/Loss (Raw)": 0.7177088260650635, "Finetune/Step": 3044, "Finetune/Step Time": 4.449946818873286} +{"Finetune/Learning Rate": 7.724497893145379e-06, "Finetune/Loss": 0.8764835596084595, "Finetune/Loss (Raw)": 0.797223687171936, "Finetune/Step": 3045, "Finetune/Step Time": 4.450405366718769} +{"Finetune/Learning Rate": 7.718432149855622e-06, "Finetune/Loss": 0.8779206275939941, "Finetune/Loss (Raw)": 1.030427098274231, "Finetune/Step": 3046, "Finetune/Step Time": 4.444577759131789} +{"Finetune/Learning Rate": 7.7123672919978e-06, "Finetune/Loss": 0.8774969577789307, "Finetune/Loss (Raw)": 0.8556188941001892, "Finetune/Step": 3047, "Finetune/Step Time": 4.450615830719471} +{"Finetune/Learning Rate": 7.706303321925577e-06, "Finetune/Loss": 0.8829015493392944, "Finetune/Loss (Raw)": 1.0389277935028076, "Finetune/Step": 3048, "Finetune/Step Time": 4.4110389444977045} +{"Finetune/Learning Rate": 7.700240241992247e-06, "Finetune/Loss": 0.8828079700469971, "Finetune/Loss (Raw)": 0.8800188302993774, "Finetune/Step": 3049, "Finetune/Step Time": 4.394374230876565} +{"Finetune/Learning Rate": 7.694178054550782e-06, "Finetune/Loss": 0.8839071989059448, "Finetune/Loss (Raw)": 0.99393630027771, "Finetune/Step": 3050, "Finetune/Step Time": 4.385678386315703} +{"Finetune/Learning Rate": 7.688116761953797e-06, "Finetune/Loss": 0.8850617408752441, "Finetune/Loss (Raw)": 1.0251606702804565, "Finetune/Step": 3051, "Finetune/Step Time": 4.364740919321775} +{"Finetune/Learning Rate": 7.682056366553559e-06, "Finetune/Loss": 0.8841211795806885, "Finetune/Loss (Raw)": 0.8729730844497681, "Finetune/Step": 3052, "Finetune/Step Time": 4.369266828522086} +{"Finetune/Learning Rate": 7.675996870701999e-06, "Finetune/Loss": 0.8838402032852173, "Finetune/Loss (Raw)": 0.9747843742370605, "Finetune/Step": 3053, "Finetune/Step Time": 4.372597049921751} +{"Finetune/Learning Rate": 7.669938276750673e-06, "Finetune/Loss": 0.8847802877426147, "Finetune/Loss (Raw)": 1.0074763298034668, "Finetune/Step": 3054, "Finetune/Step Time": 4.3471856992691755} +{"Finetune/Learning Rate": 7.663880587050812e-06, "Finetune/Loss": 0.8865785598754883, "Finetune/Loss (Raw)": 0.9198490977287292, "Finetune/Step": 3055, "Finetune/Step Time": 4.3385972417891026} +{"Finetune/Learning Rate": 7.657823803953288e-06, "Finetune/Loss": 0.8871816396713257, "Finetune/Loss (Raw)": 0.9738337397575378, "Finetune/Step": 3056, "Finetune/Step Time": 4.317553421482444} +{"Finetune/Learning Rate": 7.651767929808615e-06, "Finetune/Loss": 0.8852169513702393, "Finetune/Loss (Raw)": 0.7891260981559753, "Finetune/Step": 3057, "Finetune/Step Time": 4.330306023359299} +{"Finetune/Learning Rate": 7.645712966966973e-06, "Finetune/Loss": 0.8852394819259644, "Finetune/Loss (Raw)": 0.971079409122467, "Finetune/Step": 3058, "Finetune/Step Time": 4.321558529511094} +{"Finetune/Learning Rate": 7.639658917778156e-06, "Finetune/Loss": 0.8853132724761963, "Finetune/Loss (Raw)": 0.9451725482940674, "Finetune/Step": 3059, "Finetune/Step Time": 4.31769179366529} +{"Finetune/Learning Rate": 7.633605784591633e-06, "Finetune/Loss": 0.8856633901596069, "Finetune/Loss (Raw)": 0.9752469658851624, "Finetune/Step": 3060, "Finetune/Step Time": 4.322190660983324} +{"Finetune/Learning Rate": 7.627553569756505e-06, "Finetune/Loss": 0.8860435485839844, "Finetune/Loss (Raw)": 0.9482954144477844, "Finetune/Step": 3061, "Finetune/Step Time": 4.314150754362345} +{"Finetune/Learning Rate": 7.621502275621518e-06, "Finetune/Loss": 0.8854680061340332, "Finetune/Loss (Raw)": 0.8487895727157593, "Finetune/Step": 3062, "Finetune/Step Time": 4.342358188703656} +{"Finetune/Learning Rate": 7.615451904535066e-06, "Finetune/Loss": 0.8851092457771301, "Finetune/Loss (Raw)": 0.8565539121627808, "Finetune/Step": 3063, "Finetune/Step Time": 4.334338275715709} +{"Finetune/Learning Rate": 7.609402458845171e-06, "Finetune/Loss": 0.8842378854751587, "Finetune/Loss (Raw)": 0.9059056639671326, "Finetune/Step": 3064, "Finetune/Step Time": 4.316490489989519} +{"Finetune/Learning Rate": 7.60335394089951e-06, "Finetune/Loss": 0.8823199272155762, "Finetune/Loss (Raw)": 0.8025045990943909, "Finetune/Step": 3065, "Finetune/Step Time": 4.3276097271591425} +{"Finetune/Learning Rate": 7.597306353045393e-06, "Finetune/Loss": 0.8802792429924011, "Finetune/Loss (Raw)": 0.6803928017616272, "Finetune/Step": 3066, "Finetune/Step Time": 4.36097595281899} +{"Finetune/Learning Rate": 7.5912596976297735e-06, "Finetune/Loss": 0.8801125884056091, "Finetune/Loss (Raw)": 0.8772465586662292, "Finetune/Step": 3067, "Finetune/Step Time": 4.361497530713677} +{"Finetune/Learning Rate": 7.58521397699924e-06, "Finetune/Loss": 0.881411612033844, "Finetune/Loss (Raw)": 0.8867725133895874, "Finetune/Step": 3068, "Finetune/Step Time": 4.373125188052654} +{"Finetune/Learning Rate": 7.579169193500014e-06, "Finetune/Loss": 0.8823281526565552, "Finetune/Loss (Raw)": 0.7838095426559448, "Finetune/Step": 3069, "Finetune/Step Time": 4.370514107868075} +{"Finetune/Learning Rate": 7.573125349477964e-06, "Finetune/Loss": 0.8815975189208984, "Finetune/Loss (Raw)": 0.9273938536643982, "Finetune/Step": 3070, "Finetune/Step Time": 4.367623332887888} +{"Finetune/Learning Rate": 7.567082447278582e-06, "Finetune/Loss": 0.8821655511856079, "Finetune/Loss (Raw)": 1.0121183395385742, "Finetune/Step": 3071, "Finetune/Step Time": 4.364945555105805} +{"Finetune/Learning Rate": 7.561040489247008e-06, "Finetune/Loss": 0.8817709684371948, "Finetune/Loss (Raw)": 0.8623052835464478, "Finetune/Step": 3072, "Finetune/Step Time": 4.366580026224256} +{"Finetune/Learning Rate": 7.554999477728001e-06, "Finetune/Loss": 0.8823556900024414, "Finetune/Loss (Raw)": 0.8965940475463867, "Finetune/Step": 3073, "Finetune/Step Time": 4.357867093756795} +{"Finetune/Learning Rate": 7.548959415065961e-06, "Finetune/Loss": 0.8819907903671265, "Finetune/Loss (Raw)": 0.8622746467590332, "Finetune/Step": 3074, "Finetune/Step Time": 4.354161025956273} +{"Finetune/Learning Rate": 7.54292030360492e-06, "Finetune/Loss": 0.8775286674499512, "Finetune/Loss (Raw)": 0.29551756381988525, "Finetune/Step": 3075, "Finetune/Step Time": 4.3770204447209835} +{"Finetune/Learning Rate": 7.5368821456885375e-06, "Finetune/Loss": 0.8759621381759644, "Finetune/Loss (Raw)": 0.753133237361908, "Finetune/Step": 3076, "Finetune/Step Time": 4.363248646259308} +{"Finetune/Learning Rate": 7.530844943660111e-06, "Finetune/Loss": 0.8759192228317261, "Finetune/Loss (Raw)": 0.9354015588760376, "Finetune/Step": 3077, "Finetune/Step Time": 4.362662803381681} +{"Finetune/Learning Rate": 7.524808699862552e-06, "Finetune/Loss": 0.8751527070999146, "Finetune/Loss (Raw)": 0.8080325722694397, "Finetune/Step": 3078, "Finetune/Step Time": 4.358815532177687} +{"Finetune/Learning Rate": 7.518773416638412e-06, "Finetune/Loss": 0.878682553768158, "Finetune/Loss (Raw)": 0.756465494632721, "Finetune/Step": 3079, "Finetune/Step Time": 4.341960318386555} +{"Finetune/Learning Rate": 7.512739096329868e-06, "Finetune/Loss": 0.8772182464599609, "Finetune/Loss (Raw)": 0.7605449557304382, "Finetune/Step": 3080, "Finetune/Step Time": 4.346491374075413} +{"Finetune/Learning Rate": 7.50670574127872e-06, "Finetune/Loss": 0.8759222030639648, "Finetune/Loss (Raw)": 0.8115261793136597, "Finetune/Step": 3081, "Finetune/Step Time": 4.349653432145715} +{"Finetune/Learning Rate": 7.500673353826399e-06, "Finetune/Loss": 0.876500129699707, "Finetune/Loss (Raw)": 0.873609721660614, "Finetune/Step": 3082, "Finetune/Step Time": 4.34977419115603} +{"Finetune/Learning Rate": 7.4946419363139535e-06, "Finetune/Loss": 0.8756275773048401, "Finetune/Loss (Raw)": 0.939289391040802, "Finetune/Step": 3083, "Finetune/Step Time": 4.355329338461161} +{"Finetune/Learning Rate": 7.488611491082056e-06, "Finetune/Loss": 0.8746635317802429, "Finetune/Loss (Raw)": 0.8145039081573486, "Finetune/Step": 3084, "Finetune/Step Time": 4.3672483041882515} +{"Finetune/Learning Rate": 7.482582020471012e-06, "Finetune/Loss": 0.8760572671890259, "Finetune/Loss (Raw)": 1.0706003904342651, "Finetune/Step": 3085, "Finetune/Step Time": 4.359521504491568} +{"Finetune/Learning Rate": 7.476553526820733e-06, "Finetune/Loss": 0.8765315413475037, "Finetune/Loss (Raw)": 0.9722825288772583, "Finetune/Step": 3086, "Finetune/Step Time": 4.358747120946646} +{"Finetune/Learning Rate": 7.4705260124707705e-06, "Finetune/Loss": 0.8810784220695496, "Finetune/Loss (Raw)": 0.9023494720458984, "Finetune/Step": 3087, "Finetune/Step Time": 4.330368826165795} +{"Finetune/Learning Rate": 7.464499479760275e-06, "Finetune/Loss": 0.8796130418777466, "Finetune/Loss (Raw)": 0.7070577144622803, "Finetune/Step": 3088, "Finetune/Step Time": 4.329078273847699} +{"Finetune/Learning Rate": 7.458473931028029e-06, "Finetune/Loss": 0.880443811416626, "Finetune/Loss (Raw)": 0.9439660906791687, "Finetune/Step": 3089, "Finetune/Step Time": 4.333125531673431} +{"Finetune/Learning Rate": 7.452449368612433e-06, "Finetune/Loss": 0.878505527973175, "Finetune/Loss (Raw)": 0.8224751353263855, "Finetune/Step": 3090, "Finetune/Step Time": 4.327029561623931} +{"Finetune/Learning Rate": 7.446425794851498e-06, "Finetune/Loss": 0.8773202896118164, "Finetune/Loss (Raw)": 0.8187353014945984, "Finetune/Step": 3091, "Finetune/Step Time": 4.330435449257493} +{"Finetune/Learning Rate": 7.440403212082863e-06, "Finetune/Loss": 0.8719140291213989, "Finetune/Loss (Raw)": 0.298340767621994, "Finetune/Step": 3092, "Finetune/Step Time": 4.36665702611208} +{"Finetune/Learning Rate": 7.434381622643766e-06, "Finetune/Loss": 0.8712557554244995, "Finetune/Loss (Raw)": 0.8799253702163696, "Finetune/Step": 3093, "Finetune/Step Time": 4.373687546700239} +{"Finetune/Learning Rate": 7.42836102887107e-06, "Finetune/Loss": 0.8719868659973145, "Finetune/Loss (Raw)": 0.9087812304496765, "Finetune/Step": 3094, "Finetune/Step Time": 4.368649072945118} +{"Finetune/Learning Rate": 7.422341433101254e-06, "Finetune/Loss": 0.872783899307251, "Finetune/Loss (Raw)": 0.8816159963607788, "Finetune/Step": 3095, "Finetune/Step Time": 4.349822569638491} +{"Finetune/Learning Rate": 7.4163228376704e-06, "Finetune/Loss": 0.871925950050354, "Finetune/Loss (Raw)": 0.9035902619361877, "Finetune/Step": 3096, "Finetune/Step Time": 4.343786491081119} +{"Finetune/Learning Rate": 7.410305244914216e-06, "Finetune/Loss": 0.8675297498703003, "Finetune/Loss (Raw)": 0.30551227927207947, "Finetune/Step": 3097, "Finetune/Step Time": 4.389397885650396} +{"Finetune/Learning Rate": 7.404288657168006e-06, "Finetune/Loss": 0.8659331202507019, "Finetune/Loss (Raw)": 0.7034306526184082, "Finetune/Step": 3098, "Finetune/Step Time": 4.377408213913441} +{"Finetune/Learning Rate": 7.398273076766687e-06, "Finetune/Loss": 0.866179883480072, "Finetune/Loss (Raw)": 0.9290659427642822, "Finetune/Step": 3099, "Finetune/Step Time": 4.380798809230328} +{"Finetune/Learning Rate": 7.392258506044796e-06, "Finetune/Loss": 0.8673480153083801, "Finetune/Loss (Raw)": 1.0744012594223022, "Finetune/Step": 3100, "Finetune/Step Time": 4.382824959233403} +{"Finetune/Learning Rate": 7.386244947336463e-06, "Finetune/Loss": 0.8662704825401306, "Finetune/Loss (Raw)": 0.6456825733184814, "Finetune/Step": 3101, "Finetune/Step Time": 4.37953263707459} +{"Finetune/Learning Rate": 7.380232402975443e-06, "Finetune/Loss": 0.8666369915008545, "Finetune/Loss (Raw)": 0.9759111404418945, "Finetune/Step": 3102, "Finetune/Step Time": 4.386885743588209} +{"Finetune/Learning Rate": 7.3742208752950776e-06, "Finetune/Loss": 0.8715158700942993, "Finetune/Loss (Raw)": 1.0179482698440552, "Finetune/Step": 3103, "Finetune/Step Time": 4.360082874074578} +{"Finetune/Learning Rate": 7.3682103666283245e-06, "Finetune/Loss": 0.8722885251045227, "Finetune/Loss (Raw)": 0.9222446084022522, "Finetune/Step": 3104, "Finetune/Step Time": 4.354791140183806} +{"Finetune/Learning Rate": 7.362200879307748e-06, "Finetune/Loss": 0.8721994161605835, "Finetune/Loss (Raw)": 0.8973478674888611, "Finetune/Step": 3105, "Finetune/Step Time": 4.345368275418878} +{"Finetune/Learning Rate": 7.3561924156655105e-06, "Finetune/Loss": 0.8710653185844421, "Finetune/Loss (Raw)": 0.8457700610160828, "Finetune/Step": 3106, "Finetune/Step Time": 4.339738752692938} +{"Finetune/Learning Rate": 7.350184978033386e-06, "Finetune/Loss": 0.8719618320465088, "Finetune/Loss (Raw)": 1.006981611251831, "Finetune/Step": 3107, "Finetune/Step Time": 4.332774421200156} +{"Finetune/Learning Rate": 7.344178568742734e-06, "Finetune/Loss": 0.8716963529586792, "Finetune/Loss (Raw)": 0.8079348206520081, "Finetune/Step": 3108, "Finetune/Step Time": 4.324300387874246} +{"Finetune/Learning Rate": 7.338173190124526e-06, "Finetune/Loss": 0.8724719882011414, "Finetune/Loss (Raw)": 0.8869946002960205, "Finetune/Step": 3109, "Finetune/Step Time": 4.305867053568363} +{"Finetune/Learning Rate": 7.3321688445093395e-06, "Finetune/Loss": 0.8726224899291992, "Finetune/Loss (Raw)": 0.8402246832847595, "Finetune/Step": 3110, "Finetune/Step Time": 4.302336249500513} +{"Finetune/Learning Rate": 7.326165534227341e-06, "Finetune/Loss": 0.8715406656265259, "Finetune/Loss (Raw)": 0.8675662279129028, "Finetune/Step": 3111, "Finetune/Step Time": 4.311087243258953} +{"Finetune/Learning Rate": 7.320163261608293e-06, "Finetune/Loss": 0.8714240789413452, "Finetune/Loss (Raw)": 0.9724623560905457, "Finetune/Step": 3112, "Finetune/Step Time": 4.295249290764332} +{"Finetune/Learning Rate": 7.314162028981564e-06, "Finetune/Loss": 0.8714368343353271, "Finetune/Loss (Raw)": 0.8751013875007629, "Finetune/Step": 3113, "Finetune/Step Time": 4.308512698858976} +{"Finetune/Learning Rate": 7.308161838676113e-06, "Finetune/Loss": 0.8761642575263977, "Finetune/Loss (Raw)": 0.9355335831642151, "Finetune/Step": 3114, "Finetune/Step Time": 4.275210291147232} +{"Finetune/Learning Rate": 7.302162693020503e-06, "Finetune/Loss": 0.8762989640235901, "Finetune/Loss (Raw)": 0.8514716029167175, "Finetune/Step": 3115, "Finetune/Step Time": 4.257355876266956} +{"Finetune/Learning Rate": 7.2961645943428804e-06, "Finetune/Loss": 0.8746051788330078, "Finetune/Loss (Raw)": 0.7299700379371643, "Finetune/Step": 3116, "Finetune/Step Time": 4.28391170874238} +{"Finetune/Learning Rate": 7.290167544970989e-06, "Finetune/Loss": 0.8752580285072327, "Finetune/Loss (Raw)": 0.897515594959259, "Finetune/Step": 3117, "Finetune/Step Time": 4.27864115126431} +{"Finetune/Learning Rate": 7.284171547232168e-06, "Finetune/Loss": 0.8710941076278687, "Finetune/Loss (Raw)": 0.47609469294548035, "Finetune/Step": 3118, "Finetune/Step Time": 4.320035068318248} +{"Finetune/Learning Rate": 7.278176603453347e-06, "Finetune/Loss": 0.8704808950424194, "Finetune/Loss (Raw)": 0.8705169558525085, "Finetune/Step": 3119, "Finetune/Step Time": 4.320219000801444} +{"Finetune/Learning Rate": 7.272182715961049e-06, "Finetune/Loss": 0.8714084625244141, "Finetune/Loss (Raw)": 0.905734658241272, "Finetune/Step": 3120, "Finetune/Step Time": 4.314341144636273} +{"Finetune/Learning Rate": 7.266189887081385e-06, "Finetune/Loss": 0.8693205118179321, "Finetune/Loss (Raw)": 0.7314059734344482, "Finetune/Step": 3121, "Finetune/Step Time": 4.325216956436634} +{"Finetune/Learning Rate": 7.2601981191400484e-06, "Finetune/Loss": 0.8683333396911621, "Finetune/Loss (Raw)": 0.8918964266777039, "Finetune/Step": 3122, "Finetune/Step Time": 4.326671181246638} +{"Finetune/Learning Rate": 7.254207414462333e-06, "Finetune/Loss": 0.8677670955657959, "Finetune/Loss (Raw)": 0.833662211894989, "Finetune/Step": 3123, "Finetune/Step Time": 4.321037348359823} +{"Finetune/Learning Rate": 7.248217775373112e-06, "Finetune/Loss": 0.8686214089393616, "Finetune/Loss (Raw)": 0.9854832291603088, "Finetune/Step": 3124, "Finetune/Step Time": 4.305028231814504} +{"Finetune/Learning Rate": 7.242229204196849e-06, "Finetune/Loss": 0.869276762008667, "Finetune/Loss (Raw)": 0.9142224192619324, "Finetune/Step": 3125, "Finetune/Step Time": 4.304974002763629} +{"Finetune/Learning Rate": 7.236241703257594e-06, "Finetune/Loss": 0.8701211810112, "Finetune/Loss (Raw)": 0.9666820168495178, "Finetune/Step": 3126, "Finetune/Step Time": 4.295937296003103} +{"Finetune/Learning Rate": 7.230255274878972e-06, "Finetune/Loss": 0.8698026537895203, "Finetune/Loss (Raw)": 0.9975833296775818, "Finetune/Step": 3127, "Finetune/Step Time": 4.292251002043486} +{"Finetune/Learning Rate": 7.224269921384206e-06, "Finetune/Loss": 0.8682422637939453, "Finetune/Loss (Raw)": 0.6930981874465942, "Finetune/Step": 3128, "Finetune/Step Time": 4.323137279599905} +{"Finetune/Learning Rate": 7.218285645096088e-06, "Finetune/Loss": 0.8681204319000244, "Finetune/Loss (Raw)": 0.8429916501045227, "Finetune/Step": 3129, "Finetune/Step Time": 4.319163518026471} +{"Finetune/Learning Rate": 7.2123024483370056e-06, "Finetune/Loss": 0.867195725440979, "Finetune/Loss (Raw)": 0.8128002882003784, "Finetune/Step": 3130, "Finetune/Step Time": 4.345072822645307} +{"Finetune/Learning Rate": 7.20632033342892e-06, "Finetune/Loss": 0.8669319152832031, "Finetune/Loss (Raw)": 0.8558993339538574, "Finetune/Step": 3131, "Finetune/Step Time": 4.34143640846014} +{"Finetune/Learning Rate": 7.200339302693367e-06, "Finetune/Loss": 0.8628681302070618, "Finetune/Loss (Raw)": 0.4147876799106598, "Finetune/Step": 3132, "Finetune/Step Time": 4.380520710721612} +{"Finetune/Learning Rate": 7.194359358451475e-06, "Finetune/Loss": 0.8644199371337891, "Finetune/Loss (Raw)": 0.9227681159973145, "Finetune/Step": 3133, "Finetune/Step Time": 4.355652837082744} +{"Finetune/Learning Rate": 7.188380503023936e-06, "Finetune/Loss": 0.8638430237770081, "Finetune/Loss (Raw)": 0.8613343834877014, "Finetune/Step": 3134, "Finetune/Step Time": 4.370660737156868} +{"Finetune/Learning Rate": 7.182402738731036e-06, "Finetune/Loss": 0.865065336227417, "Finetune/Loss (Raw)": 0.9226521253585815, "Finetune/Step": 3135, "Finetune/Step Time": 4.373366799205542} +{"Finetune/Learning Rate": 7.176426067892623e-06, "Finetune/Loss": 0.8652464747428894, "Finetune/Loss (Raw)": 0.9599825143814087, "Finetune/Step": 3136, "Finetune/Step Time": 4.380606312304735} +{"Finetune/Learning Rate": 7.170450492828126e-06, "Finetune/Loss": 0.8639853000640869, "Finetune/Loss (Raw)": 0.8210558891296387, "Finetune/Step": 3137, "Finetune/Step Time": 4.396785654127598} +{"Finetune/Learning Rate": 7.164476015856552e-06, "Finetune/Loss": 0.8631515502929688, "Finetune/Loss (Raw)": 0.7543878555297852, "Finetune/Step": 3138, "Finetune/Step Time": 4.40449027903378} +{"Finetune/Learning Rate": 7.158502639296475e-06, "Finetune/Loss": 0.8621010780334473, "Finetune/Loss (Raw)": 0.8809300661087036, "Finetune/Step": 3139, "Finetune/Step Time": 4.405453449115157} +{"Finetune/Learning Rate": 7.152530365466051e-06, "Finetune/Loss": 0.8615622520446777, "Finetune/Loss (Raw)": 0.8670238256454468, "Finetune/Step": 3140, "Finetune/Step Time": 4.410292642191052} +{"Finetune/Learning Rate": 7.146559196683001e-06, "Finetune/Loss": 0.8610698580741882, "Finetune/Loss (Raw)": 0.826751708984375, "Finetune/Step": 3141, "Finetune/Step Time": 4.4064470417797565} +{"Finetune/Learning Rate": 7.140589135264614e-06, "Finetune/Loss": 0.861129641532898, "Finetune/Loss (Raw)": 0.8445969820022583, "Finetune/Step": 3142, "Finetune/Step Time": 4.416960706934333} +{"Finetune/Learning Rate": 7.13462018352776e-06, "Finetune/Loss": 0.8605481386184692, "Finetune/Loss (Raw)": 0.6865121722221375, "Finetune/Step": 3143, "Finetune/Step Time": 4.416015453636646} +{"Finetune/Learning Rate": 7.12865234378887e-06, "Finetune/Loss": 0.8612504005432129, "Finetune/Loss (Raw)": 0.996322512626648, "Finetune/Step": 3144, "Finetune/Step Time": 4.407687272876501} +{"Finetune/Learning Rate": 7.122685618363948e-06, "Finetune/Loss": 0.8628268837928772, "Finetune/Loss (Raw)": 0.9757728576660156, "Finetune/Step": 3145, "Finetune/Step Time": 4.402528012171388} +{"Finetune/Learning Rate": 7.116720009568565e-06, "Finetune/Loss": 0.8635483980178833, "Finetune/Loss (Raw)": 0.9374972581863403, "Finetune/Step": 3146, "Finetune/Step Time": 4.408401995897293} +{"Finetune/Learning Rate": 7.110755519717851e-06, "Finetune/Loss": 0.8639732003211975, "Finetune/Loss (Raw)": 0.9753478169441223, "Finetune/Step": 3147, "Finetune/Step Time": 4.415333911776543} +{"Finetune/Learning Rate": 7.104792151126515e-06, "Finetune/Loss": 0.8653175830841064, "Finetune/Loss (Raw)": 1.041300892829895, "Finetune/Step": 3148, "Finetune/Step Time": 4.43560946546495} +{"Finetune/Learning Rate": 7.098829906108819e-06, "Finetune/Loss": 0.8617209196090698, "Finetune/Loss (Raw)": 0.42117542028427124, "Finetune/Step": 3149, "Finetune/Step Time": 4.472088374197483} +{"Finetune/Learning Rate": 7.092868786978596e-06, "Finetune/Loss": 0.8610628843307495, "Finetune/Loss (Raw)": 0.8862392902374268, "Finetune/Step": 3150, "Finetune/Step Time": 4.469170901924372} +{"Finetune/Learning Rate": 7.086908796049243e-06, "Finetune/Loss": 0.8622676730155945, "Finetune/Loss (Raw)": 0.8898651599884033, "Finetune/Step": 3151, "Finetune/Step Time": 4.470007084310055} +{"Finetune/Learning Rate": 7.08094993563371e-06, "Finetune/Loss": 0.8615283966064453, "Finetune/Loss (Raw)": 0.8219062685966492, "Finetune/Step": 3152, "Finetune/Step Time": 4.464609928429127} +{"Finetune/Learning Rate": 7.074992208044518e-06, "Finetune/Loss": 0.8614518642425537, "Finetune/Loss (Raw)": 0.9194897413253784, "Finetune/Step": 3153, "Finetune/Step Time": 4.464617155492306} +{"Finetune/Learning Rate": 7.069035615593743e-06, "Finetune/Loss": 0.861095666885376, "Finetune/Loss (Raw)": 0.8807982802391052, "Finetune/Step": 3154, "Finetune/Step Time": 4.462006194517016} +{"Finetune/Learning Rate": 7.063080160593026e-06, "Finetune/Loss": 0.860936164855957, "Finetune/Loss (Raw)": 0.9621197581291199, "Finetune/Step": 3155, "Finetune/Step Time": 4.450551722198725} +{"Finetune/Learning Rate": 7.0571258453535585e-06, "Finetune/Loss": 0.8602755069732666, "Finetune/Loss (Raw)": 0.8113895058631897, "Finetune/Step": 3156, "Finetune/Step Time": 4.445390759035945} +{"Finetune/Learning Rate": 7.051172672186093e-06, "Finetune/Loss": 0.8599333763122559, "Finetune/Loss (Raw)": 1.007523775100708, "Finetune/Step": 3157, "Finetune/Step Time": 4.451286122202873} +{"Finetune/Learning Rate": 7.045220643400946e-06, "Finetune/Loss": 0.8552871942520142, "Finetune/Loss (Raw)": 0.2854260802268982, "Finetune/Step": 3158, "Finetune/Step Time": 4.477556863799691} +{"Finetune/Learning Rate": 7.039269761307977e-06, "Finetune/Loss": 0.8548368215560913, "Finetune/Loss (Raw)": 0.848664402961731, "Finetune/Step": 3159, "Finetune/Step Time": 4.4812986850738525} +{"Finetune/Learning Rate": 7.0333200282166146e-06, "Finetune/Loss": 0.8540613651275635, "Finetune/Loss (Raw)": 0.8710379600524902, "Finetune/Step": 3160, "Finetune/Step Time": 4.4717466700822115} +{"Finetune/Learning Rate": 7.027371446435829e-06, "Finetune/Loss": 0.854002833366394, "Finetune/Loss (Raw)": 0.8960192203521729, "Finetune/Step": 3161, "Finetune/Step Time": 4.490793874487281} +{"Finetune/Learning Rate": 7.0214240182741455e-06, "Finetune/Loss": 0.8490824699401855, "Finetune/Loss (Raw)": 0.4090699255466461, "Finetune/Step": 3162, "Finetune/Step Time": 4.529717797413468} +{"Finetune/Learning Rate": 7.015477746039652e-06, "Finetune/Loss": 0.850249707698822, "Finetune/Loss (Raw)": 1.0430816411972046, "Finetune/Step": 3163, "Finetune/Step Time": 4.54083370603621} +{"Finetune/Learning Rate": 7.009532632039975e-06, "Finetune/Loss": 0.8507766723632812, "Finetune/Loss (Raw)": 0.9444770812988281, "Finetune/Step": 3164, "Finetune/Step Time": 4.521908080205321} +{"Finetune/Learning Rate": 7.003588678582303e-06, "Finetune/Loss": 0.8481608033180237, "Finetune/Loss (Raw)": 0.679675281047821, "Finetune/Step": 3165, "Finetune/Step Time": 4.524244904518127} +{"Finetune/Learning Rate": 6.9976458879733635e-06, "Finetune/Loss": 0.8469277620315552, "Finetune/Loss (Raw)": 0.8249671459197998, "Finetune/Step": 3166, "Finetune/Step Time": 4.524578221142292} +{"Finetune/Learning Rate": 6.991704262519436e-06, "Finetune/Loss": 0.8475300669670105, "Finetune/Loss (Raw)": 0.913284957408905, "Finetune/Step": 3167, "Finetune/Step Time": 4.522481124848127} +{"Finetune/Learning Rate": 6.985763804526354e-06, "Finetune/Loss": 0.8481719493865967, "Finetune/Loss (Raw)": 1.0146512985229492, "Finetune/Step": 3168, "Finetune/Step Time": 4.528044881299138} +{"Finetune/Learning Rate": 6.979824516299491e-06, "Finetune/Loss": 0.8446057438850403, "Finetune/Loss (Raw)": 0.31582027673721313, "Finetune/Step": 3169, "Finetune/Step Time": 4.564282491803169} +{"Finetune/Learning Rate": 6.973886400143772e-06, "Finetune/Loss": 0.8435406684875488, "Finetune/Loss (Raw)": 0.7818142175674438, "Finetune/Step": 3170, "Finetune/Step Time": 4.559350945055485} +{"Finetune/Learning Rate": 6.9679494583636595e-06, "Finetune/Loss": 0.8474870920181274, "Finetune/Loss (Raw)": 0.8164153099060059, "Finetune/Step": 3171, "Finetune/Step Time": 4.522583492100239} +{"Finetune/Learning Rate": 6.9620136932631655e-06, "Finetune/Loss": 0.8487666249275208, "Finetune/Loss (Raw)": 0.881494402885437, "Finetune/Step": 3172, "Finetune/Step Time": 4.5042388662695885} +{"Finetune/Learning Rate": 6.956079107145845e-06, "Finetune/Loss": 0.8487570285797119, "Finetune/Loss (Raw)": 0.795994222164154, "Finetune/Step": 3173, "Finetune/Step Time": 4.515734264627099} +{"Finetune/Learning Rate": 6.950145702314795e-06, "Finetune/Loss": 0.848802924156189, "Finetune/Loss (Raw)": 1.0362988710403442, "Finetune/Step": 3174, "Finetune/Step Time": 4.521574659273028} +{"Finetune/Learning Rate": 6.9442134810726615e-06, "Finetune/Loss": 0.848964273929596, "Finetune/Loss (Raw)": 0.8762762546539307, "Finetune/Step": 3175, "Finetune/Step Time": 4.517826799303293} +{"Finetune/Learning Rate": 6.938282445721609e-06, "Finetune/Loss": 0.8489190340042114, "Finetune/Loss (Raw)": 1.033144235610962, "Finetune/Step": 3176, "Finetune/Step Time": 4.5193035919219255} +{"Finetune/Learning Rate": 6.932352598563365e-06, "Finetune/Loss": 0.8487089276313782, "Finetune/Loss (Raw)": 0.8531152009963989, "Finetune/Step": 3177, "Finetune/Step Time": 4.5283651780337095} +{"Finetune/Learning Rate": 6.92642394189919e-06, "Finetune/Loss": 0.8476588129997253, "Finetune/Loss (Raw)": 0.8595204949378967, "Finetune/Step": 3178, "Finetune/Step Time": 4.534450454637408} +{"Finetune/Learning Rate": 6.920496478029873e-06, "Finetune/Loss": 0.8480291366577148, "Finetune/Loss (Raw)": 1.0725599527359009, "Finetune/Step": 3179, "Finetune/Step Time": 4.540081929415464} +{"Finetune/Learning Rate": 6.914570209255758e-06, "Finetune/Loss": 0.8475732803344727, "Finetune/Loss (Raw)": 0.8146315813064575, "Finetune/Step": 3180, "Finetune/Step Time": 4.535650296136737} +{"Finetune/Learning Rate": 6.908645137876699e-06, "Finetune/Loss": 0.8473575711250305, "Finetune/Loss (Raw)": 0.9471679329872131, "Finetune/Step": 3181, "Finetune/Step Time": 4.530934114009142} +{"Finetune/Learning Rate": 6.902721266192112e-06, "Finetune/Loss": 0.8466510772705078, "Finetune/Loss (Raw)": 0.9170464873313904, "Finetune/Step": 3182, "Finetune/Step Time": 4.534085791558027} +{"Finetune/Learning Rate": 6.896798596500929e-06, "Finetune/Loss": 0.84607994556427, "Finetune/Loss (Raw)": 0.8467493057250977, "Finetune/Step": 3183, "Finetune/Step Time": 4.533623864874244} +{"Finetune/Learning Rate": 6.8908771311016276e-06, "Finetune/Loss": 0.8455492854118347, "Finetune/Loss (Raw)": 0.9059092402458191, "Finetune/Step": 3184, "Finetune/Step Time": 4.544304916635156} +{"Finetune/Learning Rate": 6.884956872292216e-06, "Finetune/Loss": 0.8465707302093506, "Finetune/Loss (Raw)": 0.9198579788208008, "Finetune/Step": 3185, "Finetune/Step Time": 4.538516025990248} +{"Finetune/Learning Rate": 6.87903782237022e-06, "Finetune/Loss": 0.8455904722213745, "Finetune/Loss (Raw)": 0.8456144332885742, "Finetune/Step": 3186, "Finetune/Step Time": 4.5429144613444805} +{"Finetune/Learning Rate": 6.8731199836327155e-06, "Finetune/Loss": 0.8456461429595947, "Finetune/Loss (Raw)": 0.9522958397865295, "Finetune/Step": 3187, "Finetune/Step Time": 4.541345551609993} +{"Finetune/Learning Rate": 6.867203358376295e-06, "Finetune/Loss": 0.8447837233543396, "Finetune/Loss (Raw)": 0.8648604154586792, "Finetune/Step": 3188, "Finetune/Step Time": 4.54993930645287} +{"Finetune/Learning Rate": 6.861287948897091e-06, "Finetune/Loss": 0.8439570665359497, "Finetune/Loss (Raw)": 0.8424768447875977, "Finetune/Step": 3189, "Finetune/Step Time": 4.546252554282546} +{"Finetune/Learning Rate": 6.855373757490761e-06, "Finetune/Loss": 0.8444256782531738, "Finetune/Loss (Raw)": 0.9087793827056885, "Finetune/Step": 3190, "Finetune/Step Time": 4.521795930340886} +{"Finetune/Learning Rate": 6.849460786452476e-06, "Finetune/Loss": 0.8451043963432312, "Finetune/Loss (Raw)": 0.9434303641319275, "Finetune/Step": 3191, "Finetune/Step Time": 4.54207868129015} +{"Finetune/Learning Rate": 6.843549038076951e-06, "Finetune/Loss": 0.8446305990219116, "Finetune/Loss (Raw)": 0.8452597260475159, "Finetune/Step": 3192, "Finetune/Step Time": 4.5472407434135675} +{"Finetune/Learning Rate": 6.83763851465842e-06, "Finetune/Loss": 0.8450547456741333, "Finetune/Loss (Raw)": 0.8567933440208435, "Finetune/Step": 3193, "Finetune/Step Time": 4.531555496156216} +{"Finetune/Learning Rate": 6.831729218490645e-06, "Finetune/Loss": 0.8457820415496826, "Finetune/Loss (Raw)": 0.7734889388084412, "Finetune/Step": 3194, "Finetune/Step Time": 4.5039369855076075} +{"Finetune/Learning Rate": 6.825821151866903e-06, "Finetune/Loss": 0.8464530110359192, "Finetune/Loss (Raw)": 0.9631304740905762, "Finetune/Step": 3195, "Finetune/Step Time": 4.516638798639178} +{"Finetune/Learning Rate": 6.819914317079999e-06, "Finetune/Loss": 0.8466663360595703, "Finetune/Loss (Raw)": 0.9140799045562744, "Finetune/Step": 3196, "Finetune/Step Time": 4.506003867834806} +{"Finetune/Learning Rate": 6.814008716422265e-06, "Finetune/Loss": 0.8482108116149902, "Finetune/Loss (Raw)": 0.9814923405647278, "Finetune/Step": 3197, "Finetune/Step Time": 4.491652779281139} +{"Finetune/Learning Rate": 6.808104352185544e-06, "Finetune/Loss": 0.847160816192627, "Finetune/Loss (Raw)": 0.7929978966712952, "Finetune/Step": 3198, "Finetune/Step Time": 4.500249205157161} +{"Finetune/Learning Rate": 6.8022012266612116e-06, "Finetune/Loss": 0.8463475704193115, "Finetune/Loss (Raw)": 0.90802401304245, "Finetune/Step": 3199, "Finetune/Step Time": 4.499916013330221} +{"Finetune/Learning Rate": 6.79629934214015e-06, "Finetune/Loss": 0.8479452133178711, "Finetune/Loss (Raw)": 1.0668083429336548, "Finetune/Step": 3200, "Finetune/Step Time": 4.48919271863997} +{"Finetune/Learning Rate": 6.790398700912764e-06, "Finetune/Loss": 0.847078800201416, "Finetune/Loss (Raw)": 0.7856876254081726, "Finetune/Step": 3201, "Finetune/Step Time": 4.484767211601138} +{"Finetune/Learning Rate": 6.784499305268983e-06, "Finetune/Loss": 0.8470507264137268, "Finetune/Loss (Raw)": 0.8586833477020264, "Finetune/Step": 3202, "Finetune/Step Time": 4.481414092704654} +{"Finetune/Learning Rate": 6.778601157498241e-06, "Finetune/Loss": 0.8513628840446472, "Finetune/Loss (Raw)": 0.8474740386009216, "Finetune/Step": 3203, "Finetune/Step Time": 4.44147134013474} +{"Finetune/Learning Rate": 6.772704259889503e-06, "Finetune/Loss": 0.8520022034645081, "Finetune/Loss (Raw)": 0.8349654078483582, "Finetune/Step": 3204, "Finetune/Step Time": 4.44823632016778} +{"Finetune/Learning Rate": 6.76680861473123e-06, "Finetune/Loss": 0.8510468006134033, "Finetune/Loss (Raw)": 0.8131070137023926, "Finetune/Step": 3205, "Finetune/Step Time": 4.450330171734095} +{"Finetune/Learning Rate": 6.7609142243114115e-06, "Finetune/Loss": 0.8508221507072449, "Finetune/Loss (Raw)": 0.7792825102806091, "Finetune/Step": 3206, "Finetune/Step Time": 4.45138088427484} +{"Finetune/Learning Rate": 6.755021090917545e-06, "Finetune/Loss": 0.8520275354385376, "Finetune/Loss (Raw)": 0.9107548594474792, "Finetune/Step": 3207, "Finetune/Step Time": 4.450908247381449} +{"Finetune/Learning Rate": 6.7491292168366386e-06, "Finetune/Loss": 0.8534491062164307, "Finetune/Loss (Raw)": 0.9425003528594971, "Finetune/Step": 3208, "Finetune/Step Time": 4.443952085450292} +{"Finetune/Learning Rate": 6.743238604355219e-06, "Finetune/Loss": 0.8538048267364502, "Finetune/Loss (Raw)": 0.8570635318756104, "Finetune/Step": 3209, "Finetune/Step Time": 4.446544861420989} +{"Finetune/Learning Rate": 6.737349255759312e-06, "Finetune/Loss": 0.8530863523483276, "Finetune/Loss (Raw)": 0.7816444635391235, "Finetune/Step": 3210, "Finetune/Step Time": 4.439644232392311} +{"Finetune/Learning Rate": 6.731461173334459e-06, "Finetune/Loss": 0.8530057072639465, "Finetune/Loss (Raw)": 0.928969144821167, "Finetune/Step": 3211, "Finetune/Step Time": 4.444412142038345} +{"Finetune/Learning Rate": 6.725574359365714e-06, "Finetune/Loss": 0.8497593402862549, "Finetune/Loss (Raw)": 0.3989711403846741, "Finetune/Step": 3212, "Finetune/Step Time": 4.474615056067705} +{"Finetune/Learning Rate": 6.7196888161376305e-06, "Finetune/Loss": 0.8487377166748047, "Finetune/Loss (Raw)": 0.9398233890533447, "Finetune/Step": 3213, "Finetune/Step Time": 4.472185602411628} +{"Finetune/Learning Rate": 6.71380454593428e-06, "Finetune/Loss": 0.8483797311782837, "Finetune/Loss (Raw)": 0.9264606237411499, "Finetune/Step": 3214, "Finetune/Step Time": 4.484191324561834} +{"Finetune/Learning Rate": 6.707921551039224e-06, "Finetune/Loss": 0.8474994897842407, "Finetune/Loss (Raw)": 0.789678156375885, "Finetune/Step": 3215, "Finetune/Step Time": 4.487319944426417} +{"Finetune/Learning Rate": 6.70203983373554e-06, "Finetune/Loss": 0.8476987481117249, "Finetune/Loss (Raw)": 0.7325683236122131, "Finetune/Step": 3216, "Finetune/Step Time": 4.507631065323949} +{"Finetune/Learning Rate": 6.69615939630581e-06, "Finetune/Loss": 0.8474679589271545, "Finetune/Loss (Raw)": 0.9144232869148254, "Finetune/Step": 3217, "Finetune/Step Time": 4.499312860891223} +{"Finetune/Learning Rate": 6.690280241032117e-06, "Finetune/Loss": 0.8481693863868713, "Finetune/Loss (Raw)": 0.9122583270072937, "Finetune/Step": 3218, "Finetune/Step Time": 4.497237095609307} +{"Finetune/Learning Rate": 6.684402370196048e-06, "Finetune/Loss": 0.8477805852890015, "Finetune/Loss (Raw)": 0.768962025642395, "Finetune/Step": 3219, "Finetune/Step Time": 4.481412597000599} +{"Finetune/Learning Rate": 6.678525786078684e-06, "Finetune/Loss": 0.853748083114624, "Finetune/Loss (Raw)": 1.0621877908706665, "Finetune/Step": 3220, "Finetune/Step Time": 4.4414446372538805} +{"Finetune/Learning Rate": 6.672650490960617e-06, "Finetune/Loss": 0.8532885909080505, "Finetune/Loss (Raw)": 0.8211039304733276, "Finetune/Step": 3221, "Finetune/Step Time": 4.445641594007611} +{"Finetune/Learning Rate": 6.666776487121934e-06, "Finetune/Loss": 0.8488098978996277, "Finetune/Loss (Raw)": 0.3355139493942261, "Finetune/Step": 3222, "Finetune/Step Time": 4.48320528306067} +{"Finetune/Learning Rate": 6.660903776842218e-06, "Finetune/Loss": 0.8485438227653503, "Finetune/Loss (Raw)": 0.847561240196228, "Finetune/Step": 3223, "Finetune/Step Time": 4.488882672041655} +{"Finetune/Learning Rate": 6.655032362400562e-06, "Finetune/Loss": 0.8486539721488953, "Finetune/Loss (Raw)": 0.9176824688911438, "Finetune/Step": 3224, "Finetune/Step Time": 4.490233516320586} +{"Finetune/Learning Rate": 6.649162246075538e-06, "Finetune/Loss": 0.8530389070510864, "Finetune/Loss (Raw)": 0.8667855262756348, "Finetune/Step": 3225, "Finetune/Step Time": 4.449593199416995} +{"Finetune/Learning Rate": 6.6432934301452235e-06, "Finetune/Loss": 0.854500412940979, "Finetune/Loss (Raw)": 0.8905031681060791, "Finetune/Step": 3226, "Finetune/Step Time": 4.453765403479338} +{"Finetune/Learning Rate": 6.637425916887198e-06, "Finetune/Loss": 0.8537492752075195, "Finetune/Loss (Raw)": 0.832924485206604, "Finetune/Step": 3227, "Finetune/Step Time": 4.454173319041729} +{"Finetune/Learning Rate": 6.631559708578524e-06, "Finetune/Loss": 0.8520724773406982, "Finetune/Loss (Raw)": 0.8597604632377625, "Finetune/Step": 3228, "Finetune/Step Time": 4.451959939673543} +{"Finetune/Learning Rate": 6.625694807495766e-06, "Finetune/Loss": 0.8546255826950073, "Finetune/Loss (Raw)": 0.9724830389022827, "Finetune/Step": 3229, "Finetune/Step Time": 4.436307465657592} +{"Finetune/Learning Rate": 6.619831215914974e-06, "Finetune/Loss": 0.8536092042922974, "Finetune/Loss (Raw)": 0.8458179235458374, "Finetune/Step": 3230, "Finetune/Step Time": 4.438821649178863} +{"Finetune/Learning Rate": 6.6139689361116945e-06, "Finetune/Loss": 0.8525509834289551, "Finetune/Loss (Raw)": 0.8824981451034546, "Finetune/Step": 3231, "Finetune/Step Time": 4.451397368684411} +{"Finetune/Learning Rate": 6.608107970360964e-06, "Finetune/Loss": 0.8532792925834656, "Finetune/Loss (Raw)": 1.015467882156372, "Finetune/Step": 3232, "Finetune/Step Time": 4.453403666615486} +{"Finetune/Learning Rate": 6.602248320937308e-06, "Finetune/Loss": 0.8548752069473267, "Finetune/Loss (Raw)": 1.1016274690628052, "Finetune/Step": 3233, "Finetune/Step Time": 4.4565053172409534} +{"Finetune/Learning Rate": 6.596389990114748e-06, "Finetune/Loss": 0.8542706370353699, "Finetune/Loss (Raw)": 0.7683806419372559, "Finetune/Step": 3234, "Finetune/Step Time": 4.478092391043901} +{"Finetune/Learning Rate": 6.5905329801667796e-06, "Finetune/Loss": 0.8543652296066284, "Finetune/Loss (Raw)": 1.019093632698059, "Finetune/Step": 3235, "Finetune/Step Time": 4.4883254412561655} +{"Finetune/Learning Rate": 6.584677293366396e-06, "Finetune/Loss": 0.8538051843643188, "Finetune/Loss (Raw)": 0.7362425327301025, "Finetune/Step": 3236, "Finetune/Step Time": 4.49506975710392} +{"Finetune/Learning Rate": 6.578822931986078e-06, "Finetune/Loss": 0.8544337749481201, "Finetune/Loss (Raw)": 0.9674622416496277, "Finetune/Step": 3237, "Finetune/Step Time": 4.487249322235584} +{"Finetune/Learning Rate": 6.57296989829779e-06, "Finetune/Loss": 0.8510023355484009, "Finetune/Loss (Raw)": 0.4009999632835388, "Finetune/Step": 3238, "Finetune/Step Time": 4.5286959651857615} +{"Finetune/Learning Rate": 6.567118194572972e-06, "Finetune/Loss": 0.8521594405174255, "Finetune/Loss (Raw)": 1.0156762599945068, "Finetune/Step": 3239, "Finetune/Step Time": 4.524099517613649} +{"Finetune/Learning Rate": 6.5612678230825646e-06, "Finetune/Loss": 0.8529080748558044, "Finetune/Loss (Raw)": 1.0682820081710815, "Finetune/Step": 3240, "Finetune/Step Time": 4.525828970596194} +{"Finetune/Learning Rate": 6.555418786096976e-06, "Finetune/Loss": 0.8536174893379211, "Finetune/Loss (Raw)": 0.9659064412117004, "Finetune/Step": 3241, "Finetune/Step Time": 4.514768805354834} +{"Finetune/Learning Rate": 6.549571085886111e-06, "Finetune/Loss": 0.8539476990699768, "Finetune/Loss (Raw)": 0.9778047800064087, "Finetune/Step": 3242, "Finetune/Step Time": 4.511274380609393} +{"Finetune/Learning Rate": 6.5437247247193445e-06, "Finetune/Loss": 0.8546575307846069, "Finetune/Loss (Raw)": 0.9423349499702454, "Finetune/Step": 3243, "Finetune/Step Time": 4.515663368627429} +{"Finetune/Learning Rate": 6.537879704865531e-06, "Finetune/Loss": 0.8564761877059937, "Finetune/Loss (Raw)": 0.9627448916435242, "Finetune/Step": 3244, "Finetune/Step Time": 4.479680588468909} +{"Finetune/Learning Rate": 6.532036028593012e-06, "Finetune/Loss": 0.8572091460227966, "Finetune/Loss (Raw)": 0.991339921951294, "Finetune/Step": 3245, "Finetune/Step Time": 4.484690047800541} +{"Finetune/Learning Rate": 6.526193698169603e-06, "Finetune/Loss": 0.8591527938842773, "Finetune/Loss (Raw)": 0.724877119064331, "Finetune/Step": 3246, "Finetune/Step Time": 4.4484263844788074} +{"Finetune/Learning Rate": 6.520352715862601e-06, "Finetune/Loss": 0.8589826226234436, "Finetune/Loss (Raw)": 0.8487430810928345, "Finetune/Step": 3247, "Finetune/Step Time": 4.447815779596567} +{"Finetune/Learning Rate": 6.514513083938779e-06, "Finetune/Loss": 0.8596933484077454, "Finetune/Loss (Raw)": 0.9967038035392761, "Finetune/Step": 3248, "Finetune/Step Time": 4.451527681201696} +{"Finetune/Learning Rate": 6.508674804664374e-06, "Finetune/Loss": 0.8617534637451172, "Finetune/Loss (Raw)": 0.9950964450836182, "Finetune/Step": 3249, "Finetune/Step Time": 4.440621100366116} +{"Finetune/Learning Rate": 6.50283788030512e-06, "Finetune/Loss": 0.861350953578949, "Finetune/Loss (Raw)": 0.840378999710083, "Finetune/Step": 3250, "Finetune/Step Time": 4.459439052268863} +{"Finetune/Learning Rate": 6.497002313126204e-06, "Finetune/Loss": 0.8615006804466248, "Finetune/Loss (Raw)": 0.8528295159339905, "Finetune/Step": 3251, "Finetune/Step Time": 4.486905312165618} +{"Finetune/Learning Rate": 6.491168105392303e-06, "Finetune/Loss": 0.861820638179779, "Finetune/Loss (Raw)": 1.026437520980835, "Finetune/Step": 3252, "Finetune/Step Time": 4.486820921301842} +{"Finetune/Learning Rate": 6.485335259367558e-06, "Finetune/Loss": 0.8623413443565369, "Finetune/Loss (Raw)": 0.98087078332901, "Finetune/Step": 3253, "Finetune/Step Time": 4.489325150847435} +{"Finetune/Learning Rate": 6.4795037773155765e-06, "Finetune/Loss": 0.8621349334716797, "Finetune/Loss (Raw)": 0.9402651786804199, "Finetune/Step": 3254, "Finetune/Step Time": 4.488491212949157} +{"Finetune/Learning Rate": 6.473673661499451e-06, "Finetune/Loss": 0.8615452647209167, "Finetune/Loss (Raw)": 0.9221073985099792, "Finetune/Step": 3255, "Finetune/Step Time": 4.4994248785078526} +{"Finetune/Learning Rate": 6.467844914181727e-06, "Finetune/Loss": 0.863930881023407, "Finetune/Loss (Raw)": 0.9984502792358398, "Finetune/Step": 3256, "Finetune/Step Time": 4.465068910270929} +{"Finetune/Learning Rate": 6.462017537624437e-06, "Finetune/Loss": 0.8633925318717957, "Finetune/Loss (Raw)": 0.7740831971168518, "Finetune/Step": 3257, "Finetune/Step Time": 4.475292487069964} +{"Finetune/Learning Rate": 6.4561915340890665e-06, "Finetune/Loss": 0.8653616309165955, "Finetune/Loss (Raw)": 1.0648459196090698, "Finetune/Step": 3258, "Finetune/Step Time": 4.452749939635396} +{"Finetune/Learning Rate": 6.450366905836573e-06, "Finetune/Loss": 0.865329384803772, "Finetune/Loss (Raw)": 0.8517788052558899, "Finetune/Step": 3259, "Finetune/Step Time": 4.456210881471634} +{"Finetune/Learning Rate": 6.444543655127383e-06, "Finetune/Loss": 0.8696171641349792, "Finetune/Loss (Raw)": 0.9636143445968628, "Finetune/Step": 3260, "Finetune/Step Time": 4.434261789545417} +{"Finetune/Learning Rate": 6.438721784221385e-06, "Finetune/Loss": 0.8687669038772583, "Finetune/Loss (Raw)": 0.8139362335205078, "Finetune/Step": 3261, "Finetune/Step Time": 4.43475510366261} +{"Finetune/Learning Rate": 6.432901295377936e-06, "Finetune/Loss": 0.869718074798584, "Finetune/Loss (Raw)": 0.9830896258354187, "Finetune/Step": 3262, "Finetune/Step Time": 4.421634487807751} +{"Finetune/Learning Rate": 6.427082190855855e-06, "Finetune/Loss": 0.8703323602676392, "Finetune/Loss (Raw)": 1.001280426979065, "Finetune/Step": 3263, "Finetune/Step Time": 4.426297526806593} +{"Finetune/Learning Rate": 6.4212644729134155e-06, "Finetune/Loss": 0.8694215416908264, "Finetune/Loss (Raw)": 0.8433947563171387, "Finetune/Step": 3264, "Finetune/Step Time": 4.4277042634785175} +{"Finetune/Learning Rate": 6.415448143808367e-06, "Finetune/Loss": 0.871338963508606, "Finetune/Loss (Raw)": 1.066481590270996, "Finetune/Step": 3265, "Finetune/Step Time": 4.411207107827067} +{"Finetune/Learning Rate": 6.409633205797911e-06, "Finetune/Loss": 0.8723223209381104, "Finetune/Loss (Raw)": 0.8802632093429565, "Finetune/Step": 3266, "Finetune/Step Time": 4.405192315578461} +{"Finetune/Learning Rate": 6.403819661138712e-06, "Finetune/Loss": 0.872523844242096, "Finetune/Loss (Raw)": 0.9067215323448181, "Finetune/Step": 3267, "Finetune/Step Time": 4.419338509440422} +{"Finetune/Learning Rate": 6.398007512086897e-06, "Finetune/Loss": 0.87325519323349, "Finetune/Loss (Raw)": 0.9606389403343201, "Finetune/Step": 3268, "Finetune/Step Time": 4.413441484794021} +{"Finetune/Learning Rate": 6.3921967608980375e-06, "Finetune/Loss": 0.8742921948432922, "Finetune/Loss (Raw)": 0.9594824314117432, "Finetune/Step": 3269, "Finetune/Step Time": 4.4215389508754015} +{"Finetune/Learning Rate": 6.386387409827182e-06, "Finetune/Loss": 0.8742752075195312, "Finetune/Loss (Raw)": 0.8424268364906311, "Finetune/Step": 3270, "Finetune/Step Time": 4.4110459219664335} +{"Finetune/Learning Rate": 6.38057946112882e-06, "Finetune/Loss": 0.8756437301635742, "Finetune/Loss (Raw)": 0.8616839647293091, "Finetune/Step": 3271, "Finetune/Step Time": 4.395671684294939} +{"Finetune/Learning Rate": 6.374772917056908e-06, "Finetune/Loss": 0.8752774000167847, "Finetune/Loss (Raw)": 0.9494271278381348, "Finetune/Step": 3272, "Finetune/Step Time": 4.424634685739875} +{"Finetune/Learning Rate": 6.368967779864854e-06, "Finetune/Loss": 0.8738769292831421, "Finetune/Loss (Raw)": 0.7965160608291626, "Finetune/Step": 3273, "Finetune/Step Time": 4.430395733565092} +{"Finetune/Learning Rate": 6.3631640518055085e-06, "Finetune/Loss": 0.8690735697746277, "Finetune/Loss (Raw)": 0.32266470789909363, "Finetune/Step": 3274, "Finetune/Step Time": 4.4641351122409105} +{"Finetune/Learning Rate": 6.3573617351311945e-06, "Finetune/Loss": 0.8675501942634583, "Finetune/Loss (Raw)": 0.7803598642349243, "Finetune/Step": 3275, "Finetune/Step Time": 4.46284131705761} +{"Finetune/Learning Rate": 6.351560832093673e-06, "Finetune/Loss": 0.8664029836654663, "Finetune/Loss (Raw)": 0.8944606781005859, "Finetune/Step": 3276, "Finetune/Step Time": 4.448071541264653} +{"Finetune/Learning Rate": 6.3457613449441654e-06, "Finetune/Loss": 0.8704031109809875, "Finetune/Loss (Raw)": 0.9331888556480408, "Finetune/Step": 3277, "Finetune/Step Time": 4.407434940338135} +{"Finetune/Learning Rate": 6.3399632759333364e-06, "Finetune/Loss": 0.870137095451355, "Finetune/Loss (Raw)": 0.8521835207939148, "Finetune/Step": 3278, "Finetune/Step Time": 4.411424603313208} +{"Finetune/Learning Rate": 6.3341666273113e-06, "Finetune/Loss": 0.8706696033477783, "Finetune/Loss (Raw)": 0.9580299258232117, "Finetune/Step": 3279, "Finetune/Step Time": 4.418955085799098} +{"Finetune/Learning Rate": 6.328371401327629e-06, "Finetune/Loss": 0.8719173073768616, "Finetune/Loss (Raw)": 0.981610894203186, "Finetune/Step": 3280, "Finetune/Step Time": 4.417953094467521} +{"Finetune/Learning Rate": 6.322577600231332e-06, "Finetune/Loss": 0.8727173805236816, "Finetune/Loss (Raw)": 1.0219035148620605, "Finetune/Step": 3281, "Finetune/Step Time": 4.425743330270052} +{"Finetune/Learning Rate": 6.316785226270878e-06, "Finetune/Loss": 0.8725537061691284, "Finetune/Loss (Raw)": 0.8598465323448181, "Finetune/Step": 3282, "Finetune/Step Time": 4.421522781252861} +{"Finetune/Learning Rate": 6.310994281694167e-06, "Finetune/Loss": 0.8679376840591431, "Finetune/Loss (Raw)": 0.3712708055973053, "Finetune/Step": 3283, "Finetune/Step Time": 4.460648177191615} +{"Finetune/Learning Rate": 6.305204768748551e-06, "Finetune/Loss": 0.8683346509933472, "Finetune/Loss (Raw)": 0.8621985912322998, "Finetune/Step": 3284, "Finetune/Step Time": 4.464011497795582} +{"Finetune/Learning Rate": 6.299416689680831e-06, "Finetune/Loss": 0.8680763244628906, "Finetune/Loss (Raw)": 0.9744573831558228, "Finetune/Step": 3285, "Finetune/Step Time": 4.457332167774439} +{"Finetune/Learning Rate": 6.293630046737245e-06, "Finetune/Loss": 0.8726028203964233, "Finetune/Loss (Raw)": 0.8648126721382141, "Finetune/Step": 3286, "Finetune/Step Time": 4.423682713881135} +{"Finetune/Learning Rate": 6.28784484216348e-06, "Finetune/Loss": 0.8730633854866028, "Finetune/Loss (Raw)": 0.9076231122016907, "Finetune/Step": 3287, "Finetune/Step Time": 4.430915238335729} +{"Finetune/Learning Rate": 6.282061078204657e-06, "Finetune/Loss": 0.8738622069358826, "Finetune/Loss (Raw)": 0.9732825756072998, "Finetune/Step": 3288, "Finetune/Step Time": 4.435990458354354} +{"Finetune/Learning Rate": 6.276278757105341e-06, "Finetune/Loss": 0.8732066750526428, "Finetune/Loss (Raw)": 0.8121154308319092, "Finetune/Step": 3289, "Finetune/Step Time": 4.412192467600107} +{"Finetune/Learning Rate": 6.270497881109541e-06, "Finetune/Loss": 0.8773652911186218, "Finetune/Loss (Raw)": 0.9413747191429138, "Finetune/Step": 3290, "Finetune/Step Time": 4.374103356152773} +{"Finetune/Learning Rate": 6.264718452460699e-06, "Finetune/Loss": 0.8752142190933228, "Finetune/Loss (Raw)": 0.7677438259124756, "Finetune/Step": 3291, "Finetune/Step Time": 4.369594028219581} +{"Finetune/Learning Rate": 6.258940473401705e-06, "Finetune/Loss": 0.8743849992752075, "Finetune/Loss (Raw)": 0.8383376598358154, "Finetune/Step": 3292, "Finetune/Step Time": 4.370317624881864} +{"Finetune/Learning Rate": 6.253163946174872e-06, "Finetune/Loss": 0.8762957453727722, "Finetune/Loss (Raw)": 0.9242510199546814, "Finetune/Step": 3293, "Finetune/Step Time": 4.370909819379449} +{"Finetune/Learning Rate": 6.247388873021959e-06, "Finetune/Loss": 0.8763505220413208, "Finetune/Loss (Raw)": 0.8319805264472961, "Finetune/Step": 3294, "Finetune/Step Time": 4.3975680489093065} +{"Finetune/Learning Rate": 6.2416152561841635e-06, "Finetune/Loss": 0.8756961226463318, "Finetune/Loss (Raw)": 0.8295175433158875, "Finetune/Step": 3295, "Finetune/Step Time": 4.400186782702804} +{"Finetune/Learning Rate": 6.2358430979021075e-06, "Finetune/Loss": 0.8749451041221619, "Finetune/Loss (Raw)": 0.9185174107551575, "Finetune/Step": 3296, "Finetune/Step Time": 4.401395263150334} +{"Finetune/Learning Rate": 6.230072400415863e-06, "Finetune/Loss": 0.8788021802902222, "Finetune/Loss (Raw)": 0.809526801109314, "Finetune/Step": 3297, "Finetune/Step Time": 4.387624004855752} +{"Finetune/Learning Rate": 6.224303165964913e-06, "Finetune/Loss": 0.8803861141204834, "Finetune/Loss (Raw)": 0.9845597743988037, "Finetune/Step": 3298, "Finetune/Step Time": 4.385777894407511} +{"Finetune/Learning Rate": 6.21853539678819e-06, "Finetune/Loss": 0.8770172595977783, "Finetune/Loss (Raw)": 0.38520461320877075, "Finetune/Step": 3299, "Finetune/Step Time": 4.422522641718388} +{"Finetune/Learning Rate": 6.212769095124055e-06, "Finetune/Loss": 0.8773486614227295, "Finetune/Loss (Raw)": 0.9239108562469482, "Finetune/Step": 3300, "Finetune/Step Time": 4.425301259383559} +{"Finetune/Learning Rate": 6.207004263210292e-06, "Finetune/Loss": 0.8769344091415405, "Finetune/Loss (Raw)": 0.7429717183113098, "Finetune/Step": 3301, "Finetune/Step Time": 4.414170457050204} +{"Finetune/Learning Rate": 6.2012409032841294e-06, "Finetune/Loss": 0.8763735294342041, "Finetune/Loss (Raw)": 0.9645026922225952, "Finetune/Step": 3302, "Finetune/Step Time": 4.411703063175082} +{"Finetune/Learning Rate": 6.1954790175822e-06, "Finetune/Loss": 0.8763026595115662, "Finetune/Loss (Raw)": 0.867210865020752, "Finetune/Step": 3303, "Finetune/Step Time": 4.40942745283246} +{"Finetune/Learning Rate": 6.189718608340587e-06, "Finetune/Loss": 0.8747223615646362, "Finetune/Loss (Raw)": 0.830861508846283, "Finetune/Step": 3304, "Finetune/Step Time": 4.419202703982592} +{"Finetune/Learning Rate": 6.183959677794795e-06, "Finetune/Loss": 0.8757668733596802, "Finetune/Loss (Raw)": 0.9868177771568298, "Finetune/Step": 3305, "Finetune/Step Time": 4.4093863889575005} +{"Finetune/Learning Rate": 6.1782022281797485e-06, "Finetune/Loss": 0.8755447268486023, "Finetune/Loss (Raw)": 0.8310827612876892, "Finetune/Step": 3306, "Finetune/Step Time": 4.401364099234343} +{"Finetune/Learning Rate": 6.17244626172981e-06, "Finetune/Loss": 0.8751274347305298, "Finetune/Loss (Raw)": 1.0191525220870972, "Finetune/Step": 3307, "Finetune/Step Time": 4.396189961582422} +{"Finetune/Learning Rate": 6.166691780678743e-06, "Finetune/Loss": 0.8764356374740601, "Finetune/Loss (Raw)": 0.9820753931999207, "Finetune/Step": 3308, "Finetune/Step Time": 4.402249816805124} +{"Finetune/Learning Rate": 6.1609387872597595e-06, "Finetune/Loss": 0.8767060041427612, "Finetune/Loss (Raw)": 0.981778085231781, "Finetune/Step": 3309, "Finetune/Step Time": 4.412467325106263} +{"Finetune/Learning Rate": 6.155187283705479e-06, "Finetune/Loss": 0.8758620023727417, "Finetune/Loss (Raw)": 0.8090121746063232, "Finetune/Step": 3310, "Finetune/Step Time": 4.4089667573571205} +{"Finetune/Learning Rate": 6.14943727224795e-06, "Finetune/Loss": 0.8754410743713379, "Finetune/Loss (Raw)": 0.7928712368011475, "Finetune/Step": 3311, "Finetune/Step Time": 4.417230725288391} +{"Finetune/Learning Rate": 6.1436887551186466e-06, "Finetune/Loss": 0.8748587965965271, "Finetune/Loss (Raw)": 0.8313778638839722, "Finetune/Step": 3312, "Finetune/Step Time": 4.41123635135591} +{"Finetune/Learning Rate": 6.137941734548443e-06, "Finetune/Loss": 0.8710585832595825, "Finetune/Loss (Raw)": 0.4334341883659363, "Finetune/Step": 3313, "Finetune/Step Time": 4.444375034421682} +{"Finetune/Learning Rate": 6.132196212767653e-06, "Finetune/Loss": 0.8707398772239685, "Finetune/Loss (Raw)": 0.8048141002655029, "Finetune/Step": 3314, "Finetune/Step Time": 4.4407758098095655} +{"Finetune/Learning Rate": 6.126452192005999e-06, "Finetune/Loss": 0.8719614744186401, "Finetune/Loss (Raw)": 1.1086585521697998, "Finetune/Step": 3315, "Finetune/Step Time": 4.440287314355373} +{"Finetune/Learning Rate": 6.120709674492627e-06, "Finetune/Loss": 0.8716568946838379, "Finetune/Loss (Raw)": 0.825878918170929, "Finetune/Step": 3316, "Finetune/Step Time": 4.431700672954321} +{"Finetune/Learning Rate": 6.114968662456093e-06, "Finetune/Loss": 0.8720163106918335, "Finetune/Loss (Raw)": 0.8884747624397278, "Finetune/Step": 3317, "Finetune/Step Time": 4.433059927076101} +{"Finetune/Learning Rate": 6.10922915812437e-06, "Finetune/Loss": 0.8675503730773926, "Finetune/Loss (Raw)": 0.3371467888355255, "Finetune/Step": 3318, "Finetune/Step Time": 4.4668199345469475} +{"Finetune/Learning Rate": 6.103491163724853e-06, "Finetune/Loss": 0.867778480052948, "Finetune/Loss (Raw)": 0.9726278185844421, "Finetune/Step": 3319, "Finetune/Step Time": 4.4473312851041555} +{"Finetune/Learning Rate": 6.09775468148434e-06, "Finetune/Loss": 0.8678621649742126, "Finetune/Loss (Raw)": 0.855969250202179, "Finetune/Step": 3320, "Finetune/Step Time": 4.439271297305822} +{"Finetune/Learning Rate": 6.092019713629053e-06, "Finetune/Loss": 0.8675585985183716, "Finetune/Loss (Raw)": 0.8179407715797424, "Finetune/Step": 3321, "Finetune/Step Time": 4.438917066901922} +{"Finetune/Learning Rate": 6.086286262384617e-06, "Finetune/Loss": 0.8680757284164429, "Finetune/Loss (Raw)": 0.839678168296814, "Finetune/Step": 3322, "Finetune/Step Time": 4.431966396048665} +{"Finetune/Learning Rate": 6.080554329976071e-06, "Finetune/Loss": 0.8684242963790894, "Finetune/Loss (Raw)": 1.0077495574951172, "Finetune/Step": 3323, "Finetune/Step Time": 4.432661037892103} +{"Finetune/Learning Rate": 6.074823918627872e-06, "Finetune/Loss": 0.8678229451179504, "Finetune/Loss (Raw)": 0.8371055722236633, "Finetune/Step": 3324, "Finetune/Step Time": 4.446253454312682} +{"Finetune/Learning Rate": 6.069095030563877e-06, "Finetune/Loss": 0.8664635419845581, "Finetune/Loss (Raw)": 0.8074843883514404, "Finetune/Step": 3325, "Finetune/Step Time": 4.446042509749532} +{"Finetune/Learning Rate": 6.063367668007356e-06, "Finetune/Loss": 0.8676302433013916, "Finetune/Loss (Raw)": 0.942336916923523, "Finetune/Step": 3326, "Finetune/Step Time": 4.437712375074625} +{"Finetune/Learning Rate": 6.057641833180987e-06, "Finetune/Loss": 0.8663050532341003, "Finetune/Loss (Raw)": 0.7384060621261597, "Finetune/Step": 3327, "Finetune/Step Time": 4.444926271215081} +{"Finetune/Learning Rate": 6.0519175283068534e-06, "Finetune/Loss": 0.8659624457359314, "Finetune/Loss (Raw)": 1.0229518413543701, "Finetune/Step": 3328, "Finetune/Step Time": 4.454485407099128} +{"Finetune/Learning Rate": 6.046194755606448e-06, "Finetune/Loss": 0.8677546977996826, "Finetune/Loss (Raw)": 1.015097737312317, "Finetune/Step": 3329, "Finetune/Step Time": 4.456888101994991} +{"Finetune/Learning Rate": 6.040473517300665e-06, "Finetune/Loss": 0.867216944694519, "Finetune/Loss (Raw)": 0.7898470163345337, "Finetune/Step": 3330, "Finetune/Step Time": 4.476819729432464} +{"Finetune/Learning Rate": 6.0347538156098104e-06, "Finetune/Loss": 0.8678226470947266, "Finetune/Loss (Raw)": 0.9250020980834961, "Finetune/Step": 3331, "Finetune/Step Time": 4.490725200623274} +{"Finetune/Learning Rate": 6.0290356527535836e-06, "Finetune/Loss": 0.8681933879852295, "Finetune/Loss (Raw)": 0.8824300169944763, "Finetune/Step": 3332, "Finetune/Step Time": 4.484125757589936} +{"Finetune/Learning Rate": 6.023319030951091e-06, "Finetune/Loss": 0.869135856628418, "Finetune/Loss (Raw)": 0.9337345361709595, "Finetune/Step": 3333, "Finetune/Step Time": 4.478680184110999} +{"Finetune/Learning Rate": 6.017603952420846e-06, "Finetune/Loss": 0.869952917098999, "Finetune/Loss (Raw)": 0.883866012096405, "Finetune/Step": 3334, "Finetune/Step Time": 4.479071605950594} +{"Finetune/Learning Rate": 6.011890419380756e-06, "Finetune/Loss": 0.86935955286026, "Finetune/Loss (Raw)": 0.8348096013069153, "Finetune/Step": 3335, "Finetune/Step Time": 4.457574758678675} +{"Finetune/Learning Rate": 6.0061784340481366e-06, "Finetune/Loss": 0.8687300682067871, "Finetune/Loss (Raw)": 0.8619167804718018, "Finetune/Step": 3336, "Finetune/Step Time": 4.4625809118151665} +{"Finetune/Learning Rate": 6.000467998639692e-06, "Finetune/Loss": 0.8680208325386047, "Finetune/Loss (Raw)": 0.7662909030914307, "Finetune/Step": 3337, "Finetune/Step Time": 4.472118964418769} +{"Finetune/Learning Rate": 5.994759115371531e-06, "Finetune/Loss": 0.8688347339630127, "Finetune/Loss (Raw)": 0.8858181238174438, "Finetune/Step": 3338, "Finetune/Step Time": 4.475549692288041} +{"Finetune/Learning Rate": 5.989051786459163e-06, "Finetune/Loss": 0.8689911365509033, "Finetune/Loss (Raw)": 0.9489930868148804, "Finetune/Step": 3339, "Finetune/Step Time": 4.457040751352906} +{"Finetune/Learning Rate": 5.98334601411749e-06, "Finetune/Loss": 0.8728703856468201, "Finetune/Loss (Raw)": 0.8955085873603821, "Finetune/Step": 3340, "Finetune/Step Time": 4.421116571873426} +{"Finetune/Learning Rate": 5.977641800560811e-06, "Finetune/Loss": 0.8736413717269897, "Finetune/Loss (Raw)": 1.0385133028030396, "Finetune/Step": 3341, "Finetune/Step Time": 4.42745609767735} +{"Finetune/Learning Rate": 5.971939148002819e-06, "Finetune/Loss": 0.874180793762207, "Finetune/Loss (Raw)": 0.995508074760437, "Finetune/Step": 3342, "Finetune/Step Time": 4.408430119976401} +{"Finetune/Learning Rate": 5.966238058656601e-06, "Finetune/Loss": 0.8752473592758179, "Finetune/Loss (Raw)": 0.9261985421180725, "Finetune/Step": 3343, "Finetune/Step Time": 4.39618106931448} +{"Finetune/Learning Rate": 5.960538534734641e-06, "Finetune/Loss": 0.8751828670501709, "Finetune/Loss (Raw)": 0.7243185043334961, "Finetune/Step": 3344, "Finetune/Step Time": 4.376612221822143} +{"Finetune/Learning Rate": 5.9548405784488095e-06, "Finetune/Loss": 0.8707948923110962, "Finetune/Loss (Raw)": 0.35275840759277344, "Finetune/Step": 3345, "Finetune/Step Time": 4.419808806851506} +{"Finetune/Learning Rate": 5.949144192010378e-06, "Finetune/Loss": 0.8715760111808777, "Finetune/Loss (Raw)": 1.0122421979904175, "Finetune/Step": 3346, "Finetune/Step Time": 4.423688059672713} +{"Finetune/Learning Rate": 5.94344937763e-06, "Finetune/Loss": 0.871532142162323, "Finetune/Loss (Raw)": 0.7633446455001831, "Finetune/Step": 3347, "Finetune/Step Time": 4.424419574439526} +{"Finetune/Learning Rate": 5.9377561375177164e-06, "Finetune/Loss": 0.8704781532287598, "Finetune/Loss (Raw)": 0.9272720217704773, "Finetune/Step": 3348, "Finetune/Step Time": 4.42909849062562} +{"Finetune/Learning Rate": 5.9320644738829706e-06, "Finetune/Loss": 0.8714218139648438, "Finetune/Loss (Raw)": 0.9418960809707642, "Finetune/Step": 3349, "Finetune/Step Time": 4.425356773659587} +{"Finetune/Learning Rate": 5.926374388934579e-06, "Finetune/Loss": 0.8771104216575623, "Finetune/Loss (Raw)": 1.0636606216430664, "Finetune/Step": 3350, "Finetune/Step Time": 4.393263228237629} +{"Finetune/Learning Rate": 5.920685884880762e-06, "Finetune/Loss": 0.877059280872345, "Finetune/Loss (Raw)": 0.841018795967102, "Finetune/Step": 3351, "Finetune/Step Time": 4.400061924010515} +{"Finetune/Learning Rate": 5.914998963929109e-06, "Finetune/Loss": 0.8772141933441162, "Finetune/Loss (Raw)": 0.9375091195106506, "Finetune/Step": 3352, "Finetune/Step Time": 4.404088173061609} +{"Finetune/Learning Rate": 5.9093136282866014e-06, "Finetune/Loss": 0.8780494928359985, "Finetune/Loss (Raw)": 0.9737036824226379, "Finetune/Step": 3353, "Finetune/Step Time": 4.419739672914147} +{"Finetune/Learning Rate": 5.903629880159614e-06, "Finetune/Loss": 0.8783835172653198, "Finetune/Loss (Raw)": 0.9332536458969116, "Finetune/Step": 3354, "Finetune/Step Time": 4.412287129089236} +{"Finetune/Learning Rate": 5.897947721753894e-06, "Finetune/Loss": 0.8794151544570923, "Finetune/Loss (Raw)": 0.9649799466133118, "Finetune/Step": 3355, "Finetune/Step Time": 4.404490157961845} +{"Finetune/Learning Rate": 5.89226715527458e-06, "Finetune/Loss": 0.8800064325332642, "Finetune/Loss (Raw)": 0.9354330897331238, "Finetune/Step": 3356, "Finetune/Step Time": 4.412361443042755} +{"Finetune/Learning Rate": 5.886588182926185e-06, "Finetune/Loss": 0.879148006439209, "Finetune/Loss (Raw)": 0.8626106977462769, "Finetune/Step": 3357, "Finetune/Step Time": 4.413119932636619} +{"Finetune/Learning Rate": 5.880910806912607e-06, "Finetune/Loss": 0.8788878917694092, "Finetune/Loss (Raw)": 0.8125218152999878, "Finetune/Step": 3358, "Finetune/Step Time": 4.407265018671751} +{"Finetune/Learning Rate": 5.8752350294371274e-06, "Finetune/Loss": 0.880208432674408, "Finetune/Loss (Raw)": 1.0515297651290894, "Finetune/Step": 3359, "Finetune/Step Time": 4.386715652421117} +{"Finetune/Learning Rate": 5.8695608527024075e-06, "Finetune/Loss": 0.8799839019775391, "Finetune/Loss (Raw)": 0.9867334365844727, "Finetune/Step": 3360, "Finetune/Step Time": 4.3874097019433975} +{"Finetune/Learning Rate": 5.863888278910475e-06, "Finetune/Loss": 0.8787295818328857, "Finetune/Loss (Raw)": 0.9410707354545593, "Finetune/Step": 3361, "Finetune/Step Time": 4.400619914755225} +{"Finetune/Learning Rate": 5.8582173102627526e-06, "Finetune/Loss": 0.8791476488113403, "Finetune/Loss (Raw)": 0.8218921422958374, "Finetune/Step": 3362, "Finetune/Step Time": 4.3832742013037205} +{"Finetune/Learning Rate": 5.852547948960029e-06, "Finetune/Loss": 0.8784804344177246, "Finetune/Loss (Raw)": 0.9336864352226257, "Finetune/Step": 3363, "Finetune/Step Time": 4.377860475331545} +{"Finetune/Learning Rate": 5.8468801972024755e-06, "Finetune/Loss": 0.8802449703216553, "Finetune/Loss (Raw)": 0.9621034264564514, "Finetune/Step": 3364, "Finetune/Step Time": 4.370091762393713} +{"Finetune/Learning Rate": 5.841214057189638e-06, "Finetune/Loss": 0.8794697523117065, "Finetune/Loss (Raw)": 0.868244469165802, "Finetune/Step": 3365, "Finetune/Step Time": 4.384280644357204} +{"Finetune/Learning Rate": 5.8355495311204266e-06, "Finetune/Loss": 0.8827394247055054, "Finetune/Loss (Raw)": 0.8195131421089172, "Finetune/Step": 3366, "Finetune/Step Time": 4.3561775628477335} +{"Finetune/Learning Rate": 5.829886621193141e-06, "Finetune/Loss": 0.8810878992080688, "Finetune/Loss (Raw)": 0.8042829632759094, "Finetune/Step": 3367, "Finetune/Step Time": 4.352966332808137} +{"Finetune/Learning Rate": 5.824225329605442e-06, "Finetune/Loss": 0.8761702179908752, "Finetune/Loss (Raw)": 0.43881329894065857, "Finetune/Step": 3368, "Finetune/Step Time": 4.390840461477637} +{"Finetune/Learning Rate": 5.818565658554364e-06, "Finetune/Loss": 0.8762427568435669, "Finetune/Loss (Raw)": 0.975191593170166, "Finetune/Step": 3369, "Finetune/Step Time": 4.389812501147389} +{"Finetune/Learning Rate": 5.8129076102363266e-06, "Finetune/Loss": 0.876052975654602, "Finetune/Loss (Raw)": 0.953517735004425, "Finetune/Step": 3370, "Finetune/Step Time": 4.392965979874134} +{"Finetune/Learning Rate": 5.807251186847095e-06, "Finetune/Loss": 0.8740334510803223, "Finetune/Loss (Raw)": 0.6838341951370239, "Finetune/Step": 3371, "Finetune/Step Time": 4.4089367631822824} +{"Finetune/Learning Rate": 5.801596390581817e-06, "Finetune/Loss": 0.8732811212539673, "Finetune/Loss (Raw)": 0.8664456009864807, "Finetune/Step": 3372, "Finetune/Step Time": 4.420415176078677} +{"Finetune/Learning Rate": 5.795943223635018e-06, "Finetune/Loss": 0.8729450106620789, "Finetune/Loss (Raw)": 0.9483186602592468, "Finetune/Step": 3373, "Finetune/Step Time": 4.426496664062142} +{"Finetune/Learning Rate": 5.7902916882005755e-06, "Finetune/Loss": 0.8742626905441284, "Finetune/Loss (Raw)": 0.8935322761535645, "Finetune/Step": 3374, "Finetune/Step Time": 4.428990634158254} +{"Finetune/Learning Rate": 5.784641786471742e-06, "Finetune/Loss": 0.8764250874519348, "Finetune/Loss (Raw)": 1.1255320310592651, "Finetune/Step": 3375, "Finetune/Step Time": 4.426918169483542} +{"Finetune/Learning Rate": 5.778993520641132e-06, "Finetune/Loss": 0.8765315413475037, "Finetune/Loss (Raw)": 1.0103325843811035, "Finetune/Step": 3376, "Finetune/Step Time": 4.419471802189946} +{"Finetune/Learning Rate": 5.7733468929007294e-06, "Finetune/Loss": 0.8750854730606079, "Finetune/Loss (Raw)": 0.8100000023841858, "Finetune/Step": 3377, "Finetune/Step Time": 4.4305833671242} +{"Finetune/Learning Rate": 5.7677019054418805e-06, "Finetune/Loss": 0.8761022090911865, "Finetune/Loss (Raw)": 0.9705213904380798, "Finetune/Step": 3378, "Finetune/Step Time": 4.426506493240595} +{"Finetune/Learning Rate": 5.762058560455287e-06, "Finetune/Loss": 0.8749585151672363, "Finetune/Loss (Raw)": 0.7064306735992432, "Finetune/Step": 3379, "Finetune/Step Time": 4.400967683643103} +{"Finetune/Learning Rate": 5.756416860131036e-06, "Finetune/Loss": 0.8730786442756653, "Finetune/Loss (Raw)": 0.7858171463012695, "Finetune/Step": 3380, "Finetune/Step Time": 4.3988790810108185} +{"Finetune/Learning Rate": 5.750776806658549e-06, "Finetune/Loss": 0.8722521066665649, "Finetune/Loss (Raw)": 0.8750755786895752, "Finetune/Step": 3381, "Finetune/Step Time": 4.393530556932092} +{"Finetune/Learning Rate": 5.745138402226621e-06, "Finetune/Loss": 0.8730082511901855, "Finetune/Loss (Raw)": 1.037052035331726, "Finetune/Step": 3382, "Finetune/Step Time": 4.39571707136929} +{"Finetune/Learning Rate": 5.739501649023413e-06, "Finetune/Loss": 0.872400164604187, "Finetune/Loss (Raw)": 0.844276487827301, "Finetune/Step": 3383, "Finetune/Step Time": 4.383206302300096} +{"Finetune/Learning Rate": 5.7338665492364365e-06, "Finetune/Loss": 0.8719215393066406, "Finetune/Loss (Raw)": 0.9371771812438965, "Finetune/Step": 3384, "Finetune/Step Time": 4.378997003659606} +{"Finetune/Learning Rate": 5.728233105052563e-06, "Finetune/Loss": 0.8719617128372192, "Finetune/Loss (Raw)": 0.7792295217514038, "Finetune/Step": 3385, "Finetune/Step Time": 4.368502235040069} +{"Finetune/Learning Rate": 5.722601318658021e-06, "Finetune/Loss": 0.8704938888549805, "Finetune/Loss (Raw)": 0.8769632577896118, "Finetune/Step": 3386, "Finetune/Step Time": 4.360116807743907} +{"Finetune/Learning Rate": 5.7169711922383984e-06, "Finetune/Loss": 0.8700689077377319, "Finetune/Loss (Raw)": 0.7973809242248535, "Finetune/Step": 3387, "Finetune/Step Time": 4.348332377150655} +{"Finetune/Learning Rate": 5.711342727978637e-06, "Finetune/Loss": 0.868894636631012, "Finetune/Loss (Raw)": 0.8133094906806946, "Finetune/Step": 3388, "Finetune/Step Time": 4.330526819452643} +{"Finetune/Learning Rate": 5.7057159280630316e-06, "Finetune/Loss": 0.8698091506958008, "Finetune/Loss (Raw)": 0.9309907555580139, "Finetune/Step": 3389, "Finetune/Step Time": 4.335135351866484} +{"Finetune/Learning Rate": 5.700090794675241e-06, "Finetune/Loss": 0.8695772886276245, "Finetune/Loss (Raw)": 0.9534187912940979, "Finetune/Step": 3390, "Finetune/Step Time": 4.33928787522018} +{"Finetune/Learning Rate": 5.694467329998264e-06, "Finetune/Loss": 0.8699946403503418, "Finetune/Loss (Raw)": 1.054693341255188, "Finetune/Step": 3391, "Finetune/Step Time": 4.328926471993327} +{"Finetune/Learning Rate": 5.68884553621445e-06, "Finetune/Loss": 0.8700411319732666, "Finetune/Loss (Raw)": 0.8493486046791077, "Finetune/Step": 3392, "Finetune/Step Time": 4.322604266926646} +{"Finetune/Learning Rate": 5.683225415505521e-06, "Finetune/Loss": 0.8694628477096558, "Finetune/Loss (Raw)": 0.9924575686454773, "Finetune/Step": 3393, "Finetune/Step Time": 4.331411685794592} +{"Finetune/Learning Rate": 5.67760697005253e-06, "Finetune/Loss": 0.8695573806762695, "Finetune/Loss (Raw)": 0.8923747539520264, "Finetune/Step": 3394, "Finetune/Step Time": 4.351907193660736} +{"Finetune/Learning Rate": 5.671990202035886e-06, "Finetune/Loss": 0.870142936706543, "Finetune/Loss (Raw)": 0.9816703796386719, "Finetune/Step": 3395, "Finetune/Step Time": 4.335444526746869} +{"Finetune/Learning Rate": 5.666375113635347e-06, "Finetune/Loss": 0.8685380220413208, "Finetune/Loss (Raw)": 0.7552018761634827, "Finetune/Step": 3396, "Finetune/Step Time": 4.33494059741497} +{"Finetune/Learning Rate": 5.660761707030022e-06, "Finetune/Loss": 0.8680695295333862, "Finetune/Loss (Raw)": 0.8995203375816345, "Finetune/Step": 3397, "Finetune/Step Time": 4.327591679990292} +{"Finetune/Learning Rate": 5.655149984398359e-06, "Finetune/Loss": 0.8684053421020508, "Finetune/Loss (Raw)": 0.8854060769081116, "Finetune/Step": 3398, "Finetune/Step Time": 4.320712596178055} +{"Finetune/Learning Rate": 5.649539947918159e-06, "Finetune/Loss": 0.8680949211120605, "Finetune/Loss (Raw)": 0.8219475746154785, "Finetune/Step": 3399, "Finetune/Step Time": 4.329564213752747} +{"Finetune/Learning Rate": 5.643931599766577e-06, "Finetune/Loss": 0.8671990633010864, "Finetune/Loss (Raw)": 0.8347696661949158, "Finetune/Step": 3400, "Finetune/Step Time": 4.302916716784239} +{"Finetune/Learning Rate": 5.638324942120091e-06, "Finetune/Loss": 0.8673338294029236, "Finetune/Loss (Raw)": 0.8137611150741577, "Finetune/Step": 3401, "Finetune/Step Time": 4.294745564460754} +{"Finetune/Learning Rate": 5.632719977154537e-06, "Finetune/Loss": 0.8716636896133423, "Finetune/Loss (Raw)": 0.8768852949142456, "Finetune/Step": 3402, "Finetune/Step Time": 4.255374051630497} +{"Finetune/Learning Rate": 5.6271167070451016e-06, "Finetune/Loss": 0.871956467628479, "Finetune/Loss (Raw)": 0.8178393840789795, "Finetune/Step": 3403, "Finetune/Step Time": 4.26147149130702} +{"Finetune/Learning Rate": 5.621515133966301e-06, "Finetune/Loss": 0.8719626665115356, "Finetune/Loss (Raw)": 0.8952503204345703, "Finetune/Step": 3404, "Finetune/Step Time": 4.257946686819196} +{"Finetune/Learning Rate": 5.615915260091987e-06, "Finetune/Loss": 0.870437502861023, "Finetune/Loss (Raw)": 0.7379738092422485, "Finetune/Step": 3405, "Finetune/Step Time": 4.326288526877761} +{"Finetune/Learning Rate": 5.610317087595372e-06, "Finetune/Loss": 0.870983898639679, "Finetune/Loss (Raw)": 0.9221180081367493, "Finetune/Step": 3406, "Finetune/Step Time": 4.322468275204301} +{"Finetune/Learning Rate": 5.604720618648993e-06, "Finetune/Loss": 0.8704864382743835, "Finetune/Loss (Raw)": 0.8943567872047424, "Finetune/Step": 3407, "Finetune/Step Time": 4.310146560892463} +{"Finetune/Learning Rate": 5.599125855424734e-06, "Finetune/Loss": 0.8701751232147217, "Finetune/Loss (Raw)": 0.9417608976364136, "Finetune/Step": 3408, "Finetune/Step Time": 4.310941310599446} +{"Finetune/Learning Rate": 5.593532800093808e-06, "Finetune/Loss": 0.8691301345825195, "Finetune/Loss (Raw)": 0.8881401419639587, "Finetune/Step": 3409, "Finetune/Step Time": 4.308537667617202} +{"Finetune/Learning Rate": 5.5879414548267754e-06, "Finetune/Loss": 0.8688105344772339, "Finetune/Loss (Raw)": 0.8189375996589661, "Finetune/Step": 3410, "Finetune/Step Time": 4.312135627493262} +{"Finetune/Learning Rate": 5.582351821793525e-06, "Finetune/Loss": 0.8736845850944519, "Finetune/Loss (Raw)": 0.9951498508453369, "Finetune/Step": 3411, "Finetune/Step Time": 4.272621965035796} +{"Finetune/Learning Rate": 5.576763903163281e-06, "Finetune/Loss": 0.8743162155151367, "Finetune/Loss (Raw)": 0.9430541396141052, "Finetune/Step": 3412, "Finetune/Step Time": 4.268021227791905} +{"Finetune/Learning Rate": 5.571177701104614e-06, "Finetune/Loss": 0.8747368454933167, "Finetune/Loss (Raw)": 1.028291940689087, "Finetune/Step": 3413, "Finetune/Step Time": 4.275616018101573} +{"Finetune/Learning Rate": 5.56559321778542e-06, "Finetune/Loss": 0.8760064840316772, "Finetune/Loss (Raw)": 1.0273325443267822, "Finetune/Step": 3414, "Finetune/Step Time": 4.273680754005909} +{"Finetune/Learning Rate": 5.560010455372915e-06, "Finetune/Loss": 0.8746914267539978, "Finetune/Loss (Raw)": 0.7392905354499817, "Finetune/Step": 3415, "Finetune/Step Time": 4.27066957205534} +{"Finetune/Learning Rate": 5.5544294160336735e-06, "Finetune/Loss": 0.8704074621200562, "Finetune/Loss (Raw)": 0.4249413311481476, "Finetune/Step": 3416, "Finetune/Step Time": 4.30677118524909} +{"Finetune/Learning Rate": 5.5488501019335824e-06, "Finetune/Loss": 0.8710166811943054, "Finetune/Loss (Raw)": 0.8900890946388245, "Finetune/Step": 3417, "Finetune/Step Time": 4.312946699559689} +{"Finetune/Learning Rate": 5.5432725152378675e-06, "Finetune/Loss": 0.8706182837486267, "Finetune/Loss (Raw)": 0.8903838992118835, "Finetune/Step": 3418, "Finetune/Step Time": 4.32438993267715} +{"Finetune/Learning Rate": 5.537696658111077e-06, "Finetune/Loss": 0.8716896176338196, "Finetune/Loss (Raw)": 0.9048731923103333, "Finetune/Step": 3419, "Finetune/Step Time": 4.316574689000845} +{"Finetune/Learning Rate": 5.532122532717095e-06, "Finetune/Loss": 0.8700863122940063, "Finetune/Loss (Raw)": 0.6331146359443665, "Finetune/Step": 3420, "Finetune/Step Time": 4.3460537027567625} +{"Finetune/Learning Rate": 5.52655014121913e-06, "Finetune/Loss": 0.8700287342071533, "Finetune/Loss (Raw)": 0.9168844819068909, "Finetune/Step": 3421, "Finetune/Step Time": 4.343560574576259} +{"Finetune/Learning Rate": 5.5209794857797135e-06, "Finetune/Loss": 0.8702232837677002, "Finetune/Loss (Raw)": 0.8568792343139648, "Finetune/Step": 3422, "Finetune/Step Time": 4.336766986176372} +{"Finetune/Learning Rate": 5.51541056856072e-06, "Finetune/Loss": 0.8700898885726929, "Finetune/Loss (Raw)": 0.8124483227729797, "Finetune/Step": 3423, "Finetune/Step Time": 4.335372280329466} +{"Finetune/Learning Rate": 5.509843391723332e-06, "Finetune/Loss": 0.8656718730926514, "Finetune/Loss (Raw)": 0.3530101776123047, "Finetune/Step": 3424, "Finetune/Step Time": 4.368110906332731} +{"Finetune/Learning Rate": 5.504277957428053e-06, "Finetune/Loss": 0.8661860823631287, "Finetune/Loss (Raw)": 0.8753431439399719, "Finetune/Step": 3425, "Finetune/Step Time": 4.350576642900705} +{"Finetune/Learning Rate": 5.498714267834731e-06, "Finetune/Loss": 0.8642239570617676, "Finetune/Loss (Raw)": 0.7334055304527283, "Finetune/Step": 3426, "Finetune/Step Time": 4.352832326665521} +{"Finetune/Learning Rate": 5.493152325102523e-06, "Finetune/Loss": 0.8688098192214966, "Finetune/Loss (Raw)": 0.972187340259552, "Finetune/Step": 3427, "Finetune/Step Time": 4.31774222292006} +{"Finetune/Learning Rate": 5.487592131389907e-06, "Finetune/Loss": 0.8686395287513733, "Finetune/Loss (Raw)": 0.9021183252334595, "Finetune/Step": 3428, "Finetune/Step Time": 4.325384436175227} +{"Finetune/Learning Rate": 5.482033688854689e-06, "Finetune/Loss": 0.8700175285339355, "Finetune/Loss (Raw)": 0.9193572998046875, "Finetune/Step": 3429, "Finetune/Step Time": 4.327065488323569} +{"Finetune/Learning Rate": 5.47647699965399e-06, "Finetune/Loss": 0.8689848780632019, "Finetune/Loss (Raw)": 0.8323202729225159, "Finetune/Step": 3430, "Finetune/Step Time": 4.3212148025631905} +{"Finetune/Learning Rate": 5.470922065944253e-06, "Finetune/Loss": 0.8698774576187134, "Finetune/Loss (Raw)": 0.9814687967300415, "Finetune/Step": 3431, "Finetune/Step Time": 4.325790632516146} +{"Finetune/Learning Rate": 5.465368889881235e-06, "Finetune/Loss": 0.8693883419036865, "Finetune/Loss (Raw)": 0.7682508230209351, "Finetune/Step": 3432, "Finetune/Step Time": 4.327970810234547} +{"Finetune/Learning Rate": 5.459817473620024e-06, "Finetune/Loss": 0.868415117263794, "Finetune/Loss (Raw)": 0.8622468113899231, "Finetune/Step": 3433, "Finetune/Step Time": 4.345897203311324} +{"Finetune/Learning Rate": 5.454267819315016e-06, "Finetune/Loss": 0.8685402870178223, "Finetune/Loss (Raw)": 0.84710294008255, "Finetune/Step": 3434, "Finetune/Step Time": 4.370211457833648} +{"Finetune/Learning Rate": 5.448719929119916e-06, "Finetune/Loss": 0.8672776818275452, "Finetune/Loss (Raw)": 0.8575385808944702, "Finetune/Step": 3435, "Finetune/Step Time": 4.378138663247228} +{"Finetune/Learning Rate": 5.44317380518775e-06, "Finetune/Loss": 0.8657377362251282, "Finetune/Loss (Raw)": 0.7849581241607666, "Finetune/Step": 3436, "Finetune/Step Time": 4.372100969776511} +{"Finetune/Learning Rate": 5.43762944967087e-06, "Finetune/Loss": 0.865043580532074, "Finetune/Loss (Raw)": 0.892928957939148, "Finetune/Step": 3437, "Finetune/Step Time": 4.360861836001277} +{"Finetune/Learning Rate": 5.432086864720929e-06, "Finetune/Loss": 0.8651378750801086, "Finetune/Loss (Raw)": 0.8210819363594055, "Finetune/Step": 3438, "Finetune/Step Time": 4.359570102766156} +{"Finetune/Learning Rate": 5.426546052488892e-06, "Finetune/Loss": 0.8665223121643066, "Finetune/Loss (Raw)": 0.9700800776481628, "Finetune/Step": 3439, "Finetune/Step Time": 4.364679945632815} +{"Finetune/Learning Rate": 5.421007015125046e-06, "Finetune/Loss": 0.8666262626647949, "Finetune/Loss (Raw)": 0.8446773290634155, "Finetune/Step": 3440, "Finetune/Step Time": 4.366267517209053} +{"Finetune/Learning Rate": 5.415469754778979e-06, "Finetune/Loss": 0.8688726425170898, "Finetune/Loss (Raw)": 0.7209804058074951, "Finetune/Step": 3441, "Finetune/Step Time": 4.349718179553747} +{"Finetune/Learning Rate": 5.409934273599594e-06, "Finetune/Loss": 0.8686816692352295, "Finetune/Loss (Raw)": 0.7803630828857422, "Finetune/Step": 3442, "Finetune/Step Time": 4.3625743594020605} +{"Finetune/Learning Rate": 5.404400573735105e-06, "Finetune/Loss": 0.8664412498474121, "Finetune/Loss (Raw)": 0.8218861222267151, "Finetune/Step": 3443, "Finetune/Step Time": 4.372788960114121} +{"Finetune/Learning Rate": 5.3988686573330316e-06, "Finetune/Loss": 0.8671548366546631, "Finetune/Loss (Raw)": 0.9172174334526062, "Finetune/Step": 3444, "Finetune/Step Time": 4.366590242832899} +{"Finetune/Learning Rate": 5.393338526540202e-06, "Finetune/Loss": 0.8657376170158386, "Finetune/Loss (Raw)": 0.7070748209953308, "Finetune/Step": 3445, "Finetune/Step Time": 4.384254515171051} +{"Finetune/Learning Rate": 5.387810183502749e-06, "Finetune/Loss": 0.8697832822799683, "Finetune/Loss (Raw)": 0.8549840450286865, "Finetune/Step": 3446, "Finetune/Step Time": 4.354621326550841} +{"Finetune/Learning Rate": 5.382283630366123e-06, "Finetune/Loss": 0.8699315190315247, "Finetune/Loss (Raw)": 0.9916085600852966, "Finetune/Step": 3447, "Finetune/Step Time": 4.349395776167512} +{"Finetune/Learning Rate": 5.37675886927507e-06, "Finetune/Loss": 0.8701507449150085, "Finetune/Loss (Raw)": 0.8840242028236389, "Finetune/Step": 3448, "Finetune/Step Time": 4.35480809211731} +{"Finetune/Learning Rate": 5.371235902373632e-06, "Finetune/Loss": 0.8714436292648315, "Finetune/Loss (Raw)": 0.9834375381469727, "Finetune/Step": 3449, "Finetune/Step Time": 4.349980847910047} +{"Finetune/Learning Rate": 5.365714731805174e-06, "Finetune/Loss": 0.8719294667243958, "Finetune/Loss (Raw)": 0.9018641114234924, "Finetune/Step": 3450, "Finetune/Step Time": 4.370156973600388} +{"Finetune/Learning Rate": 5.360195359712353e-06, "Finetune/Loss": 0.8706032037734985, "Finetune/Loss (Raw)": 0.8379826545715332, "Finetune/Step": 3451, "Finetune/Step Time": 4.353136699646711} +{"Finetune/Learning Rate": 5.354677788237126e-06, "Finetune/Loss": 0.8716448545455933, "Finetune/Loss (Raw)": 0.9704380035400391, "Finetune/Step": 3452, "Finetune/Step Time": 4.337346099317074} +{"Finetune/Learning Rate": 5.349162019520759e-06, "Finetune/Loss": 0.8719609975814819, "Finetune/Loss (Raw)": 0.8479589819908142, "Finetune/Step": 3453, "Finetune/Step Time": 4.335149193182588} +{"Finetune/Learning Rate": 5.343648055703812e-06, "Finetune/Loss": 0.871062159538269, "Finetune/Loss (Raw)": 0.8272779583930969, "Finetune/Step": 3454, "Finetune/Step Time": 4.343154350295663} +{"Finetune/Learning Rate": 5.338135898926147e-06, "Finetune/Loss": 0.8679448366165161, "Finetune/Loss (Raw)": 0.3393952250480652, "Finetune/Step": 3455, "Finetune/Step Time": 4.372201479971409} +{"Finetune/Learning Rate": 5.33262555132692e-06, "Finetune/Loss": 0.867293119430542, "Finetune/Loss (Raw)": 0.9395251870155334, "Finetune/Step": 3456, "Finetune/Step Time": 4.369968043640256} +{"Finetune/Learning Rate": 5.327117015044599e-06, "Finetune/Loss": 0.8663172721862793, "Finetune/Loss (Raw)": 0.89019775390625, "Finetune/Step": 3457, "Finetune/Step Time": 4.360531905665994} +{"Finetune/Learning Rate": 5.321610292216937e-06, "Finetune/Loss": 0.867164134979248, "Finetune/Loss (Raw)": 0.8982328772544861, "Finetune/Step": 3458, "Finetune/Step Time": 4.362361945211887} +{"Finetune/Learning Rate": 5.316105384980974e-06, "Finetune/Loss": 0.8666071891784668, "Finetune/Loss (Raw)": 0.8537247180938721, "Finetune/Step": 3459, "Finetune/Step Time": 4.352504473179579} +{"Finetune/Learning Rate": 5.3106022954730704e-06, "Finetune/Loss": 0.8673330545425415, "Finetune/Loss (Raw)": 0.975334644317627, "Finetune/Step": 3460, "Finetune/Step Time": 4.354219429194927} +{"Finetune/Learning Rate": 5.3051010258288624e-06, "Finetune/Loss": 0.8626635074615479, "Finetune/Loss (Raw)": 0.33603155612945557, "Finetune/Step": 3461, "Finetune/Step Time": 4.393464038148522} +{"Finetune/Learning Rate": 5.299601578183289e-06, "Finetune/Loss": 0.8630896806716919, "Finetune/Loss (Raw)": 0.9384175539016724, "Finetune/Step": 3462, "Finetune/Step Time": 4.389979215338826} +{"Finetune/Learning Rate": 5.294103954670573e-06, "Finetune/Loss": 0.8632813692092896, "Finetune/Loss (Raw)": 0.8593428134918213, "Finetune/Step": 3463, "Finetune/Step Time": 4.392542785033584} +{"Finetune/Learning Rate": 5.288608157424238e-06, "Finetune/Loss": 0.8638174533843994, "Finetune/Loss (Raw)": 0.9305400848388672, "Finetune/Step": 3464, "Finetune/Step Time": 4.3989055175334215} +{"Finetune/Learning Rate": 5.283114188577094e-06, "Finetune/Loss": 0.8643113374710083, "Finetune/Loss (Raw)": 0.8295106291770935, "Finetune/Step": 3465, "Finetune/Step Time": 4.3937666695564985} +{"Finetune/Learning Rate": 5.277622050261242e-06, "Finetune/Loss": 0.8637096881866455, "Finetune/Loss (Raw)": 0.8088014125823975, "Finetune/Step": 3466, "Finetune/Step Time": 4.392140535637736} +{"Finetune/Learning Rate": 5.272131744608078e-06, "Finetune/Loss": 0.8620221018791199, "Finetune/Loss (Raw)": 0.7329844832420349, "Finetune/Step": 3467, "Finetune/Step Time": 4.39079599827528} +{"Finetune/Learning Rate": 5.2666432737482844e-06, "Finetune/Loss": 0.860753059387207, "Finetune/Loss (Raw)": 0.733073890209198, "Finetune/Step": 3468, "Finetune/Step Time": 4.398560009896755} +{"Finetune/Learning Rate": 5.261156639811817e-06, "Finetune/Loss": 0.8596291542053223, "Finetune/Loss (Raw)": 0.8946504592895508, "Finetune/Step": 3469, "Finetune/Step Time": 4.407110143452883} +{"Finetune/Learning Rate": 5.255671844927944e-06, "Finetune/Loss": 0.8584164381027222, "Finetune/Loss (Raw)": 0.8402828574180603, "Finetune/Step": 3470, "Finetune/Step Time": 4.413958579301834} +{"Finetune/Learning Rate": 5.250188891225203e-06, "Finetune/Loss": 0.8584882020950317, "Finetune/Loss (Raw)": 0.9353806376457214, "Finetune/Step": 3471, "Finetune/Step Time": 4.4202676471322775} +{"Finetune/Learning Rate": 5.24470778083142e-06, "Finetune/Loss": 0.8590825796127319, "Finetune/Loss (Raw)": 0.8003976941108704, "Finetune/Step": 3472, "Finetune/Step Time": 4.421193273738027} +{"Finetune/Learning Rate": 5.2392285158737085e-06, "Finetune/Loss": 0.8642579317092896, "Finetune/Loss (Raw)": 1.0152101516723633, "Finetune/Step": 3473, "Finetune/Step Time": 4.402040837332606} +{"Finetune/Learning Rate": 5.233751098478465e-06, "Finetune/Loss": 0.8625393509864807, "Finetune/Loss (Raw)": 0.7922549247741699, "Finetune/Step": 3474, "Finetune/Step Time": 4.4018716011196375} +{"Finetune/Learning Rate": 5.228275530771366e-06, "Finetune/Loss": 0.8634025454521179, "Finetune/Loss (Raw)": 0.8738444447517395, "Finetune/Step": 3475, "Finetune/Step Time": 4.41403928771615} +{"Finetune/Learning Rate": 5.22280181487737e-06, "Finetune/Loss": 0.8619521856307983, "Finetune/Loss (Raw)": 0.7416210174560547, "Finetune/Step": 3476, "Finetune/Step Time": 4.418842680752277} +{"Finetune/Learning Rate": 5.217329952920727e-06, "Finetune/Loss": 0.8609648942947388, "Finetune/Loss (Raw)": 0.8155223727226257, "Finetune/Step": 3477, "Finetune/Step Time": 4.436683302745223} +{"Finetune/Learning Rate": 5.2118599470249575e-06, "Finetune/Loss": 0.8604617118835449, "Finetune/Loss (Raw)": 0.9992517232894897, "Finetune/Step": 3478, "Finetune/Step Time": 4.428846172988415} +{"Finetune/Learning Rate": 5.206391799312855e-06, "Finetune/Loss": 0.8621487021446228, "Finetune/Loss (Raw)": 1.0569589138031006, "Finetune/Step": 3479, "Finetune/Step Time": 4.4155757036060095} +{"Finetune/Learning Rate": 5.200925511906513e-06, "Finetune/Loss": 0.8620368242263794, "Finetune/Loss (Raw)": 0.9231873750686646, "Finetune/Step": 3480, "Finetune/Step Time": 4.407058486714959} +{"Finetune/Learning Rate": 5.195461086927284e-06, "Finetune/Loss": 0.8618170022964478, "Finetune/Loss (Raw)": 0.9455657005310059, "Finetune/Step": 3481, "Finetune/Step Time": 4.3837717697024345} +{"Finetune/Learning Rate": 5.189998526495808e-06, "Finetune/Loss": 0.8615032434463501, "Finetune/Loss (Raw)": 0.8930902481079102, "Finetune/Step": 3482, "Finetune/Step Time": 4.38354229554534} +{"Finetune/Learning Rate": 5.184537832731996e-06, "Finetune/Loss": 0.8608596324920654, "Finetune/Loss (Raw)": 0.8826053142547607, "Finetune/Step": 3483, "Finetune/Step Time": 4.421819422394037} +{"Finetune/Learning Rate": 5.1790790077550366e-06, "Finetune/Loss": 0.8607672452926636, "Finetune/Loss (Raw)": 0.9236021041870117, "Finetune/Step": 3484, "Finetune/Step Time": 4.432514552026987} +{"Finetune/Learning Rate": 5.173622053683395e-06, "Finetune/Loss": 0.8617875576019287, "Finetune/Loss (Raw)": 0.9932041764259338, "Finetune/Step": 3485, "Finetune/Step Time": 4.429786793887615} +{"Finetune/Learning Rate": 5.168166972634803e-06, "Finetune/Loss": 0.8627619743347168, "Finetune/Loss (Raw)": 0.9372583627700806, "Finetune/Step": 3486, "Finetune/Step Time": 4.423462443053722} +{"Finetune/Learning Rate": 5.1627137667262815e-06, "Finetune/Loss": 0.860509991645813, "Finetune/Loss (Raw)": 0.7632717490196228, "Finetune/Step": 3487, "Finetune/Step Time": 4.416460711508989} +{"Finetune/Learning Rate": 5.1572624380741044e-06, "Finetune/Loss": 0.8586850762367249, "Finetune/Loss (Raw)": 0.7531411051750183, "Finetune/Step": 3488, "Finetune/Step Time": 4.421713402494788} +{"Finetune/Learning Rate": 5.151812988793825e-06, "Finetune/Loss": 0.8589377999305725, "Finetune/Loss (Raw)": 0.9734170436859131, "Finetune/Step": 3489, "Finetune/Step Time": 4.409263519570231} +{"Finetune/Learning Rate": 5.146365421000275e-06, "Finetune/Loss": 0.8587296009063721, "Finetune/Loss (Raw)": 0.7952386736869812, "Finetune/Step": 3490, "Finetune/Step Time": 4.407661337405443} +{"Finetune/Learning Rate": 5.140919736807546e-06, "Finetune/Loss": 0.8591735363006592, "Finetune/Loss (Raw)": 0.9905179738998413, "Finetune/Step": 3491, "Finetune/Step Time": 4.403238911181688} +{"Finetune/Learning Rate": 5.135475938329002e-06, "Finetune/Loss": 0.8586822748184204, "Finetune/Loss (Raw)": 0.8992210626602173, "Finetune/Step": 3492, "Finetune/Step Time": 4.413118958473206} +{"Finetune/Learning Rate": 5.1300340276772756e-06, "Finetune/Loss": 0.8573285341262817, "Finetune/Loss (Raw)": 0.6949679851531982, "Finetune/Step": 3493, "Finetune/Step Time": 4.405376894399524} +{"Finetune/Learning Rate": 5.124594006964263e-06, "Finetune/Loss": 0.8586761951446533, "Finetune/Loss (Raw)": 0.9920170307159424, "Finetune/Step": 3494, "Finetune/Step Time": 4.398201892152429} +{"Finetune/Learning Rate": 5.1191558783011315e-06, "Finetune/Loss": 0.8597107529640198, "Finetune/Loss (Raw)": 0.936701238155365, "Finetune/Step": 3495, "Finetune/Step Time": 4.397821975871921} +{"Finetune/Learning Rate": 5.113719643798312e-06, "Finetune/Loss": 0.8636760115623474, "Finetune/Loss (Raw)": 0.9463667869567871, "Finetune/Step": 3496, "Finetune/Step Time": 4.36643529869616} +{"Finetune/Learning Rate": 5.108285305565508e-06, "Finetune/Loss": 0.8620781898498535, "Finetune/Loss (Raw)": 0.7706690430641174, "Finetune/Step": 3497, "Finetune/Step Time": 4.37521817907691} +{"Finetune/Learning Rate": 5.1028528657116715e-06, "Finetune/Loss": 0.8617538213729858, "Finetune/Loss (Raw)": 0.9120029211044312, "Finetune/Step": 3498, "Finetune/Step Time": 4.373215086758137} +{"Finetune/Learning Rate": 5.097422326345024e-06, "Finetune/Loss": 0.8638951778411865, "Finetune/Loss (Raw)": 0.9579208493232727, "Finetune/Step": 3499, "Finetune/Step Time": 4.358618687838316} +{"Finetune/Learning Rate": 5.091993689573062e-06, "Finetune/Loss": 0.8645905256271362, "Finetune/Loss (Raw)": 0.9554592370986938, "Finetune/Step": 3500, "Finetune/Step Time": 4.349165065214038} +{"Finetune/Learning Rate": 5.086566957502527e-06, "Finetune/Loss": 0.8644353747367859, "Finetune/Loss (Raw)": 0.9284509420394897, "Finetune/Step": 3501, "Finetune/Step Time": 4.334733922034502} +{"Finetune/Learning Rate": 5.081142132239429e-06, "Finetune/Loss": 0.8657227754592896, "Finetune/Loss (Raw)": 1.058326244354248, "Finetune/Step": 3502, "Finetune/Step Time": 4.335435109212995} +{"Finetune/Learning Rate": 5.075719215889039e-06, "Finetune/Loss": 0.8644288778305054, "Finetune/Loss (Raw)": 0.9599025845527649, "Finetune/Step": 3503, "Finetune/Step Time": 4.335605397820473} +{"Finetune/Learning Rate": 5.070298210555881e-06, "Finetune/Loss": 0.8631500601768494, "Finetune/Loss (Raw)": 0.8466507196426392, "Finetune/Step": 3504, "Finetune/Step Time": 4.337124740704894} +{"Finetune/Learning Rate": 5.064879118343744e-06, "Finetune/Loss": 0.864145040512085, "Finetune/Loss (Raw)": 0.9373507499694824, "Finetune/Step": 3505, "Finetune/Step Time": 4.333113504573703} +{"Finetune/Learning Rate": 5.059461941355665e-06, "Finetune/Loss": 0.8630043268203735, "Finetune/Loss (Raw)": 0.8245124220848083, "Finetune/Step": 3506, "Finetune/Step Time": 4.329790597781539} +{"Finetune/Learning Rate": 5.05404668169396e-06, "Finetune/Loss": 0.8627033233642578, "Finetune/Loss (Raw)": 0.6679046750068665, "Finetune/Step": 3507, "Finetune/Step Time": 4.344918992370367} +{"Finetune/Learning Rate": 5.048633341460174e-06, "Finetune/Loss": 0.8634318113327026, "Finetune/Loss (Raw)": 0.8790664076805115, "Finetune/Step": 3508, "Finetune/Step Time": 4.34527905844152} +{"Finetune/Learning Rate": 5.043221922755115e-06, "Finetune/Loss": 0.8641847968101501, "Finetune/Loss (Raw)": 0.9714542031288147, "Finetune/Step": 3509, "Finetune/Step Time": 4.358873328194022} +{"Finetune/Learning Rate": 5.03781242767886e-06, "Finetune/Loss": 0.8623523116111755, "Finetune/Loss (Raw)": 0.8024972081184387, "Finetune/Step": 3510, "Finetune/Step Time": 4.357336850836873} +{"Finetune/Learning Rate": 5.032404858330721e-06, "Finetune/Loss": 0.8624913692474365, "Finetune/Loss (Raw)": 0.8620676398277283, "Finetune/Step": 3511, "Finetune/Step Time": 4.364858336746693} +{"Finetune/Learning Rate": 5.026999216809275e-06, "Finetune/Loss": 0.8622453212738037, "Finetune/Loss (Raw)": 0.905690610408783, "Finetune/Step": 3512, "Finetune/Step Time": 4.362836170941591} +{"Finetune/Learning Rate": 5.021595505212342e-06, "Finetune/Loss": 0.8634843230247498, "Finetune/Loss (Raw)": 0.9378190636634827, "Finetune/Step": 3513, "Finetune/Step Time": 4.362303581088781} +{"Finetune/Learning Rate": 5.016193725637e-06, "Finetune/Loss": 0.8629045486450195, "Finetune/Loss (Raw)": 0.8027539849281311, "Finetune/Step": 3514, "Finetune/Step Time": 4.3670034389942884} +{"Finetune/Learning Rate": 5.0107938801795695e-06, "Finetune/Loss": 0.8633424043655396, "Finetune/Loss (Raw)": 0.8534235954284668, "Finetune/Step": 3515, "Finetune/Step Time": 4.373655961826444} +{"Finetune/Learning Rate": 5.005395970935627e-06, "Finetune/Loss": 0.863774299621582, "Finetune/Loss (Raw)": 0.8685932159423828, "Finetune/Step": 3516, "Finetune/Step Time": 4.374886862933636} +{"Finetune/Learning Rate": 5.000000000000003e-06, "Finetune/Loss": 0.8638824224472046, "Finetune/Loss (Raw)": 0.9448344111442566, "Finetune/Step": 3517, "Finetune/Step Time": 4.37196303717792} +{"Finetune/Learning Rate": 4.994605969466758e-06, "Finetune/Loss": 0.8633202314376831, "Finetune/Loss (Raw)": 0.8814613223075867, "Finetune/Step": 3518, "Finetune/Step Time": 4.39713410101831} +{"Finetune/Learning Rate": 4.989213881429211e-06, "Finetune/Loss": 0.8613119125366211, "Finetune/Loss (Raw)": 0.797617495059967, "Finetune/Step": 3519, "Finetune/Step Time": 4.426785297691822} +{"Finetune/Learning Rate": 4.983823737979933e-06, "Finetune/Loss": 0.8617911338806152, "Finetune/Loss (Raw)": 0.9106903076171875, "Finetune/Step": 3520, "Finetune/Step Time": 4.427834179252386} +{"Finetune/Learning Rate": 4.97843554121073e-06, "Finetune/Loss": 0.8619822263717651, "Finetune/Loss (Raw)": 1.0169148445129395, "Finetune/Step": 3521, "Finetune/Step Time": 4.4158969931304455} +{"Finetune/Learning Rate": 4.973049293212655e-06, "Finetune/Loss": 0.8611352443695068, "Finetune/Loss (Raw)": 0.7839641571044922, "Finetune/Step": 3522, "Finetune/Step Time": 4.411591367796063} +{"Finetune/Learning Rate": 4.967664996076007e-06, "Finetune/Loss": 0.8595859408378601, "Finetune/Loss (Raw)": 0.7833632826805115, "Finetune/Step": 3523, "Finetune/Step Time": 4.436542751267552} +{"Finetune/Learning Rate": 4.962282651890325e-06, "Finetune/Loss": 0.8598021268844604, "Finetune/Loss (Raw)": 0.7828744649887085, "Finetune/Step": 3524, "Finetune/Step Time": 4.445963887497783} +{"Finetune/Learning Rate": 4.9569022627443944e-06, "Finetune/Loss": 0.8609129190444946, "Finetune/Loss (Raw)": 1.0417038202285767, "Finetune/Step": 3525, "Finetune/Step Time": 4.449063628911972} +{"Finetune/Learning Rate": 4.951523830726236e-06, "Finetune/Loss": 0.8606157302856445, "Finetune/Loss (Raw)": 0.847362220287323, "Finetune/Step": 3526, "Finetune/Step Time": 4.456762386485934} +{"Finetune/Learning Rate": 4.946147357923116e-06, "Finetune/Loss": 0.8614858388900757, "Finetune/Loss (Raw)": 0.93332439661026, "Finetune/Step": 3527, "Finetune/Step Time": 4.4447521939873695} +{"Finetune/Learning Rate": 4.940772846421538e-06, "Finetune/Loss": 0.8619459271430969, "Finetune/Loss (Raw)": 0.8936588168144226, "Finetune/Step": 3528, "Finetune/Step Time": 4.455248881131411} +{"Finetune/Learning Rate": 4.935400298307241e-06, "Finetune/Loss": 0.8618388175964355, "Finetune/Loss (Raw)": 0.8000456690788269, "Finetune/Step": 3529, "Finetune/Step Time": 4.465409986674786} +{"Finetune/Learning Rate": 4.930029715665215e-06, "Finetune/Loss": 0.8620861768722534, "Finetune/Loss (Raw)": 0.9085533618927002, "Finetune/Step": 3530, "Finetune/Step Time": 4.473311817273498} +{"Finetune/Learning Rate": 4.9246611005796776e-06, "Finetune/Loss": 0.8632286787033081, "Finetune/Loss (Raw)": 0.9640762805938721, "Finetune/Step": 3531, "Finetune/Step Time": 4.461842922493815} +{"Finetune/Learning Rate": 4.919294455134071e-06, "Finetune/Loss": 0.8626546263694763, "Finetune/Loss (Raw)": 0.8217736482620239, "Finetune/Step": 3532, "Finetune/Step Time": 4.474504118785262} +{"Finetune/Learning Rate": 4.913929781411099e-06, "Finetune/Loss": 0.8594111204147339, "Finetune/Loss (Raw)": 0.3228070139884949, "Finetune/Step": 3533, "Finetune/Step Time": 4.447102915495634} +{"Finetune/Learning Rate": 4.90856708149268e-06, "Finetune/Loss": 0.8594136238098145, "Finetune/Loss (Raw)": 0.9224404096603394, "Finetune/Step": 3534, "Finetune/Step Time": 4.455146010965109} +{"Finetune/Learning Rate": 4.903206357459976e-06, "Finetune/Loss": 0.8550843000411987, "Finetune/Loss (Raw)": 0.3402038812637329, "Finetune/Step": 3535, "Finetune/Step Time": 4.493982220068574} +{"Finetune/Learning Rate": 4.897847611393378e-06, "Finetune/Loss": 0.8542948961257935, "Finetune/Loss (Raw)": 0.8407153487205505, "Finetune/Step": 3536, "Finetune/Step Time": 4.491395739838481} +{"Finetune/Learning Rate": 4.892490845372511e-06, "Finetune/Loss": 0.8553154468536377, "Finetune/Loss (Raw)": 1.0187647342681885, "Finetune/Step": 3537, "Finetune/Step Time": 4.494008839130402} +{"Finetune/Learning Rate": 4.8871360614762295e-06, "Finetune/Loss": 0.8567308187484741, "Finetune/Loss (Raw)": 1.0001120567321777, "Finetune/Step": 3538, "Finetune/Step Time": 4.4962852615863085} +{"Finetune/Learning Rate": 4.881783261782621e-06, "Finetune/Loss": 0.8562985062599182, "Finetune/Loss (Raw)": 0.9398079514503479, "Finetune/Step": 3539, "Finetune/Step Time": 4.5003037098795176} +{"Finetune/Learning Rate": 4.876432448369006e-06, "Finetune/Loss": 0.8558076024055481, "Finetune/Loss (Raw)": 0.880218505859375, "Finetune/Step": 3540, "Finetune/Step Time": 4.509588977321982} +{"Finetune/Learning Rate": 4.871083623311933e-06, "Finetune/Loss": 0.8546429872512817, "Finetune/Loss (Raw)": 0.8792240619659424, "Finetune/Step": 3541, "Finetune/Step Time": 4.500408751890063} +{"Finetune/Learning Rate": 4.865736788687164e-06, "Finetune/Loss": 0.8533843755722046, "Finetune/Loss (Raw)": 0.8662306666374207, "Finetune/Step": 3542, "Finetune/Step Time": 4.51040954887867} +{"Finetune/Learning Rate": 4.860391946569714e-06, "Finetune/Loss": 0.854408860206604, "Finetune/Loss (Raw)": 0.8704182505607605, "Finetune/Step": 3543, "Finetune/Step Time": 4.52309924736619} +{"Finetune/Learning Rate": 4.855049099033806e-06, "Finetune/Loss": 0.8582326173782349, "Finetune/Loss (Raw)": 0.9143929481506348, "Finetune/Step": 3544, "Finetune/Step Time": 4.484198046848178} +{"Finetune/Learning Rate": 4.849708248152897e-06, "Finetune/Loss": 0.8545798063278198, "Finetune/Loss (Raw)": 0.42252305150032043, "Finetune/Step": 3545, "Finetune/Step Time": 4.518973410129547} +{"Finetune/Learning Rate": 4.8443693959996655e-06, "Finetune/Loss": 0.854034960269928, "Finetune/Loss (Raw)": 0.8206437230110168, "Finetune/Step": 3546, "Finetune/Step Time": 4.515890948474407} +{"Finetune/Learning Rate": 4.839032544646016e-06, "Finetune/Loss": 0.8547706604003906, "Finetune/Loss (Raw)": 0.9990413784980774, "Finetune/Step": 3547, "Finetune/Step Time": 4.512700593098998} +{"Finetune/Learning Rate": 4.833697696163077e-06, "Finetune/Loss": 0.8571813106536865, "Finetune/Loss (Raw)": 0.9416733980178833, "Finetune/Step": 3548, "Finetune/Step Time": 4.495835769921541} +{"Finetune/Learning Rate": 4.828364852621195e-06, "Finetune/Loss": 0.8577733039855957, "Finetune/Loss (Raw)": 0.9926692247390747, "Finetune/Step": 3549, "Finetune/Step Time": 4.494347026571631} +{"Finetune/Learning Rate": 4.823034016089949e-06, "Finetune/Loss": 0.853763222694397, "Finetune/Loss (Raw)": 0.3435942232608795, "Finetune/Step": 3550, "Finetune/Step Time": 4.512242024764419} +{"Finetune/Learning Rate": 4.8177051886381345e-06, "Finetune/Loss": 0.8534670472145081, "Finetune/Loss (Raw)": 0.7745330929756165, "Finetune/Step": 3551, "Finetune/Step Time": 4.530289858579636} +{"Finetune/Learning Rate": 4.812378372333751e-06, "Finetune/Loss": 0.8577693104743958, "Finetune/Loss (Raw)": 0.9036983847618103, "Finetune/Step": 3552, "Finetune/Step Time": 4.489044327288866} +{"Finetune/Learning Rate": 4.807053569244047e-06, "Finetune/Loss": 0.856713056564331, "Finetune/Loss (Raw)": 0.7401439547538757, "Finetune/Step": 3553, "Finetune/Step Time": 4.496412308886647} +{"Finetune/Learning Rate": 4.801730781435469e-06, "Finetune/Loss": 0.8585196733474731, "Finetune/Loss (Raw)": 0.9646461009979248, "Finetune/Step": 3554, "Finetune/Step Time": 4.498859601095319} +{"Finetune/Learning Rate": 4.796410010973686e-06, "Finetune/Loss": 0.8591967225074768, "Finetune/Loss (Raw)": 1.0588514804840088, "Finetune/Step": 3555, "Finetune/Step Time": 4.5079912301152945} +{"Finetune/Learning Rate": 4.791091259923587e-06, "Finetune/Loss": 0.8548845648765564, "Finetune/Loss (Raw)": 0.35016316175460815, "Finetune/Step": 3556, "Finetune/Step Time": 4.538229566067457} +{"Finetune/Learning Rate": 4.785774530349275e-06, "Finetune/Loss": 0.8557767868041992, "Finetune/Loss (Raw)": 1.0335613489151, "Finetune/Step": 3557, "Finetune/Step Time": 4.531650269404054} +{"Finetune/Learning Rate": 4.7804598243140664e-06, "Finetune/Loss": 0.8561656475067139, "Finetune/Loss (Raw)": 0.8820996284484863, "Finetune/Step": 3558, "Finetune/Step Time": 4.533581782132387} +{"Finetune/Learning Rate": 4.775147143880496e-06, "Finetune/Loss": 0.8560122847557068, "Finetune/Loss (Raw)": 0.9618358016014099, "Finetune/Step": 3559, "Finetune/Step Time": 4.5317482352256775} +{"Finetune/Learning Rate": 4.769836491110314e-06, "Finetune/Loss": 0.8578033447265625, "Finetune/Loss (Raw)": 0.9975035190582275, "Finetune/Step": 3560, "Finetune/Step Time": 4.515522075816989} +{"Finetune/Learning Rate": 4.764527868064484e-06, "Finetune/Loss": 0.8592066764831543, "Finetune/Loss (Raw)": 1.0418757200241089, "Finetune/Step": 3561, "Finetune/Step Time": 4.4989330265671015} +{"Finetune/Learning Rate": 4.759221276803172e-06, "Finetune/Loss": 0.8582754135131836, "Finetune/Loss (Raw)": 0.7278960943222046, "Finetune/Step": 3562, "Finetune/Step Time": 4.474172800779343} +{"Finetune/Learning Rate": 4.75391671938576e-06, "Finetune/Loss": 0.8575879335403442, "Finetune/Loss (Raw)": 0.7695491313934326, "Finetune/Step": 3563, "Finetune/Step Time": 4.460452644154429} +{"Finetune/Learning Rate": 4.748614197870852e-06, "Finetune/Loss": 0.857878565788269, "Finetune/Loss (Raw)": 0.8221552968025208, "Finetune/Step": 3564, "Finetune/Step Time": 4.462737627327442} +{"Finetune/Learning Rate": 4.743313714316249e-06, "Finetune/Loss": 0.8574529886245728, "Finetune/Loss (Raw)": 0.8384562134742737, "Finetune/Step": 3565, "Finetune/Step Time": 4.465804787352681} +{"Finetune/Learning Rate": 4.738015270778965e-06, "Finetune/Loss": 0.8589112758636475, "Finetune/Loss (Raw)": 1.007741093635559, "Finetune/Step": 3566, "Finetune/Step Time": 4.464963462203741} +{"Finetune/Learning Rate": 4.7327188693152235e-06, "Finetune/Loss": 0.8582404851913452, "Finetune/Loss (Raw)": 0.8842225074768066, "Finetune/Step": 3567, "Finetune/Step Time": 4.473762311041355} +{"Finetune/Learning Rate": 4.727424511980453e-06, "Finetune/Loss": 0.8590067625045776, "Finetune/Loss (Raw)": 0.9427630305290222, "Finetune/Step": 3568, "Finetune/Step Time": 4.4788659196347} +{"Finetune/Learning Rate": 4.722132200829291e-06, "Finetune/Loss": 0.8600785732269287, "Finetune/Loss (Raw)": 0.858159065246582, "Finetune/Step": 3569, "Finetune/Step Time": 4.466600690037012} +{"Finetune/Learning Rate": 4.716841937915582e-06, "Finetune/Loss": 0.8604637384414673, "Finetune/Loss (Raw)": 0.8296746611595154, "Finetune/Step": 3570, "Finetune/Step Time": 4.4554302133619785} +{"Finetune/Learning Rate": 4.71155372529237e-06, "Finetune/Loss": 0.8612531423568726, "Finetune/Loss (Raw)": 0.9229323267936707, "Finetune/Step": 3571, "Finetune/Step Time": 4.447933791205287} +{"Finetune/Learning Rate": 4.706267565011909e-06, "Finetune/Loss": 0.8614379167556763, "Finetune/Loss (Raw)": 0.9408655762672424, "Finetune/Step": 3572, "Finetune/Step Time": 4.449687456712127} +{"Finetune/Learning Rate": 4.700983459125652e-06, "Finetune/Loss": 0.8632779121398926, "Finetune/Loss (Raw)": 0.9425917267799377, "Finetune/Step": 3573, "Finetune/Step Time": 4.431752610951662} +{"Finetune/Learning Rate": 4.695701409684264e-06, "Finetune/Loss": 0.8642126321792603, "Finetune/Loss (Raw)": 0.9746330380439758, "Finetune/Step": 3574, "Finetune/Step Time": 4.429876243695617} +{"Finetune/Learning Rate": 4.690421418737604e-06, "Finetune/Loss": 0.86324143409729, "Finetune/Loss (Raw)": 0.8672974705696106, "Finetune/Step": 3575, "Finetune/Step Time": 4.430921349674463} +{"Finetune/Learning Rate": 4.685143488334723e-06, "Finetune/Loss": 0.8639692068099976, "Finetune/Loss (Raw)": 0.977173924446106, "Finetune/Step": 3576, "Finetune/Step Time": 4.428662026301026} +{"Finetune/Learning Rate": 4.679867620523895e-06, "Finetune/Loss": 0.8631705045700073, "Finetune/Loss (Raw)": 0.8812071084976196, "Finetune/Step": 3577, "Finetune/Step Time": 4.447302779182792} +{"Finetune/Learning Rate": 4.674593817352575e-06, "Finetune/Loss": 0.8637180328369141, "Finetune/Loss (Raw)": 0.9719405770301819, "Finetune/Step": 3578, "Finetune/Step Time": 4.431982830166817} +{"Finetune/Learning Rate": 4.669322080867427e-06, "Finetune/Loss": 0.8645549416542053, "Finetune/Loss (Raw)": 0.9451073408126831, "Finetune/Step": 3579, "Finetune/Step Time": 4.433083409443498} +{"Finetune/Learning Rate": 4.664052413114305e-06, "Finetune/Loss": 0.864195704460144, "Finetune/Loss (Raw)": 0.9244604706764221, "Finetune/Step": 3580, "Finetune/Step Time": 4.440850978717208} +{"Finetune/Learning Rate": 4.658784816138266e-06, "Finetune/Loss": 0.8651790022850037, "Finetune/Loss (Raw)": 0.9738191962242126, "Finetune/Step": 3581, "Finetune/Step Time": 4.456731613725424} +{"Finetune/Learning Rate": 4.653519291983561e-06, "Finetune/Loss": 0.8652815818786621, "Finetune/Loss (Raw)": 0.8404111862182617, "Finetune/Step": 3582, "Finetune/Step Time": 4.458965301513672} +{"Finetune/Learning Rate": 4.648255842693635e-06, "Finetune/Loss": 0.8690313100814819, "Finetune/Loss (Raw)": 0.8193526864051819, "Finetune/Step": 3583, "Finetune/Step Time": 4.41997055709362} +{"Finetune/Learning Rate": 4.642994470311136e-06, "Finetune/Loss": 0.8689635992050171, "Finetune/Loss (Raw)": 0.9308642148971558, "Finetune/Step": 3584, "Finetune/Step Time": 4.415680592879653} +{"Finetune/Learning Rate": 4.6377351768779e-06, "Finetune/Loss": 0.8699966073036194, "Finetune/Loss (Raw)": 1.0224159955978394, "Finetune/Step": 3585, "Finetune/Step Time": 4.41605025716126} +{"Finetune/Learning Rate": 4.632477964434944e-06, "Finetune/Loss": 0.8689290285110474, "Finetune/Loss (Raw)": 0.7615821957588196, "Finetune/Step": 3586, "Finetune/Step Time": 4.413934791460633} +{"Finetune/Learning Rate": 4.6272228350225026e-06, "Finetune/Loss": 0.8686649799346924, "Finetune/Loss (Raw)": 0.8199371695518494, "Finetune/Step": 3587, "Finetune/Step Time": 4.407442027702928} +{"Finetune/Learning Rate": 4.621969790679984e-06, "Finetune/Loss": 0.8665371537208557, "Finetune/Loss (Raw)": 0.7029697895050049, "Finetune/Step": 3588, "Finetune/Step Time": 4.408875880762935} +{"Finetune/Learning Rate": 4.6167188334459936e-06, "Finetune/Loss": 0.8707014322280884, "Finetune/Loss (Raw)": 0.8690522313117981, "Finetune/Step": 3589, "Finetune/Step Time": 4.378027526661754} +{"Finetune/Learning Rate": 4.611469965358323e-06, "Finetune/Loss": 0.8707538843154907, "Finetune/Loss (Raw)": 0.9451415538787842, "Finetune/Step": 3590, "Finetune/Step Time": 4.381349243223667} +{"Finetune/Learning Rate": 4.606223188453956e-06, "Finetune/Loss": 0.8703039288520813, "Finetune/Loss (Raw)": 0.8017464280128479, "Finetune/Step": 3591, "Finetune/Step Time": 4.379663836210966} +{"Finetune/Learning Rate": 4.6009785047690666e-06, "Finetune/Loss": 0.8702804446220398, "Finetune/Loss (Raw)": 0.9275305867195129, "Finetune/Step": 3592, "Finetune/Step Time": 4.373604292050004} +{"Finetune/Learning Rate": 4.595735916339008e-06, "Finetune/Loss": 0.8716350793838501, "Finetune/Loss (Raw)": 1.0029053688049316, "Finetune/Step": 3593, "Finetune/Step Time": 4.364783503115177} +{"Finetune/Learning Rate": 4.590495425198335e-06, "Finetune/Loss": 0.8701133728027344, "Finetune/Loss (Raw)": 0.6140137910842896, "Finetune/Step": 3594, "Finetune/Step Time": 4.377202030271292} +{"Finetune/Learning Rate": 4.585257033380779e-06, "Finetune/Loss": 0.8713712096214294, "Finetune/Loss (Raw)": 0.8939931988716125, "Finetune/Step": 3595, "Finetune/Step Time": 4.3886416014283895} +{"Finetune/Learning Rate": 4.580020742919247e-06, "Finetune/Loss": 0.8736355304718018, "Finetune/Loss (Raw)": 1.0229078531265259, "Finetune/Step": 3596, "Finetune/Step Time": 4.385816834867001} +{"Finetune/Learning Rate": 4.5747865558458524e-06, "Finetune/Loss": 0.8686407804489136, "Finetune/Loss (Raw)": 0.2553254961967468, "Finetune/Step": 3597, "Finetune/Step Time": 4.413797572255135} +{"Finetune/Learning Rate": 4.569554474191878e-06, "Finetune/Loss": 0.868584156036377, "Finetune/Loss (Raw)": 0.8330369591712952, "Finetune/Step": 3598, "Finetune/Step Time": 4.4278069622814655} +{"Finetune/Learning Rate": 4.56432449998779e-06, "Finetune/Loss": 0.8675673007965088, "Finetune/Loss (Raw)": 0.805222749710083, "Finetune/Step": 3599, "Finetune/Step Time": 4.418830942362547} +{"Finetune/Learning Rate": 4.559096635263242e-06, "Finetune/Loss": 0.8647187948226929, "Finetune/Loss (Raw)": 0.43578076362609863, "Finetune/Step": 3600, "Finetune/Step Time": 4.44947680644691} +{"Finetune/Learning Rate": 4.553870882047064e-06, "Finetune/Loss": 0.8640589118003845, "Finetune/Loss (Raw)": 0.9307478666305542, "Finetune/Step": 3601, "Finetune/Step Time": 4.43767637014389} +{"Finetune/Learning Rate": 4.548647242367271e-06, "Finetune/Loss": 0.8644858598709106, "Finetune/Loss (Raw)": 0.8469035029411316, "Finetune/Step": 3602, "Finetune/Step Time": 4.432693621143699} +{"Finetune/Learning Rate": 4.543425718251051e-06, "Finetune/Loss": 0.8642977476119995, "Finetune/Loss (Raw)": 0.849772036075592, "Finetune/Step": 3603, "Finetune/Step Time": 4.425658017396927} +{"Finetune/Learning Rate": 4.538206311724782e-06, "Finetune/Loss": 0.8655552864074707, "Finetune/Loss (Raw)": 0.9025827646255493, "Finetune/Step": 3604, "Finetune/Step Time": 4.422265902161598} +{"Finetune/Learning Rate": 4.532989024814015e-06, "Finetune/Loss": 0.8653430342674255, "Finetune/Loss (Raw)": 0.7883501052856445, "Finetune/Step": 3605, "Finetune/Step Time": 4.408437881618738} +{"Finetune/Learning Rate": 4.527773859543466e-06, "Finetune/Loss": 0.8642925024032593, "Finetune/Loss (Raw)": 0.8647884130477905, "Finetune/Step": 3606, "Finetune/Step Time": 4.420886816456914} +{"Finetune/Learning Rate": 4.52256081793705e-06, "Finetune/Loss": 0.8631645441055298, "Finetune/Loss (Raw)": 0.9125770926475525, "Finetune/Step": 3607, "Finetune/Step Time": 4.424454547464848} +{"Finetune/Learning Rate": 4.517349902017841e-06, "Finetune/Loss": 0.862735390663147, "Finetune/Loss (Raw)": 0.8682602047920227, "Finetune/Step": 3608, "Finetune/Step Time": 4.451914118602872} +{"Finetune/Learning Rate": 4.512141113808097e-06, "Finetune/Loss": 0.862413227558136, "Finetune/Loss (Raw)": 0.904327929019928, "Finetune/Step": 3609, "Finetune/Step Time": 4.452834501862526} +{"Finetune/Learning Rate": 4.5069344553292436e-06, "Finetune/Loss": 0.8615903854370117, "Finetune/Loss (Raw)": 0.7877600193023682, "Finetune/Step": 3610, "Finetune/Step Time": 4.452349016442895} +{"Finetune/Learning Rate": 4.501729928601887e-06, "Finetune/Loss": 0.857365071773529, "Finetune/Loss (Raw)": 0.3417709171772003, "Finetune/Step": 3611, "Finetune/Step Time": 4.448651211336255} +{"Finetune/Learning Rate": 4.4965275356458e-06, "Finetune/Loss": 0.8569058179855347, "Finetune/Loss (Raw)": 0.8648228645324707, "Finetune/Step": 3612, "Finetune/Step Time": 4.4462053664028645} +{"Finetune/Learning Rate": 4.491327278479924e-06, "Finetune/Loss": 0.8552731275558472, "Finetune/Loss (Raw)": 0.7842097878456116, "Finetune/Step": 3613, "Finetune/Step Time": 4.446888163685799} +{"Finetune/Learning Rate": 4.4861291591223934e-06, "Finetune/Loss": 0.8547118902206421, "Finetune/Loss (Raw)": 0.8654196858406067, "Finetune/Step": 3614, "Finetune/Step Time": 4.450922893360257} +{"Finetune/Learning Rate": 4.4809331795904835e-06, "Finetune/Loss": 0.8558836579322815, "Finetune/Loss (Raw)": 0.913262665271759, "Finetune/Step": 3615, "Finetune/Step Time": 4.455740397796035} +{"Finetune/Learning Rate": 4.47573934190065e-06, "Finetune/Loss": 0.8560260534286499, "Finetune/Loss (Raw)": 0.771367073059082, "Finetune/Step": 3616, "Finetune/Step Time": 4.454497933387756} +{"Finetune/Learning Rate": 4.470547648068532e-06, "Finetune/Loss": 0.855073094367981, "Finetune/Loss (Raw)": 0.8514357209205627, "Finetune/Step": 3617, "Finetune/Step Time": 4.462795136496425} +{"Finetune/Learning Rate": 4.465358100108916e-06, "Finetune/Loss": 0.8553444743156433, "Finetune/Loss (Raw)": 0.8299791216850281, "Finetune/Step": 3618, "Finetune/Step Time": 4.470273654907942} +{"Finetune/Learning Rate": 4.460170700035769e-06, "Finetune/Loss": 0.8541955947875977, "Finetune/Loss (Raw)": 0.843467116355896, "Finetune/Step": 3619, "Finetune/Step Time": 4.468003164976835} +{"Finetune/Learning Rate": 4.454985449862215e-06, "Finetune/Loss": 0.8537063598632812, "Finetune/Loss (Raw)": 0.8365947604179382, "Finetune/Step": 3620, "Finetune/Step Time": 4.457560922950506} +{"Finetune/Learning Rate": 4.449802351600551e-06, "Finetune/Loss": 0.8546658158302307, "Finetune/Loss (Raw)": 0.8177725672721863, "Finetune/Step": 3621, "Finetune/Step Time": 4.451113371178508} +{"Finetune/Learning Rate": 4.444621407262237e-06, "Finetune/Loss": 0.8539518117904663, "Finetune/Loss (Raw)": 0.9006222486495972, "Finetune/Step": 3622, "Finetune/Step Time": 4.449880184605718} +{"Finetune/Learning Rate": 4.439442618857891e-06, "Finetune/Loss": 0.853783905506134, "Finetune/Loss (Raw)": 0.9152146577835083, "Finetune/Step": 3623, "Finetune/Step Time": 4.455992380157113} +{"Finetune/Learning Rate": 4.434265988397312e-06, "Finetune/Loss": 0.8539831042289734, "Finetune/Loss (Raw)": 0.9718660116195679, "Finetune/Step": 3624, "Finetune/Step Time": 4.449920861050487} +{"Finetune/Learning Rate": 4.4290915178894355e-06, "Finetune/Loss": 0.8554216623306274, "Finetune/Loss (Raw)": 0.9548061490058899, "Finetune/Step": 3625, "Finetune/Step Time": 4.440613012760878} +{"Finetune/Learning Rate": 4.423919209342376e-06, "Finetune/Loss": 0.854489803314209, "Finetune/Loss (Raw)": 0.7927162647247314, "Finetune/Step": 3626, "Finetune/Step Time": 4.4372583366930485} +{"Finetune/Learning Rate": 4.41874906476341e-06, "Finetune/Loss": 0.85406494140625, "Finetune/Loss (Raw)": 0.9035431742668152, "Finetune/Step": 3627, "Finetune/Step Time": 4.452256824821234} +{"Finetune/Learning Rate": 4.413581086158969e-06, "Finetune/Loss": 0.853522777557373, "Finetune/Loss (Raw)": 0.8860589861869812, "Finetune/Step": 3628, "Finetune/Step Time": 4.457798454910517} +{"Finetune/Learning Rate": 4.408415275534641e-06, "Finetune/Loss": 0.8527401089668274, "Finetune/Loss (Raw)": 0.8282699584960938, "Finetune/Step": 3629, "Finetune/Step Time": 4.464486246928573} +{"Finetune/Learning Rate": 4.4032516348951795e-06, "Finetune/Loss": 0.8511239290237427, "Finetune/Loss (Raw)": 0.8514509201049805, "Finetune/Step": 3630, "Finetune/Step Time": 4.4590990748256445} +{"Finetune/Learning Rate": 4.3980901662444905e-06, "Finetune/Loss": 0.8491953015327454, "Finetune/Loss (Raw)": 0.7130444645881653, "Finetune/Step": 3631, "Finetune/Step Time": 4.473134716972709} +{"Finetune/Learning Rate": 4.39293087158564e-06, "Finetune/Loss": 0.8502500057220459, "Finetune/Loss (Raw)": 0.9816595911979675, "Finetune/Step": 3632, "Finetune/Step Time": 4.4758693147450686} +{"Finetune/Learning Rate": 4.387773752920845e-06, "Finetune/Loss": 0.850631058216095, "Finetune/Loss (Raw)": 0.9861217737197876, "Finetune/Step": 3633, "Finetune/Step Time": 4.47033361159265} +{"Finetune/Learning Rate": 4.382618812251495e-06, "Finetune/Loss": 0.850996732711792, "Finetune/Loss (Raw)": 0.8713153600692749, "Finetune/Step": 3634, "Finetune/Step Time": 4.46292382851243} +{"Finetune/Learning Rate": 4.377466051578111e-06, "Finetune/Loss": 0.8518095016479492, "Finetune/Loss (Raw)": 0.7719405889511108, "Finetune/Step": 3635, "Finetune/Step Time": 4.4468734338879585} +{"Finetune/Learning Rate": 4.372315472900377e-06, "Finetune/Loss": 0.85202956199646, "Finetune/Loss (Raw)": 0.9072362184524536, "Finetune/Step": 3636, "Finetune/Step Time": 4.445893397554755} +{"Finetune/Learning Rate": 4.367167078217141e-06, "Finetune/Loss": 0.8519789576530457, "Finetune/Loss (Raw)": 0.9649754762649536, "Finetune/Step": 3637, "Finetune/Step Time": 4.431399218738079} +{"Finetune/Learning Rate": 4.36202086952639e-06, "Finetune/Loss": 0.8530499935150146, "Finetune/Loss (Raw)": 0.9395875930786133, "Finetune/Step": 3638, "Finetune/Step Time": 4.429420867934823} +{"Finetune/Learning Rate": 4.356876848825266e-06, "Finetune/Loss": 0.8531739711761475, "Finetune/Loss (Raw)": 0.8779450058937073, "Finetune/Step": 3639, "Finetune/Step Time": 4.424209155142307} +{"Finetune/Learning Rate": 4.351735018110066e-06, "Finetune/Loss": 0.8485121130943298, "Finetune/Loss (Raw)": 0.3089633882045746, "Finetune/Step": 3640, "Finetune/Step Time": 4.464152038097382} +{"Finetune/Learning Rate": 4.346595379376232e-06, "Finetune/Loss": 0.8474794626235962, "Finetune/Loss (Raw)": 0.805640697479248, "Finetune/Step": 3641, "Finetune/Step Time": 4.467605277895927} +{"Finetune/Learning Rate": 4.341457934618357e-06, "Finetune/Loss": 0.8486021757125854, "Finetune/Loss (Raw)": 0.9464632272720337, "Finetune/Step": 3642, "Finetune/Step Time": 4.470701454207301} +{"Finetune/Learning Rate": 4.336322685830181e-06, "Finetune/Loss": 0.8491430282592773, "Finetune/Loss (Raw)": 0.9226475954055786, "Finetune/Step": 3643, "Finetune/Step Time": 4.464072339236736} +{"Finetune/Learning Rate": 4.331189635004604e-06, "Finetune/Loss": 0.8488887548446655, "Finetune/Loss (Raw)": 0.8360454440116882, "Finetune/Step": 3644, "Finetune/Step Time": 4.460972910746932} +{"Finetune/Learning Rate": 4.326058784133652e-06, "Finetune/Loss": 0.8479673862457275, "Finetune/Loss (Raw)": 0.8269020318984985, "Finetune/Step": 3645, "Finetune/Step Time": 4.460443541407585} +{"Finetune/Learning Rate": 4.3209301352085075e-06, "Finetune/Loss": 0.8487626314163208, "Finetune/Loss (Raw)": 0.9832581281661987, "Finetune/Step": 3646, "Finetune/Step Time": 4.445020541548729} +{"Finetune/Learning Rate": 4.315803690219507e-06, "Finetune/Loss": 0.849707305431366, "Finetune/Loss (Raw)": 0.9185338616371155, "Finetune/Step": 3647, "Finetune/Step Time": 4.417148249223828} +{"Finetune/Learning Rate": 4.310679451156122e-06, "Finetune/Loss": 0.8500069379806519, "Finetune/Loss (Raw)": 0.949044942855835, "Finetune/Step": 3648, "Finetune/Step Time": 4.4193301517516375} +{"Finetune/Learning Rate": 4.305557420006962e-06, "Finetune/Loss": 0.849511981010437, "Finetune/Loss (Raw)": 0.9535576701164246, "Finetune/Step": 3649, "Finetune/Step Time": 4.4240916427224874} +{"Finetune/Learning Rate": 4.300437598759794e-06, "Finetune/Loss": 0.8513654470443726, "Finetune/Loss (Raw)": 1.0212031602859497, "Finetune/Step": 3650, "Finetune/Step Time": 4.4043346885591745} +{"Finetune/Learning Rate": 4.295319989401522e-06, "Finetune/Loss": 0.8510738611221313, "Finetune/Loss (Raw)": 0.7460404634475708, "Finetune/Step": 3651, "Finetune/Step Time": 4.3939707186073065} +{"Finetune/Learning Rate": 4.2902045939181855e-06, "Finetune/Loss": 0.8523219227790833, "Finetune/Loss (Raw)": 0.9426283240318298, "Finetune/Step": 3652, "Finetune/Step Time": 4.387010205537081} +{"Finetune/Learning Rate": 4.285091414294973e-06, "Finetune/Loss": 0.8516449332237244, "Finetune/Loss (Raw)": 0.955054759979248, "Finetune/Step": 3653, "Finetune/Step Time": 4.384932558983564} +{"Finetune/Learning Rate": 4.2799804525162085e-06, "Finetune/Loss": 0.8519384860992432, "Finetune/Loss (Raw)": 0.8849381804466248, "Finetune/Step": 3654, "Finetune/Step Time": 4.400747660547495} +{"Finetune/Learning Rate": 4.274871710565356e-06, "Finetune/Loss": 0.852870523929596, "Finetune/Loss (Raw)": 1.052621841430664, "Finetune/Step": 3655, "Finetune/Step Time": 4.405339512974024} +{"Finetune/Learning Rate": 4.269765190425015e-06, "Finetune/Loss": 0.8519432544708252, "Finetune/Loss (Raw)": 0.7749689817428589, "Finetune/Step": 3656, "Finetune/Step Time": 4.398374008014798} +{"Finetune/Learning Rate": 4.264660894076934e-06, "Finetune/Loss": 0.8525002002716064, "Finetune/Loss (Raw)": 0.8713318109512329, "Finetune/Step": 3657, "Finetune/Step Time": 4.392758846282959} +{"Finetune/Learning Rate": 4.25955882350199e-06, "Finetune/Loss": 0.8523474335670471, "Finetune/Loss (Raw)": 0.888999342918396, "Finetune/Step": 3658, "Finetune/Step Time": 4.4024256858974695} +{"Finetune/Learning Rate": 4.254458980680188e-06, "Finetune/Loss": 0.8518939018249512, "Finetune/Loss (Raw)": 0.9060297012329102, "Finetune/Step": 3659, "Finetune/Step Time": 4.40542272105813} +{"Finetune/Learning Rate": 4.2493613675906865e-06, "Finetune/Loss": 0.8532652854919434, "Finetune/Loss (Raw)": 0.9973007440567017, "Finetune/Step": 3660, "Finetune/Step Time": 4.3980537708848715} +{"Finetune/Learning Rate": 4.244265986211766e-06, "Finetune/Loss": 0.857637882232666, "Finetune/Loss (Raw)": 0.8825134038925171, "Finetune/Step": 3661, "Finetune/Step Time": 4.363803682848811} +{"Finetune/Learning Rate": 4.239172838520846e-06, "Finetune/Loss": 0.8580176830291748, "Finetune/Loss (Raw)": 0.9710469841957092, "Finetune/Step": 3662, "Finetune/Step Time": 4.354542620480061} +{"Finetune/Learning Rate": 4.2340819264944755e-06, "Finetune/Loss": 0.8636113405227661, "Finetune/Loss (Raw)": 1.0561970472335815, "Finetune/Step": 3663, "Finetune/Step Time": 4.313587974756956} +{"Finetune/Learning Rate": 4.22899325210834e-06, "Finetune/Loss": 0.8634639382362366, "Finetune/Loss (Raw)": 0.8218445777893066, "Finetune/Step": 3664, "Finetune/Step Time": 4.331447031348944} +{"Finetune/Learning Rate": 4.223906817337253e-06, "Finetune/Loss": 0.86260586977005, "Finetune/Loss (Raw)": 0.9089365601539612, "Finetune/Step": 3665, "Finetune/Step Time": 4.322839008644223} +{"Finetune/Learning Rate": 4.218822624155159e-06, "Finetune/Loss": 0.8619664907455444, "Finetune/Loss (Raw)": 0.918263852596283, "Finetune/Step": 3666, "Finetune/Step Time": 4.317838815972209} +{"Finetune/Learning Rate": 4.213740674535141e-06, "Finetune/Loss": 0.8579447269439697, "Finetune/Loss (Raw)": 0.42502105236053467, "Finetune/Step": 3667, "Finetune/Step Time": 4.358638491481543} +{"Finetune/Learning Rate": 4.2086609704494015e-06, "Finetune/Loss": 0.857465922832489, "Finetune/Loss (Raw)": 0.8189346194267273, "Finetune/Step": 3668, "Finetune/Step Time": 4.34358786419034} +{"Finetune/Learning Rate": 4.203583513869267e-06, "Finetune/Loss": 0.857288122177124, "Finetune/Loss (Raw)": 0.8564672470092773, "Finetune/Step": 3669, "Finetune/Step Time": 4.347216974943876} +{"Finetune/Learning Rate": 4.198508306765209e-06, "Finetune/Loss": 0.857585072517395, "Finetune/Loss (Raw)": 0.904239296913147, "Finetune/Step": 3670, "Finetune/Step Time": 4.332148805260658} +{"Finetune/Learning Rate": 4.193435351106811e-06, "Finetune/Loss": 0.858425498008728, "Finetune/Loss (Raw)": 0.9779888391494751, "Finetune/Step": 3671, "Finetune/Step Time": 4.3135408740490675} +{"Finetune/Learning Rate": 4.188364648862791e-06, "Finetune/Loss": 0.8578394651412964, "Finetune/Loss (Raw)": 0.8393853306770325, "Finetune/Step": 3672, "Finetune/Step Time": 4.320948377251625} +{"Finetune/Learning Rate": 4.183296202000988e-06, "Finetune/Loss": 0.8597627282142639, "Finetune/Loss (Raw)": 0.668698787689209, "Finetune/Step": 3673, "Finetune/Step Time": 4.284992773085833} +{"Finetune/Learning Rate": 4.178230012488368e-06, "Finetune/Loss": 0.8599846959114075, "Finetune/Loss (Raw)": 0.8490591645240784, "Finetune/Step": 3674, "Finetune/Step Time": 4.27577892318368} +{"Finetune/Learning Rate": 4.173166082291019e-06, "Finetune/Loss": 0.8551282286643982, "Finetune/Loss (Raw)": 0.3774131238460541, "Finetune/Step": 3675, "Finetune/Step Time": 4.3189388159662485} +{"Finetune/Learning Rate": 4.16810441337415e-06, "Finetune/Loss": 0.8551371097564697, "Finetune/Loss (Raw)": 0.942813515663147, "Finetune/Step": 3676, "Finetune/Step Time": 4.3219861574471} +{"Finetune/Learning Rate": 4.163045007702105e-06, "Finetune/Loss": 0.8549195528030396, "Finetune/Loss (Raw)": 0.964813768863678, "Finetune/Step": 3677, "Finetune/Step Time": 4.321749886497855} +{"Finetune/Learning Rate": 4.157987867238338e-06, "Finetune/Loss": 0.8572770357131958, "Finetune/Loss (Raw)": 0.645348072052002, "Finetune/Step": 3678, "Finetune/Step Time": 4.284177593886852} +{"Finetune/Learning Rate": 4.152932993945418e-06, "Finetune/Loss": 0.8589071035385132, "Finetune/Loss (Raw)": 0.983195960521698, "Finetune/Step": 3679, "Finetune/Step Time": 4.272235481068492} +{"Finetune/Learning Rate": 4.147880389785051e-06, "Finetune/Loss": 0.8589745163917542, "Finetune/Loss (Raw)": 0.9123218059539795, "Finetune/Step": 3680, "Finetune/Step Time": 4.277997316792607} +{"Finetune/Learning Rate": 4.142830056718052e-06, "Finetune/Loss": 0.8607659935951233, "Finetune/Loss (Raw)": 0.9694575667381287, "Finetune/Step": 3681, "Finetune/Step Time": 4.25890589132905} +{"Finetune/Learning Rate": 4.137781996704356e-06, "Finetune/Loss": 0.8617565631866455, "Finetune/Loss (Raw)": 1.0914320945739746, "Finetune/Step": 3682, "Finetune/Step Time": 4.256333149969578} +{"Finetune/Learning Rate": 4.1327362117030174e-06, "Finetune/Loss": 0.8604096174240112, "Finetune/Loss (Raw)": 0.8864405155181885, "Finetune/Step": 3683, "Finetune/Step Time": 4.245345486328006} +{"Finetune/Learning Rate": 4.127692703672207e-06, "Finetune/Loss": 0.86284339427948, "Finetune/Loss (Raw)": 0.6616908311843872, "Finetune/Step": 3684, "Finetune/Step Time": 4.225031595677137} +{"Finetune/Learning Rate": 4.1226514745692115e-06, "Finetune/Loss": 0.8614577054977417, "Finetune/Loss (Raw)": 0.8561955690383911, "Finetune/Step": 3685, "Finetune/Step Time": 4.2209875248372555} +{"Finetune/Learning Rate": 4.117612526350429e-06, "Finetune/Loss": 0.8617120385169983, "Finetune/Loss (Raw)": 0.9146486520767212, "Finetune/Step": 3686, "Finetune/Step Time": 4.225837791338563} +{"Finetune/Learning Rate": 4.112575860971384e-06, "Finetune/Loss": 0.8610653281211853, "Finetune/Loss (Raw)": 0.8790567517280579, "Finetune/Step": 3687, "Finetune/Step Time": 4.22406649030745} +{"Finetune/Learning Rate": 4.107541480386708e-06, "Finetune/Loss": 0.8596374988555908, "Finetune/Loss (Raw)": 0.8147434592247009, "Finetune/Step": 3688, "Finetune/Step Time": 4.232631916180253} +{"Finetune/Learning Rate": 4.102509386550141e-06, "Finetune/Loss": 0.8591771125793457, "Finetune/Loss (Raw)": 0.9829415082931519, "Finetune/Step": 3689, "Finetune/Step Time": 4.232309520244598} +{"Finetune/Learning Rate": 4.0974795814145384e-06, "Finetune/Loss": 0.8607234954833984, "Finetune/Loss (Raw)": 0.9258357286453247, "Finetune/Step": 3690, "Finetune/Step Time": 4.235645076259971} +{"Finetune/Learning Rate": 4.092452066931877e-06, "Finetune/Loss": 0.8571416139602661, "Finetune/Loss (Raw)": 0.3110673725605011, "Finetune/Step": 3691, "Finetune/Step Time": 4.273533409461379} +{"Finetune/Learning Rate": 4.087426845053236e-06, "Finetune/Loss": 0.8584770560264587, "Finetune/Loss (Raw)": 0.9930976033210754, "Finetune/Step": 3692, "Finetune/Step Time": 4.267474934458733} +{"Finetune/Learning Rate": 4.082403917728796e-06, "Finetune/Loss": 0.8588935136795044, "Finetune/Loss (Raw)": 0.8917598128318787, "Finetune/Step": 3693, "Finetune/Step Time": 4.272296532988548} +{"Finetune/Learning Rate": 4.077383286907869e-06, "Finetune/Loss": 0.857054591178894, "Finetune/Loss (Raw)": 0.7723590731620789, "Finetune/Step": 3694, "Finetune/Step Time": 4.2961402870714664} +{"Finetune/Learning Rate": 4.072364954538858e-06, "Finetune/Loss": 0.8588550090789795, "Finetune/Loss (Raw)": 1.1146737337112427, "Finetune/Step": 3695, "Finetune/Step Time": 4.270101051777601} +{"Finetune/Learning Rate": 4.067348922569281e-06, "Finetune/Loss": 0.8582838773727417, "Finetune/Loss (Raw)": 0.8696640133857727, "Finetune/Step": 3696, "Finetune/Step Time": 4.263263588771224} +{"Finetune/Learning Rate": 4.062335192945762e-06, "Finetune/Loss": 0.857203483581543, "Finetune/Loss (Raw)": 0.7198668718338013, "Finetune/Step": 3697, "Finetune/Step Time": 4.279328752309084} +{"Finetune/Learning Rate": 4.057323767614032e-06, "Finetune/Loss": 0.8578921556472778, "Finetune/Loss (Raw)": 0.917823076248169, "Finetune/Step": 3698, "Finetune/Step Time": 4.2858047522604465} +{"Finetune/Learning Rate": 4.052314648518925e-06, "Finetune/Loss": 0.8578734993934631, "Finetune/Loss (Raw)": 0.9205446839332581, "Finetune/Step": 3699, "Finetune/Step Time": 4.282738350331783} +{"Finetune/Learning Rate": 4.047307837604383e-06, "Finetune/Loss": 0.857712984085083, "Finetune/Loss (Raw)": 0.9203203916549683, "Finetune/Step": 3700, "Finetune/Step Time": 4.289167085662484} +{"Finetune/Learning Rate": 4.0423033368134545e-06, "Finetune/Loss": 0.8559615612030029, "Finetune/Loss (Raw)": 0.7184082269668579, "Finetune/Step": 3701, "Finetune/Step Time": 4.296678025275469} +{"Finetune/Learning Rate": 4.037301148088292e-06, "Finetune/Loss": 0.8555222749710083, "Finetune/Loss (Raw)": 0.9184097051620483, "Finetune/Step": 3702, "Finetune/Step Time": 4.285966772586107} +{"Finetune/Learning Rate": 4.032301273370132e-06, "Finetune/Loss": 0.8562345504760742, "Finetune/Loss (Raw)": 0.9584620594978333, "Finetune/Step": 3703, "Finetune/Step Time": 4.283621218055487} +{"Finetune/Learning Rate": 4.027303714599345e-06, "Finetune/Loss": 0.8557126522064209, "Finetune/Loss (Raw)": 0.9103699922561646, "Finetune/Step": 3704, "Finetune/Step Time": 4.289062703028321} +{"Finetune/Learning Rate": 4.022308473715379e-06, "Finetune/Loss": 0.8544508218765259, "Finetune/Loss (Raw)": 0.7196949124336243, "Finetune/Step": 3705, "Finetune/Step Time": 4.2707335371524096} +{"Finetune/Learning Rate": 4.017315552656788e-06, "Finetune/Loss": 0.8535802960395813, "Finetune/Loss (Raw)": 0.8605104088783264, "Finetune/Step": 3706, "Finetune/Step Time": 4.2733243349939585} +{"Finetune/Learning Rate": 4.01232495336123e-06, "Finetune/Loss": 0.8528735637664795, "Finetune/Loss (Raw)": 0.8546488285064697, "Finetune/Step": 3707, "Finetune/Step Time": 4.299026465043426} +{"Finetune/Learning Rate": 4.0073366777654574e-06, "Finetune/Loss": 0.8517535924911499, "Finetune/Loss (Raw)": 0.7811089158058167, "Finetune/Step": 3708, "Finetune/Step Time": 4.310187680646777} +{"Finetune/Learning Rate": 4.002350727805324e-06, "Finetune/Loss": 0.8520326614379883, "Finetune/Loss (Raw)": 1.0095305442810059, "Finetune/Step": 3709, "Finetune/Step Time": 4.29465851560235} +{"Finetune/Learning Rate": 3.997367105415775e-06, "Finetune/Loss": 0.8531923890113831, "Finetune/Loss (Raw)": 0.9888595342636108, "Finetune/Step": 3710, "Finetune/Step Time": 4.28138985298574} +{"Finetune/Learning Rate": 3.992385812530864e-06, "Finetune/Loss": 0.8528915047645569, "Finetune/Loss (Raw)": 0.7808422446250916, "Finetune/Step": 3711, "Finetune/Step Time": 4.292899116873741} +{"Finetune/Learning Rate": 3.987406851083735e-06, "Finetune/Loss": 0.8534521460533142, "Finetune/Loss (Raw)": 1.002621054649353, "Finetune/Step": 3712, "Finetune/Step Time": 4.29170760139823} +{"Finetune/Learning Rate": 3.982430223006614e-06, "Finetune/Loss": 0.8476929664611816, "Finetune/Loss (Raw)": 0.28524571657180786, "Finetune/Step": 3713, "Finetune/Step Time": 4.329841813072562} +{"Finetune/Learning Rate": 3.977455930230842e-06, "Finetune/Loss": 0.8487794399261475, "Finetune/Loss (Raw)": 0.9006412029266357, "Finetune/Step": 3714, "Finetune/Step Time": 4.323459209874272} +{"Finetune/Learning Rate": 3.972483974686845e-06, "Finetune/Loss": 0.8497922420501709, "Finetune/Loss (Raw)": 0.9495844841003418, "Finetune/Step": 3715, "Finetune/Step Time": 4.334459299221635} +{"Finetune/Learning Rate": 3.967514358304139e-06, "Finetune/Loss": 0.8505442142486572, "Finetune/Loss (Raw)": 0.7992178201675415, "Finetune/Step": 3716, "Finetune/Step Time": 4.35737993568182} +{"Finetune/Learning Rate": 3.962547083011338e-06, "Finetune/Loss": 0.8497656583786011, "Finetune/Loss (Raw)": 0.7694061398506165, "Finetune/Step": 3717, "Finetune/Step Time": 4.366579670459032} +{"Finetune/Learning Rate": 3.957582150736141e-06, "Finetune/Loss": 0.8484941124916077, "Finetune/Loss (Raw)": 0.7823781967163086, "Finetune/Step": 3718, "Finetune/Step Time": 4.371082143858075} +{"Finetune/Learning Rate": 3.952619563405343e-06, "Finetune/Loss": 0.8475959300994873, "Finetune/Loss (Raw)": 0.6867826581001282, "Finetune/Step": 3719, "Finetune/Step Time": 4.3817298002541065} +{"Finetune/Learning Rate": 3.9476593229448245e-06, "Finetune/Loss": 0.8481210470199585, "Finetune/Loss (Raw)": 0.9947461485862732, "Finetune/Step": 3720, "Finetune/Step Time": 4.385686198249459} +{"Finetune/Learning Rate": 3.942701431279564e-06, "Finetune/Loss": 0.8479211330413818, "Finetune/Loss (Raw)": 0.9773054718971252, "Finetune/Step": 3721, "Finetune/Step Time": 4.385026914998889} +{"Finetune/Learning Rate": 3.937745890333623e-06, "Finetune/Loss": 0.8502998352050781, "Finetune/Loss (Raw)": 0.9184979796409607, "Finetune/Step": 3722, "Finetune/Step Time": 4.376867951825261} +{"Finetune/Learning Rate": 3.932792702030139e-06, "Finetune/Loss": 0.8508892059326172, "Finetune/Loss (Raw)": 0.9694256782531738, "Finetune/Step": 3723, "Finetune/Step Time": 4.374723486602306} +{"Finetune/Learning Rate": 3.92784186829136e-06, "Finetune/Loss": 0.8493003845214844, "Finetune/Loss (Raw)": 0.8195422887802124, "Finetune/Step": 3724, "Finetune/Step Time": 4.369951790198684} +{"Finetune/Learning Rate": 3.9228933910386014e-06, "Finetune/Loss": 0.8548500537872314, "Finetune/Loss (Raw)": 0.9656821489334106, "Finetune/Step": 3725, "Finetune/Step Time": 4.334095615893602} +{"Finetune/Learning Rate": 3.917947272192273e-06, "Finetune/Loss": 0.8513434529304504, "Finetune/Loss (Raw)": 0.38419297337532043, "Finetune/Step": 3726, "Finetune/Step Time": 4.355449192225933} +{"Finetune/Learning Rate": 3.913003513671866e-06, "Finetune/Loss": 0.851544201374054, "Finetune/Loss (Raw)": 0.8309205770492554, "Finetune/Step": 3727, "Finetune/Step Time": 4.365433355793357} +{"Finetune/Learning Rate": 3.908062117395956e-06, "Finetune/Loss": 0.8534595966339111, "Finetune/Loss (Raw)": 0.6809462904930115, "Finetune/Step": 3728, "Finetune/Step Time": 4.339477762579918} +{"Finetune/Learning Rate": 3.903123085282204e-06, "Finetune/Loss": 0.853381872177124, "Finetune/Loss (Raw)": 0.9208081364631653, "Finetune/Step": 3729, "Finetune/Step Time": 4.342336507514119} +{"Finetune/Learning Rate": 3.898186419247347e-06, "Finetune/Loss": 0.8539165258407593, "Finetune/Loss (Raw)": 0.9153289198875427, "Finetune/Step": 3730, "Finetune/Step Time": 4.371323009952903} +{"Finetune/Learning Rate": 3.893252121207221e-06, "Finetune/Loss": 0.8546162843704224, "Finetune/Loss (Raw)": 0.9393471479415894, "Finetune/Step": 3731, "Finetune/Step Time": 4.371561186388135} +{"Finetune/Learning Rate": 3.888320193076719e-06, "Finetune/Loss": 0.8546134233474731, "Finetune/Loss (Raw)": 0.9022077322006226, "Finetune/Step": 3732, "Finetune/Step Time": 4.375359991565347} +{"Finetune/Learning Rate": 3.8833906367698245e-06, "Finetune/Loss": 0.8551025390625, "Finetune/Loss (Raw)": 0.8509638905525208, "Finetune/Step": 3733, "Finetune/Step Time": 4.383159760385752} +{"Finetune/Learning Rate": 3.878463454199613e-06, "Finetune/Loss": 0.8558844327926636, "Finetune/Loss (Raw)": 0.9648653864860535, "Finetune/Step": 3734, "Finetune/Step Time": 4.375388067215681} +{"Finetune/Learning Rate": 3.87353864727822e-06, "Finetune/Loss": 0.8542951345443726, "Finetune/Loss (Raw)": 0.7091512084007263, "Finetune/Step": 3735, "Finetune/Step Time": 4.375959541648626} +{"Finetune/Learning Rate": 3.8686162179168695e-06, "Finetune/Loss": 0.8559432029724121, "Finetune/Loss (Raw)": 1.0792182683944702, "Finetune/Step": 3736, "Finetune/Step Time": 4.346160687506199} +{"Finetune/Learning Rate": 3.863696168025859e-06, "Finetune/Loss": 0.8559978604316711, "Finetune/Loss (Raw)": 0.9113203287124634, "Finetune/Step": 3737, "Finetune/Step Time": 4.3645649664103985} +{"Finetune/Learning Rate": 3.858778499514565e-06, "Finetune/Loss": 0.8563526272773743, "Finetune/Loss (Raw)": 0.8331723809242249, "Finetune/Step": 3738, "Finetune/Step Time": 4.374256178736687} +{"Finetune/Learning Rate": 3.853863214291439e-06, "Finetune/Loss": 0.8602171540260315, "Finetune/Loss (Raw)": 0.836431086063385, "Finetune/Step": 3739, "Finetune/Step Time": 4.335706302896142} +{"Finetune/Learning Rate": 3.848950314264001e-06, "Finetune/Loss": 0.8603324294090271, "Finetune/Loss (Raw)": 0.879574179649353, "Finetune/Step": 3740, "Finetune/Step Time": 4.3189687971025705} +{"Finetune/Learning Rate": 3.8440398013388655e-06, "Finetune/Loss": 0.8617587089538574, "Finetune/Loss (Raw)": 0.9667750597000122, "Finetune/Step": 3741, "Finetune/Step Time": 4.318487744778395} +{"Finetune/Learning Rate": 3.839131677421694e-06, "Finetune/Loss": 0.8618183135986328, "Finetune/Loss (Raw)": 0.8730534911155701, "Finetune/Step": 3742, "Finetune/Step Time": 4.314625920727849} +{"Finetune/Learning Rate": 3.834225944417235e-06, "Finetune/Loss": 0.8618450164794922, "Finetune/Loss (Raw)": 0.9166759848594666, "Finetune/Step": 3743, "Finetune/Step Time": 4.347233112901449} +{"Finetune/Learning Rate": 3.829322604229313e-06, "Finetune/Loss": 0.858644962310791, "Finetune/Loss (Raw)": 0.3617554306983948, "Finetune/Step": 3744, "Finetune/Step Time": 4.381979402154684} +{"Finetune/Learning Rate": 3.824421658760815e-06, "Finetune/Loss": 0.8545351624488831, "Finetune/Loss (Raw)": 0.325385183095932, "Finetune/Step": 3745, "Finetune/Step Time": 4.408759726211429} +{"Finetune/Learning Rate": 3.819523109913706e-06, "Finetune/Loss": 0.8538898825645447, "Finetune/Loss (Raw)": 0.7473843693733215, "Finetune/Step": 3746, "Finetune/Step Time": 4.399496089667082} +{"Finetune/Learning Rate": 3.814626959589012e-06, "Finetune/Loss": 0.8531755208969116, "Finetune/Loss (Raw)": 0.752034604549408, "Finetune/Step": 3747, "Finetune/Step Time": 4.419555334374309} +{"Finetune/Learning Rate": 3.8097332096868368e-06, "Finetune/Loss": 0.8540031909942627, "Finetune/Loss (Raw)": 0.9425321221351624, "Finetune/Step": 3748, "Finetune/Step Time": 4.430410439148545} +{"Finetune/Learning Rate": 3.8048418621063475e-06, "Finetune/Loss": 0.85367751121521, "Finetune/Loss (Raw)": 0.7760786414146423, "Finetune/Step": 3749, "Finetune/Step Time": 4.4372917134314775} +{"Finetune/Learning Rate": 3.7999529187457775e-06, "Finetune/Loss": 0.8540653586387634, "Finetune/Loss (Raw)": 0.9502691030502319, "Finetune/Step": 3750, "Finetune/Step Time": 4.436339169740677} +{"Finetune/Learning Rate": 3.7950663815024424e-06, "Finetune/Loss": 0.8548614978790283, "Finetune/Loss (Raw)": 1.0171208381652832, "Finetune/Step": 3751, "Finetune/Step Time": 4.438119005411863} +{"Finetune/Learning Rate": 3.790182252272698e-06, "Finetune/Loss": 0.8538397550582886, "Finetune/Loss (Raw)": 0.8410775065422058, "Finetune/Step": 3752, "Finetune/Step Time": 4.440628547221422} +{"Finetune/Learning Rate": 3.785300532951983e-06, "Finetune/Loss": 0.8531568646430969, "Finetune/Loss (Raw)": 0.8674044609069824, "Finetune/Step": 3753, "Finetune/Step Time": 4.455436127260327} +{"Finetune/Learning Rate": 3.780421225434803e-06, "Finetune/Loss": 0.8539736866950989, "Finetune/Loss (Raw)": 0.8972715139389038, "Finetune/Step": 3754, "Finetune/Step Time": 4.4626232255250216} +{"Finetune/Learning Rate": 3.7755443316147177e-06, "Finetune/Loss": 0.8543267250061035, "Finetune/Loss (Raw)": 0.948731005191803, "Finetune/Step": 3755, "Finetune/Step Time": 4.45595564879477} +{"Finetune/Learning Rate": 3.7706698533843556e-06, "Finetune/Loss": 0.8529961109161377, "Finetune/Loss (Raw)": 0.7157358527183533, "Finetune/Step": 3756, "Finetune/Step Time": 4.457352431491017} +{"Finetune/Learning Rate": 3.765797792635406e-06, "Finetune/Loss": 0.8539614677429199, "Finetune/Loss (Raw)": 0.9518369436264038, "Finetune/Step": 3757, "Finetune/Step Time": 4.460601128637791} +{"Finetune/Learning Rate": 3.7609281512586203e-06, "Finetune/Loss": 0.8539676666259766, "Finetune/Loss (Raw)": 0.852242112159729, "Finetune/Step": 3758, "Finetune/Step Time": 4.475650276988745} +{"Finetune/Learning Rate": 3.756060931143811e-06, "Finetune/Loss": 0.855810284614563, "Finetune/Loss (Raw)": 0.9489002227783203, "Finetune/Step": 3759, "Finetune/Step Time": 4.460094893351197} +{"Finetune/Learning Rate": 3.7511961341798487e-06, "Finetune/Loss": 0.8547028303146362, "Finetune/Loss (Raw)": 0.8399036526679993, "Finetune/Step": 3760, "Finetune/Step Time": 4.465547950938344} +{"Finetune/Learning Rate": 3.746333762254677e-06, "Finetune/Loss": 0.8528092503547668, "Finetune/Loss (Raw)": 0.7437425851821899, "Finetune/Step": 3761, "Finetune/Step Time": 4.476445684209466} +{"Finetune/Learning Rate": 3.7414738172552744e-06, "Finetune/Loss": 0.8528506755828857, "Finetune/Loss (Raw)": 0.8766250610351562, "Finetune/Step": 3762, "Finetune/Step Time": 4.482264064252377} +{"Finetune/Learning Rate": 3.7366163010676937e-06, "Finetune/Loss": 0.8541629314422607, "Finetune/Loss (Raw)": 0.9399073123931885, "Finetune/Step": 3763, "Finetune/Step Time": 4.483436539769173} +{"Finetune/Learning Rate": 3.7317612155770467e-06, "Finetune/Loss": 0.8534532785415649, "Finetune/Loss (Raw)": 0.8163982629776001, "Finetune/Step": 3764, "Finetune/Step Time": 4.484175018966198} +{"Finetune/Learning Rate": 3.726908562667496e-06, "Finetune/Loss": 0.8525915741920471, "Finetune/Loss (Raw)": 0.8546760082244873, "Finetune/Step": 3765, "Finetune/Step Time": 4.493325339630246} +{"Finetune/Learning Rate": 3.72205834422226e-06, "Finetune/Loss": 0.8526166677474976, "Finetune/Loss (Raw)": 0.9428004622459412, "Finetune/Step": 3766, "Finetune/Step Time": 4.496544661000371} +{"Finetune/Learning Rate": 3.717210562123613e-06, "Finetune/Loss": 0.8530791401863098, "Finetune/Loss (Raw)": 0.937142014503479, "Finetune/Step": 3767, "Finetune/Step Time": 4.49997260607779} +{"Finetune/Learning Rate": 3.712365218252887e-06, "Finetune/Loss": 0.856611967086792, "Finetune/Loss (Raw)": 0.7611640095710754, "Finetune/Step": 3768, "Finetune/Step Time": 4.469714604318142} +{"Finetune/Learning Rate": 3.7075223144904638e-06, "Finetune/Loss": 0.8522554039955139, "Finetune/Loss (Raw)": 0.24800148606300354, "Finetune/Step": 3769, "Finetune/Step Time": 4.5058680810034275} +{"Finetune/Learning Rate": 3.702681852715777e-06, "Finetune/Loss": 0.8530006408691406, "Finetune/Loss (Raw)": 1.0418641567230225, "Finetune/Step": 3770, "Finetune/Step Time": 4.497293543070555} +{"Finetune/Learning Rate": 3.6978438348073253e-06, "Finetune/Loss": 0.8541531562805176, "Finetune/Loss (Raw)": 1.0701649188995361, "Finetune/Step": 3771, "Finetune/Step Time": 4.499286517500877} +{"Finetune/Learning Rate": 3.6930082626426377e-06, "Finetune/Loss": 0.8551241159439087, "Finetune/Loss (Raw)": 0.9603262543678284, "Finetune/Step": 3772, "Finetune/Step Time": 4.505390739068389} +{"Finetune/Learning Rate": 3.6881751380983066e-06, "Finetune/Loss": 0.851365864276886, "Finetune/Loss (Raw)": 0.3458460867404938, "Finetune/Step": 3773, "Finetune/Step Time": 4.54256646335125} +{"Finetune/Learning Rate": 3.6833444630499804e-06, "Finetune/Loss": 0.8509016633033752, "Finetune/Loss (Raw)": 0.9238429069519043, "Finetune/Step": 3774, "Finetune/Step Time": 4.544564969837666} +{"Finetune/Learning Rate": 3.6785162393723482e-06, "Finetune/Loss": 0.8511800765991211, "Finetune/Loss (Raw)": 0.954159677028656, "Finetune/Step": 3775, "Finetune/Step Time": 4.549721457064152} +{"Finetune/Learning Rate": 3.6736904689391417e-06, "Finetune/Loss": 0.8511098623275757, "Finetune/Loss (Raw)": 0.9400599002838135, "Finetune/Step": 3776, "Finetune/Step Time": 4.552798444405198} +{"Finetune/Learning Rate": 3.6688671536231557e-06, "Finetune/Loss": 0.8458051681518555, "Finetune/Loss (Raw)": 0.27456432580947876, "Finetune/Step": 3777, "Finetune/Step Time": 4.5913819540292025} +{"Finetune/Learning Rate": 3.6640462952962217e-06, "Finetune/Loss": 0.8450380563735962, "Finetune/Loss (Raw)": 0.9230093359947205, "Finetune/Step": 3778, "Finetune/Step Time": 4.599556451663375} +{"Finetune/Learning Rate": 3.6592278958292225e-06, "Finetune/Loss": 0.845751941204071, "Finetune/Loss (Raw)": 0.8374183773994446, "Finetune/Step": 3779, "Finetune/Step Time": 4.59915341809392} +{"Finetune/Learning Rate": 3.6544119570920845e-06, "Finetune/Loss": 0.8446319699287415, "Finetune/Loss (Raw)": 0.7992745637893677, "Finetune/Step": 3780, "Finetune/Step Time": 4.5996335707604885} +{"Finetune/Learning Rate": 3.6495984809537775e-06, "Finetune/Loss": 0.8446168899536133, "Finetune/Loss (Raw)": 0.9531183242797852, "Finetune/Step": 3781, "Finetune/Step Time": 4.596707865595818} +{"Finetune/Learning Rate": 3.6447874692823195e-06, "Finetune/Loss": 0.8442647457122803, "Finetune/Loss (Raw)": 0.8398761749267578, "Finetune/Step": 3782, "Finetune/Step Time": 4.573284586891532} +{"Finetune/Learning Rate": 3.639978923944766e-06, "Finetune/Loss": 0.8422978520393372, "Finetune/Loss (Raw)": 0.8008507490158081, "Finetune/Step": 3783, "Finetune/Step Time": 4.575551738962531} +{"Finetune/Learning Rate": 3.6351728468072265e-06, "Finetune/Loss": 0.8426075577735901, "Finetune/Loss (Raw)": 0.8146088123321533, "Finetune/Step": 3784, "Finetune/Step Time": 4.587284443899989} +{"Finetune/Learning Rate": 3.6303692397348455e-06, "Finetune/Loss": 0.8425959348678589, "Finetune/Loss (Raw)": 0.8698446154594421, "Finetune/Step": 3785, "Finetune/Step Time": 4.585516422986984} +{"Finetune/Learning Rate": 3.6255681045918e-06, "Finetune/Loss": 0.8433146476745605, "Finetune/Loss (Raw)": 0.9810042381286621, "Finetune/Step": 3786, "Finetune/Step Time": 4.569309528917074} +{"Finetune/Learning Rate": 3.6207694432413255e-06, "Finetune/Loss": 0.8434686660766602, "Finetune/Loss (Raw)": 0.9257330894470215, "Finetune/Step": 3787, "Finetune/Step Time": 4.566589705646038} +{"Finetune/Learning Rate": 3.6159732575456862e-06, "Finetune/Loss": 0.8424200415611267, "Finetune/Loss (Raw)": 0.8630793690681458, "Finetune/Step": 3788, "Finetune/Step Time": 4.566672699525952} +{"Finetune/Learning Rate": 3.6111795493661885e-06, "Finetune/Loss": 0.841683030128479, "Finetune/Loss (Raw)": 0.7881787419319153, "Finetune/Step": 3789, "Finetune/Step Time": 4.584722675383091} +{"Finetune/Learning Rate": 3.606388320563177e-06, "Finetune/Loss": 0.841458261013031, "Finetune/Loss (Raw)": 0.9422810673713684, "Finetune/Step": 3790, "Finetune/Step Time": 4.58613670617342} +{"Finetune/Learning Rate": 3.6015995729960316e-06, "Finetune/Loss": 0.8400020599365234, "Finetune/Loss (Raw)": 0.8697966933250427, "Finetune/Step": 3791, "Finetune/Step Time": 4.586148766800761} +{"Finetune/Learning Rate": 3.596813308523176e-06, "Finetune/Loss": 0.8362767696380615, "Finetune/Loss (Raw)": 0.3450095057487488, "Finetune/Step": 3792, "Finetune/Step Time": 4.619709165766835} +{"Finetune/Learning Rate": 3.592029529002059e-06, "Finetune/Loss": 0.836259126663208, "Finetune/Loss (Raw)": 0.9066707491874695, "Finetune/Step": 3793, "Finetune/Step Time": 4.638785736635327} +{"Finetune/Learning Rate": 3.5872482362891802e-06, "Finetune/Loss": 0.8363224864006042, "Finetune/Loss (Raw)": 0.9263766407966614, "Finetune/Step": 3794, "Finetune/Step Time": 4.6382747143507} +{"Finetune/Learning Rate": 3.5824694322400653e-06, "Finetune/Loss": 0.8389104008674622, "Finetune/Loss (Raw)": 0.7562726140022278, "Finetune/Step": 3795, "Finetune/Step Time": 4.607650173828006} +{"Finetune/Learning Rate": 3.5776931187092667e-06, "Finetune/Loss": 0.8395354151725769, "Finetune/Loss (Raw)": 0.8989401459693909, "Finetune/Step": 3796, "Finetune/Step Time": 4.612919673323631} +{"Finetune/Learning Rate": 3.5729192975503856e-06, "Finetune/Loss": 0.8386302590370178, "Finetune/Loss (Raw)": 0.740603506565094, "Finetune/Step": 3797, "Finetune/Step Time": 4.61726994626224} +{"Finetune/Learning Rate": 3.5681479706160483e-06, "Finetune/Loss": 0.8380270004272461, "Finetune/Loss (Raw)": 0.8270277380943298, "Finetune/Step": 3798, "Finetune/Step Time": 4.61953816562891} +{"Finetune/Learning Rate": 3.563379139757911e-06, "Finetune/Loss": 0.8367685079574585, "Finetune/Loss (Raw)": 0.8168964982032776, "Finetune/Step": 3799, "Finetune/Step Time": 4.6295731123536825} +{"Finetune/Learning Rate": 3.558612806826666e-06, "Finetune/Loss": 0.8376913666725159, "Finetune/Loss (Raw)": 0.9575139880180359, "Finetune/Step": 3800, "Finetune/Step Time": 4.62530243396759} +{"Finetune/Learning Rate": 3.553848973672033e-06, "Finetune/Loss": 0.839264452457428, "Finetune/Loss (Raw)": 0.8700569868087769, "Finetune/Step": 3801, "Finetune/Step Time": 4.625924611464143} +{"Finetune/Learning Rate": 3.549087642142762e-06, "Finetune/Loss": 0.8401079177856445, "Finetune/Loss (Raw)": 0.9570206999778748, "Finetune/Step": 3802, "Finetune/Step Time": 4.627339489758015} +{"Finetune/Learning Rate": 3.544328814086632e-06, "Finetune/Loss": 0.8449572920799255, "Finetune/Loss (Raw)": 0.9981271624565125, "Finetune/Step": 3803, "Finetune/Step Time": 4.584840754047036} +{"Finetune/Learning Rate": 3.5395724913504546e-06, "Finetune/Loss": 0.8447481393814087, "Finetune/Loss (Raw)": 0.9160516262054443, "Finetune/Step": 3804, "Finetune/Step Time": 4.569006295874715} +{"Finetune/Learning Rate": 3.5348186757800683e-06, "Finetune/Loss": 0.843875527381897, "Finetune/Loss (Raw)": 0.8531134128570557, "Finetune/Step": 3805, "Finetune/Step Time": 4.585722167044878} +{"Finetune/Learning Rate": 3.530067369220326e-06, "Finetune/Loss": 0.8457474708557129, "Finetune/Loss (Raw)": 0.8849659562110901, "Finetune/Step": 3806, "Finetune/Step Time": 4.597717082127929} +{"Finetune/Learning Rate": 3.525318573515125e-06, "Finetune/Loss": 0.8446512222290039, "Finetune/Loss (Raw)": 0.8428704738616943, "Finetune/Step": 3807, "Finetune/Step Time": 4.5998841393738985} +{"Finetune/Learning Rate": 3.520572290507378e-06, "Finetune/Loss": 0.8435391187667847, "Finetune/Loss (Raw)": 0.7699756622314453, "Finetune/Step": 3808, "Finetune/Step Time": 4.606845440343022} +{"Finetune/Learning Rate": 3.5158285220390256e-06, "Finetune/Loss": 0.8425648808479309, "Finetune/Loss (Raw)": 0.8447482585906982, "Finetune/Step": 3809, "Finetune/Step Time": 4.621550807729363} +{"Finetune/Learning Rate": 3.5110872699510313e-06, "Finetune/Loss": 0.8413830399513245, "Finetune/Loss (Raw)": 0.9401562809944153, "Finetune/Step": 3810, "Finetune/Step Time": 4.618304241448641} +{"Finetune/Learning Rate": 3.5063485360833816e-06, "Finetune/Loss": 0.8409881591796875, "Finetune/Loss (Raw)": 0.8359007239341736, "Finetune/Step": 3811, "Finetune/Step Time": 4.624531773850322} +{"Finetune/Learning Rate": 3.5016123222750863e-06, "Finetune/Loss": 0.842002272605896, "Finetune/Loss (Raw)": 0.791498064994812, "Finetune/Step": 3812, "Finetune/Step Time": 4.618817873299122} +{"Finetune/Learning Rate": 3.4968786303641764e-06, "Finetune/Loss": 0.8421245813369751, "Finetune/Loss (Raw)": 0.8718501925468445, "Finetune/Step": 3813, "Finetune/Step Time": 4.616351189091802} +{"Finetune/Learning Rate": 3.492147462187714e-06, "Finetune/Loss": 0.8415166735649109, "Finetune/Loss (Raw)": 0.8368322253227234, "Finetune/Step": 3814, "Finetune/Step Time": 4.608938565477729} +{"Finetune/Learning Rate": 3.4874188195817626e-06, "Finetune/Loss": 0.8414192795753479, "Finetune/Loss (Raw)": 0.8665881752967834, "Finetune/Step": 3815, "Finetune/Step Time": 4.62753870151937} +{"Finetune/Learning Rate": 3.4826927043814197e-06, "Finetune/Loss": 0.8419049382209778, "Finetune/Loss (Raw)": 0.8769107460975647, "Finetune/Step": 3816, "Finetune/Step Time": 4.622998207807541} +{"Finetune/Learning Rate": 3.4779691184207954e-06, "Finetune/Loss": 0.8404933214187622, "Finetune/Loss (Raw)": 0.8022602200508118, "Finetune/Step": 3817, "Finetune/Step Time": 4.620005069300532} +{"Finetune/Learning Rate": 3.473248063533028e-06, "Finetune/Loss": 0.8396481871604919, "Finetune/Loss (Raw)": 0.8176525235176086, "Finetune/Step": 3818, "Finetune/Step Time": 4.613231690600514} +{"Finetune/Learning Rate": 3.4685295415502663e-06, "Finetune/Loss": 0.8435074090957642, "Finetune/Loss (Raw)": 0.8050519824028015, "Finetune/Step": 3819, "Finetune/Step Time": 4.576207956299186} +{"Finetune/Learning Rate": 3.4638135543036656e-06, "Finetune/Loss": 0.8413442373275757, "Finetune/Loss (Raw)": 0.716208279132843, "Finetune/Step": 3820, "Finetune/Step Time": 4.58322324603796} +{"Finetune/Learning Rate": 3.4591001036234206e-06, "Finetune/Loss": 0.8406877517700195, "Finetune/Loss (Raw)": 0.8077250719070435, "Finetune/Step": 3821, "Finetune/Step Time": 4.582963299006224} +{"Finetune/Learning Rate": 3.4543891913387253e-06, "Finetune/Loss": 0.8407188653945923, "Finetune/Loss (Raw)": 0.7763474583625793, "Finetune/Step": 3822, "Finetune/Step Time": 4.567242622375488} +{"Finetune/Learning Rate": 3.449680819277792e-06, "Finetune/Loss": 0.8396539688110352, "Finetune/Loss (Raw)": 0.9783631563186646, "Finetune/Step": 3823, "Finetune/Step Time": 4.570003671571612} +{"Finetune/Learning Rate": 3.4449749892678486e-06, "Finetune/Loss": 0.8398499488830566, "Finetune/Loss (Raw)": 0.8947559595108032, "Finetune/Step": 3824, "Finetune/Step Time": 4.568835273385048} +{"Finetune/Learning Rate": 3.4402717031351363e-06, "Finetune/Loss": 0.8401645421981812, "Finetune/Loss (Raw)": 0.7601319551467896, "Finetune/Step": 3825, "Finetune/Step Time": 4.547193195670843} +{"Finetune/Learning Rate": 3.4355709627049073e-06, "Finetune/Loss": 0.8393970131874084, "Finetune/Loss (Raw)": 0.8195797204971313, "Finetune/Step": 3826, "Finetune/Step Time": 4.543594574555755} +{"Finetune/Learning Rate": 3.4308727698014243e-06, "Finetune/Loss": 0.8391801118850708, "Finetune/Loss (Raw)": 0.8927779793739319, "Finetune/Step": 3827, "Finetune/Step Time": 4.5711232256144285} +{"Finetune/Learning Rate": 3.4261771262479724e-06, "Finetune/Loss": 0.8393909931182861, "Finetune/Loss (Raw)": 0.9473171830177307, "Finetune/Step": 3828, "Finetune/Step Time": 4.569094521924853} +{"Finetune/Learning Rate": 3.4214840338668376e-06, "Finetune/Loss": 0.8359805345535278, "Finetune/Loss (Raw)": 0.28187111020088196, "Finetune/Step": 3829, "Finetune/Step Time": 4.601870024576783} +{"Finetune/Learning Rate": 3.416793494479308e-06, "Finetune/Loss": 0.8357266187667847, "Finetune/Loss (Raw)": 0.8859052062034607, "Finetune/Step": 3830, "Finetune/Step Time": 4.61132194660604} +{"Finetune/Learning Rate": 3.412105509905701e-06, "Finetune/Loss": 0.8347594141960144, "Finetune/Loss (Raw)": 0.8346591591835022, "Finetune/Step": 3831, "Finetune/Step Time": 4.619116805493832} +{"Finetune/Learning Rate": 3.407420081965327e-06, "Finetune/Loss": 0.8357224464416504, "Finetune/Loss (Raw)": 1.0336410999298096, "Finetune/Step": 3832, "Finetune/Step Time": 4.616689363494515} +{"Finetune/Learning Rate": 3.402737212476509e-06, "Finetune/Loss": 0.837584376335144, "Finetune/Loss (Raw)": 0.9580191969871521, "Finetune/Step": 3833, "Finetune/Step Time": 4.6398799531161785} +{"Finetune/Learning Rate": 3.398056903256579e-06, "Finetune/Loss": 0.8384572863578796, "Finetune/Loss (Raw)": 0.972245454788208, "Finetune/Step": 3834, "Finetune/Step Time": 4.633370218798518} +{"Finetune/Learning Rate": 3.393379156121873e-06, "Finetune/Loss": 0.8391708135604858, "Finetune/Loss (Raw)": 0.9459834694862366, "Finetune/Step": 3835, "Finetune/Step Time": 4.614370930939913} +{"Finetune/Learning Rate": 3.3887039728877336e-06, "Finetune/Loss": 0.839421808719635, "Finetune/Loss (Raw)": 0.8132322430610657, "Finetune/Step": 3836, "Finetune/Step Time": 4.598537525162101} +{"Finetune/Learning Rate": 3.3840313553685034e-06, "Finetune/Loss": 0.8381547927856445, "Finetune/Loss (Raw)": 0.8473524451255798, "Finetune/Step": 3837, "Finetune/Step Time": 4.6041614562273026} +{"Finetune/Learning Rate": 3.379361305377542e-06, "Finetune/Loss": 0.8368065357208252, "Finetune/Loss (Raw)": 0.8162798881530762, "Finetune/Step": 3838, "Finetune/Step Time": 4.602118078619242} +{"Finetune/Learning Rate": 3.374693824727204e-06, "Finetune/Loss": 0.8376003503799438, "Finetune/Loss (Raw)": 0.8824558854103088, "Finetune/Step": 3839, "Finetune/Step Time": 4.6167658399790525} +{"Finetune/Learning Rate": 3.3700289152288377e-06, "Finetune/Loss": 0.8363239169120789, "Finetune/Loss (Raw)": 0.8392351269721985, "Finetune/Step": 3840, "Finetune/Step Time": 4.617177654057741} +{"Finetune/Learning Rate": 3.3653665786928138e-06, "Finetune/Loss": 0.8419562578201294, "Finetune/Loss (Raw)": 1.0061789751052856, "Finetune/Step": 3841, "Finetune/Step Time": 4.578130563721061} +{"Finetune/Learning Rate": 3.360706816928491e-06, "Finetune/Loss": 0.8412368297576904, "Finetune/Loss (Raw)": 0.8085654973983765, "Finetune/Step": 3842, "Finetune/Step Time": 4.569731894880533} +{"Finetune/Learning Rate": 3.3560496317442305e-06, "Finetune/Loss": 0.8403104543685913, "Finetune/Loss (Raw)": 0.8310021758079529, "Finetune/Step": 3843, "Finetune/Step Time": 4.570334378629923} +{"Finetune/Learning Rate": 3.351395024947395e-06, "Finetune/Loss": 0.8406750559806824, "Finetune/Loss (Raw)": 0.8458885550498962, "Finetune/Step": 3844, "Finetune/Step Time": 4.551352294161916} +{"Finetune/Learning Rate": 3.3467429983443477e-06, "Finetune/Loss": 0.8412145972251892, "Finetune/Loss (Raw)": 0.838469386100769, "Finetune/Step": 3845, "Finetune/Step Time": 4.534934584051371} +{"Finetune/Learning Rate": 3.34209355374045e-06, "Finetune/Loss": 0.8405787944793701, "Finetune/Loss (Raw)": 0.7009925842285156, "Finetune/Step": 3846, "Finetune/Step Time": 4.531016457825899} +{"Finetune/Learning Rate": 3.337446692940055e-06, "Finetune/Loss": 0.8429210782051086, "Finetune/Loss (Raw)": 0.9865943789482117, "Finetune/Step": 3847, "Finetune/Step Time": 4.518848618492484} +{"Finetune/Learning Rate": 3.332802417746527e-06, "Finetune/Loss": 0.8377425670623779, "Finetune/Loss (Raw)": 0.3318959176540375, "Finetune/Step": 3848, "Finetune/Step Time": 4.5471152644604445} +{"Finetune/Learning Rate": 3.3281607299622175e-06, "Finetune/Loss": 0.838152289390564, "Finetune/Loss (Raw)": 1.0297508239746094, "Finetune/Step": 3849, "Finetune/Step Time": 4.562701538205147} +{"Finetune/Learning Rate": 3.3235216313884657e-06, "Finetune/Loss": 0.8357930779457092, "Finetune/Loss (Raw)": 0.616522490978241, "Finetune/Step": 3850, "Finetune/Step Time": 4.5698668751865625} +{"Finetune/Learning Rate": 3.3188851238256246e-06, "Finetune/Loss": 0.8341822624206543, "Finetune/Loss (Raw)": 0.7632381319999695, "Finetune/Step": 3851, "Finetune/Step Time": 4.561358010396361} +{"Finetune/Learning Rate": 3.3142512090730283e-06, "Finetune/Loss": 0.8349740505218506, "Finetune/Loss (Raw)": 0.9208910465240479, "Finetune/Step": 3852, "Finetune/Step Time": 4.562103042379022} +{"Finetune/Learning Rate": 3.309619888929011e-06, "Finetune/Loss": 0.8309983015060425, "Finetune/Loss (Raw)": 0.45678743720054626, "Finetune/Step": 3853, "Finetune/Step Time": 4.596212789416313} +{"Finetune/Learning Rate": 3.304991165190895e-06, "Finetune/Loss": 0.8349577188491821, "Finetune/Loss (Raw)": 0.891002357006073, "Finetune/Step": 3854, "Finetune/Step Time": 4.561055976897478} +{"Finetune/Learning Rate": 3.300365039655e-06, "Finetune/Loss": 0.833469033241272, "Finetune/Loss (Raw)": 0.6403580904006958, "Finetune/Step": 3855, "Finetune/Step Time": 4.566214442253113} +{"Finetune/Learning Rate": 3.2957415141166327e-06, "Finetune/Loss": 0.8358181715011597, "Finetune/Loss (Raw)": 0.9816460609436035, "Finetune/Step": 3856, "Finetune/Step Time": 4.565195668488741} +{"Finetune/Learning Rate": 3.2911205903700916e-06, "Finetune/Loss": 0.8314683437347412, "Finetune/Loss (Raw)": 0.36401867866516113, "Finetune/Step": 3857, "Finetune/Step Time": 4.593477176502347} +{"Finetune/Learning Rate": 3.2865022702086756e-06, "Finetune/Loss": 0.8269426822662354, "Finetune/Loss (Raw)": 0.33604490756988525, "Finetune/Step": 3858, "Finetune/Step Time": 4.606577806174755} +{"Finetune/Learning Rate": 3.281886555424656e-06, "Finetune/Loss": 0.8278424739837646, "Finetune/Loss (Raw)": 1.0545223951339722, "Finetune/Step": 3859, "Finetune/Step Time": 4.604747228324413} +{"Finetune/Learning Rate": 3.277273447809299e-06, "Finetune/Loss": 0.827695369720459, "Finetune/Loss (Raw)": 0.8833825588226318, "Finetune/Step": 3860, "Finetune/Step Time": 4.5928872767835855} +{"Finetune/Learning Rate": 3.2726629491528714e-06, "Finetune/Loss": 0.8276163935661316, "Finetune/Loss (Raw)": 0.8408598303794861, "Finetune/Step": 3861, "Finetune/Step Time": 4.578050408512354} +{"Finetune/Learning Rate": 3.268055061244614e-06, "Finetune/Loss": 0.8269942998886108, "Finetune/Loss (Raw)": 0.8852355480194092, "Finetune/Step": 3862, "Finetune/Step Time": 4.5767527632415295} +{"Finetune/Learning Rate": 3.2634497858727553e-06, "Finetune/Loss": 0.8278273344039917, "Finetune/Loss (Raw)": 0.8157784938812256, "Finetune/Step": 3863, "Finetune/Step Time": 4.586849361658096} +{"Finetune/Learning Rate": 3.2588471248245157e-06, "Finetune/Loss": 0.8265972137451172, "Finetune/Loss (Raw)": 0.9217641353607178, "Finetune/Step": 3864, "Finetune/Step Time": 4.595595613121986} +{"Finetune/Learning Rate": 3.254247079886097e-06, "Finetune/Loss": 0.8263201117515564, "Finetune/Loss (Raw)": 0.8758475184440613, "Finetune/Step": 3865, "Finetune/Step Time": 4.613017661496997} +{"Finetune/Learning Rate": 3.2496496528426867e-06, "Finetune/Loss": 0.8278799057006836, "Finetune/Loss (Raw)": 1.0328285694122314, "Finetune/Step": 3866, "Finetune/Step Time": 4.6039630472660065} +{"Finetune/Learning Rate": 3.245054845478454e-06, "Finetune/Loss": 0.8297422528266907, "Finetune/Loss (Raw)": 1.0748076438903809, "Finetune/Step": 3867, "Finetune/Step Time": 4.604175386950374} +{"Finetune/Learning Rate": 3.240462659576562e-06, "Finetune/Loss": 0.8298395872116089, "Finetune/Loss (Raw)": 0.8920369744300842, "Finetune/Step": 3868, "Finetune/Step Time": 4.607596881687641} +{"Finetune/Learning Rate": 3.23587309691914e-06, "Finetune/Loss": 0.8286227583885193, "Finetune/Loss (Raw)": 0.8110232949256897, "Finetune/Step": 3869, "Finetune/Step Time": 4.609266122803092} +{"Finetune/Learning Rate": 3.231286159287308e-06, "Finetune/Loss": 0.8281160593032837, "Finetune/Loss (Raw)": 0.8081868290901184, "Finetune/Step": 3870, "Finetune/Step Time": 4.633516687899828} +{"Finetune/Learning Rate": 3.2267018484611725e-06, "Finetune/Loss": 0.8277549147605896, "Finetune/Loss (Raw)": 0.8704517483711243, "Finetune/Step": 3871, "Finetune/Step Time": 4.5972061194479465} +{"Finetune/Learning Rate": 3.222120166219812e-06, "Finetune/Loss": 0.8289080262184143, "Finetune/Loss (Raw)": 0.5093629360198975, "Finetune/Step": 3872, "Finetune/Step Time": 4.593184104189277} +{"Finetune/Learning Rate": 3.217541114341288e-06, "Finetune/Loss": 0.832097053527832, "Finetune/Loss (Raw)": 0.7335734963417053, "Finetune/Step": 3873, "Finetune/Step Time": 4.563471766188741} +{"Finetune/Learning Rate": 3.2129646946026407e-06, "Finetune/Loss": 0.8331185579299927, "Finetune/Loss (Raw)": 0.8781324028968811, "Finetune/Step": 3874, "Finetune/Step Time": 4.563469476997852} +{"Finetune/Learning Rate": 3.2083909087798905e-06, "Finetune/Loss": 0.8340373039245605, "Finetune/Loss (Raw)": 0.869633674621582, "Finetune/Step": 3875, "Finetune/Step Time": 4.545686012133956} +{"Finetune/Learning Rate": 3.2038197586480336e-06, "Finetune/Loss": 0.833584189414978, "Finetune/Loss (Raw)": 0.8845369219779968, "Finetune/Step": 3876, "Finetune/Step Time": 4.532143317162991} +{"Finetune/Learning Rate": 3.199251245981041e-06, "Finetune/Loss": 0.8342041373252869, "Finetune/Loss (Raw)": 0.8554377555847168, "Finetune/Step": 3877, "Finetune/Step Time": 4.53993290103972} +{"Finetune/Learning Rate": 3.1946853725518724e-06, "Finetune/Loss": 0.8304716348648071, "Finetune/Loss (Raw)": 0.47251084446907043, "Finetune/Step": 3878, "Finetune/Step Time": 4.577217595651746} +{"Finetune/Learning Rate": 3.1901221401324446e-06, "Finetune/Loss": 0.8291876316070557, "Finetune/Loss (Raw)": 0.852765679359436, "Finetune/Step": 3879, "Finetune/Step Time": 4.602777004241943} +{"Finetune/Learning Rate": 3.18556155049366e-06, "Finetune/Loss": 0.8294458389282227, "Finetune/Loss (Raw)": 0.8741302490234375, "Finetune/Step": 3880, "Finetune/Step Time": 4.599571952596307} +{"Finetune/Learning Rate": 3.1810036054054005e-06, "Finetune/Loss": 0.829788863658905, "Finetune/Loss (Raw)": 0.9113044738769531, "Finetune/Step": 3881, "Finetune/Step Time": 4.5923342648893595} +{"Finetune/Learning Rate": 3.1764483066365117e-06, "Finetune/Loss": 0.83012855052948, "Finetune/Loss (Raw)": 0.9407517910003662, "Finetune/Step": 3882, "Finetune/Step Time": 4.591637894511223} +{"Finetune/Learning Rate": 3.1718956559548154e-06, "Finetune/Loss": 0.8303483128547668, "Finetune/Loss (Raw)": 0.9768577814102173, "Finetune/Step": 3883, "Finetune/Step Time": 4.57982624694705} +{"Finetune/Learning Rate": 3.1673456551271086e-06, "Finetune/Loss": 0.83116215467453, "Finetune/Loss (Raw)": 0.8199107646942139, "Finetune/Step": 3884, "Finetune/Step Time": 4.573161825537682} +{"Finetune/Learning Rate": 3.162798305919157e-06, "Finetune/Loss": 0.8289955854415894, "Finetune/Loss (Raw)": 0.6745149493217468, "Finetune/Step": 3885, "Finetune/Step Time": 4.57924378849566} +{"Finetune/Learning Rate": 3.1582536100956973e-06, "Finetune/Loss": 0.8282077312469482, "Finetune/Loss (Raw)": 0.7513918876647949, "Finetune/Step": 3886, "Finetune/Step Time": 4.565638519823551} +{"Finetune/Learning Rate": 3.1537115694204345e-06, "Finetune/Loss": 0.827072024345398, "Finetune/Loss (Raw)": 0.8035337328910828, "Finetune/Step": 3887, "Finetune/Step Time": 4.567315887659788} +{"Finetune/Learning Rate": 3.1491721856560555e-06, "Finetune/Loss": 0.8276510238647461, "Finetune/Loss (Raw)": 0.9140188097953796, "Finetune/Step": 3888, "Finetune/Step Time": 4.559680899605155} +{"Finetune/Learning Rate": 3.144635460564197e-06, "Finetune/Loss": 0.8286919593811035, "Finetune/Loss (Raw)": 0.8769828677177429, "Finetune/Step": 3889, "Finetune/Step Time": 4.556515185162425} +{"Finetune/Learning Rate": 3.1401013959054726e-06, "Finetune/Loss": 0.8289247155189514, "Finetune/Loss (Raw)": 0.906420886516571, "Finetune/Step": 3890, "Finetune/Step Time": 4.555008705705404} +{"Finetune/Learning Rate": 3.1355699934394724e-06, "Finetune/Loss": 0.8283346891403198, "Finetune/Loss (Raw)": 0.8643759489059448, "Finetune/Step": 3891, "Finetune/Step Time": 4.569955514743924} +{"Finetune/Learning Rate": 3.1310412549247403e-06, "Finetune/Loss": 0.8286327719688416, "Finetune/Loss (Raw)": 0.8545606732368469, "Finetune/Step": 3892, "Finetune/Step Time": 4.573020303621888} +{"Finetune/Learning Rate": 3.1265151821187933e-06, "Finetune/Loss": 0.82485032081604, "Finetune/Loss (Raw)": 0.37051957845687866, "Finetune/Step": 3893, "Finetune/Step Time": 4.6024407632648945} +{"Finetune/Learning Rate": 3.121991776778109e-06, "Finetune/Loss": 0.8245536088943481, "Finetune/Loss (Raw)": 0.9048212766647339, "Finetune/Step": 3894, "Finetune/Step Time": 4.600880058482289} +{"Finetune/Learning Rate": 3.1174710406581355e-06, "Finetune/Loss": 0.8244461417198181, "Finetune/Loss (Raw)": 0.9233871102333069, "Finetune/Step": 3895, "Finetune/Step Time": 4.593336025252938} +{"Finetune/Learning Rate": 3.1129529755132815e-06, "Finetune/Loss": 0.8257842063903809, "Finetune/Loss (Raw)": 0.9324333071708679, "Finetune/Step": 3896, "Finetune/Step Time": 4.581672610715032} +{"Finetune/Learning Rate": 3.108437583096918e-06, "Finetune/Loss": 0.8306900262832642, "Finetune/Loss (Raw)": 0.8759499192237854, "Finetune/Step": 3897, "Finetune/Step Time": 4.542256595566869} +{"Finetune/Learning Rate": 3.103924865161383e-06, "Finetune/Loss": 0.829257607460022, "Finetune/Loss (Raw)": 0.8585113883018494, "Finetune/Step": 3898, "Finetune/Step Time": 4.558673406019807} +{"Finetune/Learning Rate": 3.0994148234579734e-06, "Finetune/Loss": 0.828703761100769, "Finetune/Loss (Raw)": 0.9992703199386597, "Finetune/Step": 3899, "Finetune/Step Time": 4.561865052208304} +{"Finetune/Learning Rate": 3.0949074597369444e-06, "Finetune/Loss": 0.827904462814331, "Finetune/Loss (Raw)": 0.8580199480056763, "Finetune/Step": 3900, "Finetune/Step Time": 4.572361435741186} +{"Finetune/Learning Rate": 3.090402775747523e-06, "Finetune/Loss": 0.8324977159500122, "Finetune/Loss (Raw)": 0.9337785840034485, "Finetune/Step": 3901, "Finetune/Step Time": 4.533074656501412} +{"Finetune/Learning Rate": 3.0859007732378897e-06, "Finetune/Loss": 0.8328624963760376, "Finetune/Loss (Raw)": 0.970542311668396, "Finetune/Step": 3902, "Finetune/Step Time": 4.523348616436124} +{"Finetune/Learning Rate": 3.0814014539551727e-06, "Finetune/Loss": 0.8328291177749634, "Finetune/Loss (Raw)": 0.949884831905365, "Finetune/Step": 3903, "Finetune/Step Time": 4.51787175424397} +{"Finetune/Learning Rate": 3.076904819645481e-06, "Finetune/Loss": 0.8323889970779419, "Finetune/Loss (Raw)": 0.8837287425994873, "Finetune/Step": 3904, "Finetune/Step Time": 4.521624602377415} +{"Finetune/Learning Rate": 3.072410872053867e-06, "Finetune/Loss": 0.8356513381004333, "Finetune/Loss (Raw)": 0.6921360492706299, "Finetune/Step": 3905, "Finetune/Step Time": 4.496465288102627} +{"Finetune/Learning Rate": 3.067919612924343e-06, "Finetune/Loss": 0.835728108882904, "Finetune/Loss (Raw)": 0.9328402280807495, "Finetune/Step": 3906, "Finetune/Step Time": 4.529643399640918} +{"Finetune/Learning Rate": 3.0634310439998803e-06, "Finetune/Loss": 0.8381651639938354, "Finetune/Loss (Raw)": 1.1493574380874634, "Finetune/Step": 3907, "Finetune/Step Time": 4.515085399150848} +{"Finetune/Learning Rate": 3.0589451670224037e-06, "Finetune/Loss": 0.838234007358551, "Finetune/Loss (Raw)": 0.8080934882164001, "Finetune/Step": 3908, "Finetune/Step Time": 4.522548291832209} +{"Finetune/Learning Rate": 3.054461983732795e-06, "Finetune/Loss": 0.8372257947921753, "Finetune/Loss (Raw)": 0.8240620493888855, "Finetune/Step": 3909, "Finetune/Step Time": 4.528354123234749} +{"Finetune/Learning Rate": 3.0499814958708883e-06, "Finetune/Loss": 0.8381285667419434, "Finetune/Loss (Raw)": 0.9554256796836853, "Finetune/Step": 3910, "Finetune/Step Time": 4.528881607577205} +{"Finetune/Learning Rate": 3.045503705175478e-06, "Finetune/Loss": 0.8398268222808838, "Finetune/Loss (Raw)": 1.0182303190231323, "Finetune/Step": 3911, "Finetune/Step Time": 4.525152703747153} +{"Finetune/Learning Rate": 3.041028613384307e-06, "Finetune/Loss": 0.8397704362869263, "Finetune/Loss (Raw)": 0.8073989748954773, "Finetune/Step": 3912, "Finetune/Step Time": 4.522175272926688} +{"Finetune/Learning Rate": 3.036556222234063e-06, "Finetune/Loss": 0.839967668056488, "Finetune/Loss (Raw)": 0.8950840830802917, "Finetune/Step": 3913, "Finetune/Step Time": 4.526135412976146} +{"Finetune/Learning Rate": 3.032086533460402e-06, "Finetune/Loss": 0.8388378620147705, "Finetune/Loss (Raw)": 0.8363925814628601, "Finetune/Step": 3914, "Finetune/Step Time": 4.529073489829898} +{"Finetune/Learning Rate": 3.027619548797921e-06, "Finetune/Loss": 0.8386505842208862, "Finetune/Loss (Raw)": 0.9017539024353027, "Finetune/Step": 3915, "Finetune/Step Time": 4.532133569940925} +{"Finetune/Learning Rate": 3.023155269980169e-06, "Finetune/Loss": 0.8388721942901611, "Finetune/Loss (Raw)": 0.8914502859115601, "Finetune/Step": 3916, "Finetune/Step Time": 4.532903380692005} +{"Finetune/Learning Rate": 3.0186936987396442e-06, "Finetune/Loss": 0.8396739363670349, "Finetune/Loss (Raw)": 0.8908032178878784, "Finetune/Step": 3917, "Finetune/Step Time": 4.528926244005561} +{"Finetune/Learning Rate": 3.014234836807798e-06, "Finetune/Loss": 0.8397167921066284, "Finetune/Loss (Raw)": 0.9477654695510864, "Finetune/Step": 3918, "Finetune/Step Time": 4.528201762586832} +{"Finetune/Learning Rate": 3.0097786859150247e-06, "Finetune/Loss": 0.8396154642105103, "Finetune/Loss (Raw)": 0.8568326830863953, "Finetune/Step": 3919, "Finetune/Step Time": 4.538774203509092} +{"Finetune/Learning Rate": 3.005325247790668e-06, "Finetune/Loss": 0.8437595367431641, "Finetune/Loss (Raw)": 0.8754464387893677, "Finetune/Step": 3920, "Finetune/Step Time": 4.523351073265076} +{"Finetune/Learning Rate": 3.0008745241630266e-06, "Finetune/Loss": 0.8419857025146484, "Finetune/Loss (Raw)": 0.6796174049377441, "Finetune/Step": 3921, "Finetune/Step Time": 4.538137132301927} +{"Finetune/Learning Rate": 2.9964265167593386e-06, "Finetune/Loss": 0.8421447277069092, "Finetune/Loss (Raw)": 0.9467316269874573, "Finetune/Step": 3922, "Finetune/Step Time": 4.5411720238626} +{"Finetune/Learning Rate": 2.9919812273057815e-06, "Finetune/Loss": 0.8418712615966797, "Finetune/Loss (Raw)": 0.7212666869163513, "Finetune/Step": 3923, "Finetune/Step Time": 4.540848644450307} +{"Finetune/Learning Rate": 2.987538657527491e-06, "Finetune/Loss": 0.8419434428215027, "Finetune/Loss (Raw)": 0.9081860184669495, "Finetune/Step": 3924, "Finetune/Step Time": 4.555684799328446} +{"Finetune/Learning Rate": 2.9830988091485426e-06, "Finetune/Loss": 0.8432852029800415, "Finetune/Loss (Raw)": 0.9123485684394836, "Finetune/Step": 3925, "Finetune/Step Time": 4.552166927605867} +{"Finetune/Learning Rate": 2.9786616838919515e-06, "Finetune/Loss": 0.8431716561317444, "Finetune/Loss (Raw)": 0.8124861121177673, "Finetune/Step": 3926, "Finetune/Step Time": 4.551845815032721} +{"Finetune/Learning Rate": 2.9742272834796813e-06, "Finetune/Loss": 0.8434662818908691, "Finetune/Loss (Raw)": 0.8546213507652283, "Finetune/Step": 3927, "Finetune/Step Time": 4.55929453112185} +{"Finetune/Learning Rate": 2.969795609632636e-06, "Finetune/Loss": 0.8436537384986877, "Finetune/Loss (Raw)": 0.9815048575401306, "Finetune/Step": 3928, "Finetune/Step Time": 4.560072433203459} +{"Finetune/Learning Rate": 2.9653666640706614e-06, "Finetune/Loss": 0.8440152406692505, "Finetune/Loss (Raw)": 0.9163262844085693, "Finetune/Step": 3929, "Finetune/Step Time": 4.556378265842795} +{"Finetune/Learning Rate": 2.9609404485125414e-06, "Finetune/Loss": 0.8431150913238525, "Finetune/Loss (Raw)": 0.8417997360229492, "Finetune/Step": 3930, "Finetune/Step Time": 4.567479791119695} +{"Finetune/Learning Rate": 2.9565169646760108e-06, "Finetune/Loss": 0.8430265188217163, "Finetune/Loss (Raw)": 0.9867972731590271, "Finetune/Step": 3931, "Finetune/Step Time": 4.568368250504136} +{"Finetune/Learning Rate": 2.9520962142777367e-06, "Finetune/Loss": 0.8423717021942139, "Finetune/Loss (Raw)": 0.8322310447692871, "Finetune/Step": 3932, "Finetune/Step Time": 4.583133660256863} +{"Finetune/Learning Rate": 2.947678199033317e-06, "Finetune/Loss": 0.8408217430114746, "Finetune/Loss (Raw)": 0.6547209620475769, "Finetune/Step": 3933, "Finetune/Step Time": 4.591365434229374} +{"Finetune/Learning Rate": 2.943262920657306e-06, "Finetune/Loss": 0.8400619626045227, "Finetune/Loss (Raw)": 0.7877045273780823, "Finetune/Step": 3934, "Finetune/Step Time": 4.5778994876891375} +{"Finetune/Learning Rate": 2.938850380863184e-06, "Finetune/Loss": 0.8400002717971802, "Finetune/Loss (Raw)": 0.8349776864051819, "Finetune/Step": 3935, "Finetune/Step Time": 4.580533318221569} +{"Finetune/Learning Rate": 2.9344405813633724e-06, "Finetune/Loss": 0.8413870334625244, "Finetune/Loss (Raw)": 0.9474781155586243, "Finetune/Step": 3936, "Finetune/Step Time": 4.57184767909348} +{"Finetune/Learning Rate": 2.930033523869228e-06, "Finetune/Loss": 0.8417873382568359, "Finetune/Loss (Raw)": 0.8959932327270508, "Finetune/Step": 3937, "Finetune/Step Time": 4.566478269174695} +{"Finetune/Learning Rate": 2.925629210091043e-06, "Finetune/Loss": 0.8411504030227661, "Finetune/Loss (Raw)": 0.8586282730102539, "Finetune/Step": 3938, "Finetune/Step Time": 4.56671280413866} +{"Finetune/Learning Rate": 2.9212276417380468e-06, "Finetune/Loss": 0.8419386148452759, "Finetune/Loss (Raw)": 0.9367932677268982, "Finetune/Step": 3939, "Finetune/Step Time": 4.567935585975647} +{"Finetune/Learning Rate": 2.9168288205183983e-06, "Finetune/Loss": 0.842589259147644, "Finetune/Loss (Raw)": 0.8747748136520386, "Finetune/Step": 3940, "Finetune/Step Time": 4.555561909452081} +{"Finetune/Learning Rate": 2.9124327481392023e-06, "Finetune/Loss": 0.843245804309845, "Finetune/Loss (Raw)": 0.955894947052002, "Finetune/Step": 3941, "Finetune/Step Time": 4.558951297774911} +{"Finetune/Learning Rate": 2.9080394263064815e-06, "Finetune/Loss": 0.8442983627319336, "Finetune/Loss (Raw)": 0.9715517163276672, "Finetune/Step": 3942, "Finetune/Step Time": 4.565234145149589} +{"Finetune/Learning Rate": 2.9036488567252006e-06, "Finetune/Loss": 0.8436765074729919, "Finetune/Loss (Raw)": 0.7869964241981506, "Finetune/Step": 3943, "Finetune/Step Time": 4.570947770029306} +{"Finetune/Learning Rate": 2.8992610410992506e-06, "Finetune/Loss": 0.8439761400222778, "Finetune/Loss (Raw)": 0.915256917476654, "Finetune/Step": 3944, "Finetune/Step Time": 4.570526020601392} +{"Finetune/Learning Rate": 2.894875981131463e-06, "Finetune/Loss": 0.846116840839386, "Finetune/Loss (Raw)": 1.0762791633605957, "Finetune/Step": 3945, "Finetune/Step Time": 4.593060523271561} +{"Finetune/Learning Rate": 2.890493678523595e-06, "Finetune/Loss": 0.8466161489486694, "Finetune/Loss (Raw)": 0.8815578818321228, "Finetune/Step": 3946, "Finetune/Step Time": 4.60739615932107} +{"Finetune/Learning Rate": 2.8861141349763224e-06, "Finetune/Loss": 0.8474116325378418, "Finetune/Loss (Raw)": 0.9068823456764221, "Finetune/Step": 3947, "Finetune/Step Time": 4.614610968157649} +{"Finetune/Learning Rate": 2.881737352189271e-06, "Finetune/Loss": 0.8446487188339233, "Finetune/Loss (Raw)": 0.36254364252090454, "Finetune/Step": 3948, "Finetune/Step Time": 4.64869093708694} +{"Finetune/Learning Rate": 2.8773633318609796e-06, "Finetune/Loss": 0.8461419343948364, "Finetune/Loss (Raw)": 0.9988641738891602, "Finetune/Step": 3949, "Finetune/Step Time": 4.64517636038363} +{"Finetune/Learning Rate": 2.872992075688922e-06, "Finetune/Loss": 0.8476713299751282, "Finetune/Loss (Raw)": 0.9721001982688904, "Finetune/Step": 3950, "Finetune/Step Time": 4.63726400770247} +{"Finetune/Learning Rate": 2.868623585369497e-06, "Finetune/Loss": 0.8475725054740906, "Finetune/Loss (Raw)": 0.965717613697052, "Finetune/Step": 3951, "Finetune/Step Time": 4.641302280128002} +{"Finetune/Learning Rate": 2.864257862598029e-06, "Finetune/Loss": 0.8466753959655762, "Finetune/Loss (Raw)": 0.7799288034439087, "Finetune/Step": 3952, "Finetune/Step Time": 4.650204809382558} +{"Finetune/Learning Rate": 2.8598949090687723e-06, "Finetune/Loss": 0.8473403453826904, "Finetune/Loss (Raw)": 0.8452461957931519, "Finetune/Step": 3953, "Finetune/Step Time": 4.6478900369256735} +{"Finetune/Learning Rate": 2.8555347264748988e-06, "Finetune/Loss": 0.8473473191261292, "Finetune/Loss (Raw)": 0.8204694986343384, "Finetune/Step": 3954, "Finetune/Step Time": 4.641363434493542} +{"Finetune/Learning Rate": 2.8511773165085164e-06, "Finetune/Loss": 0.8466024994850159, "Finetune/Loss (Raw)": 0.7974470257759094, "Finetune/Step": 3955, "Finetune/Step Time": 4.614771721884608} +{"Finetune/Learning Rate": 2.8468226808606525e-06, "Finetune/Loss": 0.847435474395752, "Finetune/Loss (Raw)": 1.0539343357086182, "Finetune/Step": 3956, "Finetune/Step Time": 4.619858397170901} +{"Finetune/Learning Rate": 2.842470821221246e-06, "Finetune/Loss": 0.8522496819496155, "Finetune/Loss (Raw)": 0.8980900645256042, "Finetune/Step": 3957, "Finetune/Step Time": 4.5811076033860445} +{"Finetune/Learning Rate": 2.8381217392791784e-06, "Finetune/Loss": 0.8518427014350891, "Finetune/Loss (Raw)": 0.8338066339492798, "Finetune/Step": 3958, "Finetune/Step Time": 4.577422507107258} +{"Finetune/Learning Rate": 2.833775436722238e-06, "Finetune/Loss": 0.8513298034667969, "Finetune/Loss (Raw)": 0.7690133452415466, "Finetune/Step": 3959, "Finetune/Step Time": 4.589680582284927} +{"Finetune/Learning Rate": 2.829431915237144e-06, "Finetune/Loss": 0.849662184715271, "Finetune/Loss (Raw)": 0.8201838135719299, "Finetune/Step": 3960, "Finetune/Step Time": 4.592583464458585} +{"Finetune/Learning Rate": 2.825091176509529e-06, "Finetune/Loss": 0.8492114543914795, "Finetune/Loss (Raw)": 0.9003338813781738, "Finetune/Step": 3961, "Finetune/Step Time": 4.574821570888162} +{"Finetune/Learning Rate": 2.8207532222239497e-06, "Finetune/Loss": 0.8476426005363464, "Finetune/Loss (Raw)": 0.7714243531227112, "Finetune/Step": 3962, "Finetune/Step Time": 4.58261626586318} +{"Finetune/Learning Rate": 2.816418054063883e-06, "Finetune/Loss": 0.847430944442749, "Finetune/Loss (Raw)": 0.9188887476921082, "Finetune/Step": 3963, "Finetune/Step Time": 4.586660673841834} +{"Finetune/Learning Rate": 2.8120856737117187e-06, "Finetune/Loss": 0.8479016423225403, "Finetune/Loss (Raw)": 0.8734844923019409, "Finetune/Step": 3964, "Finetune/Step Time": 4.5948814637959} +{"Finetune/Learning Rate": 2.807756082848775e-06, "Finetune/Loss": 0.8489819169044495, "Finetune/Loss (Raw)": 0.9856278300285339, "Finetune/Step": 3965, "Finetune/Step Time": 4.607112027704716} +{"Finetune/Learning Rate": 2.803429283155281e-06, "Finetune/Loss": 0.84563809633255, "Finetune/Loss (Raw)": 0.3882691562175751, "Finetune/Step": 3966, "Finetune/Step Time": 4.650174915790558} +{"Finetune/Learning Rate": 2.7991052763103767e-06, "Finetune/Loss": 0.8451846241950989, "Finetune/Loss (Raw)": 0.8244142532348633, "Finetune/Step": 3967, "Finetune/Step Time": 4.6231596525758505} +{"Finetune/Learning Rate": 2.7947840639921308e-06, "Finetune/Loss": 0.8459638953208923, "Finetune/Loss (Raw)": 0.9389840364456177, "Finetune/Step": 3968, "Finetune/Step Time": 4.621252633631229} +{"Finetune/Learning Rate": 2.7904656478775196e-06, "Finetune/Loss": 0.8452321290969849, "Finetune/Loss (Raw)": 0.9125071167945862, "Finetune/Step": 3969, "Finetune/Step Time": 4.645886646583676} +{"Finetune/Learning Rate": 2.7861500296424372e-06, "Finetune/Loss": 0.8462270498275757, "Finetune/Loss (Raw)": 0.9359134435653687, "Finetune/Step": 3970, "Finetune/Step Time": 4.646529035642743} +{"Finetune/Learning Rate": 2.781837210961689e-06, "Finetune/Loss": 0.8473082780838013, "Finetune/Loss (Raw)": 0.9694112539291382, "Finetune/Step": 3971, "Finetune/Step Time": 4.638198034837842} +{"Finetune/Learning Rate": 2.7775271935089964e-06, "Finetune/Loss": 0.846954345703125, "Finetune/Loss (Raw)": 0.8005778193473816, "Finetune/Step": 3972, "Finetune/Step Time": 4.635595630854368} +{"Finetune/Learning Rate": 2.7732199789569937e-06, "Finetune/Loss": 0.8469848036766052, "Finetune/Loss (Raw)": 0.8423659205436707, "Finetune/Step": 3973, "Finetune/Step Time": 4.632207367569208} +{"Finetune/Learning Rate": 2.768915568977222e-06, "Finetune/Loss": 0.8478505611419678, "Finetune/Loss (Raw)": 0.8118133544921875, "Finetune/Step": 3974, "Finetune/Step Time": 4.634730860590935} +{"Finetune/Learning Rate": 2.7646139652401473e-06, "Finetune/Loss": 0.847570538520813, "Finetune/Loss (Raw)": 0.9507538080215454, "Finetune/Step": 3975, "Finetune/Step Time": 4.632947653532028} +{"Finetune/Learning Rate": 2.7603151694151354e-06, "Finetune/Loss": 0.851684033870697, "Finetune/Loss (Raw)": 0.8584205508232117, "Finetune/Step": 3976, "Finetune/Step Time": 4.6066855024546385} +{"Finetune/Learning Rate": 2.7560191831704584e-06, "Finetune/Loss": 0.8495388627052307, "Finetune/Loss (Raw)": 0.7551701664924622, "Finetune/Step": 3977, "Finetune/Step Time": 4.601374076679349} +{"Finetune/Learning Rate": 2.751726008173312e-06, "Finetune/Loss": 0.8497005701065063, "Finetune/Loss (Raw)": 0.6372184753417969, "Finetune/Step": 3978, "Finetune/Step Time": 4.592361692339182} +{"Finetune/Learning Rate": 2.747435646089791e-06, "Finetune/Loss": 0.850602388381958, "Finetune/Loss (Raw)": 0.8786689043045044, "Finetune/Step": 3979, "Finetune/Step Time": 4.5957651026546955} +{"Finetune/Learning Rate": 2.7431480985849023e-06, "Finetune/Loss": 0.8485294580459595, "Finetune/Loss (Raw)": 0.6555581092834473, "Finetune/Step": 3980, "Finetune/Step Time": 4.592061620205641} +{"Finetune/Learning Rate": 2.7388633673225586e-06, "Finetune/Loss": 0.8523433804512024, "Finetune/Loss (Raw)": 0.9449713230133057, "Finetune/Step": 3981, "Finetune/Step Time": 4.554665971547365} +{"Finetune/Learning Rate": 2.734581453965579e-06, "Finetune/Loss": 0.8519115447998047, "Finetune/Loss (Raw)": 0.8357298374176025, "Finetune/Step": 3982, "Finetune/Step Time": 4.559241443872452} +{"Finetune/Learning Rate": 2.730302360175693e-06, "Finetune/Loss": 0.8534380793571472, "Finetune/Loss (Raw)": 0.8357511162757874, "Finetune/Step": 3983, "Finetune/Step Time": 4.5431268233805895} +{"Finetune/Learning Rate": 2.726026087613528e-06, "Finetune/Loss": 0.8484410643577576, "Finetune/Loss (Raw)": 0.3420284688472748, "Finetune/Step": 3984, "Finetune/Step Time": 4.571041883900762} +{"Finetune/Learning Rate": 2.7217526379386327e-06, "Finetune/Loss": 0.8527498841285706, "Finetune/Loss (Raw)": 0.9155469536781311, "Finetune/Step": 3985, "Finetune/Step Time": 4.534355040639639} +{"Finetune/Learning Rate": 2.7174820128094393e-06, "Finetune/Loss": 0.858404278755188, "Finetune/Loss (Raw)": 1.0598033666610718, "Finetune/Step": 3986, "Finetune/Step Time": 4.493795186281204} +{"Finetune/Learning Rate": 2.713214213883295e-06, "Finetune/Loss": 0.8563518524169922, "Finetune/Loss (Raw)": 0.7918164730072021, "Finetune/Step": 3987, "Finetune/Step Time": 4.502425596117973} +{"Finetune/Learning Rate": 2.7089492428164556e-06, "Finetune/Loss": 0.856860876083374, "Finetune/Loss (Raw)": 0.94853276014328, "Finetune/Step": 3988, "Finetune/Step Time": 4.505453919991851} +{"Finetune/Learning Rate": 2.7046871012640687e-06, "Finetune/Loss": 0.8577899932861328, "Finetune/Loss (Raw)": 0.9597904682159424, "Finetune/Step": 3989, "Finetune/Step Time": 4.510070964694023} +{"Finetune/Learning Rate": 2.7004277908801877e-06, "Finetune/Loss": 0.8580373525619507, "Finetune/Loss (Raw)": 0.9169037342071533, "Finetune/Step": 3990, "Finetune/Step Time": 4.514426590874791} +{"Finetune/Learning Rate": 2.6961713133177705e-06, "Finetune/Loss": 0.8592395186424255, "Finetune/Loss (Raw)": 0.9696484208106995, "Finetune/Step": 3991, "Finetune/Step Time": 4.506457883864641} +{"Finetune/Learning Rate": 2.6919176702286696e-06, "Finetune/Loss": 0.8578577041625977, "Finetune/Loss (Raw)": 0.7448910474777222, "Finetune/Step": 3992, "Finetune/Step Time": 4.5181941743940115} +{"Finetune/Learning Rate": 2.687666863263644e-06, "Finetune/Loss": 0.8580629825592041, "Finetune/Loss (Raw)": 0.9021229147911072, "Finetune/Step": 3993, "Finetune/Step Time": 4.484731040894985} +{"Finetune/Learning Rate": 2.683418894072345e-06, "Finetune/Loss": 0.8560878038406372, "Finetune/Loss (Raw)": 0.7800106406211853, "Finetune/Step": 3994, "Finetune/Step Time": 4.49747034534812} +{"Finetune/Learning Rate": 2.6791737643033333e-06, "Finetune/Loss": 0.8539825677871704, "Finetune/Loss (Raw)": 0.805336058139801, "Finetune/Step": 3995, "Finetune/Step Time": 4.497199971228838} +{"Finetune/Learning Rate": 2.6749314756040547e-06, "Finetune/Loss": 0.8532663583755493, "Finetune/Loss (Raw)": 0.8003560900688171, "Finetune/Step": 3996, "Finetune/Step Time": 4.495355119928718} +{"Finetune/Learning Rate": 2.6706920296208573e-06, "Finetune/Loss": 0.8534250855445862, "Finetune/Loss (Raw)": 0.8313502073287964, "Finetune/Step": 3997, "Finetune/Step Time": 4.49971729144454} +{"Finetune/Learning Rate": 2.666455427998994e-06, "Finetune/Loss": 0.8550397157669067, "Finetune/Loss (Raw)": 1.014853596687317, "Finetune/Step": 3998, "Finetune/Step Time": 4.488878101110458} +{"Finetune/Learning Rate": 2.6622216723826035e-06, "Finetune/Loss": 0.8553897142410278, "Finetune/Loss (Raw)": 0.915256142616272, "Finetune/Step": 3999, "Finetune/Step Time": 4.5048404317349195} +{"Finetune/Learning Rate": 2.6579907644147227e-06, "Finetune/Loss": 0.8578711152076721, "Finetune/Loss (Raw)": 0.8269794583320618, "Finetune/Step": 4000, "Finetune/Step Time": 4.475070156157017} +{"Finetune/Learning Rate": 2.653762705737287e-06, "Finetune/Loss": 0.8586447238922119, "Finetune/Loss (Raw)": 0.8325979113578796, "Finetune/Step": 4001, "Finetune/Step Time": 4.486317176371813} +{"Finetune/Learning Rate": 2.6495374979911215e-06, "Finetune/Loss": 0.8580000400543213, "Finetune/Loss (Raw)": 0.7956063151359558, "Finetune/Step": 4002, "Finetune/Step Time": 4.496472056955099} +{"Finetune/Learning Rate": 2.6453151428159472e-06, "Finetune/Loss": 0.8579975366592407, "Finetune/Loss (Raw)": 0.8693183660507202, "Finetune/Step": 4003, "Finetune/Step Time": 4.508468383923173} +{"Finetune/Learning Rate": 2.641095641850375e-06, "Finetune/Loss": 0.8582473993301392, "Finetune/Loss (Raw)": 0.9165115356445312, "Finetune/Step": 4004, "Finetune/Step Time": 4.512963766232133} +{"Finetune/Learning Rate": 2.6368789967319196e-06, "Finetune/Loss": 0.8583984375, "Finetune/Loss (Raw)": 0.8747804164886475, "Finetune/Step": 4005, "Finetune/Step Time": 4.497568478807807} +{"Finetune/Learning Rate": 2.6326652090969697e-06, "Finetune/Loss": 0.8627920150756836, "Finetune/Loss (Raw)": 1.0348840951919556, "Finetune/Step": 4006, "Finetune/Step Time": 4.456346984952688} +{"Finetune/Learning Rate": 2.6284542805808144e-06, "Finetune/Loss": 0.8589751720428467, "Finetune/Loss (Raw)": 0.36420515179634094, "Finetune/Step": 4007, "Finetune/Step Time": 4.459708511829376} +{"Finetune/Learning Rate": 2.624246212817638e-06, "Finetune/Loss": 0.8581638932228088, "Finetune/Loss (Raw)": 0.7702946066856384, "Finetune/Step": 4008, "Finetune/Step Time": 4.457940815016627} +{"Finetune/Learning Rate": 2.620041007440508e-06, "Finetune/Loss": 0.8575862646102905, "Finetune/Loss (Raw)": 0.8373618721961975, "Finetune/Step": 4009, "Finetune/Step Time": 4.453232733532786} +{"Finetune/Learning Rate": 2.6158386660813807e-06, "Finetune/Loss": 0.8564703464508057, "Finetune/Loss (Raw)": 0.7979185581207275, "Finetune/Step": 4010, "Finetune/Step Time": 4.461429553106427} +{"Finetune/Learning Rate": 2.6116391903711035e-06, "Finetune/Loss": 0.856505274772644, "Finetune/Loss (Raw)": 0.9813268184661865, "Finetune/Step": 4011, "Finetune/Step Time": 4.4662013947963715} +{"Finetune/Learning Rate": 2.607442581939411e-06, "Finetune/Loss": 0.8553587198257446, "Finetune/Loss (Raw)": 0.6731463074684143, "Finetune/Step": 4012, "Finetune/Step Time": 4.48339026607573} +{"Finetune/Learning Rate": 2.603248842414925e-06, "Finetune/Loss": 0.8567920923233032, "Finetune/Loss (Raw)": 0.8579942584037781, "Finetune/Step": 4013, "Finetune/Step Time": 4.477113237604499} +{"Finetune/Learning Rate": 2.599057973425151e-06, "Finetune/Loss": 0.85724937915802, "Finetune/Loss (Raw)": 0.8099220395088196, "Finetune/Step": 4014, "Finetune/Step Time": 4.502400910481811} +{"Finetune/Learning Rate": 2.5948699765964915e-06, "Finetune/Loss": 0.8576103448867798, "Finetune/Loss (Raw)": 0.8497378826141357, "Finetune/Step": 4015, "Finetune/Step Time": 4.5029447339475155} +{"Finetune/Learning Rate": 2.5906848535542196e-06, "Finetune/Loss": 0.8574292063713074, "Finetune/Loss (Raw)": 0.8908303380012512, "Finetune/Step": 4016, "Finetune/Step Time": 4.504429703578353} +{"Finetune/Learning Rate": 2.5865026059224974e-06, "Finetune/Loss": 0.857702374458313, "Finetune/Loss (Raw)": 0.9119558930397034, "Finetune/Step": 4017, "Finetune/Step Time": 4.501123024150729} +{"Finetune/Learning Rate": 2.58232323532438e-06, "Finetune/Loss": 0.8531638383865356, "Finetune/Loss (Raw)": 0.325478732585907, "Finetune/Step": 4018, "Finetune/Step Time": 4.536413943395019} +{"Finetune/Learning Rate": 2.5781467433817973e-06, "Finetune/Loss": 0.8542587757110596, "Finetune/Loss (Raw)": 1.004530429840088, "Finetune/Step": 4019, "Finetune/Step Time": 4.527785802260041} +{"Finetune/Learning Rate": 2.573973131715565e-06, "Finetune/Loss": 0.854989230632782, "Finetune/Loss (Raw)": 0.9480593800544739, "Finetune/Step": 4020, "Finetune/Step Time": 4.526458205655217} +{"Finetune/Learning Rate": 2.569802401945378e-06, "Finetune/Loss": 0.8589545488357544, "Finetune/Loss (Raw)": 0.8780922293663025, "Finetune/Step": 4021, "Finetune/Step Time": 4.4982981607317924} +{"Finetune/Learning Rate": 2.565634555689818e-06, "Finetune/Loss": 0.8581640124320984, "Finetune/Loss (Raw)": 0.8036202788352966, "Finetune/Step": 4022, "Finetune/Step Time": 4.500992631539702} +{"Finetune/Learning Rate": 2.5614695945663415e-06, "Finetune/Loss": 0.8580053448677063, "Finetune/Loss (Raw)": 0.9030767679214478, "Finetune/Step": 4023, "Finetune/Step Time": 4.503539979457855} +{"Finetune/Learning Rate": 2.557307520191291e-06, "Finetune/Loss": 0.8582170009613037, "Finetune/Loss (Raw)": 0.9595282077789307, "Finetune/Step": 4024, "Finetune/Step Time": 4.509341241791844} +{"Finetune/Learning Rate": 2.5531483341798856e-06, "Finetune/Loss": 0.8580576181411743, "Finetune/Loss (Raw)": 0.8555523753166199, "Finetune/Step": 4025, "Finetune/Step Time": 4.509318897500634} +{"Finetune/Learning Rate": 2.548992038146224e-06, "Finetune/Loss": 0.8588995337486267, "Finetune/Loss (Raw)": 0.9662770628929138, "Finetune/Step": 4026, "Finetune/Step Time": 4.496305400505662} +{"Finetune/Learning Rate": 2.5448386337032817e-06, "Finetune/Loss": 0.8589256405830383, "Finetune/Loss (Raw)": 1.0026094913482666, "Finetune/Step": 4027, "Finetune/Step Time": 4.492378490045667} +{"Finetune/Learning Rate": 2.5406881224629175e-06, "Finetune/Loss": 0.8585911989212036, "Finetune/Loss (Raw)": 0.8152121305465698, "Finetune/Step": 4028, "Finetune/Step Time": 4.4775863997638226} +{"Finetune/Learning Rate": 2.5365405060358663e-06, "Finetune/Loss": 0.8583051562309265, "Finetune/Loss (Raw)": 0.8971675038337708, "Finetune/Step": 4029, "Finetune/Step Time": 4.477614957839251} +{"Finetune/Learning Rate": 2.532395786031726e-06, "Finetune/Loss": 0.8538212776184082, "Finetune/Loss (Raw)": 0.39660778641700745, "Finetune/Step": 4030, "Finetune/Step Time": 4.512950764968991} +{"Finetune/Learning Rate": 2.528253964058992e-06, "Finetune/Loss": 0.853136420249939, "Finetune/Loss (Raw)": 0.8622188568115234, "Finetune/Step": 4031, "Finetune/Step Time": 4.512997955083847} +{"Finetune/Learning Rate": 2.5241150417250216e-06, "Finetune/Loss": 0.8530763387680054, "Finetune/Loss (Raw)": 0.8760296106338501, "Finetune/Step": 4032, "Finetune/Step Time": 4.506444966420531} +{"Finetune/Learning Rate": 2.5199790206360486e-06, "Finetune/Loss": 0.8549326062202454, "Finetune/Loss (Raw)": 0.9297450184822083, "Finetune/Step": 4033, "Finetune/Step Time": 4.494819914922118} +{"Finetune/Learning Rate": 2.5158459023971826e-06, "Finetune/Loss": 0.8543256521224976, "Finetune/Loss (Raw)": 0.8551554083824158, "Finetune/Step": 4034, "Finetune/Step Time": 4.454912580549717} +{"Finetune/Learning Rate": 2.511715688612405e-06, "Finetune/Loss": 0.8518749475479126, "Finetune/Loss (Raw)": 0.8356611132621765, "Finetune/Step": 4035, "Finetune/Step Time": 4.461115896701813} +{"Finetune/Learning Rate": 2.5075883808845736e-06, "Finetune/Loss": 0.8536275625228882, "Finetune/Loss (Raw)": 1.0324310064315796, "Finetune/Step": 4036, "Finetune/Step Time": 4.447588723152876} +{"Finetune/Learning Rate": 2.5034639808154115e-06, "Finetune/Loss": 0.8533293604850769, "Finetune/Loss (Raw)": 0.7858912944793701, "Finetune/Step": 4037, "Finetune/Step Time": 4.4540959391742945} +{"Finetune/Learning Rate": 2.4993424900055228e-06, "Finetune/Loss": 0.8486548066139221, "Finetune/Loss (Raw)": 0.3570816218852997, "Finetune/Step": 4038, "Finetune/Step Time": 4.49146655574441} +{"Finetune/Learning Rate": 2.4952239100543808e-06, "Finetune/Loss": 0.8480433225631714, "Finetune/Loss (Raw)": 0.9399620890617371, "Finetune/Step": 4039, "Finetune/Step Time": 4.512067185714841} +{"Finetune/Learning Rate": 2.491108242560313e-06, "Finetune/Loss": 0.8489480018615723, "Finetune/Loss (Raw)": 0.9232001900672913, "Finetune/Step": 4040, "Finetune/Step Time": 4.501352099701762} +{"Finetune/Learning Rate": 2.486995489120542e-06, "Finetune/Loss": 0.8482744693756104, "Finetune/Loss (Raw)": 0.8088716268539429, "Finetune/Step": 4041, "Finetune/Step Time": 4.49592842720449} +{"Finetune/Learning Rate": 2.4828856513311405e-06, "Finetune/Loss": 0.8485136032104492, "Finetune/Loss (Raw)": 0.8670035600662231, "Finetune/Step": 4042, "Finetune/Step Time": 4.509010352194309} +{"Finetune/Learning Rate": 2.4787787307870605e-06, "Finetune/Loss": 0.8485554456710815, "Finetune/Loss (Raw)": 0.9071091413497925, "Finetune/Step": 4043, "Finetune/Step Time": 4.505703052505851} +{"Finetune/Learning Rate": 2.474674729082115e-06, "Finetune/Loss": 0.8484233617782593, "Finetune/Loss (Raw)": 0.8745425343513489, "Finetune/Step": 4044, "Finetune/Step Time": 4.497294118627906} +{"Finetune/Learning Rate": 2.470573647808987e-06, "Finetune/Loss": 0.8437486886978149, "Finetune/Loss (Raw)": 0.2924465239048004, "Finetune/Step": 4045, "Finetune/Step Time": 4.515910372138023} +{"Finetune/Learning Rate": 2.466475488559227e-06, "Finetune/Loss": 0.8427037000656128, "Finetune/Loss (Raw)": 0.8140081763267517, "Finetune/Step": 4046, "Finetune/Step Time": 4.512574529275298} +{"Finetune/Learning Rate": 2.4623802529232476e-06, "Finetune/Loss": 0.8421903848648071, "Finetune/Loss (Raw)": 0.7911173105239868, "Finetune/Step": 4047, "Finetune/Step Time": 4.514921614900231} +{"Finetune/Learning Rate": 2.458287942490335e-06, "Finetune/Loss": 0.8382717370986938, "Finetune/Loss (Raw)": 0.37386488914489746, "Finetune/Step": 4048, "Finetune/Step Time": 4.518141744658351} +{"Finetune/Learning Rate": 2.454198558848635e-06, "Finetune/Loss": 0.8388917446136475, "Finetune/Loss (Raw)": 0.7589831352233887, "Finetune/Step": 4049, "Finetune/Step Time": 4.500982109457254} +{"Finetune/Learning Rate": 2.4501121035851494e-06, "Finetune/Loss": 0.8380025029182434, "Finetune/Loss (Raw)": 0.8329036235809326, "Finetune/Step": 4050, "Finetune/Step Time": 4.499330760911107} +{"Finetune/Learning Rate": 2.4460285782857606e-06, "Finetune/Loss": 0.8384747505187988, "Finetune/Loss (Raw)": 0.7817188501358032, "Finetune/Step": 4051, "Finetune/Step Time": 4.491062605753541} +{"Finetune/Learning Rate": 2.441947984535201e-06, "Finetune/Loss": 0.8379413485527039, "Finetune/Loss (Raw)": 0.8399057388305664, "Finetune/Step": 4052, "Finetune/Step Time": 4.506268413737416} +{"Finetune/Learning Rate": 2.4378703239170707e-06, "Finetune/Loss": 0.8337440490722656, "Finetune/Loss (Raw)": 0.37510162591934204, "Finetune/Step": 4053, "Finetune/Step Time": 4.542173091322184} +{"Finetune/Learning Rate": 2.4337955980138293e-06, "Finetune/Loss": 0.834089457988739, "Finetune/Loss (Raw)": 0.8566890358924866, "Finetune/Step": 4054, "Finetune/Step Time": 4.55610385350883} +{"Finetune/Learning Rate": 2.4297238084067988e-06, "Finetune/Loss": 0.834444522857666, "Finetune/Loss (Raw)": 0.9000694751739502, "Finetune/Step": 4055, "Finetune/Step Time": 4.537717128172517} +{"Finetune/Learning Rate": 2.425654956676161e-06, "Finetune/Loss": 0.8317473530769348, "Finetune/Loss (Raw)": 0.6362689137458801, "Finetune/Step": 4056, "Finetune/Step Time": 4.55358973890543} +{"Finetune/Learning Rate": 2.421589044400955e-06, "Finetune/Loss": 0.8321895599365234, "Finetune/Loss (Raw)": 0.9729259610176086, "Finetune/Step": 4057, "Finetune/Step Time": 4.550173427909613} +{"Finetune/Learning Rate": 2.4175260731590878e-06, "Finetune/Loss": 0.8320191502571106, "Finetune/Loss (Raw)": 0.8199899196624756, "Finetune/Step": 4058, "Finetune/Step Time": 4.5362292397767305} +{"Finetune/Learning Rate": 2.413466044527318e-06, "Finetune/Loss": 0.8312619924545288, "Finetune/Loss (Raw)": 0.8898831009864807, "Finetune/Step": 4059, "Finetune/Step Time": 4.5504832081496716} +{"Finetune/Learning Rate": 2.4094089600812576e-06, "Finetune/Loss": 0.8297491073608398, "Finetune/Loss (Raw)": 0.6385852694511414, "Finetune/Step": 4060, "Finetune/Step Time": 4.5525234043598175} +{"Finetune/Learning Rate": 2.405354821395388e-06, "Finetune/Loss": 0.8309035301208496, "Finetune/Loss (Raw)": 0.8024794459342957, "Finetune/Step": 4061, "Finetune/Step Time": 4.550233475863934} +{"Finetune/Learning Rate": 2.4013036300430403e-06, "Finetune/Loss": 0.8326231241226196, "Finetune/Loss (Raw)": 1.0078129768371582, "Finetune/Step": 4062, "Finetune/Step Time": 4.553309328854084} +{"Finetune/Learning Rate": 2.3972553875964035e-06, "Finetune/Loss": 0.8288538455963135, "Finetune/Loss (Raw)": 0.3525184690952301, "Finetune/Step": 4063, "Finetune/Step Time": 4.581081474199891} +{"Finetune/Learning Rate": 2.393210095626515e-06, "Finetune/Loss": 0.8244292736053467, "Finetune/Loss (Raw)": 0.38113024830818176, "Finetune/Step": 4064, "Finetune/Step Time": 4.627044878900051} +{"Finetune/Learning Rate": 2.38916775570328e-06, "Finetune/Loss": 0.8235198855400085, "Finetune/Loss (Raw)": 0.7795904874801636, "Finetune/Step": 4065, "Finetune/Step Time": 4.6323149017989635} +{"Finetune/Learning Rate": 2.3851283693954506e-06, "Finetune/Loss": 0.8235998153686523, "Finetune/Loss (Raw)": 0.8688578605651855, "Finetune/Step": 4066, "Finetune/Step Time": 4.636046424508095} +{"Finetune/Learning Rate": 2.3810919382706297e-06, "Finetune/Loss": 0.8239328861236572, "Finetune/Loss (Raw)": 0.9794266819953918, "Finetune/Step": 4067, "Finetune/Step Time": 4.634957855567336} +{"Finetune/Learning Rate": 2.3770584638952874e-06, "Finetune/Loss": 0.824275553226471, "Finetune/Loss (Raw)": 0.9186356663703918, "Finetune/Step": 4068, "Finetune/Step Time": 4.636997122317553} +{"Finetune/Learning Rate": 2.3730279478347275e-06, "Finetune/Loss": 0.8240286707878113, "Finetune/Loss (Raw)": 0.9242956042289734, "Finetune/Step": 4069, "Finetune/Step Time": 4.650566069409251} +{"Finetune/Learning Rate": 2.3690003916531157e-06, "Finetune/Loss": 0.8228119611740112, "Finetune/Loss (Raw)": 0.8158068060874939, "Finetune/Step": 4070, "Finetune/Step Time": 4.646400602534413} +{"Finetune/Learning Rate": 2.3649757969134667e-06, "Finetune/Loss": 0.8234094381332397, "Finetune/Loss (Raw)": 0.8634774684906006, "Finetune/Step": 4071, "Finetune/Step Time": 4.6530049704015255} +{"Finetune/Learning Rate": 2.3609541651776537e-06, "Finetune/Loss": 0.8222473859786987, "Finetune/Loss (Raw)": 0.7665188908576965, "Finetune/Step": 4072, "Finetune/Step Time": 4.662849543616176} +{"Finetune/Learning Rate": 2.356935498006391e-06, "Finetune/Loss": 0.8208830952644348, "Finetune/Loss (Raw)": 0.9016432762145996, "Finetune/Step": 4073, "Finetune/Step Time": 4.645104832947254} +{"Finetune/Learning Rate": 2.3529197969592375e-06, "Finetune/Loss": 0.8216763734817505, "Finetune/Loss (Raw)": 0.9831026792526245, "Finetune/Step": 4074, "Finetune/Step Time": 4.632404208183289} +{"Finetune/Learning Rate": 2.3489070635946175e-06, "Finetune/Loss": 0.8210104703903198, "Finetune/Loss (Raw)": 0.8216476440429688, "Finetune/Step": 4075, "Finetune/Step Time": 4.620989121496677} +{"Finetune/Learning Rate": 2.3448972994697906e-06, "Finetune/Loss": 0.8234221935272217, "Finetune/Loss (Raw)": 0.6712472438812256, "Finetune/Step": 4076, "Finetune/Step Time": 4.601924000307918} +{"Finetune/Learning Rate": 2.340890506140869e-06, "Finetune/Loss": 0.8222706317901611, "Finetune/Loss (Raw)": 0.8514595031738281, "Finetune/Step": 4077, "Finetune/Step Time": 4.615638971328735} +{"Finetune/Learning Rate": 2.3368866851628102e-06, "Finetune/Loss": 0.8210279941558838, "Finetune/Loss (Raw)": 0.8130360245704651, "Finetune/Step": 4078, "Finetune/Step Time": 4.618286145851016} +{"Finetune/Learning Rate": 2.3328858380894194e-06, "Finetune/Loss": 0.821635901927948, "Finetune/Loss (Raw)": 1.0435371398925781, "Finetune/Step": 4079, "Finetune/Step Time": 4.629293726757169} +{"Finetune/Learning Rate": 2.328887966473348e-06, "Finetune/Loss": 0.8215979337692261, "Finetune/Loss (Raw)": 0.775070071220398, "Finetune/Step": 4080, "Finetune/Step Time": 4.628983853384852} +{"Finetune/Learning Rate": 2.324893071866087e-06, "Finetune/Loss": 0.821703314781189, "Finetune/Loss (Raw)": 0.8587315082550049, "Finetune/Step": 4081, "Finetune/Step Time": 4.651130074635148} +{"Finetune/Learning Rate": 2.320901155817983e-06, "Finetune/Loss": 0.8215761780738831, "Finetune/Loss (Raw)": 0.8041984438896179, "Finetune/Step": 4082, "Finetune/Step Time": 4.659127818420529} +{"Finetune/Learning Rate": 2.316912219878222e-06, "Finetune/Loss": 0.822989821434021, "Finetune/Loss (Raw)": 0.9783937931060791, "Finetune/Step": 4083, "Finetune/Step Time": 4.661695871502161} +{"Finetune/Learning Rate": 2.312926265594825e-06, "Finetune/Loss": 0.8184148669242859, "Finetune/Loss (Raw)": 0.4683396518230438, "Finetune/Step": 4084, "Finetune/Step Time": 4.6896631103008986} +{"Finetune/Learning Rate": 2.308943294514667e-06, "Finetune/Loss": 0.8179414868354797, "Finetune/Loss (Raw)": 0.8374967575073242, "Finetune/Step": 4085, "Finetune/Step Time": 4.701299339532852} +{"Finetune/Learning Rate": 2.3049633081834634e-06, "Finetune/Loss": 0.8182810544967651, "Finetune/Loss (Raw)": 0.8772639036178589, "Finetune/Step": 4086, "Finetune/Step Time": 4.708382863551378} +{"Finetune/Learning Rate": 2.3009863081457663e-06, "Finetune/Loss": 0.8200349807739258, "Finetune/Loss (Raw)": 0.9935211539268494, "Finetune/Step": 4087, "Finetune/Step Time": 4.69577026553452} +{"Finetune/Learning Rate": 2.2970122959449716e-06, "Finetune/Loss": 0.8210053443908691, "Finetune/Loss (Raw)": 0.9443938732147217, "Finetune/Step": 4088, "Finetune/Step Time": 4.696651319041848} +{"Finetune/Learning Rate": 2.2930412731233174e-06, "Finetune/Loss": 0.8205870389938354, "Finetune/Loss (Raw)": 0.8467829823493958, "Finetune/Step": 4089, "Finetune/Step Time": 4.698189387097955} +{"Finetune/Learning Rate": 2.2890732412218795e-06, "Finetune/Loss": 0.8210794925689697, "Finetune/Loss (Raw)": 0.8344600200653076, "Finetune/Step": 4090, "Finetune/Step Time": 4.690873058512807} +{"Finetune/Learning Rate": 2.2851082017805704e-06, "Finetune/Loss": 0.8204044103622437, "Finetune/Loss (Raw)": 0.8324832916259766, "Finetune/Step": 4091, "Finetune/Step Time": 4.703593784943223} +{"Finetune/Learning Rate": 2.2811461563381497e-06, "Finetune/Loss": 0.8201040029525757, "Finetune/Loss (Raw)": 0.8350328803062439, "Finetune/Step": 4092, "Finetune/Step Time": 4.69968387670815} +{"Finetune/Learning Rate": 2.2771871064322117e-06, "Finetune/Loss": 0.8200798034667969, "Finetune/Loss (Raw)": 0.982525646686554, "Finetune/Step": 4093, "Finetune/Step Time": 4.689774477854371} +{"Finetune/Learning Rate": 2.2732310535991775e-06, "Finetune/Loss": 0.8252841830253601, "Finetune/Loss (Raw)": 1.0544283390045166, "Finetune/Step": 4094, "Finetune/Step Time": 4.652004972100258} +{"Finetune/Learning Rate": 2.2692779993743207e-06, "Finetune/Loss": 0.824894368648529, "Finetune/Loss (Raw)": 0.7745256423950195, "Finetune/Step": 4095, "Finetune/Step Time": 4.659315459430218} +{"Finetune/Learning Rate": 2.265327945291742e-06, "Finetune/Loss": 0.8255382776260376, "Finetune/Loss (Raw)": 1.0213993787765503, "Finetune/Step": 4096, "Finetune/Step Time": 4.663860434666276} +{"Finetune/Learning Rate": 2.2613808928843827e-06, "Finetune/Loss": 0.8258578777313232, "Finetune/Loss (Raw)": 0.9534187316894531, "Finetune/Step": 4097, "Finetune/Step Time": 4.649475937709212} +{"Finetune/Learning Rate": 2.2574368436840144e-06, "Finetune/Loss": 0.8239107728004456, "Finetune/Loss (Raw)": 0.6866837739944458, "Finetune/Step": 4098, "Finetune/Step Time": 4.658023888245225} +{"Finetune/Learning Rate": 2.2534957992212457e-06, "Finetune/Loss": 0.8234896659851074, "Finetune/Loss (Raw)": 0.9155102372169495, "Finetune/Step": 4099, "Finetune/Step Time": 4.66136491112411} +{"Finetune/Learning Rate": 2.2495577610255203e-06, "Finetune/Loss": 0.8246943950653076, "Finetune/Loss (Raw)": 0.9547869563102722, "Finetune/Step": 4100, "Finetune/Step Time": 4.661508025601506} +{"Finetune/Learning Rate": 2.24562273062511e-06, "Finetune/Loss": 0.8250636458396912, "Finetune/Loss (Raw)": 0.8896254301071167, "Finetune/Step": 4101, "Finetune/Step Time": 4.682317057624459} +{"Finetune/Learning Rate": 2.2416907095471296e-06, "Finetune/Loss": 0.8253445625305176, "Finetune/Loss (Raw)": 0.8477725982666016, "Finetune/Step": 4102, "Finetune/Step Time": 4.678943257778883} +{"Finetune/Learning Rate": 2.237761699317519e-06, "Finetune/Loss": 0.824845552444458, "Finetune/Loss (Raw)": 0.8868762254714966, "Finetune/Step": 4103, "Finetune/Step Time": 4.681588649749756} +{"Finetune/Learning Rate": 2.2338357014610425e-06, "Finetune/Loss": 0.8248957395553589, "Finetune/Loss (Raw)": 0.8648461699485779, "Finetune/Step": 4104, "Finetune/Step Time": 4.6691973228007555} +{"Finetune/Learning Rate": 2.2299127175013113e-06, "Finetune/Loss": 0.8250324726104736, "Finetune/Loss (Raw)": 0.7726714611053467, "Finetune/Step": 4105, "Finetune/Step Time": 4.663482334464788} +{"Finetune/Learning Rate": 2.2259927489607568e-06, "Finetune/Loss": 0.826525092124939, "Finetune/Loss (Raw)": 0.8282686471939087, "Finetune/Step": 4106, "Finetune/Step Time": 4.661878451704979} +{"Finetune/Learning Rate": 2.222075797360641e-06, "Finetune/Loss": 0.8261984586715698, "Finetune/Loss (Raw)": 0.836858868598938, "Finetune/Step": 4107, "Finetune/Step Time": 4.664173459634185} +{"Finetune/Learning Rate": 2.2181618642210577e-06, "Finetune/Loss": 0.8283643126487732, "Finetune/Loss (Raw)": 0.9327948689460754, "Finetune/Step": 4108, "Finetune/Step Time": 4.663006294518709} +{"Finetune/Learning Rate": 2.2142509510609277e-06, "Finetune/Loss": 0.8273705244064331, "Finetune/Loss (Raw)": 0.8177670240402222, "Finetune/Step": 4109, "Finetune/Step Time": 4.670233679935336} +{"Finetune/Learning Rate": 2.210343059397999e-06, "Finetune/Loss": 0.8279180526733398, "Finetune/Loss (Raw)": 0.9058066606521606, "Finetune/Step": 4110, "Finetune/Step Time": 4.672257537022233} +{"Finetune/Learning Rate": 2.206438190748846e-06, "Finetune/Loss": 0.829107403755188, "Finetune/Loss (Raw)": 0.9879971146583557, "Finetune/Step": 4111, "Finetune/Step Time": 4.684133321046829} +{"Finetune/Learning Rate": 2.2025363466288796e-06, "Finetune/Loss": 0.832284688949585, "Finetune/Loss (Raw)": 0.7487166523933411, "Finetune/Step": 4112, "Finetune/Step Time": 4.657865507528186} +{"Finetune/Learning Rate": 2.1986375285523233e-06, "Finetune/Loss": 0.8307574391365051, "Finetune/Loss (Raw)": 0.7200594544410706, "Finetune/Step": 4113, "Finetune/Step Time": 4.6557679157704115} +{"Finetune/Learning Rate": 2.194741738032229e-06, "Finetune/Loss": 0.8292937278747559, "Finetune/Loss (Raw)": 0.8724474906921387, "Finetune/Step": 4114, "Finetune/Step Time": 4.657192723825574} +{"Finetune/Learning Rate": 2.1908489765804853e-06, "Finetune/Loss": 0.829930305480957, "Finetune/Loss (Raw)": 0.8732935190200806, "Finetune/Step": 4115, "Finetune/Step Time": 4.64650390855968} +{"Finetune/Learning Rate": 2.1869592457077924e-06, "Finetune/Loss": 0.827450156211853, "Finetune/Loss (Raw)": 0.6310820579528809, "Finetune/Step": 4116, "Finetune/Step Time": 4.661671409383416} +{"Finetune/Learning Rate": 2.183072546923679e-06, "Finetune/Loss": 0.8259503245353699, "Finetune/Loss (Raw)": 0.7678076028823853, "Finetune/Step": 4117, "Finetune/Step Time": 4.664831502363086} +{"Finetune/Learning Rate": 2.179188881736498e-06, "Finetune/Loss": 0.8269308805465698, "Finetune/Loss (Raw)": 1.0424145460128784, "Finetune/Step": 4118, "Finetune/Step Time": 4.657679812982678} +{"Finetune/Learning Rate": 2.1753082516534242e-06, "Finetune/Loss": 0.8270099759101868, "Finetune/Loss (Raw)": 0.9797685742378235, "Finetune/Step": 4119, "Finetune/Step Time": 4.657487513497472} +{"Finetune/Learning Rate": 2.1714306581804533e-06, "Finetune/Loss": 0.8281141519546509, "Finetune/Loss (Raw)": 0.8862283229827881, "Finetune/Step": 4120, "Finetune/Step Time": 4.652817416936159} +{"Finetune/Learning Rate": 2.1675561028224012e-06, "Finetune/Loss": 0.8279290199279785, "Finetune/Loss (Raw)": 0.8784312009811401, "Finetune/Step": 4121, "Finetune/Step Time": 4.658445727080107} +{"Finetune/Learning Rate": 2.163684587082915e-06, "Finetune/Loss": 0.8286148309707642, "Finetune/Loss (Raw)": 0.8677937388420105, "Finetune/Step": 4122, "Finetune/Step Time": 4.657442813739181} +{"Finetune/Learning Rate": 2.159816112464447e-06, "Finetune/Loss": 0.8302371501922607, "Finetune/Loss (Raw)": 1.012985110282898, "Finetune/Step": 4123, "Finetune/Step Time": 4.670207858085632} +{"Finetune/Learning Rate": 2.1559506804682762e-06, "Finetune/Loss": 0.8290407061576843, "Finetune/Loss (Raw)": 0.6472187042236328, "Finetune/Step": 4124, "Finetune/Step Time": 4.678081683814526} +{"Finetune/Learning Rate": 2.1520882925945074e-06, "Finetune/Loss": 0.8295989036560059, "Finetune/Loss (Raw)": 0.9027939438819885, "Finetune/Step": 4125, "Finetune/Step Time": 4.676156735047698} +{"Finetune/Learning Rate": 2.1482289503420537e-06, "Finetune/Loss": 0.8285378217697144, "Finetune/Loss (Raw)": 0.8790411949157715, "Finetune/Step": 4126, "Finetune/Step Time": 4.665022067725658} +{"Finetune/Learning Rate": 2.144372655208653e-06, "Finetune/Loss": 0.8284065127372742, "Finetune/Loss (Raw)": 0.8984430432319641, "Finetune/Step": 4127, "Finetune/Step Time": 4.645099051296711} +{"Finetune/Learning Rate": 2.1405194086908555e-06, "Finetune/Loss": 0.8283532857894897, "Finetune/Loss (Raw)": 0.8201663494110107, "Finetune/Step": 4128, "Finetune/Step Time": 4.640424981713295} +{"Finetune/Learning Rate": 2.1366692122840317e-06, "Finetune/Loss": 0.8282463550567627, "Finetune/Loss (Raw)": 0.8189154267311096, "Finetune/Step": 4129, "Finetune/Step Time": 4.6210123132914305} +{"Finetune/Learning Rate": 2.1328220674823686e-06, "Finetune/Loss": 0.8285607695579529, "Finetune/Loss (Raw)": 0.8358510732650757, "Finetune/Step": 4130, "Finetune/Step Time": 4.61175337433815} +{"Finetune/Learning Rate": 2.1289779757788655e-06, "Finetune/Loss": 0.8286619186401367, "Finetune/Loss (Raw)": 0.8822590708732605, "Finetune/Step": 4131, "Finetune/Step Time": 4.605450948700309} +{"Finetune/Learning Rate": 2.1251369386653454e-06, "Finetune/Loss": 0.828460693359375, "Finetune/Loss (Raw)": 0.8907573223114014, "Finetune/Step": 4132, "Finetune/Step Time": 4.6072142496705055} +{"Finetune/Learning Rate": 2.121298957632434e-06, "Finetune/Loss": 0.828208863735199, "Finetune/Loss (Raw)": 0.8425498008728027, "Finetune/Step": 4133, "Finetune/Step Time": 4.614812785759568} +{"Finetune/Learning Rate": 2.117464034169575e-06, "Finetune/Loss": 0.82684326171875, "Finetune/Loss (Raw)": 0.860086977481842, "Finetune/Step": 4134, "Finetune/Step Time": 4.631220178678632} +{"Finetune/Learning Rate": 2.113632169765033e-06, "Finetune/Loss": 0.8301501870155334, "Finetune/Loss (Raw)": 0.7874879837036133, "Finetune/Step": 4135, "Finetune/Step Time": 4.593132583424449} +{"Finetune/Learning Rate": 2.1098033659058792e-06, "Finetune/Loss": 0.8306605815887451, "Finetune/Loss (Raw)": 0.8356225490570068, "Finetune/Step": 4136, "Finetune/Step Time": 4.5934955179691315} +{"Finetune/Learning Rate": 2.105977624077995e-06, "Finetune/Loss": 0.8303043842315674, "Finetune/Loss (Raw)": 0.7917736172676086, "Finetune/Step": 4137, "Finetune/Step Time": 4.5910409688949585} +{"Finetune/Learning Rate": 2.102154945766076e-06, "Finetune/Loss": 0.831489622592926, "Finetune/Loss (Raw)": 0.9496307373046875, "Finetune/Step": 4138, "Finetune/Step Time": 4.576846167445183} +{"Finetune/Learning Rate": 2.0983353324536316e-06, "Finetune/Loss": 0.8299745917320251, "Finetune/Loss (Raw)": 0.787402331829071, "Finetune/Step": 4139, "Finetune/Step Time": 4.566948480904102} +{"Finetune/Learning Rate": 2.0945187856229778e-06, "Finetune/Loss": 0.8310496807098389, "Finetune/Loss (Raw)": 0.8107579350471497, "Finetune/Step": 4140, "Finetune/Step Time": 4.556865688413382} +{"Finetune/Learning Rate": 2.0907053067552384e-06, "Finetune/Loss": 0.8273589611053467, "Finetune/Loss (Raw)": 0.38558098673820496, "Finetune/Step": 4141, "Finetune/Step Time": 4.586899694055319} +{"Finetune/Learning Rate": 2.0868948973303593e-06, "Finetune/Loss": 0.8270763158798218, "Finetune/Loss (Raw)": 0.7737425565719604, "Finetune/Step": 4142, "Finetune/Step Time": 4.563928814604878} +{"Finetune/Learning Rate": 2.0830875588270773e-06, "Finetune/Loss": 0.826540470123291, "Finetune/Loss (Raw)": 0.7811537384986877, "Finetune/Step": 4143, "Finetune/Step Time": 4.560688350349665} +{"Finetune/Learning Rate": 2.0792832927229465e-06, "Finetune/Loss": 0.8218497037887573, "Finetune/Loss (Raw)": 0.2904030978679657, "Finetune/Step": 4144, "Finetune/Step Time": 4.595113817602396} +{"Finetune/Learning Rate": 2.0754821004943337e-06, "Finetune/Loss": 0.8204846382141113, "Finetune/Loss (Raw)": 0.7372381687164307, "Finetune/Step": 4145, "Finetune/Step Time": 4.6032838597893715} +{"Finetune/Learning Rate": 2.071683983616406e-06, "Finetune/Loss": 0.8250308036804199, "Finetune/Loss (Raw)": 0.9073837995529175, "Finetune/Step": 4146, "Finetune/Step Time": 4.558668289333582} +{"Finetune/Learning Rate": 2.0678889435631333e-06, "Finetune/Loss": 0.8237994909286499, "Finetune/Loss (Raw)": 0.846920371055603, "Finetune/Step": 4147, "Finetune/Step Time": 4.553118826821446} +{"Finetune/Learning Rate": 2.064096981807302e-06, "Finetune/Loss": 0.8234328031539917, "Finetune/Loss (Raw)": 0.9011226296424866, "Finetune/Step": 4148, "Finetune/Step Time": 4.553450021892786} +{"Finetune/Learning Rate": 2.060308099820496e-06, "Finetune/Loss": 0.8231188058853149, "Finetune/Loss (Raw)": 0.8379021286964417, "Finetune/Step": 4149, "Finetune/Step Time": 4.542114349082112} +{"Finetune/Learning Rate": 2.056522299073108e-06, "Finetune/Loss": 0.8228189945220947, "Finetune/Loss (Raw)": 0.7652419805526733, "Finetune/Step": 4150, "Finetune/Step Time": 4.572506036609411} +{"Finetune/Learning Rate": 2.052739581034332e-06, "Finetune/Loss": 0.8225024342536926, "Finetune/Loss (Raw)": 0.8625569343566895, "Finetune/Step": 4151, "Finetune/Step Time": 4.577812736853957} +{"Finetune/Learning Rate": 2.0489599471721678e-06, "Finetune/Loss": 0.8217077255249023, "Finetune/Loss (Raw)": 0.8578085899353027, "Finetune/Step": 4152, "Finetune/Step Time": 4.5825827941298485} +{"Finetune/Learning Rate": 2.0451833989534165e-06, "Finetune/Loss": 0.8215100169181824, "Finetune/Loss (Raw)": 0.8302480578422546, "Finetune/Step": 4153, "Finetune/Step Time": 4.5849329829216} +{"Finetune/Learning Rate": 2.0414099378436812e-06, "Finetune/Loss": 0.8208233118057251, "Finetune/Loss (Raw)": 0.8783703446388245, "Finetune/Step": 4154, "Finetune/Step Time": 4.589054748415947} +{"Finetune/Learning Rate": 2.0376395653073735e-06, "Finetune/Loss": 0.8193987607955933, "Finetune/Loss (Raw)": 0.8202770948410034, "Finetune/Step": 4155, "Finetune/Step Time": 4.597263464704156} +{"Finetune/Learning Rate": 2.0338722828077007e-06, "Finetune/Loss": 0.8197583556175232, "Finetune/Loss (Raw)": 0.8612354397773743, "Finetune/Step": 4156, "Finetune/Step Time": 4.5960503201931715} +{"Finetune/Learning Rate": 2.030108091806664e-06, "Finetune/Loss": 0.8203048706054688, "Finetune/Loss (Raw)": 0.9671143889427185, "Finetune/Step": 4157, "Finetune/Step Time": 4.596545562148094} +{"Finetune/Learning Rate": 2.0263469937650813e-06, "Finetune/Loss": 0.8236484527587891, "Finetune/Loss (Raw)": 0.8245983719825745, "Finetune/Step": 4158, "Finetune/Step Time": 4.562279103323817} +{"Finetune/Learning Rate": 2.0225889901425565e-06, "Finetune/Loss": 0.824195146560669, "Finetune/Loss (Raw)": 0.9321896433830261, "Finetune/Step": 4159, "Finetune/Step Time": 4.56203593313694} +{"Finetune/Learning Rate": 2.0188340823975005e-06, "Finetune/Loss": 0.82499760389328, "Finetune/Loss (Raw)": 0.9787429571151733, "Finetune/Step": 4160, "Finetune/Step Time": 4.5550917983055115} +{"Finetune/Learning Rate": 2.0150822719871167e-06, "Finetune/Loss": 0.8248003125190735, "Finetune/Loss (Raw)": 0.904496431350708, "Finetune/Step": 4161, "Finetune/Step Time": 4.55244455114007} +{"Finetune/Learning Rate": 2.011333560367411e-06, "Finetune/Loss": 0.8255298733711243, "Finetune/Loss (Raw)": 0.9485333561897278, "Finetune/Step": 4162, "Finetune/Step Time": 4.554060615599155} +{"Finetune/Learning Rate": 2.007587948993185e-06, "Finetune/Loss": 0.8245521783828735, "Finetune/Loss (Raw)": 0.7105194330215454, "Finetune/Step": 4163, "Finetune/Step Time": 4.565096959471703} +{"Finetune/Learning Rate": 2.003845439318035e-06, "Finetune/Loss": 0.8211429715156555, "Finetune/Loss (Raw)": 0.5960517525672913, "Finetune/Step": 4164, "Finetune/Step Time": 4.577198758721352} +{"Finetune/Learning Rate": 2.000106032794361e-06, "Finetune/Loss": 0.8228747844696045, "Finetune/Loss (Raw)": 1.0075608491897583, "Finetune/Step": 4165, "Finetune/Step Time": 4.569721482694149} +{"Finetune/Learning Rate": 1.996369730873353e-06, "Finetune/Loss": 0.8274484872817993, "Finetune/Loss (Raw)": 0.9425123333930969, "Finetune/Step": 4166, "Finetune/Step Time": 4.530482776463032} +{"Finetune/Learning Rate": 1.99263653500499e-06, "Finetune/Loss": 0.8262840509414673, "Finetune/Loss (Raw)": 0.7909190654754639, "Finetune/Step": 4167, "Finetune/Step Time": 4.514893818646669} +{"Finetune/Learning Rate": 1.9889064466380593e-06, "Finetune/Loss": 0.8245518207550049, "Finetune/Loss (Raw)": 0.7014775276184082, "Finetune/Step": 4168, "Finetune/Step Time": 4.512757109478116} +{"Finetune/Learning Rate": 1.985179467220133e-06, "Finetune/Loss": 0.8211579322814941, "Finetune/Loss (Raw)": 0.37445613741874695, "Finetune/Step": 4169, "Finetune/Step Time": 4.551583405584097} +{"Finetune/Learning Rate": 1.9814555981975803e-06, "Finetune/Loss": 0.820406436920166, "Finetune/Loss (Raw)": 0.7708051204681396, "Finetune/Step": 4170, "Finetune/Step Time": 4.5433459505438805} +{"Finetune/Learning Rate": 1.9777348410155606e-06, "Finetune/Loss": 0.820357084274292, "Finetune/Loss (Raw)": 0.9007993340492249, "Finetune/Step": 4171, "Finetune/Step Time": 4.545921329408884} +{"Finetune/Learning Rate": 1.9740171971180278e-06, "Finetune/Loss": 0.8205745816230774, "Finetune/Loss (Raw)": 0.9023755192756653, "Finetune/Step": 4172, "Finetune/Step Time": 4.566244134679437} +{"Finetune/Learning Rate": 1.9703026679477253e-06, "Finetune/Loss": 0.8247315883636475, "Finetune/Loss (Raw)": 0.8245490789413452, "Finetune/Step": 4173, "Finetune/Step Time": 4.533222410827875} +{"Finetune/Learning Rate": 1.9665912549461898e-06, "Finetune/Loss": 0.8211761713027954, "Finetune/Loss (Raw)": 0.35891616344451904, "Finetune/Step": 4174, "Finetune/Step Time": 4.573082296177745} +{"Finetune/Learning Rate": 1.962882959553751e-06, "Finetune/Loss": 0.8210663795471191, "Finetune/Loss (Raw)": 0.7770587801933289, "Finetune/Step": 4175, "Finetune/Step Time": 4.562235409393907} +{"Finetune/Learning Rate": 1.959177783209527e-06, "Finetune/Loss": 0.8243947625160217, "Finetune/Loss (Raw)": 0.7998967170715332, "Finetune/Step": 4176, "Finetune/Step Time": 4.536629840731621} +{"Finetune/Learning Rate": 1.955475727351417e-06, "Finetune/Loss": 0.8259462118148804, "Finetune/Loss (Raw)": 0.9575688242912292, "Finetune/Step": 4177, "Finetune/Step Time": 4.541695723310113} +{"Finetune/Learning Rate": 1.9517767934161224e-06, "Finetune/Loss": 0.8265396356582642, "Finetune/Loss (Raw)": 0.9088627696037292, "Finetune/Step": 4178, "Finetune/Step Time": 4.544003022834659} +{"Finetune/Learning Rate": 1.9480809828391266e-06, "Finetune/Loss": 0.8272842168807983, "Finetune/Loss (Raw)": 0.8770302534103394, "Finetune/Step": 4179, "Finetune/Step Time": 4.55669479817152} +{"Finetune/Learning Rate": 1.9443882970547003e-06, "Finetune/Loss": 0.8281906247138977, "Finetune/Loss (Raw)": 0.9559211134910583, "Finetune/Step": 4180, "Finetune/Step Time": 4.530969604849815} +{"Finetune/Learning Rate": 1.940698737495904e-06, "Finetune/Loss": 0.831601619720459, "Finetune/Loss (Raw)": 0.811715304851532, "Finetune/Step": 4181, "Finetune/Step Time": 4.4963200725615025} +{"Finetune/Learning Rate": 1.937012305594583e-06, "Finetune/Loss": 0.8302234411239624, "Finetune/Loss (Raw)": 0.6802792549133301, "Finetune/Step": 4182, "Finetune/Step Time": 4.497455507516861} +{"Finetune/Learning Rate": 1.93332900278137e-06, "Finetune/Loss": 0.8304895162582397, "Finetune/Loss (Raw)": 0.9341251254081726, "Finetune/Step": 4183, "Finetune/Step Time": 4.496704787015915} +{"Finetune/Learning Rate": 1.9296488304856785e-06, "Finetune/Loss": 0.8325549364089966, "Finetune/Loss (Raw)": 0.9006403088569641, "Finetune/Step": 4184, "Finetune/Step Time": 4.474268667399883} +{"Finetune/Learning Rate": 1.92597179013572e-06, "Finetune/Loss": 0.8324646949768066, "Finetune/Loss (Raw)": 0.9613792896270752, "Finetune/Step": 4185, "Finetune/Step Time": 4.483344729989767} +{"Finetune/Learning Rate": 1.9222978831584793e-06, "Finetune/Loss": 0.8334074020385742, "Finetune/Loss (Raw)": 0.9406561851501465, "Finetune/Step": 4186, "Finetune/Step Time": 4.4967693798244} +{"Finetune/Learning Rate": 1.9186271109797215e-06, "Finetune/Loss": 0.8327085971832275, "Finetune/Loss (Raw)": 0.800434947013855, "Finetune/Step": 4187, "Finetune/Step Time": 4.49025684595108} +{"Finetune/Learning Rate": 1.9149594750240087e-06, "Finetune/Loss": 0.8338272571563721, "Finetune/Loss (Raw)": 0.7817762494087219, "Finetune/Step": 4188, "Finetune/Step Time": 4.48116359859705} +{"Finetune/Learning Rate": 1.911294976714676e-06, "Finetune/Loss": 0.8338133096694946, "Finetune/Loss (Raw)": 0.8006981611251831, "Finetune/Step": 4189, "Finetune/Step Time": 4.458547813817859} +{"Finetune/Learning Rate": 1.9076336174738476e-06, "Finetune/Loss": 0.8312706351280212, "Finetune/Loss (Raw)": 0.6823504567146301, "Finetune/Step": 4190, "Finetune/Step Time": 4.458835834637284} +{"Finetune/Learning Rate": 1.9039753987224164e-06, "Finetune/Loss": 0.8356571197509766, "Finetune/Loss (Raw)": 0.9139801263809204, "Finetune/Step": 4191, "Finetune/Step Time": 4.424378719180822} +{"Finetune/Learning Rate": 1.9003203218800748e-06, "Finetune/Loss": 0.8406002521514893, "Finetune/Loss (Raw)": 1.0138485431671143, "Finetune/Step": 4192, "Finetune/Step Time": 4.3886019587516785} +{"Finetune/Learning Rate": 1.8966683883652826e-06, "Finetune/Loss": 0.8369029760360718, "Finetune/Loss (Raw)": 0.30634260177612305, "Finetune/Step": 4193, "Finetune/Step Time": 4.417871564626694} +{"Finetune/Learning Rate": 1.8930195995952827e-06, "Finetune/Loss": 0.83709716796875, "Finetune/Loss (Raw)": 0.8937175273895264, "Finetune/Step": 4194, "Finetune/Step Time": 4.417688312008977} +{"Finetune/Learning Rate": 1.8893739569861058e-06, "Finetune/Loss": 0.8365141153335571, "Finetune/Loss (Raw)": 0.9048005938529968, "Finetune/Step": 4195, "Finetune/Step Time": 4.41828029230237} +{"Finetune/Learning Rate": 1.8857314619525469e-06, "Finetune/Loss": 0.8362652063369751, "Finetune/Loss (Raw)": 0.8867664337158203, "Finetune/Step": 4196, "Finetune/Step Time": 4.415321983397007} +{"Finetune/Learning Rate": 1.8820921159081896e-06, "Finetune/Loss": 0.833549976348877, "Finetune/Loss (Raw)": 0.576750636100769, "Finetune/Step": 4197, "Finetune/Step Time": 4.423970587551594} +{"Finetune/Learning Rate": 1.8784559202653907e-06, "Finetune/Loss": 0.8346242308616638, "Finetune/Loss (Raw)": 0.9533151984214783, "Finetune/Step": 4198, "Finetune/Step Time": 4.422209432348609} +{"Finetune/Learning Rate": 1.8748228764352915e-06, "Finetune/Loss": 0.835555911064148, "Finetune/Loss (Raw)": 0.9827276468276978, "Finetune/Step": 4199, "Finetune/Step Time": 4.391780598089099} +{"Finetune/Learning Rate": 1.8711929858278066e-06, "Finetune/Loss": 0.8365907669067383, "Finetune/Loss (Raw)": 0.8989783525466919, "Finetune/Step": 4200, "Finetune/Step Time": 4.3881944213062525} +{"Finetune/Learning Rate": 1.8675662498516167e-06, "Finetune/Loss": 0.8360999226570129, "Finetune/Loss (Raw)": 0.8388177752494812, "Finetune/Step": 4201, "Finetune/Step Time": 4.39875359646976} +{"Finetune/Learning Rate": 1.863942669914196e-06, "Finetune/Loss": 0.8348140716552734, "Finetune/Loss (Raw)": 0.8185139894485474, "Finetune/Step": 4202, "Finetune/Step Time": 4.401769468560815} +{"Finetune/Learning Rate": 1.8603222474217819e-06, "Finetune/Loss": 0.8344756364822388, "Finetune/Loss (Raw)": 0.7783259749412537, "Finetune/Step": 4203, "Finetune/Step Time": 4.403167741373181} +{"Finetune/Learning Rate": 1.8567049837793894e-06, "Finetune/Loss": 0.8354876041412354, "Finetune/Loss (Raw)": 0.8007832765579224, "Finetune/Step": 4204, "Finetune/Step Time": 4.383216772228479} +{"Finetune/Learning Rate": 1.8530908803908088e-06, "Finetune/Loss": 0.8358262777328491, "Finetune/Loss (Raw)": 0.894808828830719, "Finetune/Step": 4205, "Finetune/Step Time": 4.364990180358291} +{"Finetune/Learning Rate": 1.8494799386586015e-06, "Finetune/Loss": 0.8359708189964294, "Finetune/Loss (Raw)": 0.8315396308898926, "Finetune/Step": 4206, "Finetune/Step Time": 4.363818019628525} +{"Finetune/Learning Rate": 1.8458721599841056e-06, "Finetune/Loss": 0.8349366784095764, "Finetune/Loss (Raw)": 0.9111650586128235, "Finetune/Step": 4207, "Finetune/Step Time": 4.357457373291254} +{"Finetune/Learning Rate": 1.8422675457674254e-06, "Finetune/Loss": 0.8353500962257385, "Finetune/Loss (Raw)": 0.8279829621315002, "Finetune/Step": 4208, "Finetune/Step Time": 4.346703914925456} +{"Finetune/Learning Rate": 1.8386660974074466e-06, "Finetune/Loss": 0.836918830871582, "Finetune/Loss (Raw)": 1.0595282316207886, "Finetune/Step": 4209, "Finetune/Step Time": 4.324626110494137} +{"Finetune/Learning Rate": 1.8350678163018221e-06, "Finetune/Loss": 0.8371985554695129, "Finetune/Loss (Raw)": 0.840008020401001, "Finetune/Step": 4210, "Finetune/Step Time": 4.320236958563328} +{"Finetune/Learning Rate": 1.831472703846966e-06, "Finetune/Loss": 0.8368497490882874, "Finetune/Loss (Raw)": 0.9337483048439026, "Finetune/Step": 4211, "Finetune/Step Time": 4.319493466988206} +{"Finetune/Learning Rate": 1.827880761438079e-06, "Finetune/Loss": 0.8392489552497864, "Finetune/Loss (Raw)": 0.7754319310188293, "Finetune/Step": 4212, "Finetune/Step Time": 4.280159175395966} +{"Finetune/Learning Rate": 1.8242919904691214e-06, "Finetune/Loss": 0.8382355570793152, "Finetune/Loss (Raw)": 0.7077838182449341, "Finetune/Step": 4213, "Finetune/Step Time": 4.278262523934245} +{"Finetune/Learning Rate": 1.820706392332824e-06, "Finetune/Loss": 0.8383030891418457, "Finetune/Loss (Raw)": 0.8859091401100159, "Finetune/Step": 4214, "Finetune/Step Time": 4.268734283745289} +{"Finetune/Learning Rate": 1.8171239684206898e-06, "Finetune/Loss": 0.8372592926025391, "Finetune/Loss (Raw)": 0.8599164485931396, "Finetune/Step": 4215, "Finetune/Step Time": 4.294903948903084} +{"Finetune/Learning Rate": 1.8135447201229861e-06, "Finetune/Loss": 0.8366228342056274, "Finetune/Loss (Raw)": 0.8629274368286133, "Finetune/Step": 4216, "Finetune/Step Time": 4.284504884853959} +{"Finetune/Learning Rate": 1.8099686488287483e-06, "Finetune/Loss": 0.8373593091964722, "Finetune/Loss (Raw)": 0.9410529136657715, "Finetune/Step": 4217, "Finetune/Step Time": 4.292129458859563} +{"Finetune/Learning Rate": 1.8063957559257794e-06, "Finetune/Loss": 0.8374335765838623, "Finetune/Loss (Raw)": 0.8439613580703735, "Finetune/Step": 4218, "Finetune/Step Time": 4.296894354745746} +{"Finetune/Learning Rate": 1.8028260428006538e-06, "Finetune/Loss": 0.8367536067962646, "Finetune/Loss (Raw)": 0.7454535961151123, "Finetune/Step": 4219, "Finetune/Step Time": 4.280023083090782} +{"Finetune/Learning Rate": 1.7992595108387068e-06, "Finetune/Loss": 0.83705073595047, "Finetune/Loss (Raw)": 0.873056173324585, "Finetune/Step": 4220, "Finetune/Step Time": 4.269370285794139} +{"Finetune/Learning Rate": 1.795696161424032e-06, "Finetune/Loss": 0.8371290564537048, "Finetune/Loss (Raw)": 0.992554247379303, "Finetune/Step": 4221, "Finetune/Step Time": 4.265429778024554} +{"Finetune/Learning Rate": 1.7921359959395046e-06, "Finetune/Loss": 0.8363894820213318, "Finetune/Loss (Raw)": 0.9597682952880859, "Finetune/Step": 4222, "Finetune/Step Time": 4.26951564475894} +{"Finetune/Learning Rate": 1.7885790157667526e-06, "Finetune/Loss": 0.8378781080245972, "Finetune/Loss (Raw)": 0.9650698900222778, "Finetune/Step": 4223, "Finetune/Step Time": 4.261269548907876} +{"Finetune/Learning Rate": 1.7850252222861697e-06, "Finetune/Loss": 0.8361696004867554, "Finetune/Loss (Raw)": 0.8027083277702332, "Finetune/Step": 4224, "Finetune/Step Time": 4.266464430838823} +{"Finetune/Learning Rate": 1.7814746168769138e-06, "Finetune/Loss": 0.8349189758300781, "Finetune/Loss (Raw)": 0.7933343052864075, "Finetune/Step": 4225, "Finetune/Step Time": 4.258879626169801} +{"Finetune/Learning Rate": 1.7779272009169068e-06, "Finetune/Loss": 0.8372454643249512, "Finetune/Loss (Raw)": 0.9844818115234375, "Finetune/Step": 4226, "Finetune/Step Time": 4.251861764118075} +{"Finetune/Learning Rate": 1.77438297578283e-06, "Finetune/Loss": 0.8383541107177734, "Finetune/Loss (Raw)": 1.057405948638916, "Finetune/Step": 4227, "Finetune/Step Time": 4.251298625022173} +{"Finetune/Learning Rate": 1.7708419428501256e-06, "Finetune/Loss": 0.8375627994537354, "Finetune/Loss (Raw)": 0.8535075187683105, "Finetune/Step": 4228, "Finetune/Step Time": 4.253284677863121} +{"Finetune/Learning Rate": 1.7673041034930083e-06, "Finetune/Loss": 0.8376826047897339, "Finetune/Loss (Raw)": 0.9049544334411621, "Finetune/Step": 4229, "Finetune/Step Time": 4.236069777980447} +{"Finetune/Learning Rate": 1.7637694590844357e-06, "Finetune/Loss": 0.8331674933433533, "Finetune/Loss (Raw)": 0.2698402404785156, "Finetune/Step": 4230, "Finetune/Step Time": 4.270098987966776} +{"Finetune/Learning Rate": 1.7602380109961348e-06, "Finetune/Loss": 0.8313637971878052, "Finetune/Loss (Raw)": 0.6560068130493164, "Finetune/Step": 4231, "Finetune/Step Time": 4.315656943246722} +{"Finetune/Learning Rate": 1.7567097605985972e-06, "Finetune/Loss": 0.8315615057945251, "Finetune/Loss (Raw)": 0.8901463747024536, "Finetune/Step": 4232, "Finetune/Step Time": 4.323998127132654} +{"Finetune/Learning Rate": 1.7531847092610644e-06, "Finetune/Loss": 0.8315974473953247, "Finetune/Loss (Raw)": 0.7772817611694336, "Finetune/Step": 4233, "Finetune/Step Time": 4.3299928195774555} +{"Finetune/Learning Rate": 1.7496628583515395e-06, "Finetune/Loss": 0.8336992263793945, "Finetune/Loss (Raw)": 1.097292184829712, "Finetune/Step": 4234, "Finetune/Step Time": 4.332828132435679} +{"Finetune/Learning Rate": 1.7461442092367864e-06, "Finetune/Loss": 0.8341145515441895, "Finetune/Loss (Raw)": 0.8900173902511597, "Finetune/Step": 4235, "Finetune/Step Time": 4.332308666780591} +{"Finetune/Learning Rate": 1.7426287632823213e-06, "Finetune/Loss": 0.8324750065803528, "Finetune/Loss (Raw)": 0.7229311466217041, "Finetune/Step": 4236, "Finetune/Step Time": 4.341214576736093} +{"Finetune/Learning Rate": 1.7391165218524232e-06, "Finetune/Loss": 0.833875298500061, "Finetune/Loss (Raw)": 0.9970065355300903, "Finetune/Step": 4237, "Finetune/Step Time": 4.333974378183484} +{"Finetune/Learning Rate": 1.735607486310119e-06, "Finetune/Loss": 0.8342811465263367, "Finetune/Loss (Raw)": 0.9577597379684448, "Finetune/Step": 4238, "Finetune/Step Time": 4.321857925504446} +{"Finetune/Learning Rate": 1.7321016580172068e-06, "Finetune/Loss": 0.83362877368927, "Finetune/Loss (Raw)": 0.9044884443283081, "Finetune/Step": 4239, "Finetune/Step Time": 4.3116976004093885} +{"Finetune/Learning Rate": 1.728599038334221e-06, "Finetune/Loss": 0.8342605829238892, "Finetune/Loss (Raw)": 0.8295894265174866, "Finetune/Step": 4240, "Finetune/Step Time": 4.299924697726965} +{"Finetune/Learning Rate": 1.7250996286204602e-06, "Finetune/Loss": 0.8352803587913513, "Finetune/Loss (Raw)": 0.8505933880805969, "Finetune/Step": 4241, "Finetune/Step Time": 4.301451899111271} +{"Finetune/Learning Rate": 1.7216034302339824e-06, "Finetune/Loss": 0.8357597589492798, "Finetune/Loss (Raw)": 0.9338117241859436, "Finetune/Step": 4242, "Finetune/Step Time": 4.300403643399477} +{"Finetune/Learning Rate": 1.718110444531591e-06, "Finetune/Loss": 0.8358192443847656, "Finetune/Loss (Raw)": 0.8809066414833069, "Finetune/Step": 4243, "Finetune/Step Time": 4.32065493799746} +{"Finetune/Learning Rate": 1.7146206728688465e-06, "Finetune/Loss": 0.8377597332000732, "Finetune/Loss (Raw)": 0.8794623017311096, "Finetune/Step": 4244, "Finetune/Step Time": 4.3043263386935} +{"Finetune/Learning Rate": 1.7111341166000594e-06, "Finetune/Loss": 0.838683009147644, "Finetune/Loss (Raw)": 0.8859855532646179, "Finetune/Step": 4245, "Finetune/Step Time": 4.307443145662546} +{"Finetune/Learning Rate": 1.7076507770782957e-06, "Finetune/Loss": 0.8372759819030762, "Finetune/Loss (Raw)": 0.8623141050338745, "Finetune/Step": 4246, "Finetune/Step Time": 4.34413749538362} +{"Finetune/Learning Rate": 1.7041706556553717e-06, "Finetune/Loss": 0.8363605737686157, "Finetune/Loss (Raw)": 0.8625962734222412, "Finetune/Step": 4247, "Finetune/Step Time": 4.340725500136614} +{"Finetune/Learning Rate": 1.7006937536818503e-06, "Finetune/Loss": 0.8317188620567322, "Finetune/Loss (Raw)": 0.2920903265476227, "Finetune/Step": 4248, "Finetune/Step Time": 4.372585253790021} +{"Finetune/Learning Rate": 1.6972200725070587e-06, "Finetune/Loss": 0.8320680260658264, "Finetune/Loss (Raw)": 0.9231289625167847, "Finetune/Step": 4249, "Finetune/Step Time": 4.372289139777422} +{"Finetune/Learning Rate": 1.6937496134790554e-06, "Finetune/Loss": 0.8309253454208374, "Finetune/Loss (Raw)": 0.7215316891670227, "Finetune/Step": 4250, "Finetune/Step Time": 4.368530001491308} +{"Finetune/Learning Rate": 1.6902823779446586e-06, "Finetune/Loss": 0.8288003206253052, "Finetune/Loss (Raw)": 0.7409723401069641, "Finetune/Step": 4251, "Finetune/Step Time": 4.356543188914657} +{"Finetune/Learning Rate": 1.686818367249441e-06, "Finetune/Loss": 0.8319514393806458, "Finetune/Loss (Raw)": 1.050567388534546, "Finetune/Step": 4252, "Finetune/Step Time": 4.352592626586556} +{"Finetune/Learning Rate": 1.6833575827377135e-06, "Finetune/Loss": 0.8299574255943298, "Finetune/Loss (Raw)": 0.647565484046936, "Finetune/Step": 4253, "Finetune/Step Time": 4.372526109218597} +{"Finetune/Learning Rate": 1.6799000257525388e-06, "Finetune/Loss": 0.828246533870697, "Finetune/Loss (Raw)": 0.6600400805473328, "Finetune/Step": 4254, "Finetune/Step Time": 4.388069437816739} +{"Finetune/Learning Rate": 1.6764456976357279e-06, "Finetune/Loss": 0.8272691965103149, "Finetune/Loss (Raw)": 0.7733491659164429, "Finetune/Step": 4255, "Finetune/Step Time": 4.394106460735202} +{"Finetune/Learning Rate": 1.6729945997278386e-06, "Finetune/Loss": 0.827964723110199, "Finetune/Loss (Raw)": 0.9091888070106506, "Finetune/Step": 4256, "Finetune/Step Time": 4.410014942288399} +{"Finetune/Learning Rate": 1.6695467333681748e-06, "Finetune/Loss": 0.829096257686615, "Finetune/Loss (Raw)": 0.9637550711631775, "Finetune/Step": 4257, "Finetune/Step Time": 4.414349220693111} +{"Finetune/Learning Rate": 1.6661020998947818e-06, "Finetune/Loss": 0.8279034495353699, "Finetune/Loss (Raw)": 0.683172345161438, "Finetune/Step": 4258, "Finetune/Step Time": 4.420773083344102} +{"Finetune/Learning Rate": 1.6626607006444628e-06, "Finetune/Loss": 0.8288708925247192, "Finetune/Loss (Raw)": 1.0060855150222778, "Finetune/Step": 4259, "Finetune/Step Time": 4.424584988504648} +{"Finetune/Learning Rate": 1.6592225369527515e-06, "Finetune/Loss": 0.8284658193588257, "Finetune/Loss (Raw)": 0.8389108180999756, "Finetune/Step": 4260, "Finetune/Step Time": 4.421718515455723} +{"Finetune/Learning Rate": 1.6557876101539305e-06, "Finetune/Loss": 0.8291492462158203, "Finetune/Loss (Raw)": 0.9300317764282227, "Finetune/Step": 4261, "Finetune/Step Time": 4.4217237532138824} +{"Finetune/Learning Rate": 1.6523559215810336e-06, "Finetune/Loss": 0.8289975523948669, "Finetune/Loss (Raw)": 0.8406639695167542, "Finetune/Step": 4262, "Finetune/Step Time": 4.41207136772573} +{"Finetune/Learning Rate": 1.64892747256583e-06, "Finetune/Loss": 0.8309059143066406, "Finetune/Loss (Raw)": 1.031764030456543, "Finetune/Step": 4263, "Finetune/Step Time": 4.409654112532735} +{"Finetune/Learning Rate": 1.645502264438833e-06, "Finetune/Loss": 0.8304104804992676, "Finetune/Loss (Raw)": 0.7722018957138062, "Finetune/Step": 4264, "Finetune/Step Time": 4.4208679515868425} +{"Finetune/Learning Rate": 1.6420802985292995e-06, "Finetune/Loss": 0.8312717080116272, "Finetune/Loss (Raw)": 0.9020177125930786, "Finetune/Step": 4265, "Finetune/Step Time": 4.428359266370535} +{"Finetune/Learning Rate": 1.638661576165227e-06, "Finetune/Loss": 0.8312625885009766, "Finetune/Loss (Raw)": 0.9484630227088928, "Finetune/Step": 4266, "Finetune/Step Time": 4.427762750536203} +{"Finetune/Learning Rate": 1.6352460986733564e-06, "Finetune/Loss": 0.8310911655426025, "Finetune/Loss (Raw)": 0.7654621005058289, "Finetune/Step": 4267, "Finetune/Step Time": 4.4417790081351995} +{"Finetune/Learning Rate": 1.6318338673791655e-06, "Finetune/Loss": 0.8310215473175049, "Finetune/Loss (Raw)": 0.8018434047698975, "Finetune/Step": 4268, "Finetune/Step Time": 4.43250747025013} +{"Finetune/Learning Rate": 1.6284248836068772e-06, "Finetune/Loss": 0.8336091637611389, "Finetune/Loss (Raw)": 0.7167945504188538, "Finetune/Step": 4269, "Finetune/Step Time": 4.428562343120575} +{"Finetune/Learning Rate": 1.6250191486794497e-06, "Finetune/Loss": 0.834098219871521, "Finetune/Loss (Raw)": 0.836342453956604, "Finetune/Step": 4270, "Finetune/Step Time": 4.428918274119496} +{"Finetune/Learning Rate": 1.6216166639185804e-06, "Finetune/Loss": 0.8305546045303345, "Finetune/Loss (Raw)": 0.3275671601295471, "Finetune/Step": 4271, "Finetune/Step Time": 4.46927479468286} +{"Finetune/Learning Rate": 1.618217430644713e-06, "Finetune/Loss": 0.8354017734527588, "Finetune/Loss (Raw)": 0.9108443856239319, "Finetune/Step": 4272, "Finetune/Step Time": 4.432590862736106} +{"Finetune/Learning Rate": 1.6148214501770221e-06, "Finetune/Loss": 0.8354451060295105, "Finetune/Loss (Raw)": 0.7427829504013062, "Finetune/Step": 4273, "Finetune/Step Time": 4.4205312971025705} +{"Finetune/Learning Rate": 1.6114287238334148e-06, "Finetune/Loss": 0.8350311517715454, "Finetune/Loss (Raw)": 0.8543994426727295, "Finetune/Step": 4274, "Finetune/Step Time": 4.426219092682004} +{"Finetune/Learning Rate": 1.6080392529305489e-06, "Finetune/Loss": 0.8348393440246582, "Finetune/Loss (Raw)": 0.8223706483840942, "Finetune/Step": 4275, "Finetune/Step Time": 4.437112454324961} +{"Finetune/Learning Rate": 1.60465303878381e-06, "Finetune/Loss": 0.8349663019180298, "Finetune/Loss (Raw)": 0.9173784852027893, "Finetune/Step": 4276, "Finetune/Step Time": 4.43791182897985} +{"Finetune/Learning Rate": 1.6012700827073213e-06, "Finetune/Loss": 0.8349604606628418, "Finetune/Loss (Raw)": 0.8371446132659912, "Finetune/Step": 4277, "Finetune/Step Time": 4.460140708833933} +{"Finetune/Learning Rate": 1.5978903860139416e-06, "Finetune/Loss": 0.8362058997154236, "Finetune/Loss (Raw)": 0.92465740442276, "Finetune/Step": 4278, "Finetune/Step Time": 4.429008953273296} +{"Finetune/Learning Rate": 1.5945139500152662e-06, "Finetune/Loss": 0.8357266783714294, "Finetune/Loss (Raw)": 0.8012174963951111, "Finetune/Step": 4279, "Finetune/Step Time": 4.426449337974191} +{"Finetune/Learning Rate": 1.5911407760216235e-06, "Finetune/Loss": 0.8363721370697021, "Finetune/Loss (Raw)": 0.9404281973838806, "Finetune/Step": 4280, "Finetune/Step Time": 4.4201818853616714} +{"Finetune/Learning Rate": 1.5877708653420742e-06, "Finetune/Loss": 0.8376396894454956, "Finetune/Loss (Raw)": 0.9924969673156738, "Finetune/Step": 4281, "Finetune/Step Time": 4.4205384105443954} +{"Finetune/Learning Rate": 1.58440421928442e-06, "Finetune/Loss": 0.8381942510604858, "Finetune/Loss (Raw)": 0.9493503570556641, "Finetune/Step": 4282, "Finetune/Step Time": 4.4191401563584805} +{"Finetune/Learning Rate": 1.581040839155189e-06, "Finetune/Loss": 0.8386063575744629, "Finetune/Loss (Raw)": 0.8730237483978271, "Finetune/Step": 4283, "Finetune/Step Time": 4.424249041825533} +{"Finetune/Learning Rate": 1.5776807262596371e-06, "Finetune/Loss": 0.8389760255813599, "Finetune/Loss (Raw)": 0.9085565209388733, "Finetune/Step": 4284, "Finetune/Step Time": 4.42812679708004} +{"Finetune/Learning Rate": 1.574323881901767e-06, "Finetune/Loss": 0.8366290926933289, "Finetune/Loss (Raw)": 0.6667050123214722, "Finetune/Step": 4285, "Finetune/Step Time": 4.462810598313808} +{"Finetune/Learning Rate": 1.5709703073843008e-06, "Finetune/Loss": 0.8371872901916504, "Finetune/Loss (Raw)": 0.8960464596748352, "Finetune/Step": 4286, "Finetune/Step Time": 4.4541887156665325} +{"Finetune/Learning Rate": 1.5676200040086941e-06, "Finetune/Loss": 0.8369839191436768, "Finetune/Loss (Raw)": 0.9061630964279175, "Finetune/Step": 4287, "Finetune/Step Time": 4.45743559114635} +{"Finetune/Learning Rate": 1.564272973075136e-06, "Finetune/Loss": 0.8371149301528931, "Finetune/Loss (Raw)": 0.9955158233642578, "Finetune/Step": 4288, "Finetune/Step Time": 4.47176337428391} +{"Finetune/Learning Rate": 1.560929215882544e-06, "Finetune/Loss": 0.8363896608352661, "Finetune/Loss (Raw)": 0.8116587996482849, "Finetune/Step": 4289, "Finetune/Step Time": 4.479021077975631} +{"Finetune/Learning Rate": 1.5575887337285644e-06, "Finetune/Loss": 0.8365655541419983, "Finetune/Loss (Raw)": 0.9710471630096436, "Finetune/Step": 4290, "Finetune/Step Time": 4.4803976491093636} +{"Finetune/Learning Rate": 1.5542515279095704e-06, "Finetune/Loss": 0.8382502794265747, "Finetune/Loss (Raw)": 0.926163911819458, "Finetune/Step": 4291, "Finetune/Step Time": 4.464974757283926} +{"Finetune/Learning Rate": 1.5509175997206705e-06, "Finetune/Loss": 0.840802788734436, "Finetune/Loss (Raw)": 0.9227730631828308, "Finetune/Step": 4292, "Finetune/Step Time": 4.460741013288498} +{"Finetune/Learning Rate": 1.5475869504556996e-06, "Finetune/Loss": 0.8400112390518188, "Finetune/Loss (Raw)": 0.9062379598617554, "Finetune/Step": 4293, "Finetune/Step Time": 4.468849942088127} +{"Finetune/Learning Rate": 1.5442595814072081e-06, "Finetune/Loss": 0.8394477367401123, "Finetune/Loss (Raw)": 0.870392382144928, "Finetune/Step": 4294, "Finetune/Step Time": 4.490523222833872} +{"Finetune/Learning Rate": 1.540935493866491e-06, "Finetune/Loss": 0.8356258869171143, "Finetune/Loss (Raw)": 0.3017195761203766, "Finetune/Step": 4295, "Finetune/Step Time": 4.526253793388605} +{"Finetune/Learning Rate": 1.53761468912356e-06, "Finetune/Loss": 0.8378105163574219, "Finetune/Loss (Raw)": 0.9811112880706787, "Finetune/Step": 4296, "Finetune/Step Time": 4.523577658459544} +{"Finetune/Learning Rate": 1.5342971684671537e-06, "Finetune/Loss": 0.8418747186660767, "Finetune/Loss (Raw)": 0.8946709036827087, "Finetune/Step": 4297, "Finetune/Step Time": 4.487225515767932} +{"Finetune/Learning Rate": 1.530982933184737e-06, "Finetune/Loss": 0.8413779735565186, "Finetune/Loss (Raw)": 0.70721834897995, "Finetune/Step": 4298, "Finetune/Step Time": 4.481562554836273} +{"Finetune/Learning Rate": 1.527671984562501e-06, "Finetune/Loss": 0.841679573059082, "Finetune/Loss (Raw)": 0.939407467842102, "Finetune/Step": 4299, "Finetune/Step Time": 4.488587100058794} +{"Finetune/Learning Rate": 1.524364323885359e-06, "Finetune/Loss": 0.8419898748397827, "Finetune/Loss (Raw)": 0.9420924186706543, "Finetune/Step": 4300, "Finetune/Step Time": 4.473429538309574} +{"Finetune/Learning Rate": 1.5210599524369484e-06, "Finetune/Loss": 0.8384010791778564, "Finetune/Loss (Raw)": 0.36518922448158264, "Finetune/Step": 4301, "Finetune/Step Time": 4.507500750944018} +{"Finetune/Learning Rate": 1.517758871499635e-06, "Finetune/Loss": 0.8431549072265625, "Finetune/Loss (Raw)": 0.9674004316329956, "Finetune/Step": 4302, "Finetune/Step Time": 4.473120965063572} +{"Finetune/Learning Rate": 1.5144610823545035e-06, "Finetune/Loss": 0.8448036313056946, "Finetune/Loss (Raw)": 0.9880971312522888, "Finetune/Step": 4303, "Finetune/Step Time": 4.473026320338249} +{"Finetune/Learning Rate": 1.511166586281355e-06, "Finetune/Loss": 0.8412293195724487, "Finetune/Loss (Raw)": 0.34238430857658386, "Finetune/Step": 4304, "Finetune/Step Time": 4.49909321218729} +{"Finetune/Learning Rate": 1.507875384558727e-06, "Finetune/Loss": 0.8397786021232605, "Finetune/Loss (Raw)": 0.7718784809112549, "Finetune/Step": 4305, "Finetune/Step Time": 4.490697305649519} +{"Finetune/Learning Rate": 1.504587478463867e-06, "Finetune/Loss": 0.8391712307929993, "Finetune/Loss (Raw)": 0.8311214447021484, "Finetune/Step": 4306, "Finetune/Step Time": 4.49613912217319} +{"Finetune/Learning Rate": 1.5013028692727483e-06, "Finetune/Loss": 0.8376098275184631, "Finetune/Loss (Raw)": 0.67717045545578, "Finetune/Step": 4307, "Finetune/Step Time": 4.4850417375564575} +{"Finetune/Learning Rate": 1.4980215582600621e-06, "Finetune/Loss": 0.8369875550270081, "Finetune/Loss (Raw)": 0.8762656450271606, "Finetune/Step": 4308, "Finetune/Step Time": 4.485848993062973} +{"Finetune/Learning Rate": 1.4947435466992222e-06, "Finetune/Loss": 0.837313711643219, "Finetune/Loss (Raw)": 0.8534661531448364, "Finetune/Step": 4309, "Finetune/Step Time": 4.490326622501016} +{"Finetune/Learning Rate": 1.4914688358623608e-06, "Finetune/Loss": 0.8347469568252563, "Finetune/Loss (Raw)": 0.3517412543296814, "Finetune/Step": 4310, "Finetune/Step Time": 4.518235970288515} +{"Finetune/Learning Rate": 1.488197427020327e-06, "Finetune/Loss": 0.834282636642456, "Finetune/Loss (Raw)": 0.8746861219406128, "Finetune/Step": 4311, "Finetune/Step Time": 4.525002481415868} +{"Finetune/Learning Rate": 1.4849293214426975e-06, "Finetune/Loss": 0.8348764777183533, "Finetune/Loss (Raw)": 0.9766510128974915, "Finetune/Step": 4312, "Finetune/Step Time": 4.5331610180437565} +{"Finetune/Learning Rate": 1.481664520397753e-06, "Finetune/Loss": 0.8349905014038086, "Finetune/Loss (Raw)": 0.9759767651557922, "Finetune/Step": 4313, "Finetune/Step Time": 4.537912365049124} +{"Finetune/Learning Rate": 1.4784030251524994e-06, "Finetune/Loss": 0.8348818421363831, "Finetune/Loss (Raw)": 0.9267446994781494, "Finetune/Step": 4314, "Finetune/Step Time": 4.534343406558037} +{"Finetune/Learning Rate": 1.4751448369726652e-06, "Finetune/Loss": 0.8352397680282593, "Finetune/Loss (Raw)": 0.8462517857551575, "Finetune/Step": 4315, "Finetune/Step Time": 4.5256907641887665} +{"Finetune/Learning Rate": 1.4718899571226842e-06, "Finetune/Loss": 0.8354336619377136, "Finetune/Loss (Raw)": 0.8065920472145081, "Finetune/Step": 4316, "Finetune/Step Time": 4.529115483164787} +{"Finetune/Learning Rate": 1.468638386865716e-06, "Finetune/Loss": 0.8358337879180908, "Finetune/Loss (Raw)": 0.8519186973571777, "Finetune/Step": 4317, "Finetune/Step Time": 4.532526774331927} +{"Finetune/Learning Rate": 1.465390127463625e-06, "Finetune/Loss": 0.8366762399673462, "Finetune/Loss (Raw)": 0.7901769280433655, "Finetune/Step": 4318, "Finetune/Step Time": 4.536372546106577} +{"Finetune/Learning Rate": 1.4621451801770038e-06, "Finetune/Loss": 0.8352200984954834, "Finetune/Loss (Raw)": 0.727597177028656, "Finetune/Step": 4319, "Finetune/Step Time": 4.541974194347858} +{"Finetune/Learning Rate": 1.4589035462651502e-06, "Finetune/Loss": 0.8337363004684448, "Finetune/Loss (Raw)": 0.8239279985427856, "Finetune/Step": 4320, "Finetune/Step Time": 4.540039511397481} +{"Finetune/Learning Rate": 1.4556652269860772e-06, "Finetune/Loss": 0.836859941482544, "Finetune/Loss (Raw)": 0.7061680555343628, "Finetune/Step": 4321, "Finetune/Step Time": 4.514101402834058} +{"Finetune/Learning Rate": 1.4524302235965203e-06, "Finetune/Loss": 0.8377290964126587, "Finetune/Loss (Raw)": 1.0049664974212646, "Finetune/Step": 4322, "Finetune/Step Time": 4.517112361267209} +{"Finetune/Learning Rate": 1.4491985373519146e-06, "Finetune/Loss": 0.8361214399337769, "Finetune/Loss (Raw)": 0.6990240812301636, "Finetune/Step": 4323, "Finetune/Step Time": 4.514527043327689} +{"Finetune/Learning Rate": 1.4459701695064155e-06, "Finetune/Loss": 0.8355529308319092, "Finetune/Loss (Raw)": 0.8139892816543579, "Finetune/Step": 4324, "Finetune/Step Time": 4.513874506577849} +{"Finetune/Learning Rate": 1.4427451213128873e-06, "Finetune/Loss": 0.8362559676170349, "Finetune/Loss (Raw)": 0.6667399406433105, "Finetune/Step": 4325, "Finetune/Step Time": 4.509363567456603} +{"Finetune/Learning Rate": 1.4395233940229147e-06, "Finetune/Loss": 0.8346298336982727, "Finetune/Loss (Raw)": 0.7451765537261963, "Finetune/Step": 4326, "Finetune/Step Time": 4.517484966665506} +{"Finetune/Learning Rate": 1.4363049888867853e-06, "Finetune/Loss": 0.832505464553833, "Finetune/Loss (Raw)": 0.7108116745948792, "Finetune/Step": 4327, "Finetune/Step Time": 4.534427495673299} +{"Finetune/Learning Rate": 1.4330899071534942e-06, "Finetune/Loss": 0.8321802616119385, "Finetune/Loss (Raw)": 0.8573430776596069, "Finetune/Step": 4328, "Finetune/Step Time": 4.528329383581877} +{"Finetune/Learning Rate": 1.4298781500707558e-06, "Finetune/Loss": 0.8322965502738953, "Finetune/Loss (Raw)": 0.8537082076072693, "Finetune/Step": 4329, "Finetune/Step Time": 4.5176291931420565} +{"Finetune/Learning Rate": 1.426669718884991e-06, "Finetune/Loss": 0.8333983421325684, "Finetune/Loss (Raw)": 0.9595383405685425, "Finetune/Step": 4330, "Finetune/Step Time": 4.516938671469688} +{"Finetune/Learning Rate": 1.4234646148413277e-06, "Finetune/Loss": 0.834691047668457, "Finetune/Loss (Raw)": 0.9437916278839111, "Finetune/Step": 4331, "Finetune/Step Time": 4.516944479197264} +{"Finetune/Learning Rate": 1.420262839183605e-06, "Finetune/Loss": 0.8353865742683411, "Finetune/Loss (Raw)": 0.8898153901100159, "Finetune/Step": 4332, "Finetune/Step Time": 4.519468173384666} +{"Finetune/Learning Rate": 1.4170643931543693e-06, "Finetune/Loss": 0.8359860181808472, "Finetune/Loss (Raw)": 0.9715364575386047, "Finetune/Step": 4333, "Finetune/Step Time": 4.520897274836898} +{"Finetune/Learning Rate": 1.4138692779948749e-06, "Finetune/Loss": 0.8368237018585205, "Finetune/Loss (Raw)": 0.9387648105621338, "Finetune/Step": 4334, "Finetune/Step Time": 4.518727509304881} +{"Finetune/Learning Rate": 1.4106774949450808e-06, "Finetune/Loss": 0.8367910385131836, "Finetune/Loss (Raw)": 0.9069774150848389, "Finetune/Step": 4335, "Finetune/Step Time": 4.513795085251331} +{"Finetune/Learning Rate": 1.4074890452436607e-06, "Finetune/Loss": 0.8381430506706238, "Finetune/Loss (Raw)": 1.001046061515808, "Finetune/Step": 4336, "Finetune/Step Time": 4.529996642842889} +{"Finetune/Learning Rate": 1.4043039301279904e-06, "Finetune/Loss": 0.835024893283844, "Finetune/Loss (Raw)": 0.660402774810791, "Finetune/Step": 4337, "Finetune/Step Time": 4.552471101284027} +{"Finetune/Learning Rate": 1.4011221508341433e-06, "Finetune/Loss": 0.8352420926094055, "Finetune/Loss (Raw)": 0.8678116202354431, "Finetune/Step": 4338, "Finetune/Step Time": 4.552155531942844} +{"Finetune/Learning Rate": 1.3979437085969127e-06, "Finetune/Loss": 0.8341620564460754, "Finetune/Loss (Raw)": 0.7955074310302734, "Finetune/Step": 4339, "Finetune/Step Time": 4.552735103294253} +{"Finetune/Learning Rate": 1.3947686046497878e-06, "Finetune/Loss": 0.8347831964492798, "Finetune/Loss (Raw)": 0.8549361228942871, "Finetune/Step": 4340, "Finetune/Step Time": 4.56715102493763} +{"Finetune/Learning Rate": 1.3915968402249647e-06, "Finetune/Loss": 0.8367531299591064, "Finetune/Loss (Raw)": 0.9599245190620422, "Finetune/Step": 4341, "Finetune/Step Time": 4.558810232207179} +{"Finetune/Learning Rate": 1.3884284165533423e-06, "Finetune/Loss": 0.8363391757011414, "Finetune/Loss (Raw)": 0.8329318761825562, "Finetune/Step": 4342, "Finetune/Step Time": 4.604557391256094} +{"Finetune/Learning Rate": 1.3852633348645262e-06, "Finetune/Loss": 0.835871696472168, "Finetune/Loss (Raw)": 0.8000754714012146, "Finetune/Step": 4343, "Finetune/Step Time": 4.567548617720604} +{"Finetune/Learning Rate": 1.38210159638682e-06, "Finetune/Loss": 0.8361470699310303, "Finetune/Loss (Raw)": 0.8981815576553345, "Finetune/Step": 4344, "Finetune/Step Time": 4.577277336269617} +{"Finetune/Learning Rate": 1.3789432023472315e-06, "Finetune/Loss": 0.8354848623275757, "Finetune/Loss (Raw)": 0.8562904596328735, "Finetune/Step": 4345, "Finetune/Step Time": 4.565959919244051} +{"Finetune/Learning Rate": 1.3757881539714768e-06, "Finetune/Loss": 0.8357043266296387, "Finetune/Loss (Raw)": 0.8720541596412659, "Finetune/Step": 4346, "Finetune/Step Time": 4.569366667419672} +{"Finetune/Learning Rate": 1.3726364524839687e-06, "Finetune/Loss": 0.8373653888702393, "Finetune/Loss (Raw)": 0.958059549331665, "Finetune/Step": 4347, "Finetune/Step Time": 4.561700763180852} +{"Finetune/Learning Rate": 1.369488099107814e-06, "Finetune/Loss": 0.8376750946044922, "Finetune/Loss (Raw)": 0.9126976728439331, "Finetune/Step": 4348, "Finetune/Step Time": 4.567620882764459} +{"Finetune/Learning Rate": 1.3663430950648326e-06, "Finetune/Loss": 0.8375457525253296, "Finetune/Loss (Raw)": 0.9760053157806396, "Finetune/Step": 4349, "Finetune/Step Time": 4.569690104573965} +{"Finetune/Learning Rate": 1.363201441575539e-06, "Finetune/Loss": 0.8350991010665894, "Finetune/Loss (Raw)": 0.64659583568573, "Finetune/Step": 4350, "Finetune/Step Time": 4.580916903913021} +{"Finetune/Learning Rate": 1.360063139859147e-06, "Finetune/Loss": 0.8345859050750732, "Finetune/Loss (Raw)": 0.8993739485740662, "Finetune/Step": 4351, "Finetune/Step Time": 4.580112868919969} +{"Finetune/Learning Rate": 1.3569281911335686e-06, "Finetune/Loss": 0.8350030183792114, "Finetune/Loss (Raw)": 0.8561046123504639, "Finetune/Step": 4352, "Finetune/Step Time": 4.5720813646912575} +{"Finetune/Learning Rate": 1.3537965966154177e-06, "Finetune/Loss": 0.8359116315841675, "Finetune/Loss (Raw)": 0.9096347689628601, "Finetune/Step": 4353, "Finetune/Step Time": 4.5704514142125845} +{"Finetune/Learning Rate": 1.3506683575200053e-06, "Finetune/Loss": 0.8340636491775513, "Finetune/Loss (Raw)": 0.7479487061500549, "Finetune/Step": 4354, "Finetune/Step Time": 4.574932252988219} +{"Finetune/Learning Rate": 1.347543475061337e-06, "Finetune/Loss": 0.8327392339706421, "Finetune/Loss (Raw)": 0.887876570224762, "Finetune/Step": 4355, "Finetune/Step Time": 4.571155807003379} +{"Finetune/Learning Rate": 1.344421950452126e-06, "Finetune/Loss": 0.8330205678939819, "Finetune/Loss (Raw)": 0.8895150423049927, "Finetune/Step": 4356, "Finetune/Step Time": 4.569419572129846} +{"Finetune/Learning Rate": 1.341303784903767e-06, "Finetune/Loss": 0.8331006169319153, "Finetune/Loss (Raw)": 0.9152000546455383, "Finetune/Step": 4357, "Finetune/Step Time": 4.569499308243394} +{"Finetune/Learning Rate": 1.3381889796263602e-06, "Finetune/Loss": 0.8380622863769531, "Finetune/Loss (Raw)": 0.9049389362335205, "Finetune/Step": 4358, "Finetune/Step Time": 4.540528807789087} +{"Finetune/Learning Rate": 1.335077535828706e-06, "Finetune/Loss": 0.8398129343986511, "Finetune/Loss (Raw)": 0.8800838589668274, "Finetune/Step": 4359, "Finetune/Step Time": 4.494440054520965} +{"Finetune/Learning Rate": 1.3319694547182916e-06, "Finetune/Loss": 0.8392201662063599, "Finetune/Loss (Raw)": 0.8142798542976379, "Finetune/Step": 4360, "Finetune/Step Time": 4.486665835604072} +{"Finetune/Learning Rate": 1.328864737501302e-06, "Finetune/Loss": 0.841453492641449, "Finetune/Loss (Raw)": 1.0631415843963623, "Finetune/Step": 4361, "Finetune/Step Time": 4.495142603293061} +{"Finetune/Learning Rate": 1.325763385382618e-06, "Finetune/Loss": 0.8392287492752075, "Finetune/Loss (Raw)": 0.8125256896018982, "Finetune/Step": 4362, "Finetune/Step Time": 4.497992506250739} +{"Finetune/Learning Rate": 1.3226653995658144e-06, "Finetune/Loss": 0.8387838006019592, "Finetune/Loss (Raw)": 0.8330641984939575, "Finetune/Step": 4363, "Finetune/Step Time": 4.49494256451726} +{"Finetune/Learning Rate": 1.319570781253159e-06, "Finetune/Loss": 0.8391754627227783, "Finetune/Loss (Raw)": 0.7730634212493896, "Finetune/Step": 4364, "Finetune/Step Time": 4.506878472864628} +{"Finetune/Learning Rate": 1.3164795316456091e-06, "Finetune/Loss": 0.8373692035675049, "Finetune/Loss (Raw)": 0.7658042907714844, "Finetune/Step": 4365, "Finetune/Step Time": 4.521684385836124} +{"Finetune/Learning Rate": 1.313391651942827e-06, "Finetune/Loss": 0.8369913101196289, "Finetune/Loss (Raw)": 0.9093865752220154, "Finetune/Step": 4366, "Finetune/Step Time": 4.521439880132675} +{"Finetune/Learning Rate": 1.3103071433431501e-06, "Finetune/Loss": 0.8361058235168457, "Finetune/Loss (Raw)": 0.7911504507064819, "Finetune/Step": 4367, "Finetune/Step Time": 4.527826752513647} +{"Finetune/Learning Rate": 1.307226007043617e-06, "Finetune/Loss": 0.8368110656738281, "Finetune/Loss (Raw)": 0.919867217540741, "Finetune/Step": 4368, "Finetune/Step Time": 4.524121010676026} +{"Finetune/Learning Rate": 1.30414824423996e-06, "Finetune/Loss": 0.8365674018859863, "Finetune/Loss (Raw)": 0.8193975687026978, "Finetune/Step": 4369, "Finetune/Step Time": 4.531165791675448} +{"Finetune/Learning Rate": 1.301073856126598e-06, "Finetune/Loss": 0.8367813229560852, "Finetune/Loss (Raw)": 0.9611950516700745, "Finetune/Step": 4370, "Finetune/Step Time": 4.533408340066671} +{"Finetune/Learning Rate": 1.2980028438966396e-06, "Finetune/Loss": 0.8375709652900696, "Finetune/Loss (Raw)": 0.9819778203964233, "Finetune/Step": 4371, "Finetune/Step Time": 4.511701060459018} +{"Finetune/Learning Rate": 1.2949352087418864e-06, "Finetune/Loss": 0.8326455354690552, "Finetune/Loss (Raw)": 0.2490111142396927, "Finetune/Step": 4372, "Finetune/Step Time": 4.5522911716252565} +{"Finetune/Learning Rate": 1.291870951852826e-06, "Finetune/Loss": 0.8316166400909424, "Finetune/Loss (Raw)": 0.7542814612388611, "Finetune/Step": 4373, "Finetune/Step Time": 4.548841584473848} +{"Finetune/Learning Rate": 1.288810074418637e-06, "Finetune/Loss": 0.83134526014328, "Finetune/Loss (Raw)": 0.8275847434997559, "Finetune/Step": 4374, "Finetune/Step Time": 4.519183307886124} +{"Finetune/Learning Rate": 1.2857525776271852e-06, "Finetune/Loss": 0.8308285474777222, "Finetune/Loss (Raw)": 0.7964553833007812, "Finetune/Step": 4375, "Finetune/Step Time": 4.526682591065764} +{"Finetune/Learning Rate": 1.2826984626650297e-06, "Finetune/Loss": 0.8357905149459839, "Finetune/Loss (Raw)": 0.9272230267524719, "Finetune/Step": 4376, "Finetune/Step Time": 4.496632322669029} +{"Finetune/Learning Rate": 1.2796477307174082e-06, "Finetune/Loss": 0.8354133367538452, "Finetune/Loss (Raw)": 0.874844491481781, "Finetune/Step": 4377, "Finetune/Step Time": 4.491446882486343} +{"Finetune/Learning Rate": 1.2766003829682504e-06, "Finetune/Loss": 0.8365719318389893, "Finetune/Loss (Raw)": 0.8698328733444214, "Finetune/Step": 4378, "Finetune/Step Time": 4.5015123672783375} +{"Finetune/Learning Rate": 1.273556420600175e-06, "Finetune/Loss": 0.8367332220077515, "Finetune/Loss (Raw)": 0.7616210579872131, "Finetune/Step": 4379, "Finetune/Step Time": 4.518722135573626} +{"Finetune/Learning Rate": 1.2705158447944843e-06, "Finetune/Loss": 0.8361818194389343, "Finetune/Loss (Raw)": 0.9799860119819641, "Finetune/Step": 4380, "Finetune/Step Time": 4.5215078722685575} +{"Finetune/Learning Rate": 1.2674786567311648e-06, "Finetune/Loss": 0.838152289390564, "Finetune/Loss (Raw)": 0.8997858762741089, "Finetune/Step": 4381, "Finetune/Step Time": 4.508797317743301} +{"Finetune/Learning Rate": 1.2644448575888913e-06, "Finetune/Loss": 0.8402571678161621, "Finetune/Loss (Raw)": 0.9294639825820923, "Finetune/Step": 4382, "Finetune/Step Time": 4.497592410072684} +{"Finetune/Learning Rate": 1.2614144485450208e-06, "Finetune/Loss": 0.8398980498313904, "Finetune/Loss (Raw)": 0.7273810505867004, "Finetune/Step": 4383, "Finetune/Step Time": 4.512084120884538} +{"Finetune/Learning Rate": 1.2583874307755961e-06, "Finetune/Loss": 0.8395759463310242, "Finetune/Loss (Raw)": 0.8679613471031189, "Finetune/Step": 4384, "Finetune/Step Time": 4.49854645319283} +{"Finetune/Learning Rate": 1.2553638054553418e-06, "Finetune/Loss": 0.8383457660675049, "Finetune/Loss (Raw)": 0.8062945604324341, "Finetune/Step": 4385, "Finetune/Step Time": 4.491296794265509} +{"Finetune/Learning Rate": 1.252343573757674e-06, "Finetune/Loss": 0.8395500183105469, "Finetune/Loss (Raw)": 0.8373183012008667, "Finetune/Step": 4386, "Finetune/Step Time": 4.508242981508374} +{"Finetune/Learning Rate": 1.2493267368546802e-06, "Finetune/Loss": 0.8398686051368713, "Finetune/Loss (Raw)": 1.0468603372573853, "Finetune/Step": 4387, "Finetune/Step Time": 4.495581034570932} +{"Finetune/Learning Rate": 1.2463132959171343e-06, "Finetune/Loss": 0.8403652906417847, "Finetune/Loss (Raw)": 0.9024816751480103, "Finetune/Step": 4388, "Finetune/Step Time": 4.5058665089309216} +{"Finetune/Learning Rate": 1.2433032521145006e-06, "Finetune/Loss": 0.8402539491653442, "Finetune/Loss (Raw)": 0.9157875776290894, "Finetune/Step": 4389, "Finetune/Step Time": 4.504485810175538} +{"Finetune/Learning Rate": 1.2402966066149135e-06, "Finetune/Loss": 0.840151309967041, "Finetune/Loss (Raw)": 0.8275215029716492, "Finetune/Step": 4390, "Finetune/Step Time": 4.503418138250709} +{"Finetune/Learning Rate": 1.2372933605851944e-06, "Finetune/Loss": 0.838711142539978, "Finetune/Loss (Raw)": 0.8474229574203491, "Finetune/Step": 4391, "Finetune/Step Time": 4.511678969487548} +{"Finetune/Learning Rate": 1.2342935151908453e-06, "Finetune/Loss": 0.8354445695877075, "Finetune/Loss (Raw)": 0.354085236787796, "Finetune/Step": 4392, "Finetune/Step Time": 4.543004959821701} +{"Finetune/Learning Rate": 1.231297071596046e-06, "Finetune/Loss": 0.8341724872589111, "Finetune/Loss (Raw)": 0.7391897439956665, "Finetune/Step": 4393, "Finetune/Step Time": 4.550824753940105} +{"Finetune/Learning Rate": 1.2283040309636596e-06, "Finetune/Loss": 0.833146870136261, "Finetune/Loss (Raw)": 0.8171800971031189, "Finetune/Step": 4394, "Finetune/Step Time": 4.560573538765311} +{"Finetune/Learning Rate": 1.2253143944552258e-06, "Finetune/Loss": 0.8337881565093994, "Finetune/Loss (Raw)": 0.8475468158721924, "Finetune/Step": 4395, "Finetune/Step Time": 4.550636686384678} +{"Finetune/Learning Rate": 1.2223281632309636e-06, "Finetune/Loss": 0.8350518941879272, "Finetune/Loss (Raw)": 0.9636037349700928, "Finetune/Step": 4396, "Finetune/Step Time": 4.559311358258128} +{"Finetune/Learning Rate": 1.2193453384497722e-06, "Finetune/Loss": 0.8327266573905945, "Finetune/Loss (Raw)": 0.419165700674057, "Finetune/Step": 4397, "Finetune/Step Time": 4.565057376399636} +{"Finetune/Learning Rate": 1.2163659212692246e-06, "Finetune/Loss": 0.8331242799758911, "Finetune/Loss (Raw)": 0.8872396945953369, "Finetune/Step": 4398, "Finetune/Step Time": 4.567523656412959} +{"Finetune/Learning Rate": 1.2133899128455796e-06, "Finetune/Loss": 0.8368306159973145, "Finetune/Loss (Raw)": 0.8019762635231018, "Finetune/Step": 4399, "Finetune/Step Time": 4.530591003596783} +{"Finetune/Learning Rate": 1.2104173143337672e-06, "Finetune/Loss": 0.8365623950958252, "Finetune/Loss (Raw)": 0.8765097260475159, "Finetune/Step": 4400, "Finetune/Step Time": 4.538067746907473} +{"Finetune/Learning Rate": 1.2074481268873894e-06, "Finetune/Loss": 0.8333694338798523, "Finetune/Loss (Raw)": 0.33408743143081665, "Finetune/Step": 4401, "Finetune/Step Time": 4.578289743512869} +{"Finetune/Learning Rate": 1.2044823516587345e-06, "Finetune/Loss": 0.8329132795333862, "Finetune/Loss (Raw)": 0.7960073351860046, "Finetune/Step": 4402, "Finetune/Step Time": 4.596719261258841} +{"Finetune/Learning Rate": 1.2015199897987616e-06, "Finetune/Loss": 0.8336977362632751, "Finetune/Loss (Raw)": 0.9227840304374695, "Finetune/Step": 4403, "Finetune/Step Time": 4.581015355885029} +{"Finetune/Learning Rate": 1.198561042457105e-06, "Finetune/Loss": 0.8344284892082214, "Finetune/Loss (Raw)": 1.0109084844589233, "Finetune/Step": 4404, "Finetune/Step Time": 4.581829549744725} +{"Finetune/Learning Rate": 1.195605510782073e-06, "Finetune/Loss": 0.834300696849823, "Finetune/Loss (Raw)": 0.8207906484603882, "Finetune/Step": 4405, "Finetune/Step Time": 4.572182033210993} +{"Finetune/Learning Rate": 1.192653395920652e-06, "Finetune/Loss": 0.8329342007637024, "Finetune/Loss (Raw)": 0.7497426867485046, "Finetune/Step": 4406, "Finetune/Step Time": 4.569475166499615} +{"Finetune/Learning Rate": 1.189704699018499e-06, "Finetune/Loss": 0.8327000737190247, "Finetune/Loss (Raw)": 0.7712559103965759, "Finetune/Step": 4407, "Finetune/Step Time": 4.567982221022248} +{"Finetune/Learning Rate": 1.186759421219943e-06, "Finetune/Loss": 0.8332592248916626, "Finetune/Loss (Raw)": 1.0119999647140503, "Finetune/Step": 4408, "Finetune/Step Time": 4.567156279459596} +{"Finetune/Learning Rate": 1.1838175636679938e-06, "Finetune/Loss": 0.8318711519241333, "Finetune/Loss (Raw)": 0.8148239254951477, "Finetune/Step": 4409, "Finetune/Step Time": 4.566050058230758} +{"Finetune/Learning Rate": 1.1808791275043297e-06, "Finetune/Loss": 0.8313215970993042, "Finetune/Loss (Raw)": 0.8790045976638794, "Finetune/Step": 4410, "Finetune/Step Time": 4.5627414640039206} +{"Finetune/Learning Rate": 1.1779441138692926e-06, "Finetune/Loss": 0.8311407566070557, "Finetune/Loss (Raw)": 0.8498750329017639, "Finetune/Step": 4411, "Finetune/Step Time": 4.555776154622436} +{"Finetune/Learning Rate": 1.1750125239019118e-06, "Finetune/Loss": 0.8303192853927612, "Finetune/Loss (Raw)": 0.8034089207649231, "Finetune/Step": 4412, "Finetune/Step Time": 4.552061673253775} +{"Finetune/Learning Rate": 1.1720843587398756e-06, "Finetune/Loss": 0.8335967659950256, "Finetune/Loss (Raw)": 1.0862237215042114, "Finetune/Step": 4413, "Finetune/Step Time": 4.515196476131678} +{"Finetune/Learning Rate": 1.1691596195195498e-06, "Finetune/Loss": 0.8328818082809448, "Finetune/Loss (Raw)": 0.804532527923584, "Finetune/Step": 4414, "Finetune/Step Time": 4.514541316777468} +{"Finetune/Learning Rate": 1.1662383073759686e-06, "Finetune/Loss": 0.8331109285354614, "Finetune/Loss (Raw)": 0.9354951977729797, "Finetune/Step": 4415, "Finetune/Step Time": 4.519492590799928} +{"Finetune/Learning Rate": 1.1633204234428352e-06, "Finetune/Loss": 0.8311716318130493, "Finetune/Loss (Raw)": 0.7472741007804871, "Finetune/Step": 4416, "Finetune/Step Time": 4.525867074728012} +{"Finetune/Learning Rate": 1.1604059688525238e-06, "Finetune/Loss": 0.8323338627815247, "Finetune/Loss (Raw)": 0.9604324698448181, "Finetune/Step": 4417, "Finetune/Step Time": 4.526482073590159} +{"Finetune/Learning Rate": 1.157494944736075e-06, "Finetune/Loss": 0.8314144611358643, "Finetune/Loss (Raw)": 0.8533632755279541, "Finetune/Step": 4418, "Finetune/Step Time": 4.526176953688264} +{"Finetune/Learning Rate": 1.1545873522232055e-06, "Finetune/Loss": 0.8313840627670288, "Finetune/Loss (Raw)": 0.922266960144043, "Finetune/Step": 4419, "Finetune/Step Time": 4.531539050862193} +{"Finetune/Learning Rate": 1.1516831924422945e-06, "Finetune/Loss": 0.8301199674606323, "Finetune/Loss (Raw)": 0.760969340801239, "Finetune/Step": 4420, "Finetune/Step Time": 4.543241145089269} +{"Finetune/Learning Rate": 1.1487824665203828e-06, "Finetune/Loss": 0.8294891119003296, "Finetune/Loss (Raw)": 0.8254887461662292, "Finetune/Step": 4421, "Finetune/Step Time": 4.54088356718421} +{"Finetune/Learning Rate": 1.1458851755831934e-06, "Finetune/Loss": 0.8296389579772949, "Finetune/Loss (Raw)": 0.889571487903595, "Finetune/Step": 4422, "Finetune/Step Time": 4.5237370524555445} +{"Finetune/Learning Rate": 1.1429913207551046e-06, "Finetune/Loss": 0.8334647417068481, "Finetune/Loss (Raw)": 0.791419267654419, "Finetune/Step": 4423, "Finetune/Step Time": 4.490106962621212} +{"Finetune/Learning Rate": 1.1401009031591658e-06, "Finetune/Loss": 0.8283891677856445, "Finetune/Loss (Raw)": 0.3314449191093445, "Finetune/Step": 4424, "Finetune/Step Time": 4.532268464565277} +{"Finetune/Learning Rate": 1.137213923917092e-06, "Finetune/Loss": 0.8276848793029785, "Finetune/Loss (Raw)": 0.8045212626457214, "Finetune/Step": 4425, "Finetune/Step Time": 4.549229549244046} +{"Finetune/Learning Rate": 1.1343303841492626e-06, "Finetune/Loss": 0.8292530179023743, "Finetune/Loss (Raw)": 0.9079447984695435, "Finetune/Step": 4426, "Finetune/Step Time": 4.545131899416447} +{"Finetune/Learning Rate": 1.1314502849747244e-06, "Finetune/Loss": 0.8279712200164795, "Finetune/Loss (Raw)": 0.7753268480300903, "Finetune/Step": 4427, "Finetune/Step Time": 4.553515430539846} +{"Finetune/Learning Rate": 1.1285736275111836e-06, "Finetune/Loss": 0.8275585174560547, "Finetune/Loss (Raw)": 0.889271080493927, "Finetune/Step": 4428, "Finetune/Step Time": 4.558710590004921} +{"Finetune/Learning Rate": 1.1257004128750205e-06, "Finetune/Loss": 0.831455409526825, "Finetune/Loss (Raw)": 0.8639904856681824, "Finetune/Step": 4429, "Finetune/Step Time": 4.520270079374313} +{"Finetune/Learning Rate": 1.122830642181273e-06, "Finetune/Loss": 0.8292204737663269, "Finetune/Loss (Raw)": 0.6813284158706665, "Finetune/Step": 4430, "Finetune/Step Time": 4.560392394661903} +{"Finetune/Learning Rate": 1.119964316543637e-06, "Finetune/Loss": 0.828328549861908, "Finetune/Loss (Raw)": 0.8739351034164429, "Finetune/Step": 4431, "Finetune/Step Time": 4.567836590111256} +{"Finetune/Learning Rate": 1.1171014370744837e-06, "Finetune/Loss": 0.8325276970863342, "Finetune/Loss (Raw)": 0.8798714876174927, "Finetune/Step": 4432, "Finetune/Step Time": 4.53489969484508} +{"Finetune/Learning Rate": 1.114242004884839e-06, "Finetune/Loss": 0.8320531845092773, "Finetune/Loss (Raw)": 0.7111465930938721, "Finetune/Step": 4433, "Finetune/Step Time": 4.538555530831218} +{"Finetune/Learning Rate": 1.111386021084394e-06, "Finetune/Loss": 0.8330487608909607, "Finetune/Loss (Raw)": 0.9585525989532471, "Finetune/Step": 4434, "Finetune/Step Time": 4.533829914405942} +{"Finetune/Learning Rate": 1.1085334867814946e-06, "Finetune/Loss": 0.83453369140625, "Finetune/Loss (Raw)": 0.867235541343689, "Finetune/Step": 4435, "Finetune/Step Time": 4.528763961046934} +{"Finetune/Learning Rate": 1.1056844030831603e-06, "Finetune/Loss": 0.833562970161438, "Finetune/Loss (Raw)": 0.7520178556442261, "Finetune/Step": 4436, "Finetune/Step Time": 4.540305718779564} +{"Finetune/Learning Rate": 1.1028387710950616e-06, "Finetune/Loss": 0.8332764506340027, "Finetune/Loss (Raw)": 0.8167897462844849, "Finetune/Step": 4437, "Finetune/Step Time": 4.540262712165713} +{"Finetune/Learning Rate": 1.0999965919215327e-06, "Finetune/Loss": 0.8373456597328186, "Finetune/Loss (Raw)": 0.8725975155830383, "Finetune/Step": 4438, "Finetune/Step Time": 4.497468117624521} +{"Finetune/Learning Rate": 1.0971578666655713e-06, "Finetune/Loss": 0.8385318517684937, "Finetune/Loss (Raw)": 1.0265225172042847, "Finetune/Step": 4439, "Finetune/Step Time": 4.4999823700636625} +{"Finetune/Learning Rate": 1.0943225964288272e-06, "Finetune/Loss": 0.8387142419815063, "Finetune/Loss (Raw)": 0.9999983310699463, "Finetune/Step": 4440, "Finetune/Step Time": 4.497915629297495} +{"Finetune/Learning Rate": 1.091490782311614e-06, "Finetune/Loss": 0.83867347240448, "Finetune/Loss (Raw)": 0.9707574248313904, "Finetune/Step": 4441, "Finetune/Step Time": 4.497061587870121} +{"Finetune/Learning Rate": 1.088662425412903e-06, "Finetune/Loss": 0.8386772274971008, "Finetune/Loss (Raw)": 0.9272302985191345, "Finetune/Step": 4442, "Finetune/Step Time": 4.505484364926815} +{"Finetune/Learning Rate": 1.0858375268303267e-06, "Finetune/Loss": 0.8386213779449463, "Finetune/Loss (Raw)": 0.8390946388244629, "Finetune/Step": 4443, "Finetune/Step Time": 4.5056918654590845} +{"Finetune/Learning Rate": 1.0830160876601737e-06, "Finetune/Loss": 0.8386398553848267, "Finetune/Loss (Raw)": 0.8089612722396851, "Finetune/Step": 4444, "Finetune/Step Time": 4.495957354083657} +{"Finetune/Learning Rate": 1.0801981089973844e-06, "Finetune/Loss": 0.8376830220222473, "Finetune/Loss (Raw)": 0.7294395565986633, "Finetune/Step": 4445, "Finetune/Step Time": 4.493251483887434} +{"Finetune/Learning Rate": 1.0773835919355669e-06, "Finetune/Loss": 0.8390798568725586, "Finetune/Loss (Raw)": 0.9689753651618958, "Finetune/Step": 4446, "Finetune/Step Time": 4.489977428689599} +{"Finetune/Learning Rate": 1.0745725375669768e-06, "Finetune/Loss": 0.8399389982223511, "Finetune/Loss (Raw)": 0.8375676274299622, "Finetune/Step": 4447, "Finetune/Step Time": 4.479942632839084} +{"Finetune/Learning Rate": 1.0717649469825298e-06, "Finetune/Loss": 0.8402834534645081, "Finetune/Loss (Raw)": 0.8680188655853271, "Finetune/Step": 4448, "Finetune/Step Time": 4.479240098968148} +{"Finetune/Learning Rate": 1.0689608212718005e-06, "Finetune/Loss": 0.8425279855728149, "Finetune/Loss (Raw)": 0.9934677481651306, "Finetune/Step": 4449, "Finetune/Step Time": 4.465494807809591} +{"Finetune/Learning Rate": 1.0661601615230122e-06, "Finetune/Loss": 0.8407086133956909, "Finetune/Loss (Raw)": 0.7720797657966614, "Finetune/Step": 4450, "Finetune/Step Time": 4.4589400719851255} +{"Finetune/Learning Rate": 1.0633629688230452e-06, "Finetune/Loss": 0.8412631154060364, "Finetune/Loss (Raw)": 0.7700060606002808, "Finetune/Step": 4451, "Finetune/Step Time": 4.463757712393999} +{"Finetune/Learning Rate": 1.0605692442574356e-06, "Finetune/Loss": 0.8376255631446838, "Finetune/Loss (Raw)": 0.3483816087245941, "Finetune/Step": 4452, "Finetune/Step Time": 4.502148101106286} +{"Finetune/Learning Rate": 1.0577789889103763e-06, "Finetune/Loss": 0.838268518447876, "Finetune/Loss (Raw)": 0.7490348219871521, "Finetune/Step": 4453, "Finetune/Step Time": 4.488252559676766} +{"Finetune/Learning Rate": 1.054992203864712e-06, "Finetune/Loss": 0.8392865657806396, "Finetune/Loss (Raw)": 0.8754861950874329, "Finetune/Step": 4454, "Finetune/Step Time": 4.4867062624543905} +{"Finetune/Learning Rate": 1.0522088902019324e-06, "Finetune/Loss": 0.83946692943573, "Finetune/Loss (Raw)": 0.7338975667953491, "Finetune/Step": 4455, "Finetune/Step Time": 4.483599454164505} +{"Finetune/Learning Rate": 1.049429049002193e-06, "Finetune/Loss": 0.8391397595405579, "Finetune/Loss (Raw)": 0.8154675960540771, "Finetune/Step": 4456, "Finetune/Step Time": 4.484545458108187} +{"Finetune/Learning Rate": 1.0466526813442957e-06, "Finetune/Loss": 0.8385503888130188, "Finetune/Loss (Raw)": 0.7782738208770752, "Finetune/Step": 4457, "Finetune/Step Time": 4.495910372585058} +{"Finetune/Learning Rate": 1.0438797883056927e-06, "Finetune/Loss": 0.8375959396362305, "Finetune/Loss (Raw)": 0.8373663425445557, "Finetune/Step": 4458, "Finetune/Step Time": 4.515782384201884} +{"Finetune/Learning Rate": 1.0411103709624903e-06, "Finetune/Loss": 0.8367840051651001, "Finetune/Loss (Raw)": 0.8398686051368713, "Finetune/Step": 4459, "Finetune/Step Time": 4.521577490493655} +{"Finetune/Learning Rate": 1.0383444303894453e-06, "Finetune/Loss": 0.8373063802719116, "Finetune/Loss (Raw)": 0.9566710591316223, "Finetune/Step": 4460, "Finetune/Step Time": 4.5217650812119246} +{"Finetune/Learning Rate": 1.0355819676599655e-06, "Finetune/Loss": 0.835877537727356, "Finetune/Loss (Raw)": 0.7886478304862976, "Finetune/Step": 4461, "Finetune/Step Time": 4.520612319931388} +{"Finetune/Learning Rate": 1.0328229838461069e-06, "Finetune/Loss": 0.8356747031211853, "Finetune/Loss (Raw)": 0.9128050804138184, "Finetune/Step": 4462, "Finetune/Step Time": 4.533208968117833} +{"Finetune/Learning Rate": 1.0300674800185795e-06, "Finetune/Loss": 0.8337603807449341, "Finetune/Loss (Raw)": 0.6619402170181274, "Finetune/Step": 4463, "Finetune/Step Time": 4.542560493573546} +{"Finetune/Learning Rate": 1.0273154572467436e-06, "Finetune/Loss": 0.8326192498207092, "Finetune/Loss (Raw)": 0.8549832701683044, "Finetune/Step": 4464, "Finetune/Step Time": 4.5455266404896975} +{"Finetune/Learning Rate": 1.0245669165985973e-06, "Finetune/Loss": 0.8336796164512634, "Finetune/Loss (Raw)": 0.7961295247077942, "Finetune/Step": 4465, "Finetune/Step Time": 4.522262131795287} +{"Finetune/Learning Rate": 1.0218218591408013e-06, "Finetune/Loss": 0.8333560824394226, "Finetune/Loss (Raw)": 0.826398491859436, "Finetune/Step": 4466, "Finetune/Step Time": 4.5206101182848215} +{"Finetune/Learning Rate": 1.0190802859386585e-06, "Finetune/Loss": 0.8342353701591492, "Finetune/Loss (Raw)": 0.9080554246902466, "Finetune/Step": 4467, "Finetune/Step Time": 4.5147391352802515} +{"Finetune/Learning Rate": 1.0163421980561184e-06, "Finetune/Loss": 0.8341580033302307, "Finetune/Loss (Raw)": 0.8450315594673157, "Finetune/Step": 4468, "Finetune/Step Time": 4.501500951126218} +{"Finetune/Learning Rate": 1.0136075965557812e-06, "Finetune/Loss": 0.8332525491714478, "Finetune/Loss (Raw)": 0.8440338373184204, "Finetune/Step": 4469, "Finetune/Step Time": 4.501899825409055} +{"Finetune/Learning Rate": 1.0108764824988903e-06, "Finetune/Loss": 0.8332892656326294, "Finetune/Loss (Raw)": 0.8376213312149048, "Finetune/Step": 4470, "Finetune/Step Time": 4.458523396402597} +{"Finetune/Learning Rate": 1.008148856945339e-06, "Finetune/Loss": 0.8332011699676514, "Finetune/Loss (Raw)": 0.7888025045394897, "Finetune/Step": 4471, "Finetune/Step Time": 4.471810158342123} +{"Finetune/Learning Rate": 1.0054247209536628e-06, "Finetune/Loss": 0.8329566717147827, "Finetune/Loss (Raw)": 0.866881787776947, "Finetune/Step": 4472, "Finetune/Step Time": 4.462413404136896} +{"Finetune/Learning Rate": 1.0027040755810513e-06, "Finetune/Loss": 0.8343832492828369, "Finetune/Loss (Raw)": 1.0389035940170288, "Finetune/Step": 4473, "Finetune/Step Time": 4.460283657535911} +{"Finetune/Learning Rate": 9.999869218833313e-07, "Finetune/Loss": 0.8346440196037292, "Finetune/Loss (Raw)": 0.9054296016693115, "Finetune/Step": 4474, "Finetune/Step Time": 4.453021641820669} +{"Finetune/Learning Rate": 9.972732609149738e-07, "Finetune/Loss": 0.833200991153717, "Finetune/Loss (Raw)": 0.773348867893219, "Finetune/Step": 4475, "Finetune/Step Time": 4.474183337762952} +{"Finetune/Learning Rate": 9.945630937291006e-07, "Finetune/Loss": 0.8333220481872559, "Finetune/Loss (Raw)": 0.9282006025314331, "Finetune/Step": 4476, "Finetune/Step Time": 4.4752100463956594} +{"Finetune/Learning Rate": 9.918564213774762e-07, "Finetune/Loss": 0.8319048285484314, "Finetune/Loss (Raw)": 0.7946003079414368, "Finetune/Step": 4477, "Finetune/Step Time": 4.476772082969546} +{"Finetune/Learning Rate": 9.891532449105045e-07, "Finetune/Loss": 0.8341924548149109, "Finetune/Loss (Raw)": 0.9394091963768005, "Finetune/Step": 4478, "Finetune/Step Time": 4.459150968119502} +{"Finetune/Learning Rate": 9.864535653772367e-07, "Finetune/Loss": 0.8340229392051697, "Finetune/Loss (Raw)": 0.8776734471321106, "Finetune/Step": 4479, "Finetune/Step Time": 4.468640495091677} +{"Finetune/Learning Rate": 9.837573838253655e-07, "Finetune/Loss": 0.8338162899017334, "Finetune/Loss (Raw)": 0.8296488523483276, "Finetune/Step": 4480, "Finetune/Step Time": 4.473066460341215} +{"Finetune/Learning Rate": 9.810647013012265e-07, "Finetune/Loss": 0.8332396149635315, "Finetune/Loss (Raw)": 0.8358262181282043, "Finetune/Step": 4481, "Finetune/Step Time": 4.4743617586791515} +{"Finetune/Learning Rate": 9.783755188497944e-07, "Finetune/Loss": 0.8336966037750244, "Finetune/Loss (Raw)": 0.8064472675323486, "Finetune/Step": 4482, "Finetune/Step Time": 4.492666745558381} +{"Finetune/Learning Rate": 9.75689837514696e-07, "Finetune/Loss": 0.8334965109825134, "Finetune/Loss (Raw)": 0.862261950969696, "Finetune/Step": 4483, "Finetune/Step Time": 4.4895229786634445} +{"Finetune/Learning Rate": 9.730076583381841e-07, "Finetune/Loss": 0.833654522895813, "Finetune/Loss (Raw)": 0.9097423553466797, "Finetune/Step": 4484, "Finetune/Step Time": 4.50475406460464} +{"Finetune/Learning Rate": 9.703289823611606e-07, "Finetune/Loss": 0.8331778645515442, "Finetune/Loss (Raw)": 0.854188859462738, "Finetune/Step": 4485, "Finetune/Step Time": 4.505561728030443} +{"Finetune/Learning Rate": 9.676538106231725e-07, "Finetune/Loss": 0.8328490853309631, "Finetune/Loss (Raw)": 0.8628520369529724, "Finetune/Step": 4486, "Finetune/Step Time": 4.502256305888295} +{"Finetune/Learning Rate": 9.649821441623986e-07, "Finetune/Loss": 0.8285186290740967, "Finetune/Loss (Raw)": 0.325787216424942, "Finetune/Step": 4487, "Finetune/Step Time": 4.540499670431018} +{"Finetune/Learning Rate": 9.623139840156604e-07, "Finetune/Loss": 0.8294792175292969, "Finetune/Loss (Raw)": 0.9372320175170898, "Finetune/Step": 4488, "Finetune/Step Time": 4.547287300229073} +{"Finetune/Learning Rate": 9.596493312184197e-07, "Finetune/Loss": 0.8263280391693115, "Finetune/Loss (Raw)": 0.6597946286201477, "Finetune/Step": 4489, "Finetune/Step Time": 4.537109045311809} +{"Finetune/Learning Rate": 9.569881868047747e-07, "Finetune/Loss": 0.8257620930671692, "Finetune/Loss (Raw)": 0.7400816082954407, "Finetune/Step": 4490, "Finetune/Step Time": 4.540926413610578} +{"Finetune/Learning Rate": 9.543305518074652e-07, "Finetune/Loss": 0.8253874778747559, "Finetune/Loss (Raw)": 0.7851139307022095, "Finetune/Step": 4491, "Finetune/Step Time": 4.540360499173403} +{"Finetune/Learning Rate": 9.516764272578638e-07, "Finetune/Loss": 0.8264139294624329, "Finetune/Loss (Raw)": 0.9044487476348877, "Finetune/Step": 4492, "Finetune/Step Time": 4.522633347660303} +{"Finetune/Learning Rate": 9.49025814185992e-07, "Finetune/Loss": 0.8267289996147156, "Finetune/Loss (Raw)": 0.806136429309845, "Finetune/Step": 4493, "Finetune/Step Time": 4.51796717569232} +{"Finetune/Learning Rate": 9.463787136204938e-07, "Finetune/Loss": 0.825088381767273, "Finetune/Loss (Raw)": 0.6993821859359741, "Finetune/Step": 4494, "Finetune/Step Time": 4.522811064496636} +{"Finetune/Learning Rate": 9.437351265886585e-07, "Finetune/Loss": 0.8259038329124451, "Finetune/Loss (Raw)": 0.8955267071723938, "Finetune/Step": 4495, "Finetune/Step Time": 4.5223193019628525} +{"Finetune/Learning Rate": 9.410950541164143e-07, "Finetune/Loss": 0.82471764087677, "Finetune/Loss (Raw)": 0.7680420875549316, "Finetune/Step": 4496, "Finetune/Step Time": 4.530088119208813} +{"Finetune/Learning Rate": 9.384584972283184e-07, "Finetune/Loss": 0.8255558013916016, "Finetune/Loss (Raw)": 0.9266810417175293, "Finetune/Step": 4497, "Finetune/Step Time": 4.53273781388998} +{"Finetune/Learning Rate": 9.358254569475689e-07, "Finetune/Loss": 0.8240485191345215, "Finetune/Loss (Raw)": 0.7682589888572693, "Finetune/Step": 4498, "Finetune/Step Time": 4.537418065592647} +{"Finetune/Learning Rate": 9.331959342959951e-07, "Finetune/Loss": 0.8238106966018677, "Finetune/Loss (Raw)": 0.9515366554260254, "Finetune/Step": 4499, "Finetune/Step Time": 4.537157641723752} +{"Finetune/Learning Rate": 9.305699302940662e-07, "Finetune/Loss": 0.8285006880760193, "Finetune/Loss (Raw)": 0.8493305444717407, "Finetune/Step": 4500, "Finetune/Step Time": 4.504287047311664} +{"Finetune/Learning Rate": 9.279474459608806e-07, "Finetune/Loss": 0.829976499080658, "Finetune/Loss (Raw)": 0.9431890845298767, "Finetune/Step": 4501, "Finetune/Step Time": 4.502869240939617} +{"Finetune/Learning Rate": 9.253284823141728e-07, "Finetune/Loss": 0.8291546106338501, "Finetune/Loss (Raw)": 0.7223753333091736, "Finetune/Step": 4502, "Finetune/Step Time": 4.502211021259427} +{"Finetune/Learning Rate": 9.227130403703178e-07, "Finetune/Loss": 0.8302940130233765, "Finetune/Loss (Raw)": 0.9423058032989502, "Finetune/Step": 4503, "Finetune/Step Time": 4.4976977575570345} +{"Finetune/Learning Rate": 9.201011211443101e-07, "Finetune/Loss": 0.8296232223510742, "Finetune/Loss (Raw)": 0.8413516283035278, "Finetune/Step": 4504, "Finetune/Step Time": 4.487042019143701} +{"Finetune/Learning Rate": 9.174927256497846e-07, "Finetune/Loss": 0.8291918635368347, "Finetune/Loss (Raw)": 0.8196354508399963, "Finetune/Step": 4505, "Finetune/Step Time": 4.486118424683809} +{"Finetune/Learning Rate": 9.148878548990137e-07, "Finetune/Loss": 0.8302590250968933, "Finetune/Loss (Raw)": 1.006428837776184, "Finetune/Step": 4506, "Finetune/Step Time": 4.471754593774676} +{"Finetune/Learning Rate": 9.122865099028944e-07, "Finetune/Loss": 0.826880693435669, "Finetune/Loss (Raw)": 0.32920095324516296, "Finetune/Step": 4507, "Finetune/Step Time": 4.491255907341838} +{"Finetune/Learning Rate": 9.09688691670958e-07, "Finetune/Loss": 0.8259537220001221, "Finetune/Loss (Raw)": 0.8613330125808716, "Finetune/Step": 4508, "Finetune/Step Time": 4.48451672680676} +{"Finetune/Learning Rate": 9.070944012113669e-07, "Finetune/Loss": 0.8250542283058167, "Finetune/Loss (Raw)": 0.7846492528915405, "Finetune/Step": 4509, "Finetune/Step Time": 4.4827348459511995} +{"Finetune/Learning Rate": 9.045036395309148e-07, "Finetune/Loss": 0.8252332210540771, "Finetune/Loss (Raw)": 0.9523738026618958, "Finetune/Step": 4510, "Finetune/Step Time": 4.479157090187073} +{"Finetune/Learning Rate": 9.019164076350262e-07, "Finetune/Loss": 0.8274122476577759, "Finetune/Loss (Raw)": 1.0062918663024902, "Finetune/Step": 4511, "Finetune/Step Time": 4.4720046650618315} +{"Finetune/Learning Rate": 8.993327065277524e-07, "Finetune/Loss": 0.8234636783599854, "Finetune/Loss (Raw)": 0.36254504323005676, "Finetune/Step": 4512, "Finetune/Step Time": 4.511971469968557} +{"Finetune/Learning Rate": 8.967525372117825e-07, "Finetune/Loss": 0.823656439781189, "Finetune/Loss (Raw)": 0.8309729099273682, "Finetune/Step": 4513, "Finetune/Step Time": 4.517121994867921} +{"Finetune/Learning Rate": 8.941759006884265e-07, "Finetune/Loss": 0.823002815246582, "Finetune/Loss (Raw)": 0.7536507248878479, "Finetune/Step": 4514, "Finetune/Step Time": 4.4935681242495775} +{"Finetune/Learning Rate": 8.916027979576247e-07, "Finetune/Loss": 0.8214026689529419, "Finetune/Loss (Raw)": 0.8420342803001404, "Finetune/Step": 4515, "Finetune/Step Time": 4.4967752024531364} +{"Finetune/Learning Rate": 8.890332300179516e-07, "Finetune/Loss": 0.8203553557395935, "Finetune/Loss (Raw)": 0.768436074256897, "Finetune/Step": 4516, "Finetune/Step Time": 4.485086975619197} +{"Finetune/Learning Rate": 8.864671978666084e-07, "Finetune/Loss": 0.819031834602356, "Finetune/Loss (Raw)": 0.7463732361793518, "Finetune/Step": 4517, "Finetune/Step Time": 4.481150867417455} +{"Finetune/Learning Rate": 8.839047024994152e-07, "Finetune/Loss": 0.820331871509552, "Finetune/Loss (Raw)": 0.9939282536506653, "Finetune/Step": 4518, "Finetune/Step Time": 4.476563462987542} +{"Finetune/Learning Rate": 8.813457449108309e-07, "Finetune/Loss": 0.8210564851760864, "Finetune/Loss (Raw)": 0.9401759505271912, "Finetune/Step": 4519, "Finetune/Step Time": 4.46951493807137} +{"Finetune/Learning Rate": 8.787903260939368e-07, "Finetune/Loss": 0.8255442380905151, "Finetune/Loss (Raw)": 0.9285107851028442, "Finetune/Step": 4520, "Finetune/Step Time": 4.42890202999115} +{"Finetune/Learning Rate": 8.762384470404417e-07, "Finetune/Loss": 0.8274868130683899, "Finetune/Loss (Raw)": 0.9878445267677307, "Finetune/Step": 4521, "Finetune/Step Time": 4.4163766745477915} +{"Finetune/Learning Rate": 8.736901087406779e-07, "Finetune/Loss": 0.8276360034942627, "Finetune/Loss (Raw)": 0.8362764716148376, "Finetune/Step": 4522, "Finetune/Step Time": 4.413558974862099} +{"Finetune/Learning Rate": 8.711453121836066e-07, "Finetune/Loss": 0.8278051614761353, "Finetune/Loss (Raw)": 0.8691973686218262, "Finetune/Step": 4523, "Finetune/Step Time": 4.41748533770442} +{"Finetune/Learning Rate": 8.686040583568134e-07, "Finetune/Loss": 0.8269431591033936, "Finetune/Loss (Raw)": 0.8532644510269165, "Finetune/Step": 4524, "Finetune/Step Time": 4.411627184599638} +{"Finetune/Learning Rate": 8.660663482465093e-07, "Finetune/Loss": 0.8301504850387573, "Finetune/Loss (Raw)": 0.8297115564346313, "Finetune/Step": 4525, "Finetune/Step Time": 4.380368942394853} +{"Finetune/Learning Rate": 8.63532182837531e-07, "Finetune/Loss": 0.8284496068954468, "Finetune/Loss (Raw)": 0.6695186495780945, "Finetune/Step": 4526, "Finetune/Step Time": 4.375537306070328} +{"Finetune/Learning Rate": 8.610015631133395e-07, "Finetune/Loss": 0.8263792395591736, "Finetune/Loss (Raw)": 0.5369764566421509, "Finetune/Step": 4527, "Finetune/Step Time": 4.394130198284984} +{"Finetune/Learning Rate": 8.584744900560149e-07, "Finetune/Loss": 0.8260723352432251, "Finetune/Loss (Raw)": 0.8372214436531067, "Finetune/Step": 4528, "Finetune/Step Time": 4.384581031277776} +{"Finetune/Learning Rate": 8.559509646462693e-07, "Finetune/Loss": 0.8306045532226562, "Finetune/Loss (Raw)": 0.9142117500305176, "Finetune/Step": 4529, "Finetune/Step Time": 4.348601324483752} +{"Finetune/Learning Rate": 8.534309878634317e-07, "Finetune/Loss": 0.8319036364555359, "Finetune/Loss (Raw)": 0.9622920155525208, "Finetune/Step": 4530, "Finetune/Step Time": 4.331973662599921} +{"Finetune/Learning Rate": 8.509145606854552e-07, "Finetune/Loss": 0.8324314951896667, "Finetune/Loss (Raw)": 0.9903457760810852, "Finetune/Step": 4531, "Finetune/Step Time": 4.338967552408576} +{"Finetune/Learning Rate": 8.484016840889176e-07, "Finetune/Loss": 0.8296346664428711, "Finetune/Loss (Raw)": 0.6529232263565063, "Finetune/Step": 4532, "Finetune/Step Time": 4.34968963265419} +{"Finetune/Learning Rate": 8.458923590490165e-07, "Finetune/Loss": 0.8301229476928711, "Finetune/Loss (Raw)": 0.8832823038101196, "Finetune/Step": 4533, "Finetune/Step Time": 4.341703075915575} +{"Finetune/Learning Rate": 8.433865865395718e-07, "Finetune/Loss": 0.8269003033638, "Finetune/Loss (Raw)": 0.33724138140678406, "Finetune/Step": 4534, "Finetune/Step Time": 4.3819313775748014} +{"Finetune/Learning Rate": 8.408843675330225e-07, "Finetune/Loss": 0.8267049193382263, "Finetune/Loss (Raw)": 0.7462544441223145, "Finetune/Step": 4535, "Finetune/Step Time": 4.390218386426568} +{"Finetune/Learning Rate": 8.383857030004361e-07, "Finetune/Loss": 0.8255385160446167, "Finetune/Loss (Raw)": 0.8627002835273743, "Finetune/Step": 4536, "Finetune/Step Time": 4.400625124573708} +{"Finetune/Learning Rate": 8.358905939114948e-07, "Finetune/Loss": 0.8259549736976624, "Finetune/Loss (Raw)": 0.8681315779685974, "Finetune/Step": 4537, "Finetune/Step Time": 4.398417646065354} +{"Finetune/Learning Rate": 8.333990412344961e-07, "Finetune/Loss": 0.8261232376098633, "Finetune/Loss (Raw)": 0.9005371332168579, "Finetune/Step": 4538, "Finetune/Step Time": 4.398046847432852} +{"Finetune/Learning Rate": 8.30911045936369e-07, "Finetune/Loss": 0.8254882097244263, "Finetune/Loss (Raw)": 0.7685907483100891, "Finetune/Step": 4539, "Finetune/Step Time": 4.399846613407135} +{"Finetune/Learning Rate": 8.284266089826531e-07, "Finetune/Loss": 0.8216375112533569, "Finetune/Loss (Raw)": 0.31052058935165405, "Finetune/Step": 4540, "Finetune/Step Time": 4.439750276505947} +{"Finetune/Learning Rate": 8.259457313375097e-07, "Finetune/Loss": 0.8191112875938416, "Finetune/Loss (Raw)": 0.76286780834198, "Finetune/Step": 4541, "Finetune/Step Time": 4.461907461285591} +{"Finetune/Learning Rate": 8.234684139637205e-07, "Finetune/Loss": 0.818626880645752, "Finetune/Loss (Raw)": 0.7425251007080078, "Finetune/Step": 4542, "Finetune/Step Time": 4.4633286241441965} +{"Finetune/Learning Rate": 8.209946578226836e-07, "Finetune/Loss": 0.8178408145904541, "Finetune/Loss (Raw)": 0.8348881006240845, "Finetune/Step": 4543, "Finetune/Step Time": 4.457946145907044} +{"Finetune/Learning Rate": 8.185244638744139e-07, "Finetune/Loss": 0.8191328048706055, "Finetune/Loss (Raw)": 0.9126438498497009, "Finetune/Step": 4544, "Finetune/Step Time": 4.44456223398447} +{"Finetune/Learning Rate": 8.160578330775459e-07, "Finetune/Loss": 0.8187613487243652, "Finetune/Loss (Raw)": 0.9128865003585815, "Finetune/Step": 4545, "Finetune/Step Time": 4.455909743905067} +{"Finetune/Learning Rate": 8.135947663893329e-07, "Finetune/Loss": 0.8194432258605957, "Finetune/Loss (Raw)": 0.940639078617096, "Finetune/Step": 4546, "Finetune/Step Time": 4.455043146386743} +{"Finetune/Learning Rate": 8.111352647656423e-07, "Finetune/Loss": 0.8194514513015747, "Finetune/Loss (Raw)": 0.9233225584030151, "Finetune/Step": 4547, "Finetune/Step Time": 4.44834646768868} +{"Finetune/Learning Rate": 8.086793291609562e-07, "Finetune/Loss": 0.8179978132247925, "Finetune/Loss (Raw)": 0.5749022364616394, "Finetune/Step": 4548, "Finetune/Step Time": 4.494307471439242} +{"Finetune/Learning Rate": 8.062269605283779e-07, "Finetune/Loss": 0.8172372579574585, "Finetune/Loss (Raw)": 0.7281357645988464, "Finetune/Step": 4549, "Finetune/Step Time": 4.500284764915705} +{"Finetune/Learning Rate": 8.037781598196225e-07, "Finetune/Loss": 0.8179943561553955, "Finetune/Loss (Raw)": 0.9864764213562012, "Finetune/Step": 4550, "Finetune/Step Time": 4.502712242305279} +{"Finetune/Learning Rate": 8.013329279850235e-07, "Finetune/Loss": 0.8189247846603394, "Finetune/Loss (Raw)": 0.9105260968208313, "Finetune/Step": 4551, "Finetune/Step Time": 4.500235460698605} +{"Finetune/Learning Rate": 7.988912659735249e-07, "Finetune/Loss": 0.82167649269104, "Finetune/Loss (Raw)": 0.6836643815040588, "Finetune/Step": 4552, "Finetune/Step Time": 4.472691148519516} +{"Finetune/Learning Rate": 7.964531747326898e-07, "Finetune/Loss": 0.8225888013839722, "Finetune/Loss (Raw)": 0.9212952256202698, "Finetune/Step": 4553, "Finetune/Step Time": 4.452644811943173} +{"Finetune/Learning Rate": 7.940186552086937e-07, "Finetune/Loss": 0.8235488533973694, "Finetune/Loss (Raw)": 1.0308282375335693, "Finetune/Step": 4554, "Finetune/Step Time": 4.451311847195029} +{"Finetune/Learning Rate": 7.915877083463242e-07, "Finetune/Loss": 0.8243550062179565, "Finetune/Loss (Raw)": 0.8785088062286377, "Finetune/Step": 4555, "Finetune/Step Time": 4.452660318464041} +{"Finetune/Learning Rate": 7.891603350889888e-07, "Finetune/Loss": 0.8256983160972595, "Finetune/Loss (Raw)": 1.0612186193466187, "Finetune/Step": 4556, "Finetune/Step Time": 4.4500023405998945} +{"Finetune/Learning Rate": 7.867365363787005e-07, "Finetune/Loss": 0.8250091671943665, "Finetune/Loss (Raw)": 0.7757832407951355, "Finetune/Step": 4557, "Finetune/Step Time": 4.452668892219663} +{"Finetune/Learning Rate": 7.843163131560871e-07, "Finetune/Loss": 0.8247699737548828, "Finetune/Loss (Raw)": 0.6507095694541931, "Finetune/Step": 4558, "Finetune/Step Time": 4.4254568964242935} +{"Finetune/Learning Rate": 7.818996663603917e-07, "Finetune/Loss": 0.8254761695861816, "Finetune/Loss (Raw)": 0.9643304944038391, "Finetune/Step": 4559, "Finetune/Step Time": 4.426576901227236} +{"Finetune/Learning Rate": 7.794865969294685e-07, "Finetune/Loss": 0.8250874280929565, "Finetune/Loss (Raw)": 0.8301079869270325, "Finetune/Step": 4560, "Finetune/Step Time": 4.421923700720072} +{"Finetune/Learning Rate": 7.770771057997828e-07, "Finetune/Loss": 0.826543927192688, "Finetune/Loss (Raw)": 0.8975809812545776, "Finetune/Step": 4561, "Finetune/Step Time": 4.411942915990949} +{"Finetune/Learning Rate": 7.746711939064066e-07, "Finetune/Loss": 0.8263980150222778, "Finetune/Loss (Raw)": 0.9398773908615112, "Finetune/Step": 4562, "Finetune/Step Time": 4.4119370095431805} +{"Finetune/Learning Rate": 7.722688621830333e-07, "Finetune/Loss": 0.8220183849334717, "Finetune/Loss (Raw)": 0.3066461384296417, "Finetune/Step": 4563, "Finetune/Step Time": 4.450118785724044} +{"Finetune/Learning Rate": 7.698701115619578e-07, "Finetune/Loss": 0.8227871656417847, "Finetune/Loss (Raw)": 0.8504130244255066, "Finetune/Step": 4564, "Finetune/Step Time": 4.4338715728372335} +{"Finetune/Learning Rate": 7.674749429740868e-07, "Finetune/Loss": 0.8225768804550171, "Finetune/Loss (Raw)": 0.789878785610199, "Finetune/Step": 4565, "Finetune/Step Time": 4.428990248590708} +{"Finetune/Learning Rate": 7.650833573489436e-07, "Finetune/Loss": 0.8222753405570984, "Finetune/Loss (Raw)": 0.8339992165565491, "Finetune/Step": 4566, "Finetune/Step Time": 4.431477598845959} +{"Finetune/Learning Rate": 7.626953556146521e-07, "Finetune/Loss": 0.8218058347702026, "Finetune/Loss (Raw)": 0.9664318561553955, "Finetune/Step": 4567, "Finetune/Step Time": 4.423043709248304} +{"Finetune/Learning Rate": 7.603109386979501e-07, "Finetune/Loss": 0.8211914300918579, "Finetune/Loss (Raw)": 0.9213501214981079, "Finetune/Step": 4568, "Finetune/Step Time": 4.420222237706184} +{"Finetune/Learning Rate": 7.579301075241808e-07, "Finetune/Loss": 0.8214430809020996, "Finetune/Loss (Raw)": 1.002967357635498, "Finetune/Step": 4569, "Finetune/Step Time": 4.412344068288803} +{"Finetune/Learning Rate": 7.555528630173014e-07, "Finetune/Loss": 0.8214752674102783, "Finetune/Loss (Raw)": 0.9313549995422363, "Finetune/Step": 4570, "Finetune/Step Time": 4.398493757471442} +{"Finetune/Learning Rate": 7.531792060998766e-07, "Finetune/Loss": 0.8214379549026489, "Finetune/Loss (Raw)": 0.8343155980110168, "Finetune/Step": 4571, "Finetune/Step Time": 4.416189668700099} +{"Finetune/Learning Rate": 7.508091376930682e-07, "Finetune/Loss": 0.8217394948005676, "Finetune/Loss (Raw)": 0.8475545644760132, "Finetune/Step": 4572, "Finetune/Step Time": 4.421996233984828} +{"Finetune/Learning Rate": 7.484426587166615e-07, "Finetune/Loss": 0.8222876191139221, "Finetune/Loss (Raw)": 0.7996020913124084, "Finetune/Step": 4573, "Finetune/Step Time": 4.448338959366083} +{"Finetune/Learning Rate": 7.460797700890365e-07, "Finetune/Loss": 0.8216875791549683, "Finetune/Loss (Raw)": 0.8921661376953125, "Finetune/Step": 4574, "Finetune/Step Time": 4.442068304866552} +{"Finetune/Learning Rate": 7.437204727271852e-07, "Finetune/Loss": 0.821439266204834, "Finetune/Loss (Raw)": 0.8057878017425537, "Finetune/Step": 4575, "Finetune/Step Time": 4.456423884257674} +{"Finetune/Learning Rate": 7.41364767546705e-07, "Finetune/Loss": 0.8208247423171997, "Finetune/Loss (Raw)": 0.7893587350845337, "Finetune/Step": 4576, "Finetune/Step Time": 4.457898661494255} +{"Finetune/Learning Rate": 7.390126554617982e-07, "Finetune/Loss": 0.8207801580429077, "Finetune/Loss (Raw)": 0.9877657294273376, "Finetune/Step": 4577, "Finetune/Step Time": 4.4607483353465796} +{"Finetune/Learning Rate": 7.366641373852745e-07, "Finetune/Loss": 0.822167158126831, "Finetune/Loss (Raw)": 0.9496081471443176, "Finetune/Step": 4578, "Finetune/Step Time": 4.464967356994748} +{"Finetune/Learning Rate": 7.343192142285471e-07, "Finetune/Loss": 0.8194230198860168, "Finetune/Loss (Raw)": 0.41875675320625305, "Finetune/Step": 4579, "Finetune/Step Time": 4.492937417700887} +{"Finetune/Learning Rate": 7.319778869016358e-07, "Finetune/Loss": 0.8187979459762573, "Finetune/Loss (Raw)": 0.268371045589447, "Finetune/Step": 4580, "Finetune/Step Time": 4.4984567277133465} +{"Finetune/Learning Rate": 7.296401563131661e-07, "Finetune/Loss": 0.8194512128829956, "Finetune/Loss (Raw)": 0.8326531648635864, "Finetune/Step": 4581, "Finetune/Step Time": 4.513028290122747} +{"Finetune/Learning Rate": 7.273060233703599e-07, "Finetune/Loss": 0.8195527791976929, "Finetune/Loss (Raw)": 0.8884882926940918, "Finetune/Step": 4582, "Finetune/Step Time": 4.516419151797891} +{"Finetune/Learning Rate": 7.249754889790539e-07, "Finetune/Loss": 0.8213604688644409, "Finetune/Loss (Raw)": 0.9652814269065857, "Finetune/Step": 4583, "Finetune/Step Time": 4.503059426322579} +{"Finetune/Learning Rate": 7.2264855404368e-07, "Finetune/Loss": 0.821121096611023, "Finetune/Loss (Raw)": 0.7848246097564697, "Finetune/Step": 4584, "Finetune/Step Time": 4.509907653555274} +{"Finetune/Learning Rate": 7.203252194672783e-07, "Finetune/Loss": 0.8208373188972473, "Finetune/Loss (Raw)": 0.7419534921646118, "Finetune/Step": 4585, "Finetune/Step Time": 4.4981006141752005} +{"Finetune/Learning Rate": 7.180054861514885e-07, "Finetune/Loss": 0.8200783729553223, "Finetune/Loss (Raw)": 0.7402259111404419, "Finetune/Step": 4586, "Finetune/Step Time": 4.483590317890048} +{"Finetune/Learning Rate": 7.156893549965538e-07, "Finetune/Loss": 0.8205952644348145, "Finetune/Loss (Raw)": 0.9060260057449341, "Finetune/Step": 4587, "Finetune/Step Time": 4.479563340544701} +{"Finetune/Learning Rate": 7.133768269013186e-07, "Finetune/Loss": 0.821117639541626, "Finetune/Loss (Raw)": 1.0235413312911987, "Finetune/Step": 4588, "Finetune/Step Time": 4.503208927810192} +{"Finetune/Learning Rate": 7.110679027632295e-07, "Finetune/Loss": 0.8214819431304932, "Finetune/Loss (Raw)": 0.8352735042572021, "Finetune/Step": 4589, "Finetune/Step Time": 4.503095902502537} +{"Finetune/Learning Rate": 7.087625834783385e-07, "Finetune/Loss": 0.8206661343574524, "Finetune/Loss (Raw)": 0.8083814382553101, "Finetune/Step": 4590, "Finetune/Step Time": 4.493526680395007} +{"Finetune/Learning Rate": 7.064608699412944e-07, "Finetune/Loss": 0.8226331472396851, "Finetune/Loss (Raw)": 0.9137174487113953, "Finetune/Step": 4591, "Finetune/Step Time": 4.4776691775768995} +{"Finetune/Learning Rate": 7.041627630453418e-07, "Finetune/Loss": 0.8182476758956909, "Finetune/Loss (Raw)": 0.2936411499977112, "Finetune/Step": 4592, "Finetune/Step Time": 4.497619725763798} +{"Finetune/Learning Rate": 7.018682636823371e-07, "Finetune/Loss": 0.8191050291061401, "Finetune/Loss (Raw)": 0.9058749675750732, "Finetune/Step": 4593, "Finetune/Step Time": 4.503213873133063} +{"Finetune/Learning Rate": 6.99577372742728e-07, "Finetune/Loss": 0.8158540725708008, "Finetune/Loss (Raw)": 0.41027289628982544, "Finetune/Step": 4594, "Finetune/Step Time": 4.541429720818996} +{"Finetune/Learning Rate": 6.972900911155656e-07, "Finetune/Loss": 0.8159000277519226, "Finetune/Loss (Raw)": 0.9139382839202881, "Finetune/Step": 4595, "Finetune/Step Time": 4.543740600347519} +{"Finetune/Learning Rate": 6.950064196884987e-07, "Finetune/Loss": 0.8166168928146362, "Finetune/Loss (Raw)": 0.9367865324020386, "Finetune/Step": 4596, "Finetune/Step Time": 4.5505977142602205} +{"Finetune/Learning Rate": 6.927263593477751e-07, "Finetune/Loss": 0.8159328103065491, "Finetune/Loss (Raw)": 0.7564780712127686, "Finetune/Step": 4597, "Finetune/Step Time": 4.551910143345594} +{"Finetune/Learning Rate": 6.904499109782426e-07, "Finetune/Loss": 0.816100001335144, "Finetune/Loss (Raw)": 0.8590242862701416, "Finetune/Step": 4598, "Finetune/Step Time": 4.549029076471925} +{"Finetune/Learning Rate": 6.881770754633444e-07, "Finetune/Loss": 0.8161752820014954, "Finetune/Loss (Raw)": 0.7984347939491272, "Finetune/Step": 4599, "Finetune/Step Time": 4.543176220729947} +{"Finetune/Learning Rate": 6.859078536851283e-07, "Finetune/Loss": 0.8158438205718994, "Finetune/Loss (Raw)": 0.8244551420211792, "Finetune/Step": 4600, "Finetune/Step Time": 4.547032320871949} +{"Finetune/Learning Rate": 6.836422465242298e-07, "Finetune/Loss": 0.8133707046508789, "Finetune/Loss (Raw)": 0.7223477959632874, "Finetune/Step": 4601, "Finetune/Step Time": 4.552563393488526} +{"Finetune/Learning Rate": 6.81380254859888e-07, "Finetune/Loss": 0.8142812848091125, "Finetune/Loss (Raw)": 1.0219765901565552, "Finetune/Step": 4602, "Finetune/Step Time": 4.554504916071892} +{"Finetune/Learning Rate": 6.791218795699406e-07, "Finetune/Loss": 0.8152980804443359, "Finetune/Loss (Raw)": 0.9034996032714844, "Finetune/Step": 4603, "Finetune/Step Time": 4.547057626768947} +{"Finetune/Learning Rate": 6.768671215308165e-07, "Finetune/Loss": 0.8146125078201294, "Finetune/Loss (Raw)": 0.8404479622840881, "Finetune/Step": 4604, "Finetune/Step Time": 4.541123621165752} +{"Finetune/Learning Rate": 6.746159816175435e-07, "Finetune/Loss": 0.8109910488128662, "Finetune/Loss (Raw)": 0.3310554623603821, "Finetune/Step": 4605, "Finetune/Step Time": 4.572720354422927} +{"Finetune/Learning Rate": 6.723684607037451e-07, "Finetune/Loss": 0.8106144666671753, "Finetune/Loss (Raw)": 0.8912020921707153, "Finetune/Step": 4606, "Finetune/Step Time": 4.580765452235937} +{"Finetune/Learning Rate": 6.701245596616402e-07, "Finetune/Loss": 0.8101328611373901, "Finetune/Loss (Raw)": 0.8160325288772583, "Finetune/Step": 4607, "Finetune/Step Time": 4.572855239734054} +{"Finetune/Learning Rate": 6.678842793620433e-07, "Finetune/Loss": 0.8102726936340332, "Finetune/Loss (Raw)": 0.8475491404533386, "Finetune/Step": 4608, "Finetune/Step Time": 4.568963818252087} +{"Finetune/Learning Rate": 6.656476206743623e-07, "Finetune/Loss": 0.8118367195129395, "Finetune/Loss (Raw)": 1.0360239744186401, "Finetune/Step": 4609, "Finetune/Step Time": 4.567491101101041} +{"Finetune/Learning Rate": 6.634145844666051e-07, "Finetune/Loss": 0.8119099140167236, "Finetune/Loss (Raw)": 0.8158058524131775, "Finetune/Step": 4610, "Finetune/Step Time": 4.5407060123980045} +{"Finetune/Learning Rate": 6.611851716053641e-07, "Finetune/Loss": 0.8074204325675964, "Finetune/Loss (Raw)": 0.28761807084083557, "Finetune/Step": 4611, "Finetune/Step Time": 4.579552952200174} +{"Finetune/Learning Rate": 6.589593829558316e-07, "Finetune/Loss": 0.8071944713592529, "Finetune/Loss (Raw)": 0.8808144927024841, "Finetune/Step": 4612, "Finetune/Step Time": 4.561023382470012} +{"Finetune/Learning Rate": 6.567372193817967e-07, "Finetune/Loss": 0.8068406581878662, "Finetune/Loss (Raw)": 0.8089052438735962, "Finetune/Step": 4613, "Finetune/Step Time": 4.589684408158064} +{"Finetune/Learning Rate": 6.545186817456339e-07, "Finetune/Loss": 0.8060528039932251, "Finetune/Loss (Raw)": 0.7620059847831726, "Finetune/Step": 4614, "Finetune/Step Time": 4.5963296834379435} +{"Finetune/Learning Rate": 6.523037709083158e-07, "Finetune/Loss": 0.8109736442565918, "Finetune/Loss (Raw)": 0.955651044845581, "Finetune/Step": 4615, "Finetune/Step Time": 4.563401509076357} +{"Finetune/Learning Rate": 6.500924877294057e-07, "Finetune/Loss": 0.8100669384002686, "Finetune/Loss (Raw)": 0.8211730718612671, "Finetune/Step": 4616, "Finetune/Step Time": 4.560541005805135} +{"Finetune/Learning Rate": 6.478848330670607e-07, "Finetune/Loss": 0.810877799987793, "Finetune/Loss (Raw)": 0.7635862827301025, "Finetune/Step": 4617, "Finetune/Step Time": 4.5524503998458385} +{"Finetune/Learning Rate": 6.456808077780263e-07, "Finetune/Loss": 0.8119525909423828, "Finetune/Loss (Raw)": 0.877647876739502, "Finetune/Step": 4618, "Finetune/Step Time": 4.544008569791913} +{"Finetune/Learning Rate": 6.434804127176409e-07, "Finetune/Loss": 0.8136799931526184, "Finetune/Loss (Raw)": 1.0062288045883179, "Finetune/Step": 4619, "Finetune/Step Time": 4.545737588778138} +{"Finetune/Learning Rate": 6.4128364873984e-07, "Finetune/Loss": 0.813014566898346, "Finetune/Loss (Raw)": 0.8192740082740784, "Finetune/Step": 4620, "Finetune/Step Time": 4.5493396278470755} +{"Finetune/Learning Rate": 6.390905166971406e-07, "Finetune/Loss": 0.8144844770431519, "Finetune/Loss (Raw)": 0.9942843317985535, "Finetune/Step": 4621, "Finetune/Step Time": 4.539126856252551} +{"Finetune/Learning Rate": 6.369010174406531e-07, "Finetune/Loss": 0.815870463848114, "Finetune/Loss (Raw)": 0.8767914175987244, "Finetune/Step": 4622, "Finetune/Step Time": 4.535937121137977} +{"Finetune/Learning Rate": 6.347151518200834e-07, "Finetune/Loss": 0.8164845705032349, "Finetune/Loss (Raw)": 0.9741291999816895, "Finetune/Step": 4623, "Finetune/Step Time": 4.533406907692552} +{"Finetune/Learning Rate": 6.325329206837217e-07, "Finetune/Loss": 0.8170249462127686, "Finetune/Loss (Raw)": 0.8372132778167725, "Finetune/Step": 4624, "Finetune/Step Time": 4.526243044063449} +{"Finetune/Learning Rate": 6.303543248784483e-07, "Finetune/Loss": 0.8168088793754578, "Finetune/Loss (Raw)": 0.899021565914154, "Finetune/Step": 4625, "Finetune/Step Time": 4.526934193447232} +{"Finetune/Learning Rate": 6.281793652497359e-07, "Finetune/Loss": 0.8135174512863159, "Finetune/Loss (Raw)": 0.34695515036582947, "Finetune/Step": 4626, "Finetune/Step Time": 4.560211822390556} +{"Finetune/Learning Rate": 6.260080426416415e-07, "Finetune/Loss": 0.81287682056427, "Finetune/Loss (Raw)": 0.8695423007011414, "Finetune/Step": 4627, "Finetune/Step Time": 4.569957887753844} +{"Finetune/Learning Rate": 6.238403578968144e-07, "Finetune/Loss": 0.8123610019683838, "Finetune/Loss (Raw)": 0.783294141292572, "Finetune/Step": 4628, "Finetune/Step Time": 4.573885360732675} +{"Finetune/Learning Rate": 6.216763118564883e-07, "Finetune/Loss": 0.8116129040718079, "Finetune/Loss (Raw)": 0.8474417328834534, "Finetune/Step": 4629, "Finetune/Step Time": 4.571838140487671} +{"Finetune/Learning Rate": 6.195159053604926e-07, "Finetune/Loss": 0.8121604919433594, "Finetune/Loss (Raw)": 0.7924695014953613, "Finetune/Step": 4630, "Finetune/Step Time": 4.580329053103924} +{"Finetune/Learning Rate": 6.173591392472334e-07, "Finetune/Loss": 0.8103449940681458, "Finetune/Loss (Raw)": 0.7099205851554871, "Finetune/Step": 4631, "Finetune/Step Time": 4.5939954705536366} +{"Finetune/Learning Rate": 6.152060143537109e-07, "Finetune/Loss": 0.8058756589889526, "Finetune/Loss (Raw)": 0.2692737281322479, "Finetune/Step": 4632, "Finetune/Step Time": 4.62798242084682} +{"Finetune/Learning Rate": 6.130565315155124e-07, "Finetune/Loss": 0.805850088596344, "Finetune/Loss (Raw)": 0.8163617849349976, "Finetune/Step": 4633, "Finetune/Step Time": 4.640604484826326} +{"Finetune/Learning Rate": 6.109106915668106e-07, "Finetune/Loss": 0.8050683736801147, "Finetune/Loss (Raw)": 0.9063682556152344, "Finetune/Step": 4634, "Finetune/Step Time": 4.640700990334153} +{"Finetune/Learning Rate": 6.087684953403616e-07, "Finetune/Loss": 0.8078078627586365, "Finetune/Loss (Raw)": 0.6798601150512695, "Finetune/Step": 4635, "Finetune/Step Time": 4.621087299659848} +{"Finetune/Learning Rate": 6.066299436675117e-07, "Finetune/Loss": 0.8077943921089172, "Finetune/Loss (Raw)": 0.8596047759056091, "Finetune/Step": 4636, "Finetune/Step Time": 4.62343237362802} +{"Finetune/Learning Rate": 6.04495037378191e-07, "Finetune/Loss": 0.8086550235748291, "Finetune/Loss (Raw)": 0.894815981388092, "Finetune/Step": 4637, "Finetune/Step Time": 4.616750149056315} +{"Finetune/Learning Rate": 6.023637773009139e-07, "Finetune/Loss": 0.8097232580184937, "Finetune/Loss (Raw)": 1.0890969038009644, "Finetune/Step": 4638, "Finetune/Step Time": 4.620198922231793} +{"Finetune/Learning Rate": 6.002361642627807e-07, "Finetune/Loss": 0.8074100017547607, "Finetune/Loss (Raw)": 0.7101947665214539, "Finetune/Step": 4639, "Finetune/Step Time": 4.643710717558861} +{"Finetune/Learning Rate": 5.981121990894789e-07, "Finetune/Loss": 0.8120641112327576, "Finetune/Loss (Raw)": 0.9582718014717102, "Finetune/Step": 4640, "Finetune/Step Time": 4.604838099330664} +{"Finetune/Learning Rate": 5.959918826052757e-07, "Finetune/Loss": 0.8123065829277039, "Finetune/Loss (Raw)": 0.8620162010192871, "Finetune/Step": 4641, "Finetune/Step Time": 4.608868120238185} +{"Finetune/Learning Rate": 5.938752156330241e-07, "Finetune/Loss": 0.8132116198539734, "Finetune/Loss (Raw)": 0.8694958090782166, "Finetune/Step": 4642, "Finetune/Step Time": 4.616115663200617} +{"Finetune/Learning Rate": 5.917621989941624e-07, "Finetune/Loss": 0.813957154750824, "Finetune/Loss (Raw)": 0.9374598264694214, "Finetune/Step": 4643, "Finetune/Step Time": 4.613981219008565} +{"Finetune/Learning Rate": 5.896528335087137e-07, "Finetune/Loss": 0.8108686208724976, "Finetune/Loss (Raw)": 0.3731047809123993, "Finetune/Step": 4644, "Finetune/Step Time": 4.6548601277172565} +{"Finetune/Learning Rate": 5.875471199952754e-07, "Finetune/Loss": 0.8127841949462891, "Finetune/Loss (Raw)": 0.991573691368103, "Finetune/Step": 4645, "Finetune/Step Time": 4.652883805334568} +{"Finetune/Learning Rate": 5.854450592710382e-07, "Finetune/Loss": 0.8109406232833862, "Finetune/Loss (Raw)": 0.7579492926597595, "Finetune/Step": 4646, "Finetune/Step Time": 4.671241957694292} +{"Finetune/Learning Rate": 5.833466521517695e-07, "Finetune/Loss": 0.8105628490447998, "Finetune/Loss (Raw)": 0.8918196558952332, "Finetune/Step": 4647, "Finetune/Step Time": 4.681077988818288} +{"Finetune/Learning Rate": 5.812518994518213e-07, "Finetune/Loss": 0.810418963432312, "Finetune/Loss (Raw)": 0.9100879430770874, "Finetune/Step": 4648, "Finetune/Step Time": 4.685922488570213} +{"Finetune/Learning Rate": 5.791608019841244e-07, "Finetune/Loss": 0.8100965023040771, "Finetune/Loss (Raw)": 0.946567177772522, "Finetune/Step": 4649, "Finetune/Step Time": 4.68225928209722} +{"Finetune/Learning Rate": 5.770733605601953e-07, "Finetune/Loss": 0.809673547744751, "Finetune/Loss (Raw)": 0.7821399569511414, "Finetune/Step": 4650, "Finetune/Step Time": 4.689845273271203} +{"Finetune/Learning Rate": 5.74989575990127e-07, "Finetune/Loss": 0.8096910715103149, "Finetune/Loss (Raw)": 0.8714406490325928, "Finetune/Step": 4651, "Finetune/Step Time": 4.6830413937568665} +{"Finetune/Learning Rate": 5.729094490825948e-07, "Finetune/Loss": 0.8095517158508301, "Finetune/Loss (Raw)": 0.8354343175888062, "Finetune/Step": 4652, "Finetune/Step Time": 4.681173548102379} +{"Finetune/Learning Rate": 5.708329806448587e-07, "Finetune/Loss": 0.8054182529449463, "Finetune/Loss (Raw)": 0.300628662109375, "Finetune/Step": 4653, "Finetune/Step Time": 4.7121013551950455} +{"Finetune/Learning Rate": 5.687601714827551e-07, "Finetune/Loss": 0.8068970441818237, "Finetune/Loss (Raw)": 0.8587986826896667, "Finetune/Step": 4654, "Finetune/Step Time": 4.717170458287001} +{"Finetune/Learning Rate": 5.666910224006971e-07, "Finetune/Loss": 0.8081893920898438, "Finetune/Loss (Raw)": 0.7023934125900269, "Finetune/Step": 4655, "Finetune/Step Time": 4.701289393007755} +{"Finetune/Learning Rate": 5.646255342016848e-07, "Finetune/Loss": 0.8050521016120911, "Finetune/Loss (Raw)": 0.43564602732658386, "Finetune/Step": 4656, "Finetune/Step Time": 4.742481889203191} +{"Finetune/Learning Rate": 5.625637076872914e-07, "Finetune/Loss": 0.8044480085372925, "Finetune/Loss (Raw)": 0.8368967175483704, "Finetune/Step": 4657, "Finetune/Step Time": 4.739649156108499} +{"Finetune/Learning Rate": 5.605055436576746e-07, "Finetune/Loss": 0.8042109608650208, "Finetune/Loss (Raw)": 0.9319441914558411, "Finetune/Step": 4658, "Finetune/Step Time": 4.735163617879152} +{"Finetune/Learning Rate": 5.584510429115641e-07, "Finetune/Loss": 0.8036825656890869, "Finetune/Loss (Raw)": 0.9227094054222107, "Finetune/Step": 4659, "Finetune/Step Time": 4.731867797672749} +{"Finetune/Learning Rate": 5.564002062462737e-07, "Finetune/Loss": 0.8058905005455017, "Finetune/Loss (Raw)": 0.9355467557907104, "Finetune/Step": 4660, "Finetune/Step Time": 4.738130018115044} +{"Finetune/Learning Rate": 5.54353034457693e-07, "Finetune/Loss": 0.8051807880401611, "Finetune/Loss (Raw)": 0.7924399971961975, "Finetune/Step": 4661, "Finetune/Step Time": 4.740156568586826} +{"Finetune/Learning Rate": 5.523095283402868e-07, "Finetune/Loss": 0.8095134496688843, "Finetune/Loss (Raw)": 0.8918238878250122, "Finetune/Step": 4662, "Finetune/Step Time": 4.700343249365687} +{"Finetune/Learning Rate": 5.502696886871039e-07, "Finetune/Loss": 0.8108370304107666, "Finetune/Loss (Raw)": 0.915665864944458, "Finetune/Step": 4663, "Finetune/Step Time": 4.689066210761666} +{"Finetune/Learning Rate": 5.482335162897657e-07, "Finetune/Loss": 0.8108597993850708, "Finetune/Loss (Raw)": 0.865615725517273, "Finetune/Step": 4664, "Finetune/Step Time": 4.6767535004764795} +{"Finetune/Learning Rate": 5.462010119384665e-07, "Finetune/Loss": 0.8102916479110718, "Finetune/Loss (Raw)": 0.7954092025756836, "Finetune/Step": 4665, "Finetune/Step Time": 4.676627518609166} +{"Finetune/Learning Rate": 5.441721764219853e-07, "Finetune/Loss": 0.8088972568511963, "Finetune/Loss (Raw)": 0.7220611572265625, "Finetune/Step": 4666, "Finetune/Step Time": 4.695706939324737} +{"Finetune/Learning Rate": 5.42147010527675e-07, "Finetune/Loss": 0.8104400634765625, "Finetune/Loss (Raw)": 0.9660610556602478, "Finetune/Step": 4667, "Finetune/Step Time": 4.6825906075537205} +{"Finetune/Learning Rate": 5.401255150414598e-07, "Finetune/Loss": 0.814610481262207, "Finetune/Loss (Raw)": 0.8443394303321838, "Finetune/Step": 4668, "Finetune/Step Time": 4.65403095073998} +{"Finetune/Learning Rate": 5.381076907478433e-07, "Finetune/Loss": 0.8113888502120972, "Finetune/Loss (Raw)": 0.3504917621612549, "Finetune/Step": 4669, "Finetune/Step Time": 4.673232721164823} +{"Finetune/Learning Rate": 5.360935384299049e-07, "Finetune/Loss": 0.8125897645950317, "Finetune/Loss (Raw)": 0.8962528109550476, "Finetune/Step": 4670, "Finetune/Step Time": 4.67228738963604} +{"Finetune/Learning Rate": 5.34083058869297e-07, "Finetune/Loss": 0.8123294115066528, "Finetune/Loss (Raw)": 0.8015587329864502, "Finetune/Step": 4671, "Finetune/Step Time": 4.66713278926909} +{"Finetune/Learning Rate": 5.320762528462453e-07, "Finetune/Loss": 0.8120876550674438, "Finetune/Loss (Raw)": 0.8816987872123718, "Finetune/Step": 4672, "Finetune/Step Time": 4.680706111714244} +{"Finetune/Learning Rate": 5.30073121139556e-07, "Finetune/Loss": 0.8115416765213013, "Finetune/Loss (Raw)": 0.8429990410804749, "Finetune/Step": 4673, "Finetune/Step Time": 4.664017511531711} +{"Finetune/Learning Rate": 5.28073664526606e-07, "Finetune/Loss": 0.8115575909614563, "Finetune/Loss (Raw)": 0.9426764845848083, "Finetune/Step": 4674, "Finetune/Step Time": 4.687924163416028} +{"Finetune/Learning Rate": 5.260778837833391e-07, "Finetune/Loss": 0.8112860321998596, "Finetune/Loss (Raw)": 0.8885640501976013, "Finetune/Step": 4675, "Finetune/Step Time": 4.690116768702865} +{"Finetune/Learning Rate": 5.240857796842846e-07, "Finetune/Loss": 0.8132930994033813, "Finetune/Loss (Raw)": 0.8318057060241699, "Finetune/Step": 4676, "Finetune/Step Time": 4.6279989667236805} +{"Finetune/Learning Rate": 5.220973530025375e-07, "Finetune/Loss": 0.8146911263465881, "Finetune/Loss (Raw)": 0.9070838093757629, "Finetune/Step": 4677, "Finetune/Step Time": 4.613057412207127} +{"Finetune/Learning Rate": 5.201126045097671e-07, "Finetune/Loss": 0.81419837474823, "Finetune/Loss (Raw)": 0.9234071969985962, "Finetune/Step": 4678, "Finetune/Step Time": 4.633100681006908} +{"Finetune/Learning Rate": 5.181315349762162e-07, "Finetune/Loss": 0.8128336668014526, "Finetune/Loss (Raw)": 0.7358334064483643, "Finetune/Step": 4679, "Finetune/Step Time": 4.632800994440913} +{"Finetune/Learning Rate": 5.161541451706975e-07, "Finetune/Loss": 0.8154527544975281, "Finetune/Loss (Raw)": 1.018918514251709, "Finetune/Step": 4680, "Finetune/Step Time": 4.640858301892877} +{"Finetune/Learning Rate": 5.141804358605995e-07, "Finetune/Loss": 0.815092921257019, "Finetune/Loss (Raw)": 0.8752295970916748, "Finetune/Step": 4681, "Finetune/Step Time": 4.657764391973615} +{"Finetune/Learning Rate": 5.122104078118773e-07, "Finetune/Loss": 0.8140195608139038, "Finetune/Loss (Raw)": 0.8934426307678223, "Finetune/Step": 4682, "Finetune/Step Time": 4.66855064034462} +{"Finetune/Learning Rate": 5.102440617890658e-07, "Finetune/Loss": 0.8142884969711304, "Finetune/Loss (Raw)": 0.9129301309585571, "Finetune/Step": 4683, "Finetune/Step Time": 4.652526026591659} +{"Finetune/Learning Rate": 5.082813985552604e-07, "Finetune/Loss": 0.8133518695831299, "Finetune/Loss (Raw)": 0.9413291811943054, "Finetune/Step": 4684, "Finetune/Step Time": 4.650246063247323} +{"Finetune/Learning Rate": 5.063224188721327e-07, "Finetune/Loss": 0.8144301176071167, "Finetune/Loss (Raw)": 0.9137972593307495, "Finetune/Step": 4685, "Finetune/Step Time": 4.653059588745236} +{"Finetune/Learning Rate": 5.043671234999271e-07, "Finetune/Loss": 0.8151190280914307, "Finetune/Loss (Raw)": 0.738897979259491, "Finetune/Step": 4686, "Finetune/Step Time": 4.651533721014857} +{"Finetune/Learning Rate": 5.02415513197454e-07, "Finetune/Loss": 0.8144464492797852, "Finetune/Loss (Raw)": 0.8782405853271484, "Finetune/Step": 4687, "Finetune/Step Time": 4.642586138099432} +{"Finetune/Learning Rate": 5.004675887220978e-07, "Finetune/Loss": 0.8164759874343872, "Finetune/Loss (Raw)": 1.0898773670196533, "Finetune/Step": 4688, "Finetune/Step Time": 4.639634793624282} +{"Finetune/Learning Rate": 4.985233508298037e-07, "Finetune/Loss": 0.8171733021736145, "Finetune/Loss (Raw)": 0.9868426322937012, "Finetune/Step": 4689, "Finetune/Step Time": 4.630624113604426} +{"Finetune/Learning Rate": 4.965828002750994e-07, "Finetune/Loss": 0.815950870513916, "Finetune/Loss (Raw)": 0.7834069728851318, "Finetune/Step": 4690, "Finetune/Step Time": 4.644720673561096} +{"Finetune/Learning Rate": 4.946459378110701e-07, "Finetune/Loss": 0.8211503028869629, "Finetune/Loss (Raw)": 0.97217857837677, "Finetune/Step": 4691, "Finetune/Step Time": 4.606372855603695} +{"Finetune/Learning Rate": 4.927127641893759e-07, "Finetune/Loss": 0.8225001096725464, "Finetune/Loss (Raw)": 1.023177981376648, "Finetune/Step": 4692, "Finetune/Step Time": 4.606850240379572} +{"Finetune/Learning Rate": 4.907832801602452e-07, "Finetune/Loss": 0.8233636617660522, "Finetune/Loss (Raw)": 0.9004167318344116, "Finetune/Step": 4693, "Finetune/Step Time": 4.618365969508886} +{"Finetune/Learning Rate": 4.888574864724715e-07, "Finetune/Loss": 0.8245721459388733, "Finetune/Loss (Raw)": 0.9886854290962219, "Finetune/Step": 4694, "Finetune/Step Time": 4.612377850338817} +{"Finetune/Learning Rate": 4.869353838734169e-07, "Finetune/Loss": 0.8246772289276123, "Finetune/Loss (Raw)": 0.9798796772956848, "Finetune/Step": 4695, "Finetune/Step Time": 4.617250023409724} +{"Finetune/Learning Rate": 4.850169731090127e-07, "Finetune/Loss": 0.8235906362533569, "Finetune/Loss (Raw)": 0.7822709083557129, "Finetune/Step": 4696, "Finetune/Step Time": 4.628826098516583} +{"Finetune/Learning Rate": 4.831022549237574e-07, "Finetune/Loss": 0.8225986957550049, "Finetune/Loss (Raw)": 0.8760021328926086, "Finetune/Step": 4697, "Finetune/Step Time": 4.624106999486685} +{"Finetune/Learning Rate": 4.811912300607169e-07, "Finetune/Loss": 0.8225373029708862, "Finetune/Loss (Raw)": 0.9234933257102966, "Finetune/Step": 4698, "Finetune/Step Time": 4.634123658761382} +{"Finetune/Learning Rate": 4.792838992615179e-07, "Finetune/Loss": 0.8235108256340027, "Finetune/Loss (Raw)": 0.9589247703552246, "Finetune/Step": 4699, "Finetune/Step Time": 4.629750965163112} +{"Finetune/Learning Rate": 4.773802632663627e-07, "Finetune/Loss": 0.8238256573677063, "Finetune/Loss (Raw)": 0.8878515362739563, "Finetune/Step": 4700, "Finetune/Step Time": 4.6248014736920595} +{"Finetune/Learning Rate": 4.7548032281401367e-07, "Finetune/Loss": 0.8246575593948364, "Finetune/Loss (Raw)": 0.9060892462730408, "Finetune/Step": 4701, "Finetune/Step Time": 4.6021096259355545} +{"Finetune/Learning Rate": 4.735840786418011e-07, "Finetune/Loss": 0.8242419362068176, "Finetune/Loss (Raw)": 0.8389655947685242, "Finetune/Step": 4702, "Finetune/Step Time": 4.604174247011542} +{"Finetune/Learning Rate": 4.716915314856196e-07, "Finetune/Loss": 0.8255040645599365, "Finetune/Loss (Raw)": 0.9673435688018799, "Finetune/Step": 4703, "Finetune/Step Time": 4.586916845291853} +{"Finetune/Learning Rate": 4.698026820799306e-07, "Finetune/Loss": 0.8251713514328003, "Finetune/Loss (Raw)": 0.7467659711837769, "Finetune/Step": 4704, "Finetune/Step Time": 4.587031090632081} +{"Finetune/Learning Rate": 4.6791753115775884e-07, "Finetune/Loss": 0.824955940246582, "Finetune/Loss (Raw)": 0.9601938128471375, "Finetune/Step": 4705, "Finetune/Step Time": 4.588075764477253} +{"Finetune/Learning Rate": 4.660360794506946e-07, "Finetune/Loss": 0.8239479064941406, "Finetune/Loss (Raw)": 0.8205806016921997, "Finetune/Step": 4706, "Finetune/Step Time": 4.5814425107091665} +{"Finetune/Learning Rate": 4.641583276888939e-07, "Finetune/Loss": 0.8265541791915894, "Finetune/Loss (Raw)": 0.7523587942123413, "Finetune/Step": 4707, "Finetune/Step Time": 4.544235004112124} +{"Finetune/Learning Rate": 4.622842766010749e-07, "Finetune/Loss": 0.8304400444030762, "Finetune/Loss (Raw)": 0.765763521194458, "Finetune/Step": 4708, "Finetune/Step Time": 4.500277766957879} +{"Finetune/Learning Rate": 4.604139269145191e-07, "Finetune/Loss": 0.8296454548835754, "Finetune/Loss (Raw)": 0.7309460043907166, "Finetune/Step": 4709, "Finetune/Step Time": 4.498956965282559} +{"Finetune/Learning Rate": 4.585472793550738e-07, "Finetune/Loss": 0.8304402828216553, "Finetune/Loss (Raw)": 0.9902271628379822, "Finetune/Step": 4710, "Finetune/Step Time": 4.489146010950208} +{"Finetune/Learning Rate": 4.5668433464714835e-07, "Finetune/Loss": 0.8295931220054626, "Finetune/Loss (Raw)": 0.8568422794342041, "Finetune/Step": 4711, "Finetune/Step Time": 4.489570755511522} +{"Finetune/Learning Rate": 4.548250935137144e-07, "Finetune/Loss": 0.8296846747398376, "Finetune/Loss (Raw)": 0.7965449690818787, "Finetune/Step": 4712, "Finetune/Step Time": 4.503062766045332} +{"Finetune/Learning Rate": 4.5296955667630926e-07, "Finetune/Loss": 0.8313422799110413, "Finetune/Loss (Raw)": 0.9541264772415161, "Finetune/Step": 4713, "Finetune/Step Time": 4.505373924970627} +{"Finetune/Learning Rate": 4.51117724855028e-07, "Finetune/Loss": 0.8320209383964539, "Finetune/Loss (Raw)": 0.8270971179008484, "Finetune/Step": 4714, "Finetune/Step Time": 4.508644444867969} +{"Finetune/Learning Rate": 4.492695987685314e-07, "Finetune/Loss": 0.8320453763008118, "Finetune/Loss (Raw)": 0.909153401851654, "Finetune/Step": 4715, "Finetune/Step Time": 4.511489210650325} +{"Finetune/Learning Rate": 4.474251791340412e-07, "Finetune/Loss": 0.8307166695594788, "Finetune/Loss (Raw)": 0.8534631729125977, "Finetune/Step": 4716, "Finetune/Step Time": 4.493204763159156} +{"Finetune/Learning Rate": 4.4558446666734147e-07, "Finetune/Loss": 0.8311201333999634, "Finetune/Loss (Raw)": 0.8869218230247498, "Finetune/Step": 4717, "Finetune/Step Time": 4.49099975079298} +{"Finetune/Learning Rate": 4.437474620827764e-07, "Finetune/Loss": 0.8298583030700684, "Finetune/Loss (Raw)": 0.6468574404716492, "Finetune/Step": 4718, "Finetune/Step Time": 4.504304643720388} +{"Finetune/Learning Rate": 4.419141660932491e-07, "Finetune/Loss": 0.8285993337631226, "Finetune/Loss (Raw)": 0.7525786757469177, "Finetune/Step": 4719, "Finetune/Step Time": 4.504034195095301} +{"Finetune/Learning Rate": 4.4008457941022933e-07, "Finetune/Loss": 0.8309880495071411, "Finetune/Loss (Raw)": 0.5993955731391907, "Finetune/Step": 4720, "Finetune/Step Time": 4.501235608011484} +{"Finetune/Learning Rate": 4.382587027437435e-07, "Finetune/Loss": 0.8300920128822327, "Finetune/Loss (Raw)": 0.7911795377731323, "Finetune/Step": 4721, "Finetune/Step Time": 4.505096573382616} +{"Finetune/Learning Rate": 4.364365368023771e-07, "Finetune/Loss": 0.8333975672721863, "Finetune/Loss (Raw)": 0.8333854675292969, "Finetune/Step": 4722, "Finetune/Step Time": 4.468329977244139} +{"Finetune/Learning Rate": 4.3461808229327773e-07, "Finetune/Loss": 0.833102822303772, "Finetune/Loss (Raw)": 0.8762158751487732, "Finetune/Step": 4723, "Finetune/Step Time": 4.46713101118803} +{"Finetune/Learning Rate": 4.3280333992215206e-07, "Finetune/Loss": 0.8321609497070312, "Finetune/Loss (Raw)": 0.8162181377410889, "Finetune/Step": 4724, "Finetune/Step Time": 4.489164534956217} +{"Finetune/Learning Rate": 4.3099231039326785e-07, "Finetune/Loss": 0.8329712152481079, "Finetune/Loss (Raw)": 0.8601996302604675, "Finetune/Step": 4725, "Finetune/Step Time": 4.491858180612326} +{"Finetune/Learning Rate": 4.291849944094462e-07, "Finetune/Loss": 0.8320746421813965, "Finetune/Loss (Raw)": 0.7442551255226135, "Finetune/Step": 4726, "Finetune/Step Time": 4.494331354275346} +{"Finetune/Learning Rate": 4.273813926720771e-07, "Finetune/Loss": 0.8323257565498352, "Finetune/Loss (Raw)": 0.8305824995040894, "Finetune/Step": 4727, "Finetune/Step Time": 4.490520412102342} +{"Finetune/Learning Rate": 4.2558150588109747e-07, "Finetune/Loss": 0.8328121900558472, "Finetune/Loss (Raw)": 0.8867212533950806, "Finetune/Step": 4728, "Finetune/Step Time": 4.492769438773394} +{"Finetune/Learning Rate": 4.237853347350107e-07, "Finetune/Loss": 0.8353977799415588, "Finetune/Loss (Raw)": 1.0532951354980469, "Finetune/Step": 4729, "Finetune/Step Time": 4.488768681883812} +{"Finetune/Learning Rate": 4.219928799308759e-07, "Finetune/Loss": 0.834292471408844, "Finetune/Loss (Raw)": 0.880499541759491, "Finetune/Step": 4730, "Finetune/Step Time": 4.484148181974888} +{"Finetune/Learning Rate": 4.2020414216431106e-07, "Finetune/Loss": 0.8333152532577515, "Finetune/Loss (Raw)": 0.7784187197685242, "Finetune/Step": 4731, "Finetune/Step Time": 4.482967644929886} +{"Finetune/Learning Rate": 4.184191221294875e-07, "Finetune/Loss": 0.8333377242088318, "Finetune/Loss (Raw)": 0.8433231115341187, "Finetune/Step": 4732, "Finetune/Step Time": 4.483245305716991} +{"Finetune/Learning Rate": 4.16637820519139e-07, "Finetune/Loss": 0.8378755450248718, "Finetune/Loss (Raw)": 0.911893904209137, "Finetune/Step": 4733, "Finetune/Step Time": 4.439806334674358} +{"Finetune/Learning Rate": 4.1486023802455253e-07, "Finetune/Loss": 0.8370808362960815, "Finetune/Loss (Raw)": 0.7894757390022278, "Finetune/Step": 4734, "Finetune/Step Time": 4.445902058854699} +{"Finetune/Learning Rate": 4.1308637533557294e-07, "Finetune/Loss": 0.8380962014198303, "Finetune/Loss (Raw)": 0.9460037350654602, "Finetune/Step": 4735, "Finetune/Step Time": 4.446084437891841} +{"Finetune/Learning Rate": 4.1131623314060175e-07, "Finetune/Loss": 0.8370324969291687, "Finetune/Loss (Raw)": 0.711394190788269, "Finetune/Step": 4736, "Finetune/Step Time": 4.455545281991363} +{"Finetune/Learning Rate": 4.095498121265995e-07, "Finetune/Loss": 0.8350608348846436, "Finetune/Loss (Raw)": 0.7836452126502991, "Finetune/Step": 4737, "Finetune/Step Time": 4.465147703886032} +{"Finetune/Learning Rate": 4.077871129790767e-07, "Finetune/Loss": 0.8363513350486755, "Finetune/Loss (Raw)": 0.9810010194778442, "Finetune/Step": 4738, "Finetune/Step Time": 4.474044067785144} +{"Finetune/Learning Rate": 4.060281363821017e-07, "Finetune/Loss": 0.8402050733566284, "Finetune/Loss (Raw)": 0.7808903455734253, "Finetune/Step": 4739, "Finetune/Step Time": 4.438707038760185} +{"Finetune/Learning Rate": 4.042728830183018e-07, "Finetune/Loss": 0.8397407531738281, "Finetune/Loss (Raw)": 0.8213821053504944, "Finetune/Step": 4740, "Finetune/Step Time": 4.44199038669467} +{"Finetune/Learning Rate": 4.025213535688544e-07, "Finetune/Loss": 0.8387628793716431, "Finetune/Loss (Raw)": 0.6837352514266968, "Finetune/Step": 4741, "Finetune/Step Time": 4.425240272656083} +{"Finetune/Learning Rate": 4.007735487134956e-07, "Finetune/Loss": 0.8387817144393921, "Finetune/Loss (Raw)": 0.764421284198761, "Finetune/Step": 4742, "Finetune/Step Time": 4.435653341934085} +{"Finetune/Learning Rate": 3.99029469130513e-07, "Finetune/Loss": 0.8370141983032227, "Finetune/Loss (Raw)": 0.7294063568115234, "Finetune/Step": 4743, "Finetune/Step Time": 4.4455553106963634} +{"Finetune/Learning Rate": 3.972891154967506e-07, "Finetune/Loss": 0.8384203910827637, "Finetune/Loss (Raw)": 1.001165509223938, "Finetune/Step": 4744, "Finetune/Step Time": 4.446719352155924} +{"Finetune/Learning Rate": 3.95552488487605e-07, "Finetune/Loss": 0.8384626507759094, "Finetune/Loss (Raw)": 0.7689962983131409, "Finetune/Step": 4745, "Finetune/Step Time": 4.465985558927059} +{"Finetune/Learning Rate": 3.9381958877702684e-07, "Finetune/Loss": 0.8377432227134705, "Finetune/Loss (Raw)": 0.7855620980262756, "Finetune/Step": 4746, "Finetune/Step Time": 4.490032583475113} +{"Finetune/Learning Rate": 3.920904170375239e-07, "Finetune/Loss": 0.837868869304657, "Finetune/Loss (Raw)": 1.0223133563995361, "Finetune/Step": 4747, "Finetune/Step Time": 4.4950765408575535} +{"Finetune/Learning Rate": 3.903649739401494e-07, "Finetune/Loss": 0.8387458920478821, "Finetune/Loss (Raw)": 0.9315325021743774, "Finetune/Step": 4748, "Finetune/Step Time": 4.490490393713117} +{"Finetune/Learning Rate": 3.8864326015451536e-07, "Finetune/Loss": 0.8375046253204346, "Finetune/Loss (Raw)": 0.8353977799415588, "Finetune/Step": 4749, "Finetune/Step Time": 4.486966764554381} +{"Finetune/Learning Rate": 3.8692527634878853e-07, "Finetune/Loss": 0.8373541831970215, "Finetune/Loss (Raw)": 0.8575370907783508, "Finetune/Step": 4750, "Finetune/Step Time": 4.48957896232605} +{"Finetune/Learning Rate": 3.8521102318968106e-07, "Finetune/Loss": 0.8365829586982727, "Finetune/Loss (Raw)": 0.8754124045372009, "Finetune/Step": 4751, "Finetune/Step Time": 4.505204018205404} +{"Finetune/Learning Rate": 3.835005013424631e-07, "Finetune/Loss": 0.8370249271392822, "Finetune/Loss (Raw)": 0.8937796354293823, "Finetune/Step": 4752, "Finetune/Step Time": 4.5081185307353735} +{"Finetune/Learning Rate": 3.8179371147095354e-07, "Finetune/Loss": 0.8360105156898499, "Finetune/Loss (Raw)": 0.7691798806190491, "Finetune/Step": 4753, "Finetune/Step Time": 4.499517349526286} +{"Finetune/Learning Rate": 3.80090654237526e-07, "Finetune/Loss": 0.8404194116592407, "Finetune/Loss (Raw)": 0.911303699016571, "Finetune/Step": 4754, "Finetune/Step Time": 4.466390689834952} +{"Finetune/Learning Rate": 3.7839133030310174e-07, "Finetune/Loss": 0.8408524990081787, "Finetune/Loss (Raw)": 0.9249650835990906, "Finetune/Step": 4755, "Finetune/Step Time": 4.466240903362632} +{"Finetune/Learning Rate": 3.766957403271554e-07, "Finetune/Loss": 0.8414967656135559, "Finetune/Loss (Raw)": 0.865767776966095, "Finetune/Step": 4756, "Finetune/Step Time": 4.459005048498511} +{"Finetune/Learning Rate": 3.750038849677162e-07, "Finetune/Loss": 0.8416702747344971, "Finetune/Loss (Raw)": 0.869642972946167, "Finetune/Step": 4757, "Finetune/Step Time": 4.457925982773304} +{"Finetune/Learning Rate": 3.7331576488135456e-07, "Finetune/Loss": 0.8420910835266113, "Finetune/Loss (Raw)": 0.8463466763496399, "Finetune/Step": 4758, "Finetune/Step Time": 4.4485013745725155} +{"Finetune/Learning Rate": 3.716313807231986e-07, "Finetune/Loss": 0.843051016330719, "Finetune/Loss (Raw)": 0.8327823281288147, "Finetune/Step": 4759, "Finetune/Step Time": 4.433770008385181} +{"Finetune/Learning Rate": 3.699507331469276e-07, "Finetune/Loss": 0.8476550579071045, "Finetune/Loss (Raw)": 0.8585885763168335, "Finetune/Step": 4760, "Finetune/Step Time": 4.397148951888084} +{"Finetune/Learning Rate": 3.682738228047655e-07, "Finetune/Loss": 0.8471618890762329, "Finetune/Loss (Raw)": 0.7532415986061096, "Finetune/Step": 4761, "Finetune/Step Time": 4.400009907782078} +{"Finetune/Learning Rate": 3.666006503474895e-07, "Finetune/Loss": 0.8462424874305725, "Finetune/Loss (Raw)": 0.7886866927146912, "Finetune/Step": 4762, "Finetune/Step Time": 4.399061976000667} +{"Finetune/Learning Rate": 3.6493121642442454e-07, "Finetune/Loss": 0.8477603793144226, "Finetune/Loss (Raw)": 0.8741463422775269, "Finetune/Step": 4763, "Finetune/Step Time": 4.383544415235519} +{"Finetune/Learning Rate": 3.6326552168344574e-07, "Finetune/Loss": 0.8476371765136719, "Finetune/Loss (Raw)": 0.843832790851593, "Finetune/Step": 4764, "Finetune/Step Time": 4.387361513450742} +{"Finetune/Learning Rate": 3.616035667709761e-07, "Finetune/Loss": 0.8466439843177795, "Finetune/Loss (Raw)": 0.7676897048950195, "Finetune/Step": 4765, "Finetune/Step Time": 4.387747002765536} +{"Finetune/Learning Rate": 3.5994535233198844e-07, "Finetune/Loss": 0.8461056351661682, "Finetune/Loss (Raw)": 1.0201915502548218, "Finetune/Step": 4766, "Finetune/Step Time": 4.395210929214954} +{"Finetune/Learning Rate": 3.582908790100037e-07, "Finetune/Loss": 0.8474438786506653, "Finetune/Loss (Raw)": 0.8814867734909058, "Finetune/Step": 4767, "Finetune/Step Time": 4.360717054456472} +{"Finetune/Learning Rate": 3.566401474470904e-07, "Finetune/Loss": 0.8462573289871216, "Finetune/Loss (Raw)": 0.8063938021659851, "Finetune/Step": 4768, "Finetune/Step Time": 4.3656773418188095} +{"Finetune/Learning Rate": 3.5499315828386393e-07, "Finetune/Loss": 0.8459336757659912, "Finetune/Loss (Raw)": 0.8205930590629578, "Finetune/Step": 4769, "Finetune/Step Time": 4.359020598232746} +{"Finetune/Learning Rate": 3.533499121594919e-07, "Finetune/Loss": 0.8441001176834106, "Finetune/Loss (Raw)": 0.6347975730895996, "Finetune/Step": 4770, "Finetune/Step Time": 4.356992859393358} +{"Finetune/Learning Rate": 3.5171040971168414e-07, "Finetune/Loss": 0.8444587588310242, "Finetune/Loss (Raw)": 0.9833618998527527, "Finetune/Step": 4771, "Finetune/Step Time": 4.365027707070112} +{"Finetune/Learning Rate": 3.500746515766973e-07, "Finetune/Loss": 0.8475149869918823, "Finetune/Loss (Raw)": 0.7643054723739624, "Finetune/Step": 4772, "Finetune/Step Time": 4.338149260729551} +{"Finetune/Learning Rate": 3.4844263838934023e-07, "Finetune/Loss": 0.844626784324646, "Finetune/Loss (Raw)": 0.6218803524971008, "Finetune/Step": 4773, "Finetune/Step Time": 4.338200064375997} +{"Finetune/Learning Rate": 3.468143707829641e-07, "Finetune/Loss": 0.8455723524093628, "Finetune/Loss (Raw)": 0.8789848685264587, "Finetune/Step": 4774, "Finetune/Step Time": 4.323592383414507} +{"Finetune/Learning Rate": 3.45189849389469e-07, "Finetune/Loss": 0.846118688583374, "Finetune/Loss (Raw)": 0.9617547392845154, "Finetune/Step": 4775, "Finetune/Step Time": 4.315548291429877} +{"Finetune/Learning Rate": 3.4356907483929835e-07, "Finetune/Loss": 0.8452727198600769, "Finetune/Loss (Raw)": 0.801800012588501, "Finetune/Step": 4776, "Finetune/Step Time": 4.334501422941685} +{"Finetune/Learning Rate": 3.419520477614435e-07, "Finetune/Loss": 0.8447731733322144, "Finetune/Loss (Raw)": 0.8826237320899963, "Finetune/Step": 4777, "Finetune/Step Time": 4.335649877786636} +{"Finetune/Learning Rate": 3.403387687834425e-07, "Finetune/Loss": 0.8409228324890137, "Finetune/Loss (Raw)": 0.28930190205574036, "Finetune/Step": 4778, "Finetune/Step Time": 4.364602409303188} +{"Finetune/Learning Rate": 3.3872923853137453e-07, "Finetune/Loss": 0.8419870138168335, "Finetune/Loss (Raw)": 1.0076541900634766, "Finetune/Step": 4779, "Finetune/Step Time": 4.366523498669267} +{"Finetune/Learning Rate": 3.371234576298699e-07, "Finetune/Loss": 0.8419091701507568, "Finetune/Loss (Raw)": 0.8254654407501221, "Finetune/Step": 4780, "Finetune/Step Time": 4.371535474434495} +{"Finetune/Learning Rate": 3.355214267021012e-07, "Finetune/Loss": 0.8457385897636414, "Finetune/Loss (Raw)": 0.7907969951629639, "Finetune/Step": 4781, "Finetune/Step Time": 4.340318817645311} +{"Finetune/Learning Rate": 3.3392314636978117e-07, "Finetune/Loss": 0.8454962968826294, "Finetune/Loss (Raw)": 0.8277927041053772, "Finetune/Step": 4782, "Finetune/Step Time": 4.341839894652367} +{"Finetune/Learning Rate": 3.323286172531748e-07, "Finetune/Loss": 0.8428641557693481, "Finetune/Loss (Raw)": 0.3654715120792389, "Finetune/Step": 4783, "Finetune/Step Time": 4.376466931775212} +{"Finetune/Learning Rate": 3.30737839971087e-07, "Finetune/Loss": 0.8462890982627869, "Finetune/Loss (Raw)": 0.8740398287773132, "Finetune/Step": 4784, "Finetune/Step Time": 4.336648553609848} +{"Finetune/Learning Rate": 3.2915081514086624e-07, "Finetune/Loss": 0.8478612899780273, "Finetune/Loss (Raw)": 1.0381351709365845, "Finetune/Step": 4785, "Finetune/Step Time": 4.338366497308016} +{"Finetune/Learning Rate": 3.2756754337840757e-07, "Finetune/Loss": 0.8475283980369568, "Finetune/Loss (Raw)": 0.8893395662307739, "Finetune/Step": 4786, "Finetune/Step Time": 4.340817291289568} +{"Finetune/Learning Rate": 3.2598802529814734e-07, "Finetune/Loss": 0.8469448089599609, "Finetune/Loss (Raw)": 0.8480017781257629, "Finetune/Step": 4787, "Finetune/Step Time": 4.336491089314222} +{"Finetune/Learning Rate": 3.2441226151306403e-07, "Finetune/Loss": 0.8459805250167847, "Finetune/Loss (Raw)": 0.8121167421340942, "Finetune/Step": 4788, "Finetune/Step Time": 4.345560062676668} +{"Finetune/Learning Rate": 3.228402526346819e-07, "Finetune/Loss": 0.8453562259674072, "Finetune/Loss (Raw)": 0.7125381827354431, "Finetune/Step": 4789, "Finetune/Step Time": 4.344991696998477} +{"Finetune/Learning Rate": 3.212719992730684e-07, "Finetune/Loss": 0.8457257747650146, "Finetune/Loss (Raw)": 0.9391227960586548, "Finetune/Step": 4790, "Finetune/Step Time": 4.350126406177878} +{"Finetune/Learning Rate": 3.197075020368312e-07, "Finetune/Loss": 0.8409358859062195, "Finetune/Loss (Raw)": 0.3025640845298767, "Finetune/Step": 4791, "Finetune/Step Time": 4.390200333669782} +{"Finetune/Learning Rate": 3.1814676153311907e-07, "Finetune/Loss": 0.842104434967041, "Finetune/Loss (Raw)": 1.01518976688385, "Finetune/Step": 4792, "Finetune/Step Time": 4.394297935068607} +{"Finetune/Learning Rate": 3.165897783676275e-07, "Finetune/Loss": 0.8430188894271851, "Finetune/Loss (Raw)": 0.9124575257301331, "Finetune/Step": 4793, "Finetune/Step Time": 4.3928904086351395} +{"Finetune/Learning Rate": 3.1503655314458983e-07, "Finetune/Loss": 0.8437125086784363, "Finetune/Loss (Raw)": 0.8108476400375366, "Finetune/Step": 4794, "Finetune/Step Time": 4.374195199459791} +{"Finetune/Learning Rate": 3.134870864667827e-07, "Finetune/Loss": 0.8422856330871582, "Finetune/Loss (Raw)": 0.7834131121635437, "Finetune/Step": 4795, "Finetune/Step Time": 4.406114799901843} +{"Finetune/Learning Rate": 3.1194137893552523e-07, "Finetune/Loss": 0.8429926633834839, "Finetune/Loss (Raw)": 0.934846043586731, "Finetune/Step": 4796, "Finetune/Step Time": 4.400297217071056} +{"Finetune/Learning Rate": 3.1039943115067417e-07, "Finetune/Loss": 0.8472869992256165, "Finetune/Loss (Raw)": 0.9001637101173401, "Finetune/Step": 4797, "Finetune/Step Time": 4.361489029601216} +{"Finetune/Learning Rate": 3.088612437106309e-07, "Finetune/Loss": 0.8457017540931702, "Finetune/Loss (Raw)": 0.6933459639549255, "Finetune/Step": 4798, "Finetune/Step Time": 4.377817112952471} +{"Finetune/Learning Rate": 3.0732681721233446e-07, "Finetune/Loss": 0.8462802767753601, "Finetune/Loss (Raw)": 0.8756051659584045, "Finetune/Step": 4799, "Finetune/Step Time": 4.383057365193963} +{"Finetune/Learning Rate": 3.057961522512676e-07, "Finetune/Loss": 0.8476625680923462, "Finetune/Loss (Raw)": 1.0586284399032593, "Finetune/Step": 4800, "Finetune/Step Time": 4.366607502102852} +{"Finetune/Learning Rate": 3.0426924942145166e-07, "Finetune/Loss": 0.8483589291572571, "Finetune/Loss (Raw)": 0.9321370124816895, "Finetune/Step": 4801, "Finetune/Step Time": 4.360109461471438} +{"Finetune/Learning Rate": 3.027461093154449e-07, "Finetune/Loss": 0.8478897213935852, "Finetune/Loss (Raw)": 0.8826228976249695, "Finetune/Step": 4802, "Finetune/Step Time": 4.3341077119112015} +{"Finetune/Learning Rate": 3.012267325243512e-07, "Finetune/Loss": 0.8481060862541199, "Finetune/Loss (Raw)": 0.9162524938583374, "Finetune/Step": 4803, "Finetune/Step Time": 4.339179325848818} +{"Finetune/Learning Rate": 2.9971111963780994e-07, "Finetune/Loss": 0.8486529588699341, "Finetune/Loss (Raw)": 0.9018104672431946, "Finetune/Step": 4804, "Finetune/Step Time": 4.341957213357091} +{"Finetune/Learning Rate": 2.981992712440007e-07, "Finetune/Loss": 0.8489868640899658, "Finetune/Loss (Raw)": 0.9498186707496643, "Finetune/Step": 4805, "Finetune/Step Time": 4.348271293565631} +{"Finetune/Learning Rate": 2.96691187929643e-07, "Finetune/Loss": 0.8487727046012878, "Finetune/Loss (Raw)": 0.8959981203079224, "Finetune/Step": 4806, "Finetune/Step Time": 4.329312885180116} +{"Finetune/Learning Rate": 2.9518687027999427e-07, "Finetune/Loss": 0.8495383858680725, "Finetune/Loss (Raw)": 0.8338370323181152, "Finetune/Step": 4807, "Finetune/Step Time": 4.330852707847953} +{"Finetune/Learning Rate": 2.936863188788497e-07, "Finetune/Loss": 0.8490831255912781, "Finetune/Loss (Raw)": 0.9606487154960632, "Finetune/Step": 4808, "Finetune/Step Time": 4.316310152411461} +{"Finetune/Learning Rate": 2.921895343085446e-07, "Finetune/Loss": 0.8474172353744507, "Finetune/Loss (Raw)": 0.6620004773139954, "Finetune/Step": 4809, "Finetune/Step Time": 4.31078509055078} +{"Finetune/Learning Rate": 2.9069651714995426e-07, "Finetune/Loss": 0.8469120264053345, "Finetune/Loss (Raw)": 0.8287732601165771, "Finetune/Step": 4810, "Finetune/Step Time": 4.301723398268223} +{"Finetune/Learning Rate": 2.8920726798248645e-07, "Finetune/Loss": 0.8467708826065063, "Finetune/Loss (Raw)": 0.8948566317558289, "Finetune/Step": 4811, "Finetune/Step Time": 4.322834832593799} +{"Finetune/Learning Rate": 2.8772178738408985e-07, "Finetune/Loss": 0.8462398052215576, "Finetune/Loss (Raw)": 0.8733527660369873, "Finetune/Step": 4812, "Finetune/Step Time": 4.322663310915232} +{"Finetune/Learning Rate": 2.862400759312522e-07, "Finetune/Loss": 0.8428326845169067, "Finetune/Loss (Raw)": 0.4776913821697235, "Finetune/Step": 4813, "Finetune/Step Time": 4.356342785060406} +{"Finetune/Learning Rate": 2.847621341989959e-07, "Finetune/Loss": 0.8438918590545654, "Finetune/Loss (Raw)": 0.8744681477546692, "Finetune/Step": 4814, "Finetune/Step Time": 4.346461718901992} +{"Finetune/Learning Rate": 2.832879627608831e-07, "Finetune/Loss": 0.8443558216094971, "Finetune/Loss (Raw)": 0.9376247525215149, "Finetune/Step": 4815, "Finetune/Step Time": 4.344031248241663} +{"Finetune/Learning Rate": 2.8181756218900626e-07, "Finetune/Loss": 0.8422324061393738, "Finetune/Loss (Raw)": 0.8180803656578064, "Finetune/Step": 4816, "Finetune/Step Time": 4.360313611105084} +{"Finetune/Learning Rate": 2.803509330540033e-07, "Finetune/Loss": 0.8415151834487915, "Finetune/Loss (Raw)": 0.8950424790382385, "Finetune/Step": 4817, "Finetune/Step Time": 4.364701796323061} +{"Finetune/Learning Rate": 2.788880759250423e-07, "Finetune/Loss": 0.842898964881897, "Finetune/Loss (Raw)": 0.9605258703231812, "Finetune/Step": 4818, "Finetune/Step Time": 4.355331284925342} +{"Finetune/Learning Rate": 2.7742899136983026e-07, "Finetune/Loss": 0.842736005783081, "Finetune/Loss (Raw)": 0.9513306617736816, "Finetune/Step": 4819, "Finetune/Step Time": 4.355275949463248} +{"Finetune/Learning Rate": 2.759736799546109e-07, "Finetune/Loss": 0.8418465852737427, "Finetune/Loss (Raw)": 0.9093194603919983, "Finetune/Step": 4820, "Finetune/Step Time": 4.357679694890976} +{"Finetune/Learning Rate": 2.745221422441602e-07, "Finetune/Loss": 0.8421890735626221, "Finetune/Loss (Raw)": 0.9442605972290039, "Finetune/Step": 4821, "Finetune/Step Time": 4.3450015261769295} +{"Finetune/Learning Rate": 2.73074378801792e-07, "Finetune/Loss": 0.8413329720497131, "Finetune/Loss (Raw)": 0.8791072368621826, "Finetune/Step": 4822, "Finetune/Step Time": 4.353167125955224} +{"Finetune/Learning Rate": 2.7163039018935357e-07, "Finetune/Loss": 0.8410341143608093, "Finetune/Loss (Raw)": 0.9416251182556152, "Finetune/Step": 4823, "Finetune/Step Time": 4.351413382217288} +{"Finetune/Learning Rate": 2.701901769672322e-07, "Finetune/Loss": 0.8372244238853455, "Finetune/Loss (Raw)": 0.29462623596191406, "Finetune/Step": 4824, "Finetune/Step Time": 4.374732930213213} +{"Finetune/Learning Rate": 2.687537396943451e-07, "Finetune/Loss": 0.8371038436889648, "Finetune/Loss (Raw)": 0.8605786561965942, "Finetune/Step": 4825, "Finetune/Step Time": 4.37908873334527} +{"Finetune/Learning Rate": 2.673210789281433e-07, "Finetune/Loss": 0.8369281888008118, "Finetune/Loss (Raw)": 0.9010040760040283, "Finetune/Step": 4826, "Finetune/Step Time": 4.372281733900309} +{"Finetune/Learning Rate": 2.658921952246163e-07, "Finetune/Loss": 0.836746871471405, "Finetune/Loss (Raw)": 0.9357126355171204, "Finetune/Step": 4827, "Finetune/Step Time": 4.361862260848284} +{"Finetune/Learning Rate": 2.6446708913828725e-07, "Finetune/Loss": 0.8371927738189697, "Finetune/Loss (Raw)": 0.9449292421340942, "Finetune/Step": 4828, "Finetune/Step Time": 4.368831733241677} +{"Finetune/Learning Rate": 2.6304576122221035e-07, "Finetune/Loss": 0.8366572856903076, "Finetune/Loss (Raw)": 0.8375462293624878, "Finetune/Step": 4829, "Finetune/Step Time": 4.375273102894425} +{"Finetune/Learning Rate": 2.616282120279745e-07, "Finetune/Loss": 0.8369721174240112, "Finetune/Loss (Raw)": 0.879264771938324, "Finetune/Step": 4830, "Finetune/Step Time": 4.373227687552571} +{"Finetune/Learning Rate": 2.602144421057051e-07, "Finetune/Loss": 0.8321927189826965, "Finetune/Loss (Raw)": 0.3555770814418793, "Finetune/Step": 4831, "Finetune/Step Time": 4.416069207713008} +{"Finetune/Learning Rate": 2.5880445200405667e-07, "Finetune/Loss": 0.8322498798370361, "Finetune/Loss (Raw)": 0.754080593585968, "Finetune/Step": 4832, "Finetune/Step Time": 4.433508077636361} +{"Finetune/Learning Rate": 2.573982422702193e-07, "Finetune/Loss": 0.8313509821891785, "Finetune/Loss (Raw)": 0.845138430595398, "Finetune/Step": 4833, "Finetune/Step Time": 4.436817968264222} +{"Finetune/Learning Rate": 2.559958134499163e-07, "Finetune/Loss": 0.8326765298843384, "Finetune/Loss (Raw)": 0.9902559518814087, "Finetune/Step": 4834, "Finetune/Step Time": 4.440920144319534} +{"Finetune/Learning Rate": 2.545971660874036e-07, "Finetune/Loss": 0.8340895175933838, "Finetune/Loss (Raw)": 0.9332144260406494, "Finetune/Step": 4835, "Finetune/Step Time": 4.443019419908524} +{"Finetune/Learning Rate": 2.532023007254658e-07, "Finetune/Loss": 0.8356532454490662, "Finetune/Loss (Raw)": 0.9659205675125122, "Finetune/Step": 4836, "Finetune/Step Time": 4.441402900964022} +{"Finetune/Learning Rate": 2.518112179054266e-07, "Finetune/Loss": 0.8366365432739258, "Finetune/Loss (Raw)": 0.8568026423454285, "Finetune/Step": 4837, "Finetune/Step Time": 4.423075081780553} +{"Finetune/Learning Rate": 2.504239181671353e-07, "Finetune/Loss": 0.8364260196685791, "Finetune/Loss (Raw)": 0.9632925987243652, "Finetune/Step": 4838, "Finetune/Step Time": 4.424290971830487} +{"Finetune/Learning Rate": 2.490404020489767e-07, "Finetune/Loss": 0.8360520601272583, "Finetune/Loss (Raw)": 0.8089690804481506, "Finetune/Step": 4839, "Finetune/Step Time": 4.430916739627719} +{"Finetune/Learning Rate": 2.476606700878681e-07, "Finetune/Loss": 0.8367890119552612, "Finetune/Loss (Raw)": 0.8908789753913879, "Finetune/Step": 4840, "Finetune/Step Time": 4.421437939628959} +{"Finetune/Learning Rate": 2.4628472281925443e-07, "Finetune/Loss": 0.8356850147247314, "Finetune/Loss (Raw)": 0.8128189444541931, "Finetune/Step": 4841, "Finetune/Step Time": 4.4332061018794775} +{"Finetune/Learning Rate": 2.449125607771141e-07, "Finetune/Loss": 0.8358619213104248, "Finetune/Loss (Raw)": 0.8497365713119507, "Finetune/Step": 4842, "Finetune/Step Time": 4.443335454910994} +{"Finetune/Learning Rate": 2.435441844939557e-07, "Finetune/Loss": 0.8369052410125732, "Finetune/Loss (Raw)": 1.0426973104476929, "Finetune/Step": 4843, "Finetune/Step Time": 4.4401641599833965} +{"Finetune/Learning Rate": 2.4217959450082207e-07, "Finetune/Loss": 0.8366541266441345, "Finetune/Loss (Raw)": 0.8213163614273071, "Finetune/Step": 4844, "Finetune/Step Time": 4.454360444098711} +{"Finetune/Learning Rate": 2.408187913272841e-07, "Finetune/Loss": 0.8375728130340576, "Finetune/Loss (Raw)": 1.004514455795288, "Finetune/Step": 4845, "Finetune/Step Time": 4.456369433552027} +{"Finetune/Learning Rate": 2.3946177550143835e-07, "Finetune/Loss": 0.8398010730743408, "Finetune/Loss (Raw)": 0.9320740103721619, "Finetune/Step": 4846, "Finetune/Step Time": 4.467594223096967} +{"Finetune/Learning Rate": 2.3810854754992008e-07, "Finetune/Loss": 0.8364356756210327, "Finetune/Loss (Raw)": 0.3218132257461548, "Finetune/Step": 4847, "Finetune/Step Time": 4.508329447358847} +{"Finetune/Learning Rate": 2.3675910799789036e-07, "Finetune/Loss": 0.8376932144165039, "Finetune/Loss (Raw)": 0.7603530287742615, "Finetune/Step": 4848, "Finetune/Step Time": 4.471617434173822} +{"Finetune/Learning Rate": 2.354134573690392e-07, "Finetune/Loss": 0.8373990058898926, "Finetune/Loss (Raw)": 0.7535182237625122, "Finetune/Step": 4849, "Finetune/Step Time": 4.487314919009805} +{"Finetune/Learning Rate": 2.3407159618558772e-07, "Finetune/Loss": 0.8383615016937256, "Finetune/Loss (Raw)": 0.9565901756286621, "Finetune/Step": 4850, "Finetune/Step Time": 4.49574377015233} +{"Finetune/Learning Rate": 2.3273352496828605e-07, "Finetune/Loss": 0.8386369943618774, "Finetune/Loss (Raw)": 0.9114760756492615, "Finetune/Step": 4851, "Finetune/Step Time": 4.5012539979070425} +{"Finetune/Learning Rate": 2.3139924423641325e-07, "Finetune/Loss": 0.8387051224708557, "Finetune/Loss (Raw)": 0.8249428868293762, "Finetune/Step": 4852, "Finetune/Step Time": 4.493807747960091} +{"Finetune/Learning Rate": 2.3006875450777733e-07, "Finetune/Loss": 0.8398164510726929, "Finetune/Loss (Raw)": 1.002454161643982, "Finetune/Step": 4853, "Finetune/Step Time": 4.500448327511549} +{"Finetune/Learning Rate": 2.287420562987175e-07, "Finetune/Loss": 0.8388173580169678, "Finetune/Loss (Raw)": 0.6163656115531921, "Finetune/Step": 4854, "Finetune/Step Time": 4.503355737775564} +{"Finetune/Learning Rate": 2.2741915012409744e-07, "Finetune/Loss": 0.8393292427062988, "Finetune/Loss (Raw)": 0.8960999250411987, "Finetune/Step": 4855, "Finetune/Step Time": 4.504709277302027} +{"Finetune/Learning Rate": 2.2610003649731093e-07, "Finetune/Loss": 0.8393706679344177, "Finetune/Loss (Raw)": 0.8920281529426575, "Finetune/Step": 4856, "Finetune/Step Time": 4.499472867697477} +{"Finetune/Learning Rate": 2.2478471593028073e-07, "Finetune/Loss": 0.8386430740356445, "Finetune/Loss (Raw)": 0.9601569771766663, "Finetune/Step": 4857, "Finetune/Step Time": 4.4981443248689175} +{"Finetune/Learning Rate": 2.234731889334585e-07, "Finetune/Loss": 0.8395678400993347, "Finetune/Loss (Raw)": 0.9988735914230347, "Finetune/Step": 4858, "Finetune/Step Time": 4.507488712668419} +{"Finetune/Learning Rate": 2.221654560158204e-07, "Finetune/Loss": 0.8404241800308228, "Finetune/Loss (Raw)": 0.8880358338356018, "Finetune/Step": 4859, "Finetune/Step Time": 4.49338211491704} +{"Finetune/Learning Rate": 2.2086151768487273e-07, "Finetune/Loss": 0.8399783372879028, "Finetune/Loss (Raw)": 0.7862547636032104, "Finetune/Step": 4860, "Finetune/Step Time": 4.491971716284752} +{"Finetune/Learning Rate": 2.195613744466485e-07, "Finetune/Loss": 0.8395230770111084, "Finetune/Loss (Raw)": 0.853620171546936, "Finetune/Step": 4861, "Finetune/Step Time": 4.506789145991206} +{"Finetune/Learning Rate": 2.182650268057085e-07, "Finetune/Loss": 0.8408329486846924, "Finetune/Loss (Raw)": 0.9571349024772644, "Finetune/Step": 4862, "Finetune/Step Time": 4.491888131946325} +{"Finetune/Learning Rate": 2.1697247526513698e-07, "Finetune/Loss": 0.8410978317260742, "Finetune/Loss (Raw)": 0.979912281036377, "Finetune/Step": 4863, "Finetune/Step Time": 4.496882503852248} +{"Finetune/Learning Rate": 2.1568372032655384e-07, "Finetune/Loss": 0.8427815437316895, "Finetune/Loss (Raw)": 0.9269083738327026, "Finetune/Step": 4864, "Finetune/Step Time": 4.502191636711359} +{"Finetune/Learning Rate": 2.143987624900945e-07, "Finetune/Loss": 0.8437612056732178, "Finetune/Loss (Raw)": 0.9090367555618286, "Finetune/Step": 4865, "Finetune/Step Time": 4.496351500973105} +{"Finetune/Learning Rate": 2.1311760225442678e-07, "Finetune/Loss": 0.8421354293823242, "Finetune/Loss (Raw)": 0.7729033827781677, "Finetune/Step": 4866, "Finetune/Step Time": 4.488291027024388} +{"Finetune/Learning Rate": 2.118402401167452e-07, "Finetune/Loss": 0.8387773036956787, "Finetune/Loss (Raw)": 0.35105544328689575, "Finetune/Step": 4867, "Finetune/Step Time": 4.527139030396938} +{"Finetune/Learning Rate": 2.1056667657276876e-07, "Finetune/Loss": 0.8382596969604492, "Finetune/Loss (Raw)": 0.7551169991493225, "Finetune/Step": 4868, "Finetune/Step Time": 4.52635277621448} +{"Finetune/Learning Rate": 2.0929691211674319e-07, "Finetune/Loss": 0.8390457630157471, "Finetune/Loss (Raw)": 0.7843589186668396, "Finetune/Step": 4869, "Finetune/Step Time": 4.525346713140607} +{"Finetune/Learning Rate": 2.0803094724143879e-07, "Finetune/Loss": 0.8391715884208679, "Finetune/Loss (Raw)": 0.7805289030075073, "Finetune/Step": 4870, "Finetune/Step Time": 4.5109008606523275} +{"Finetune/Learning Rate": 2.0676878243815145e-07, "Finetune/Loss": 0.8408256769180298, "Finetune/Loss (Raw)": 0.9411263465881348, "Finetune/Step": 4871, "Finetune/Step Time": 4.505313703790307} +{"Finetune/Learning Rate": 2.055104181967027e-07, "Finetune/Loss": 0.8394885063171387, "Finetune/Loss (Raw)": 0.830004870891571, "Finetune/Step": 4872, "Finetune/Step Time": 4.499809343367815} +{"Finetune/Learning Rate": 2.0425585500543854e-07, "Finetune/Loss": 0.8397637009620667, "Finetune/Loss (Raw)": 0.8042252063751221, "Finetune/Step": 4873, "Finetune/Step Time": 4.496884349733591} +{"Finetune/Learning Rate": 2.0300509335123286e-07, "Finetune/Loss": 0.8388286828994751, "Finetune/Loss (Raw)": 0.6658838987350464, "Finetune/Step": 4874, "Finetune/Step Time": 4.4813085943460464} +{"Finetune/Learning Rate": 2.0175813371948072e-07, "Finetune/Loss": 0.8373183012008667, "Finetune/Loss (Raw)": 0.8289823532104492, "Finetune/Step": 4875, "Finetune/Step Time": 4.485324997454882} +{"Finetune/Learning Rate": 2.0051497659409947e-07, "Finetune/Loss": 0.8372832536697388, "Finetune/Loss (Raw)": 0.9270434975624084, "Finetune/Step": 4876, "Finetune/Step Time": 4.478906800970435} +{"Finetune/Learning Rate": 1.9927562245753874e-07, "Finetune/Loss": 0.8383558392524719, "Finetune/Loss (Raw)": 0.972694456577301, "Finetune/Step": 4877, "Finetune/Step Time": 4.486082162708044} +{"Finetune/Learning Rate": 1.9804007179076602e-07, "Finetune/Loss": 0.838227391242981, "Finetune/Loss (Raw)": 0.8410931825637817, "Finetune/Step": 4878, "Finetune/Step Time": 4.493258373811841} +{"Finetune/Learning Rate": 1.968083250732744e-07, "Finetune/Loss": 0.8385106921195984, "Finetune/Loss (Raw)": 0.9116725325584412, "Finetune/Step": 4879, "Finetune/Step Time": 4.479946656152606} +{"Finetune/Learning Rate": 1.9558038278308044e-07, "Finetune/Loss": 0.8389065861701965, "Finetune/Loss (Raw)": 0.944450855255127, "Finetune/Step": 4880, "Finetune/Step Time": 4.476462448015809} +{"Finetune/Learning Rate": 1.9435624539672515e-07, "Finetune/Loss": 0.8411493301391602, "Finetune/Loss (Raw)": 1.0562546253204346, "Finetune/Step": 4881, "Finetune/Step Time": 4.47086077183485} +{"Finetune/Learning Rate": 1.9313591338927185e-07, "Finetune/Loss": 0.8400395512580872, "Finetune/Loss (Raw)": 0.7692558169364929, "Finetune/Step": 4882, "Finetune/Step Time": 4.473001813516021} +{"Finetune/Learning Rate": 1.9191938723430614e-07, "Finetune/Loss": 0.8386737704277039, "Finetune/Loss (Raw)": 0.7501458525657654, "Finetune/Step": 4883, "Finetune/Step Time": 4.464105708524585} +{"Finetune/Learning Rate": 1.9070666740394151e-07, "Finetune/Loss": 0.8395769596099854, "Finetune/Loss (Raw)": 0.9813735485076904, "Finetune/Step": 4884, "Finetune/Step Time": 4.466970467939973} +{"Finetune/Learning Rate": 1.8949775436880924e-07, "Finetune/Loss": 0.8393277525901794, "Finetune/Loss (Raw)": 0.8377474546432495, "Finetune/Step": 4885, "Finetune/Step Time": 4.4708863366395235} +{"Finetune/Learning Rate": 1.8829264859806296e-07, "Finetune/Loss": 0.8402615189552307, "Finetune/Loss (Raw)": 0.9658602476119995, "Finetune/Step": 4886, "Finetune/Step Time": 4.477208001539111} +{"Finetune/Learning Rate": 1.8709135055938298e-07, "Finetune/Loss": 0.8406859636306763, "Finetune/Loss (Raw)": 0.8871150612831116, "Finetune/Step": 4887, "Finetune/Step Time": 4.48374380543828} +{"Finetune/Learning Rate": 1.8589386071896975e-07, "Finetune/Loss": 0.8412775993347168, "Finetune/Loss (Raw)": 0.9343150854110718, "Finetune/Step": 4888, "Finetune/Step Time": 4.493702115491033} +{"Finetune/Learning Rate": 1.8470017954154262e-07, "Finetune/Loss": 0.8421855568885803, "Finetune/Loss (Raw)": 0.8694655299186707, "Finetune/Step": 4889, "Finetune/Step Time": 4.482973352074623} +{"Finetune/Learning Rate": 1.8351030749034992e-07, "Finetune/Loss": 0.8426705598831177, "Finetune/Loss (Raw)": 0.8507637977600098, "Finetune/Step": 4890, "Finetune/Step Time": 4.5020211935043335} +{"Finetune/Learning Rate": 1.8232424502715672e-07, "Finetune/Loss": 0.8419698476791382, "Finetune/Loss (Raw)": 0.7844574451446533, "Finetune/Step": 4891, "Finetune/Step Time": 4.499285686761141} +{"Finetune/Learning Rate": 1.811419926122493e-07, "Finetune/Loss": 0.8422130346298218, "Finetune/Loss (Raw)": 0.8749567866325378, "Finetune/Step": 4892, "Finetune/Step Time": 4.495003834366798} +{"Finetune/Learning Rate": 1.7996355070443727e-07, "Finetune/Loss": 0.8439979553222656, "Finetune/Loss (Raw)": 0.996168315410614, "Finetune/Step": 4893, "Finetune/Step Time": 4.4947399608790874} +{"Finetune/Learning Rate": 1.787889197610515e-07, "Finetune/Loss": 0.8424110412597656, "Finetune/Loss (Raw)": 0.8170647621154785, "Finetune/Step": 4894, "Finetune/Step Time": 4.495727909728885} +{"Finetune/Learning Rate": 1.776181002379429e-07, "Finetune/Loss": 0.8416292071342468, "Finetune/Loss (Raw)": 0.7814108729362488, "Finetune/Step": 4895, "Finetune/Step Time": 4.497472412884235} +{"Finetune/Learning Rate": 1.7645109258948245e-07, "Finetune/Loss": 0.8425617218017578, "Finetune/Loss (Raw)": 0.9257524013519287, "Finetune/Step": 4896, "Finetune/Step Time": 4.489774089306593} +{"Finetune/Learning Rate": 1.7528789726856566e-07, "Finetune/Loss": 0.8415658473968506, "Finetune/Loss (Raw)": 0.693119466304779, "Finetune/Step": 4897, "Finetune/Step Time": 4.495401242747903} +{"Finetune/Learning Rate": 1.7412851472660585e-07, "Finetune/Loss": 0.84257972240448, "Finetune/Loss (Raw)": 0.7645747661590576, "Finetune/Step": 4898, "Finetune/Step Time": 4.496410857886076} +{"Finetune/Learning Rate": 1.7297294541353317e-07, "Finetune/Loss": 0.8424456119537354, "Finetune/Loss (Raw)": 0.966206431388855, "Finetune/Step": 4899, "Finetune/Step Time": 4.493881015107036} +{"Finetune/Learning Rate": 1.7182118977780548e-07, "Finetune/Loss": 0.8431013822555542, "Finetune/Loss (Raw)": 0.8482380509376526, "Finetune/Step": 4900, "Finetune/Step Time": 4.480985211208463} +{"Finetune/Learning Rate": 1.706732482663942e-07, "Finetune/Loss": 0.8447107076644897, "Finetune/Loss (Raw)": 0.8278779983520508, "Finetune/Step": 4901, "Finetune/Step Time": 4.479367513209581} +{"Finetune/Learning Rate": 1.6952912132479404e-07, "Finetune/Loss": 0.8442631363868713, "Finetune/Loss (Raw)": 0.8216887712478638, "Finetune/Step": 4902, "Finetune/Step Time": 4.478283917531371} +{"Finetune/Learning Rate": 1.6838880939701875e-07, "Finetune/Loss": 0.8432474732398987, "Finetune/Loss (Raw)": 0.8317531943321228, "Finetune/Step": 4903, "Finetune/Step Time": 4.478078909218311} +{"Finetune/Learning Rate": 1.6725231292559985e-07, "Finetune/Loss": 0.8436330556869507, "Finetune/Loss (Raw)": 0.8511459231376648, "Finetune/Step": 4904, "Finetune/Step Time": 4.456969076767564} +{"Finetune/Learning Rate": 1.661196323515901e-07, "Finetune/Loss": 0.8439908027648926, "Finetune/Loss (Raw)": 0.9284193515777588, "Finetune/Step": 4905, "Finetune/Step Time": 4.460764076560736} +{"Finetune/Learning Rate": 1.6499076811456015e-07, "Finetune/Loss": 0.8482470512390137, "Finetune/Loss (Raw)": 0.8341044783592224, "Finetune/Step": 4906, "Finetune/Step Time": 4.424257267266512} +{"Finetune/Learning Rate": 1.6386572065260065e-07, "Finetune/Loss": 0.8463060259819031, "Finetune/Loss (Raw)": 0.7592064142227173, "Finetune/Step": 4907, "Finetune/Step Time": 4.4313237108290195} +{"Finetune/Learning Rate": 1.6274449040232233e-07, "Finetune/Loss": 0.8467649817466736, "Finetune/Loss (Raw)": 0.8842090964317322, "Finetune/Step": 4908, "Finetune/Step Time": 4.449291596189141} +{"Finetune/Learning Rate": 1.6162707779884933e-07, "Finetune/Loss": 0.847240686416626, "Finetune/Loss (Raw)": 0.8516892194747925, "Finetune/Step": 4909, "Finetune/Step Time": 4.451202850788832} +{"Finetune/Learning Rate": 1.6051348327583038e-07, "Finetune/Loss": 0.8477632999420166, "Finetune/Loss (Raw)": 0.8946823477745056, "Finetune/Step": 4910, "Finetune/Step Time": 4.440980549901724} +{"Finetune/Learning Rate": 1.5940370726542864e-07, "Finetune/Loss": 0.8514670729637146, "Finetune/Loss (Raw)": 0.8395571112632751, "Finetune/Step": 4911, "Finetune/Step Time": 4.40146054700017} +{"Finetune/Learning Rate": 1.5829775019832628e-07, "Finetune/Loss": 0.8516553640365601, "Finetune/Loss (Raw)": 0.8981361389160156, "Finetune/Step": 4912, "Finetune/Step Time": 4.4083979446440935} +{"Finetune/Learning Rate": 1.571956125037244e-07, "Finetune/Loss": 0.8507994413375854, "Finetune/Loss (Raw)": 0.9285866618156433, "Finetune/Step": 4913, "Finetune/Step Time": 4.416864277794957} +{"Finetune/Learning Rate": 1.5609729460934087e-07, "Finetune/Loss": 0.8470504283905029, "Finetune/Loss (Raw)": 0.4094543755054474, "Finetune/Step": 4914, "Finetune/Step Time": 4.452770793810487} +{"Finetune/Learning Rate": 1.550027969414114e-07, "Finetune/Loss": 0.8462193012237549, "Finetune/Loss (Raw)": 0.7416256666183472, "Finetune/Step": 4915, "Finetune/Step Time": 4.455914434045553} +{"Finetune/Learning Rate": 1.5391211992468736e-07, "Finetune/Loss": 0.846514105796814, "Finetune/Loss (Raw)": 0.8498529195785522, "Finetune/Step": 4916, "Finetune/Step Time": 4.429728923365474} +{"Finetune/Learning Rate": 1.5282526398244236e-07, "Finetune/Loss": 0.8441405892372131, "Finetune/Loss (Raw)": 0.40872514247894287, "Finetune/Step": 4917, "Finetune/Step Time": 4.46596547216177} +{"Finetune/Learning Rate": 1.5174222953646344e-07, "Finetune/Loss": 0.8437317609786987, "Finetune/Loss (Raw)": 0.8867931365966797, "Finetune/Step": 4918, "Finetune/Step Time": 4.46047087199986} +{"Finetune/Learning Rate": 1.5066301700705334e-07, "Finetune/Loss": 0.84807288646698, "Finetune/Loss (Raw)": 0.8582237362861633, "Finetune/Step": 4919, "Finetune/Step Time": 4.422756880521774} +{"Finetune/Learning Rate": 1.4958762681303362e-07, "Finetune/Loss": 0.8472270965576172, "Finetune/Loss (Raw)": 0.9069347381591797, "Finetune/Step": 4920, "Finetune/Step Time": 4.418551757931709} +{"Finetune/Learning Rate": 1.4851605937174384e-07, "Finetune/Loss": 0.8471707105636597, "Finetune/Loss (Raw)": 0.9052343964576721, "Finetune/Step": 4921, "Finetune/Step Time": 4.431460320949554} +{"Finetune/Learning Rate": 1.474483150990369e-07, "Finetune/Loss": 0.8467559814453125, "Finetune/Loss (Raw)": 0.7577613592147827, "Finetune/Step": 4922, "Finetune/Step Time": 4.432677136734128} +{"Finetune/Learning Rate": 1.4638439440928354e-07, "Finetune/Loss": 0.8469275236129761, "Finetune/Loss (Raw)": 0.8053774237632751, "Finetune/Step": 4923, "Finetune/Step Time": 4.407121933996677} +{"Finetune/Learning Rate": 1.4532429771537237e-07, "Finetune/Loss": 0.8463689684867859, "Finetune/Loss (Raw)": 0.8633520007133484, "Finetune/Step": 4924, "Finetune/Step Time": 4.407989002764225} +{"Finetune/Learning Rate": 1.4426802542870433e-07, "Finetune/Loss": 0.845761775970459, "Finetune/Loss (Raw)": 0.8224401473999023, "Finetune/Step": 4925, "Finetune/Step Time": 4.407827703282237} +{"Finetune/Learning Rate": 1.4321557795919705e-07, "Finetune/Loss": 0.8466768860816956, "Finetune/Loss (Raw)": 0.8104861378669739, "Finetune/Step": 4926, "Finetune/Step Time": 4.391969388350844} +{"Finetune/Learning Rate": 1.4216695571528828e-07, "Finetune/Loss": 0.8461397290229797, "Finetune/Loss (Raw)": 0.8068429231643677, "Finetune/Step": 4927, "Finetune/Step Time": 4.389985598623753} +{"Finetune/Learning Rate": 1.411221591039269e-07, "Finetune/Loss": 0.8429707288742065, "Finetune/Loss (Raw)": 0.6530005931854248, "Finetune/Step": 4928, "Finetune/Step Time": 4.412873659282923} +{"Finetune/Learning Rate": 1.4008118853057529e-07, "Finetune/Loss": 0.8421712517738342, "Finetune/Loss (Raw)": 0.8298006057739258, "Finetune/Step": 4929, "Finetune/Step Time": 4.41710814088583} +{"Finetune/Learning Rate": 1.3904404439921582e-07, "Finetune/Loss": 0.8421817421913147, "Finetune/Loss (Raw)": 0.8839640617370605, "Finetune/Step": 4930, "Finetune/Step Time": 4.4162584859877825} +{"Finetune/Learning Rate": 1.3801072711234541e-07, "Finetune/Loss": 0.8423614501953125, "Finetune/Loss (Raw)": 0.9392565488815308, "Finetune/Step": 4931, "Finetune/Step Time": 4.413411593064666} +{"Finetune/Learning Rate": 1.369812370709711e-07, "Finetune/Loss": 0.8423367142677307, "Finetune/Loss (Raw)": 0.8986462354660034, "Finetune/Step": 4932, "Finetune/Step Time": 4.41075543127954} +{"Finetune/Learning Rate": 1.3595557467461994e-07, "Finetune/Loss": 0.8411194086074829, "Finetune/Loss (Raw)": 0.7940039038658142, "Finetune/Step": 4933, "Finetune/Step Time": 4.420999715104699} +{"Finetune/Learning Rate": 1.3493374032133134e-07, "Finetune/Loss": 0.8413324356079102, "Finetune/Loss (Raw)": 0.9232685565948486, "Finetune/Step": 4934, "Finetune/Step Time": 4.416061948984861} +{"Finetune/Learning Rate": 1.3391573440765915e-07, "Finetune/Loss": 0.8421087861061096, "Finetune/Loss (Raw)": 0.93320232629776, "Finetune/Step": 4935, "Finetune/Step Time": 4.4147983603179455} +{"Finetune/Learning Rate": 1.329015573286696e-07, "Finetune/Loss": 0.8407409191131592, "Finetune/Loss (Raw)": 0.7855639457702637, "Finetune/Step": 4936, "Finetune/Step Time": 4.4198071248829365} +{"Finetune/Learning Rate": 1.3189120947794897e-07, "Finetune/Loss": 0.8434544801712036, "Finetune/Loss (Raw)": 1.009336233139038, "Finetune/Step": 4937, "Finetune/Step Time": 4.407648349180818} +{"Finetune/Learning Rate": 1.308846912475914e-07, "Finetune/Loss": 0.8440766334533691, "Finetune/Loss (Raw)": 0.9084092378616333, "Finetune/Step": 4938, "Finetune/Step Time": 4.419258244335651} +{"Finetune/Learning Rate": 1.2988200302820552e-07, "Finetune/Loss": 0.8397579789161682, "Finetune/Loss (Raw)": 0.3420681655406952, "Finetune/Step": 4939, "Finetune/Step Time": 4.432147378101945} +{"Finetune/Learning Rate": 1.2888314520891676e-07, "Finetune/Loss": 0.838298499584198, "Finetune/Loss (Raw)": 0.686539351940155, "Finetune/Step": 4940, "Finetune/Step Time": 4.432740367949009} +{"Finetune/Learning Rate": 1.2788811817736168e-07, "Finetune/Loss": 0.8404427766799927, "Finetune/Loss (Raw)": 0.7521637082099915, "Finetune/Step": 4941, "Finetune/Step Time": 4.395218323916197} +{"Finetune/Learning Rate": 1.2689692231969364e-07, "Finetune/Loss": 0.8399729132652283, "Finetune/Loss (Raw)": 0.8143206834793091, "Finetune/Step": 4942, "Finetune/Step Time": 4.401745880022645} +{"Finetune/Learning Rate": 1.2590955802057047e-07, "Finetune/Loss": 0.83877032995224, "Finetune/Loss (Raw)": 0.7836960554122925, "Finetune/Step": 4943, "Finetune/Step Time": 4.418382128700614} +{"Finetune/Learning Rate": 1.2492602566317457e-07, "Finetune/Loss": 0.8395471572875977, "Finetune/Loss (Raw)": 0.9175142645835876, "Finetune/Step": 4944, "Finetune/Step Time": 4.4106190372258425} +{"Finetune/Learning Rate": 1.2394632562919283e-07, "Finetune/Loss": 0.8399147391319275, "Finetune/Loss (Raw)": 0.942088782787323, "Finetune/Step": 4945, "Finetune/Step Time": 4.410639168694615} +{"Finetune/Learning Rate": 1.2297045829882892e-07, "Finetune/Loss": 0.8391119837760925, "Finetune/Loss (Raw)": 0.857774019241333, "Finetune/Step": 4946, "Finetune/Step Time": 4.416197065263987} +{"Finetune/Learning Rate": 1.2199842405079766e-07, "Finetune/Loss": 0.8388038277626038, "Finetune/Loss (Raw)": 0.9118853211402893, "Finetune/Step": 4947, "Finetune/Step Time": 4.417444065213203} +{"Finetune/Learning Rate": 1.2103022326232505e-07, "Finetune/Loss": 0.8380763530731201, "Finetune/Loss (Raw)": 0.8162070512771606, "Finetune/Step": 4948, "Finetune/Step Time": 4.415048234164715} +{"Finetune/Learning Rate": 1.2006585630915391e-07, "Finetune/Loss": 0.836979866027832, "Finetune/Loss (Raw)": 0.8039116263389587, "Finetune/Step": 4949, "Finetune/Step Time": 4.413096658885479} +{"Finetune/Learning Rate": 1.1910532356553261e-07, "Finetune/Loss": 0.8364455699920654, "Finetune/Loss (Raw)": 0.8107134699821472, "Finetune/Step": 4950, "Finetune/Step Time": 4.419961614534259} +{"Finetune/Learning Rate": 1.1814862540422744e-07, "Finetune/Loss": 0.8361732959747314, "Finetune/Loss (Raw)": 0.9067696928977966, "Finetune/Step": 4951, "Finetune/Step Time": 4.418600661680102} +{"Finetune/Learning Rate": 1.1719576219651585e-07, "Finetune/Loss": 0.840971827507019, "Finetune/Loss (Raw)": 0.9088477492332458, "Finetune/Step": 4952, "Finetune/Step Time": 4.3899230025708675} +{"Finetune/Learning Rate": 1.1624673431218092e-07, "Finetune/Loss": 0.8406202793121338, "Finetune/Loss (Raw)": 0.815578043460846, "Finetune/Step": 4953, "Finetune/Step Time": 4.3909277599304914} +{"Finetune/Learning Rate": 1.1530154211952583e-07, "Finetune/Loss": 0.8407349586486816, "Finetune/Loss (Raw)": 0.915683388710022, "Finetune/Step": 4954, "Finetune/Step Time": 4.44202215410769} +{"Finetune/Learning Rate": 1.1436018598535937e-07, "Finetune/Loss": 0.8401814103126526, "Finetune/Loss (Raw)": 0.8648529052734375, "Finetune/Step": 4955, "Finetune/Step Time": 4.444794775918126} +{"Finetune/Learning Rate": 1.1342266627500375e-07, "Finetune/Loss": 0.8400875329971313, "Finetune/Loss (Raw)": 0.9329100847244263, "Finetune/Step": 4956, "Finetune/Step Time": 4.432483920827508} +{"Finetune/Learning Rate": 1.1248898335229241e-07, "Finetune/Loss": 0.8411601781845093, "Finetune/Loss (Raw)": 0.9748468399047852, "Finetune/Step": 4957, "Finetune/Step Time": 4.422183791175485} +{"Finetune/Learning Rate": 1.1155913757956881e-07, "Finetune/Loss": 0.8405971527099609, "Finetune/Loss (Raw)": 0.807195782661438, "Finetune/Step": 4958, "Finetune/Step Time": 4.439286019653082} +{"Finetune/Learning Rate": 1.1063312931768877e-07, "Finetune/Loss": 0.8412717580795288, "Finetune/Loss (Raw)": 0.44193512201309204, "Finetune/Step": 4959, "Finetune/Step Time": 4.438771249726415} +{"Finetune/Learning Rate": 1.0971095892601702e-07, "Finetune/Loss": 0.8418334722518921, "Finetune/Loss (Raw)": 0.8259750008583069, "Finetune/Step": 4960, "Finetune/Step Time": 4.412164920940995} +{"Finetune/Learning Rate": 1.0879262676243174e-07, "Finetune/Loss": 0.8415561318397522, "Finetune/Loss (Raw)": 0.8096376061439514, "Finetune/Step": 4961, "Finetune/Step Time": 4.410095019266009} +{"Finetune/Learning Rate": 1.0787813318331786e-07, "Finetune/Loss": 0.8410043120384216, "Finetune/Loss (Raw)": 0.919625997543335, "Finetune/Step": 4962, "Finetune/Step Time": 4.409219237044454} +{"Finetune/Learning Rate": 1.0696747854357259e-07, "Finetune/Loss": 0.8403270244598389, "Finetune/Loss (Raw)": 0.8465191721916199, "Finetune/Step": 4963, "Finetune/Step Time": 4.408073401078582} +{"Finetune/Learning Rate": 1.0606066319660435e-07, "Finetune/Loss": 0.840106725692749, "Finetune/Loss (Raw)": 0.9377276301383972, "Finetune/Step": 4964, "Finetune/Step Time": 4.41159924864769} +{"Finetune/Learning Rate": 1.0515768749432942e-07, "Finetune/Loss": 0.8409655690193176, "Finetune/Loss (Raw)": 0.9667333364486694, "Finetune/Step": 4965, "Finetune/Step Time": 4.412332469597459} +{"Finetune/Learning Rate": 1.0425855178717636e-07, "Finetune/Loss": 0.839688777923584, "Finetune/Loss (Raw)": 0.7998618483543396, "Finetune/Step": 4966, "Finetune/Step Time": 4.415636571124196} +{"Finetune/Learning Rate": 1.0336325642408052e-07, "Finetune/Loss": 0.8406146764755249, "Finetune/Loss (Raw)": 0.9274808168411255, "Finetune/Step": 4967, "Finetune/Step Time": 4.407786751165986} +{"Finetune/Learning Rate": 1.024718017524895e-07, "Finetune/Loss": 0.8405636548995972, "Finetune/Loss (Raw)": 0.8843539357185364, "Finetune/Step": 4968, "Finetune/Step Time": 4.402069620788097} +{"Finetune/Learning Rate": 1.015841881183588e-07, "Finetune/Loss": 0.8410347104072571, "Finetune/Loss (Raw)": 0.8731107115745544, "Finetune/Step": 4969, "Finetune/Step Time": 4.399732757359743} +{"Finetune/Learning Rate": 1.0070041586615398e-07, "Finetune/Loss": 0.8417377471923828, "Finetune/Loss (Raw)": 0.9397241473197937, "Finetune/Step": 4970, "Finetune/Step Time": 4.3819189835339785} +{"Finetune/Learning Rate": 9.98204853388518e-08, "Finetune/Loss": 0.8414291143417358, "Finetune/Loss (Raw)": 1.0031930208206177, "Finetune/Step": 4971, "Finetune/Step Time": 4.393169149756432} +{"Finetune/Learning Rate": 9.894439687793466e-08, "Finetune/Loss": 0.8417137265205383, "Finetune/Loss (Raw)": 0.8577412962913513, "Finetune/Step": 4972, "Finetune/Step Time": 4.392069324851036} +{"Finetune/Learning Rate": 9.807215082339395e-08, "Finetune/Loss": 0.8416393995285034, "Finetune/Loss (Raw)": 0.9950112104415894, "Finetune/Step": 4973, "Finetune/Step Time": 4.392253803089261} +{"Finetune/Learning Rate": 9.720374751373329e-08, "Finetune/Loss": 0.8367338180541992, "Finetune/Loss (Raw)": 0.30415481328964233, "Finetune/Step": 4974, "Finetune/Step Time": 4.4036681316792965} +{"Finetune/Learning Rate": 9.633918728596203e-08, "Finetune/Loss": 0.8404766321182251, "Finetune/Loss (Raw)": 0.8008959293365479, "Finetune/Step": 4975, "Finetune/Step Time": 4.368091827258468} +{"Finetune/Learning Rate": 9.547847047559955e-08, "Finetune/Loss": 0.842262864112854, "Finetune/Loss (Raw)": 0.9889919757843018, "Finetune/Step": 4976, "Finetune/Step Time": 4.370712300762534} +{"Finetune/Learning Rate": 9.462159741667309e-08, "Finetune/Loss": 0.8421977758407593, "Finetune/Loss (Raw)": 0.74518883228302, "Finetune/Step": 4977, "Finetune/Step Time": 4.369935506954789} +{"Finetune/Learning Rate": 9.376856844171889e-08, "Finetune/Loss": 0.8410014510154724, "Finetune/Loss (Raw)": 0.8034592866897583, "Finetune/Step": 4978, "Finetune/Step Time": 4.365876909345388} +{"Finetune/Learning Rate": 9.291938388178102e-08, "Finetune/Loss": 0.8401511907577515, "Finetune/Loss (Raw)": 0.802631676197052, "Finetune/Step": 4979, "Finetune/Step Time": 4.367782151326537} +{"Finetune/Learning Rate": 9.207404406640919e-08, "Finetune/Loss": 0.8391927480697632, "Finetune/Loss (Raw)": 0.7022668719291687, "Finetune/Step": 4980, "Finetune/Step Time": 4.361031571403146} +{"Finetune/Learning Rate": 9.123254932366655e-08, "Finetune/Loss": 0.8373611569404602, "Finetune/Loss (Raw)": 0.7680152058601379, "Finetune/Step": 4981, "Finetune/Step Time": 4.3577572628855705} +{"Finetune/Learning Rate": 9.039489998011853e-08, "Finetune/Loss": 0.8399841785430908, "Finetune/Loss (Raw)": 0.9521112442016602, "Finetune/Step": 4982, "Finetune/Step Time": 4.3594826515764} +{"Finetune/Learning Rate": 8.956109636084065e-08, "Finetune/Loss": 0.8362162709236145, "Finetune/Loss (Raw)": 0.41380390524864197, "Finetune/Step": 4983, "Finetune/Step Time": 4.397124340757728} +{"Finetune/Learning Rate": 8.873113878941631e-08, "Finetune/Loss": 0.8354170322418213, "Finetune/Loss (Raw)": 0.7897238731384277, "Finetune/Step": 4984, "Finetune/Step Time": 4.40828868560493} +{"Finetune/Learning Rate": 8.790502758793562e-08, "Finetune/Loss": 0.8358616232872009, "Finetune/Loss (Raw)": 1.017071008682251, "Finetune/Step": 4985, "Finetune/Step Time": 4.404908388853073} +{"Finetune/Learning Rate": 8.70827630769977e-08, "Finetune/Loss": 0.8335120677947998, "Finetune/Loss (Raw)": 0.698126494884491, "Finetune/Step": 4986, "Finetune/Step Time": 4.397220220416784} +{"Finetune/Learning Rate": 8.626434557570507e-08, "Finetune/Loss": 0.8337658643722534, "Finetune/Loss (Raw)": 0.9205241799354553, "Finetune/Step": 4987, "Finetune/Step Time": 4.404603622853756} +{"Finetune/Learning Rate": 8.544977540167032e-08, "Finetune/Loss": 0.8349438309669495, "Finetune/Loss (Raw)": 0.9370279908180237, "Finetune/Step": 4988, "Finetune/Step Time": 4.412787409499288} +{"Finetune/Learning Rate": 8.463905287101282e-08, "Finetune/Loss": 0.8354259729385376, "Finetune/Loss (Raw)": 0.9153469800949097, "Finetune/Step": 4989, "Finetune/Step Time": 4.404222344979644} +{"Finetune/Learning Rate": 8.383217829835755e-08, "Finetune/Loss": 0.8346506357192993, "Finetune/Loss (Raw)": 0.8578917980194092, "Finetune/Step": 4990, "Finetune/Step Time": 4.407226134091616} +{"Finetune/Learning Rate": 8.302915199683737e-08, "Finetune/Loss": 0.8334447145462036, "Finetune/Loss (Raw)": 0.8255525827407837, "Finetune/Step": 4991, "Finetune/Step Time": 4.400812711566687} +{"Finetune/Learning Rate": 8.222997427809076e-08, "Finetune/Loss": 0.8338992595672607, "Finetune/Loss (Raw)": 0.9850815534591675, "Finetune/Step": 4992, "Finetune/Step Time": 4.401040144264698} +{"Finetune/Learning Rate": 8.143464545226298e-08, "Finetune/Loss": 0.831484854221344, "Finetune/Loss (Raw)": 0.5999979972839355, "Finetune/Step": 4993, "Finetune/Step Time": 4.414822658523917} +{"Finetune/Learning Rate": 8.064316582800491e-08, "Finetune/Loss": 0.8319645524024963, "Finetune/Loss (Raw)": 0.8343085050582886, "Finetune/Step": 4994, "Finetune/Step Time": 4.421417731791735} +{"Finetune/Learning Rate": 7.98555357124764e-08, "Finetune/Loss": 0.834060788154602, "Finetune/Loss (Raw)": 0.619364321231842, "Finetune/Step": 4995, "Finetune/Step Time": 4.385816374793649} +{"Finetune/Learning Rate": 7.907175541133849e-08, "Finetune/Loss": 0.8360230922698975, "Finetune/Loss (Raw)": 1.0062899589538574, "Finetune/Step": 4996, "Finetune/Step Time": 4.386916346848011} +{"Finetune/Learning Rate": 7.829182522876344e-08, "Finetune/Loss": 0.8371192216873169, "Finetune/Loss (Raw)": 0.9246757626533508, "Finetune/Step": 4997, "Finetune/Step Time": 4.371597155928612} +{"Finetune/Learning Rate": 7.75157454674258e-08, "Finetune/Loss": 0.8395552635192871, "Finetune/Loss (Raw)": 1.0923404693603516, "Finetune/Step": 4998, "Finetune/Step Time": 4.368996446952224} +{"Finetune/Learning Rate": 7.674351642850797e-08, "Finetune/Loss": 0.8399075269699097, "Finetune/Loss (Raw)": 0.9862138628959656, "Finetune/Step": 4999, "Finetune/Step Time": 4.359499532729387} +{"Finetune/Learning Rate": 7.597513841169469e-08, "Finetune/Loss": 0.8361871838569641, "Finetune/Loss (Raw)": 0.35379886627197266, "Finetune/Step": 5000, "Finetune/Step Time": 4.3991527166217566} +{"Finetune/Learning Rate": 7.521061171518184e-08, "Finetune/Loss": 0.8371690511703491, "Finetune/Loss (Raw)": 0.9299099445343018, "Finetune/Step": 5001, "Finetune/Step Time": 4.381337208673358} +{"Finetune/Learning Rate": 7.444993663566325e-08, "Finetune/Loss": 0.837273120880127, "Finetune/Loss (Raw)": 0.6792067289352417, "Finetune/Step": 5002, "Finetune/Step Time": 4.382668374106288} +{"Finetune/Learning Rate": 7.369311346834496e-08, "Finetune/Loss": 0.8370131850242615, "Finetune/Loss (Raw)": 0.7957034707069397, "Finetune/Step": 5003, "Finetune/Step Time": 4.370445942506194} +{"Finetune/Learning Rate": 7.294014250693538e-08, "Finetune/Loss": 0.8355292081832886, "Finetune/Loss (Raw)": 0.7370908260345459, "Finetune/Step": 5004, "Finetune/Step Time": 4.383814137428999} +{"Finetune/Learning Rate": 7.21910240436463e-08, "Finetune/Loss": 0.8346377015113831, "Finetune/Loss (Raw)": 0.8585906028747559, "Finetune/Step": 5005, "Finetune/Step Time": 4.385967453941703} +{"Finetune/Learning Rate": 7.144575836919632e-08, "Finetune/Loss": 0.8350609540939331, "Finetune/Loss (Raw)": 0.8952632546424866, "Finetune/Step": 5006, "Finetune/Step Time": 4.378177328035235} +{"Finetune/Learning Rate": 7.07043457728096e-08, "Finetune/Loss": 0.8340110182762146, "Finetune/Loss (Raw)": 0.7772814035415649, "Finetune/Step": 5007, "Finetune/Step Time": 4.382620804011822} +{"Finetune/Learning Rate": 6.996678654221267e-08, "Finetune/Loss": 0.8327704668045044, "Finetune/Loss (Raw)": 0.7856545448303223, "Finetune/Step": 5008, "Finetune/Step Time": 4.389646843075752} +{"Finetune/Learning Rate": 6.923308096363879e-08, "Finetune/Loss": 0.8323642015457153, "Finetune/Loss (Raw)": 1.0042542219161987, "Finetune/Step": 5009, "Finetune/Step Time": 4.3922239281237125} +{"Finetune/Learning Rate": 6.850322932182352e-08, "Finetune/Loss": 0.8336499929428101, "Finetune/Loss (Raw)": 0.9338427782058716, "Finetune/Step": 5010, "Finetune/Step Time": 4.383030444383621} +{"Finetune/Learning Rate": 6.777723190000918e-08, "Finetune/Loss": 0.8344736695289612, "Finetune/Loss (Raw)": 0.8555765748023987, "Finetune/Step": 5011, "Finetune/Step Time": 4.384103205054998} +{"Finetune/Learning Rate": 6.705508897994151e-08, "Finetune/Loss": 0.8334101438522339, "Finetune/Loss (Raw)": 0.8452376127243042, "Finetune/Step": 5012, "Finetune/Step Time": 4.382135586813092} +{"Finetune/Learning Rate": 6.633680084186856e-08, "Finetune/Loss": 0.832579493522644, "Finetune/Loss (Raw)": 0.7314318418502808, "Finetune/Step": 5013, "Finetune/Step Time": 4.380687642842531} +{"Finetune/Learning Rate": 6.562236776454512e-08, "Finetune/Loss": 0.8311997652053833, "Finetune/Loss (Raw)": 0.7892524003982544, "Finetune/Step": 5014, "Finetune/Step Time": 4.375891963019967} +{"Finetune/Learning Rate": 6.491179002522829e-08, "Finetune/Loss": 0.8302794694900513, "Finetune/Loss (Raw)": 0.7693220376968384, "Finetune/Step": 5015, "Finetune/Step Time": 4.364905098453164} +{"Finetune/Learning Rate": 6.420506789967973e-08, "Finetune/Loss": 0.8296833038330078, "Finetune/Loss (Raw)": 0.8579966425895691, "Finetune/Step": 5016, "Finetune/Step Time": 4.380012065172195} +{"Finetune/Learning Rate": 6.350220166216337e-08, "Finetune/Loss": 0.8289788961410522, "Finetune/Loss (Raw)": 0.7793120741844177, "Finetune/Step": 5017, "Finetune/Step Time": 4.379395833238959} +{"Finetune/Learning Rate": 6.280319158544989e-08, "Finetune/Loss": 0.8281995058059692, "Finetune/Loss (Raw)": 0.7509976625442505, "Finetune/Step": 5018, "Finetune/Step Time": 4.367492720484734} +{"Finetune/Learning Rate": 6.21080379408101e-08, "Finetune/Loss": 0.82818603515625, "Finetune/Loss (Raw)": 0.7827337384223938, "Finetune/Step": 5019, "Finetune/Step Time": 4.366145711392164} +{"Finetune/Learning Rate": 6.14167409980193e-08, "Finetune/Loss": 0.8240585327148438, "Finetune/Loss (Raw)": 0.3466310501098633, "Finetune/Step": 5020, "Finetune/Step Time": 4.403809066861868} +{"Finetune/Learning Rate": 6.072930102535734e-08, "Finetune/Loss": 0.8234555721282959, "Finetune/Loss (Raw)": 0.9189959168434143, "Finetune/Step": 5021, "Finetune/Step Time": 4.402948314324021} +{"Finetune/Learning Rate": 6.004571828960525e-08, "Finetune/Loss": 0.8240050077438354, "Finetune/Loss (Raw)": 0.8873928785324097, "Finetune/Step": 5022, "Finetune/Step Time": 4.385040279477835} +{"Finetune/Learning Rate": 5.936599305604862e-08, "Finetune/Loss": 0.8247430324554443, "Finetune/Loss (Raw)": 0.8758726716041565, "Finetune/Step": 5023, "Finetune/Step Time": 4.391660571098328} +{"Finetune/Learning Rate": 5.869012558847642e-08, "Finetune/Loss": 0.8239086270332336, "Finetune/Loss (Raw)": 0.8189470171928406, "Finetune/Step": 5024, "Finetune/Step Time": 4.3951763696968555} +{"Finetune/Learning Rate": 5.801811614917885e-08, "Finetune/Loss": 0.8249483108520508, "Finetune/Loss (Raw)": 0.826202929019928, "Finetune/Step": 5025, "Finetune/Step Time": 4.399816008284688} +{"Finetune/Learning Rate": 5.73499649989484e-08, "Finetune/Loss": 0.8261667490005493, "Finetune/Loss (Raw)": 0.9205299615859985, "Finetune/Step": 5026, "Finetune/Step Time": 4.3916340712457895} +{"Finetune/Learning Rate": 5.6685672397083226e-08, "Finetune/Loss": 0.8257687091827393, "Finetune/Loss (Raw)": 0.9152608513832092, "Finetune/Step": 5027, "Finetune/Step Time": 4.389347996562719} +{"Finetune/Learning Rate": 5.602523860138154e-08, "Finetune/Loss": 0.8267133235931396, "Finetune/Loss (Raw)": 0.9691466093063354, "Finetune/Step": 5028, "Finetune/Step Time": 4.3916108924895525} +{"Finetune/Learning Rate": 5.536866386814499e-08, "Finetune/Loss": 0.8271691799163818, "Finetune/Loss (Raw)": 0.8862301111221313, "Finetune/Step": 5029, "Finetune/Step Time": 4.393402619287372} +{"Finetune/Learning Rate": 5.471594845217865e-08, "Finetune/Loss": 0.8271552324295044, "Finetune/Loss (Raw)": 0.8199039697647095, "Finetune/Step": 5030, "Finetune/Step Time": 4.397058930248022} +{"Finetune/Learning Rate": 5.4067092606785445e-08, "Finetune/Loss": 0.827637791633606, "Finetune/Loss (Raw)": 0.8935151696205139, "Finetune/Step": 5031, "Finetune/Step Time": 4.398544939234853} +{"Finetune/Learning Rate": 5.342209658377728e-08, "Finetune/Loss": 0.8283490538597107, "Finetune/Loss (Raw)": 0.9421879053115845, "Finetune/Step": 5032, "Finetune/Step Time": 4.396016497164965} +{"Finetune/Learning Rate": 5.278096063346172e-08, "Finetune/Loss": 0.827725350856781, "Finetune/Loss (Raw)": 0.8485879898071289, "Finetune/Step": 5033, "Finetune/Step Time": 4.395357117056847} +{"Finetune/Learning Rate": 5.214368500465305e-08, "Finetune/Loss": 0.8283262252807617, "Finetune/Loss (Raw)": 0.9110118746757507, "Finetune/Step": 5034, "Finetune/Step Time": 4.387399420142174} +{"Finetune/Learning Rate": 5.151026994466457e-08, "Finetune/Loss": 0.8285595178604126, "Finetune/Loss (Raw)": 0.789076566696167, "Finetune/Step": 5035, "Finetune/Step Time": 4.396925941109657} +{"Finetune/Learning Rate": 5.088071569931185e-08, "Finetune/Loss": 0.8280531764030457, "Finetune/Loss (Raw)": 0.8193914294242859, "Finetune/Step": 5036, "Finetune/Step Time": 4.370648978278041} +{"Finetune/Learning Rate": 5.025502251291392e-08, "Finetune/Loss": 0.8289717435836792, "Finetune/Loss (Raw)": 0.9692731499671936, "Finetune/Step": 5037, "Finetune/Step Time": 4.366049086675048} +{"Finetune/Learning Rate": 4.963319062828875e-08, "Finetune/Loss": 0.8295283317565918, "Finetune/Loss (Raw)": 0.96592116355896, "Finetune/Step": 5038, "Finetune/Step Time": 4.3740290477871895} +{"Finetune/Learning Rate": 4.901522028675776e-08, "Finetune/Loss": 0.8313407897949219, "Finetune/Loss (Raw)": 1.0715534687042236, "Finetune/Step": 5039, "Finetune/Step Time": 4.379170209169388} +{"Finetune/Learning Rate": 4.8401111728143545e-08, "Finetune/Loss": 0.8300299644470215, "Finetune/Loss (Raw)": 0.7303480505943298, "Finetune/Step": 5040, "Finetune/Step Time": 4.389103749766946} +{"Finetune/Learning Rate": 4.779086519076992e-08, "Finetune/Loss": 0.8304903507232666, "Finetune/Loss (Raw)": 0.98750901222229, "Finetune/Step": 5041, "Finetune/Step Time": 4.377338221296668} +{"Finetune/Learning Rate": 4.7184480911459665e-08, "Finetune/Loss": 0.8347893357276917, "Finetune/Loss (Raw)": 0.9597273468971252, "Finetune/Step": 5042, "Finetune/Step Time": 4.352194268256426} +{"Finetune/Learning Rate": 4.65819591255412e-08, "Finetune/Loss": 0.8357779383659363, "Finetune/Loss (Raw)": 0.8681709170341492, "Finetune/Step": 5043, "Finetune/Step Time": 4.360629014670849} +{"Finetune/Learning Rate": 4.5983300066839706e-08, "Finetune/Loss": 0.8357361555099487, "Finetune/Loss (Raw)": 0.8445031046867371, "Finetune/Step": 5044, "Finetune/Step Time": 4.363904979079962} +{"Finetune/Learning Rate": 4.538850396768379e-08, "Finetune/Loss": 0.8348195552825928, "Finetune/Loss (Raw)": 0.2914057672023773, "Finetune/Step": 5045, "Finetune/Step Time": 4.361348483711481} +{"Finetune/Learning Rate": 4.479757105890104e-08, "Finetune/Loss": 0.8304967880249023, "Finetune/Loss (Raw)": 0.3334707021713257, "Finetune/Step": 5046, "Finetune/Step Time": 4.399000534787774} +{"Finetune/Learning Rate": 4.4210501569823536e-08, "Finetune/Loss": 0.8307459354400635, "Finetune/Loss (Raw)": 0.8901211619377136, "Finetune/Step": 5047, "Finetune/Step Time": 4.3988087475299835} +{"Finetune/Learning Rate": 4.362729572827906e-08, "Finetune/Loss": 0.8311554193496704, "Finetune/Loss (Raw)": 0.9593442678451538, "Finetune/Step": 5048, "Finetune/Step Time": 4.399425562471151} +{"Finetune/Learning Rate": 4.304795376059989e-08, "Finetune/Loss": 0.8306992650032043, "Finetune/Loss (Raw)": 0.8468457460403442, "Finetune/Step": 5049, "Finetune/Step Time": 4.391507923603058} +{"Finetune/Learning Rate": 4.2472475891617295e-08, "Finetune/Loss": 0.8313307762145996, "Finetune/Loss (Raw)": 0.8385979533195496, "Finetune/Step": 5050, "Finetune/Step Time": 4.402536636218429} +{"Finetune/Learning Rate": 4.1900862344661506e-08, "Finetune/Loss": 0.8310396671295166, "Finetune/Loss (Raw)": 0.7681149840354919, "Finetune/Step": 5051, "Finetune/Step Time": 4.417246533557773} +{"Finetune/Learning Rate": 4.133311334156509e-08, "Finetune/Loss": 0.8271937370300293, "Finetune/Loss (Raw)": 0.3710678219795227, "Finetune/Step": 5052, "Finetune/Step Time": 4.451539505273104} +{"Finetune/Learning Rate": 4.076922910266179e-08, "Finetune/Loss": 0.8268970251083374, "Finetune/Loss (Raw)": 0.7844709157943726, "Finetune/Step": 5053, "Finetune/Step Time": 4.459871262311935} +{"Finetune/Learning Rate": 4.020920984678323e-08, "Finetune/Loss": 0.8284047842025757, "Finetune/Loss (Raw)": 1.0034677982330322, "Finetune/Step": 5054, "Finetune/Step Time": 4.466524804010987} +{"Finetune/Learning Rate": 3.965305579126111e-08, "Finetune/Loss": 0.8285892009735107, "Finetune/Loss (Raw)": 0.8304505348205566, "Finetune/Step": 5055, "Finetune/Step Time": 4.46671561524272} +{"Finetune/Learning Rate": 3.9100767151928345e-08, "Finetune/Loss": 0.8304425477981567, "Finetune/Loss (Raw)": 0.8902348279953003, "Finetune/Step": 5056, "Finetune/Step Time": 4.441686825826764} +{"Finetune/Learning Rate": 3.855234414311904e-08, "Finetune/Loss": 0.8307645320892334, "Finetune/Loss (Raw)": 0.8710101842880249, "Finetune/Step": 5057, "Finetune/Step Time": 4.445200156420469} +{"Finetune/Learning Rate": 3.800778697766294e-08, "Finetune/Loss": 0.8307268023490906, "Finetune/Loss (Raw)": 0.8791370987892151, "Finetune/Step": 5058, "Finetune/Step Time": 4.449342457577586} +{"Finetune/Learning Rate": 3.746709586689323e-08, "Finetune/Loss": 0.8308688998222351, "Finetune/Loss (Raw)": 0.9574506878852844, "Finetune/Step": 5059, "Finetune/Step Time": 4.4472257271409035} +{"Finetune/Learning Rate": 3.693027102064206e-08, "Finetune/Loss": 0.8314513564109802, "Finetune/Loss (Raw)": 0.973193347454071, "Finetune/Step": 5060, "Finetune/Step Time": 4.447136100381613} +{"Finetune/Learning Rate": 3.639731264724056e-08, "Finetune/Loss": 0.8323503136634827, "Finetune/Loss (Raw)": 0.909073531627655, "Finetune/Step": 5061, "Finetune/Step Time": 4.429855138063431} +{"Finetune/Learning Rate": 3.586822095351772e-08, "Finetune/Loss": 0.8312357664108276, "Finetune/Loss (Raw)": 0.7806078791618347, "Finetune/Step": 5062, "Finetune/Step Time": 4.434693435207009} +{"Finetune/Learning Rate": 3.5342996144805966e-08, "Finetune/Loss": 0.8312915563583374, "Finetune/Loss (Raw)": 0.9403433799743652, "Finetune/Step": 5063, "Finetune/Step Time": 4.431712582707405} +{"Finetune/Learning Rate": 3.482163842493447e-08, "Finetune/Loss": 0.8310772180557251, "Finetune/Loss (Raw)": 0.7581304907798767, "Finetune/Step": 5064, "Finetune/Step Time": 4.427087903022766} +{"Finetune/Learning Rate": 3.4304147996230275e-08, "Finetune/Loss": 0.8306916952133179, "Finetune/Loss (Raw)": 0.9599770903587341, "Finetune/Step": 5065, "Finetune/Step Time": 4.433740044012666} +{"Finetune/Learning Rate": 3.379052505952274e-08, "Finetune/Loss": 0.8308775424957275, "Finetune/Loss (Raw)": 0.9322086572647095, "Finetune/Step": 5066, "Finetune/Step Time": 4.424870945513248} +{"Finetune/Learning Rate": 3.328076981413908e-08, "Finetune/Loss": 0.8346585035324097, "Finetune/Loss (Raw)": 0.8260299563407898, "Finetune/Step": 5067, "Finetune/Step Time": 4.389336809515953} +{"Finetune/Learning Rate": 3.2774882457904386e-08, "Finetune/Loss": 0.8363844156265259, "Finetune/Loss (Raw)": 0.9074501991271973, "Finetune/Step": 5068, "Finetune/Step Time": 4.382049564272165} +{"Finetune/Learning Rate": 3.227286318714384e-08, "Finetune/Loss": 0.8365635275840759, "Finetune/Loss (Raw)": 0.7750950455665588, "Finetune/Step": 5069, "Finetune/Step Time": 4.383468061685562} +{"Finetune/Learning Rate": 3.177471219668271e-08, "Finetune/Loss": 0.8365020751953125, "Finetune/Loss (Raw)": 0.8064553141593933, "Finetune/Step": 5070, "Finetune/Step Time": 4.385954538360238} +{"Finetune/Learning Rate": 3.1280429679841905e-08, "Finetune/Loss": 0.8368211984634399, "Finetune/Loss (Raw)": 0.8245394825935364, "Finetune/Step": 5071, "Finetune/Step Time": 4.387630745768547} +{"Finetune/Learning Rate": 3.079001582844354e-08, "Finetune/Loss": 0.8380129337310791, "Finetune/Loss (Raw)": 1.0700584650039673, "Finetune/Step": 5072, "Finetune/Step Time": 4.389698572456837} +{"Finetune/Learning Rate": 3.0303470832806494e-08, "Finetune/Loss": 0.8376766443252563, "Finetune/Loss (Raw)": 0.8990468978881836, "Finetune/Step": 5073, "Finetune/Step Time": 4.393783053383231} +{"Finetune/Learning Rate": 2.9820794881751934e-08, "Finetune/Loss": 0.8383381366729736, "Finetune/Loss (Raw)": 0.9424399733543396, "Finetune/Step": 5074, "Finetune/Step Time": 4.38440558873117} +{"Finetune/Learning Rate": 2.9341988162595593e-08, "Finetune/Loss": 0.8340728878974915, "Finetune/Loss (Raw)": 0.3659380376338959, "Finetune/Step": 5075, "Finetune/Step Time": 4.4217327907681465} +{"Finetune/Learning Rate": 2.886705086115216e-08, "Finetune/Loss": 0.8349471092224121, "Finetune/Loss (Raw)": 0.928102970123291, "Finetune/Step": 5076, "Finetune/Step Time": 4.420563271269202} +{"Finetune/Learning Rate": 2.8395983161736418e-08, "Finetune/Loss": 0.835299015045166, "Finetune/Loss (Raw)": 0.848953366279602, "Finetune/Step": 5077, "Finetune/Step Time": 4.44219290278852} +{"Finetune/Learning Rate": 2.7928785247161027e-08, "Finetune/Loss": 0.835671067237854, "Finetune/Loss (Raw)": 0.8583378195762634, "Finetune/Step": 5078, "Finetune/Step Time": 4.425775304436684} +{"Finetune/Learning Rate": 2.7465457298736508e-08, "Finetune/Loss": 0.8364098072052002, "Finetune/Loss (Raw)": 1.001327633857727, "Finetune/Step": 5079, "Finetune/Step Time": 4.426108136773109} +{"Finetune/Learning Rate": 2.700599949626903e-08, "Finetune/Loss": 0.8367933034896851, "Finetune/Loss (Raw)": 0.9579350352287292, "Finetune/Step": 5080, "Finetune/Step Time": 4.418196342885494} +{"Finetune/Learning Rate": 2.6550412018067072e-08, "Finetune/Loss": 0.8360082507133484, "Finetune/Loss (Raw)": 0.7150914072990417, "Finetune/Step": 5081, "Finetune/Step Time": 4.414839996024966} +{"Finetune/Learning Rate": 2.6098695040935874e-08, "Finetune/Loss": 0.8315543532371521, "Finetune/Loss (Raw)": 0.3455895483493805, "Finetune/Step": 5082, "Finetune/Step Time": 4.4032551776617765} +{"Finetune/Learning Rate": 2.5650848740177425e-08, "Finetune/Loss": 0.8314650058746338, "Finetune/Loss (Raw)": 0.8534170985221863, "Finetune/Step": 5083, "Finetune/Step Time": 4.40799630060792} +{"Finetune/Learning Rate": 2.5206873289590484e-08, "Finetune/Loss": 0.8300736546516418, "Finetune/Loss (Raw)": 0.7548145651817322, "Finetune/Step": 5084, "Finetune/Step Time": 4.418507941067219} +{"Finetune/Learning Rate": 2.4766768861475e-08, "Finetune/Loss": 0.8291909098625183, "Finetune/Loss (Raw)": 0.8618581295013428, "Finetune/Step": 5085, "Finetune/Step Time": 4.421615632250905} +{"Finetune/Learning Rate": 2.4330535626627684e-08, "Finetune/Loss": 0.830585241317749, "Finetune/Loss (Raw)": 0.985663115978241, "Finetune/Step": 5086, "Finetune/Step Time": 4.408956728875637} +{"Finetune/Learning Rate": 2.389817375433978e-08, "Finetune/Loss": 0.8341128826141357, "Finetune/Loss (Raw)": 0.8934760689735413, "Finetune/Step": 5087, "Finetune/Step Time": 4.392682043835521} +{"Finetune/Learning Rate": 2.3469683412403745e-08, "Finetune/Loss": 0.8354483842849731, "Finetune/Loss (Raw)": 0.9969210624694824, "Finetune/Step": 5088, "Finetune/Step Time": 4.392320454120636} +{"Finetune/Learning Rate": 2.304506476710877e-08, "Finetune/Loss": 0.8363698720932007, "Finetune/Loss (Raw)": 0.9275822639465332, "Finetune/Step": 5089, "Finetune/Step Time": 4.391230177134275} +{"Finetune/Learning Rate": 2.262431798323972e-08, "Finetune/Loss": 0.8354846239089966, "Finetune/Loss (Raw)": 0.8063145875930786, "Finetune/Step": 5090, "Finetune/Step Time": 4.392613437026739} +{"Finetune/Learning Rate": 2.2207443224081528e-08, "Finetune/Loss": 0.8362076878547668, "Finetune/Loss (Raw)": 0.9390721917152405, "Finetune/Step": 5091, "Finetune/Step Time": 4.400722619146109} +{"Finetune/Learning Rate": 2.1794440651414782e-08, "Finetune/Loss": 0.8314065933227539, "Finetune/Loss (Raw)": 0.32319533824920654, "Finetune/Step": 5092, "Finetune/Step Time": 4.435341738164425} +{"Finetune/Learning Rate": 2.1385310425517945e-08, "Finetune/Loss": 0.8308771848678589, "Finetune/Loss (Raw)": 0.8989596366882324, "Finetune/Step": 5093, "Finetune/Step Time": 4.436798429116607} +{"Finetune/Learning Rate": 2.0980052705165123e-08, "Finetune/Loss": 0.8309289813041687, "Finetune/Loss (Raw)": 0.8065024018287659, "Finetune/Step": 5094, "Finetune/Step Time": 4.446427706629038} +{"Finetune/Learning Rate": 2.05786676476305e-08, "Finetune/Loss": 0.8291163444519043, "Finetune/Loss (Raw)": 0.6954550743103027, "Finetune/Step": 5095, "Finetune/Step Time": 4.459032598882914} +{"Finetune/Learning Rate": 2.0181155408682816e-08, "Finetune/Loss": 0.828400731086731, "Finetune/Loss (Raw)": 0.7927621006965637, "Finetune/Step": 5096, "Finetune/Step Time": 4.481387754902244} +{"Finetune/Learning Rate": 1.978751614258978e-08, "Finetune/Loss": 0.8265419006347656, "Finetune/Loss (Raw)": 0.6351743936538696, "Finetune/Step": 5097, "Finetune/Step Time": 4.471707422286272} +{"Finetune/Learning Rate": 1.9397750002113634e-08, "Finetune/Loss": 0.8248436450958252, "Finetune/Loss (Raw)": 0.722352921962738, "Finetune/Step": 5098, "Finetune/Step Time": 4.473325438797474} +{"Finetune/Learning Rate": 1.901185713851672e-08, "Finetune/Loss": 0.8231092691421509, "Finetune/Loss (Raw)": 0.7811936140060425, "Finetune/Step": 5099, "Finetune/Step Time": 4.460773542523384} +{"Finetune/Learning Rate": 1.8629837701554797e-08, "Finetune/Loss": 0.8223804235458374, "Finetune/Loss (Raw)": 0.7644434571266174, "Finetune/Step": 5100, "Finetune/Step Time": 4.448164604604244} +{"Finetune/Learning Rate": 1.825169183948372e-08, "Finetune/Loss": 0.820777416229248, "Finetune/Loss (Raw)": 0.789828360080719, "Finetune/Step": 5101, "Finetune/Step Time": 4.455558830872178} +{"Finetune/Learning Rate": 1.7877419699053876e-08, "Finetune/Loss": 0.8254000544548035, "Finetune/Loss (Raw)": 0.8958514928817749, "Finetune/Step": 5102, "Finetune/Step Time": 4.419600246474147} +{"Finetune/Learning Rate": 1.7507021425512417e-08, "Finetune/Loss": 0.8265924453735352, "Finetune/Loss (Raw)": 0.9535276889801025, "Finetune/Step": 5103, "Finetune/Step Time": 4.4144647270441055} +{"Finetune/Learning Rate": 1.7140497162605464e-08, "Finetune/Loss": 0.823825478553772, "Finetune/Loss (Raw)": 0.6348150968551636, "Finetune/Step": 5104, "Finetune/Step Time": 4.41647619754076} +{"Finetune/Learning Rate": 1.677784705257146e-08, "Finetune/Loss": 0.8248533606529236, "Finetune/Loss (Raw)": 0.8767578601837158, "Finetune/Step": 5105, "Finetune/Step Time": 4.395020488649607} +{"Finetune/Learning Rate": 1.6419071236148943e-08, "Finetune/Loss": 0.8262604475021362, "Finetune/Loss (Raw)": 0.9835705161094666, "Finetune/Step": 5106, "Finetune/Step Time": 4.396082546561956} +{"Finetune/Learning Rate": 1.6064169852572088e-08, "Finetune/Loss": 0.8274363279342651, "Finetune/Loss (Raw)": 0.9531437754631042, "Finetune/Step": 5107, "Finetune/Step Time": 4.395170236006379} +{"Finetune/Learning Rate": 1.5713143039571833e-08, "Finetune/Loss": 0.8285804986953735, "Finetune/Loss (Raw)": 0.8487195372581482, "Finetune/Step": 5108, "Finetune/Step Time": 4.379152303561568} +{"Finetune/Learning Rate": 1.5365990933373653e-08, "Finetune/Loss": 0.8300318717956543, "Finetune/Loss (Raw)": 0.9537899494171143, "Finetune/Step": 5109, "Finetune/Step Time": 4.376467319205403} +{"Finetune/Learning Rate": 1.5022713668699785e-08, "Finetune/Loss": 0.829411506652832, "Finetune/Loss (Raw)": 0.8727086186408997, "Finetune/Step": 5110, "Finetune/Step Time": 4.388015795499086} +{"Finetune/Learning Rate": 1.4683311378771437e-08, "Finetune/Loss": 0.8320477604866028, "Finetune/Loss (Raw)": 0.7512425184249878, "Finetune/Step": 5111, "Finetune/Step Time": 4.374392628669739} +{"Finetune/Learning Rate": 1.434778419530325e-08, "Finetune/Loss": 0.8285955190658569, "Finetune/Loss (Raw)": 0.34783634543418884, "Finetune/Step": 5112, "Finetune/Step Time": 4.409016879275441} +{"Finetune/Learning Rate": 1.401613224850551e-08, "Finetune/Loss": 0.8229290843009949, "Finetune/Loss (Raw)": 0.2917655110359192, "Finetune/Step": 5113, "Finetune/Step Time": 4.453128416091204} +{"Finetune/Learning Rate": 1.3688355667087483e-08, "Finetune/Loss": 0.8241205215454102, "Finetune/Loss (Raw)": 0.8506343960762024, "Finetune/Step": 5114, "Finetune/Step Time": 4.45430395565927} +{"Finetune/Learning Rate": 1.3364454578252972e-08, "Finetune/Loss": 0.8229960203170776, "Finetune/Loss (Raw)": 0.7765845060348511, "Finetune/Step": 5115, "Finetune/Step Time": 4.446256557479501} +{"Finetune/Learning Rate": 1.3044429107700319e-08, "Finetune/Loss": 0.8177051544189453, "Finetune/Loss (Raw)": 0.25980037450790405, "Finetune/Step": 5116, "Finetune/Step Time": 4.478378940373659} +{"Finetune/Learning Rate": 1.272827937962573e-08, "Finetune/Loss": 0.8186798095703125, "Finetune/Loss (Raw)": 1.0400996208190918, "Finetune/Step": 5117, "Finetune/Step Time": 4.48027410171926} +{"Finetune/Learning Rate": 1.2416005516722174e-08, "Finetune/Loss": 0.8196377754211426, "Finetune/Loss (Raw)": 0.9805132746696472, "Finetune/Step": 5118, "Finetune/Step Time": 4.4828097727149725} +{"Finetune/Learning Rate": 1.2107607640174934e-08, "Finetune/Loss": 0.8212075233459473, "Finetune/Loss (Raw)": 1.0264766216278076, "Finetune/Step": 5119, "Finetune/Step Time": 4.483240989968181} +{"Finetune/Learning Rate": 1.1803085869667164e-08, "Finetune/Loss": 0.816213846206665, "Finetune/Loss (Raw)": 0.34588733315467834, "Finetune/Step": 5120, "Finetune/Step Time": 4.505808539688587} +{"Finetune/Learning Rate": 1.1502440323379882e-08, "Finetune/Loss": 0.8178915977478027, "Finetune/Loss (Raw)": 0.8147517442703247, "Finetune/Step": 5121, "Finetune/Step Time": 4.491824772208929} +{"Finetune/Learning Rate": 1.1205671117985318e-08, "Finetune/Loss": 0.8174785375595093, "Finetune/Loss (Raw)": 0.7814390659332275, "Finetune/Step": 5122, "Finetune/Step Time": 4.483623472973704} +{"Finetune/Learning Rate": 1.0912778368655786e-08, "Finetune/Loss": 0.8196341395378113, "Finetune/Loss (Raw)": 0.8952866792678833, "Finetune/Step": 5123, "Finetune/Step Time": 4.485916519537568} +{"Finetune/Learning Rate": 1.0623762189057029e-08, "Finetune/Loss": 0.8191262483596802, "Finetune/Loss (Raw)": 0.9412758350372314, "Finetune/Step": 5124, "Finetune/Step Time": 4.487269351258874} +{"Finetune/Learning Rate": 1.0338622691349332e-08, "Finetune/Loss": 0.819657564163208, "Finetune/Loss (Raw)": 0.9926855564117432, "Finetune/Step": 5125, "Finetune/Step Time": 4.4923101272434} +{"Finetune/Learning Rate": 1.005735998619084e-08, "Finetune/Loss": 0.8141653537750244, "Finetune/Loss (Raw)": 0.38933610916137695, "Finetune/Step": 5126, "Finetune/Step Time": 4.52970895729959} +{"Finetune/Learning Rate": 9.779974182733131e-09, "Finetune/Loss": 0.8130999803543091, "Finetune/Loss (Raw)": 0.8498526215553284, "Finetune/Step": 5127, "Finetune/Step Time": 4.543040892109275} +{"Finetune/Learning Rate": 9.506465388624541e-09, "Finetune/Loss": 0.8170369267463684, "Finetune/Loss (Raw)": 0.8577202558517456, "Finetune/Step": 5128, "Finetune/Step Time": 4.507787067443132} +{"Finetune/Learning Rate": 9.236833710010163e-09, "Finetune/Loss": 0.8120721578598022, "Finetune/Loss (Raw)": 0.29441753029823303, "Finetune/Step": 5129, "Finetune/Step Time": 4.548326963558793} +{"Finetune/Learning Rate": 8.971079251526293e-09, "Finetune/Loss": 0.8133145570755005, "Finetune/Loss (Raw)": 0.838237464427948, "Finetune/Step": 5130, "Finetune/Step Time": 4.543524095788598} +{"Finetune/Learning Rate": 8.709202116308214e-09, "Finetune/Loss": 0.8127025365829468, "Finetune/Loss (Raw)": 0.7173643708229065, "Finetune/Step": 5131, "Finetune/Step Time": 4.549197781831026} +{"Finetune/Learning Rate": 8.451202405985736e-09, "Finetune/Loss": 0.812767505645752, "Finetune/Loss (Raw)": 0.7454062700271606, "Finetune/Step": 5132, "Finetune/Step Time": 4.543325236067176} +{"Finetune/Learning Rate": 8.197080220683217e-09, "Finetune/Loss": 0.8129066228866577, "Finetune/Loss (Raw)": 0.8763946294784546, "Finetune/Step": 5133, "Finetune/Step Time": 4.538034228608012} +{"Finetune/Learning Rate": 7.946835659020657e-09, "Finetune/Loss": 0.8128384351730347, "Finetune/Loss (Raw)": 0.8865438103675842, "Finetune/Step": 5134, "Finetune/Step Time": 4.535767547786236} +{"Finetune/Learning Rate": 7.700468818111483e-09, "Finetune/Loss": 0.8124603629112244, "Finetune/Loss (Raw)": 0.7288865447044373, "Finetune/Step": 5135, "Finetune/Step Time": 4.529324624687433} +{"Finetune/Learning Rate": 7.457979793568105e-09, "Finetune/Loss": 0.8143719434738159, "Finetune/Loss (Raw)": 1.030330777168274, "Finetune/Step": 5136, "Finetune/Step Time": 4.533931275829673} +{"Finetune/Learning Rate": 7.219368679495242e-09, "Finetune/Loss": 0.8129345178604126, "Finetune/Loss (Raw)": 0.8202704191207886, "Finetune/Step": 5137, "Finetune/Step Time": 4.537165354937315} +{"Finetune/Learning Rate": 6.984635568492159e-09, "Finetune/Loss": 0.8120810985565186, "Finetune/Loss (Raw)": 0.8246058821678162, "Finetune/Step": 5138, "Finetune/Step Time": 4.556466469541192} +{"Finetune/Learning Rate": 6.753780551654876e-09, "Finetune/Loss": 0.8080614805221558, "Finetune/Loss (Raw)": 0.34106001257896423, "Finetune/Step": 5139, "Finetune/Step Time": 4.598684566095471} +{"Finetune/Learning Rate": 6.526803718572838e-09, "Finetune/Loss": 0.8086637258529663, "Finetune/Loss (Raw)": 0.922329843044281, "Finetune/Step": 5140, "Finetune/Step Time": 4.593483254313469} +{"Finetune/Learning Rate": 6.3037051573333615e-09, "Finetune/Loss": 0.8105660676956177, "Finetune/Loss (Raw)": 0.9749357104301453, "Finetune/Step": 5141, "Finetune/Step Time": 4.59054946154356} +{"Finetune/Learning Rate": 6.0844849545149685e-09, "Finetune/Loss": 0.8108144998550415, "Finetune/Loss (Raw)": 0.8210381865501404, "Finetune/Step": 5142, "Finetune/Step Time": 4.588898088783026} +{"Finetune/Learning Rate": 5.869143195192939e-09, "Finetune/Loss": 0.8121753931045532, "Finetune/Loss (Raw)": 0.9435288906097412, "Finetune/Step": 5143, "Finetune/Step Time": 4.591943457722664} +{"Finetune/Learning Rate": 5.657679962938201e-09, "Finetune/Loss": 0.8107047080993652, "Finetune/Loss (Raw)": 0.6697371006011963, "Finetune/Step": 5144, "Finetune/Step Time": 4.597339851781726} +{"Finetune/Learning Rate": 5.4500953398139985e-09, "Finetune/Loss": 0.81099933385849, "Finetune/Loss (Raw)": 0.817028820514679, "Finetune/Step": 5145, "Finetune/Step Time": 4.603729521855712} +{"Finetune/Learning Rate": 5.246389406381447e-09, "Finetune/Loss": 0.8129738569259644, "Finetune/Loss (Raw)": 1.0037307739257812, "Finetune/Step": 5146, "Finetune/Step Time": 4.598757032305002} +{"Finetune/Learning Rate": 5.046562241693975e-09, "Finetune/Loss": 0.8135635852813721, "Finetune/Loss (Raw)": 0.8582311272621155, "Finetune/Step": 5147, "Finetune/Step Time": 4.601247871294618} +{"Finetune/Learning Rate": 4.850613923300662e-09, "Finetune/Loss": 0.8161011934280396, "Finetune/Loss (Raw)": 0.6714416742324829, "Finetune/Step": 5148, "Finetune/Step Time": 4.5680431965738535} +{"Finetune/Learning Rate": 4.658544527244013e-09, "Finetune/Loss": 0.8153092861175537, "Finetune/Loss (Raw)": 0.8176299929618835, "Finetune/Step": 5149, "Finetune/Step Time": 4.574448000639677} +{"Finetune/Learning Rate": 4.470354128065513e-09, "Finetune/Loss": 0.8148144483566284, "Finetune/Loss (Raw)": 0.824053943157196, "Finetune/Step": 5150, "Finetune/Step Time": 4.581139570102096} +{"Finetune/Learning Rate": 4.286042798794521e-09, "Finetune/Loss": 0.8142101168632507, "Finetune/Loss (Raw)": 0.7985205054283142, "Finetune/Step": 5151, "Finetune/Step Time": 4.577329143881798} +{"Finetune/Learning Rate": 4.105610610961597e-09, "Finetune/Loss": 0.8138933181762695, "Finetune/Loss (Raw)": 0.7783885598182678, "Finetune/Step": 5152, "Finetune/Step Time": 4.579677872359753} +{"Finetune/Learning Rate": 3.929057634587397e-09, "Finetune/Loss": 0.8107369542121887, "Finetune/Loss (Raw)": 0.4221915006637573, "Finetune/Step": 5153, "Finetune/Step Time": 4.607222380116582} +{"Finetune/Learning Rate": 3.756383938189334e-09, "Finetune/Loss": 0.80644291639328, "Finetune/Loss (Raw)": 0.3708984851837158, "Finetune/Step": 5154, "Finetune/Step Time": 4.647094409912825} +{"Finetune/Learning Rate": 3.587589588778251e-09, "Finetune/Loss": 0.805219829082489, "Finetune/Loss (Raw)": 0.7587025761604309, "Finetune/Step": 5155, "Finetune/Step Time": 4.647682927548885} +{"Finetune/Learning Rate": 3.4226746518606356e-09, "Finetune/Loss": 0.8051056265830994, "Finetune/Loss (Raw)": 0.9545271396636963, "Finetune/Step": 5156, "Finetune/Step Time": 4.660141693428159} +{"Finetune/Learning Rate": 3.2616391914364056e-09, "Finetune/Loss": 0.804855465888977, "Finetune/Loss (Raw)": 0.8542138338088989, "Finetune/Step": 5157, "Finetune/Step Time": 4.6598742343485355} +{"Finetune/Learning Rate": 3.1044832699989035e-09, "Finetune/Loss": 0.8050566911697388, "Finetune/Loss (Raw)": 0.8456632494926453, "Finetune/Step": 5158, "Finetune/Step Time": 4.6569743528962135} +{"Finetune/Learning Rate": 2.9512069485393426e-09, "Finetune/Loss": 0.8054972887039185, "Finetune/Loss (Raw)": 0.9499050974845886, "Finetune/Step": 5159, "Finetune/Step Time": 4.664990469813347} +{"Finetune/Learning Rate": 2.801810286540141e-09, "Finetune/Loss": 0.8059453964233398, "Finetune/Loss (Raw)": 0.9995396137237549, "Finetune/Step": 5160, "Finetune/Step Time": 4.664004750549793} +{"Finetune/Learning Rate": 2.656293341980476e-09, "Finetune/Loss": 0.8064711093902588, "Finetune/Loss (Raw)": 0.9158810973167419, "Finetune/Step": 5161, "Finetune/Step Time": 4.675686737522483} +{"Finetune/Learning Rate": 2.5146561713307315e-09, "Finetune/Loss": 0.8055146932601929, "Finetune/Loss (Raw)": 0.7885924577713013, "Finetune/Step": 5162, "Finetune/Step Time": 4.6809109300374985} +{"Finetune/Learning Rate": 2.3768988295580497e-09, "Finetune/Loss": 0.8064201474189758, "Finetune/Loss (Raw)": 0.9049788117408752, "Finetune/Step": 5163, "Finetune/Step Time": 4.664959365502} +{"Finetune/Learning Rate": 2.2430213701252203e-09, "Finetune/Loss": 0.8075495958328247, "Finetune/Loss (Raw)": 0.9639620780944824, "Finetune/Step": 5164, "Finetune/Step Time": 4.678877834230661} +{"Finetune/Learning Rate": 2.1130238449851292e-09, "Finetune/Loss": 0.8054789304733276, "Finetune/Loss (Raw)": 0.7042245864868164, "Finetune/Step": 5165, "Finetune/Step Time": 4.68296604976058} +{"Finetune/Learning Rate": 1.986906304588532e-09, "Finetune/Loss": 0.8049970269203186, "Finetune/Loss (Raw)": 0.904239296913147, "Finetune/Step": 5166, "Finetune/Step Time": 4.6954181510955095} +{"Finetune/Learning Rate": 1.8646687978784992e-09, "Finetune/Loss": 0.8027597665786743, "Finetune/Loss (Raw)": 0.7851850986480713, "Finetune/Step": 5167, "Finetune/Step Time": 4.693584725260735} +{"Finetune/Learning Rate": 1.7463113722937518e-09, "Finetune/Loss": 0.8043332695960999, "Finetune/Loss (Raw)": 0.9317524433135986, "Finetune/Step": 5168, "Finetune/Step Time": 4.6781405713409185} +{"Finetune/Learning Rate": 1.631834073765326e-09, "Finetune/Loss": 0.8038536310195923, "Finetune/Loss (Raw)": 0.9261149764060974, "Finetune/Step": 5169, "Finetune/Step Time": 4.679192654788494} +{"Finetune/Learning Rate": 1.5212369467210164e-09, "Finetune/Loss": 0.801976203918457, "Finetune/Loss (Raw)": 0.7194188237190247, "Finetune/Step": 5170, "Finetune/Step Time": 4.675962558016181} +{"Finetune/Learning Rate": 1.4145200340809352e-09, "Finetune/Loss": 0.8029478788375854, "Finetune/Loss (Raw)": 0.992546558380127, "Finetune/Step": 5171, "Finetune/Step Time": 4.670950384810567} +{"Finetune/Learning Rate": 1.3116833772597315e-09, "Finetune/Loss": 0.8029988408088684, "Finetune/Loss (Raw)": 0.8510234355926514, "Finetune/Step": 5172, "Finetune/Step Time": 4.667661244049668} +{"Finetune/Learning Rate": 1.2127270161654824e-09, "Finetune/Loss": 0.806919515132904, "Finetune/Loss (Raw)": 0.7932579517364502, "Finetune/Step": 5173, "Finetune/Step Time": 4.629203263670206} +{"Finetune/Learning Rate": 1.1176509892030229e-09, "Finetune/Loss": 0.8128305673599243, "Finetune/Loss (Raw)": 1.0900852680206299, "Finetune/Step": 5174, "Finetune/Step Time": 4.590821634978056} +{"Finetune/Learning Rate": 1.0264553332683946e-09, "Finetune/Loss": 0.8127164244651794, "Finetune/Loss (Raw)": 0.8755130171775818, "Finetune/Step": 5175, "Finetune/Step Time": 4.596906682476401} +{"Finetune/Learning Rate": 9.39140083752177e-10, "Finetune/Loss": 0.8077717423439026, "Finetune/Loss (Raw)": 0.32641705870628357, "Finetune/Step": 5176, "Finetune/Step Time": 4.634846523404121} +{"Finetune/Learning Rate": 8.557052745405969e-10, "Finetune/Loss": 0.8088918924331665, "Finetune/Loss (Raw)": 0.9902295470237732, "Finetune/Step": 5177, "Finetune/Step Time": 4.628754645586014} +{"Finetune/Learning Rate": 7.761509380133092e-10, "Finetune/Loss": 0.8082351684570312, "Finetune/Loss (Raw)": 0.7545346617698669, "Finetune/Step": 5178, "Finetune/Step Time": 4.6133845541626215} +{"Finetune/Learning Rate": 7.004771050433956e-10, "Finetune/Loss": 0.8084178566932678, "Finetune/Loss (Raw)": 0.791502058506012, "Finetune/Step": 5179, "Finetune/Step Time": 4.614786064252257} +{"Finetune/Learning Rate": 6.286838049984755e-10, "Finetune/Loss": 0.8115443587303162, "Finetune/Loss (Raw)": 0.7712556719779968, "Finetune/Step": 5180, "Finetune/Step Time": 4.583662662655115} +{"Finetune/Learning Rate": 5.607710657407061e-10, "Finetune/Loss": 0.8123983144760132, "Finetune/Loss (Raw)": 0.89377760887146, "Finetune/Step": 5181, "Finetune/Step Time": 4.578683080151677} +{"Finetune/Learning Rate": 4.967389136245615e-10, "Finetune/Loss": 0.8100610375404358, "Finetune/Loss (Raw)": 0.7043018341064453, "Finetune/Step": 5182, "Finetune/Step Time": 4.581395488232374} +{"Finetune/Learning Rate": 4.3658737350016354e-10, "Finetune/Loss": 0.8106035590171814, "Finetune/Loss (Raw)": 0.8998928666114807, "Finetune/Step": 5183, "Finetune/Step Time": 4.585553912445903} +{"Finetune/Learning Rate": 3.8031646871106166e-10, "Finetune/Loss": 0.8102539777755737, "Finetune/Loss (Raw)": 0.845483660697937, "Finetune/Step": 5184, "Finetune/Step Time": 4.595458876341581} +{"Finetune/Learning Rate": 3.279262210953427e-10, "Finetune/Loss": 0.8111215233802795, "Finetune/Loss (Raw)": 0.9820560812950134, "Finetune/Step": 5185, "Finetune/Step Time": 4.60675629042089} +{"Finetune/Learning Rate": 2.794166509845209e-10, "Finetune/Loss": 0.8072167634963989, "Finetune/Loss (Raw)": 0.37933504581451416, "Finetune/Step": 5186, "Finetune/Step Time": 4.642488958314061} +{"Finetune/Learning Rate": 2.3478777720353785e-10, "Finetune/Loss": 0.8054985404014587, "Finetune/Loss (Raw)": 0.7375116944313049, "Finetune/Step": 5187, "Finetune/Step Time": 4.6607756074517965} +{"Finetune/Learning Rate": 1.940396170718728e-10, "Finetune/Loss": 0.8041858077049255, "Finetune/Loss (Raw)": 0.8051595091819763, "Finetune/Step": 5188, "Finetune/Step Time": 4.662024637684226} +{"Finetune/Learning Rate": 1.5717218640465272e-10, "Finetune/Loss": 0.8029948472976685, "Finetune/Loss (Raw)": 0.7566375732421875, "Finetune/Step": 5189, "Finetune/Step Time": 4.674514800310135} +{"Finetune/Learning Rate": 1.2418549950821145e-10, "Finetune/Loss": 0.8026789426803589, "Finetune/Loss (Raw)": 0.7401713132858276, "Finetune/Step": 5190, "Finetune/Step Time": 4.672358253970742} +{"Finetune/Learning Rate": 9.507956918453076e-11, "Finetune/Loss": 0.8009384870529175, "Finetune/Loss (Raw)": 0.7175663113594055, "Finetune/Step": 5191, "Finetune/Step Time": 4.684629820287228} +{"Finetune/Learning Rate": 6.985440672790945e-11, "Finetune/Loss": 0.8015296459197998, "Finetune/Loss (Raw)": 0.833795428276062, "Finetune/Step": 5192, "Finetune/Step Time": 4.680379576981068} +{"Finetune/Learning Rate": 4.851002192940435e-11, "Finetune/Loss": 0.7996881604194641, "Finetune/Loss (Raw)": 0.7242684364318848, "Finetune/Step": 5193, "Finetune/Step Time": 4.691946497187018} +{"Finetune/Learning Rate": 3.1046423071279254e-11, "Finetune/Loss": 0.79502272605896, "Finetune/Loss (Raw)": 0.33503589034080505, "Finetune/Step": 5194, "Finetune/Step Time": 4.729959037154913} +{"Finetune/Learning Rate": 1.7463616930335493e-11, "Finetune/Loss": 0.7951947450637817, "Finetune/Loss (Raw)": 0.8480529189109802, "Finetune/Step": 5195, "Finetune/Step Time": 4.72809693031013} +{"Finetune/Learning Rate": 7.761608780132434e-12, "Finetune/Loss": 0.794326901435852, "Finetune/Loss (Raw)": 0.7963572144508362, "Finetune/Step": 5196, "Finetune/Step Time": 4.733793431892991} +{"Finetune/Learning Rate": 1.9404023832159115e-12, "Finetune/Loss": 0.7950717210769653, "Finetune/Loss (Raw)": 0.8704362511634827, "Finetune/Step": 5197, "Finetune/Step Time": 4.726126262918115} +{"Finetune/Learning Rate": 0.0, "Finetune/Loss": 0.7954542636871338, "Finetune/Loss (Raw)": 0.8554208278656006, "Finetune/Step": 5198, "Finetune/Step Time": 4.717184504494071} diff --git a/qformer2_32/run-metrics.jsonl b/qformer2_32/run-metrics.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..656cfb462e27aa2a6b19f63299a8df9dda053d10 --- /dev/null +++ b/qformer2_32/run-metrics.jsonl @@ -0,0 +1 @@ +{"hparams": {"dataset": {"align_stage_components": ["download/llava-laion-cc-sbu-558k/chat.json", "download/llava-laion-cc-sbu-558k"], "dataset_id": "llava-v15", "dataset_root_dir": "data", "finetune_stage_components": ["download/llava-v1.5-instruct/llava_v1_5_mix665k.json", "download/llava-v1.5-instruct"], "type": "llava-v15"}, "hf_token": ".hf_token", "max_length": 4096, "model": {"align_epochs": 1, "align_global_batch_size": 256, "align_learning_rate": 0.001, "align_lr_scheduler_type": "linear-warmup+cosine-decay", "align_max_grad_norm": 1.0, "align_max_steps": null, "align_per_device_batch_size": 16, "align_train_strategy": "fsdp-shard-grad-op", "align_warmup_ratio": 0.03, "align_weight_decay": 0.0, "arch_specifier": "qformer2_32", "enable_gradient_checkpointing": true, "enable_mixed_precision_training": true, "finetune_epochs": 1, "finetune_global_batch_size": 128, "finetune_learning_rate": 2e-05, "finetune_lr_scheduler_type": "linear-warmup+cosine-decay", "finetune_max_grad_norm": 1.0, "finetune_max_steps": null, "finetune_per_device_batch_size": 16, "finetune_train_strategy": "fsdp-full-shard", "finetune_warmup_ratio": 0.03, "finetune_weight_decay": 0.1, "image_resize_strategy": "letterbox", "llm_backbone_id": "vicuna-v15-7b", "llm_max_length": 2048, "model_id": "one-stage+7b_qformer2_32", "reduce_in_full_precision": false, "type": "one-stage+7b", "vision_backbone_id": "clip-vit-l-336px"}, "pretrained_checkpoint": null, "run_id": "qformer2_32", "run_root_dir": "runs", "seed": 7, "stage": "finetune", "trackers": ["jsonl", "wandb"], "wandb_entity": "lilei_stones", "wandb_project": "hf-vlms"}, "run_id": "qformer2_32"} diff --git a/qformer2_32/wandb/debug-internal.log b/qformer2_32/wandb/debug-internal.log new file mode 120000 index 0000000000000000000000000000000000000000..c36aa121639ac7757e750e239e4ca6177767b23a --- /dev/null +++ b/qformer2_32/wandb/debug-internal.log @@ -0,0 +1 @@ +run-20240312_142855-srovbvi2/logs/debug-internal.log \ No newline at end of file diff --git a/qformer2_32/wandb/debug.log b/qformer2_32/wandb/debug.log new file mode 120000 index 0000000000000000000000000000000000000000..2311d88847df1f7525650df1806f01a4fed3bb54 --- /dev/null +++ b/qformer2_32/wandb/debug.log @@ -0,0 +1 @@ +run-20240312_142855-srovbvi2/logs/debug.log \ No newline at end of file diff --git a/qformer2_32/wandb/latest-run b/qformer2_32/wandb/latest-run new file mode 120000 index 0000000000000000000000000000000000000000..a01bc5683e42d98c722a50158b908f54e99bca16 --- /dev/null +++ b/qformer2_32/wandb/latest-run @@ -0,0 +1 @@ +run-20240312_142855-srovbvi2 \ No newline at end of file diff --git a/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/conda-environment.yaml b/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/conda-environment.yaml new file mode 100644 index 0000000000000000000000000000000000000000..799a6f3a49522423b58a3ae238b95584ecfe64c1 --- /dev/null +++ b/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/conda-environment.yaml @@ -0,0 +1,194 @@ +name: pvlm +channels: + - pytorch + - nvidia + - defaults +dependencies: + - _libgcc_mutex=0.1=main + - _openmp_mutex=5.1=1_gnu + - blas=1.0=mkl + - bzip2=1.0.8=h7b6447c_0 + - ca-certificates=2023.12.12=h06a4308_0 + - certifi=2024.2.2=py310h06a4308_0 + - charset-normalizer=2.0.4=pyhd3eb1b0_0 + - cuda-cudart=11.8.89=0 + - cuda-cupti=11.8.87=0 + - cuda-libraries=11.8.0=0 + - cuda-nvrtc=11.8.89=0 + - cuda-nvtx=11.8.86=0 + - cuda-runtime=11.8.0=0 + - ffmpeg=4.3=hf484d3e_0 + - filelock=3.13.1=py310h06a4308_0 + - freetype=2.12.1=h4a9f257_0 + - gmp=6.2.1=h295c915_3 + - gmpy2=2.1.2=py310heeb90bb_0 + - gnutls=3.6.15=he1e5248_0 + - idna=3.4=py310h06a4308_0 + - intel-openmp=2023.1.0=hdb19cb5_46306 + - jinja2=3.1.3=py310h06a4308_0 + - jpeg=9e=h5eee18b_1 + - lame=3.100=h7b6447c_0 + - lcms2=2.12=h3be6417_0 + - ld_impl_linux-64=2.38=h1181459_1 + - lerc=3.0=h295c915_0 + - libcublas=11.11.3.6=0 + - libcufft=10.9.0.58=0 + - libcufile=1.8.1.2=0 + - libcurand=10.3.4.107=0 + - libcusolver=11.4.1.48=0 + - libcusparse=11.7.5.86=0 + - libdeflate=1.17=h5eee18b_1 + - libffi=3.4.4=h6a678d5_0 + - libgcc-ng=11.2.0=h1234567_1 + - libgomp=11.2.0=h1234567_1 + - libiconv=1.16=h7f8727e_2 + - libidn2=2.3.4=h5eee18b_0 + - libjpeg-turbo=2.0.0=h9bf148f_0 + - libnpp=11.8.0.86=0 + - libnvjpeg=11.9.0.86=0 + - libpng=1.6.39=h5eee18b_0 + - libstdcxx-ng=11.2.0=h1234567_1 + - libtasn1=4.19.0=h5eee18b_0 + - libtiff=4.5.1=h6a678d5_0 + - libunistring=0.9.10=h27cfd23_0 + - libuuid=1.41.5=h5eee18b_0 + - libwebp-base=1.3.2=h5eee18b_0 + - llvm-openmp=14.0.6=h9e868ea_0 + - lz4-c=1.9.4=h6a678d5_0 + - markupsafe=2.1.3=py310h5eee18b_0 + - mkl=2023.1.0=h213fc3f_46344 + - mkl-service=2.4.0=py310h5eee18b_1 + - mkl_fft=1.3.8=py310h5eee18b_0 + - mkl_random=1.2.4=py310hdb19cb5_0 + - mpc=1.1.0=h10f8cd9_1 + - mpfr=4.0.2=hb69a4c5_1 + - mpmath=1.3.0=py310h06a4308_0 + - ncurses=6.4=h6a678d5_0 + - nettle=3.7.3=hbbd107a_1 + - networkx=3.1=py310h06a4308_0 + - numpy=1.26.4=py310h5f9d8c6_0 + - numpy-base=1.26.4=py310hb5e798b_0 + - openh264=2.1.1=h4ff587b_0 + - openjpeg=2.4.0=h3ad879b_0 + - openssl=3.0.13=h7f8727e_0 + - pillow=10.2.0=py310h5eee18b_0 + - pip=23.3.1=py310h06a4308_0 + - python=3.10.13=h955ad1f_0 + - pytorch=2.2.1=py3.10_cuda11.8_cudnn8.7.0_0 + - pytorch-cuda=11.8=h7e8668a_5 + - pytorch-mutex=1.0=cuda + - pyyaml=6.0.1=py310h5eee18b_0 + - readline=8.2=h5eee18b_0 + - requests=2.31.0=py310h06a4308_1 + - setuptools=68.2.2=py310h06a4308_0 + - sqlite=3.41.2=h5eee18b_0 + - sympy=1.12=py310h06a4308_0 + - tbb=2021.8.0=hdb19cb5_0 + - tk=8.6.12=h1ccaba5_0 + - torchaudio=2.2.1=py310_cu118 + - torchtriton=2.2.0=py310 + - torchvision=0.17.1=py310_cu118 + - typing_extensions=4.9.0=py310h06a4308_1 + - urllib3=2.1.0=py310h06a4308_0 + - wheel=0.41.2=py310h06a4308_0 + - xz=5.4.5=h5eee18b_0 + - yaml=0.2.5=h7b6447c_0 + - zlib=1.2.13=h5eee18b_0 + - zstd=1.5.5=hc292b87_0 + - pip: + - accelerate==0.27.2 + - aiohttp==3.9.3 + - aiosignal==1.3.1 + - annotated-types==0.6.0 + - appdirs==1.4.4 + - async-timeout==4.0.3 + - attrs==23.2.0 + - beautifulsoup4==4.12.3 + - blis==0.7.11 + - catalogue==2.0.10 + - click==8.1.7 + - cloudpathlib==0.16.0 + - confection==0.1.4 + - contourpy==1.2.0 + - cycler==0.12.1 + - cymem==2.0.8 + - datasets==2.18.0 + - deepspeed==0.13.4 + - dill==0.3.8 + - docker-pycreds==0.4.0 + - draccus==0.7.1 + - einops==0.7.0 + - flash-attn==2.5.5 + - fonttools==4.49.0 + - frozenlist==1.4.1 + - fsspec==2024.2.0 + - gdown==5.1.0 + - gitdb==4.0.11 + - gitpython==3.1.42 + - hjson==3.1.0 + - huggingface-hub==0.20.3 + - install==1.3.5 + - jsonlines==4.0.0 + - kiwisolver==1.4.5 + - langcodes==3.3.0 + - markdown-it-py==3.0.0 + - matplotlib==3.8.3 + - mdurl==0.1.2 + - mergedeep==1.3.4 + - multidict==6.0.5 + - multiprocess==0.70.16 + - murmurhash==1.0.10 + - mypy-extensions==1.0.0 + - ninja==1.11.1.1 + - packaging==23.2 + - pandas==2.2.1 + - preshed==3.0.9 + - prismatic==0.0.1 + - protobuf==4.25.3 + - psutil==5.9.8 + - py-cpuinfo==9.0.0 + - pyarrow==15.0.0 + - pyarrow-hotfix==0.6 + - pycocoevalcap==1.2 + - pycocotools==2.0.7 + - pydantic==2.6.3 + - pydantic-core==2.16.3 + - pygments==2.17.2 + - pynvml==11.5.0 + - pyparsing==3.1.1 + - pysocks==1.7.1 + - pythainlp==5.0.1 + - python-dateutil==2.9.0.post0 + - pytz==2024.1 + - pyyaml-include==1.3.2 + - regex==2023.12.25 + - rich==13.7.0 + - safetensors==0.4.2 + - sentencepiece==0.2.0 + - sentry-sdk==1.40.5 + - setproctitle==1.3.3 + - six==1.16.0 + - smart-open==6.4.0 + - smmap==5.0.1 + - soupsieve==2.5 + - spacy==3.7.4 + - spacy-legacy==3.0.12 + - spacy-loggers==1.0.5 + - srsly==2.4.8 + - sudachidict-core==20240109 + - sudachipy==0.6.8 + - thinc==8.2.3 + - timm==0.9.16 + - tokenizers==0.15.2 + - toml==0.10.2 + - tqdm==4.66.2 + - transformers==4.38.1 + - typer==0.9.0 + - typing-inspect==0.9.0 + - tzdata==2024.1 + - wandb==0.16.3 + - wasabi==1.1.2 + - weasel==0.3.4 + - xxhash==3.4.1 + - yarl==1.9.4 +prefix: /home/lilei/anaconda3/envs/pvlm diff --git a/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/config.yaml b/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3718012ef2590aff7e033237cc9ecc3df67401e4 --- /dev/null +++ b/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/config.yaml @@ -0,0 +1,119 @@ +wandb_version: 1 + +model: + desc: null + value: + type: one-stage+7b + model_id: one-stage+7b_qformer2_32 + arch_specifier: qformer2_32 + vision_backbone_id: clip-vit-l-336px + llm_backbone_id: vicuna-v15-7b + image_resize_strategy: letterbox + llm_max_length: 2048 + align_epochs: 1 + align_max_steps: null + align_global_batch_size: 256 + align_per_device_batch_size: 16 + align_learning_rate: 0.001 + align_weight_decay: 0.0 + align_max_grad_norm: 1.0 + align_lr_scheduler_type: linear-warmup+cosine-decay + align_warmup_ratio: 0.03 + align_train_strategy: fsdp-shard-grad-op + finetune_epochs: 1 + finetune_max_steps: null + finetune_global_batch_size: 128 + finetune_per_device_batch_size: 16 + finetune_learning_rate: 2.0e-05 + finetune_weight_decay: 0.1 + finetune_max_grad_norm: 1.0 + finetune_lr_scheduler_type: linear-warmup+cosine-decay + finetune_warmup_ratio: 0.03 + finetune_train_strategy: fsdp-full-shard + enable_gradient_checkpointing: true + enable_mixed_precision_training: true + reduce_in_full_precision: false +dataset: + desc: null + value: + type: llava-v15 + dataset_id: llava-v15 + align_stage_components: + - download/llava-laion-cc-sbu-558k/chat.json + - download/llava-laion-cc-sbu-558k + finetune_stage_components: + - download/llava-v1.5-instruct/llava_v1_5_mix665k.json + - download/llava-v1.5-instruct + dataset_root_dir: data +stage: + desc: null + value: finetune +pretrained_checkpoint: + desc: null + value: null +run_id: + desc: null + value: qformer2_32 +run_root_dir: + desc: null + value: runs +seed: + desc: null + value: 7 +hf_token: + desc: null + value: .hf_token +trackers: + desc: null + value: + - jsonl + - wandb +wandb_project: + desc: null + value: hf-vlms +wandb_entity: + desc: null + value: lilei_stones +max_length: + desc: null + value: 4096 +_wandb: + desc: null + value: + python_version: 3.10.13 + cli_version: 0.16.3 + framework: huggingface + huggingface_version: 4.38.1 + is_jupyter_run: false + is_kaggle_kernel: false + start_time: 1710253735.064287 + t: + 1: + - 1 + - 11 + - 41 + - 49 + - 51 + - 55 + - 63 + - 71 + 2: + - 1 + - 11 + - 41 + - 49 + - 51 + - 55 + - 63 + - 71 + 3: + - 2 + - 13 + - 16 + - 23 + 4: 3.10.13 + 5: 0.16.3 + 6: 4.38.1 + 8: + - 5 + 13: linux-x86_64 diff --git a/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log b/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..297a01c96dbdffa360582cdefe68b541ed33df20 --- /dev/null +++ b/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log @@ -0,0 +1,5198 @@ + +03/12 [14:29:00] INFO  | >> [*] Starting Training Loop ]8;id=72103;file:///home/lilei/prismatic-vlms/scripts/pretrain.py\pretrain.py]8;;\:]8;id=880770;file:///home/lilei/prismatic-vlms/scripts/pretrain.py#226\226]8;;\ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +=>> [Global Step] 005198 =>> LR :: 0.000000 -- Loss :: 0.7955: 100%|█████████████████████████████████████████████| 5198/5198 [6:30:45<00:00, 4.12s/it] +03/12 [21:04:14] INFO  | >> [*] Done with Training =>> Finalizing Metrics ]8;id=696414;file:///home/lilei/prismatic-vlms/scripts/pretrain.py\pretrain.py]8;;\:]8;id=68157;file:///home/lilei/prismatic-vlms/scripts/pretrain.py#230\230]8;;\ \ No newline at end of file diff --git a/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/requirements.txt b/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..52215be56573f749927d1265d23e43a5fe36d5b1 --- /dev/null +++ b/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/requirements.txt @@ -0,0 +1,122 @@ +accelerate==0.27.2 +aiohttp==3.9.3 +aiosignal==1.3.1 +annotated-types==0.6.0 +appdirs==1.4.4 +async-timeout==4.0.3 +attrs==23.2.0 +beautifulsoup4==4.12.3 +blis==0.7.11 +catalogue==2.0.10 +certifi==2024.2.2 +charset-normalizer==2.0.4 +click==8.1.7 +cloudpathlib==0.16.0 +confection==0.1.4 +contourpy==1.2.0 +cycler==0.12.1 +cymem==2.0.8 +datasets==2.18.0 +deepspeed==0.13.4 +dill==0.3.8 +docker-pycreds==0.4.0 +draccus==0.7.1 +einops==0.7.0 +filelock==3.13.1 +flash-attn==2.5.5 +fonttools==4.49.0 +frozenlist==1.4.1 +fsspec==2024.2.0 +gdown==5.1.0 +gitdb==4.0.11 +gitpython==3.1.42 +gmpy2==2.1.2 +hjson==3.1.0 +huggingface-hub==0.20.3 +idna==3.4 +install==1.3.5 +jinja2==3.1.3 +jsonlines==4.0.0 +kiwisolver==1.4.5 +langcodes==3.3.0 +markdown-it-py==3.0.0 +markupsafe==2.1.3 +matplotlib==3.8.3 +mdurl==0.1.2 +mergedeep==1.3.4 +mkl-fft==1.3.8 +mkl-random==1.2.4 +mkl-service==2.4.0 +mpmath==1.3.0 +multidict==6.0.5 +multiprocess==0.70.16 +murmurhash==1.0.10 +mypy-extensions==1.0.0 +networkx==3.1 +ninja==1.11.1.1 +numpy==1.26.4 +ocifs==1.3.1 +packaging==23.2 +pandas==2.2.1 +pillow==10.2.0 +pip==23.3.1 +preshed==3.0.9 +prismatic==0.0.1 +protobuf==4.25.3 +psutil==5.9.8 +py-cpuinfo==9.0.0 +pyarrow-hotfix==0.6 +pyarrow==15.0.0 +pycocoevalcap==1.2 +pycocotools==2.0.7 +pydantic-core==2.16.3 +pydantic==2.6.3 +pygments==2.17.2 +pynvml==11.5.0 +pyparsing==3.1.1 +pysocks==1.7.1 +pythainlp==5.0.1 +python-dateutil==2.9.0.post0 +pytz==2024.1 +pyyaml-include==1.3.2 +pyyaml==6.0.1 +regex==2023.12.25 +requests==2.31.0 +rich==13.7.0 +safetensors==0.4.2 +sentencepiece==0.2.0 +sentry-sdk==1.40.5 +setproctitle==1.3.3 +setuptools==68.2.2 +six==1.16.0 +smart-open==6.4.0 +smmap==5.0.1 +soupsieve==2.5 +spacy-legacy==3.0.12 +spacy-loggers==1.0.5 +spacy==3.7.4 +srsly==2.4.8 +sudachidict-core==20240109 +sudachipy==0.6.8 +sympy==1.12 +thinc==8.2.3 +timm==0.9.16 +tokenizers==0.15.2 +toml==0.10.2 +torch==2.2.1 +torchaudio==2.2.1 +torchvision==0.17.1 +tqdm==4.66.2 +transformers==4.38.1 +triton==2.2.0 +typer==0.9.0 +typing-extensions==4.9.0 +typing-inspect==0.9.0 +tzdata==2024.1 +urllib3==2.1.0 +wandb==0.16.3 +wasabi==1.1.2 +weasel==0.3.4 +wheel==0.41.2 +xxhash==3.4.1 +yarl==1.9.4 \ No newline at end of file diff --git a/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-metadata.json b/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..682dabbd01e1e1ddef1290d901888bd0da9e8541 --- /dev/null +++ b/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-metadata.json @@ -0,0 +1,1360 @@ +{ + "os": "Linux-5.15.0-1027-oracle-x86_64-with-glibc2.31", + "python": "3.10.13", + "heartbeatAt": "2024-03-12T14:28:55.726177", + "startedAt": "2024-03-12T14:28:55.018301", + "docker": null, + "cuda": null, + "args": [ + "--model.type", + "one-stage+7b", + "--model.model_id", + "one-stage+7b_qformer2_32", + "--run_id", + "qformer2_32", + "--model.arch_specifier", + "qformer2_32" + ], + "state": "running", + "program": "/home/lilei/prismatic-vlms/scripts/pretrain.py", + "codePathLocal": "scripts/pretrain.py", + "codePath": "scripts/pretrain.py", + "git": { + "remote": "https://github.com/TRI-ML/prismatic-vlms.git", + "commit": "b1c33c284c6e76d908c29e5545b00f5b8b4c601f" + }, + "email": "tobiaslee@qq.com", + "root": "/home/lilei/prismatic-vlms", + "host": "compute-permanent-node-590", + "username": "lilei", + "executable": "/home/lilei/anaconda3/envs/pvlm/bin/python", + "cpu_count": 128, + "cpu_count_logical": 255, + "cpu_freq": { + "current": 2571.3575529411764, + "min": 1500.0, + "max": 2550.0 + }, + "cpu_freq_per_core": [ + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 3248.75, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 3249.597, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 3248.662, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 3248.777, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 3248.983, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 3248.315, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 3249.081, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 3247.977, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 3006.851, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + } + ], + "disk": { + "/": { + "total": 242.1086883544922, + "used": 52.20142364501953 + } + }, + "gpu": "NVIDIA A100-SXM4-80GB", + "gpu_count": 8, + "gpu_devices": [ + { + "name": "NVIDIA A100-SXM4-80GB", + "memory_total": 85899345920 + }, + { + "name": "NVIDIA A100-SXM4-80GB", + "memory_total": 85899345920 + }, + { + "name": "NVIDIA A100-SXM4-80GB", + "memory_total": 85899345920 + }, + { + "name": "NVIDIA A100-SXM4-80GB", + "memory_total": 85899345920 + }, + { + "name": "NVIDIA A100-SXM4-80GB", + "memory_total": 85899345920 + }, + { + "name": "NVIDIA A100-SXM4-80GB", + "memory_total": 85899345920 + }, + { + "name": "NVIDIA A100-SXM4-80GB", + "memory_total": 85899345920 + }, + { + "name": "NVIDIA A100-SXM4-80GB", + "memory_total": 85899345920 + } + ], + "memory": { + "total": 2015.6942710876465 + } +} diff --git a/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json b/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json new file mode 100644 index 0000000000000000000000000000000000000000..2ecaae4e92dd4d0a2b06d3681268648cdacb829e --- /dev/null +++ b/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json @@ -0,0 +1 @@ +{"Finetune/Step": 5198, "Finetune/Loss": 0.7954542636871338, "Finetune/Loss (Raw)": 0.8554208278656006, "Finetune/Learning Rate": 0.0, "Finetune/Step Time": 4.717184504494071, "_timestamp": 1710277188.6693704, "_runtime": 23453.605083465576, "_step": 5198, "_wandb": {"runtime": 23718}} \ No newline at end of file diff --git a/qformer2_32/wandb/run-20240312_142855-srovbvi2/logs/debug-internal.log b/qformer2_32/wandb/run-20240312_142855-srovbvi2/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..f87d098ddb8334dba80f5caa3dd0fc0381dcf357 --- /dev/null +++ b/qformer2_32/wandb/run-20240312_142855-srovbvi2/logs/debug-internal.log @@ -0,0 +1,41260 @@ +2024-03-12 14:28:55,064 INFO StreamThr :452226 [internal.py:wandb_internal():86] W&B internal server running at pid: 452226, started at: 2024-03-12 14:28:55.062357 +2024-03-12 14:28:55,065 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status +2024-03-12 14:28:55,078 INFO WriterThread:452226 [datastore.py:open_for_write():87] open: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/run-srovbvi2.wandb +2024-03-12 14:28:55,080 DEBUG SenderThread:452226 [sender.py:send():382] send: header +2024-03-12 14:28:55,098 DEBUG SenderThread:452226 [sender.py:send():382] send: run +2024-03-12 14:28:55,575 INFO SenderThread:452226 [dir_watcher.py:__init__():211] watching files in: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files +2024-03-12 14:28:55,575 INFO SenderThread:452226 [sender.py:_start_run_threads():1136] run started: srovbvi2 with start time 1710253735.064287 +2024-03-12 14:28:55,589 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: check_version +2024-03-12 14:28:55,589 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: check_version +2024-03-12 14:28:55,643 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: run_start +2024-03-12 14:28:55,654 DEBUG HandlerThread:452226 [system_info.py:__init__():27] System info init +2024-03-12 14:28:55,654 DEBUG HandlerThread:452226 [system_info.py:__init__():42] System info init done +2024-03-12 14:28:55,654 INFO HandlerThread:452226 [system_monitor.py:start():194] Starting system monitor +2024-03-12 14:28:55,654 INFO SystemMonitor:452226 [system_monitor.py:_start():158] Starting system asset monitoring threads +2024-03-12 14:28:55,654 INFO HandlerThread:452226 [system_monitor.py:probe():214] Collecting system info +2024-03-12 14:28:55,655 INFO SystemMonitor:452226 [interfaces.py:start():190] Started cpu monitoring +2024-03-12 14:28:55,655 INFO SystemMonitor:452226 [interfaces.py:start():190] Started disk monitoring +2024-03-12 14:28:55,656 INFO SystemMonitor:452226 [interfaces.py:start():190] Started gpu monitoring +2024-03-12 14:28:55,656 INFO SystemMonitor:452226 [interfaces.py:start():190] Started memory monitoring +2024-03-12 14:28:55,656 INFO SystemMonitor:452226 [interfaces.py:start():190] Started network monitoring +2024-03-12 14:28:55,726 DEBUG HandlerThread:452226 [system_info.py:probe():151] Probing system +2024-03-12 14:28:55,731 DEBUG HandlerThread:452226 [system_info.py:_probe_git():136] Probing git +2024-03-12 14:28:55,750 DEBUG HandlerThread:452226 [system_info.py:_probe_git():144] Probing git done +2024-03-12 14:28:55,750 DEBUG HandlerThread:452226 [system_info.py:probe():199] Probing system done +2024-03-12 14:28:55,750 DEBUG HandlerThread:452226 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-1027-oracle-x86_64-with-glibc2.31', 'python': '3.10.13', 'heartbeatAt': '2024-03-12T14:28:55.726177', 'startedAt': '2024-03-12T14:28:55.018301', 'docker': None, 'cuda': None, 'args': ('--model.type', 'one-stage+7b', '--model.model_id', 'one-stage+7b_qformer2_32', '--run_id', 'qformer2_32', '--model.arch_specifier', 'qformer2_32'), 'state': 'running', 'program': '/home/lilei/prismatic-vlms/scripts/pretrain.py', 'codePathLocal': 'scripts/pretrain.py', 'codePath': 'scripts/pretrain.py', 'git': {'remote': 'https://github.com/TRI-ML/prismatic-vlms.git', 'commit': 'b1c33c284c6e76d908c29e5545b00f5b8b4c601f'}, 'email': 'tobiaslee@qq.com', 'root': '/home/lilei/prismatic-vlms', 'host': 'compute-permanent-node-590', 'username': 'lilei', 'executable': '/home/lilei/anaconda3/envs/pvlm/bin/python', 'cpu_count': 128, 'cpu_count_logical': 255, 'cpu_freq': {'current': 2571.3575529411764, 'min': 1500.0, 'max': 2550.0}, 'cpu_freq_per_core': [{'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 3248.75, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 3249.597, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 3248.662, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 3248.777, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 3248.983, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 3248.315, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 3249.081, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 3247.977, 'min': 1500.0, 'max': 2550.0}, {'current': 3006.851, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}], 'disk': {'/': {'total': 242.1086883544922, 'used': 52.20142364501953}}, 'gpu': 'NVIDIA A100-SXM4-80GB', 'gpu_count': 8, 'gpu_devices': [{'name': 'NVIDIA A100-SXM4-80GB', 'memory_total': 85899345920}, {'name': 'NVIDIA A100-SXM4-80GB', 'memory_total': 85899345920}, {'name': 'NVIDIA A100-SXM4-80GB', 'memory_total': 85899345920}, {'name': 'NVIDIA A100-SXM4-80GB', 'memory_total': 85899345920}, {'name': 'NVIDIA A100-SXM4-80GB', 'memory_total': 85899345920}, {'name': 'NVIDIA A100-SXM4-80GB', 'memory_total': 85899345920}, {'name': 'NVIDIA A100-SXM4-80GB', 'memory_total': 85899345920}, {'name': 'NVIDIA A100-SXM4-80GB', 'memory_total': 85899345920}], 'memory': {'total': 2015.6942710876465}} +2024-03-12 14:28:55,751 INFO HandlerThread:452226 [system_monitor.py:probe():224] Finished collecting system info +2024-03-12 14:28:55,751 INFO HandlerThread:452226 [system_monitor.py:probe():227] Publishing system info +2024-03-12 14:28:55,751 DEBUG HandlerThread:452226 [system_info.py:_save_conda():208] Saving list of conda packages installed into the current environment +2024-03-12 14:28:56,578 INFO Thread-12 :452226 [dir_watcher.py:_on_file_created():271] file/dir created: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/conda-environment.yaml +2024-03-12 14:29:00,119 DEBUG HandlerThread:452226 [system_info.py:_save_conda():220] Saving conda packages done +2024-03-12 14:29:00,126 INFO HandlerThread:452226 [system_monitor.py:probe():229] Finished publishing system info +2024-03-12 14:29:00,131 DEBUG SenderThread:452226 [sender.py:send():382] send: files +2024-03-12 14:29:00,131 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-metadata.json with policy now +2024-03-12 14:29:00,131 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:29:00,142 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: python_packages +2024-03-12 14:29:00,142 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: python_packages +2024-03-12 14:29:00,144 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:29:00,144 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:29:00,150 DEBUG SenderThread:452226 [sender.py:send():382] send: telemetry +2024-03-12 14:29:00,150 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:29:00,581 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/conda-environment.yaml +2024-03-12 14:29:00,581 INFO Thread-12 :452226 [dir_watcher.py:_on_file_created():271] file/dir created: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/requirements.txt +2024-03-12 14:29:00,581 INFO Thread-12 :452226 [dir_watcher.py:_on_file_created():271] file/dir created: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:29:00,581 INFO Thread-12 :452226 [dir_watcher.py:_on_file_created():271] file/dir created: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-metadata.json +2024-03-12 14:29:01,144 INFO wandb-upload_0:452226 [upload_job.py:push():131] Uploaded file /tmp/tmpsj_hfqi9wandb/lzgq4p2i-wandb-metadata.json +2024-03-12 14:29:02,584 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:29:05,570 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:29:05,588 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:29:10,570 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:29:15,142 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:29:15,143 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:29:15,143 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:29:16,346 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:29:21,347 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:29:25,783 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:29:26,829 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:29:27,607 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:29:27,607 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/config.yaml +2024-03-12 14:29:30,142 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:29:30,143 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:29:30,143 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:29:32,453 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:29:32,527 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:29:32,528 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:29:32,528 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:29:32,532 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:29:32,611 INFO Thread-12 :452226 [dir_watcher.py:_on_file_created():271] file/dir created: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:29:33,613 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:29:35,757 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:29:35,758 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:29:35,758 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:29:35,763 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:29:36,615 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:29:37,618 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:29:37,764 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:29:39,152 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:29:39,153 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:29:39,153 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:29:39,158 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:29:39,619 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:29:41,622 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:29:43,159 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:29:43,587 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:29:43,587 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:29:43,588 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:29:43,593 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:29:43,623 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:29:45,142 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:29:45,143 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:29:45,143 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:29:45,626 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:29:47,475 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:29:47,476 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:29:47,476 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:29:47,481 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:29:47,627 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:29:48,518 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:29:49,631 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:29:51,405 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:29:51,406 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:29:51,406 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:29:51,411 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:29:51,632 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:29:53,637 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:29:54,412 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:29:55,656 DEBUG SystemMonitor:452226 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2024-03-12 14:29:55,659 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 14:29:56,557 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:29:56,558 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:29:56,558 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:29:56,563 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:29:56,640 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:29:57,643 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:29:59,564 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:30:00,143 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:30:00,143 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:30:00,144 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:30:00,584 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:30:00,584 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:30:00,585 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:30:00,589 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:30:00,644 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:30:01,647 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:30:04,627 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:30:07,017 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:30:07,018 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:30:07,018 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:30:07,023 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:30:07,652 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:30:07,653 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:30:10,023 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:30:10,660 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:30:10,661 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:30:10,661 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:30:10,666 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:30:11,657 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:30:11,657 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:30:14,451 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:30:14,452 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:30:14,452 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:30:14,457 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:30:14,659 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:30:15,143 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:30:15,143 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:30:15,144 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:30:15,426 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:30:15,661 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:30:18,195 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:30:18,196 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:30:18,196 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:30:18,201 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:30:18,663 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:30:19,665 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:30:21,238 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:30:21,617 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:30:21,618 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:30:21,618 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:30:21,623 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:30:21,666 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:30:23,670 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:30:25,661 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 14:30:26,458 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:30:26,459 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:30:26,459 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:30:26,459 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:30:26,464 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:30:26,671 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:30:27,675 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:30:30,143 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:30:30,143 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:30:30,143 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:30:31,461 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:30:31,461 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:30:31,462 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:30:31,462 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:30:31,467 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:30:31,677 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:30:33,680 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:30:35,442 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:30:35,443 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:30:35,443 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:30:35,448 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:30:35,682 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:30:37,449 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:30:37,684 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:30:39,819 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:30:39,820 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:30:39,820 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:30:39,825 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:30:40,686 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:30:41,688 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:30:42,825 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:30:43,269 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:30:43,270 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:30:43,270 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:30:43,275 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:30:43,690 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:30:45,143 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:30:45,143 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:30:45,143 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:30:45,694 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:30:46,474 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:30:46,475 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:30:46,475 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:30:46,480 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:30:46,695 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:30:47,697 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:30:48,519 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:30:51,494 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:30:51,495 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:30:51,495 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:30:51,508 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:30:51,700 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:30:53,703 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:30:54,509 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:30:55,385 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:30:55,385 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:30:55,385 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:30:55,390 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:30:55,664 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 14:30:55,704 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:30:57,707 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:30:59,665 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:31:00,143 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:31:00,143 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:31:00,144 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:31:03,659 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:31:03,660 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:31:03,660 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:31:03,665 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:31:03,711 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:31:04,702 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:31:05,714 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:31:06,911 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:31:06,912 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:31:06,912 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:31:06,917 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:31:07,717 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:31:07,717 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:31:09,918 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:31:10,865 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:31:10,866 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:31:10,866 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:31:10,870 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:31:11,721 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:31:11,721 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:31:15,143 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:31:15,143 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:31:15,143 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:31:15,393 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:31:19,517 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:31:19,517 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:31:19,518 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:31:19,523 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:31:19,725 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:31:20,558 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:31:21,728 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:31:22,494 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:31:22,495 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:31:22,495 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:31:22,500 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:31:22,729 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:31:23,732 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:31:25,666 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 14:31:25,667 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:31:25,783 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:31:25,784 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:31:25,784 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:31:25,789 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:31:26,733 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:31:27,737 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:31:30,143 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:31:30,144 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:31:30,144 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:31:30,554 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:31:30,555 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:31:30,555 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:31:30,560 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:31:30,739 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:31:31,598 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:31:31,741 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:31:35,423 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:31:35,424 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:31:35,424 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:31:35,429 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:31:35,743 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:31:37,430 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:31:37,746 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:31:39,488 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:31:39,488 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:31:39,489 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:31:39,494 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:31:39,747 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:31:41,750 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:31:42,494 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:31:45,143 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:31:45,144 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:31:45,144 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:31:45,311 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:31:45,391 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:31:45,392 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:31:45,396 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:31:45,754 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:31:47,757 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:31:48,397 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:31:49,636 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:31:49,644 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:31:49,644 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:31:49,649 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:31:49,758 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:31:51,761 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:31:53,349 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:31:53,349 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:31:53,350 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:31:53,355 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:31:53,762 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:31:54,355 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:31:55,668 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 14:31:55,765 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:31:57,392 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:31:57,392 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:31:57,392 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:31:57,397 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:31:57,767 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:31:59,398 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:31:59,770 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:32:00,143 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:32:00,143 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:32:00,145 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:32:02,713 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:32:02,714 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:32:02,714 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:32:02,719 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:32:02,772 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:32:03,774 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:32:04,758 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:32:09,531 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:32:09,532 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:32:09,533 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:32:09,537 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:32:09,778 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:32:10,538 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:32:11,781 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:32:12,823 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:32:12,824 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:32:12,824 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:32:12,829 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:32:13,784 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:32:13,784 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:32:15,143 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:32:15,144 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:32:15,145 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:32:16,114 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:32:16,115 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:32:16,115 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:32:16,115 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:32:16,120 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:32:16,786 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:32:17,789 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:32:21,159 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:32:21,273 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:32:21,274 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:32:21,274 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:32:21,279 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:32:21,793 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:32:21,793 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:32:24,058 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:32:24,059 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:32:24,059 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:32:24,064 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:32:24,795 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:32:25,670 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 14:32:25,798 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:32:26,671 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:32:27,332 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:32:27,333 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:32:27,333 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:32:27,338 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:32:27,801 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:32:29,804 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:32:30,143 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:32:30,144 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:32:30,145 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:32:32,070 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:32:32,070 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:32:32,070 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:32:32,071 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:32:32,075 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:32:32,806 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:32:33,808 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:32:36,592 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:32:36,593 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:32:36,593 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:32:36,599 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:32:36,811 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:32:37,600 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:32:37,812 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:32:42,600 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:32:44,027 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:32:44,028 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:32:44,028 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:32:44,033 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:32:44,818 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:32:45,144 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:32:45,144 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:32:45,145 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:32:45,820 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:32:48,409 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:32:53,410 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:32:53,582 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:32:53,583 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:32:53,583 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:32:53,588 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:32:53,824 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:32:55,677 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 14:32:55,827 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:32:58,002 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:32:58,003 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:32:58,003 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:32:58,008 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:32:58,829 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:32:59,008 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:32:59,831 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:33:00,144 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:33:00,144 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:33:00,144 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:33:03,303 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:33:03,304 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:33:03,304 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:33:03,309 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:33:03,834 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:33:04,346 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:33:05,837 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:33:07,059 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:33:07,059 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:33:07,060 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:33:07,064 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:33:07,841 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:33:07,841 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:33:10,065 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:33:10,251 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:33:10,252 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:33:10,252 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:33:10,257 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:33:10,842 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:33:11,845 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:33:14,576 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:33:14,577 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:33:14,577 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:33:14,582 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:33:14,847 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:33:15,144 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:33:15,144 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:33:15,146 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:33:15,362 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:33:15,849 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:33:18,993 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:33:18,993 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:33:18,994 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:33:18,998 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:33:19,853 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:33:19,854 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:33:21,034 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:33:24,582 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:33:24,582 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:33:24,583 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:33:24,587 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:33:24,857 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:33:25,674 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 14:33:25,859 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:33:26,676 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:33:29,170 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:33:29,171 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:33:29,171 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:33:29,176 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:33:29,865 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:33:29,865 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:33:30,144 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:33:30,144 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:33:30,145 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:33:32,445 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:33:34,257 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:33:34,258 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:33:34,258 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:33:34,263 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:33:34,868 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:33:35,870 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:33:38,303 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:33:39,699 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:33:39,700 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:33:39,700 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:33:39,705 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:33:39,873 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:33:41,876 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:33:43,342 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:33:43,343 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:33:43,343 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:33:43,343 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:33:43,348 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:33:43,879 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:33:43,879 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:33:45,144 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:33:45,145 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:33:45,145 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:33:48,443 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:33:49,769 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:33:49,769 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:33:49,769 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:33:49,774 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:33:49,884 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:33:51,887 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:33:53,811 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:33:55,119 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:33:55,120 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:33:55,120 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:33:55,125 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:33:55,682 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 14:33:55,891 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:33:55,891 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:33:58,576 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:33:58,577 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:33:58,577 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:33:58,582 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:33:58,893 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:33:59,583 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:33:59,895 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:34:00,144 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:34:00,144 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:34:00,146 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:34:02,640 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:34:02,640 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:34:02,640 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:34:02,645 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:34:02,897 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:34:03,900 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:34:04,682 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:34:05,996 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:34:05,996 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:34:05,996 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:34:06,001 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:34:06,901 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:34:07,904 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:34:09,636 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:34:09,637 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:34:09,637 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:34:09,642 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:34:09,905 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:34:10,643 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:34:11,908 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:34:15,144 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:34:15,145 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:34:15,145 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:34:16,442 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:34:16,687 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:34:16,688 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:34:16,688 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:34:16,693 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:34:16,911 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:34:17,914 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:34:21,180 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:34:21,181 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:34:21,181 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:34:21,187 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:34:21,918 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:34:21,918 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:34:22,187 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:34:25,122 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:34:25,123 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:34:25,123 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:34:25,128 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:34:25,678 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 14:34:25,923 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:34:25,923 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:34:27,680 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:34:28,365 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:34:28,365 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:34:28,366 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:34:28,370 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:34:28,926 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:34:29,928 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:34:30,144 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:34:30,145 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:34:30,145 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:34:33,027 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:34:33,028 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:34:33,028 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:34:33,028 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:34:33,033 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:34:33,932 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:34:33,933 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:34:36,252 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:34:36,253 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:34:36,253 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:34:36,260 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:34:36,934 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:34:37,937 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:34:38,260 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:34:39,708 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:34:39,708 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:34:39,709 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:34:39,713 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:34:39,938 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:34:41,941 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:34:43,275 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:34:43,276 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:34:43,276 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:34:43,276 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:34:43,281 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:34:43,944 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:34:43,944 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:34:45,144 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:34:45,145 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:34:45,146 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:34:46,430 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:34:46,430 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:34:46,431 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:34:46,435 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:34:46,947 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:34:47,950 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:34:48,474 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:34:49,889 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:34:49,889 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:34:49,890 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:34:49,894 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:34:49,951 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:34:51,954 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:34:53,429 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:34:53,429 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:34:53,430 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:34:53,434 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:34:53,957 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:34:53,958 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:34:54,435 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:34:55,685 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 14:34:57,725 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:34:57,726 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:34:57,726 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:34:57,730 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:34:57,960 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:34:59,731 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:34:59,963 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:35:00,144 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:35:00,145 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:35:00,146 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:35:01,995 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:35:01,996 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:35:01,996 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:35:02,000 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:35:02,965 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:35:03,967 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:35:05,039 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:35:05,061 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:35:05,062 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:35:05,062 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:35:05,067 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:35:05,970 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:35:05,971 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:35:08,279 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:35:08,280 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:35:08,280 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:35:08,285 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:35:08,972 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:35:09,975 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:35:10,286 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:35:11,759 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:35:11,760 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:35:11,760 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:35:11,765 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:35:11,976 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:35:13,979 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:35:15,144 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:35:15,145 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:35:15,145 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:35:15,394 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:35:16,897 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:35:16,897 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:35:16,897 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:35:16,902 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:35:16,980 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:35:17,982 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:35:20,939 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:35:21,380 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:35:21,381 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:35:21,381 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:35:21,386 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:35:21,987 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:35:21,987 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:35:24,738 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:35:24,738 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:35:24,739 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:35:24,744 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:35:24,989 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:35:25,682 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 14:35:25,991 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:35:26,683 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:35:30,063 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:35:30,064 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:35:30,064 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:35:30,069 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:35:30,145 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:35:30,145 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:35:30,146 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:35:30,996 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:35:31,998 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:35:32,375 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:35:34,902 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:35:34,903 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:35:34,903 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:35:34,908 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:35:35,000 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:35:36,002 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:35:37,947 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:35:38,403 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:35:38,404 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:35:38,404 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:35:38,409 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:35:39,004 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:35:40,006 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:35:43,410 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:35:45,145 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:35:45,145 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:35:45,146 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:35:47,655 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:35:47,656 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:35:47,656 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:35:47,661 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:35:48,012 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:35:48,698 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:35:50,015 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:35:51,121 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:35:51,122 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:35:51,122 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:35:51,127 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:35:52,018 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:35:52,018 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:35:54,128 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:35:55,685 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 14:35:56,485 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:35:56,486 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:35:56,486 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:35:56,491 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:35:57,021 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:35:58,023 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:35:59,492 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:35:59,992 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:35:59,993 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:35:59,993 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:35:59,998 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:36:00,025 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:36:00,145 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:36:00,145 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:36:00,146 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:36:02,028 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:36:05,359 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:36:05,384 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:36:05,385 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:36:05,385 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:36:05,389 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:36:06,032 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:36:06,032 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:36:10,327 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:36:10,328 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:36:10,328 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:36:10,333 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:36:11,035 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:36:11,334 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:36:12,037 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:36:13,986 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:36:13,987 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:36:13,987 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:36:13,992 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:36:14,039 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:36:15,145 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:36:15,145 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:36:15,145 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:36:16,042 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:36:16,354 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:36:17,943 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:36:17,943 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:36:17,943 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:36:17,948 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:36:18,043 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:36:20,046 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:36:21,003 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:36:21,004 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:36:21,004 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:36:21,009 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:36:21,046 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:36:22,010 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:36:22,049 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:36:24,525 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:36:24,526 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:36:24,526 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:36:24,531 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:36:25,051 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:36:25,687 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 14:36:26,054 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:36:27,688 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:36:29,434 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:36:29,435 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:36:29,435 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:36:29,440 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:36:30,059 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:36:30,059 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:36:30,145 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:36:30,145 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:36:30,145 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:36:33,444 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:36:33,640 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:36:33,641 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:36:33,641 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:36:33,647 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:36:34,061 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:36:36,064 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:36:37,175 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:36:37,176 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:36:37,176 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:36:37,181 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:36:38,067 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:36:38,068 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:36:39,181 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:36:44,182 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:36:45,145 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:36:45,145 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:36:45,146 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:36:45,432 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:36:45,432 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:36:45,432 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:36:45,438 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:36:46,075 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:36:46,075 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:36:49,397 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:36:49,398 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:36:49,399 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:36:49,399 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:36:49,403 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:36:50,080 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:36:50,081 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:36:54,178 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:36:54,178 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:36:54,178 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:36:54,183 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:36:55,083 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:36:55,184 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:36:55,689 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 14:36:56,086 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:36:57,123 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:36:57,124 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:36:57,124 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:36:57,129 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:36:58,089 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:36:58,089 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:37:00,145 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:37:00,145 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:37:00,145 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:37:00,412 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:37:05,413 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:37:05,547 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:37:05,547 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:37:05,547 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:37:05,552 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:37:06,096 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:37:06,096 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:37:08,829 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:37:08,830 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:37:08,830 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:37:08,835 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:37:09,097 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:37:10,100 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:37:10,836 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:37:11,710 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:37:11,710 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:37:11,711 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:37:11,715 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:37:12,101 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:37:14,104 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:37:15,077 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:37:15,078 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:37:15,078 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:37:15,083 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:37:15,105 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:37:15,145 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:37:15,145 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:37:15,147 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:37:16,107 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:37:16,420 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:37:18,556 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:37:18,557 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:37:18,557 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:37:18,562 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:37:19,109 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:37:20,112 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:37:21,598 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:37:22,279 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:37:22,280 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:37:22,280 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:37:22,285 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:37:23,113 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:37:24,116 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:37:25,688 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:37:25,689 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:37:25,689 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:37:25,694 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:37:25,697 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 14:37:26,117 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:37:26,698 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:37:28,122 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:37:28,944 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:37:28,944 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:37:28,944 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:37:28,950 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:37:29,122 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:37:30,125 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:37:30,145 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:37:30,146 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:37:30,147 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:37:32,399 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:37:34,131 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:37:34,132 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:37:34,132 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:37:34,137 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:37:35,127 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:37:36,130 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:37:38,175 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:37:38,923 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:37:38,924 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:37:38,925 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:37:38,929 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:37:39,132 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:37:40,134 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:37:43,837 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:37:43,838 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:37:43,838 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:37:43,838 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:37:43,843 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:37:44,136 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:37:45,145 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:37:45,146 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:37:45,147 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:37:46,141 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:37:49,427 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:37:53,209 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:37:53,210 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:37:53,210 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:37:53,215 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:37:54,148 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:37:54,148 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:37:55,254 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:37:55,694 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 14:37:56,669 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:37:56,669 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:37:56,670 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:37:56,675 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:37:57,150 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:37:58,152 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:38:00,145 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:38:00,146 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:38:00,147 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:38:00,441 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:38:02,564 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:38:02,565 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:38:02,565 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:38:02,570 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:38:03,156 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:38:04,158 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:38:05,606 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:38:06,614 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:38:06,615 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:38:06,615 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:38:06,620 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:38:07,160 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:38:08,162 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:38:09,976 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:38:09,977 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:38:09,977 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:38:09,981 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:38:10,164 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:38:10,982 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:38:12,167 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:38:13,538 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:38:13,538 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:38:13,538 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:38:13,543 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:38:14,169 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:38:14,170 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:38:15,145 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:38:15,146 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:38:15,146 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:38:16,443 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:38:17,699 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:38:17,700 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:38:17,700 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:38:17,705 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:38:18,172 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:38:20,175 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:38:21,743 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:38:24,178 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:38:24,179 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:38:24,179 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:38:24,184 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:38:25,178 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:38:25,702 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 14:38:26,180 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:38:27,543 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:38:27,544 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:38:27,544 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:38:27,544 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:38:27,549 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:38:28,184 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:38:28,184 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:38:30,146 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:38:30,146 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:38:30,146 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:38:30,992 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:38:30,993 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:38:30,993 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:38:30,998 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:38:31,186 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:38:32,188 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:38:33,034 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:38:34,778 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:38:34,778 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:38:34,779 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:38:34,784 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:38:35,190 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:38:36,193 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:38:38,530 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:38:38,531 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:38:38,531 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:38:38,531 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:38:38,536 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:38:39,195 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:38:40,197 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:38:43,537 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:38:44,531 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:38:44,532 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:38:44,532 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:38:44,537 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:38:45,147 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:38:45,147 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:38:45,147 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:38:45,202 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:38:46,204 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:38:47,870 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:38:47,870 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:38:47,870 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:38:47,876 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:38:48,205 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:38:48,914 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:38:50,208 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:38:51,540 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:38:51,541 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:38:51,541 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:38:51,546 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:38:52,211 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:38:52,211 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:38:54,547 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:38:55,017 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:38:55,018 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:38:55,018 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:38:55,023 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:38:55,213 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:38:55,699 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 14:38:56,215 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:38:59,568 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:38:59,568 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:38:59,569 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:38:59,569 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:38:59,574 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:39:00,146 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:39:00,146 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:39:00,147 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:39:00,219 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:39:00,220 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:39:02,741 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:39:02,741 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:39:02,742 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:39:02,747 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:39:03,222 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:39:04,224 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:39:04,787 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:39:06,160 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:39:06,160 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:39:06,161 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:39:06,166 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:39:06,225 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:39:08,229 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:39:10,167 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:39:11,750 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:39:11,751 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:39:11,751 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:39:11,756 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:39:12,231 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:39:14,234 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:39:15,149 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:39:15,150 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:39:15,150 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:39:15,238 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:39:15,404 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:39:15,404 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:39:15,404 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:39:15,409 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:39:16,238 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:39:16,238 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:39:19,372 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:39:19,373 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:39:19,373 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:39:19,378 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:39:20,242 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:39:20,242 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:39:21,379 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:39:22,673 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:39:22,674 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:39:22,674 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:39:22,679 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:39:23,244 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:39:24,247 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:39:25,706 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 14:39:26,708 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:39:26,925 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:39:26,926 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:39:26,926 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:39:26,931 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:39:27,249 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:39:28,253 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:39:29,767 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:39:29,768 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:39:29,768 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:39:29,774 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:39:30,149 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:39:30,149 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:39:30,149 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:39:30,256 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:39:30,257 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:39:32,372 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:39:33,535 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:39:33,535 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:39:33,535 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:39:33,541 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:39:34,261 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:39:34,262 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:39:36,559 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:39:36,560 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:39:36,560 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:39:36,566 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:39:37,263 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:39:37,567 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:39:38,266 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:39:39,471 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:39:39,472 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:39:39,472 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:39:39,477 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:39:40,269 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:39:40,269 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:39:43,478 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:39:45,149 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:39:45,149 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:39:45,149 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:39:48,049 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:39:48,050 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:39:48,050 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:39:48,055 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:39:48,276 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:39:49,094 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:39:50,279 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:39:54,095 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:39:54,728 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:39:54,729 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:39:54,729 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:39:54,734 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:39:55,282 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:39:55,703 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 14:39:56,285 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:39:59,705 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:40:00,149 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:40:00,149 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:40:00,150 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:40:03,770 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:40:03,770 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:40:03,770 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:40:03,776 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:40:04,292 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:40:04,293 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:40:04,814 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:40:09,815 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:40:12,174 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:40:12,175 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:40:12,175 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:40:12,181 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:40:12,297 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:40:14,300 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:40:15,149 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:40:15,149 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:40:15,150 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:40:15,293 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:40:15,420 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:40:15,420 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:40:15,420 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:40:15,425 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:40:16,304 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:40:16,304 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:40:19,531 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:40:19,532 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:40:19,532 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:40:19,537 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:40:20,308 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:40:20,308 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:40:20,538 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:40:23,064 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:40:23,064 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:40:23,064 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:40:23,069 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:40:23,310 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:40:24,313 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:40:25,706 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 14:40:25,707 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:40:26,729 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:40:26,730 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:40:26,730 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:40:26,735 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:40:27,315 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:40:28,318 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:40:30,149 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:40:30,149 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:40:30,151 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:40:31,098 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:40:31,098 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:40:31,098 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:40:31,098 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:40:31,103 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:40:31,320 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:40:32,322 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:40:35,197 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:40:35,198 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:40:35,198 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:40:35,207 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:40:35,325 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:40:36,207 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:40:36,333 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:40:39,114 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:40:39,115 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:40:39,115 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:40:39,123 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:40:39,335 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:40:40,341 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:40:42,124 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:40:44,224 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:40:44,225 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:40:44,225 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:40:44,240 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:40:44,346 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:40:45,150 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:40:45,150 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:40:45,150 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:40:46,363 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:40:47,441 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:40:47,817 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:40:47,817 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:40:47,817 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:40:47,824 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:40:48,375 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:40:48,375 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:40:51,906 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:40:51,906 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:40:51,907 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:40:51,920 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:40:52,378 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:40:52,921 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:40:54,384 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:40:55,708 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 14:40:58,709 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:41:00,150 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:41:00,150 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:41:00,150 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:41:00,421 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:41:00,421 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:41:00,422 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:41:00,432 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:41:01,390 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:41:02,395 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:41:03,927 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:41:03,927 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:41:03,928 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:41:03,928 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:41:03,938 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:41:04,396 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:41:06,403 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:41:07,652 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:41:07,653 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:41:07,653 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:41:07,667 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:41:08,409 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:41:08,409 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:41:09,668 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:41:10,698 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:41:10,698 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:41:10,698 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:41:10,709 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:41:11,411 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:41:12,417 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:41:14,710 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:41:15,150 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:41:15,150 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:41:15,150 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:41:15,171 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:41:15,371 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:41:15,371 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:41:15,375 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:41:15,420 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:41:16,428 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:41:19,018 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:41:19,018 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:41:19,019 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:41:19,024 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:41:19,431 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:41:20,025 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:41:20,435 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:41:22,511 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:41:22,511 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:41:22,512 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:41:22,519 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:41:23,438 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:41:24,444 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:41:25,520 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:41:25,710 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 14:41:26,639 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:41:26,639 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:41:26,640 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:41:26,650 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:41:27,447 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:41:28,454 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:41:30,150 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:41:30,150 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:41:30,150 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:41:31,367 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:41:31,664 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:41:31,664 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:41:31,664 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:41:31,675 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:41:32,465 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:41:32,465 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:41:35,557 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:41:35,558 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:41:35,558 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:41:35,570 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:41:36,473 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:41:36,473 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:41:36,570 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:41:39,094 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:41:39,095 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:41:39,095 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:41:39,103 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:41:39,475 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:41:40,481 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:41:42,103 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:41:42,644 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:41:42,645 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:41:42,645 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:41:42,653 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:41:43,487 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:41:44,496 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:41:45,150 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:41:45,150 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:41:45,150 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:41:46,355 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:41:46,355 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:41:46,355 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:41:46,366 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:41:46,501 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:41:47,398 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:41:48,505 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:41:49,797 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:41:49,797 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:41:49,797 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:41:49,803 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:41:50,511 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:41:50,511 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:41:52,804 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:41:55,718 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 14:41:57,910 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:41:57,910 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:41:57,911 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:41:57,911 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:41:57,919 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:41:58,552 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:41:58,552 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:42:00,150 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:42:00,150 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:42:00,150 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:42:02,395 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:42:02,396 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:42:02,396 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:42:02,405 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:42:02,557 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:42:03,438 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:42:04,564 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:42:07,180 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:42:07,181 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:42:07,181 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:42:07,188 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:42:07,565 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:42:08,575 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:42:09,189 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:42:10,617 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:42:10,618 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:42:10,618 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:42:10,626 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:42:11,577 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:42:12,582 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:42:14,626 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:42:14,848 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:42:14,849 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:42:14,849 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:42:14,855 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:42:15,150 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:42:15,150 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:42:15,150 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:42:15,585 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:42:16,591 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:42:18,141 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:42:18,142 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:42:18,142 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:42:18,152 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:42:18,593 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:42:20,186 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:42:20,602 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:42:25,187 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:42:25,714 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 14:42:26,555 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:42:26,556 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:42:26,556 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:42:26,569 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:42:26,607 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:42:28,615 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:42:30,151 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:42:30,151 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:42:30,151 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:42:30,411 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:42:30,657 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:42:30,657 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:42:30,658 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:42:30,669 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:42:31,618 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:42:32,623 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:42:34,813 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:42:34,813 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:42:34,814 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:42:34,823 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:42:35,625 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:42:35,824 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:42:36,631 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:42:38,955 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:42:38,956 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:42:38,956 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:42:38,964 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:42:39,634 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:42:40,639 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:42:40,964 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:42:44,345 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:42:44,346 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:42:44,346 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:42:44,352 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:42:44,642 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:42:45,150 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:42:45,150 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:42:45,151 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:42:46,451 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:42:46,651 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:42:47,422 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:42:47,422 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:42:47,422 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:42:47,429 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:42:47,651 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:42:48,654 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:42:51,467 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:42:52,584 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:42:52,585 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:42:52,585 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:42:52,592 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:42:52,658 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:42:54,663 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:42:55,729 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 14:42:56,238 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:42:56,239 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:42:56,239 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:42:56,248 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:42:56,664 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:42:57,248 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:42:58,670 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:42:59,980 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:42:59,980 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:42:59,981 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:42:59,987 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:43:00,150 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:43:00,150 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:43:00,151 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:43:00,678 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:43:00,679 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:43:02,397 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:43:04,068 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:43:04,069 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:43:04,069 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:43:04,079 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:43:04,682 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:43:06,687 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:43:08,111 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:43:09,263 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:43:09,264 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:43:09,264 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:43:09,272 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:43:09,689 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:43:10,695 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:43:12,810 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:43:12,811 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:43:12,811 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:43:12,824 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:43:13,697 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:43:13,825 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:43:14,703 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:43:15,150 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:43:15,151 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:43:15,151 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:43:16,069 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:43:16,069 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:43:16,069 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:43:16,077 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:43:16,705 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:43:18,713 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:43:19,110 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:43:21,410 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:43:21,411 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:43:21,411 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:43:21,416 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:43:21,716 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:43:22,721 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:43:24,417 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:43:25,529 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:43:25,530 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:43:25,530 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:43:25,538 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:43:25,718 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 14:43:25,724 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:43:26,779 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:43:29,720 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:43:29,837 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:43:29,837 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:43:29,837 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:43:29,844 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:43:30,151 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:43:30,151 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:43:30,151 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:43:30,788 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:43:30,788 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:43:34,321 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:43:34,322 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:43:34,322 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:43:34,330 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:43:34,792 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:43:35,366 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:43:36,801 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:43:37,979 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:43:37,979 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:43:37,979 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:43:37,987 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:43:38,811 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:43:38,811 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:43:40,987 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:43:41,798 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:43:41,799 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:43:41,799 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:43:41,808 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:43:41,813 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:43:42,820 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:43:45,151 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:43:45,151 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:43:45,151 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:43:45,952 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:43:45,953 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:43:45,953 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:43:45,960 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:43:45,994 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:43:46,832 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:43:46,832 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:43:50,043 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:43:50,044 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:43:50,044 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:43:50,051 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:43:50,843 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:43:50,843 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:43:51,052 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:43:54,224 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:43:54,225 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:43:54,226 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:43:54,232 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:43:54,846 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:43:55,729 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 14:43:56,730 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:43:56,852 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:43:57,587 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:43:57,587 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:43:57,588 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:43:57,595 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:43:57,853 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:43:58,859 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:44:00,151 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:44:00,151 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:44:00,151 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:44:01,780 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:44:01,780 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:44:01,780 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:44:01,780 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:44:01,787 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:44:01,861 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:44:02,866 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:44:05,396 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:44:05,397 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:44:05,397 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:44:05,405 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:44:05,868 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:44:06,872 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:44:07,405 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:44:09,504 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:44:09,504 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:44:09,504 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:44:09,511 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:44:09,874 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:44:10,878 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:44:12,512 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:44:13,336 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:44:13,337 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:44:13,337 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:44:13,343 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:44:13,881 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:44:14,887 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:44:15,151 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:44:15,151 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:44:15,151 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:44:17,054 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:44:17,055 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:44:17,055 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:44:17,063 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:44:17,889 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:44:18,098 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:44:18,893 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:44:20,554 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:44:20,555 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:44:20,555 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:44:20,561 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:44:20,895 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:44:22,902 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:44:23,562 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:44:25,723 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 14:44:28,724 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:44:29,840 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:44:29,841 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:44:29,841 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:44:29,848 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:44:29,909 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:44:30,151 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:44:30,151 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:44:30,151 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:44:30,913 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:44:34,205 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:44:34,206 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:44:34,206 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:44:34,206 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:44:34,214 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:44:34,916 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:44:36,922 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:44:39,251 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:44:40,520 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:44:40,520 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:44:40,521 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:44:40,527 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:44:40,925 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:44:42,930 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:44:43,793 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:44:43,793 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:44:43,794 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:44:43,799 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:44:43,931 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:44:44,800 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:44:44,935 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:44:45,151 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:44:45,151 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:44:45,194 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:44:47,528 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:44:47,528 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:44:47,528 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:44:47,535 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:44:47,940 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:44:48,943 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:44:50,570 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:44:55,571 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:44:55,725 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 14:44:55,770 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:44:55,771 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:44:55,771 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:44:55,778 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:44:55,950 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:44:56,953 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:45:00,122 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:45:00,123 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:45:00,123 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:45:00,129 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:45:00,152 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:45:00,152 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:45:00,152 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:45:00,957 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:45:00,957 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:45:01,446 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:45:04,651 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:45:04,652 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:45:04,652 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:45:04,659 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:45:04,960 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:45:06,694 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:45:06,966 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:45:08,609 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:45:08,615 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:45:08,615 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:45:08,621 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:45:08,967 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:45:10,971 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:45:12,621 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:45:14,357 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:45:14,358 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:45:14,358 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:45:14,363 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:45:14,973 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:45:15,151 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:45:15,152 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:45:15,152 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:45:16,977 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:45:17,884 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:45:17,885 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:45:17,885 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:45:17,885 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:45:17,891 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:45:17,977 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:45:18,980 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:45:22,927 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:45:23,038 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:45:23,039 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:45:23,039 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:45:23,045 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:45:23,984 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:45:24,987 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:45:25,727 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 14:45:26,332 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:45:26,333 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:45:26,333 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:45:26,339 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:45:26,989 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:45:28,340 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:45:28,994 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:45:30,151 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:45:30,152 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:45:30,152 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:45:30,433 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:45:30,433 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:45:30,433 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:45:30,439 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:45:30,995 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:45:32,998 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:45:33,478 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:45:33,877 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:45:33,878 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:45:33,878 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:45:33,883 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:45:33,999 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:45:35,002 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:45:37,226 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:45:37,226 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:45:37,226 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:45:37,232 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:45:38,003 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:45:39,006 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:45:39,232 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:45:41,501 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:45:41,501 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:45:41,502 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:45:41,507 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:45:42,008 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:45:43,011 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:45:44,508 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:45:45,151 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:45:45,152 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:45:45,152 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:45:45,489 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:45:45,490 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:45:45,490 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:45:45,495 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:45:46,014 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:45:47,017 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:45:49,156 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:45:49,157 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:45:49,157 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:45:49,162 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:45:50,018 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:45:50,163 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:45:51,021 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:45:55,163 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:45:55,729 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 14:45:57,695 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:45:57,695 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:45:57,696 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:45:57,700 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:45:58,025 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:45:59,028 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:46:00,152 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:46:00,152 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:46:00,194 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:46:00,438 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:46:01,983 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:46:01,983 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:46:01,983 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:46:01,988 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:46:02,030 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:46:03,032 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:46:05,050 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:46:05,051 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:46:05,051 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:46:05,056 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:46:06,034 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:46:06,057 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:46:07,037 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:46:08,387 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:46:08,387 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:46:08,388 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:46:08,392 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:46:09,038 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:46:11,041 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:46:11,393 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:46:11,628 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:46:11,629 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:46:11,629 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:46:11,634 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:46:12,042 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:46:13,044 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:46:15,152 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:46:15,152 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:46:15,152 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:46:15,591 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:46:15,592 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:46:15,592 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:46:15,597 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:46:16,046 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:46:16,634 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:46:17,049 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:46:18,852 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:46:18,852 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:46:18,853 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:46:18,857 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:46:19,050 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:46:21,053 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:46:21,858 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:46:22,535 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:46:22,540 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:46:22,541 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:46:22,545 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:46:23,054 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:46:25,058 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:46:25,732 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 14:46:26,278 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:46:26,279 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:46:26,279 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:46:26,284 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:46:27,059 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:46:27,284 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:46:29,064 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:46:30,152 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:46:30,152 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:46:30,152 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:46:32,469 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:46:34,710 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:46:34,710 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:46:34,710 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:46:34,715 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:46:35,067 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:46:37,070 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:46:37,755 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:46:40,466 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:46:40,466 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:46:40,467 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:46:40,472 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:46:41,073 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:46:43,074 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:46:43,472 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:46:45,152 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:46:45,152 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:46:45,152 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:46:49,081 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:46:49,081 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:46:49,082 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:46:49,082 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:46:49,087 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:46:50,079 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:46:51,082 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:46:52,795 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:46:52,795 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:46:52,795 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:46:52,800 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:46:53,083 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:46:54,801 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:46:55,086 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:46:55,734 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 14:47:00,152 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:47:00,152 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:47:00,152 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:47:00,152 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:47:00,493 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:47:00,494 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:47:00,494 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:47:00,499 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:47:01,089 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:47:03,093 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:47:04,227 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:47:04,228 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:47:04,228 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:47:04,233 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:47:05,096 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:47:05,096 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:47:05,233 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:47:07,380 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:47:07,381 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:47:07,381 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:47:07,386 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:47:08,098 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:47:09,101 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:47:10,387 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:47:11,220 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:47:11,221 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:47:11,221 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:47:11,226 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:47:12,102 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:47:13,105 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:47:14,508 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:47:14,509 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:47:14,509 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:47:14,514 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:47:15,106 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:47:15,152 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:47:15,152 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:47:15,194 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:47:16,371 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:47:17,109 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:47:18,965 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:47:18,965 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:47:18,966 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:47:18,970 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:47:19,110 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:47:21,113 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:47:22,010 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:47:25,104 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:47:25,105 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:47:25,105 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:47:25,110 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:47:25,116 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:47:25,747 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 14:47:27,119 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:47:27,749 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:47:28,382 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:47:28,383 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:47:28,383 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:47:28,388 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:47:29,121 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:47:30,152 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:47:30,152 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:47:30,152 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:47:31,124 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:47:31,477 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:47:31,478 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:47:31,478 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:47:31,483 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:47:32,125 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:47:33,127 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:47:33,483 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:47:38,281 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:47:38,282 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:47:38,282 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:47:38,287 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:47:39,132 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:47:39,133 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:47:39,287 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:47:41,932 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:47:41,933 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:47:41,933 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:47:41,938 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:47:42,134 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:47:43,137 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:47:44,939 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:47:45,152 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:47:45,152 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:47:45,153 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:47:46,479 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:47:46,480 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:47:46,480 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:47:46,484 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:47:47,141 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:47:49,144 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:47:50,366 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:47:50,367 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:47:50,367 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:47:50,367 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:47:50,372 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:47:51,145 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:47:53,148 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:47:53,819 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:47:53,820 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:47:53,820 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:47:53,827 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:47:54,149 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:47:55,152 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:47:55,739 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 14:47:55,740 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:47:59,072 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:47:59,073 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:47:59,073 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:47:59,078 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:47:59,154 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:48:00,152 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:48:00,153 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:48:00,153 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:48:01,156 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:48:01,381 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:48:02,703 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:48:02,703 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:48:02,703 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:48:02,708 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:48:03,157 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:48:05,160 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:48:05,829 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:48:05,830 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:48:05,830 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:48:05,835 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:48:06,161 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:48:06,836 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:48:07,164 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:48:09,433 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:48:09,434 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:48:09,434 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:48:09,439 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:48:10,165 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:48:11,168 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:48:12,440 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:48:15,152 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:48:15,153 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:48:15,153 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:48:17,455 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:48:17,788 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:48:17,788 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:48:17,788 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:48:17,793 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:48:18,172 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:48:19,174 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:48:21,495 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:48:21,496 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:48:21,496 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:48:21,501 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:48:22,176 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:48:22,502 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:48:23,176 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:48:24,943 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:48:24,944 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:48:24,944 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:48:24,949 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:48:25,178 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:48:25,743 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 14:48:27,181 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:48:27,744 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:48:29,483 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:48:29,484 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:48:29,484 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:48:29,489 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:48:30,152 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:48:30,153 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:48:30,186 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:48:30,194 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:48:31,189 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:48:32,947 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:48:32,947 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:48:32,948 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:48:32,948 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:48:32,952 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:48:33,190 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:48:35,193 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:48:37,991 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:48:38,897 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:48:38,897 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:48:38,898 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:48:38,902 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:48:39,195 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:48:41,198 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:48:42,398 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:48:42,399 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:48:42,399 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:48:42,404 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:48:43,201 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:48:43,201 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:48:43,404 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:48:45,153 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:48:45,153 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:48:45,153 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:48:47,205 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:48:47,206 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:48:47,206 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:48:47,211 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:48:48,205 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:48:49,207 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:48:49,250 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:48:51,093 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:48:51,093 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:48:51,094 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:48:51,099 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:48:51,209 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:48:53,212 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:48:55,099 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:48:55,743 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 14:48:56,203 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:48:56,203 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:48:56,203 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:48:56,208 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:48:56,213 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:48:57,216 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:48:59,975 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:48:59,976 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:48:59,976 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:48:59,985 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:49:00,152 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:49:00,153 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:49:00,153 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:49:00,154 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:49:00,217 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:49:01,221 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:49:04,234 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:49:04,235 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:49:04,235 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:49:04,240 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:49:05,223 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:49:05,223 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:49:05,240 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:49:07,734 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:49:07,735 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:49:07,735 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:49:07,740 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:49:08,225 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:49:09,228 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:49:10,740 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:49:11,207 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:49:11,207 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:49:11,207 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:49:11,212 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:49:11,229 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:49:13,232 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:49:14,967 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:49:14,968 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:49:14,968 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:49:14,973 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:49:15,153 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:49:15,153 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:49:15,153 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:49:15,233 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:49:16,460 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:49:17,236 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:49:19,044 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:49:19,045 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:49:19,045 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:49:19,050 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:49:19,237 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:49:21,240 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:49:22,050 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:49:24,559 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:49:24,560 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:49:24,560 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:49:24,566 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:49:25,242 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:49:25,745 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 14:49:27,243 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:49:27,746 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:49:28,483 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:49:28,484 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:49:28,484 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:49:28,489 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:49:29,246 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:49:30,153 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:49:30,153 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:49:30,153 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:49:31,249 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:49:32,347 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:49:32,347 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:49:32,347 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:49:32,352 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:49:33,252 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:49:33,253 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:49:33,393 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:49:36,453 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:49:36,453 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:49:36,454 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:49:36,458 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:49:37,256 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:49:37,257 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:49:38,458 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:49:40,223 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:49:40,224 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:49:40,224 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:49:40,229 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:49:40,258 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:49:41,261 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:49:44,229 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:49:45,153 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:49:45,153 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:49:45,153 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:49:47,935 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:49:47,936 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:49:47,936 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:49:47,941 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:49:48,266 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:49:49,268 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:49:49,942 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:49:51,512 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:49:51,513 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:49:51,513 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:49:51,518 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:49:52,270 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:49:53,272 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:49:55,012 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:49:55,013 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:49:55,014 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:49:55,014 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:49:55,019 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:49:55,274 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:49:55,747 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 14:49:57,276 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:49:59,767 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:49:59,768 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:49:59,768 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:49:59,773 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:50:00,153 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:50:00,153 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:50:00,153 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:50:00,153 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:50:00,278 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:50:01,280 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:50:03,658 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:50:03,658 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:50:03,659 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:50:03,664 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:50:04,283 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:50:05,286 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:50:05,703 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:50:06,895 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:50:06,896 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:50:06,896 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:50:06,901 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:50:07,287 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:50:09,290 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:50:10,399 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:50:10,400 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:50:10,400 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:50:10,405 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:50:11,293 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:50:11,293 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:50:11,405 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:50:14,141 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:50:14,141 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:50:14,141 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:50:14,146 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:50:14,295 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:50:15,153 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:50:15,154 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:50:15,154 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:50:15,297 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:50:17,374 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:50:17,375 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:50:17,375 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:50:17,375 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:50:17,380 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:50:18,299 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:50:19,301 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:50:22,419 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:50:23,134 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:50:23,135 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:50:23,135 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:50:23,140 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:50:23,304 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:50:25,307 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:50:25,749 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 14:50:27,751 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:50:28,723 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:50:28,723 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:50:28,724 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:50:28,728 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:50:29,311 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:50:30,153 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:50:30,153 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:50:30,154 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:50:31,314 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:50:32,878 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:50:32,879 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:50:32,879 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:50:32,879 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:50:32,884 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:50:33,315 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:50:35,318 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:50:37,923 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:50:38,266 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:50:38,266 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:50:38,266 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:50:38,271 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:50:38,320 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:50:39,323 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:50:42,608 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:50:42,609 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:50:42,609 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:50:42,614 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:50:43,325 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:50:43,614 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:50:45,153 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:50:45,153 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:50:45,154 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:50:45,328 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:50:47,569 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:50:47,570 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:50:47,570 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:50:47,574 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:50:48,331 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:50:49,333 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:50:49,614 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:50:50,928 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:50:50,932 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:50:50,933 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:50:50,937 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:50:51,334 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:50:53,337 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:50:54,501 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:50:54,502 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:50:54,502 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:50:54,507 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:50:55,341 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:50:55,341 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:50:55,507 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:50:55,752 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 14:50:57,847 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:50:57,847 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:50:57,847 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:50:57,852 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:50:58,342 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:50:59,345 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:51:00,153 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:51:00,154 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:51:00,154 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:51:01,463 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:51:02,168 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:51:02,169 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:51:02,169 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:51:02,174 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:51:02,347 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:51:03,349 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:51:06,553 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:51:06,554 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:51:06,554 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:51:06,554 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:51:06,559 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:51:07,354 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:51:07,354 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:51:10,842 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:51:10,843 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:51:10,843 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:51:10,848 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:51:11,356 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:51:11,848 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:51:13,359 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:51:14,622 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:51:14,623 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:51:14,623 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:51:14,628 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:51:15,153 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:51:15,154 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:51:15,154 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:51:15,360 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:51:17,363 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:51:17,457 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:51:21,117 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:51:21,118 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:51:21,118 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:51:21,123 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:51:21,365 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:51:23,124 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:51:23,368 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:51:25,754 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 14:51:28,755 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:51:30,153 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:51:30,154 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:51:30,154 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:51:30,197 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:51:30,446 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:51:30,446 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:51:30,450 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:51:31,376 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:51:31,376 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:51:34,451 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:51:38,585 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:51:38,586 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:51:38,586 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:51:38,591 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:51:39,383 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:51:39,383 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:51:39,592 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:51:44,592 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:51:45,154 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:51:45,154 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:51:45,154 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:51:47,037 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:51:47,038 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:51:47,038 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:51:47,043 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:51:47,388 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:51:49,391 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:51:50,083 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:51:52,321 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:51:52,321 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:51:52,322 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:51:52,327 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:51:52,393 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:51:53,395 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:51:55,327 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:51:55,759 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 14:52:00,154 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:52:00,154 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:52:00,154 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:52:00,418 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:52:00,748 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:52:00,749 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:52:00,749 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:52:00,754 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:52:01,399 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:52:03,403 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:52:04,342 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:52:04,342 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:52:04,343 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:52:04,347 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:52:04,403 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:52:05,404 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:52:06,348 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:52:07,498 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:52:07,499 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:52:07,499 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:52:07,504 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:52:08,406 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:52:09,408 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:52:11,505 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:52:11,879 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:52:11,880 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:52:11,880 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:52:11,885 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:52:12,410 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:52:13,412 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:52:15,154 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:52:15,154 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:52:15,154 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:52:16,982 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:52:16,983 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:52:16,983 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:52:16,983 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:52:16,988 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:52:17,414 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:52:19,417 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:52:21,103 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:52:21,104 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:52:21,104 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:52:21,109 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:52:21,418 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:52:22,109 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:52:23,421 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:52:25,759 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 14:52:27,377 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:52:27,378 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:52:27,378 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:52:27,379 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:52:27,384 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:52:27,424 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:52:29,428 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:52:30,154 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:52:30,154 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:52:30,154 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:52:31,130 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:52:31,131 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:52:31,131 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:52:31,136 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:52:31,429 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:52:33,174 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:52:33,432 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:52:36,218 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:52:36,218 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:52:36,219 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:52:36,223 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:52:36,434 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:52:37,437 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:52:38,224 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:52:40,683 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:52:40,684 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:52:40,684 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:52:40,689 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:52:41,439 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:52:43,440 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:52:43,690 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:52:45,154 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:52:45,154 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:52:45,154 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:52:45,162 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:52:45,351 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:52:45,351 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:52:45,356 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:52:45,441 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:52:47,446 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:52:48,223 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:52:48,224 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:52:48,224 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:52:48,229 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:52:48,446 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:52:49,229 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:52:49,449 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:52:52,519 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:52:52,520 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:52:52,520 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:52:52,525 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:52:53,453 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:52:53,453 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:52:54,526 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:52:55,764 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 14:52:59,765 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:53:00,154 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:53:00,154 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:53:00,154 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:53:00,921 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:53:00,921 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:53:00,921 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:53:00,927 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:53:01,457 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:53:03,461 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:53:04,963 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:53:05,048 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:53:05,048 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:53:05,048 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:53:05,053 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:53:05,462 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:53:07,465 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:53:08,961 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:53:08,962 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:53:08,962 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:53:08,967 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:53:09,466 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:53:09,967 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:53:11,469 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:53:14,134 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:53:14,134 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:53:14,134 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:53:14,139 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:53:14,471 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:53:15,140 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:53:15,154 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:53:15,154 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:53:15,154 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:53:15,473 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:53:18,218 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:53:18,218 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:53:18,219 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:53:18,223 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:53:18,475 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:53:19,478 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:53:20,224 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:53:25,224 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:53:25,763 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 14:53:26,684 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:53:26,685 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:53:26,685 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:53:26,689 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:53:27,485 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:53:27,485 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:53:30,154 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:53:30,154 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:53:30,155 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:53:30,384 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:53:30,565 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:53:30,566 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:53:30,566 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:53:30,570 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:53:31,490 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:53:31,490 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:53:33,996 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:53:33,997 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:53:33,998 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:53:34,002 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:53:34,492 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:53:35,492 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:53:36,003 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:53:38,158 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:53:38,159 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:53:38,159 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:53:38,164 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:53:38,494 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:53:39,496 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:53:41,165 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:53:41,821 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:53:41,822 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:53:41,822 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:53:41,827 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:53:42,498 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:53:43,501 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:53:45,154 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:53:45,154 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:53:45,155 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:53:45,291 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:53:45,462 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:53:45,462 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:53:45,467 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:53:45,502 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:53:46,468 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:53:47,506 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:53:48,645 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:53:48,646 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:53:48,646 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:53:48,651 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:53:49,509 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:53:49,510 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:53:51,651 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:53:53,352 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:53:53,353 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:53:53,353 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:53:53,358 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:53:53,512 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:53:55,515 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:53:55,765 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 14:53:56,767 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:53:59,354 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:53:59,355 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:53:59,355 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:53:59,360 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:53:59,517 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:54:00,154 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:54:00,155 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:54:00,155 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:54:01,520 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:54:02,363 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:54:03,917 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:54:03,918 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:54:03,918 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:54:03,923 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:54:04,522 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:54:05,525 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:54:07,963 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:54:08,148 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:54:08,148 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:54:08,148 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:54:08,153 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:54:08,526 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:54:09,529 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:54:13,154 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:54:13,608 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:54:13,609 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:54:13,609 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:54:13,614 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:54:14,532 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:54:15,154 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:54:15,155 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:54:15,155 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:54:15,534 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:54:17,844 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:54:17,845 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:54:17,845 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:54:17,850 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:54:18,536 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:54:18,886 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:54:19,538 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:54:21,396 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:54:21,397 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:54:21,397 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:54:21,402 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:54:21,539 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:54:23,543 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:54:24,403 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:54:25,768 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 14:54:26,212 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:54:26,213 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:54:26,213 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:54:26,218 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:54:26,545 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:54:27,547 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:54:29,930 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:54:29,931 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:54:29,931 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:54:29,931 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:54:29,936 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:54:30,155 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:54:30,155 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:54:30,155 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:54:30,550 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:54:31,553 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:54:33,048 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:54:33,048 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:54:33,049 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:54:33,054 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:54:33,554 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:54:35,055 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:54:35,557 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:54:37,526 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:54:37,526 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:54:37,527 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:54:37,532 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:54:37,558 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:54:39,561 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:54:40,532 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:54:42,314 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:54:42,315 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:54:42,315 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:54:42,320 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:54:42,563 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:54:43,565 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:54:45,155 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:54:45,155 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:54:45,155 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:54:46,458 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:54:46,629 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:54:46,629 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:54:46,630 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:54:46,635 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:54:47,571 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:54:47,571 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:54:50,716 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:54:50,717 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:54:50,717 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:54:50,722 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:54:51,573 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:54:51,573 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:54:51,723 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:54:54,087 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:54:54,088 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:54:54,088 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:54:54,093 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:54:54,575 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:54:55,577 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:54:55,769 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 14:54:56,771 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:54:57,327 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:54:57,328 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:54:57,328 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:54:57,333 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:54:57,578 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:54:59,581 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:55:00,155 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:55:00,155 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:55:00,155 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:55:00,625 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:55:00,626 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:55:00,626 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:55:00,631 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:55:01,584 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:55:01,584 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:55:02,671 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:55:04,252 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:55:04,253 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:55:04,253 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:55:04,258 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:55:04,586 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:55:05,589 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:55:08,259 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:55:09,155 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:55:09,156 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:55:09,156 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:55:09,161 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:55:09,591 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:55:11,594 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:55:13,190 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:55:13,190 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:55:13,191 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:55:13,196 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:55:13,595 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:55:14,196 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:55:15,155 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:55:15,155 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:55:15,155 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:55:15,599 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:55:18,964 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:55:18,965 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:55:18,965 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:55:18,970 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:55:19,601 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:55:20,007 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:55:21,604 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:55:24,351 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:55:24,351 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:55:24,352 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:55:24,357 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:55:24,606 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:55:25,357 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:55:25,609 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:55:25,774 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 14:55:30,155 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:55:30,155 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:55:30,155 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:55:30,164 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:55:30,407 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:55:30,408 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:55:30,408 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:55:30,413 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:55:30,613 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:55:31,615 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:55:35,414 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:55:35,563 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:55:35,564 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:55:35,564 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:55:35,569 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:55:35,618 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:55:37,621 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:55:40,308 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:55:40,309 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:55:40,310 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:55:40,314 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:55:40,622 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:55:41,315 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:55:41,625 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:55:44,387 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:55:44,388 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:55:44,388 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:55:44,393 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:55:44,626 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:55:45,155 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:55:45,155 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:55:45,155 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:55:45,629 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:55:46,438 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:55:48,533 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:55:48,534 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:55:48,534 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:55:48,539 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:55:48,632 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:55:49,634 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:55:51,579 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:55:53,023 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:55:53,024 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:55:53,024 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:55:53,029 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:55:53,636 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:55:55,639 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:55:55,774 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 14:55:56,775 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:55:57,321 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:55:57,322 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:55:57,322 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:55:57,327 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:55:57,640 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:55:59,643 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:56:00,155 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:56:00,155 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:56:00,155 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:56:02,431 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:56:03,083 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:56:03,083 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:56:03,084 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:56:03,088 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:56:03,646 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:56:05,649 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:56:08,127 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:56:11,369 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:56:11,370 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:56:11,370 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:56:11,375 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:56:11,652 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:56:13,375 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:56:13,655 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:56:15,062 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:56:15,063 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:56:15,063 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:56:15,068 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:56:15,155 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:56:15,155 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:56:15,155 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:56:15,657 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:56:17,660 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:56:18,410 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:56:18,808 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:56:18,809 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:56:18,810 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:56:18,815 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:56:19,663 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:56:19,663 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:56:22,130 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:56:22,130 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:56:22,131 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:56:22,136 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:56:22,665 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:56:23,667 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:56:24,136 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:56:25,782 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 14:56:26,912 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:56:26,913 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:56:26,913 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:56:26,918 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:56:27,671 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:56:27,672 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:56:29,918 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:56:30,157 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:56:30,157 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:56:30,157 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:56:30,220 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:56:30,426 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:56:30,426 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:56:30,431 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:56:30,674 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:56:31,677 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:56:33,753 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:56:33,753 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:56:33,754 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:56:33,759 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:56:34,679 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:56:35,681 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:56:35,760 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:56:37,508 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:56:37,509 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:56:37,509 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:56:37,514 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:56:37,682 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:56:39,685 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:56:41,515 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:56:41,882 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:56:41,883 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:56:41,883 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:56:41,888 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:56:42,687 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:56:43,689 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:56:45,157 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:56:45,157 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:56:45,157 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:56:45,754 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:56:45,755 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:56:45,755 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:56:45,760 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:56:46,692 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:56:46,798 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:56:47,695 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:56:49,534 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:56:49,535 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:56:49,535 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:56:49,539 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:56:49,696 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:56:51,699 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:56:52,540 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:56:52,859 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:56:52,860 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:56:52,860 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:56:52,865 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:56:53,702 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:56:53,702 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:56:55,778 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 14:56:56,588 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:56:56,589 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:56:56,590 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:56:56,594 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:56:56,704 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:56:57,595 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:56:57,705 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:57:00,157 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:57:00,157 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:57:00,157 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:57:00,258 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:57:00,348 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:57:00,348 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:57:00,353 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:57:00,706 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:57:01,709 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:57:03,354 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:57:05,450 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:57:05,450 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:57:05,451 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:57:05,455 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:57:05,711 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:57:07,714 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:57:08,456 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:57:13,457 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:57:13,897 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:57:13,898 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:57:13,899 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:57:13,904 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:57:14,718 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:57:15,157 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:57:15,157 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:57:15,157 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:57:15,721 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:57:19,368 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:57:22,904 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:57:22,905 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:57:22,906 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:57:22,910 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:57:23,727 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:57:23,727 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:57:24,947 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:57:25,783 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 14:57:26,136 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:57:26,137 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:57:26,137 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:57:26,142 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:57:26,729 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:57:27,732 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:57:30,143 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:57:30,157 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:57:30,157 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:57:30,157 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:57:30,539 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:57:30,539 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:57:30,540 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:57:30,544 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:57:30,735 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:57:31,737 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:57:33,792 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:57:33,793 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:57:33,793 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:57:33,798 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:57:34,739 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:57:35,740 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:57:35,798 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:57:37,023 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:57:37,024 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:57:37,024 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:57:37,030 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:57:37,741 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:57:39,744 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:57:41,030 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:57:42,183 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:57:42,183 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:57:42,184 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:57:42,188 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:57:42,746 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:57:43,748 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:57:45,157 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:57:45,157 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:57:45,158 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:57:46,261 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:57:46,262 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:57:46,262 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:57:46,262 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:57:46,267 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:57:46,751 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:57:47,754 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:57:51,075 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:57:51,076 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:57:51,077 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:57:51,081 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:57:51,756 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:57:52,082 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:57:53,759 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:57:55,783 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 14:57:57,784 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:58:00,157 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:58:00,157 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:58:00,158 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:58:00,347 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:58:00,417 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:58:00,417 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:58:00,422 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:58:00,763 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:58:01,765 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:58:03,422 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:58:08,423 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:58:08,554 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:58:08,555 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:58:08,555 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:58:08,560 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:58:08,770 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:58:09,772 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:58:13,599 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:58:15,157 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:58:15,157 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:58:15,158 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:58:16,819 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:58:16,820 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:58:16,820 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:58:16,824 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:58:17,778 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:58:17,779 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:58:18,862 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:58:19,900 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:58:19,900 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:58:19,900 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:58:19,905 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:58:20,780 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:58:21,782 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:58:23,906 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:58:24,382 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:58:24,383 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:58:24,383 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:58:24,388 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:58:24,784 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:58:25,785 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 14:58:25,788 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:58:28,124 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:58:28,125 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:58:28,125 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:58:28,130 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:58:28,790 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:58:29,130 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:58:29,794 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:58:30,157 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:58:30,157 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:58:30,158 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:58:31,234 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:58:31,234 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:58:31,234 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:58:31,239 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:58:31,795 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:58:33,798 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:58:34,278 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:58:34,578 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:58:34,578 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:58:34,578 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:58:34,583 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:58:34,799 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:58:35,801 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:58:37,706 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:58:37,707 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:58:37,707 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:58:37,712 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:58:37,802 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:58:39,712 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:58:39,805 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:58:41,012 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:58:41,013 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:58:41,013 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:58:41,017 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:58:41,808 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:58:41,809 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:58:45,018 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:58:45,157 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:58:45,158 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:58:45,158 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:58:45,715 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:58:45,715 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:58:45,716 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:58:45,721 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:58:45,811 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:58:47,815 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:58:50,759 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:58:51,079 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:58:51,080 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:58:51,080 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:58:51,085 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:58:51,819 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:58:51,819 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:58:55,787 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 14:58:55,788 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:58:57,289 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:58:57,290 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:58:57,290 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:58:57,294 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:58:57,823 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:58:59,826 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:59:00,157 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:59:00,158 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:59:00,158 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:59:00,703 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:59:00,703 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:59:00,704 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:59:00,708 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:59:00,826 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:59:01,746 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:59:01,829 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:59:04,297 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:59:04,297 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:59:04,298 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:59:04,302 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:59:04,831 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:59:05,834 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:59:07,303 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:59:10,005 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:59:10,006 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:59:10,006 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:59:10,011 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:59:10,836 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:59:11,839 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:59:13,012 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:59:13,245 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:59:13,245 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:59:13,245 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:59:13,250 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:59:13,840 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:59:15,158 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:59:15,158 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:59:15,158 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:59:15,843 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:59:17,194 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:59:17,194 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:59:17,194 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:59:17,199 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:59:17,844 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:59:18,238 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:59:19,848 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:59:20,808 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:59:20,809 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:59:20,809 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:59:20,814 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:59:20,848 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:59:21,851 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:59:23,815 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:59:25,161 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:59:25,162 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:59:25,162 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:59:25,168 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:59:25,790 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 14:59:25,853 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:59:27,856 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:59:29,791 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:59:30,158 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:59:30,158 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:59:30,158 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:59:30,583 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:59:30,583 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:59:30,583 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:59:30,588 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:59:30,859 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:59:31,862 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:59:34,356 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:59:34,357 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:59:34,358 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:59:34,363 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:59:34,863 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:59:35,363 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:59:35,866 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:59:37,891 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:59:37,892 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:59:37,893 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:59:37,898 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:59:38,868 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:59:39,868 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:59:40,898 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:59:43,038 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:59:43,039 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:59:43,039 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:59:43,043 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:59:43,872 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:59:43,873 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:59:45,158 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:59:45,158 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:59:45,158 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:59:46,370 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:59:51,371 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:59:51,517 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:59:51,518 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:59:51,518 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:59:51,524 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:59:51,878 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 14:59:53,882 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 14:59:55,795 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 14:59:56,797 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:59:59,838 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:59:59,839 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 14:59:59,839 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:59:59,844 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:59:59,885 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:00:00,158 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:00:00,158 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:00:00,158 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:00:01,888 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:00:02,435 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:00:03,741 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:00:03,741 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:00:03,742 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:00:03,747 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:00:03,889 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:00:05,892 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:00:07,069 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:00:07,069 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:00:07,070 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:00:07,074 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:00:07,893 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:00:07,894 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:00:08,075 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:00:10,303 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:00:10,304 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:00:10,304 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:00:10,308 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:00:10,895 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:00:11,897 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:00:13,309 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:00:14,413 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:00:14,414 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:00:14,414 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:00:14,419 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:00:14,899 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:00:15,158 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:00:15,158 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:00:15,201 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:00:15,901 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:00:18,380 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:00:23,381 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:00:23,765 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:00:23,766 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:00:23,766 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:00:23,770 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:00:23,906 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:00:25,795 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 15:00:25,909 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:00:26,634 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:00:26,635 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:00:26,635 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:00:26,640 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:00:26,911 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:00:27,913 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:00:28,640 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:00:30,158 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:00:30,158 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:00:30,158 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:00:30,311 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:00:30,394 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:00:30,395 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:00:30,399 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:00:30,916 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:00:31,919 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:00:33,951 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:00:33,952 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:00:33,952 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:00:33,952 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:00:33,957 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:00:34,920 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:00:35,923 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:00:37,572 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:00:37,573 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:00:37,573 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:00:37,578 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:00:37,924 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:00:39,578 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:00:39,927 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:00:41,293 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:00:41,294 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:00:41,294 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:00:41,299 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:00:41,929 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:00:43,932 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:00:44,656 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:00:44,656 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:00:44,656 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:00:44,657 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:00:44,661 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:00:44,932 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:00:45,158 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:00:45,158 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:00:45,158 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:00:45,935 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:00:50,236 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:00:50,237 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:00:50,237 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:00:50,237 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:00:50,243 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:00:50,939 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:00:51,942 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:00:55,279 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:00:55,799 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 15:00:58,768 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:00:58,769 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:00:58,769 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:00:58,774 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:00:58,946 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:00:59,948 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:01:00,158 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:01:00,158 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:01:00,159 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:01:00,451 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:01:02,047 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:01:02,048 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:01:02,048 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:01:02,053 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:01:02,950 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:01:03,953 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:01:06,091 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:01:07,162 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:01:07,162 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:01:07,163 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:01:07,167 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:01:07,957 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:01:07,957 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:01:10,698 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:01:10,698 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:01:10,699 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:01:10,703 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:01:10,959 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:01:11,704 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:01:11,961 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:01:14,920 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:01:14,921 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:01:14,921 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:01:14,926 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:01:14,963 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:01:15,158 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:01:15,158 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:01:15,159 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:01:15,966 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:01:17,455 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:01:22,456 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:01:23,248 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:01:23,248 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:01:23,249 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:01:23,253 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:01:23,970 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:01:25,799 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 15:01:25,974 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:01:27,123 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:01:27,124 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:01:27,124 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:01:27,128 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:01:27,977 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:01:27,977 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:01:28,129 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:01:30,158 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:01:30,159 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:01:30,159 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:01:31,680 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:01:31,680 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:01:31,681 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:01:31,686 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:01:31,981 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:01:33,723 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:01:33,984 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:01:34,939 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:01:34,940 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:01:34,940 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:01:34,945 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:01:34,984 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:01:35,987 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:01:38,898 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:01:38,899 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:01:38,899 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:01:38,899 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:01:38,904 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:01:38,988 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:01:39,991 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:01:43,418 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:01:43,419 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:01:43,420 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:01:43,425 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:01:43,993 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:01:44,425 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:01:45,158 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:01:45,158 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:01:45,159 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:01:45,997 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:01:47,870 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:01:47,871 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:01:47,871 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:01:47,876 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:01:47,999 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:01:49,914 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:01:50,002 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:01:52,443 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:01:52,444 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:01:52,444 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:01:52,449 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:01:53,004 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:01:54,007 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:01:55,450 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:01:55,801 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 15:01:55,902 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:01:55,903 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:01:55,903 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:01:55,908 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:01:56,008 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:01:58,011 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:02:00,159 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:02:00,159 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:02:00,159 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:02:00,270 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:02:00,420 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:02:00,420 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:02:00,425 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:02:01,013 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:02:01,426 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:02:02,015 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:02:03,491 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:02:03,492 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:02:03,492 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:02:03,497 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:02:04,017 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:02:06,020 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:02:06,498 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:02:07,200 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:02:07,201 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:02:07,202 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:02:07,210 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:02:08,023 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:02:08,023 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:02:10,948 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:02:10,949 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:02:10,949 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:02:10,954 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:02:11,024 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:02:11,954 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:02:12,027 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:02:14,458 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:02:14,459 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:02:14,459 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:02:14,464 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:02:15,029 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:02:15,159 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:02:15,159 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:02:15,159 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:02:16,031 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:02:17,415 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:02:19,465 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:02:19,466 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:02:19,466 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:02:19,471 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:02:20,034 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:02:22,037 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:02:22,472 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:02:23,225 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:02:23,226 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:02:23,226 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:02:23,231 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:02:24,040 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:02:24,040 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:02:25,803 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 15:02:27,804 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:02:28,805 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:02:28,806 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:02:28,806 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:02:28,811 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:02:29,043 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:02:30,047 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:02:30,159 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:02:30,159 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:02:30,159 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:02:33,439 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:02:37,109 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:02:37,110 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:02:37,110 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:02:37,115 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:02:38,054 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:02:38,054 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:02:39,154 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:02:42,786 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:02:42,786 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:02:42,787 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:02:42,791 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:02:43,056 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:02:44,059 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:02:44,792 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:02:45,159 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:02:45,159 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:02:45,159 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:02:46,137 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:02:46,138 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:02:46,138 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:02:46,143 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:02:47,062 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:02:48,063 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:02:50,183 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:02:51,039 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:02:51,040 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:02:51,040 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:02:51,045 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:02:51,064 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:02:52,067 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:02:55,282 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:02:55,283 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:02:55,283 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:02:55,283 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:02:55,288 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:02:55,805 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 15:02:56,071 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:02:56,071 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:02:59,264 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:02:59,265 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:02:59,265 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:02:59,270 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:03:00,075 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:03:00,075 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:03:00,159 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:03:00,159 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:03:00,202 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:03:00,419 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:03:03,190 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:03:03,190 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:03:03,191 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:03:03,195 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:03:04,080 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:03:04,080 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:03:06,235 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:03:07,197 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:03:07,198 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:03:07,198 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:03:07,203 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:03:08,084 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:03:08,085 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:03:11,052 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:03:11,052 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:03:11,053 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:03:11,057 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:03:11,086 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:03:12,058 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:03:12,088 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:03:15,159 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:03:15,159 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:03:15,159 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:03:15,691 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:03:15,696 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:03:15,696 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:03:15,700 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:03:16,091 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:03:17,734 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:03:18,094 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:03:18,878 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:03:18,879 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:03:18,879 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:03:18,883 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:03:19,095 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:03:20,097 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:03:21,984 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:03:21,984 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:03:21,985 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:03:21,990 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:03:22,099 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:03:22,990 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:03:24,102 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:03:25,807 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 15:03:27,298 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:03:27,299 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:03:27,299 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:03:27,304 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:03:28,107 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:03:28,107 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:03:28,305 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:03:30,159 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:03:30,159 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:03:30,160 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:03:30,530 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:03:30,531 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:03:30,531 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:03:30,535 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:03:31,110 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:03:32,112 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:03:33,574 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:03:38,562 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:03:38,562 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:03:38,563 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:03:38,567 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:03:39,116 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:03:39,568 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:03:40,119 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:03:42,207 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:03:42,207 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:03:42,208 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:03:42,212 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:03:43,120 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:03:44,123 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:03:45,159 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:03:45,159 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:03:45,159 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:03:45,160 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:03:46,288 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:03:46,288 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:03:46,289 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:03:46,293 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:03:47,126 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:03:48,128 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:03:50,331 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:03:50,777 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:03:50,778 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:03:50,778 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:03:50,783 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:03:51,130 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:03:52,132 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:03:55,784 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:03:55,809 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 15:03:56,229 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:03:56,229 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:03:56,230 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:03:56,234 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:03:57,135 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:03:58,138 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:04:00,159 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:04:00,159 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:04:00,160 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:04:00,202 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:04:00,441 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:04:00,441 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:04:00,446 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:04:01,139 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:04:01,447 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:04:02,142 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:04:03,744 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:04:03,745 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:04:03,745 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:04:03,750 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:04:04,144 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:04:06,147 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:04:06,751 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:04:07,859 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:04:07,859 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:04:07,860 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:04:07,865 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:04:08,148 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:04:10,152 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:04:11,866 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:04:15,159 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:04:15,159 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:04:15,160 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:04:16,448 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:04:16,449 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:04:16,449 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:04:16,454 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:04:17,157 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:04:17,490 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:04:18,159 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:04:20,599 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:04:20,599 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:04:20,599 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:04:20,605 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:04:21,161 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:04:22,164 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:04:22,605 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:04:25,630 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:04:25,631 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:04:25,631 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:04:25,636 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:04:25,815 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 15:04:26,166 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:04:27,817 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:04:28,170 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:04:30,159 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:04:30,160 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:04:30,160 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:04:30,404 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:04:30,404 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:04:30,404 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:04:30,409 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:04:31,173 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:04:32,175 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:04:33,446 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:04:36,225 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:04:36,226 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:04:36,226 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:04:36,231 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:04:37,178 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:04:38,181 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:04:39,231 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:04:44,232 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:04:44,645 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:04:44,646 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:04:44,646 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:04:44,651 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:04:45,159 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:04:45,160 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:04:45,160 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:04:45,185 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:04:46,188 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:04:48,101 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:04:48,102 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:04:48,102 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:04:48,107 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:04:48,191 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:04:50,146 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:04:50,194 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:04:53,250 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:04:53,250 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:04:53,251 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:04:53,256 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:04:54,198 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:04:54,198 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:04:55,256 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:04:55,814 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 15:04:56,803 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:04:56,804 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:04:56,804 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:04:56,809 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:04:57,200 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:04:58,202 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:05:00,094 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:05:00,094 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:05:00,095 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:05:00,100 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:05:00,160 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:05:00,160 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:05:00,201 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:05:00,203 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:05:00,377 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:05:02,206 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:05:04,473 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:05:04,473 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:05:04,473 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:05:04,478 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:05:05,208 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:05:05,514 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:05:06,211 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:05:08,332 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:05:08,333 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:05:08,333 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:05:08,338 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:05:09,213 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:05:10,215 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:05:11,339 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:05:12,248 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:05:12,249 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:05:12,249 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:05:12,254 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:05:13,217 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:05:14,219 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:05:15,160 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:05:15,160 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:05:15,160 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:05:16,373 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:05:18,298 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:05:18,299 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:05:18,299 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:05:18,305 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:05:19,223 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:05:20,227 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:05:21,130 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:05:21,131 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:05:21,131 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:05:21,144 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:05:21,228 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:05:22,144 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:05:22,238 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:05:24,609 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:05:24,609 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:05:24,610 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:05:24,615 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:05:25,240 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:05:25,826 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 15:05:26,248 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:05:27,829 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:05:28,367 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:05:28,368 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:05:28,368 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:05:28,375 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:05:29,250 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:05:30,160 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:05:30,160 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:05:30,160 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:05:30,254 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:05:32,959 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:05:32,959 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:05:32,959 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:05:32,959 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:05:32,967 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:05:33,256 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:05:34,258 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:05:38,003 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:05:38,151 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:05:38,151 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:05:38,151 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:05:38,162 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:05:38,297 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:05:40,301 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:05:43,163 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:05:44,055 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:05:44,056 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:05:44,056 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:05:44,061 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:05:44,304 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:05:45,160 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:05:45,160 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:05:45,201 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:05:46,308 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:05:48,451 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:05:48,784 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:05:48,785 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:05:48,785 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:05:48,791 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:05:49,312 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:05:50,318 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:05:53,827 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:05:55,041 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:05:55,042 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:05:55,042 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:05:55,047 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:05:55,321 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:05:55,818 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 15:05:56,327 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:05:59,820 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:05:59,955 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:05:59,956 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:05:59,956 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:05:59,965 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:06:00,160 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:06:00,160 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:06:00,201 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:06:00,329 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:06:02,335 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:06:03,575 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:06:03,575 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:06:03,576 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:06:03,580 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:06:04,337 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:06:05,618 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:06:06,342 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:06:06,912 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:06:06,912 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:06:06,912 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:06:06,923 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:06:07,343 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:06:08,347 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:06:10,847 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:06:10,848 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:06:10,848 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:06:10,848 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:06:10,858 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:06:11,350 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:06:12,354 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:06:15,160 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:06:15,160 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:06:15,160 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:06:16,433 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:06:16,433 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:06:16,433 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:06:16,434 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:06:16,441 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:06:17,357 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:06:18,361 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:06:21,475 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:06:23,368 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:06:23,369 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:06:23,369 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:06:23,377 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:06:24,372 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:06:24,373 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:06:25,824 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 15:06:26,826 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:06:27,643 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:06:27,644 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:06:27,644 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:06:27,651 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:06:28,376 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:06:30,160 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:06:30,160 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:06:30,161 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:06:30,385 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:06:31,790 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:06:31,790 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:06:31,790 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:06:31,797 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:06:31,834 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:06:32,386 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:06:34,394 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:06:36,835 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:06:37,566 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:06:37,566 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:06:37,566 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:06:37,572 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:06:38,402 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:06:38,403 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:06:41,453 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:06:41,454 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:06:41,454 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:06:41,463 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:06:42,405 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:06:42,405 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:06:42,463 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:06:45,160 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:06:45,160 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:06:45,161 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:06:46,986 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:06:46,987 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:06:46,987 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:06:46,995 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:06:47,412 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:06:48,030 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:06:48,416 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:06:50,617 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:06:50,618 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:06:50,618 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:06:50,625 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:06:51,418 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:06:52,423 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:06:53,625 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:06:54,189 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:06:54,190 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:06:54,190 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:06:54,199 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:06:54,425 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:06:55,823 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 15:06:56,431 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:06:58,023 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:06:58,024 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:06:58,024 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:06:58,031 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:06:58,432 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:06:59,031 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:07:00,160 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:07:00,160 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:07:00,161 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:07:00,438 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:07:02,999 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:07:03,000 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:07:03,000 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:07:03,007 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:07:03,442 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:07:04,042 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:07:04,442 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:07:08,555 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:07:08,555 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:07:08,556 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:07:08,561 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:07:09,446 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:07:09,562 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:07:10,452 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:07:14,297 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:07:14,298 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:07:14,298 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:07:14,303 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:07:14,454 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:07:15,160 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:07:15,160 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:07:15,160 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:07:15,161 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:07:16,458 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:07:19,640 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:07:19,640 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:07:19,640 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:07:19,646 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:07:20,461 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:07:20,461 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:07:20,682 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:07:23,608 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:07:23,609 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:07:23,609 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:07:23,615 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:07:24,466 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:07:24,467 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:07:25,825 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 15:07:25,826 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:07:26,930 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:07:26,930 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:07:26,931 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:07:26,937 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:07:27,469 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:07:28,472 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:07:30,160 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:07:30,161 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:07:30,161 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:07:31,378 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:07:32,367 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:07:32,367 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:07:32,367 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:07:32,372 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:07:32,477 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:07:34,480 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:07:36,411 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:07:36,967 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:07:36,968 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:07:36,968 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:07:36,973 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:07:37,482 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:07:38,485 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:07:40,271 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:07:40,272 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:07:40,273 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:07:40,279 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:07:40,486 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:07:42,280 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:07:42,488 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:07:45,000 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:07:45,001 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:07:45,001 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:07:45,006 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:07:45,161 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:07:45,161 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:07:45,205 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:07:45,490 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:07:46,492 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:07:47,391 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:07:48,549 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:07:48,550 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:07:48,550 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:07:48,555 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:07:49,496 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:07:50,499 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:07:52,503 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:07:52,504 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:07:52,504 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:07:52,504 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:07:52,509 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:07:53,501 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:07:54,504 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:07:55,827 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 15:07:56,734 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:07:56,734 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:07:56,735 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:07:56,740 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:07:57,506 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:07:57,740 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:07:58,509 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:08:00,161 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:08:00,161 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:08:00,161 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:08:01,115 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:08:01,116 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:08:01,116 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:08:01,121 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:08:01,511 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:08:02,514 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:08:03,159 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:08:07,288 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:08:07,289 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:08:07,289 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:08:07,294 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:08:07,518 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:08:08,295 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:08:08,521 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:08:13,295 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:08:15,161 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:08:15,161 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:08:15,201 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:08:15,895 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:08:15,895 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:08:15,895 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:08:15,901 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:08:16,526 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:08:18,527 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:08:18,939 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:08:20,860 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:08:20,860 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:08:20,861 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:08:20,866 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:08:21,529 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:08:22,532 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:08:24,091 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:08:24,092 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:08:24,092 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:08:24,092 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:08:24,097 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:08:24,533 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:08:25,829 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 15:08:26,537 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:08:27,622 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:08:27,623 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:08:27,623 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:08:27,628 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:08:28,540 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:08:28,540 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:08:29,628 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:08:30,161 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:08:30,161 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:08:30,201 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:08:32,736 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:08:32,737 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:08:32,737 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:08:32,742 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:08:33,545 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:08:34,547 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:08:34,778 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:08:37,422 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:08:37,422 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:08:37,423 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:08:37,427 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:08:37,549 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:08:38,552 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:08:40,428 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:08:41,797 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:08:41,798 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:08:41,798 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:08:41,803 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:08:42,554 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:08:44,557 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:08:45,161 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:08:45,161 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:08:45,162 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:08:45,633 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:08:45,633 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:08:45,634 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:08:45,634 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:08:45,638 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:08:46,560 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:08:46,560 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:08:49,828 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:08:49,829 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:08:49,829 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:08:49,834 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:08:50,564 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:08:50,835 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:08:52,567 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:08:52,890 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:08:52,890 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:08:52,891 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:08:52,896 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:08:53,567 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:08:54,570 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:08:55,834 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 15:08:55,836 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:08:56,443 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:08:56,443 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:08:56,444 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:08:56,450 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:08:56,571 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:08:58,574 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:09:00,161 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:09:00,161 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:09:00,162 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:09:01,322 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:09:01,323 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:09:01,323 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:09:01,323 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:09:01,327 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:09:01,576 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:09:02,579 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:09:04,533 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:09:04,533 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:09:04,534 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:09:04,539 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:09:04,580 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:09:06,539 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:09:06,582 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:09:07,511 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:09:07,512 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:09:07,512 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:09:07,517 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:09:07,582 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:09:08,585 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:09:11,659 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:09:11,660 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:09:11,660 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:09:11,660 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:09:11,665 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:09:12,589 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:09:12,590 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:09:15,161 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:09:15,161 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:09:15,205 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:09:15,280 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:09:15,368 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:09:15,368 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:09:15,373 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:09:15,591 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:09:16,594 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:09:17,374 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:09:18,569 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:09:18,570 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:09:18,570 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:09:18,575 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:09:18,595 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:09:20,599 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:09:21,849 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:09:21,850 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:09:21,850 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:09:21,855 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:09:22,600 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:09:22,855 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:09:24,604 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:09:25,834 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 15:09:26,513 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:09:26,514 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:09:26,514 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:09:26,519 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:09:26,605 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:09:28,519 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:09:28,608 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:09:30,161 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:09:30,161 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:09:30,205 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:09:30,994 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:09:30,995 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:09:30,995 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:09:31,000 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:09:31,611 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:09:32,614 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:09:34,038 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:09:35,809 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:09:35,810 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:09:35,810 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:09:35,815 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:09:36,618 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:09:36,618 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:09:39,816 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:09:42,462 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:09:42,463 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:09:42,463 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:09:42,468 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:09:42,621 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:09:44,624 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:09:45,161 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:09:45,161 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:09:45,162 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:09:45,163 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:09:46,033 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:09:46,034 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:09:46,034 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:09:46,039 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:09:46,626 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:09:48,630 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:09:50,530 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:09:50,530 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:09:50,531 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:09:50,531 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:09:50,535 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:09:50,631 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:09:52,634 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:09:53,771 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:09:53,772 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:09:53,772 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:09:53,777 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:09:54,637 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:09:54,637 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:09:55,777 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:09:55,839 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 15:09:57,038 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:09:57,039 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:09:57,039 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:09:57,044 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:09:57,639 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:09:58,642 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:10:00,161 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:10:00,162 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:10:00,163 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:10:00,956 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:10:00,957 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:10:00,957 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:10:00,957 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:10:00,962 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:10:01,644 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:10:02,646 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:10:04,479 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:10:04,479 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:10:04,480 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:10:04,484 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:10:04,648 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:10:06,485 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:10:06,649 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:10:08,117 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:10:08,118 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:10:08,118 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:10:08,123 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:10:08,650 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:10:10,653 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:10:11,910 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:10:11,911 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:10:11,911 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:10:11,911 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:10:11,916 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:10:12,655 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:10:14,658 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:10:15,161 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:10:15,162 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:10:15,163 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:10:15,248 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:10:15,414 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:10:15,414 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:10:15,419 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:10:15,658 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:10:16,660 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:10:17,419 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:10:18,810 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:10:18,810 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:10:18,811 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:10:18,819 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:10:19,662 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:10:20,665 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:10:22,819 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:10:23,287 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:10:23,287 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:10:23,287 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:10:23,292 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:10:23,667 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:10:24,669 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:10:25,838 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 15:10:27,839 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:10:28,582 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:10:28,583 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:10:28,583 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:10:28,588 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:10:28,672 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:10:30,162 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:10:30,162 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:10:30,205 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:10:30,676 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:10:33,070 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:10:33,071 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:10:33,071 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:10:33,071 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:10:33,076 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:10:33,678 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:10:34,680 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:10:35,910 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:10:35,910 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:10:35,910 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:10:35,915 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:10:36,684 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:10:36,684 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:10:38,916 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:10:39,585 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:10:39,586 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:10:39,586 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:10:39,591 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:10:39,686 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:10:40,688 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:10:43,321 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:10:43,322 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:10:43,322 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:10:43,327 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:10:43,690 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:10:44,328 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:10:44,693 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:10:45,162 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:10:45,162 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:10:45,205 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:10:46,964 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:10:46,964 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:10:46,964 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:10:46,969 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:10:47,696 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:10:48,698 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:10:50,007 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:10:50,483 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:10:50,484 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:10:50,484 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:10:50,489 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:10:50,699 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:10:52,702 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:10:55,490 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:10:55,843 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 15:10:56,723 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:10:56,724 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:10:56,724 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:10:56,729 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:10:57,705 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:10:58,708 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:11:00,162 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:11:00,162 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:11:00,205 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:11:01,465 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:11:01,630 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:11:01,630 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:11:01,631 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:11:01,635 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:11:01,709 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:11:02,712 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:11:05,129 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:11:05,130 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:11:05,130 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:11:05,135 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:11:05,714 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:11:06,714 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:11:07,135 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:11:12,136 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:11:13,824 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:11:13,825 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:11:13,825 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:11:13,830 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:11:14,721 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:11:14,722 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:11:15,162 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:11:15,162 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:11:15,205 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:11:17,373 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:11:17,560 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:11:17,560 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:11:17,560 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:11:17,565 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:11:17,723 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:11:18,726 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:11:21,319 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:11:21,320 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:11:21,320 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:11:21,325 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:11:21,727 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:11:22,728 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:11:23,326 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:11:25,038 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:11:25,039 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:11:25,039 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:11:25,045 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:11:25,730 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:11:25,843 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 15:11:26,733 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:11:28,844 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:11:29,117 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:11:29,118 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:11:29,118 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:11:29,124 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:11:29,734 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:11:30,162 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:11:30,162 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:11:30,205 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:11:30,739 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:11:32,466 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:11:32,466 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:11:32,466 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:11:32,471 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:11:32,740 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:11:34,510 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:11:34,743 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:11:37,390 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:11:37,391 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:11:37,391 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:11:37,396 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:11:37,745 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:11:38,747 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:11:40,396 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:11:43,167 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:11:43,168 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:11:43,168 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:11:43,173 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:11:43,750 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:11:44,753 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:11:45,162 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:11:45,162 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:11:45,205 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:11:45,398 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:11:46,876 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:11:46,876 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:11:46,876 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:11:46,881 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:11:47,756 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:11:48,758 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:11:50,919 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:11:51,553 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:11:51,553 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:11:51,554 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:11:51,558 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:11:51,760 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:11:52,762 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:11:55,845 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 15:11:56,846 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:11:59,773 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:11:59,777 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:11:59,777 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:11:59,782 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:12:00,162 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:12:00,162 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:12:00,205 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:12:00,769 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:12:00,769 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:12:02,427 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:12:05,203 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:12:05,204 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:12:05,204 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:12:05,209 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:12:05,772 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:12:06,775 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:12:08,246 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:12:09,254 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:12:09,255 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:12:09,255 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:12:09,260 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:12:09,777 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:12:10,778 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:12:13,261 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:12:14,823 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:12:14,824 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:12:14,824 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:12:14,829 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:12:15,162 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:12:15,163 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:12:15,205 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:12:15,780 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:12:16,783 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:12:18,129 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:12:18,130 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:12:18,130 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:12:18,135 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:12:18,784 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:12:19,174 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:12:20,787 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:12:23,119 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:12:23,120 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:12:23,120 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:12:23,126 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:12:23,789 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:12:24,792 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:12:25,126 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:12:25,847 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 15:12:26,293 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:12:26,294 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:12:26,294 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:12:26,299 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:12:26,793 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:12:28,794 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:12:30,162 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:12:30,162 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:12:30,163 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:12:30,205 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:12:33,105 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:12:33,106 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:12:33,106 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:12:33,111 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:12:33,800 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:12:34,802 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:12:36,151 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:12:38,548 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:12:38,549 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:12:38,549 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:12:38,554 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:12:38,805 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:12:40,808 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:12:41,555 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:12:42,026 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:12:42,027 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:12:42,027 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:12:42,032 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:12:42,811 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:12:42,811 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:12:45,163 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:12:45,163 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:12:45,205 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:12:45,327 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:12:45,443 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:12:45,444 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:12:45,449 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:12:45,812 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:12:46,815 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:12:47,449 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:12:52,325 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:12:52,326 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:12:52,326 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:12:52,331 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:12:52,819 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:12:53,331 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:12:54,822 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:12:55,831 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:12:55,831 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:12:55,832 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:12:55,836 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:12:55,849 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 15:12:56,825 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:12:56,825 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:12:58,850 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:12:59,482 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:12:59,482 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:12:59,483 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:12:59,487 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:12:59,827 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:13:00,163 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:13:00,163 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:13:00,205 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:13:00,829 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:13:04,355 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:13:04,356 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:13:04,356 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:13:04,356 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:13:04,361 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:13:04,832 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:13:06,835 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:13:08,212 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:13:08,213 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:13:08,213 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:13:08,218 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:13:08,836 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:13:10,218 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:13:10,839 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:13:12,550 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:13:12,550 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:13:12,551 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:13:12,555 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:13:12,841 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:13:14,844 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:13:15,163 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:13:15,163 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:13:15,205 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:13:15,454 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:13:16,496 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:13:16,497 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:13:16,497 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:13:16,502 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:13:16,845 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:13:18,848 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:13:20,541 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:13:25,071 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:13:25,071 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:13:25,072 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:13:25,076 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:13:25,851 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 15:13:25,852 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:13:25,853 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:13:26,856 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:13:28,360 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:13:28,361 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:13:28,361 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:13:28,366 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:13:28,857 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:13:30,163 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:13:30,163 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:13:30,205 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:13:30,861 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:13:31,474 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:13:34,852 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:13:34,853 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:13:34,853 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:13:34,858 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:13:34,863 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:13:36,864 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:13:36,894 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:13:38,185 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:13:38,186 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:13:38,186 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:13:38,191 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:13:38,866 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:13:40,869 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:13:41,886 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:13:41,887 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:13:41,887 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:13:41,892 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:13:42,871 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:13:42,872 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:13:42,893 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:13:45,163 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:13:45,163 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:13:45,205 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:13:46,214 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:13:46,214 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:13:46,214 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:13:46,219 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:13:46,874 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:13:48,258 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:13:48,878 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:13:49,663 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:13:49,663 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:13:49,664 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:13:49,670 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:13:49,878 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:13:50,881 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:13:53,670 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:13:55,582 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:13:55,583 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:13:55,583 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:13:55,588 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:13:55,853 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 15:13:55,883 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:13:56,886 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:13:58,854 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:13:59,115 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:13:59,115 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:13:59,115 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:13:59,120 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:13:59,887 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:14:00,163 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:14:00,163 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:14:00,205 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:14:00,890 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:14:02,611 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:14:02,611 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:14:02,611 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:14:02,616 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:14:02,891 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:14:04,654 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:14:04,894 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:14:06,201 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:14:06,202 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:14:06,202 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:14:06,207 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:14:06,896 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:14:08,899 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:14:10,130 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:14:10,131 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:14:10,131 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:14:10,132 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:14:10,136 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:14:10,902 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:14:10,902 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:14:15,137 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:14:15,163 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:14:15,163 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:14:15,205 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:14:16,293 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:14:16,293 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:14:16,293 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:14:16,298 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:14:16,905 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:14:18,908 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:14:19,156 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:14:19,156 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:14:19,156 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:14:19,161 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:14:19,909 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:14:20,161 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:14:20,911 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:14:23,224 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:14:23,225 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:14:23,225 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:14:23,230 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:14:23,913 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:14:24,915 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:14:25,231 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:14:25,860 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 15:14:29,591 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:14:29,592 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:14:29,592 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:14:29,597 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:14:29,918 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:14:30,163 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:14:30,163 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:14:30,205 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:14:30,392 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:14:30,922 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:14:33,089 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:14:33,089 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:14:33,089 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:14:33,094 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:14:33,924 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:14:34,924 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:14:36,131 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:14:37,483 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:14:37,484 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:14:37,484 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:14:37,489 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:14:37,926 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:14:38,929 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:14:41,262 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:14:41,263 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:14:41,263 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:14:41,263 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:14:41,268 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:14:41,930 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:14:42,933 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:14:45,163 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:14:45,164 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:14:45,206 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:14:46,463 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:14:46,710 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:14:46,710 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:14:46,711 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:14:46,716 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:14:46,935 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:14:48,939 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:14:51,706 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:14:51,707 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:14:51,707 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:14:51,708 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:14:51,712 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:14:51,941 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:14:52,943 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:14:55,857 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 15:14:56,859 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:14:58,981 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:14:58,982 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:14:58,982 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:14:58,987 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:14:59,947 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:15:00,163 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:15:00,164 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:15:00,205 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:15:00,950 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:15:02,338 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:15:02,339 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:15:02,339 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:15:02,339 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:15:02,343 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:15:02,951 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:15:04,954 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:15:06,387 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:15:06,387 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:15:06,388 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:15:06,393 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:15:06,957 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:15:07,393 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:15:08,960 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:15:10,370 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:15:10,370 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:15:10,371 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:15:10,376 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:15:10,962 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:15:12,964 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:15:13,376 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:15:13,440 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:15:13,441 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:15:13,441 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:15:13,446 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:15:13,965 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:15:14,968 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:15:15,164 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:15:15,164 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:15:15,205 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:15:17,925 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:15:17,926 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:15:17,926 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:15:17,931 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:15:17,969 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:15:18,970 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:15:18,972 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:15:23,100 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:15:23,101 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:15:23,101 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:15:23,108 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:15:23,975 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:15:24,109 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:15:24,977 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:15:25,862 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 15:15:26,204 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:15:26,204 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:15:26,204 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:15:26,209 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:15:26,980 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:15:26,981 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:15:29,210 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:15:30,164 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:15:30,164 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:15:30,205 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:15:30,603 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:15:30,604 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:15:30,604 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:15:30,609 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:15:30,984 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:15:32,987 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:15:34,245 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:15:34,246 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:15:34,246 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:15:34,246 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:15:34,251 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:15:34,990 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:15:34,990 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:15:37,183 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:15:37,183 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:15:37,184 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:15:37,189 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:15:37,992 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:15:38,995 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:15:40,189 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:15:42,025 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:15:42,026 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:15:42,026 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:15:42,031 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:15:42,999 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:15:42,999 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:15:45,164 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:15:45,164 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:15:45,205 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:15:45,414 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:15:48,283 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:15:48,283 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:15:48,283 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:15:48,288 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:15:49,004 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:15:51,005 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:15:51,327 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:15:51,377 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:15:51,377 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:15:51,378 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:15:51,382 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:15:52,005 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:15:53,008 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:15:55,862 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 15:15:56,863 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:15:57,886 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:15:57,887 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:15:57,887 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:15:57,892 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:15:58,010 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:15:59,013 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:16:00,164 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:16:00,164 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:16:00,205 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:16:02,136 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:16:02,137 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:16:02,137 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:16:02,137 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:16:02,142 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:16:03,017 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:16:03,017 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:16:06,288 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:16:06,289 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:16:06,289 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:16:06,294 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:16:07,020 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:16:07,294 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:16:09,023 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:16:10,015 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:16:10,016 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:16:10,016 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:16:10,021 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:16:10,024 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:16:11,026 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:16:13,022 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:16:13,699 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:16:13,699 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:16:13,700 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:16:13,705 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:16:14,028 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:16:15,031 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:16:15,164 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:16:15,164 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:16:15,205 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:16:16,934 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:16:16,935 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:16:16,935 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:16:16,940 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:16:17,032 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:16:18,978 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:16:19,035 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:16:21,028 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:16:21,028 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:16:21,028 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:16:21,033 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:16:21,036 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:16:23,039 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:16:24,034 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:16:25,524 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:16:25,525 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:16:25,525 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:16:25,530 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:16:25,867 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 15:16:26,041 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:16:27,044 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:16:29,051 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:16:29,052 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:16:29,052 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:16:29,053 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:16:29,057 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:16:30,046 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:16:30,164 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:16:30,164 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:16:30,205 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:16:31,049 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:16:32,445 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:16:32,446 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:16:32,446 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:16:32,451 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:16:33,051 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:16:34,490 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:16:35,054 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:16:36,027 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:16:36,028 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:16:36,028 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:16:36,033 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:16:36,054 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:16:37,057 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:16:39,661 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:16:39,662 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:16:39,662 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:16:39,662 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:16:39,667 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:16:40,059 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:16:41,061 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:16:44,152 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:16:44,153 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:16:44,154 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:16:44,159 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:16:45,065 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:16:45,066 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:16:45,159 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:16:45,164 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:16:45,164 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:16:45,205 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:16:47,667 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:16:47,667 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:16:47,667 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:16:47,672 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:16:48,068 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:16:49,071 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:16:50,710 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:16:51,907 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:16:51,908 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:16:51,908 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:16:51,913 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:16:52,072 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:16:53,075 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:16:55,866 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 15:16:55,867 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:16:58,398 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:16:58,399 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:16:58,399 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:16:58,404 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:16:59,079 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:17:00,164 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:17:00,164 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:17:00,205 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:17:01,080 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:17:01,363 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:17:01,642 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:17:01,643 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:17:01,643 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:17:01,648 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:17:02,080 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:17:03,083 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:17:06,071 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:17:06,072 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:17:06,072 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:17:06,077 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:17:06,085 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:17:07,078 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:17:07,087 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:17:09,540 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:17:09,540 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:17:09,541 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:17:09,546 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:17:10,089 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:17:11,091 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:17:12,546 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:17:13,069 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:17:13,070 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:17:13,070 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:17:13,075 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:17:13,093 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:17:15,096 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:17:15,164 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:17:15,164 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:17:15,205 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:17:17,618 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:17:17,618 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:17:17,618 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:17:17,618 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:17:17,624 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:17:18,098 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:17:19,100 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:17:22,663 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:17:25,868 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 15:17:26,145 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:17:26,146 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:17:26,146 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:17:26,151 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:17:27,107 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:17:27,107 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:17:28,152 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:17:29,613 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:17:29,614 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:17:29,614 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:17:29,619 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:17:30,109 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:17:30,164 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:17:30,165 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:17:30,205 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:17:31,110 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:17:33,452 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:17:33,814 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:17:33,815 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:17:33,815 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:17:33,820 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:17:34,112 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:17:35,115 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:17:36,878 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:17:36,878 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:17:36,878 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:17:36,883 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:17:37,116 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:17:38,884 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:17:39,119 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:17:42,519 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:17:42,519 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:17:42,519 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:17:42,524 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:17:43,121 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:17:44,525 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:17:45,124 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:17:45,164 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:17:45,165 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:17:45,205 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:17:47,606 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:17:47,607 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:17:47,607 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:17:47,612 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:17:48,127 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:17:49,130 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:17:49,650 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:17:52,755 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:17:52,756 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:17:52,756 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:17:52,761 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:17:53,132 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:17:54,761 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:17:55,135 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:17:55,870 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 15:17:57,120 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:17:57,121 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:17:57,121 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:17:57,126 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:17:57,136 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:17:59,139 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:18:00,127 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:18:00,165 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:18:00,165 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:18:00,205 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:18:02,691 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:18:02,691 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:18:02,691 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:18:02,696 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:18:03,141 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:18:05,145 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:18:05,735 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:18:08,003 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:18:08,004 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:18:08,004 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:18:08,009 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:18:08,146 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:18:09,149 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:18:11,010 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:18:11,612 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:18:11,612 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:18:11,613 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:18:11,617 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:18:12,150 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:18:13,153 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:18:15,165 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:18:15,165 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:18:15,205 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:18:16,286 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:18:16,287 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:18:16,287 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:18:16,287 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:18:16,292 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:18:17,157 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:18:17,157 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:18:20,095 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:18:20,096 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:18:20,096 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:18:20,101 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:18:20,159 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:18:21,161 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:18:22,101 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:18:24,438 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:18:24,439 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:18:24,439 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:18:24,444 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:18:25,165 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:18:25,166 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:18:25,872 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 15:18:27,873 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:18:27,914 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:18:27,915 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:18:27,915 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:18:27,920 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:18:28,167 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:18:29,170 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:18:30,165 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:18:30,165 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:18:30,205 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:18:33,427 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:18:34,690 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:18:34,690 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:18:34,690 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:18:34,695 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:18:35,174 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:18:37,177 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:18:38,554 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:18:38,555 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:18:38,555 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:18:38,556 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:18:38,560 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:18:39,181 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:18:41,184 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:18:43,561 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:18:44,614 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:18:44,615 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:18:44,615 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:18:44,620 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:18:45,165 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:18:45,165 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:18:45,186 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:18:45,205 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:18:47,189 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:18:48,604 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:18:48,605 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:18:48,605 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:18:48,605 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:18:48,610 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:18:49,191 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:18:51,194 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:18:52,201 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:18:52,202 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:18:52,202 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:18:52,208 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:18:53,197 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:18:53,197 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:18:54,208 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:18:55,372 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:18:55,376 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:18:55,376 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:18:55,381 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:18:55,874 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 15:18:56,199 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:18:57,201 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:18:58,803 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:18:58,804 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:18:58,804 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:18:58,809 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:18:59,202 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:18:59,809 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:19:00,165 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:19:00,165 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:19:00,209 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:19:01,205 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:19:03,810 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:19:03,810 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:19:03,811 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:19:03,816 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:19:04,207 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:19:04,854 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:19:05,210 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:19:08,388 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:19:08,389 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:19:08,389 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:19:08,394 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:19:09,214 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:19:09,214 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:19:10,395 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:19:11,872 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:19:11,872 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:19:11,872 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:19:11,877 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:19:12,215 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:19:13,218 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:19:15,165 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:19:15,165 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:19:15,209 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:19:16,390 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:19:16,879 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:19:16,879 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:19:16,880 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:19:16,884 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:19:17,220 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:19:19,223 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:19:20,803 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:19:20,804 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:19:20,804 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:19:20,809 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:19:21,224 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:19:21,809 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:19:23,227 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:19:23,886 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:19:23,887 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:19:23,888 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:19:23,893 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:19:24,228 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:19:25,230 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:19:25,876 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 15:19:26,878 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:19:27,774 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:19:27,774 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:19:27,775 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:19:27,779 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:19:28,232 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:19:29,235 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:19:30,165 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:19:30,165 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:19:30,209 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:19:31,218 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:19:31,219 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:19:31,219 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:19:31,225 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:19:31,236 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:19:32,261 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:19:33,239 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:19:34,544 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:19:34,544 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:19:34,545 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:19:34,549 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:19:35,240 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:19:37,243 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:19:37,550 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:19:38,138 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:19:38,138 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:19:38,139 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:19:38,144 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:19:38,244 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:19:39,246 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:19:41,662 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:19:41,663 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:19:41,663 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:19:41,668 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:19:42,248 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:19:42,668 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:19:43,248 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:19:45,165 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:19:45,166 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:19:45,209 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:19:46,497 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:19:46,498 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:19:46,498 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:19:46,503 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:19:47,252 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:19:47,253 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:19:48,541 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:19:51,192 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:19:51,193 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:19:51,193 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:19:51,198 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:19:51,256 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:19:53,259 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:19:54,199 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:19:55,881 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 15:19:59,697 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:19:59,697 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:19:59,698 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:19:59,698 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:19:59,703 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:20:00,165 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:20:00,166 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:20:00,209 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:20:00,263 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:20:01,265 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:20:04,205 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:20:04,205 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:20:04,205 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:20:04,211 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:20:04,267 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:20:05,245 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:20:05,267 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:20:08,504 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:20:08,505 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:20:08,505 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:20:08,510 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:20:09,271 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:20:09,272 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:20:10,511 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:20:11,872 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:20:11,873 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:20:11,873 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:20:11,878 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:20:12,273 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:20:13,276 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:20:15,165 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:20:15,166 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:20:15,209 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:20:15,416 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:20:15,455 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:20:15,455 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:20:15,459 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:20:16,277 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:20:16,460 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:20:17,280 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:20:18,559 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:20:18,560 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:20:18,560 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:20:18,565 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:20:19,283 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:20:19,283 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:20:21,566 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:20:22,476 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:20:22,476 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:20:22,477 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:20:22,482 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:20:23,287 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:20:23,287 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:20:25,880 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 15:20:25,973 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:20:25,973 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:20:25,974 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:20:25,979 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:20:26,288 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:20:26,979 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:20:27,289 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:20:30,166 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:20:30,166 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:20:30,209 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:20:31,416 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:20:31,417 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:20:31,417 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:20:31,422 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:20:32,293 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:20:32,457 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:20:33,296 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:20:36,651 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:20:36,652 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:20:36,652 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:20:36,657 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:20:37,298 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:20:37,657 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:20:39,301 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:20:39,977 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:20:39,978 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:20:39,978 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:20:39,983 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:20:40,301 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:20:41,304 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:20:42,984 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:20:43,702 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:20:43,703 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:20:43,703 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:20:43,708 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:20:44,305 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:20:45,166 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:20:45,166 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:20:45,209 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:20:45,308 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:20:48,414 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:20:48,993 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:20:48,993 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:20:48,993 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:20:48,998 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:20:49,311 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:20:51,314 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:20:54,034 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:20:55,885 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 15:20:58,957 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:20:58,958 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:20:58,958 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:20:58,963 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:20:59,319 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:20:59,963 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:21:00,166 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:21:00,166 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:21:00,209 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:21:01,322 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:21:02,462 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:21:02,462 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:21:02,462 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:21:02,467 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:21:03,325 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:21:03,326 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:21:05,505 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:21:09,660 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:21:09,661 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:21:09,661 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:21:09,667 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:21:10,330 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:21:10,667 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:21:11,331 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:21:14,575 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:21:14,576 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:21:14,576 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:21:14,581 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:21:15,166 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:21:15,166 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:21:15,209 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:21:15,335 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:21:15,335 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:21:16,469 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:21:21,469 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:21:23,009 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:21:23,010 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:21:23,010 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:21:23,015 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:21:23,339 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:21:25,342 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:21:25,884 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 15:21:26,886 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:21:28,457 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:21:28,458 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:21:28,458 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:21:28,463 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:21:29,346 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:21:29,347 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:21:30,166 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:21:30,166 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:21:30,209 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:21:32,433 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:21:33,186 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:21:33,187 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:21:33,187 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:21:33,192 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:21:33,350 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:21:35,353 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:21:38,230 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:21:38,735 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:21:38,736 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:21:38,736 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:21:38,742 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:21:39,355 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:21:41,358 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:21:41,878 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:21:41,879 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:21:41,879 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:21:41,884 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:21:42,359 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:21:43,361 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:21:43,885 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:21:45,166 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:21:45,166 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:21:45,209 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:21:47,794 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:21:47,795 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:21:47,795 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:21:47,800 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:21:48,365 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:21:49,367 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:21:49,837 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:21:53,624 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:21:53,625 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:21:53,625 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:21:53,630 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:21:54,370 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:21:55,373 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:21:55,630 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:21:55,886 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 15:22:00,166 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:22:00,166 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:22:00,209 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:22:01,419 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:22:02,830 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:22:02,830 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:22:02,831 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:22:02,835 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:22:03,377 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:22:05,378 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:22:06,874 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:22:07,001 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:22:07,002 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:22:07,002 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:22:07,007 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:22:07,380 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:22:09,383 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:22:11,984 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:22:11,985 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:22:11,985 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:22:11,985 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:22:11,990 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:22:12,385 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:22:13,388 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:22:15,166 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:22:15,166 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:22:15,209 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:22:17,423 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:22:18,360 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:22:18,361 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:22:18,361 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:22:18,366 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:22:18,391 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:22:19,395 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:22:21,737 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:22:21,738 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:22:21,738 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:22:21,743 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:22:22,396 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:22:22,743 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:22:23,399 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:22:25,889 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 15:22:26,034 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:22:26,035 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:22:26,035 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:22:26,040 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:22:26,401 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:22:27,404 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:22:28,040 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:22:29,712 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:22:29,713 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:22:29,713 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:22:29,718 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:22:30,166 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:22:30,166 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:22:30,209 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:22:30,406 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:22:31,410 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:22:33,535 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:22:33,876 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:22:33,876 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:22:33,877 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:22:33,882 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:22:34,411 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:22:35,414 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:22:37,626 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:22:37,627 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:22:37,627 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:22:37,632 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:22:38,416 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:22:38,632 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:22:39,419 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:22:40,941 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:22:40,941 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:22:40,941 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:22:40,946 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:22:41,420 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:22:43,423 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:22:43,947 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:22:45,166 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:22:45,166 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:22:45,209 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:22:46,594 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:22:46,594 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:22:46,595 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:22:46,600 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:22:47,427 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:22:47,428 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:22:49,637 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:22:52,229 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:22:52,230 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:22:52,230 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:22:52,235 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:22:52,432 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:22:53,434 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:22:55,235 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:22:55,891 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 15:22:56,226 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:22:56,227 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:22:56,227 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:22:56,232 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:22:56,436 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:22:57,439 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:23:00,166 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:23:00,167 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:23:00,209 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:23:00,385 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:23:01,055 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:23:01,055 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:23:01,056 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:23:01,061 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:23:01,441 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:23:03,444 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:23:05,784 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:23:05,785 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:23:05,785 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:23:05,786 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:23:05,791 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:23:06,446 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:23:07,449 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:23:09,062 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:23:09,062 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:23:09,063 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:23:09,067 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:23:09,450 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:23:11,068 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:23:11,457 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:23:13,985 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:23:13,986 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:23:13,986 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:23:13,992 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:23:14,459 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:23:15,167 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:23:15,167 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:23:15,209 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:23:15,462 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:23:16,492 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:23:18,199 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:23:18,200 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:23:18,200 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:23:18,208 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:23:18,464 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:23:19,468 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:23:22,241 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:23:22,585 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:23:22,586 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:23:22,586 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:23:22,594 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:23:23,474 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:23:23,474 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:23:25,900 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 15:23:26,303 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:23:26,303 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:23:26,303 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:23:26,313 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:23:26,475 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:23:27,314 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:23:27,477 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:23:30,167 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:23:30,167 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:23:30,209 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:23:32,376 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:23:32,700 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:23:32,701 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:23:32,701 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:23:32,709 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:23:33,487 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:23:33,488 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:23:37,742 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:23:39,571 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:23:39,572 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:23:39,572 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:23:39,581 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:23:40,492 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:23:41,495 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:23:42,694 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:23:42,695 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:23:42,696 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:23:42,705 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:23:43,499 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:23:43,500 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:23:43,705 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:23:45,167 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:23:45,167 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:23:45,209 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:23:48,269 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:23:48,269 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:23:48,270 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:23:48,276 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:23:48,505 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:23:49,309 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:23:49,509 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:23:51,982 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:23:51,982 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:23:51,983 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:23:51,988 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:23:52,511 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:23:53,512 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:23:54,988 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:23:55,895 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 15:23:56,631 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:23:56,632 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:23:56,632 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:23:56,638 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:23:57,517 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:23:57,517 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:24:00,166 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:24:00,167 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:24:00,167 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:24:00,209 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:24:00,370 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:24:00,376 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:24:00,376 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:24:00,385 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:24:00,519 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:24:01,522 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:24:05,386 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:24:06,105 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:24:06,106 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:24:06,106 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:24:06,112 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:24:06,526 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:24:07,529 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:24:11,094 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:24:11,095 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:24:11,095 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:24:11,096 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:24:11,101 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:24:11,531 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:24:13,535 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:24:15,056 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:24:15,057 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:24:15,058 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:24:15,063 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:24:15,167 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:24:15,167 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:24:15,209 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:24:15,537 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:24:16,398 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:24:17,541 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:24:19,005 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:24:19,006 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:24:19,006 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:24:19,011 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:24:19,542 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:24:21,546 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:24:22,049 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:24:25,902 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 15:24:26,768 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:24:26,769 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:24:26,769 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:24:26,776 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:24:27,550 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:24:27,550 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:24:27,776 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:24:30,167 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:24:30,167 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:24:30,209 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:24:31,995 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:24:31,996 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:24:31,996 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:24:32,001 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:24:32,555 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:24:33,037 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:24:33,558 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:24:35,516 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:24:35,517 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:24:35,517 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:24:35,522 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:24:35,559 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:24:37,562 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:24:38,523 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:24:39,851 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:24:39,852 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:24:39,852 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:24:39,857 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:24:40,564 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:24:41,567 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:24:43,858 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:24:45,167 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:24:45,167 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:24:45,209 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:24:47,839 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:24:47,840 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:24:47,840 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:24:47,846 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:24:48,573 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:24:48,881 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:24:49,576 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:24:51,369 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:24:51,370 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:24:51,370 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:24:51,375 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:24:51,577 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:24:53,581 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:24:54,376 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:24:55,017 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:24:55,018 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:24:55,018 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:24:55,024 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:24:55,582 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:24:55,899 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 15:24:57,585 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:24:59,900 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:25:00,167 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:25:00,167 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:25:00,209 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:25:03,681 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:25:03,682 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:25:03,682 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:25:03,687 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:25:04,589 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:25:05,590 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:25:05,725 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:25:08,351 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:25:08,352 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:25:08,352 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:25:08,357 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:25:08,592 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:25:09,594 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:25:11,358 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:25:15,167 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:25:15,167 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:25:15,209 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:25:15,976 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:25:15,977 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:25:15,977 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:25:15,982 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:25:16,599 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:25:17,017 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:25:17,601 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:25:19,744 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:25:19,744 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:25:19,745 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:25:19,750 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:25:20,603 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:25:21,605 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:25:22,751 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:25:23,461 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:25:23,462 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:25:23,462 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:25:23,467 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:25:23,606 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:25:25,610 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:25:25,904 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 15:25:27,209 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:25:27,210 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:25:27,210 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:25:27,215 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:25:27,611 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:25:28,215 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:25:29,614 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:25:30,168 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:25:30,168 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:25:30,209 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:25:32,478 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:25:32,479 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:25:32,479 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:25:32,484 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:25:32,617 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:25:33,521 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:25:33,618 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:25:36,098 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:25:36,098 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:25:36,099 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:25:36,104 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:25:36,619 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:25:37,622 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:25:39,104 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:25:39,766 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:25:39,767 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:25:39,767 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:25:39,772 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:25:40,624 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:25:41,627 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:25:43,064 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:25:43,064 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:25:43,065 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:25:43,070 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:25:43,628 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:25:45,070 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:25:45,168 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:25:45,168 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:25:45,209 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:25:45,631 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:25:48,210 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:25:48,211 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:25:48,211 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:25:48,216 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:25:48,634 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:25:49,637 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:25:50,253 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:25:53,174 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:25:53,175 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:25:53,175 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:25:53,180 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:25:53,639 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:25:55,640 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:25:55,903 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 15:25:55,905 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:25:57,570 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:25:57,571 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:25:57,571 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:25:57,576 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:25:57,641 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:25:59,644 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:26:00,168 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:26:00,168 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:26:00,209 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:26:01,451 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:26:01,533 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:26:01,534 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:26:01,534 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:26:01,539 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:26:01,645 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:26:03,648 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:26:06,578 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:26:09,867 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:26:09,868 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:26:09,868 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:26:09,873 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:26:10,653 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:26:11,655 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:26:11,874 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:26:13,120 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:26:13,121 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:26:13,121 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:26:13,126 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:26:13,656 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:26:15,168 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:26:15,168 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:26:15,209 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:26:15,659 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:26:17,388 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:26:19,978 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:26:19,978 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:26:19,978 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:26:19,983 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:26:20,662 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:26:21,664 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:26:23,021 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:26:23,285 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:26:23,286 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:26:23,286 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:26:23,292 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:26:23,666 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:26:25,669 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:26:25,906 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 15:26:28,405 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:26:28,406 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:26:28,406 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:26:28,406 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:26:28,411 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:26:28,673 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:26:29,675 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:26:30,168 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:26:30,168 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:26:30,209 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:26:32,867 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:26:32,867 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:26:32,867 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:26:32,872 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:26:33,682 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:26:33,682 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:26:33,909 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:26:36,319 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:26:36,320 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:26:36,320 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:26:36,325 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:26:36,685 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:26:37,688 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:26:39,056 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:26:39,056 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:26:39,057 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:26:39,057 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:26:39,062 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:26:39,689 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:26:41,694 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:26:43,166 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:26:43,167 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:26:43,167 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:26:43,172 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:26:43,695 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:26:44,173 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:26:45,168 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:26:45,168 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:26:45,209 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:26:45,700 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:26:49,435 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:26:52,950 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:26:52,951 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:26:52,951 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:26:52,956 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:26:53,711 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:26:53,711 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:26:54,997 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:26:55,907 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 15:26:57,621 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:26:57,621 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:26:57,622 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:26:57,627 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:26:57,716 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:26:59,720 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:27:00,167 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:27:00,168 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:27:00,168 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:27:00,209 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:27:01,277 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:27:01,278 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:27:01,278 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:27:01,283 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:27:01,722 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:27:03,727 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:27:04,957 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:27:04,958 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:27:04,958 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:27:04,963 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:27:05,728 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:27:05,729 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:27:05,963 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:27:10,925 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:27:10,926 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:27:10,926 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:27:10,931 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:27:11,738 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:27:11,738 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:27:11,931 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:27:15,168 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:27:15,168 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:27:15,209 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:27:15,471 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:27:15,477 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:27:15,477 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:27:15,482 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:27:15,742 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:27:17,482 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:27:17,747 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:27:20,248 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:27:20,249 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:27:20,249 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:27:20,255 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:27:20,750 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:27:21,753 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:27:23,255 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:27:25,910 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 15:27:26,371 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:27:26,372 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:27:26,372 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:27:26,377 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:27:26,757 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:27:27,760 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:27:28,377 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:27:30,168 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:27:30,168 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:27:30,209 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:27:32,251 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:27:32,252 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:27:32,252 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:27:32,257 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:27:32,766 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:27:33,767 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:27:34,293 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:27:36,357 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:27:36,358 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:27:36,358 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:27:36,363 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:27:36,768 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:27:37,771 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:27:39,364 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:27:39,683 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:27:39,684 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:27:39,684 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:27:39,689 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:27:39,772 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:27:41,775 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:27:42,616 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:27:42,617 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:27:42,617 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:27:42,622 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:27:42,776 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:27:43,778 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:27:44,623 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:27:45,168 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:27:45,169 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:27:45,209 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:27:48,822 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:27:48,822 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:27:48,822 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:27:48,827 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:27:49,785 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:27:49,785 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:27:49,865 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:27:54,340 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:27:54,341 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:27:54,341 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:27:54,346 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:27:54,787 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:27:55,346 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:27:55,790 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:27:55,915 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 15:27:58,981 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:27:58,982 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:27:58,983 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:27:58,987 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:27:59,794 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:27:59,794 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:28:00,169 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:28:00,169 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:28:00,209 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:28:00,369 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:28:05,370 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:28:05,469 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:28:05,469 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:28:05,469 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:28:05,474 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:28:05,798 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:28:07,801 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:28:09,113 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:28:09,113 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:28:09,114 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:28:09,119 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:28:09,802 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:28:11,119 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:28:11,804 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:28:14,034 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:28:14,035 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:28:14,035 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:28:14,041 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:28:14,806 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:28:15,169 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:28:15,169 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:28:15,169 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:28:15,809 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:28:16,464 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:28:21,465 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:28:22,563 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:28:22,564 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:28:22,564 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:28:22,569 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:28:22,812 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:28:23,815 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:28:25,914 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 15:28:26,480 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:28:26,481 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:28:26,481 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:28:26,481 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:28:26,486 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:28:26,817 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:28:27,817 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:28:30,169 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:28:30,169 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:28:30,209 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:28:32,405 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:28:34,140 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:28:34,140 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:28:34,141 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:28:34,145 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:28:34,822 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:28:35,825 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:28:38,181 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:28:43,182 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:28:44,173 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:28:44,174 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:28:44,174 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:28:44,179 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:28:44,830 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:28:45,169 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:28:45,169 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:28:45,209 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:28:45,832 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:28:48,447 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:28:49,778 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:28:49,778 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:28:49,778 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:28:49,783 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:28:49,835 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:28:51,838 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:28:53,822 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:28:55,919 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 15:28:55,951 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:28:55,952 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:28:55,952 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:28:55,957 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:28:56,841 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:28:57,843 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:28:58,958 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:29:00,102 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:29:00,103 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:29:00,103 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:29:00,109 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:29:00,169 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:29:00,170 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:29:00,170 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:29:00,845 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:29:01,848 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:29:04,387 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:29:07,366 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:29:07,367 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:29:07,367 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:29:07,373 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:29:07,851 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:29:09,854 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:29:10,373 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:29:10,880 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:29:10,881 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:29:10,881 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:29:10,886 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:29:11,857 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:29:11,858 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:29:14,354 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:29:14,355 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:29:14,356 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:29:14,360 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:29:14,859 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:29:15,169 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:29:15,169 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:29:15,213 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:29:15,499 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:29:15,861 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:29:20,499 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:29:21,306 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:29:21,307 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:29:21,307 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:29:21,312 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:29:21,865 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:29:23,868 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:29:24,886 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:29:24,887 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:29:24,887 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:29:24,892 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:29:25,869 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:29:25,869 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:29:25,893 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:29:25,918 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 15:29:28,759 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:29:28,760 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:29:28,760 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:29:28,766 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:29:28,871 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:29:29,873 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:29:30,169 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:29:30,169 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:29:30,213 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:29:31,462 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:29:33,675 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:29:33,676 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:29:33,676 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:29:33,681 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:29:33,877 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:29:35,880 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:29:36,717 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:29:37,418 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:29:37,418 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:29:37,419 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:29:37,423 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:29:37,881 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:29:39,884 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:29:42,424 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:29:42,861 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:29:42,862 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:29:42,862 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:29:42,867 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:29:42,886 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:29:43,888 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:29:45,169 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:29:45,169 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:29:45,213 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:29:46,320 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:29:46,320 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:29:46,321 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:29:46,326 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:29:46,890 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:29:47,891 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:29:48,361 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:29:50,353 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:29:50,354 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:29:50,354 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:29:50,359 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:29:50,893 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:29:51,896 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:29:54,360 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:29:54,412 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:29:54,413 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:29:54,413 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:29:54,418 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:29:54,898 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:29:55,900 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:29:55,920 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 15:29:58,911 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:29:58,912 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:29:58,912 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:29:58,918 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:29:59,903 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:29:59,903 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:29:59,918 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:30:00,169 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:30:00,169 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:30:00,213 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:30:02,964 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:30:02,965 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:30:02,965 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:30:02,970 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:30:03,907 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:30:03,907 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:30:05,005 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:30:09,563 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:30:09,564 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:30:09,564 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:30:09,569 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:30:09,910 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:30:10,569 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:30:11,913 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:30:13,629 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:30:13,634 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:30:13,634 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:30:13,639 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:30:13,915 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:30:15,169 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:30:15,170 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:30:15,213 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:30:15,918 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:30:16,434 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:30:19,083 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:30:19,084 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:30:19,084 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:30:19,089 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:30:19,922 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:30:19,922 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:30:22,125 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:30:22,587 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:30:22,587 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:30:22,588 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:30:22,593 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:30:22,924 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:30:23,926 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:30:25,922 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 15:30:27,433 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:30:27,434 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:30:27,434 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:30:27,434 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:30:27,439 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:30:27,929 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:30:29,934 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:30:30,170 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:30:30,170 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:30:30,213 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:30:32,477 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:30:33,580 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:30:33,581 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:30:33,581 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:30:33,586 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:30:33,937 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:30:35,940 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:30:36,830 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:30:36,830 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:30:36,830 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:30:36,835 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:30:36,941 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:30:37,836 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:30:37,943 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:30:42,811 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:30:42,812 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:30:42,812 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:30:42,817 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:30:42,946 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:30:43,818 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:30:43,948 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:30:45,170 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:30:45,170 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:30:45,213 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:30:46,819 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:30:46,820 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:30:46,820 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:30:46,825 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:30:46,950 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:30:47,952 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:30:48,861 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:30:50,359 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:30:50,360 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:30:50,360 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:30:50,365 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:30:50,955 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:30:51,958 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:30:53,872 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:30:53,873 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:30:53,873 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:30:53,873 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:30:53,878 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:30:53,959 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:30:55,924 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 15:30:55,962 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:30:56,983 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:30:56,984 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:30:56,984 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:30:56,989 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:30:57,965 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:30:57,965 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:30:58,989 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:31:00,170 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:31:00,170 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:31:00,213 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:31:00,867 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:31:00,868 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:31:00,868 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:31:00,874 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:31:00,966 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:31:01,969 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:31:04,657 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:31:04,658 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:31:04,658 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:31:04,658 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:31:04,663 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:31:04,971 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:31:05,973 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:31:08,623 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:31:08,623 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:31:08,624 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:31:08,628 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:31:08,975 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:31:09,978 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:31:10,629 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:31:12,174 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:31:12,175 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:31:12,175 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:31:12,180 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:31:12,979 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:31:13,982 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:31:15,170 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:31:15,170 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:31:15,213 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:31:16,217 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:31:16,217 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:31:16,217 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:31:16,218 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:31:16,222 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:31:16,984 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:31:17,986 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:31:21,258 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:31:24,712 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:31:24,713 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:31:24,713 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:31:24,718 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:31:24,990 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:31:25,930 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 15:31:25,993 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:31:26,931 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:31:28,190 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:31:28,191 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:31:28,191 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:31:28,196 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:31:28,995 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:31:29,995 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:31:30,170 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:31:30,170 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:31:30,213 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:31:32,412 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:31:32,944 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:31:32,945 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:31:32,945 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:31:32,950 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:31:32,998 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:31:34,000 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:31:37,985 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:31:38,610 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:31:38,611 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:31:38,611 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:31:38,616 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:31:39,003 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:31:40,006 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:31:42,721 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:31:42,721 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:31:42,721 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:31:42,726 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:31:43,007 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:31:43,727 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:31:44,010 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:31:45,170 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:31:45,170 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:31:45,213 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:31:46,409 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:31:46,410 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:31:46,410 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:31:46,415 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:31:47,011 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:31:48,014 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:31:49,453 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:31:50,150 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:31:50,150 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:31:50,151 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:31:50,155 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:31:51,019 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:31:52,021 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:31:54,622 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:31:54,623 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:31:54,623 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:31:54,623 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:31:54,628 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:31:55,023 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:31:55,929 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 15:31:56,025 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:31:58,417 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:31:58,418 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:31:58,418 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:31:58,423 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:31:59,027 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:32:00,028 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:32:00,168 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:32:00,170 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:32:00,170 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:32:00,213 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:32:02,653 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:32:02,653 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:32:02,653 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:32:02,659 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:32:03,029 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:32:04,032 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:32:05,693 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:32:10,694 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:32:10,926 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:32:10,927 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:32:10,927 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:32:10,932 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:32:11,036 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:32:14,039 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:32:15,170 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:32:15,170 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:32:15,213 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:32:16,103 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:32:16,104 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:32:16,104 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:32:16,104 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:32:16,109 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:32:17,041 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:32:18,044 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:32:19,976 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:32:19,977 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:32:19,977 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:32:19,981 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:32:20,045 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:32:21,982 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:32:22,048 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:32:25,937 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 15:32:26,271 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:32:26,272 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:32:26,272 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:32:26,277 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:32:27,050 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:32:27,277 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:32:28,053 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:32:29,580 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:32:29,580 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:32:29,581 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:32:29,586 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:32:30,054 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:32:30,170 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:32:30,170 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:32:30,213 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:32:32,058 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:32:32,402 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:32:37,403 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:32:38,039 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:32:38,040 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:32:38,040 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:32:38,045 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:32:38,062 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:32:40,065 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:32:42,659 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:32:42,660 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:32:42,660 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:32:42,660 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:32:42,665 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:32:43,066 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:32:44,069 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:32:45,170 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:32:45,171 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:32:45,213 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:32:47,012 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:32:47,013 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:32:47,013 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:32:47,018 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:32:47,070 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:32:48,053 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:32:50,075 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:32:51,176 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:32:51,177 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:32:51,177 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:32:51,182 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:32:52,076 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:32:53,182 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:32:54,080 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:32:55,932 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 15:32:56,097 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:32:56,097 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:32:56,097 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:32:56,102 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:32:57,081 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:32:58,084 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:32:59,103 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:33:00,170 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:33:00,170 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:33:00,213 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:33:00,692 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:33:00,692 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:33:00,692 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:33:00,697 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:33:01,085 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:33:02,088 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:33:04,733 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:33:05,303 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:33:05,303 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:33:05,303 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:33:05,308 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:33:06,090 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:33:08,094 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:33:09,495 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:33:09,496 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:33:09,496 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:33:09,502 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:33:10,095 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:33:10,502 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:33:12,098 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:33:13,886 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:33:13,886 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:33:13,887 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:33:13,892 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:33:14,099 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:33:15,170 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:33:15,171 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:33:15,213 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:33:16,102 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:33:16,425 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:33:18,136 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:33:18,136 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:33:18,136 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:33:18,141 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:33:19,104 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:33:20,106 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:33:22,177 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:33:23,134 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:33:23,135 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:33:23,135 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:33:23,140 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:33:24,108 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:33:25,937 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 15:33:26,111 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:33:27,295 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:33:27,295 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:33:27,296 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:33:27,296 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:33:27,301 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:33:28,113 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:33:30,116 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:33:30,170 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:33:30,171 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:33:30,213 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:33:30,494 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:33:30,494 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:33:30,494 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:33:30,500 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:33:31,116 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:33:32,120 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:33:32,537 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:33:33,722 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:33:33,723 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:33:33,723 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:33:33,728 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:33:34,121 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:33:36,124 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:33:37,729 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:33:42,346 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:33:42,347 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:33:42,347 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:33:42,353 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:33:43,127 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:33:43,354 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:33:44,130 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:33:45,171 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:33:45,171 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:33:45,213 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:33:47,154 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:33:47,154 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:33:47,155 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:33:47,160 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:33:48,132 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:33:49,197 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:33:50,136 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:33:51,398 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:33:51,399 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:33:51,399 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:33:51,404 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:33:52,138 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:33:54,140 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:33:54,404 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:33:55,937 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 15:33:56,288 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:33:56,288 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:33:56,289 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:33:56,293 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:33:57,142 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:33:58,145 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:34:00,169 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:34:00,171 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:34:00,171 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:34:00,213 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:34:00,357 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:34:00,476 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:34:00,476 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:34:00,481 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:34:01,146 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:34:02,149 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:34:03,857 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:34:03,858 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:34:03,858 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:34:03,863 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:34:04,150 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:34:05,863 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:34:06,154 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:34:07,163 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:34:07,163 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:34:07,164 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:34:07,168 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:34:08,155 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:34:10,158 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:34:11,169 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:34:12,117 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:34:12,118 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:34:12,118 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:34:12,123 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:34:12,159 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:34:14,162 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:34:15,171 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:34:15,171 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:34:15,213 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:34:16,486 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:34:20,453 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:34:20,453 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:34:20,453 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:34:20,458 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:34:21,167 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:34:21,494 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:34:22,168 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:34:24,165 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:34:24,166 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:34:24,166 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:34:24,171 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:34:24,171 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:34:25,939 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 15:34:26,174 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:34:26,940 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:34:27,933 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:34:27,934 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:34:27,934 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:34:27,939 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:34:28,175 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:34:30,171 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:34:30,171 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:34:30,178 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:34:30,213 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:34:31,526 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:34:31,527 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:34:31,527 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:34:31,532 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:34:32,181 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:34:32,570 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:34:34,184 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:34:37,305 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:34:37,306 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:34:37,306 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:34:37,311 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:34:38,186 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:34:38,312 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:34:40,189 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:34:41,170 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:34:41,170 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:34:41,171 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:34:41,176 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:34:41,190 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:34:44,176 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:34:44,193 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:34:45,068 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:34:45,069 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:34:45,069 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:34:45,074 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:34:45,172 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:34:45,172 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:34:45,194 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:34:45,213 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:34:48,197 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:34:48,909 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:34:48,910 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:34:48,910 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:34:48,915 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:34:49,199 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:34:49,954 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:34:50,202 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:34:53,948 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:34:53,949 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:34:53,949 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:34:53,954 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:34:54,204 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:34:54,955 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:34:55,941 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 15:34:56,207 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:34:57,706 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:34:57,707 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:34:57,707 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:34:57,712 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:34:58,208 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:35:00,169 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:35:00,171 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:35:00,171 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:35:00,211 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:35:00,213 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:35:03,980 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:35:03,980 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:35:03,980 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:35:03,986 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:35:04,213 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:35:06,022 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:35:06,217 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:35:07,792 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:35:07,793 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:35:07,793 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:35:07,798 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:35:08,218 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:35:10,221 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:35:11,334 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:35:11,334 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:35:11,335 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:35:11,335 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:35:11,339 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:35:12,222 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:35:14,225 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:35:15,090 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:35:15,090 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:35:15,091 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:35:15,096 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:35:15,171 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:35:15,171 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:35:15,213 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:35:15,226 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:35:16,480 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:35:18,230 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:35:21,481 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:35:21,494 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:35:21,494 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:35:21,494 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:35:21,499 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:35:22,232 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:35:24,235 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:35:25,644 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:35:25,645 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:35:25,645 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:35:25,650 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:35:25,943 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 15:35:26,236 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:35:26,944 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:35:28,240 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:35:29,710 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:35:29,711 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:35:29,711 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:35:29,716 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:35:30,171 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:35:30,172 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:35:30,213 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:35:30,241 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:35:32,245 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:35:32,390 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:35:33,603 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:35:33,603 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:35:33,604 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:35:33,608 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:35:34,246 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:35:36,249 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:35:36,673 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:35:36,674 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:35:36,674 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:35:36,679 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:35:37,250 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:35:37,680 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:35:38,252 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:35:40,974 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:35:40,975 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:35:40,975 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:35:40,980 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:35:41,254 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:35:42,257 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:35:42,981 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:35:45,171 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:35:45,172 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:35:45,213 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:35:46,105 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:35:46,106 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:35:46,106 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:35:46,112 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:35:46,259 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:35:48,149 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:35:48,262 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:35:49,627 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:35:49,627 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:35:49,628 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:35:49,633 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:35:50,264 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:35:52,267 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:35:53,633 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:35:54,554 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:35:54,555 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:35:54,555 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:35:54,560 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:35:55,269 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:35:55,949 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 15:35:56,271 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:35:58,698 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:35:58,699 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:35:58,699 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:35:58,699 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:35:58,704 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:35:59,273 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:36:00,172 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:36:00,172 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:36:00,213 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:36:00,275 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:36:04,413 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:36:05,695 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:36:05,695 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:36:05,695 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:36:05,700 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:36:06,279 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:36:08,282 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:36:09,738 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:36:10,053 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:36:10,054 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:36:10,054 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:36:10,059 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:36:10,283 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:36:12,286 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:36:15,060 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:36:15,172 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:36:15,172 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:36:15,213 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:36:15,247 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:36:15,411 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:36:15,411 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:36:15,416 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:36:16,289 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:36:18,292 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:36:18,446 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:36:18,446 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:36:18,447 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:36:18,451 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:36:19,292 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:36:20,293 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:36:20,452 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:36:21,719 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:36:21,719 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:36:21,720 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:36:21,724 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:36:22,294 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:36:24,297 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:36:25,058 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:36:25,058 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:36:25,059 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:36:25,064 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:36:25,298 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:36:25,948 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 15:36:25,949 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:36:26,300 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:36:30,172 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:36:30,172 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:36:30,213 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:36:31,420 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:36:33,497 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:36:33,497 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:36:33,497 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:36:33,502 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:36:34,306 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:36:36,309 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:36:36,537 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:36:38,369 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:36:38,370 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:36:38,370 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:36:38,375 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:36:39,311 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:36:40,314 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:36:42,376 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:36:45,172 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:36:45,172 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:36:45,213 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:36:46,789 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:36:46,790 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:36:46,790 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:36:46,795 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:36:47,317 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:36:47,833 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:36:48,320 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:36:50,292 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:36:50,293 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:36:50,293 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:36:50,298 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:36:50,322 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:36:52,325 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:36:53,299 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:36:55,639 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:36:55,640 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:36:55,640 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:36:55,645 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:36:55,953 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 15:36:56,327 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:36:58,331 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:36:58,954 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:36:59,374 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:36:59,375 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:36:59,375 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:36:59,380 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:37:00,172 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:37:00,172 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:37:00,213 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:37:00,332 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:37:02,335 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:37:03,981 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:37:03,982 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:37:03,982 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:37:03,982 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:37:03,987 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:37:04,336 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:37:06,340 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:37:08,151 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:37:08,151 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:37:08,152 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:37:08,157 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:37:08,341 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:37:09,157 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:37:10,344 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:37:11,371 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:37:11,371 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:37:11,372 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:37:11,377 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:37:12,345 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:37:14,348 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:37:14,377 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:37:15,172 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:37:15,172 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:37:15,213 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:37:16,196 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:37:16,196 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:37:16,196 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:37:16,201 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:37:16,349 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:37:18,353 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:37:20,238 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:37:21,578 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:37:21,579 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:37:21,579 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:37:21,584 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:37:22,355 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:37:24,358 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:37:25,347 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:37:25,349 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:37:25,349 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:37:25,349 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:37:25,354 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:37:25,359 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:37:25,952 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 15:37:28,362 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:37:30,172 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:37:30,172 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:37:30,213 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:37:30,375 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:37:31,331 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:37:31,331 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:37:31,332 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:37:31,336 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:37:31,364 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:37:34,369 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:37:36,374 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:37:39,687 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:37:39,688 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:37:39,689 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:37:39,693 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:37:40,372 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:37:41,694 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:37:42,375 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:37:45,172 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:37:45,172 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:37:45,213 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:37:47,425 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:37:52,426 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:37:53,412 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:37:53,413 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:37:53,413 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:37:53,418 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:37:54,383 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:37:55,954 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 15:37:56,386 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:37:57,533 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:37:57,537 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:37:57,537 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:37:57,537 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:37:57,542 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:37:58,387 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:38:00,172 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:38:00,172 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:38:00,213 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:38:00,390 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:38:01,146 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:38:01,146 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:38:01,146 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:38:01,151 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:38:01,391 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:38:02,393 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:38:03,189 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:38:05,136 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:38:05,136 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:38:05,137 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:38:05,142 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:38:05,395 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:38:06,397 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:38:09,142 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:38:12,534 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:38:12,535 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:38:12,535 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:38:12,540 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:38:13,401 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:38:14,403 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:38:14,540 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:38:15,172 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:38:15,173 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:38:15,213 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:38:17,396 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:38:17,397 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:38:17,397 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:38:17,402 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:38:17,405 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:38:20,409 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:38:20,437 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:38:20,627 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:38:20,628 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:38:20,628 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:38:20,633 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:38:21,409 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:38:22,412 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:38:24,677 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:38:24,677 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:38:24,678 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:38:24,683 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:38:25,414 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:38:25,683 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:38:25,956 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 15:38:26,417 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:38:27,958 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:38:27,959 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:38:27,959 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:38:27,964 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:38:28,418 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:38:30,172 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:38:30,173 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:38:30,213 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:38:30,421 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:38:31,461 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:38:31,520 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:38:31,521 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:38:31,521 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:38:31,526 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:38:32,424 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:38:34,427 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:38:35,994 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:38:35,994 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:38:35,995 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:38:35,999 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:38:36,428 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:38:37,000 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:38:38,431 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:38:40,668 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:38:40,669 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:38:40,669 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:38:40,674 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:38:41,433 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:38:42,433 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:38:42,674 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:38:44,399 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:38:44,399 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:38:44,400 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:38:44,405 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:38:44,434 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:38:45,173 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:38:45,173 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:38:45,213 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:38:46,437 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:38:48,418 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:38:48,950 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:38:48,950 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:38:48,950 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:38:48,955 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:38:49,440 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:38:50,443 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:38:53,994 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:38:54,348 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:38:54,348 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:38:54,349 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:38:54,354 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:38:54,445 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:38:55,958 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 15:38:56,448 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:38:58,370 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:38:58,371 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:38:58,371 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:38:58,376 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:38:58,449 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:38:59,376 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:39:00,173 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:39:00,173 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:39:00,213 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:39:00,452 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:39:01,497 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:39:01,497 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:39:01,497 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:39:01,502 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:39:02,453 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:39:04,454 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:39:04,537 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:39:05,874 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:39:05,874 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:39:05,874 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:39:05,879 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:39:06,456 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:39:08,459 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:39:09,880 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:39:14,252 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:39:14,253 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:39:14,253 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:39:14,259 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:39:14,462 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:39:15,171 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:39:15,173 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:39:15,173 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:39:15,213 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:39:16,466 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:39:18,354 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:39:18,355 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:39:18,355 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:39:18,360 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:39:18,467 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:39:20,397 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:39:20,468 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:39:24,413 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:39:24,414 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:39:24,414 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:39:24,418 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:39:24,470 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:39:25,419 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:39:25,960 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 15:39:26,473 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:39:29,180 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:39:29,180 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:39:29,181 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:39:29,186 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:39:29,475 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:39:30,173 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:39:30,173 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:39:30,213 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:39:30,420 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:39:30,478 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:39:32,755 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:39:32,755 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:39:32,756 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:39:32,761 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:39:33,481 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:39:34,483 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:39:35,797 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:39:36,448 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:39:36,449 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:39:36,449 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:39:36,454 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:39:36,484 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:39:38,485 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:39:40,803 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:39:40,804 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:39:40,804 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:39:40,805 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:39:40,810 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:39:41,487 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:39:42,490 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:39:45,173 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:39:45,173 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:39:45,217 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:39:45,344 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:39:45,435 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:39:45,435 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:39:45,440 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:39:45,491 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:39:46,441 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:39:48,495 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:39:51,441 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:39:51,749 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:39:51,750 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:39:51,750 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:39:51,755 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:39:52,498 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:39:54,501 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:39:55,962 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 15:39:56,403 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:39:56,404 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:39:56,404 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:39:56,409 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:39:56,502 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:39:57,410 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:39:58,506 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:39:59,923 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:39:59,924 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:39:59,924 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:39:59,929 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:40:00,173 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:40:00,173 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:40:00,217 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:40:00,507 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:40:02,465 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:40:02,510 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:40:02,844 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:40:02,845 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:40:02,845 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:40:02,850 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:40:03,510 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:40:04,513 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:40:06,495 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:40:06,496 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:40:06,497 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:40:06,502 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:40:06,515 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:40:07,502 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:40:08,516 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:40:11,047 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:40:11,048 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:40:11,048 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:40:11,053 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:40:11,517 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:40:12,520 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:40:13,053 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:40:14,164 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:40:14,165 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:40:14,165 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:40:14,170 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:40:14,521 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:40:15,173 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:40:15,173 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:40:15,217 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:40:16,524 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:40:18,426 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:40:18,445 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:40:18,445 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:40:18,446 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:40:18,451 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:40:18,525 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:40:20,529 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:40:23,486 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:40:24,450 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:40:24,451 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:40:24,451 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:40:24,456 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:40:24,531 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:40:25,967 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 15:40:26,534 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:40:28,969 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:40:29,039 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:40:29,040 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:40:29,040 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:40:29,045 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:40:29,536 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:40:30,173 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:40:30,173 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:40:30,217 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:40:30,538 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:40:32,316 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:40:32,317 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:40:32,317 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:40:32,322 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:40:32,540 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:40:34,358 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:40:34,544 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:40:37,289 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:40:37,289 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:40:37,290 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:40:37,295 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:40:37,545 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:40:38,548 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:40:40,295 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:40:40,663 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:40:40,663 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:40:40,664 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:40:40,669 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:40:41,549 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:40:42,552 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:40:44,741 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:40:44,742 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:40:44,742 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:40:44,747 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:40:45,173 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:40:45,174 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:40:45,217 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:40:45,464 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:40:45,553 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:40:46,556 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:40:48,336 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:40:48,337 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:40:48,337 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:40:48,342 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:40:48,557 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:40:50,561 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:40:51,378 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:40:51,803 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:40:51,804 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:40:51,805 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:40:51,809 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:40:52,562 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:40:54,565 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:40:55,478 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:40:55,479 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:40:55,480 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:40:55,484 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:40:55,566 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:40:55,966 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 15:40:56,968 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:40:58,576 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:41:00,160 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:41:00,161 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:41:00,161 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:41:00,166 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:41:00,173 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:41:00,174 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:41:00,217 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:41:00,577 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:41:02,420 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:41:02,580 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:41:04,023 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:41:04,023 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:41:04,024 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:41:04,028 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:41:04,581 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:41:06,585 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:41:06,951 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:41:06,952 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:41:06,952 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:41:06,957 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:41:07,586 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:41:07,958 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:41:08,588 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:41:12,138 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:41:12,139 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:41:12,140 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:41:12,144 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:41:12,590 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:41:13,145 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:41:14,593 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:41:15,174 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:41:15,174 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:41:15,217 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:41:17,776 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:41:17,776 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:41:17,777 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:41:17,781 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:41:18,596 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:41:18,817 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:41:20,599 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:41:23,743 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:41:23,743 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:41:23,744 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:41:23,749 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:41:24,601 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:41:24,749 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:41:25,974 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 15:41:26,604 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:41:29,975 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:41:30,174 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:41:30,174 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:41:30,217 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:41:32,258 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:41:32,259 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:41:32,259 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:41:32,265 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:41:32,608 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:41:34,611 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:41:35,301 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:41:35,338 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:41:35,338 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:41:35,338 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:41:35,343 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:41:35,612 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:41:36,614 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:41:39,062 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:41:39,062 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:41:39,063 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:41:39,068 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:41:39,616 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:41:40,617 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:41:41,068 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:41:41,886 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:41:41,887 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:41:41,887 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:41:41,892 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:41:42,618 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:41:44,621 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:41:45,174 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:41:45,174 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:41:45,217 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:41:46,237 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:41:46,238 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:41:46,238 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:41:46,238 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:41:46,243 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:41:46,622 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:41:48,625 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:41:49,617 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:41:49,617 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:41:49,618 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:41:49,622 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:41:49,627 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:41:51,623 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:41:52,631 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:41:53,107 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:41:53,108 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:41:53,108 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:41:53,113 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:41:53,632 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:41:54,634 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:41:55,970 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 15:41:56,972 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:41:57,149 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:41:57,150 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:41:57,150 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:41:57,155 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:41:57,636 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:41:58,638 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:42:00,174 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:42:00,174 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:42:00,217 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:42:00,713 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:42:00,714 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:42:00,714 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:42:00,718 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:42:01,640 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:42:02,640 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:42:02,757 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:42:05,626 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:42:05,626 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:42:05,627 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:42:05,633 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:42:05,642 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:42:08,633 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:42:08,646 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:42:10,457 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:42:10,457 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:42:10,457 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:42:10,462 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:42:10,647 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:42:12,650 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:42:13,777 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:42:13,777 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:42:13,778 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:42:13,778 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:42:13,782 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:42:14,651 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:42:15,174 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:42:15,174 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:42:15,217 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:42:16,654 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:42:17,464 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:42:17,464 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:42:17,464 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:42:17,469 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:42:17,654 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:42:18,657 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:42:19,505 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:42:21,092 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:42:21,093 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:42:21,093 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:42:21,098 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:42:21,658 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:42:22,661 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:42:25,099 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:42:25,306 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:42:25,307 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:42:25,307 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:42:25,312 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:42:25,663 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:42:25,976 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 15:42:26,665 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:42:28,780 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:42:28,781 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:42:28,781 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:42:28,786 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:42:29,667 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:42:30,236 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:42:30,236 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:42:30,236 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:42:30,237 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:42:30,668 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:42:32,802 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:42:32,802 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:42:32,802 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:42:32,808 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:42:33,671 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:42:34,673 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:42:35,845 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:42:37,075 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:42:37,075 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:42:37,075 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:42:37,081 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:42:37,675 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:42:38,677 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:42:41,082 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:42:42,724 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:42:42,725 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:42:42,725 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:42:42,730 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:42:43,680 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:42:44,683 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:42:45,236 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:42:45,236 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:42:45,277 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:42:46,530 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:42:47,861 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:42:47,862 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:42:47,862 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:42:47,867 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:42:48,685 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:42:50,689 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:42:51,818 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:42:51,818 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:42:51,819 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:42:51,819 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:42:51,824 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:42:52,690 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:42:54,693 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:42:55,975 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 15:42:56,976 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:42:58,687 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:42:58,688 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:42:58,688 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:42:58,693 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:42:58,695 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:43:00,236 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:43:00,236 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:43:00,236 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:43:00,698 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:43:02,433 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:43:03,624 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:43:03,625 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:43:03,625 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:43:03,630 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:43:03,700 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:43:06,704 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:43:07,665 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:43:10,444 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:43:10,445 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:43:10,445 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:43:10,450 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:43:10,706 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:43:12,709 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:43:13,450 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:43:15,236 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:43:15,236 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:43:15,237 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:43:18,510 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:43:18,719 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:43:18,720 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:43:18,720 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:43:18,725 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:43:19,713 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:43:20,715 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:43:22,526 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:43:22,527 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:43:22,527 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:43:22,531 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:43:22,716 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:43:23,532 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:43:24,719 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:43:25,977 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 15:43:28,423 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:43:28,424 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:43:28,424 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:43:28,429 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:43:28,722 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:43:29,430 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:43:30,236 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:43:30,236 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:43:30,237 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:43:30,725 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:43:34,364 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:43:34,364 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:43:34,364 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:43:34,369 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:43:34,729 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:43:35,409 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:43:36,732 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:43:37,936 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:43:37,937 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:43:37,937 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:43:37,942 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:43:38,733 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:43:40,736 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:43:40,943 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:43:42,430 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:43:42,431 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:43:42,431 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:43:42,437 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:43:42,737 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:43:44,740 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:43:45,236 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:43:45,237 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:43:45,237 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:43:45,282 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:43:45,544 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:43:45,544 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:43:45,549 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:43:45,741 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:43:46,550 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:43:46,744 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:43:49,445 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:43:49,446 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:43:49,446 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:43:49,451 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:43:49,746 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:43:50,747 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:43:52,452 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:43:52,772 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:43:52,773 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:43:52,773 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:43:52,778 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:43:53,749 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:43:54,752 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:43:55,979 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 15:43:56,121 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:43:56,122 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:43:56,122 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:43:56,128 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:43:56,753 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:43:58,128 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:43:58,756 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:43:59,600 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:43:59,601 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:43:59,601 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:43:59,606 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:43:59,756 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:44:00,237 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:44:00,237 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:44:00,237 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:44:00,759 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:44:03,531 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:44:04,033 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:44:04,033 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:44:04,033 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:44:04,038 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:44:04,761 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:44:06,765 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:44:09,029 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:44:09,030 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:44:09,030 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:44:09,030 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:44:09,035 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:44:09,766 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:44:10,769 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:44:12,568 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:44:12,569 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:44:12,569 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:44:12,575 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:44:12,770 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:44:14,576 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:44:14,771 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:44:15,237 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:44:15,237 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:44:15,238 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:44:16,657 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:44:16,657 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:44:16,658 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:44:16,662 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:44:16,772 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:44:18,775 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:44:19,701 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:44:22,067 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:44:22,068 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:44:22,068 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:44:22,073 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:44:22,777 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:44:24,780 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:44:25,074 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:44:25,981 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 15:44:26,166 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:44:26,167 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:44:26,167 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:44:26,172 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:44:26,781 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:44:28,785 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:44:30,173 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:44:30,237 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:44:30,237 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:44:30,277 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:44:30,651 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:44:30,652 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:44:30,652 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:44:30,657 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:44:30,786 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:44:32,790 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:44:35,022 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:44:35,023 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:44:35,023 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:44:35,028 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:44:35,792 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:44:36,028 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:44:36,794 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:44:38,939 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:44:38,940 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:44:38,940 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:44:38,945 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:44:39,796 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:44:40,798 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:44:41,945 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:44:42,421 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:44:42,422 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:44:42,422 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:44:42,427 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:44:42,799 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:44:44,802 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:44:45,237 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:44:45,237 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:44:45,237 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:44:47,496 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:44:50,065 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:44:50,065 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:44:50,065 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:44:50,070 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:44:50,807 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:44:52,809 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:44:53,105 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:44:54,847 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:44:54,848 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:44:54,848 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:44:54,853 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:44:55,811 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:44:55,983 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 15:44:56,813 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:44:58,296 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:44:58,297 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:44:58,297 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:44:58,297 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:44:58,302 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:44:58,815 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:45:00,237 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:45:00,237 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:45:00,237 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:45:00,818 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:45:02,669 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:45:02,669 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:45:02,669 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:45:02,674 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:45:02,819 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:45:03,709 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:45:04,822 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:45:06,455 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:45:06,455 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:45:06,456 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:45:06,461 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:45:06,823 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:45:08,826 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:45:09,462 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:45:12,272 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:45:12,273 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:45:12,273 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:45:12,278 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:45:12,828 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:45:14,831 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:45:15,237 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:45:15,237 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:45:15,237 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:45:15,281 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:45:15,580 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:45:15,581 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:45:15,581 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:45:15,586 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:45:15,832 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:45:16,834 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:45:18,483 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:45:18,484 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:45:18,484 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:45:18,489 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:45:18,835 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:45:20,489 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:45:20,838 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:45:25,489 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:45:25,985 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 15:45:27,015 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:45:27,016 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:45:27,016 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:45:27,021 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:45:27,842 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:45:28,845 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:45:30,237 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:45:30,237 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:45:30,281 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:45:30,496 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:45:31,372 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:45:31,372 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:45:31,373 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:45:31,377 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:45:31,847 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:45:32,850 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:45:36,414 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:45:36,922 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:45:36,922 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:45:36,922 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:45:36,927 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:45:37,853 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:45:38,856 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:45:40,365 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:45:40,366 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:45:40,366 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:45:40,371 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:45:40,857 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:45:42,372 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:45:42,860 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:45:44,489 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:45:44,490 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:45:44,490 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:45:44,495 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:45:44,861 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:45:45,237 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:45:45,237 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:45:45,237 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:45:46,864 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:45:47,539 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:45:50,468 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:45:50,469 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:45:50,469 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:45:50,474 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:45:50,867 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:45:52,868 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:45:53,509 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:45:55,150 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:45:55,151 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:45:55,151 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:45:55,156 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:45:55,870 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:45:55,990 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 15:45:56,872 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:45:58,992 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:45:59,440 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:45:59,440 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:45:59,441 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:45:59,446 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:45:59,874 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:46:00,237 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:46:00,238 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:46:00,238 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:46:00,876 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:46:04,531 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:46:07,881 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:46:07,882 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:46:07,882 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:46:07,886 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:46:08,881 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:46:09,925 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:46:10,884 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:46:11,339 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:46:11,340 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:46:11,340 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:46:11,345 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:46:11,885 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:46:12,887 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:46:14,424 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:46:14,425 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:46:14,425 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:46:14,430 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:46:14,888 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:46:15,237 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:46:15,237 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:46:15,238 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:46:15,238 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:46:16,891 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:46:18,642 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:46:18,643 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:46:18,643 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:46:18,648 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:46:18,892 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:46:20,686 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:46:20,893 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:46:21,947 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:46:21,948 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:46:21,948 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:46:21,953 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:46:22,895 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:46:24,898 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:46:25,954 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:46:25,989 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 15:46:26,934 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:46:26,935 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:46:26,935 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:46:26,940 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:46:27,899 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:46:28,902 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:46:30,238 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:46:30,238 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:46:30,238 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:46:31,456 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:46:35,528 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:46:35,528 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:46:35,529 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:46:35,533 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:46:35,907 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:46:36,569 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:46:36,908 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:46:38,848 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:46:38,849 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:46:38,849 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:46:38,854 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:46:38,909 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:46:40,912 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:46:41,854 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:46:42,513 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:46:42,514 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:46:42,514 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:46:42,519 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:46:42,913 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:46:44,916 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:46:45,238 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:46:45,238 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:46:45,238 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:46:47,513 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:46:51,216 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:46:51,216 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:46:51,216 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:46:51,221 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:46:51,921 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:46:52,923 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:46:53,257 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:46:55,379 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:46:55,380 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:46:55,380 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:46:55,385 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:46:55,925 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:46:55,994 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 15:46:56,927 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:46:58,916 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:46:58,917 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:46:58,917 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:46:58,917 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:46:58,922 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:46:58,928 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:47:00,238 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:47:00,238 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:47:00,238 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:47:00,932 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:47:03,677 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:47:03,678 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:47:03,678 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:47:03,683 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:47:03,933 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:47:04,721 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:47:04,936 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:47:08,449 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:47:08,450 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:47:08,451 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:47:08,456 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:47:08,939 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:47:10,456 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:47:10,941 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:47:14,332 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:47:14,333 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:47:14,333 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:47:14,337 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:47:14,944 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:47:15,238 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:47:15,238 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:47:15,238 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:47:15,473 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:47:16,947 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:47:18,392 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:47:18,392 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:47:18,393 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:47:18,397 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:47:18,948 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:47:20,949 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:47:21,434 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:47:22,525 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:47:22,526 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:47:22,526 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:47:22,532 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:47:22,950 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:47:24,954 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:47:25,993 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 15:47:26,992 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:47:26,993 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:47:26,994 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:47:26,994 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:47:26,999 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:47:27,955 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:47:28,958 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:47:30,238 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:47:30,238 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:47:30,238 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:47:32,499 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:47:35,642 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:47:35,643 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:47:35,643 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:47:35,648 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:47:35,964 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:47:36,966 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:47:37,685 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:47:40,118 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:47:40,119 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:47:40,119 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:47:40,125 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:47:40,968 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:47:42,972 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:47:43,125 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:47:44,709 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:47:44,710 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:47:44,710 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:47:44,715 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:47:44,973 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:47:45,238 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:47:45,238 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:47:45,238 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:47:46,976 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:47:48,477 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:47:49,232 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:47:49,232 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:47:49,233 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:47:49,237 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:47:49,979 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:47:50,981 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:47:53,546 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:47:53,547 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:47:53,547 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:47:53,547 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:47:53,552 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:47:53,983 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:47:54,985 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:47:55,995 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 15:47:58,997 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:48:00,238 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:48:00,238 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:48:00,238 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:48:02,927 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:48:02,927 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:48:02,928 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:48:02,932 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:48:02,989 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:48:04,969 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:48:04,992 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:48:09,970 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:48:11,029 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:48:11,030 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:48:11,031 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:48:11,035 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:48:11,997 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:48:12,999 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:48:15,036 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:48:15,241 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:48:15,241 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:48:15,241 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:48:15,306 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:48:15,524 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:48:15,524 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:48:15,529 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:48:16,001 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:48:17,003 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:48:20,173 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:48:20,174 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:48:20,174 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:48:20,174 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:48:20,179 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:48:21,005 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:48:23,008 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:48:24,360 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:48:24,361 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:48:24,361 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:48:24,366 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:48:25,009 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:48:25,366 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:48:25,998 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 15:48:27,012 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:48:29,217 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:48:29,218 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:48:29,218 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:48:29,223 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:48:30,014 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:48:30,240 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:48:30,241 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:48:30,242 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:48:30,482 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:48:31,017 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:48:32,433 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:48:32,433 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:48:32,434 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:48:32,438 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:48:33,019 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:48:35,022 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:48:36,046 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:48:36,047 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:48:36,047 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:48:36,048 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:48:36,052 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:48:37,023 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:48:39,026 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:48:39,991 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:48:39,992 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:48:39,992 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:48:39,997 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:48:40,027 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:48:41,029 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:48:41,998 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:48:43,230 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:48:43,230 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:48:43,231 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:48:43,239 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:48:44,031 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:48:45,034 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:48:45,240 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:48:45,241 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:48:45,241 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:48:47,317 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:48:47,318 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:48:47,318 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:48:47,318 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:48:47,323 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:48:48,035 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:48:49,038 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:48:52,362 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:48:56,000 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 15:48:56,668 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:48:56,669 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:48:56,669 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:48:56,674 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:48:57,043 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:48:57,675 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:48:59,046 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:49:00,240 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:49:00,241 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:49:00,242 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:49:01,031 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:49:01,031 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:49:01,032 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:49:01,037 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:49:01,047 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:49:03,048 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:49:03,074 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:49:07,447 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:49:07,448 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:49:07,449 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:49:07,454 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:49:08,052 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:49:08,454 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:49:09,054 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:49:11,785 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:49:11,785 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:49:11,786 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:49:11,791 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:49:12,056 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:49:13,058 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:49:13,791 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:49:15,241 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:49:15,241 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:49:15,242 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:49:15,503 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:49:15,536 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:49:15,536 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:49:15,541 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:49:16,060 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:49:17,062 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:49:19,541 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:49:19,856 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:49:19,857 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:49:19,857 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:49:19,862 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:49:20,064 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:49:21,067 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:49:23,100 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:49:23,101 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:49:23,101 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:49:23,106 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:49:24,068 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:49:25,069 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:49:25,107 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:49:26,005 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 15:49:28,244 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:49:28,245 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:49:28,246 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:49:28,250 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:49:29,071 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:49:30,241 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:49:30,241 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:49:30,241 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:49:30,515 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:49:31,074 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:49:33,177 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:49:33,177 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:49:33,178 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:49:33,182 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:49:34,077 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:49:35,080 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:49:36,218 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:49:37,487 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:49:37,488 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:49:37,488 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:49:37,493 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:49:38,082 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:49:39,084 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:49:41,494 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:49:42,262 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:49:42,263 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:49:42,263 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:49:42,268 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:49:43,086 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:49:45,089 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:49:45,241 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:49:45,241 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:49:45,241 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:49:46,453 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:49:46,454 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:49:46,454 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:49:46,459 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:49:47,090 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:49:47,493 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:49:49,093 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:49:49,907 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:49:49,907 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:49:49,908 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:49:49,913 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:49:50,095 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:49:51,098 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:49:52,913 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:49:53,121 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:49:53,122 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:49:53,122 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:49:53,127 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:49:54,100 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:49:55,102 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:49:56,004 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 15:49:56,025 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:49:56,026 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:49:56,026 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:49:56,032 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:49:56,102 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:49:57,105 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:49:58,032 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:49:59,762 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:49:59,762 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:49:59,763 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:49:59,768 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:50:00,107 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:50:00,241 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:50:00,241 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:50:00,241 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:50:01,109 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:50:03,553 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:50:05,169 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:50:05,170 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:50:05,170 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:50:05,175 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:50:06,112 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:50:07,114 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:50:09,214 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:50:09,230 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:50:09,231 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:50:09,231 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:50:09,236 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:50:10,116 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:50:11,119 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:50:12,754 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:50:12,755 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:50:12,757 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:50:12,762 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:50:13,120 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:50:14,763 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:50:15,123 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:50:15,241 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:50:15,241 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:50:15,241 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:50:16,641 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:50:16,641 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:50:16,642 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:50:16,647 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:50:17,124 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:50:19,127 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:50:20,686 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:50:20,957 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:50:20,957 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:50:20,958 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:50:20,963 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:50:21,129 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:50:23,132 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:50:24,427 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:50:24,428 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:50:24,428 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:50:24,433 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:50:25,133 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:50:26,011 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 15:50:26,013 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:50:27,136 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:50:27,857 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:50:27,858 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:50:27,858 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:50:27,863 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:50:28,137 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:50:29,139 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:50:30,241 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:50:30,241 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:50:30,241 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:50:31,165 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:50:31,166 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:50:31,166 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:50:31,166 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:50:31,171 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:50:32,141 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:50:33,145 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:50:35,732 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:50:35,733 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:50:35,733 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:50:35,738 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:50:36,147 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:50:36,738 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:50:37,149 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:50:39,660 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:50:39,661 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:50:39,661 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:50:39,666 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:50:40,151 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:50:41,153 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:50:42,667 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:50:45,241 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:50:45,241 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:50:45,242 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:50:46,708 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:50:46,708 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:50:46,709 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:50:46,713 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:50:47,157 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:50:47,749 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:50:49,160 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:50:49,976 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:50:49,977 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:50:49,977 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:50:49,982 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:50:50,161 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:50:51,164 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:50:52,983 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:50:53,444 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:50:53,444 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:50:53,445 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:50:53,449 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:50:54,165 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:50:55,168 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:50:56,008 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 15:50:57,706 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:50:57,706 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:50:57,707 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:50:57,712 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:50:58,170 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:50:58,713 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:50:59,171 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:51:00,241 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:51:00,242 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:51:00,242 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:51:00,728 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:51:00,729 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:51:00,729 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:51:00,734 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:51:01,173 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:51:03,175 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:51:03,770 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:51:04,477 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:51:04,478 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:51:04,478 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:51:04,483 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:51:05,177 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:51:07,180 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:51:09,484 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:51:09,763 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:51:09,763 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:51:09,764 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:51:09,769 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:51:10,182 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:51:11,184 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:51:13,289 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:51:13,290 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:51:13,290 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:51:13,295 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:51:14,186 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:51:15,186 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:51:15,241 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:51:15,241 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:51:15,242 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:51:15,533 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:51:16,963 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:51:16,964 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:51:16,964 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:51:16,969 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:51:17,187 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:51:19,190 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:51:21,006 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:51:21,728 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:51:21,728 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:51:21,729 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:51:21,734 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:51:22,192 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:51:23,194 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:51:24,976 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:51:24,977 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:51:24,978 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:51:24,983 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:51:25,195 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:51:26,013 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 15:51:26,015 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:51:27,198 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:51:29,310 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:51:29,311 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:51:29,312 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:51:29,317 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:51:30,200 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:51:30,241 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:51:30,242 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:51:30,242 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:51:31,203 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:51:31,481 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:51:33,256 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:51:33,257 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:51:33,257 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:51:33,262 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:51:34,206 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:51:35,208 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:51:37,298 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:51:39,290 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:51:39,291 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:51:39,291 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:51:39,296 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:51:40,211 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:51:41,214 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:51:42,508 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:51:42,509 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:51:42,510 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:51:42,510 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:51:42,515 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:51:43,215 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:51:45,218 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:51:45,241 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:51:45,242 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:51:45,242 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:51:47,539 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:51:47,912 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:51:47,912 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:51:47,912 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:51:47,917 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:51:48,219 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:51:49,222 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:51:51,485 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:51:51,485 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:51:51,486 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:51:51,491 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:51:52,225 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:51:53,227 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:51:53,491 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:51:54,672 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:51:54,673 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:51:54,673 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:51:54,678 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:51:55,228 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:51:56,012 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 15:51:57,231 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:51:58,418 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:51:58,418 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:51:58,419 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:51:58,424 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:51:59,232 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:51:59,424 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:52:00,242 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:52:00,242 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:52:00,242 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:52:01,235 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:52:02,925 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:52:02,926 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:52:02,926 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:52:02,931 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:52:03,237 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:52:04,969 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:52:05,240 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:52:06,143 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:52:06,143 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:52:06,144 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:52:06,148 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:52:06,240 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:52:07,243 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:52:10,149 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:52:11,579 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:52:11,580 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:52:11,580 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:52:11,585 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:52:12,245 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:52:13,248 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:52:14,856 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:52:14,857 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:52:14,857 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:52:14,862 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:52:15,242 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:52:15,242 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:52:15,242 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:52:15,249 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:52:15,533 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:52:17,252 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:52:18,086 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:52:18,087 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:52:18,087 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:52:18,092 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:52:18,253 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:52:19,256 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:52:21,130 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:52:22,103 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:52:22,104 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:52:22,105 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:52:22,109 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:52:22,257 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:52:23,260 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:52:26,017 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 15:52:26,101 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:52:26,101 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:52:26,102 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:52:26,107 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:52:26,262 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:52:27,107 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:52:27,264 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:52:29,158 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:52:29,159 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:52:29,159 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:52:29,164 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:52:29,266 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:52:30,242 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:52:30,242 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:52:30,242 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:52:31,269 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:52:32,462 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:52:33,389 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:52:33,389 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:52:33,390 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:52:33,395 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:52:34,272 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:52:35,275 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:52:38,430 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:52:38,826 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:52:38,827 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:52:38,827 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:52:38,832 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:52:39,277 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:52:41,280 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:52:43,833 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:52:45,242 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:52:45,242 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:52:45,242 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:52:45,542 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:52:45,545 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:52:45,545 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:52:45,550 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:52:46,283 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:52:47,285 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:52:48,866 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:52:48,866 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:52:48,867 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:52:48,867 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:52:48,872 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:52:49,287 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:52:51,291 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:52:53,628 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:52:53,629 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:52:53,629 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:52:53,634 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:52:54,293 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:52:54,634 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:52:55,295 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:52:56,016 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 15:52:57,495 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:52:57,497 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:52:57,497 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:52:57,502 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:52:58,297 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:52:59,299 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:53:00,242 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:53:00,242 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:53:00,242 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:53:00,541 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:53:01,015 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:53:01,016 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:53:01,016 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:53:01,022 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:53:01,301 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:53:03,304 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:53:05,315 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:53:05,316 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:53:05,316 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:53:05,321 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:53:06,306 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:53:06,322 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:53:07,308 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:53:10,197 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:53:10,198 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:53:10,198 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:53:10,203 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:53:10,310 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:53:11,313 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:53:12,204 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:53:15,242 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:53:15,242 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:53:15,242 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:53:15,772 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:53:15,772 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:53:15,773 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:53:15,777 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:53:16,315 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:53:17,318 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:53:17,814 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:53:20,298 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:53:20,299 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:53:20,300 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:53:20,305 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:53:20,319 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:53:21,322 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:53:23,305 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:53:23,714 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:53:23,715 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:53:23,715 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:53:23,720 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:53:24,325 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:53:25,327 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:53:26,018 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 15:53:27,195 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:53:27,196 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:53:27,197 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:53:27,201 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:53:27,328 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:53:29,202 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:53:29,332 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:53:30,242 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:53:30,242 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:53:30,242 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:53:31,995 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:53:31,998 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:53:31,999 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:53:32,005 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:53:32,333 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:53:33,337 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:53:35,038 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:53:35,342 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:53:35,343 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:53:35,344 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:53:35,349 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:53:36,339 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:53:37,342 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:53:38,431 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:53:38,431 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:53:38,432 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:53:38,437 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:53:39,343 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:53:40,437 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:53:41,346 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:53:41,988 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:53:41,989 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:53:41,989 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:53:41,994 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:53:42,346 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:53:43,349 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:53:45,071 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:53:45,072 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:53:45,073 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:53:45,077 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:53:45,242 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:53:45,242 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:53:45,243 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:53:45,350 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:53:45,527 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:53:47,353 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:53:50,527 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:53:52,748 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:53:52,749 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:53:52,749 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:53:52,754 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:53:53,358 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:53:55,361 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:53:55,790 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:53:56,020 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 15:53:57,767 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:53:57,767 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:53:57,768 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:53:57,773 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:53:58,363 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:53:59,366 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:54:00,242 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:54:00,243 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:54:00,243 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:54:01,246 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:54:01,247 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:54:01,247 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:54:01,247 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:54:01,252 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:54:01,367 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:54:03,370 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:54:05,483 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:54:05,484 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:54:05,485 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:54:05,489 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:54:06,372 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:54:06,490 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:54:07,374 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:54:09,021 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:54:09,022 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:54:09,022 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:54:09,027 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:54:09,376 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:54:11,379 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:54:12,028 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:54:14,192 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:54:14,193 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:54:14,193 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:54:14,198 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:54:14,381 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:54:15,242 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:54:15,243 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:54:15,243 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:54:15,383 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:54:17,503 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:54:18,050 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:54:18,050 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:54:18,051 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:54:18,056 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:54:18,385 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:54:19,387 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:54:21,759 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:54:21,759 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:54:21,760 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:54:21,765 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:54:22,389 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:54:22,765 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:54:23,391 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:54:26,023 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 15:54:27,345 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:54:27,346 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:54:27,346 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:54:27,351 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:54:27,394 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:54:28,351 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:54:29,397 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:54:30,242 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:54:30,243 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:54:30,243 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:54:32,987 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:54:32,987 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:54:32,988 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:54:32,993 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:54:33,401 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:54:34,030 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:54:35,404 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:54:36,953 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:54:36,953 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:54:36,954 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:54:36,959 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:54:37,405 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:54:39,406 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:54:39,959 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:54:40,294 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:54:40,295 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:54:40,295 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:54:40,300 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:54:40,407 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:54:41,409 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:54:43,852 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:54:43,856 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:54:43,856 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:54:43,861 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:54:44,411 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:54:45,243 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:54:45,243 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:54:45,243 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:54:45,414 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:54:45,462 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:54:46,958 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:54:46,959 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:54:46,959 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:54:46,964 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:54:47,415 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:54:49,418 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:54:51,003 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:54:51,465 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:54:51,466 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:54:51,467 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:54:51,471 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:54:52,421 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:54:53,423 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:54:54,317 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:54:54,317 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:54:54,318 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:54:54,323 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:54:54,424 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:54:55,426 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:54:56,025 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 15:54:56,026 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:54:59,719 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:54:59,719 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:54:59,720 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:54:59,725 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:55:00,243 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:55:00,243 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:55:00,243 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:55:00,430 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:55:01,432 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:55:01,526 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:55:03,035 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:55:03,035 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:55:03,036 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:55:03,041 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:55:03,434 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:55:05,435 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:55:06,722 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:55:06,723 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:55:06,723 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:55:06,723 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:55:06,728 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:55:07,436 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:55:09,440 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:55:10,194 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:55:10,195 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:55:10,196 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:55:10,201 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:55:10,440 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:55:11,443 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:55:12,201 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:55:13,879 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:55:13,879 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:55:13,880 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:55:13,885 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:55:14,444 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:55:15,243 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:55:15,243 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:55:15,243 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:55:15,447 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:55:17,533 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:55:17,942 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:55:17,942 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:55:17,943 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:55:17,947 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:55:18,449 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:55:19,451 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:55:21,862 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:55:21,863 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:55:21,864 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:55:21,869 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:55:22,453 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:55:22,869 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:55:23,455 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:55:25,379 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:55:25,380 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:55:25,380 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:55:25,385 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:55:25,456 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:55:26,027 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 15:55:27,460 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:55:28,028 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:55:30,044 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:55:30,044 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:55:30,045 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:55:30,052 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:55:30,243 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:55:30,243 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:55:30,243 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:55:30,462 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:55:31,464 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:55:33,492 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:55:34,964 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:55:34,964 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:55:34,965 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:55:34,970 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:55:35,468 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:55:37,471 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:55:38,015 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:55:38,016 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:55:38,016 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:55:38,021 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:55:38,472 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:55:39,022 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:55:39,474 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:55:42,256 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:55:42,256 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:55:42,257 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:55:42,262 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:55:42,476 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:55:43,478 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:55:44,263 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:55:45,243 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:55:45,243 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:55:45,243 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:55:49,451 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:55:50,736 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:55:50,736 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:55:50,737 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:55:50,741 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:55:51,484 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:55:53,487 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:55:54,779 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:55:55,891 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:55:55,894 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:55:55,895 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:55:55,899 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:55:56,029 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 15:55:56,489 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:55:57,491 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:55:59,784 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:55:59,784 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:55:59,785 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:55:59,785 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:55:59,790 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:56:00,243 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:56:00,243 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:56:00,243 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:56:00,493 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:56:01,493 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:56:03,505 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:56:03,506 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:56:03,506 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:56:03,511 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:56:04,495 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:56:05,498 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:56:05,545 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:56:08,426 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:56:08,427 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:56:08,427 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:56:08,432 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:56:08,500 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:56:09,502 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:56:11,433 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:56:12,941 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:56:12,942 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:56:12,942 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:56:12,947 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:56:13,504 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:56:15,243 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:56:15,243 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:56:15,244 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:56:15,507 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:56:16,513 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:56:21,513 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:56:22,457 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:56:22,458 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:56:22,458 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:56:22,463 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:56:22,511 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:56:23,513 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:56:26,031 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 15:56:27,032 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:56:27,201 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:56:27,201 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:56:27,202 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:56:27,207 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:56:27,516 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:56:29,519 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:56:30,243 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:56:30,243 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:56:30,244 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:56:30,742 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:56:30,743 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:56:30,743 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:56:30,748 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:56:31,521 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:56:32,786 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:56:33,525 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:56:34,696 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:56:34,697 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:56:34,697 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:56:34,702 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:56:35,526 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:56:37,534 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:56:38,703 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:56:40,910 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:56:40,911 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:56:40,911 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:56:40,917 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:56:41,536 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:56:43,539 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:56:43,918 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:56:45,243 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:56:45,243 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:56:45,244 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:56:46,722 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:56:46,722 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:56:46,723 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:56:46,729 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:56:47,542 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:56:49,543 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:56:49,766 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:56:54,766 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:56:55,185 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:56:55,185 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:56:55,186 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:56:55,191 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:56:55,547 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:56:56,037 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 15:56:57,550 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:57:00,038 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:57:00,243 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:57:00,244 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:57:00,244 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:57:03,502 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:57:03,503 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:57:03,503 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:57:03,508 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:57:03,553 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:57:05,547 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:57:05,555 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:57:06,784 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:57:06,786 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:57:06,786 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:57:06,791 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:57:07,556 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:57:09,559 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:57:10,408 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:57:10,409 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:57:10,409 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:57:10,414 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:57:10,560 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:57:11,414 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:57:11,562 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:57:14,539 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:57:14,540 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:57:14,540 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:57:14,546 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:57:14,564 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:57:15,243 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:57:15,244 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:57:15,244 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:57:15,566 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:57:16,533 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:57:18,081 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:57:18,081 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:57:18,082 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:57:18,087 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:57:18,568 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:57:19,570 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:57:22,127 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:57:24,512 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:57:24,512 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:57:24,513 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:57:24,518 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:57:24,573 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:57:25,576 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:57:26,036 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 15:57:28,037 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:57:30,216 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:57:30,217 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:57:30,217 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:57:30,222 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:57:30,244 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:57:30,244 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:57:30,244 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:57:30,579 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:57:31,581 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:57:33,534 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:57:34,945 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:57:34,946 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:57:34,946 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:57:34,951 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:57:35,585 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:57:37,588 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:57:38,991 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:57:41,528 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:57:41,529 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:57:41,529 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:57:41,535 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:57:41,590 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:57:43,593 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:57:44,535 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:57:45,244 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:57:45,244 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:57:45,244 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:57:45,532 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:57:45,544 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:57:45,545 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:57:45,549 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:57:45,594 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:57:47,597 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:57:49,550 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:57:53,849 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:57:53,850 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:57:53,850 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:57:53,855 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:57:54,602 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:57:54,894 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:57:55,605 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:57:56,040 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 15:57:58,841 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:57:58,842 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:57:58,842 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:57:58,847 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:57:59,607 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:58:00,244 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:58:00,244 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:58:00,244 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:58:00,487 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:58:01,610 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:58:04,391 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:58:04,392 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:58:04,392 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:58:04,397 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:58:04,612 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:58:05,612 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:58:06,434 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:58:08,155 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:58:08,156 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:58:08,156 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:58:08,162 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:58:08,614 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:58:09,617 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:58:11,636 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:58:11,637 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:58:11,638 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:58:11,638 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:58:11,643 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:58:12,618 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:58:13,621 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:58:15,244 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:58:15,244 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:58:15,245 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:58:17,500 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:58:20,183 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:58:20,184 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:58:20,184 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:58:20,189 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:58:20,625 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:58:21,627 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:58:23,227 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:58:23,230 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:58:23,231 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:58:23,231 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:58:23,236 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:58:23,628 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:58:25,631 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:58:26,039 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 15:58:27,639 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:58:27,640 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:58:27,641 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:58:27,646 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:58:28,633 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:58:28,646 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:58:29,636 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:58:30,244 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:58:30,244 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:58:30,244 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:58:31,363 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:58:31,364 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:58:31,364 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:58:31,369 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:58:31,637 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:58:33,641 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:58:34,407 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:58:39,407 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:58:39,633 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:58:39,634 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:58:39,634 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:58:39,639 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:58:39,645 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:58:41,648 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:58:44,640 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:58:45,079 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:58:45,079 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:58:45,080 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:58:45,085 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:58:45,244 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:58:45,244 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:58:45,246 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:58:45,650 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:58:47,653 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:58:48,114 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:58:48,115 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:58:48,115 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:58:48,120 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:58:48,653 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:58:49,656 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:58:50,158 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:58:55,159 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:58:56,041 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 15:58:56,700 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:58:56,701 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:58:56,701 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:58:56,707 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:58:57,663 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:58:57,663 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:59:00,244 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:59:00,244 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:59:00,246 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:59:00,490 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:59:05,043 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:59:05,043 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:59:05,044 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:59:05,048 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:59:05,667 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:59:06,086 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:59:07,671 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:59:09,089 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:59:09,089 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:59:09,090 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:59:09,095 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:59:09,672 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:59:11,095 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:59:11,675 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:59:15,244 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:59:15,245 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:59:15,245 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:59:15,415 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:59:15,526 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:59:15,526 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:59:15,531 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:59:15,677 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:59:16,531 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:59:17,680 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:59:19,365 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:59:19,366 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:59:19,366 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:59:19,371 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:59:19,681 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:59:21,684 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:59:22,372 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:59:23,657 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:59:23,658 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:59:23,658 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:59:23,663 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:59:23,686 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:59:25,689 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:59:26,048 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 15:59:27,432 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:59:27,433 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:59:27,434 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:59:27,434 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:59:27,440 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:59:27,690 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:59:29,693 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:59:30,244 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:59:30,245 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:59:30,245 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:59:32,554 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:59:34,313 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:59:34,314 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:59:34,314 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:59:34,319 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:59:34,697 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:59:35,700 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:59:37,579 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:59:37,580 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:59:37,581 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:59:37,581 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:59:37,586 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:59:37,701 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:59:39,702 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:59:42,032 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:59:42,033 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:59:42,034 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:59:42,038 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:59:42,703 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:59:43,039 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:59:43,706 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:59:45,244 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:59:45,245 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:59:45,245 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:59:48,458 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:59:50,668 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:59:50,669 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:59:50,669 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:59:50,674 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:59:50,711 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:59:51,714 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:59:53,711 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:59:54,611 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:59:54,612 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:59:54,612 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:59:54,617 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:59:54,715 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 15:59:55,718 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 15:59:56,046 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 15:59:58,861 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:59:58,862 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 15:59:58,862 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:59:58,863 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:59:58,869 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:59:59,720 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:00:00,244 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:00:00,245 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:00:00,245 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:00:01,721 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:00:02,359 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:00:02,359 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:00:02,360 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:00:02,365 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:00:02,722 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:00:03,724 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:00:04,402 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:00:05,865 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:00:05,865 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:00:05,866 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:00:05,871 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:00:06,726 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:00:07,729 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:00:09,871 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:00:10,327 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:00:10,328 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:00:10,328 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:00:10,333 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:00:10,730 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:00:11,733 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:00:13,684 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:00:13,685 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:00:13,686 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:00:13,691 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:00:13,734 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:00:15,245 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:00:15,245 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:00:15,245 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:00:15,499 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:00:15,737 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:00:18,654 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:00:18,655 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:00:18,655 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:00:18,661 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:00:18,739 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:00:19,742 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:00:20,698 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:00:21,890 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:00:21,891 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:00:21,891 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:00:21,896 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:00:22,743 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:00:23,746 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:00:25,506 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:00:25,507 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:00:25,507 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:00:25,512 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:00:25,747 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:00:26,051 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 16:00:26,052 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:00:27,750 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:00:28,387 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:00:28,388 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:00:28,388 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:00:28,393 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:00:28,751 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:00:29,753 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:00:30,245 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:00:30,245 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:00:30,245 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:00:31,522 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:00:32,344 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:00:32,344 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:00:32,344 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:00:32,349 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:00:32,755 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:00:33,759 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:00:35,273 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:00:35,274 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:00:35,274 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:00:35,279 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:00:35,760 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:00:37,280 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:00:37,763 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:00:39,078 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:00:39,079 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:00:39,079 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:00:39,084 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:00:39,764 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:00:41,767 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:00:43,085 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:00:43,781 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:00:43,782 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:00:43,782 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:00:43,787 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:00:44,769 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:00:45,245 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:00:45,245 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:00:45,245 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:00:45,771 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:00:48,169 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:00:48,170 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:00:48,170 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:00:48,170 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:00:48,175 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:00:48,773 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:00:49,775 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:00:52,250 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:00:52,251 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:00:52,252 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:00:52,256 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:00:52,778 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:00:53,257 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:00:53,781 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:00:56,050 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 16:00:56,796 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:00:56,796 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:00:56,797 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:00:56,802 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:00:57,785 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:00:57,785 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:00:58,802 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:01:00,245 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:01:00,245 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:01:00,245 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:01:02,619 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:01:02,619 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:01:02,620 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:01:02,626 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:01:02,788 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:01:03,790 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:01:04,662 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:01:05,709 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:01:05,710 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:01:05,710 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:01:05,715 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:01:05,791 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:01:07,793 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:01:09,715 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:01:10,038 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:01:10,039 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:01:10,040 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:01:10,044 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:01:10,795 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:01:11,797 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:01:14,495 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:01:14,496 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:01:14,496 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:01:14,501 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:01:14,799 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:01:15,245 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:01:15,245 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:01:15,245 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:01:15,551 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:01:15,801 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:01:18,178 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:01:18,179 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:01:18,179 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:01:18,184 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:01:18,803 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:01:19,804 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:01:21,223 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:01:21,446 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:01:21,447 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:01:21,447 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:01:21,452 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:01:21,805 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:01:23,808 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:01:25,081 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:01:25,082 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:01:25,082 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:01:25,087 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:01:25,809 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:01:26,055 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 16:01:27,056 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:01:27,812 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:01:28,944 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:01:28,945 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:01:28,945 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:01:28,950 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:01:29,814 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:01:30,245 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:01:30,245 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:01:30,246 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:01:31,817 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:01:32,424 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:01:32,424 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:01:32,425 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:01:32,425 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:01:32,430 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:01:32,818 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:01:33,821 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:01:37,467 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:01:38,062 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:01:38,063 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:01:38,063 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:01:38,068 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:01:38,824 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:01:39,827 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:01:41,790 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:01:41,791 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:01:41,791 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:01:41,797 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:01:41,828 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:01:42,797 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:01:43,831 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:01:45,245 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:01:45,245 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:01:45,246 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:01:48,539 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:01:50,260 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:01:50,261 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:01:50,261 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:01:50,266 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:01:50,836 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:01:51,839 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:01:53,800 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:01:53,801 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:01:53,801 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:01:53,801 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:01:53,806 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:01:53,840 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:01:55,843 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:01:56,054 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 16:01:57,421 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:01:57,422 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:01:57,422 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:01:57,427 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:01:57,844 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:01:59,427 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:01:59,847 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:02:00,245 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:02:00,245 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:02:00,247 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:02:04,510 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:02:05,884 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:02:05,884 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:02:05,885 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:02:05,889 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:02:06,851 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:02:07,854 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:02:08,969 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:02:08,970 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:02:08,970 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:02:08,975 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:02:09,855 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:02:09,976 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:02:11,858 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:02:14,169 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:02:14,170 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:02:14,170 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:02:14,176 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:02:14,860 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:02:15,176 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:02:15,245 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:02:15,246 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:02:15,247 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:02:15,862 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:02:19,579 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:02:19,580 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:02:19,580 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:02:19,585 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:02:19,865 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:02:20,622 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:02:21,868 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:02:22,870 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:02:22,871 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:02:22,871 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:02:22,876 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:02:23,871 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:02:23,871 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:02:25,877 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:02:26,059 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 16:02:26,436 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:02:26,437 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:02:26,437 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:02:26,442 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:02:26,873 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:02:27,875 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:02:29,993 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:02:29,994 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:02:29,994 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:02:29,999 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:02:30,245 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:02:30,246 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:02:30,246 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:02:30,877 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:02:31,480 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:02:31,880 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:02:35,399 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:02:35,399 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:02:35,400 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:02:35,405 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:02:35,884 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:02:37,442 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:02:37,887 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:02:39,763 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:02:39,764 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:02:39,764 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:02:39,769 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:02:39,888 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:02:41,891 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:02:42,770 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:02:45,162 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:02:45,163 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:02:45,163 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:02:45,168 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:02:45,245 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:02:45,246 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:02:45,246 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:02:45,893 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:02:47,894 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:02:48,470 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:02:48,471 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:02:48,471 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:02:48,471 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:02:48,476 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:02:48,895 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:02:49,898 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:02:53,515 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:02:56,058 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 16:02:56,949 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:02:56,950 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:02:56,950 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:02:56,955 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:02:57,905 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:02:57,905 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:02:58,955 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:03:00,246 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:03:00,246 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:03:00,246 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:03:00,330 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:03:00,541 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:03:00,542 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:03:00,546 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:03:00,907 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:03:01,909 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:03:04,352 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:03:04,353 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:03:04,354 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:03:04,354 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:03:04,359 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:03:04,911 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:03:05,914 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:03:07,460 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:03:07,461 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:03:07,462 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:03:07,467 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:03:07,916 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:03:09,468 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:03:09,917 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:03:13,081 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:03:13,082 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:03:13,082 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:03:13,087 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:03:13,919 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:03:15,087 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:03:15,246 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:03:15,246 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:03:15,246 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:03:15,923 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:03:16,323 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:03:16,324 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:03:16,324 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:03:16,329 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:03:16,923 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:03:17,926 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:03:19,333 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:03:19,334 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:03:19,334 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:03:19,339 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:03:19,927 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:03:20,340 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:03:21,930 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:03:24,431 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:03:24,432 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:03:24,432 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:03:24,437 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:03:24,932 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:03:25,438 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:03:25,932 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:03:26,061 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 16:03:28,119 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:03:28,120 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:03:28,120 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:03:28,125 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:03:28,934 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:03:29,937 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:03:30,246 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:03:30,246 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:03:30,246 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:03:30,525 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:03:32,966 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:03:32,967 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:03:32,967 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:03:32,972 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:03:33,943 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:03:33,943 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:03:36,015 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:03:36,226 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:03:36,227 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:03:36,227 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:03:36,232 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:03:36,945 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:03:37,947 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:03:39,870 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:03:39,871 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:03:39,872 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:03:39,878 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:03:39,948 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:03:41,879 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:03:41,951 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:03:43,612 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:03:43,613 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:03:43,613 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:03:43,619 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:03:43,952 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:03:45,246 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:03:45,246 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:03:45,246 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:03:45,955 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:03:47,515 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:03:47,948 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:03:47,948 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:03:47,949 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:03:47,954 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:03:47,956 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:03:49,960 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:03:51,736 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:03:51,737 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:03:51,738 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:03:51,743 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:03:51,962 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:03:52,743 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:03:53,965 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:03:54,767 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:03:54,768 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:03:54,768 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:03:54,773 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:03:54,966 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:03:55,968 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:03:56,063 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 16:03:58,064 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:04:00,246 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:04:00,246 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:04:00,247 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:04:01,041 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:04:01,042 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:04:01,042 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:04:01,047 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:04:01,972 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:04:03,086 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:04:03,973 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:04:04,387 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:04:04,388 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:04:04,388 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:04:04,393 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:04:04,974 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:04:05,976 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:04:08,394 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:04:09,738 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:04:09,738 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:04:09,739 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:04:09,744 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:04:09,979 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:04:11,982 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:04:13,470 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:04:13,471 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:04:13,471 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:04:13,472 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:04:13,477 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:04:13,983 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:04:15,246 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:04:15,246 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:04:15,247 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:04:15,986 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:04:16,576 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:04:16,577 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:04:16,577 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:04:16,582 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:04:16,987 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:04:17,989 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:04:18,618 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:04:20,538 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:04:20,539 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:04:20,539 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:04:20,544 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:04:20,991 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:04:21,994 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:04:23,801 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:04:23,802 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:04:23,802 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:04:23,802 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:04:23,807 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:04:23,995 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:04:25,997 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:04:26,064 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 16:04:29,065 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:04:29,521 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:04:29,522 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:04:29,522 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:04:29,527 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:04:30,000 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:04:30,246 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:04:30,247 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:04:30,247 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:04:32,003 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:04:33,583 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:04:33,583 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:04:33,584 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:04:33,589 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:04:34,005 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:04:34,626 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:04:36,008 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:04:36,936 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:04:36,937 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:04:36,937 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:04:36,942 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:04:37,009 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:04:38,011 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:04:39,943 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:04:40,980 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:04:40,980 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:04:40,981 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:04:40,986 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:04:41,013 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:04:42,013 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:04:44,986 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:04:45,246 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:04:45,246 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:04:45,247 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:04:47,143 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:04:47,143 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:04:47,143 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:04:47,148 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:04:48,017 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:04:50,020 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:04:50,186 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:04:51,071 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:04:51,072 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:04:51,072 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:04:51,077 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:04:52,024 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:04:52,024 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:04:55,777 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:04:55,778 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:04:55,778 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:04:55,778 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:04:55,784 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:04:56,026 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:04:56,066 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 16:04:58,029 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:04:59,769 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:04:59,770 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:04:59,770 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:04:59,776 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:05:00,030 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:05:00,246 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:05:00,247 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:05:00,247 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:05:01,497 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:05:02,037 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:05:03,039 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:05:03,040 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:05:03,040 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:05:03,045 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:05:04,040 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:05:04,040 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:05:06,173 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:05:06,174 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:05:06,174 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:05:06,179 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:05:07,042 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:05:07,180 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:05:08,044 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:05:09,802 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:05:09,803 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:05:09,803 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:05:09,808 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:05:10,046 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:05:12,049 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:05:12,808 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:05:13,513 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:05:13,514 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:05:13,514 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:05:13,519 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:05:14,050 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:05:15,246 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:05:15,247 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:05:15,247 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:05:16,053 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:05:18,493 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:05:22,057 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:05:22,058 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:05:22,058 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:05:22,063 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:05:23,057 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:05:24,057 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:05:24,102 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:05:26,068 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 16:05:27,396 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:05:27,397 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:05:27,397 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:05:27,402 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:05:28,059 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:05:29,402 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:05:30,063 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:05:30,247 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:05:30,247 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:05:30,247 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:05:30,341 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:05:30,549 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:05:30,550 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:05:30,555 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:05:31,064 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:05:32,066 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:05:33,981 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:05:33,982 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:05:33,982 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:05:33,987 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:05:34,068 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:05:34,988 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:05:36,071 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:05:37,089 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:05:37,089 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:05:37,090 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:05:37,095 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:05:38,075 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:05:38,075 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:05:40,095 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:05:41,699 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:05:41,699 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:05:41,700 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:05:41,704 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:05:42,077 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:05:44,080 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:05:45,002 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:05:45,003 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:05:45,003 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:05:45,008 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:05:45,080 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:05:45,247 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:05:45,247 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:05:45,248 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:05:45,490 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:05:46,083 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:05:49,950 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:05:49,950 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:05:49,950 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:05:49,955 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:05:50,085 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:05:50,994 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:05:52,089 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:05:55,525 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:05:55,526 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:05:55,526 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:05:55,531 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:05:56,074 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 16:05:56,075 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:05:56,091 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:05:58,094 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:05:58,668 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:05:58,669 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:05:58,669 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:05:58,674 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:05:59,095 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:06:00,097 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:06:00,247 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:06:00,247 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:06:00,248 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:06:01,451 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:06:02,171 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:06:02,172 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:06:02,172 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:06:02,177 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:06:03,099 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:06:04,102 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:06:05,951 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:06:05,952 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:06:05,952 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:06:05,957 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:06:06,103 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:06:06,958 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:06:08,107 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:06:11,149 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:06:11,150 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:06:11,151 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:06:11,155 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:06:12,111 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:06:12,111 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:06:12,156 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:06:15,247 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:06:15,247 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:06:15,247 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:06:15,522 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:06:15,556 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:06:15,557 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:06:15,561 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:06:16,113 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:06:17,562 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:06:18,116 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:06:19,620 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:06:19,620 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:06:19,621 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:06:19,625 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:06:20,117 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:06:22,120 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:06:22,662 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:06:23,260 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:06:23,261 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:06:23,261 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:06:23,266 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:06:24,122 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:06:26,073 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 16:06:26,125 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:06:27,043 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:06:27,044 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:06:27,044 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:06:27,049 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:06:27,125 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:06:28,050 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:06:28,126 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:06:30,247 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:06:30,247 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:06:30,247 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:06:30,697 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:06:30,698 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:06:30,698 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:06:30,703 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:06:31,128 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:06:32,131 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:06:33,742 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:06:35,593 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:06:35,594 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:06:35,594 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:06:35,599 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:06:36,134 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:06:38,137 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:06:39,308 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:06:39,309 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:06:39,309 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:06:39,310 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:06:39,314 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:06:40,138 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:06:42,141 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:06:42,926 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:06:42,927 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:06:42,927 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:06:42,932 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:06:43,142 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:06:44,144 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:06:44,932 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:06:45,247 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:06:45,247 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:06:45,248 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:06:48,208 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:06:48,209 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:06:48,209 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:06:48,214 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:06:49,147 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:06:50,148 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:06:50,250 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:06:51,680 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:06:51,681 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:06:51,681 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:06:51,686 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:06:52,151 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:06:54,154 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:06:55,686 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:06:55,746 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:06:55,747 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:06:55,747 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:06:55,752 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:06:56,078 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 16:06:56,155 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:06:58,158 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:06:59,427 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:06:59,427 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:06:59,428 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:06:59,433 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:07:00,159 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:07:00,247 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:07:00,248 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:07:00,248 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:07:01,556 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:07:02,162 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:07:03,432 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:07:03,433 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:07:03,433 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:07:03,438 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:07:04,163 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:07:06,166 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:07:07,474 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:07:09,005 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:07:09,006 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:07:09,006 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:07:09,011 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:07:09,169 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:07:10,172 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:07:12,235 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:07:12,236 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:07:12,236 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:07:12,242 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:07:13,173 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:07:13,243 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:07:14,176 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:07:15,247 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:07:15,247 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:07:15,249 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:07:16,385 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:07:16,385 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:07:16,385 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:07:16,390 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:07:17,177 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:07:18,180 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:07:18,426 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:07:21,093 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:07:21,094 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:07:21,094 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:07:21,100 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:07:21,182 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:07:22,184 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:07:24,100 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:07:26,077 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 16:07:26,378 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:07:26,379 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:07:26,379 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:07:26,384 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:07:27,187 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:07:28,190 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:07:29,384 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:07:30,247 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:07:30,248 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:07:30,249 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:07:30,743 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:07:30,744 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:07:30,744 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:07:30,749 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:07:31,192 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:07:32,194 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:07:34,786 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:07:35,959 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:07:35,960 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:07:35,960 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:07:35,965 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:07:36,198 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:07:38,201 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:07:38,974 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:07:38,975 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:07:38,975 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:07:38,980 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:07:39,202 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:07:39,980 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:07:40,202 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:07:43,325 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:07:43,326 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:07:43,326 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:07:43,331 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:07:44,205 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:07:45,247 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:07:45,248 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:07:45,248 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:07:45,462 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:07:46,208 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:07:46,640 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:07:46,641 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:07:46,641 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:07:46,646 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:07:47,208 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:07:48,211 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:07:50,682 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:07:52,652 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:07:52,652 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:07:52,653 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:07:52,658 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:07:53,215 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:07:54,217 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:07:56,085 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 16:07:56,086 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:07:59,294 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:07:59,295 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:07:59,295 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:07:59,300 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:08:00,222 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:08:00,222 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:08:00,248 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:08:00,248 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:08:00,249 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:08:01,490 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:08:02,657 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:08:02,658 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:08:02,658 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:08:02,663 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:08:03,224 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:08:04,226 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:08:06,397 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:08:06,398 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:08:06,398 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:08:06,403 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:08:07,229 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:08:07,404 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:08:08,231 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:08:10,465 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:08:10,466 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:08:10,466 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:08:10,471 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:08:11,233 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:08:12,235 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:08:12,471 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:08:15,248 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:08:15,248 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:08:15,248 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:08:17,255 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:08:17,256 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:08:17,256 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:08:17,261 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:08:18,241 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:08:18,241 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:08:18,298 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:08:20,538 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:08:20,539 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:08:20,539 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:08:20,544 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:08:21,242 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:08:22,245 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:08:23,544 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:08:24,732 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:08:24,733 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:08:24,733 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:08:24,738 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:08:25,247 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:08:26,081 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 16:08:26,249 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:08:29,083 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:08:30,248 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:08:30,248 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:08:30,248 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:08:30,266 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:08:30,486 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:08:30,486 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:08:30,491 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:08:31,253 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:08:32,255 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:08:34,320 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:08:34,321 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:08:34,321 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:08:34,322 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:08:34,326 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:08:35,258 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:08:36,260 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:08:38,421 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:08:38,422 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:08:38,422 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:08:38,427 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:08:39,262 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:08:39,427 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:08:40,265 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:08:44,428 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:08:44,586 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:08:44,586 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:08:44,587 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:08:44,591 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:08:45,248 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:08:45,248 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:08:45,249 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:08:45,268 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:08:46,270 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:08:48,811 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:08:48,812 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:08:48,812 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:08:48,817 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:08:49,272 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:08:49,854 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:08:50,274 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:08:52,855 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:08:52,855 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:08:52,856 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:08:52,860 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:08:53,277 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:08:54,280 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:08:54,861 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:08:56,086 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 16:08:56,776 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:08:56,776 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:08:56,776 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:08:56,782 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:08:57,281 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:08:58,284 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:09:00,248 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:09:00,248 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:09:00,248 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:09:00,497 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:09:01,313 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:09:01,314 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:09:01,314 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:09:01,319 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:09:02,288 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:09:02,288 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:09:04,564 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:09:04,564 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:09:04,565 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:09:04,569 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:09:05,290 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:09:05,570 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:09:06,292 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:09:08,179 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:09:08,179 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:09:08,180 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:09:08,184 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:09:08,294 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:09:10,297 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:09:11,185 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:09:11,418 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:09:11,419 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:09:11,419 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:09:11,424 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:09:12,298 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:09:14,301 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:09:15,248 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:09:15,248 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:09:15,248 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:09:15,921 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:09:15,921 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:09:15,922 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:09:15,926 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:09:16,302 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:09:16,966 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:09:18,305 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:09:19,644 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:09:19,644 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:09:19,645 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:09:19,650 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:09:20,306 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:09:22,309 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:09:22,650 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:09:23,315 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:09:23,315 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:09:23,316 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:09:23,320 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:09:24,312 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:09:24,312 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:09:26,086 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 16:09:26,592 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:09:26,593 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:09:26,593 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:09:26,598 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:09:27,314 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:09:28,317 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:09:28,598 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:09:30,248 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:09:30,248 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:09:30,248 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:09:30,504 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:09:30,504 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:09:30,504 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:09:30,509 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:09:31,319 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:09:32,321 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:09:34,546 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:09:35,590 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:09:35,590 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:09:35,591 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:09:35,595 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:09:36,325 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:09:38,328 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:09:38,682 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:09:38,683 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:09:38,683 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:09:38,688 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:09:39,329 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:09:39,689 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:09:40,331 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:09:44,689 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:09:45,248 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:09:45,248 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:09:45,249 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:09:46,403 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:09:46,404 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:09:46,404 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:09:46,409 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:09:47,335 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:09:48,338 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:09:49,644 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:09:49,645 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:09:49,645 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:09:49,650 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:09:50,339 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:09:50,650 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:09:52,344 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:09:53,415 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:09:53,416 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:09:53,416 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:09:53,421 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:09:54,347 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:09:54,347 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:09:56,088 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 16:09:56,089 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:09:56,738 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:09:56,739 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:09:56,739 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:09:56,745 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:09:57,349 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:09:58,351 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:09:59,991 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:09:59,992 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:09:59,992 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:09:59,997 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:10:00,248 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:10:00,249 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:10:00,250 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:10:00,352 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:10:01,510 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:10:02,356 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:10:04,968 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:10:04,968 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:10:04,968 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:10:04,973 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:10:05,357 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:10:06,360 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:10:07,010 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:10:09,067 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:10:09,068 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:10:09,068 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:10:09,073 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:10:09,362 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:10:10,364 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:10:12,074 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:10:12,598 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:10:12,599 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:10:12,599 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:10:12,604 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:10:13,366 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:10:14,369 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:10:15,248 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:10:15,249 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:10:15,249 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:10:15,831 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:10:15,831 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:10:15,831 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:10:15,836 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:10:16,370 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:10:17,874 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:10:18,371 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:10:19,746 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:10:19,747 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:10:19,747 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:10:19,752 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:10:20,372 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:10:22,375 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:10:23,752 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:10:24,349 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:10:24,350 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:10:24,350 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:10:24,355 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:10:24,376 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:10:26,090 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 16:10:26,379 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:10:29,091 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:10:30,248 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:10:30,249 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:10:30,249 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:10:32,853 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:10:32,854 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:10:32,854 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:10:32,858 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:10:33,384 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:10:34,385 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:10:34,898 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:10:37,178 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:10:37,179 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:10:37,179 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:10:37,184 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:10:37,387 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:10:38,390 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:10:40,185 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:10:41,869 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:10:41,870 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:10:41,870 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:10:41,875 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:10:42,392 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:10:44,396 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:10:45,248 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:10:45,249 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:10:45,249 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:10:45,470 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:10:45,502 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:10:45,503 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:10:45,503 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:10:45,507 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:10:46,397 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:10:48,400 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:10:50,144 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:10:50,145 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:10:50,145 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:10:50,150 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:10:50,401 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:10:51,150 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:10:52,405 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:10:56,092 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 16:10:56,723 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:10:56,724 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:10:56,725 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:10:56,725 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:10:56,729 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:10:57,408 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:10:58,410 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:11:00,249 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:11:00,249 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:11:00,249 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:11:00,864 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:11:00,865 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:11:00,865 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:11:00,870 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:11:01,412 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:11:01,906 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:11:02,415 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:11:04,416 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:11:04,416 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:11:04,417 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:11:04,422 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:11:05,416 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:11:06,419 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:11:07,423 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:11:08,219 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:11:08,219 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:11:08,220 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:11:08,224 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:11:08,420 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:11:10,423 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:11:13,225 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:11:13,947 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:11:13,948 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:11:13,948 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:11:13,953 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:11:14,426 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:11:15,249 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:11:15,249 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:11:15,249 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:11:16,429 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:11:18,472 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:11:22,420 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:11:22,420 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:11:22,420 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:11:22,425 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:11:22,432 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:11:24,435 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:11:24,461 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:11:25,160 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:11:25,161 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:11:25,161 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:11:25,166 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:11:25,436 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:11:26,097 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 16:11:26,438 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:11:29,542 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:11:29,543 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:11:29,543 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:11:29,543 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:11:29,548 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:11:30,249 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:11:30,249 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:11:30,249 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:11:30,443 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:11:30,443 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:11:33,262 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:11:33,262 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:11:33,263 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:11:33,267 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:11:33,445 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:11:34,449 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:11:35,305 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:11:40,306 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:11:41,854 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:11:41,855 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:11:41,855 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:11:41,861 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:11:42,453 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:11:44,456 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:11:45,249 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:11:45,249 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:11:45,249 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:11:45,477 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:11:45,510 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:11:45,511 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:11:45,511 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:11:45,516 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:11:46,459 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:11:46,459 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:11:49,847 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:11:49,847 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:11:49,848 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:11:49,852 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:11:50,461 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:11:50,853 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:11:52,466 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:11:53,370 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:11:53,371 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:11:53,371 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:11:53,376 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:11:53,466 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:11:54,469 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:11:56,096 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 16:11:56,097 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:11:57,984 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:11:57,985 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:11:57,985 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:11:57,990 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:11:58,471 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:12:00,249 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:12:00,249 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:12:00,249 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:12:00,474 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:12:01,498 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:12:02,908 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:12:02,909 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:12:02,909 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:12:02,914 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:12:03,476 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:12:04,478 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:12:06,950 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:12:07,182 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:12:07,182 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:12:07,183 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:12:07,187 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:12:07,481 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:12:08,483 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:12:12,188 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:12:12,393 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:12:12,393 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:12:12,394 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:12:12,398 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:12:12,485 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:12:14,489 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:12:15,249 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:12:15,250 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:12:15,250 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:12:16,083 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:12:16,084 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:12:16,084 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:12:16,089 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:12:16,490 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:12:18,125 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:12:18,493 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:12:19,328 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:12:19,329 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:12:19,329 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:12:19,334 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:12:19,493 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:12:20,496 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:12:22,788 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:12:22,789 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:12:22,789 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:12:22,794 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:12:23,497 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:12:23,795 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:12:24,500 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:12:26,101 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 16:12:26,103 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:12:26,104 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:12:26,104 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:12:26,109 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:12:26,501 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:12:28,504 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:12:29,109 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:12:30,249 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:12:30,250 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:12:30,250 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:12:32,726 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:12:32,726 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:12:32,726 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:12:32,731 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:12:33,507 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:12:34,509 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:12:34,769 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:12:36,682 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:12:36,683 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:12:36,683 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:12:36,688 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:12:37,511 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:12:38,513 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:12:40,335 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:12:40,336 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:12:40,336 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:12:40,336 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:12:40,341 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:12:40,514 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:12:42,517 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:12:45,249 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:12:45,250 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:12:45,250 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:12:45,506 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:12:46,909 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:12:46,910 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:12:46,910 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:12:46,916 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:12:47,520 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:12:48,522 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:12:50,910 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:12:50,911 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:12:50,911 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:12:50,911 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:12:50,916 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:12:51,525 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:12:52,526 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:12:55,911 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:12:55,912 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:12:55,912 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:12:55,912 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:12:55,916 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:12:56,100 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 16:12:56,528 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:12:58,531 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:12:59,872 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:12:59,873 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:12:59,873 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:12:59,878 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:13:00,249 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:13:00,250 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:13:00,250 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:13:00,532 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:13:01,525 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:13:02,535 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:13:04,280 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:13:04,281 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:13:04,281 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:13:04,286 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:13:04,536 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:13:06,539 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:13:07,321 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:13:12,322 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:13:13,441 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:13:13,442 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:13:13,442 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:13:13,447 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:13:13,543 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:13:14,546 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:13:15,250 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:13:15,250 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:13:15,250 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:13:17,308 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:13:17,309 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:13:17,309 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:13:17,314 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:13:17,349 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:13:17,548 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:13:18,550 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:13:21,845 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:13:21,846 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:13:21,846 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:13:21,851 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:13:22,552 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:13:22,852 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:13:24,555 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:13:24,933 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:13:24,934 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:13:24,934 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:13:24,939 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:13:25,556 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:13:26,105 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 16:13:26,558 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:13:28,106 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:13:30,250 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:13:30,250 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:13:30,250 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:13:33,406 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:13:33,407 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:13:33,407 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:13:33,407 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:13:33,412 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:13:33,562 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:13:34,566 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:13:38,256 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:13:38,256 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:13:38,256 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:13:38,261 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:13:38,568 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:13:39,262 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:13:40,571 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:13:43,674 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:13:43,675 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:13:43,675 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:13:43,680 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:13:44,575 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:13:44,575 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:13:44,681 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:13:45,250 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:13:45,250 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:13:45,250 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:13:49,626 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:13:49,626 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:13:49,626 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:13:49,631 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:13:50,580 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:13:50,580 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:13:50,669 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:13:52,896 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:13:52,897 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:13:52,897 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:13:52,902 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:13:53,583 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:13:54,585 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:13:55,903 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:13:56,104 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 16:13:56,790 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:13:56,790 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:13:56,791 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:13:56,795 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:13:57,587 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:13:58,589 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:14:00,250 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:14:00,250 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:14:00,250 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:14:01,400 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:14:01,401 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:14:01,401 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:14:01,401 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:14:01,405 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:14:01,591 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:14:02,593 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:14:05,462 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:14:05,463 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:14:05,463 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:14:05,468 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:14:05,595 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:14:06,469 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:14:06,596 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:14:09,689 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:14:09,689 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:14:09,690 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:14:09,694 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:14:10,600 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:14:10,600 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:14:11,695 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:14:12,859 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:14:12,860 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:14:12,861 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:14:12,866 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:14:13,602 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:14:14,604 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:14:15,250 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:14:15,250 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:14:15,250 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:14:16,907 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:14:16,907 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:14:16,908 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:14:16,908 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:14:16,913 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:14:17,606 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:14:18,608 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:14:20,936 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:14:20,937 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:14:20,937 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:14:20,942 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:14:21,610 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:14:21,942 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:14:22,612 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:14:24,198 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:14:24,199 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:14:24,199 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:14:24,204 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:14:24,613 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:14:26,109 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 16:14:26,616 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:14:27,110 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:14:27,647 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:14:27,647 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:14:27,648 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:14:27,653 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:14:28,619 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:14:28,620 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:14:30,250 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:14:30,250 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:14:30,250 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:14:31,991 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:14:31,991 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:14:31,992 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:14:31,997 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:14:32,622 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:14:33,033 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:14:34,626 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:14:37,715 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:14:37,716 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:14:37,716 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:14:37,721 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:14:38,630 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:14:38,630 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:14:38,721 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:14:43,722 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:14:45,250 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:14:45,250 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:14:45,251 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:14:45,545 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:14:45,552 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:14:45,552 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:14:45,557 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:14:45,634 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:14:46,637 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:14:49,557 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:14:53,877 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:14:53,878 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:14:53,878 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:14:53,883 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:14:54,642 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:14:54,921 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:14:56,108 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 16:14:56,645 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:14:57,527 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:14:57,528 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:14:57,528 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:14:57,533 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:14:57,646 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:14:58,648 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:15:00,250 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:15:00,251 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:15:00,252 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:15:00,505 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:15:00,811 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:15:00,811 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:15:00,811 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:15:00,816 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:15:01,650 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:15:02,652 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:15:05,415 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:15:05,416 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:15:05,416 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:15:05,421 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:15:05,654 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:15:06,422 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:15:06,657 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:15:11,422 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:15:11,487 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:15:11,488 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:15:11,488 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:15:11,493 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:15:11,660 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:15:12,662 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:15:14,805 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:15:14,806 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:15:14,806 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:15:14,811 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:15:15,250 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:15:15,250 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:15:15,252 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:15:15,664 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:15:16,504 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:15:16,664 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:15:18,912 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:15:18,912 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:15:18,913 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:15:18,917 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:15:19,666 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:15:20,668 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:15:21,955 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:15:23,299 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:15:23,300 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:15:23,300 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:15:23,304 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:15:23,670 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:15:24,672 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:15:26,110 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 16:15:27,112 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:15:27,898 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:15:27,899 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:15:27,899 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:15:27,904 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:15:28,675 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:15:30,250 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:15:30,251 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:15:30,251 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:15:30,678 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:15:32,656 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:15:36,223 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:15:36,223 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:15:36,223 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:15:36,228 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:15:36,683 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:15:38,265 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:15:38,685 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:15:39,732 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:15:39,733 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:15:39,733 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:15:39,738 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:15:40,689 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:15:40,689 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:15:43,025 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:15:43,025 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:15:43,026 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:15:43,031 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:15:43,690 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:15:44,031 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:15:44,693 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:15:45,251 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:15:45,251 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:15:45,251 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:15:48,019 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:15:48,022 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:15:48,022 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:15:48,027 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:15:48,695 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:15:49,061 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:15:50,698 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:15:52,213 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:15:52,214 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:15:52,214 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:15:52,219 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:15:52,700 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:15:54,219 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:15:54,701 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:15:56,113 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 16:16:00,114 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:16:00,251 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:16:00,251 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:16:00,252 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:16:00,620 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:16:00,621 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:16:00,621 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:16:00,626 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:16:00,704 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:16:02,707 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:16:04,543 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:16:04,544 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:16:04,544 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:16:04,548 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:16:04,708 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:16:05,549 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:16:06,711 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:16:08,054 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:16:08,055 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:16:08,055 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:16:08,060 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:16:08,713 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:16:10,714 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:16:11,060 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:16:15,251 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:16:15,251 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:16:15,251 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:16:15,336 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:16:15,479 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:16:15,479 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:16:15,484 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:16:15,717 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:16:16,484 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:16:16,719 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:16:18,838 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:16:18,839 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:16:18,839 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:16:18,844 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:16:19,721 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:16:20,723 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:16:21,844 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:16:22,487 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:16:22,488 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:16:22,488 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:16:22,493 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:16:22,725 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:16:24,728 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:16:26,115 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 16:16:26,600 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:16:26,600 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:16:26,600 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:16:26,605 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:16:26,729 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:16:27,606 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:16:28,732 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:16:30,251 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:16:30,251 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:16:30,251 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:16:30,558 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:16:30,559 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:16:30,559 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:16:30,564 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:16:30,733 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:16:32,734 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:16:33,601 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:16:34,520 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:16:34,520 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:16:34,521 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:16:34,526 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:16:34,737 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:16:36,740 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:16:39,150 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:16:39,151 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:16:39,151 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:16:39,151 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:16:39,156 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:16:39,741 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:16:40,744 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:16:42,751 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:16:42,752 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:16:42,752 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:16:42,757 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:16:43,746 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:16:44,746 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:16:44,757 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:16:45,251 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:16:45,251 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:16:45,252 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:16:46,845 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:16:46,845 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:16:46,846 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:16:46,850 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:16:47,748 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:16:48,750 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:16:49,889 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:16:50,285 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:16:50,286 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:16:50,286 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:16:50,291 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:16:50,752 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:16:52,756 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:16:54,276 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:16:54,276 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:16:54,277 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:16:54,286 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:16:54,757 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:16:55,286 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:16:56,122 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 16:16:56,760 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:16:57,773 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:16:57,773 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:16:57,774 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:16:57,779 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:16:58,768 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:16:58,769 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:17:00,251 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:17:00,251 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:17:00,251 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:17:00,554 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:17:02,059 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:17:02,059 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:17:02,059 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:17:02,064 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:17:02,771 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:17:04,774 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:17:06,101 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:17:06,155 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:17:06,155 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:17:06,156 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:17:06,160 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:17:06,775 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:17:08,778 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:17:09,800 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:17:09,801 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:17:09,801 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:17:09,806 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:17:10,781 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:17:10,781 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:17:11,806 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:17:14,981 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:17:14,982 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:17:14,982 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:17:14,987 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:17:15,251 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:17:15,251 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:17:15,251 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:17:15,784 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:17:16,786 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:17:17,593 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:17:19,663 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:17:19,664 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:17:19,664 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:17:19,668 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:17:19,788 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:17:20,790 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:17:22,705 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:17:23,184 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:17:23,185 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:17:23,185 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:17:23,190 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:17:23,792 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:17:24,794 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:17:26,119 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 16:17:28,120 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:17:28,194 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:17:28,194 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:17:28,195 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:17:28,200 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:17:28,797 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:17:30,251 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:17:30,252 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:17:30,252 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:17:30,800 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:17:32,946 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:17:32,947 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:17:32,947 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:17:32,952 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:17:33,802 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:17:33,989 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:17:34,805 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:17:37,817 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:17:37,818 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:17:37,818 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:17:37,823 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:17:38,809 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:17:38,810 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:17:39,824 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:17:42,554 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:17:42,554 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:17:42,555 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:17:42,559 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:17:42,812 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:17:44,815 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:17:45,251 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:17:45,252 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:17:45,252 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:17:45,486 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:17:46,670 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:17:46,670 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:17:46,671 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:17:46,675 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:17:46,816 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:17:48,820 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:17:50,713 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:17:50,827 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:17:50,827 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:17:50,827 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:17:50,832 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:17:51,822 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:17:52,825 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:17:55,833 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:17:56,124 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 16:17:59,159 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:17:59,160 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:17:59,160 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:17:59,165 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:17:59,829 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:18:00,251 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:18:00,252 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:18:00,252 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:18:00,831 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:18:01,533 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:18:03,421 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:18:03,422 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:18:03,422 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:18:03,427 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:18:03,833 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:18:04,836 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:18:07,465 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:18:08,861 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:18:08,862 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:18:08,862 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:18:08,867 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:18:09,839 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:18:10,841 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:18:12,290 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:18:12,290 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:18:12,290 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:18:12,295 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:18:12,843 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:18:13,296 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:18:14,847 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:18:15,252 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:18:15,252 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:18:15,253 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:18:15,703 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:18:15,704 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:18:15,704 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:18:15,709 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:18:15,847 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:18:16,849 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:18:18,745 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:18:19,321 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:18:19,321 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:18:19,322 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:18:19,327 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:18:19,851 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:18:20,854 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:18:24,327 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:18:26,123 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 16:18:26,151 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:18:26,151 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:18:26,152 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:18:26,156 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:18:26,857 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:18:28,860 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:18:29,884 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:18:29,884 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:18:29,884 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:18:29,885 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:18:29,889 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:18:30,252 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:18:30,252 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:18:30,253 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:18:30,863 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:18:30,864 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:18:33,952 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:18:33,953 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:18:33,953 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:18:33,959 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:18:34,869 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:18:34,869 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:18:34,993 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:18:38,225 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:18:38,226 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:18:38,226 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:18:38,231 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:18:38,871 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:18:40,232 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:18:40,874 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:18:43,248 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:18:43,249 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:18:43,249 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:18:43,253 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:18:43,876 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:18:44,878 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:18:45,252 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:18:45,252 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:18:45,253 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:18:45,536 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:18:46,794 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:18:46,795 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:18:46,795 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:18:46,800 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:18:46,879 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:18:48,882 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:18:50,837 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:18:52,996 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:18:52,997 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:18:52,997 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:18:53,002 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:18:53,886 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:18:54,889 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:18:56,003 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:18:56,128 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 16:18:56,849 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:18:56,850 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:18:56,850 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:18:56,855 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:18:56,890 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:18:58,893 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:19:00,159 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:19:00,160 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:19:00,160 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:19:00,165 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:19:00,252 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:19:00,252 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:19:00,253 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:19:00,894 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:19:01,475 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:19:02,897 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:19:04,256 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:19:04,256 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:19:04,256 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:19:04,261 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:19:04,898 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:19:06,899 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:19:07,298 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:19:12,298 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:19:12,490 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:19:12,491 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:19:12,491 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:19:12,496 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:19:12,903 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:19:14,906 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:19:15,252 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:19:15,252 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:19:15,252 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:19:16,167 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:19:16,168 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:19:16,168 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:19:16,173 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:19:16,907 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:19:18,209 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:19:18,910 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:19:19,055 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:19:19,056 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:19:19,056 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:19:19,061 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:19:19,911 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:19:20,913 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:19:23,451 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:19:23,452 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:19:23,452 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:19:23,452 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:19:23,457 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:19:23,915 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:19:24,918 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:19:26,127 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 16:19:26,925 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:19:26,926 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:19:26,926 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:19:26,932 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:19:27,920 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:19:28,921 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:19:28,932 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:19:30,252 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:19:30,252 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:19:30,252 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:19:30,422 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:19:30,543 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:19:30,544 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:19:30,548 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:19:30,922 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:19:32,925 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:19:33,932 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:19:33,933 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:19:33,933 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:19:33,934 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:19:33,938 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:19:34,929 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:19:34,930 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:19:38,939 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:19:42,562 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:19:42,562 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:19:42,563 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:19:42,568 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:19:42,934 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:19:44,568 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:19:44,937 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:19:45,252 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:19:45,252 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:19:45,253 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:19:45,544 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:19:45,547 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:19:45,548 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:19:45,553 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:19:45,937 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:19:46,940 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:19:49,882 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:19:49,882 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:19:49,882 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:19:49,883 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:19:49,887 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:19:49,941 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:19:50,944 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:19:53,593 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:19:53,594 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:19:53,594 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:19:53,599 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:19:53,947 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:19:54,949 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:19:55,600 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:19:56,129 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 16:19:56,863 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:19:56,864 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:19:56,865 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:19:56,869 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:19:56,950 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:19:58,953 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:20:00,252 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:20:00,253 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:20:00,253 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:20:00,656 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:20:00,657 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:20:00,657 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:20:00,657 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:20:00,662 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:20:00,955 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:20:02,958 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:20:04,028 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:20:04,029 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:20:04,030 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:20:04,035 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:20:04,961 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:20:04,961 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:20:06,035 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:20:08,389 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:20:08,390 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:20:08,390 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:20:08,395 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:20:08,963 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:20:10,966 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:20:11,396 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:20:12,172 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:20:12,172 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:20:12,173 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:20:12,178 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:20:12,968 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:20:14,970 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:20:15,252 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:20:15,253 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:20:15,253 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:20:16,334 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:20:16,335 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:20:16,335 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:20:16,340 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:20:16,972 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:20:17,378 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:20:18,975 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:20:22,379 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:20:22,919 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:20:22,920 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:20:22,920 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:20:22,925 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:20:22,977 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:20:24,980 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:20:26,131 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 16:20:26,657 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:20:26,658 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:20:26,658 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:20:26,664 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:20:26,981 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:20:27,665 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:20:28,989 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:20:30,252 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:20:30,253 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:20:30,253 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:20:33,544 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:20:36,116 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:20:36,116 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:20:36,116 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:20:36,121 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:20:36,997 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:20:36,997 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:20:39,159 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:20:39,222 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:20:39,222 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:20:39,223 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:20:39,227 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:20:39,999 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:20:41,001 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:20:42,479 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:20:42,480 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:20:42,480 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:20:42,485 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:20:43,003 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:20:44,486 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:20:45,006 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:20:45,253 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:20:45,253 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:20:45,253 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:20:45,748 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:20:45,748 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:20:45,749 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:20:45,753 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:20:46,006 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:20:47,009 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:20:49,791 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:20:50,094 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:20:50,094 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:20:50,095 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:20:50,100 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:20:51,013 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:20:51,013 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:20:53,273 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:20:53,274 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:20:53,274 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:20:53,279 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:20:54,016 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:20:55,016 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:20:55,280 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:20:56,133 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 16:20:58,018 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:20:58,018 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:20:58,018 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:20:58,023 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:20:59,021 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:20:59,021 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:21:00,252 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:21:00,253 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:21:00,254 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:21:00,486 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:21:01,876 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:21:01,877 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:21:01,877 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:21:01,882 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:21:02,023 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:21:03,025 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:21:05,918 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:21:10,244 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:21:10,244 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:21:10,245 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:21:10,249 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:21:11,030 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:21:11,250 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:21:13,033 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:21:13,720 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:21:13,721 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:21:13,721 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:21:13,726 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:21:14,034 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:21:15,036 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:21:15,253 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:21:15,253 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:21:15,253 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:21:16,539 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:21:19,849 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:21:19,853 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:21:19,853 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:21:19,858 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:21:20,039 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:21:21,041 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:21:21,890 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:21:23,364 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:21:23,365 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:21:23,365 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:21:23,373 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:21:24,044 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:21:25,046 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:21:26,135 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 16:21:27,136 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:21:28,536 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:21:28,536 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:21:28,536 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:21:28,542 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:21:29,048 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:21:30,253 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:21:30,253 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:21:30,253 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:21:31,052 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:21:31,832 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:21:31,833 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:21:31,833 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:21:31,838 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:21:32,052 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:21:32,838 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:21:33,053 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:21:37,839 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:21:38,495 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:21:38,496 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:21:38,496 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:21:38,501 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:21:39,057 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:21:41,060 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:21:41,955 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:21:41,956 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:21:41,956 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:21:41,961 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:21:42,061 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:21:42,961 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:21:43,063 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:21:45,253 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:21:45,253 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:21:45,253 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:21:45,462 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:21:45,463 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:21:45,463 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:21:45,468 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:21:46,065 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:21:47,068 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:21:48,468 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:21:53,469 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:21:53,924 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:21:53,924 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:21:53,925 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:21:53,930 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:21:54,073 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:21:55,075 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:21:56,137 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 16:21:58,426 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:21:58,427 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:21:58,427 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:21:58,432 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:21:59,078 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:21:59,433 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:22:00,253 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:22:00,253 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:22:00,253 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:22:01,081 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:22:01,893 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:22:01,894 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:22:01,894 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:22:01,899 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:22:02,081 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:22:03,084 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:22:04,938 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:22:06,822 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:22:06,823 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:22:06,823 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:22:06,828 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:22:07,086 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:22:09,089 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:22:10,829 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:22:13,666 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:22:13,667 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:22:13,667 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:22:13,672 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:22:14,092 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:22:15,094 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:22:15,253 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:22:15,254 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:22:15,255 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:22:16,487 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:22:19,374 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:22:19,374 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:22:19,374 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:22:19,379 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:22:20,097 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:22:21,100 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:22:22,418 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:22:22,447 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:22:22,448 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:22:22,448 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:22:22,453 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:22:23,101 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:22:25,104 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:22:26,143 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 16:22:28,144 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:22:30,253 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:22:30,253 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:22:30,253 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:22:30,918 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:22:30,919 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:22:30,919 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:22:30,924 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:22:31,107 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:22:33,111 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:22:33,962 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:22:34,446 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:22:34,446 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:22:34,447 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:22:34,452 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:22:35,113 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:22:37,116 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:22:39,136 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:22:39,136 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:22:39,137 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:22:39,137 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:22:39,142 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:22:40,118 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:22:41,120 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:22:42,440 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:22:42,441 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:22:42,441 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:22:42,446 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:22:43,121 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:22:44,447 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:22:45,125 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:22:45,253 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:22:45,254 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:22:45,254 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:22:46,613 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:22:46,614 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:22:46,614 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:22:46,619 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:22:47,126 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:22:49,130 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:22:49,658 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:22:50,940 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:22:50,941 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:22:50,941 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:22:50,947 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:22:51,131 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:22:53,135 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:22:54,947 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:22:56,142 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 16:22:59,432 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:22:59,432 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:22:59,433 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:22:59,437 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:23:00,139 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:23:00,253 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:23:00,254 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:23:00,254 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:23:00,462 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:23:01,141 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:23:05,462 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:23:07,946 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:23:07,946 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:23:07,946 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:23:07,952 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:23:08,145 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:23:09,148 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:23:10,990 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:23:15,169 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:23:15,169 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:23:15,169 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:23:15,174 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:23:15,253 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:23:15,253 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:23:15,255 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:23:16,151 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:23:16,460 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:23:17,154 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:23:18,816 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:23:18,817 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:23:18,817 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:23:18,822 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:23:19,155 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:23:21,158 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:23:21,858 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:23:22,122 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:23:22,123 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:23:22,123 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:23:22,128 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:23:22,159 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:23:23,161 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:23:26,150 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 16:23:27,151 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:23:27,226 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:23:27,226 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:23:27,227 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:23:27,232 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:23:28,164 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:23:29,167 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:23:30,151 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:23:30,152 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:23:30,152 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:23:30,157 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:23:30,167 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:23:30,253 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:23:30,254 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:23:30,255 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:23:31,171 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:23:32,512 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:23:34,961 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:23:34,962 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:23:34,962 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:23:34,966 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:23:35,175 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:23:37,177 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:23:38,006 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:23:38,186 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:23:38,186 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:23:38,187 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:23:38,192 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:23:39,181 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:23:39,181 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:23:42,007 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:23:42,008 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:23:42,008 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:23:42,013 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:23:42,182 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:23:43,014 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:23:43,185 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:23:45,151 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:23:45,152 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:23:45,152 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:23:45,157 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:23:45,186 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:23:45,254 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:23:45,254 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:23:45,255 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:23:47,189 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:23:48,607 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:23:50,311 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:23:50,312 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:23:50,312 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:23:50,317 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:23:51,193 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:23:51,193 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:23:54,292 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:23:54,293 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:23:54,293 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:23:54,293 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:23:54,298 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:23:55,198 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:23:55,198 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:23:56,146 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 16:23:59,011 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:23:59,011 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:23:59,012 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:23:59,017 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:23:59,201 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:24:00,017 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:24:00,254 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:24:00,254 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:24:00,255 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:24:01,203 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:24:02,440 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:24:02,440 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:24:02,441 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:24:02,448 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:24:03,205 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:24:05,208 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:24:05,482 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:24:07,741 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:24:07,742 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:24:07,742 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:24:07,747 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:24:08,210 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:24:09,212 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:24:10,748 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:24:11,472 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:24:11,472 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:24:11,473 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:24:11,479 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:24:12,214 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:24:13,214 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:24:15,206 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:24:15,206 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:24:15,207 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:24:15,211 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:24:15,215 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:24:15,254 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:24:15,254 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:24:15,255 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:24:16,565 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:24:17,219 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:24:18,769 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:24:18,769 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:24:18,769 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:24:18,774 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:24:19,220 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:24:21,223 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:24:21,810 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:24:22,189 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:24:22,190 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:24:22,190 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:24:22,195 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:24:22,223 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:24:23,226 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:24:26,060 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:24:26,061 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:24:26,061 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:24:26,066 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:24:26,151 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 16:24:26,228 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:24:27,152 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:24:27,230 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:24:30,254 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:24:30,254 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:24:30,254 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:24:32,471 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:24:34,437 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:24:34,438 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:24:34,438 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:24:34,443 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:24:35,236 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:24:37,239 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:24:37,482 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:24:37,939 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:24:37,939 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:24:37,939 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:24:37,945 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:24:38,240 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:24:39,242 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:24:42,945 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:24:43,047 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:24:43,048 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:24:43,048 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:24:43,053 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:24:43,245 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:24:45,248 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:24:45,254 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:24:45,254 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:24:45,254 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:24:48,146 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:24:48,146 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:24:48,146 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:24:48,146 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:24:48,151 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:24:48,249 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:24:49,252 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:24:53,191 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:24:55,028 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:24:55,029 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:24:55,029 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:24:55,034 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:24:55,256 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:24:56,150 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 16:24:57,259 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:24:59,151 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:24:59,613 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:24:59,613 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:24:59,614 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:24:59,619 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:25:00,254 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:25:00,254 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:25:00,255 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:25:00,261 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:25:01,263 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:25:02,852 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:25:02,853 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:25:02,853 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:25:02,858 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:25:03,265 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:25:04,859 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:25:05,267 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:25:06,338 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:25:06,339 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:25:06,339 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:25:06,344 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:25:07,271 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:25:07,271 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:25:10,344 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:25:14,817 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:25:14,818 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:25:14,818 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:25:14,824 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:25:15,254 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:25:15,254 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:25:15,255 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:25:15,275 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:25:15,491 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:25:17,278 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:25:18,309 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:25:18,309 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:25:18,309 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:25:18,314 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:25:19,282 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:25:19,283 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:25:21,350 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:25:24,052 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:25:24,053 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:25:24,053 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:25:24,058 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:25:24,285 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:25:25,288 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:25:26,152 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 16:25:27,153 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:25:28,892 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:25:28,893 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:25:28,893 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:25:28,898 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:25:29,290 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:25:30,254 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:25:30,255 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:25:30,255 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:25:31,293 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:25:32,502 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:25:33,401 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:25:33,402 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:25:33,402 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:25:33,407 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:25:34,295 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:25:35,299 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:25:37,552 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:25:37,553 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:25:37,553 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:25:37,553 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:25:37,558 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:25:38,301 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:25:39,303 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:25:42,057 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:25:42,058 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:25:42,058 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:25:42,063 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:25:42,305 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:25:43,064 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:25:43,307 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:25:45,254 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:25:45,255 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:25:45,255 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:25:45,724 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:25:45,724 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:25:45,724 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:25:45,730 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:25:46,309 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:25:47,311 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:25:48,766 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:25:50,380 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:25:50,386 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:25:50,386 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:25:50,390 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:25:51,316 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:25:51,316 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:25:53,931 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:25:53,932 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:25:53,932 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:25:53,932 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:25:53,937 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:25:54,318 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:25:55,321 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:25:56,154 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 16:25:59,155 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:26:00,254 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:26:00,255 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:26:00,255 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:26:02,378 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:26:02,379 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:26:02,379 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:26:02,384 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:26:03,327 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:26:03,327 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:26:04,422 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:26:05,831 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:26:05,832 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:26:05,832 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:26:05,837 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:26:06,329 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:26:07,331 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:26:09,442 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:26:09,443 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:26:09,443 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:26:09,444 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:26:09,448 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:26:10,333 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:26:11,336 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:26:13,723 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:26:13,724 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:26:13,724 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:26:13,729 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:26:14,337 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:26:14,729 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:26:15,255 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:26:15,255 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:26:15,255 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:26:15,340 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:26:19,254 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:26:19,255 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:26:19,255 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:26:19,261 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:26:19,342 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:26:20,261 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:26:21,345 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:26:22,756 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:26:22,757 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:26:22,757 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:26:22,762 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:26:23,346 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:26:25,350 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:26:25,763 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:26:26,156 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 16:26:26,382 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:26:26,383 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:26:26,383 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:26:26,388 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:26:27,353 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:26:27,353 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:26:30,255 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:26:30,255 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:26:30,255 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:26:30,275 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:26:30,544 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:26:30,544 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:26:30,549 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:26:31,355 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:26:31,355 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:26:31,550 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:26:34,286 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:26:34,286 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:26:34,287 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:26:34,292 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:26:34,357 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:26:35,361 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:26:37,292 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:26:38,458 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:26:38,458 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:26:38,458 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:26:38,463 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:26:39,365 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:26:39,366 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:26:42,464 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:26:42,945 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:26:42,946 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:26:42,946 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:26:42,951 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:26:43,368 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:26:45,255 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:26:45,255 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:26:45,256 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:26:45,371 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:26:47,525 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:26:52,240 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:26:52,240 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:26:52,240 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:26:52,245 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:26:52,376 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:26:53,282 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:26:53,379 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:26:56,137 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:26:56,138 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:26:56,138 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:26:56,143 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:26:56,162 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 16:26:56,380 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:26:57,383 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:26:59,163 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:26:59,509 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:26:59,510 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:26:59,510 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:26:59,515 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:27:00,255 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:27:00,255 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:27:00,256 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:27:00,384 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:27:01,387 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:27:03,252 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:27:03,253 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:27:03,253 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:27:03,259 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:27:03,388 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:27:04,294 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:27:05,391 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:27:09,213 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:27:09,214 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:27:09,214 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:27:09,219 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:27:09,393 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:27:10,219 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:27:11,396 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:27:12,709 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:27:12,710 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:27:12,710 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:27:12,715 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:27:13,398 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:27:15,255 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:27:15,255 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:27:15,256 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:27:15,401 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:27:15,521 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:27:16,551 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:27:16,552 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:27:16,552 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:27:16,556 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:27:17,404 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:27:17,405 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:27:20,594 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:27:22,177 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:27:22,178 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:27:22,178 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:27:22,183 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:27:22,407 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:27:23,410 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:27:26,160 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 16:27:26,162 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:27:26,846 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:27:26,846 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:27:26,846 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:27:26,851 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:27:27,412 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:27:29,415 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:27:30,255 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:27:30,255 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:27:30,256 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:27:30,404 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:27:30,487 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:27:30,487 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:27:30,492 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:27:31,417 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:27:31,417 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:27:31,492 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:27:34,066 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:27:34,067 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:27:34,067 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:27:34,072 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:27:34,418 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:27:35,422 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:27:37,073 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:27:39,537 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:27:39,538 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:27:39,538 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:27:39,543 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:27:40,425 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:27:41,427 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:27:42,543 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:27:43,306 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:27:43,307 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:27:43,307 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:27:43,312 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:27:43,428 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:27:45,255 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:27:45,255 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:27:45,256 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:27:45,432 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:27:47,856 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:27:47,856 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:27:47,857 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:27:47,857 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:27:47,862 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:27:48,433 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:27:49,436 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:27:52,899 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:27:54,289 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:27:54,290 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:27:54,290 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:27:54,296 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:27:54,440 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:27:55,442 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:27:56,166 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 16:27:57,976 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:27:57,977 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:27:57,977 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:27:57,977 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:27:57,982 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:27:58,444 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:27:59,447 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:28:00,255 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:28:00,255 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:28:00,257 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:28:01,282 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:28:01,283 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:28:01,283 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:28:01,288 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:28:01,448 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:28:03,326 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:28:03,451 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:28:05,618 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:28:05,619 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:28:05,619 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:28:05,624 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:28:06,452 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:28:07,455 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:28:08,625 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:28:10,041 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:28:10,041 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:28:10,042 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:28:10,046 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:28:10,457 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:28:11,460 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:28:13,303 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:28:13,303 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:28:13,304 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:28:13,309 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:28:13,461 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:28:14,309 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:28:15,255 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:28:15,255 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:28:15,257 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:28:15,464 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:28:19,229 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:28:19,230 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:28:19,230 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:28:19,235 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:28:19,466 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:28:20,274 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:28:21,469 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:28:22,991 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:28:22,992 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:28:22,992 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:28:22,997 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:28:23,471 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:28:25,474 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:28:25,997 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:28:26,164 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 16:28:26,683 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:28:26,684 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:28:26,684 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:28:26,689 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:28:27,477 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:28:27,477 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:28:30,151 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:28:30,152 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:28:30,152 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:28:30,157 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:28:30,255 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:28:30,256 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:28:30,256 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:28:30,479 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:28:31,470 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:28:31,482 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:28:35,748 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:28:35,749 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:28:35,749 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:28:35,754 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:28:36,485 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:28:36,790 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:28:37,488 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:28:39,189 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:28:39,190 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:28:39,190 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:28:39,195 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:28:39,489 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:28:41,490 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:28:42,195 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:28:45,011 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:28:45,012 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:28:45,012 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:28:45,016 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:28:45,255 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:28:45,256 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:28:45,256 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:28:45,492 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:28:47,462 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:28:47,495 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:28:48,758 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:28:48,759 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:28:48,759 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:28:48,764 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:28:49,497 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:28:51,500 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:28:52,378 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:28:52,378 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:28:52,379 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:28:52,383 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:28:52,501 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:28:53,384 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:28:53,504 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:28:55,308 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:28:55,308 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:28:55,309 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:28:55,314 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:28:55,505 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:28:56,170 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 16:28:57,508 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:28:59,171 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:28:59,714 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:28:59,715 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:28:59,715 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:28:59,720 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:29:00,256 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:29:00,256 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:29:00,256 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:29:00,510 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:29:01,513 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:29:04,459 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:29:07,038 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:29:07,039 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:29:07,039 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:29:07,044 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:29:07,516 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:29:09,520 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:29:10,083 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:29:10,944 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:29:10,944 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:29:10,945 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:29:10,950 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:29:11,521 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:29:13,524 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:29:14,509 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:29:14,509 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:29:14,510 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:29:14,515 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:29:14,525 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:29:15,256 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:29:15,256 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:29:15,256 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:29:15,525 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:29:15,542 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:29:18,433 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:29:18,434 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:29:18,435 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:29:18,440 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:29:18,527 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:29:19,530 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:29:21,440 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:29:22,119 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:29:22,120 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:29:22,120 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:29:22,125 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:29:22,531 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:29:23,534 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:29:25,672 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:29:25,673 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:29:25,673 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:29:25,679 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:29:26,169 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 16:29:26,536 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:29:27,170 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:29:27,538 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:29:30,256 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:29:30,256 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:29:30,256 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:29:32,481 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:29:35,611 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:29:35,612 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:29:35,612 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:29:35,617 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:29:36,545 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:29:37,545 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:29:37,617 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:29:39,075 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:29:39,075 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:29:39,075 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:29:39,081 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:29:39,547 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:29:41,550 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:29:43,081 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:29:44,204 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:29:44,204 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:29:44,205 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:29:44,209 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:29:44,552 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:29:45,256 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:29:45,256 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:29:45,256 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:29:45,554 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:29:48,519 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:29:49,384 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:29:49,385 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:29:49,385 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:29:49,391 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:29:49,556 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:29:51,560 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:29:53,933 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:29:53,934 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:29:53,934 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:29:53,934 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:29:53,940 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:29:54,562 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:29:55,565 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:29:56,171 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 16:29:59,172 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:29:59,281 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:29:59,282 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:29:59,282 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:29:59,287 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:29:59,567 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:30:00,256 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:30:00,256 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:30:00,256 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:30:01,571 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:30:03,948 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:30:03,949 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:30:03,949 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:30:03,954 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:30:04,572 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:30:04,954 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:30:05,575 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:30:07,430 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:30:07,434 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:30:07,434 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:30:07,439 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:30:07,576 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:30:09,580 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:30:10,440 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:30:12,296 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:30:12,297 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:30:12,297 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:30:12,302 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:30:12,581 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:30:13,584 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:30:15,256 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:30:15,256 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:30:15,256 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:30:15,585 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:30:16,033 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:30:16,034 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:30:16,034 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:30:16,040 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:30:16,586 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:30:17,588 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:30:20,368 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:30:20,369 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:30:20,369 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:30:20,374 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:30:20,590 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:30:21,374 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:30:21,591 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:30:24,655 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:30:24,656 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:30:24,656 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:30:24,661 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:30:25,595 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:30:25,595 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:30:26,173 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 16:30:27,174 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:30:29,443 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:30:29,443 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:30:29,444 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:30:29,449 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:30:29,597 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:30:30,256 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:30:30,256 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:30:30,257 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:30:31,600 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:30:32,511 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:30:37,511 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:30:38,003 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:30:38,003 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:30:38,004 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:30:38,014 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:30:38,605 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:30:39,608 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:30:42,432 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:30:42,432 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:30:42,433 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:30:42,437 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:30:42,610 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:30:43,438 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:30:43,610 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:30:45,256 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:30:45,256 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:30:45,257 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:30:48,517 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:30:51,422 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:30:51,422 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:30:51,423 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:30:51,428 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:30:51,614 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:30:53,619 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:30:54,428 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:30:55,761 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:30:55,762 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:30:55,762 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:30:55,767 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:30:56,175 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 16:30:56,620 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:30:57,623 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:30:59,930 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:30:59,931 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:30:59,931 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:30:59,931 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:30:59,936 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:31:00,256 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:31:00,256 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:31:00,257 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:31:00,624 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:31:01,627 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:31:03,736 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:31:03,737 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:31:03,737 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:31:03,742 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:31:04,628 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:31:05,631 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:31:05,743 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:31:08,235 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:31:08,236 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:31:08,237 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:31:08,241 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:31:08,633 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:31:09,636 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:31:11,242 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:31:15,256 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:31:15,256 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:31:15,257 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:31:15,601 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:31:15,602 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:31:15,602 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:31:15,607 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:31:15,639 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:31:16,646 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:31:17,642 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:31:19,561 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:31:19,561 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:31:19,562 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:31:19,567 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:31:19,643 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:31:21,646 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:31:22,567 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:31:26,180 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 16:31:28,011 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:31:28,012 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:31:28,012 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:31:28,012 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:31:28,017 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:31:28,650 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:31:29,652 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:31:30,256 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:31:30,257 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:31:30,257 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:31:31,642 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:31:31,643 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:31:31,643 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:31:31,648 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:31:31,653 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:31:33,649 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:31:33,657 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:31:35,167 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:31:35,168 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:31:35,168 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:31:35,173 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:31:35,659 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:31:37,662 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:31:39,174 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:31:41,461 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:31:41,461 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:31:41,461 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:31:41,473 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:31:41,664 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:31:43,667 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:31:44,473 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:31:45,249 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:31:45,249 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:31:45,250 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:31:45,255 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:31:45,257 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:31:45,257 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:31:45,257 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:31:45,668 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:31:47,671 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:31:49,531 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:31:50,177 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:31:50,178 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:31:50,178 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:31:50,183 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:31:50,672 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:31:51,675 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:31:53,505 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:31:53,506 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:31:53,506 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:31:53,511 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:31:53,677 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:31:55,512 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:31:55,680 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:31:56,179 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 16:31:58,057 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:31:58,058 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:31:58,058 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:31:58,063 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:31:58,682 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:31:59,684 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:32:00,257 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:32:00,257 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:32:00,257 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:32:00,527 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:32:02,139 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:32:02,139 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:32:02,139 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:32:02,144 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:32:02,686 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:32:03,689 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:32:06,182 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:32:06,668 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:32:06,669 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:32:06,669 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:32:06,674 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:32:06,690 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:32:07,693 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:32:10,169 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:32:10,169 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:32:10,170 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:32:10,174 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:32:10,695 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:32:11,695 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:32:12,175 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:32:13,923 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:32:13,924 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:32:13,924 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:32:13,929 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:32:14,697 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:32:15,257 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:32:15,257 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:32:15,257 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:32:15,700 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:32:17,223 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:32:17,224 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:32:17,224 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:32:17,225 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:32:17,229 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:32:17,702 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:32:19,705 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:32:21,339 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:32:21,340 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:32:21,340 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:32:21,345 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:32:21,706 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:32:22,345 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:32:23,709 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:32:24,567 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:32:24,567 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:32:24,568 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:32:24,573 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:32:24,710 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:32:25,713 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:32:26,184 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 16:32:28,185 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:32:28,638 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:32:28,638 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:32:28,639 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:32:28,644 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:32:28,714 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:32:29,717 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:32:30,257 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:32:30,257 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:32:30,257 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:32:32,126 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:32:32,127 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:32:32,127 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:32:32,132 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:32:32,719 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:32:33,719 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:32:34,133 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:32:36,215 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:32:36,216 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:32:36,216 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:32:36,221 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:32:36,723 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:32:37,725 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:32:39,221 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:32:39,348 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:32:39,349 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:32:39,349 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:32:39,354 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:32:39,726 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:32:41,729 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:32:42,584 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:32:42,584 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:32:42,585 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:32:42,592 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:32:42,730 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:32:43,732 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:32:44,592 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:32:45,257 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:32:45,257 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:32:45,257 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:32:45,885 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:32:45,886 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:32:45,886 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:32:45,891 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:32:46,734 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:32:47,737 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:32:49,390 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:32:49,391 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:32:49,391 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:32:49,396 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:32:49,738 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:32:50,396 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:32:51,741 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:32:52,653 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:32:52,654 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:32:52,654 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:32:52,659 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:32:52,743 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:32:53,745 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:32:55,659 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:32:56,183 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 16:32:57,098 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:32:57,098 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:32:57,099 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:32:57,104 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:32:57,747 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:32:59,750 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:33:00,257 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:33:00,257 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:33:00,257 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:33:00,842 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:33:00,843 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:33:00,843 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:33:00,843 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:33:00,848 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:33:01,753 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:33:01,754 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:33:03,939 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:33:03,940 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:33:03,940 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:33:03,945 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:33:04,755 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:33:05,757 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:33:05,945 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:33:07,027 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:33:07,028 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:33:07,028 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:33:07,033 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:33:07,759 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:33:09,762 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:33:11,034 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:33:13,685 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:33:13,686 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:33:13,686 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:33:13,691 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:33:13,764 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:33:15,257 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:33:15,257 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:33:15,258 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:33:15,767 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:33:16,547 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:33:17,117 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:33:17,118 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:33:17,118 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:33:17,123 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:33:17,769 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:33:19,771 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:33:21,091 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:33:21,092 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:33:21,092 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:33:21,097 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:33:21,773 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:33:22,097 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:33:23,776 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:33:25,045 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:33:25,045 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:33:25,046 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:33:25,050 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:33:25,777 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:33:26,188 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 16:33:27,189 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:33:27,780 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:33:28,550 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:33:28,550 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:33:28,551 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:33:28,556 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:33:28,780 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:33:29,783 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:33:30,257 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:33:30,257 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:33:30,258 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:33:31,886 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:33:31,887 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:33:31,887 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:33:31,892 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:33:32,785 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:33:32,930 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:33:33,787 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:33:34,978 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:33:34,978 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:33:34,979 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:33:34,983 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:33:35,791 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:33:35,792 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:33:37,984 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:33:38,951 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:33:38,951 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:33:38,952 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:33:38,956 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:33:39,795 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:33:39,796 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:33:43,560 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:33:43,561 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:33:43,561 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:33:43,561 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:33:43,566 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:33:43,798 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:33:45,257 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:33:45,257 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:33:45,258 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:33:45,801 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:33:49,521 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:33:50,434 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:33:50,435 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:33:50,435 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:33:50,440 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:33:50,803 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:33:51,806 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:33:54,219 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:33:54,220 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:33:54,220 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:33:54,225 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:33:54,809 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:33:55,225 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:33:55,811 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:33:56,187 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 16:33:58,116 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:33:58,116 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:33:58,116 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:33:58,122 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:33:58,813 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:33:59,816 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:34:00,257 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:34:00,257 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:34:00,258 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:34:00,560 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:34:03,468 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:34:03,468 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:34:03,469 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:34:03,474 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:34:03,818 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:34:05,819 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:34:06,510 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:34:07,525 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:34:07,525 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:34:07,526 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:34:07,530 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:34:07,820 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:34:09,823 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:34:11,373 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:34:11,373 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:34:11,374 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:34:11,378 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:34:11,825 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:34:12,379 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:34:13,828 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:34:14,948 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:34:14,948 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:34:14,949 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:34:14,954 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:34:15,257 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:34:15,258 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:34:15,258 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:34:15,829 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:34:17,494 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:34:17,832 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:34:20,270 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:34:20,271 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:34:20,271 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:34:20,276 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:34:20,834 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:34:21,836 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:34:23,276 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:34:25,116 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:34:25,117 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:34:25,117 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:34:25,122 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:34:25,839 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:34:26,195 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 16:34:27,841 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:34:28,590 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:34:28,590 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:34:28,590 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:34:28,591 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:34:28,596 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:34:28,842 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:34:29,845 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:34:30,258 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:34:30,258 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:34:30,258 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:34:33,519 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:34:33,520 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:34:33,520 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:34:33,525 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:34:33,847 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:34:34,525 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:34:35,851 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:34:39,526 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:34:41,849 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:34:41,850 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:34:41,850 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:34:41,855 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:34:41,855 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:34:43,859 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:34:44,856 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:34:45,258 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:34:45,258 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:34:45,258 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:34:45,715 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:34:45,716 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:34:45,716 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:34:45,721 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:34:45,860 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:34:47,863 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:34:49,001 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:34:49,002 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:34:49,002 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:34:49,007 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:34:49,864 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:34:50,007 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:34:51,867 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:34:52,848 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:34:52,849 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:34:52,849 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:34:52,854 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:34:52,868 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:34:55,855 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:34:55,872 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:34:56,192 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 16:34:56,845 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:34:56,846 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:34:56,846 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:34:56,851 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:34:56,872 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:34:59,876 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:35:00,258 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:35:00,258 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:35:00,258 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:35:00,997 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:35:00,998 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:35:00,998 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:35:00,998 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:35:01,003 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:35:01,877 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:35:03,880 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:35:04,491 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:35:04,492 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:35:04,492 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:35:04,498 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:35:04,881 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:35:05,883 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:35:06,498 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:35:09,553 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:35:09,553 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:35:09,554 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:35:09,558 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:35:09,886 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:35:11,559 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:35:11,889 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:35:14,468 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:35:14,469 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:35:14,469 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:35:14,474 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:35:14,891 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:35:15,258 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:35:15,258 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:35:15,258 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:35:15,893 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:35:16,572 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:35:18,198 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:35:18,199 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:35:18,199 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:35:18,204 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:35:18,895 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:35:19,897 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:35:21,751 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:35:21,752 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:35:21,752 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:35:21,753 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:35:21,757 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:35:21,899 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:35:23,902 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:35:24,533 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:35:24,534 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:35:24,534 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:35:24,539 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:35:24,902 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:35:25,905 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:35:26,197 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 16:35:27,198 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:35:27,838 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:35:27,838 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:35:27,839 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:35:27,844 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:35:27,906 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:35:29,909 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:35:30,258 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:35:30,258 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:35:30,258 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:35:32,095 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:35:32,096 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:35:32,096 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:35:32,101 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:35:32,912 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:35:33,102 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:35:33,914 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:35:35,803 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:35:35,804 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:35:35,804 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:35:35,809 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:35:35,917 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:35:37,920 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:35:38,810 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:35:39,794 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:35:39,795 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:35:39,795 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:35:39,800 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:35:39,921 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:35:41,924 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:35:44,284 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:35:44,285 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:35:44,285 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:35:44,285 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:35:44,290 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:35:44,925 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:35:45,258 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:35:45,258 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:35:45,258 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:35:45,928 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:35:49,555 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:35:50,010 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:35:50,011 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:35:50,011 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:35:50,016 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:35:50,931 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:35:51,933 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:35:54,605 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:35:54,606 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:35:54,606 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:35:54,606 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:35:54,611 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:35:54,936 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:35:55,939 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:35:56,196 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 16:35:59,403 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:35:59,404 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:35:59,404 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:35:59,410 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:35:59,941 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:36:00,258 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:36:00,258 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:36:00,259 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:36:00,549 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:36:01,944 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:36:04,011 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:36:04,012 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:36:04,012 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:36:04,017 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:36:04,946 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:36:05,948 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:36:06,018 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:36:07,903 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:36:07,903 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:36:07,904 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:36:07,909 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:36:07,949 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:36:09,953 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:36:11,909 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:36:12,126 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:36:12,127 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:36:12,127 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:36:12,133 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:36:12,954 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:36:13,957 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:36:15,258 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:36:15,258 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:36:15,259 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:36:15,846 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:36:15,847 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:36:15,847 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:36:15,852 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:36:15,958 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:36:17,853 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:36:17,961 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:36:22,853 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:36:24,539 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:36:24,539 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:36:24,540 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:36:24,544 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:36:24,965 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:36:25,968 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:36:26,198 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 16:36:28,199 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:36:29,016 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:36:29,016 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:36:29,017 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:36:29,022 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:36:29,970 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:36:30,258 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:36:30,258 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:36:30,259 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:36:31,974 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:36:32,652 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:36:32,653 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:36:32,653 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:36:32,658 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:36:32,975 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:36:33,694 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:36:33,975 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:36:36,579 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:36:36,579 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:36:36,580 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:36:36,585 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:36:36,978 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:36:37,981 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:36:39,586 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:36:42,017 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:36:42,021 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:36:42,021 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:36:42,026 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:36:42,983 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:36:43,986 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:36:45,026 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:36:45,258 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:36:45,258 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:36:45,259 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:36:45,961 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:36:45,962 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:36:45,962 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:36:45,967 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:36:45,987 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:36:47,990 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:36:51,007 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:36:51,064 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:36:51,065 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:36:51,065 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:36:51,070 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:36:51,992 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:36:53,997 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:36:56,071 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:36:56,171 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:36:56,172 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:36:56,172 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:36:56,177 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:36:56,200 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 16:36:56,998 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:36:58,001 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:37:00,214 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:37:00,215 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:37:00,215 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:37:00,220 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:37:00,258 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:37:00,259 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:37:00,259 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:37:01,003 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:37:01,576 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:37:02,005 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:37:04,538 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:37:04,539 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:37:04,539 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:37:04,544 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:37:05,007 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:37:06,009 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:37:07,545 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:37:12,545 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:37:12,966 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:37:12,967 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:37:12,967 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:37:12,972 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:37:13,013 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:37:15,259 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:37:15,259 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:37:15,259 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:37:16,017 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:37:16,306 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:37:16,307 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:37:16,307 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:37:16,312 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:37:17,018 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:37:18,020 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:37:18,313 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:37:20,564 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:37:20,565 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:37:20,565 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:37:20,570 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:37:21,022 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:37:22,024 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:37:23,571 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:37:26,202 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 16:37:27,003 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:37:27,004 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:37:27,004 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:37:27,010 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:37:27,027 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:37:29,011 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:37:30,031 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:37:30,259 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:37:30,259 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:37:30,259 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:37:33,376 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:37:33,377 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:37:33,377 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:37:33,382 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:37:34,033 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:37:34,382 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:37:36,038 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:37:37,994 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:37:37,995 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:37:37,995 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:37:38,000 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:37:38,039 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:37:40,000 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:37:40,040 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:37:43,146 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:37:43,147 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:37:43,148 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:37:43,152 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:37:44,042 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:37:45,153 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:37:45,259 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:37:45,259 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:37:45,259 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:37:46,045 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:37:46,189 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:37:46,190 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:37:46,190 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:37:46,195 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:37:47,046 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:37:48,048 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:37:49,639 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:37:49,640 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:37:49,640 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:37:49,645 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:37:50,049 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:37:50,645 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:37:52,053 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:37:53,762 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:37:53,762 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:37:53,763 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:37:53,768 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:37:54,055 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:37:55,768 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:37:56,056 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:37:56,207 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 16:37:57,307 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:37:57,308 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:37:57,309 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:37:57,313 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:37:58,057 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:38:00,060 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:38:00,259 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:38:00,259 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:38:00,259 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:38:01,476 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:38:02,649 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:38:02,650 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:38:02,650 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:38:02,655 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:38:03,062 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:38:04,064 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:38:06,320 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:38:06,321 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:38:06,321 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:38:06,326 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:38:07,066 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:38:07,327 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:38:08,069 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:38:09,674 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:38:09,674 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:38:09,675 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:38:09,679 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:38:10,070 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:38:12,074 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:38:12,680 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:38:15,259 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:38:15,259 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:38:15,259 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:38:15,655 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:38:15,656 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:38:15,656 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:38:15,661 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:38:16,076 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:38:18,077 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:38:18,661 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:38:23,662 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:38:24,752 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:38:24,753 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:38:24,753 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:38:24,758 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:38:25,081 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:38:26,084 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:38:26,206 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 16:38:27,848 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:38:27,848 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:38:27,849 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:38:27,862 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:38:28,085 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:38:28,862 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:38:30,088 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:38:30,259 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:38:30,259 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:38:30,259 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:38:31,094 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:38:31,095 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:38:31,095 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:38:31,100 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:38:32,089 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:38:34,090 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:38:34,139 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:38:35,059 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:38:35,060 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:38:35,060 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:38:35,065 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:38:35,091 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:38:38,130 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:38:39,841 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:38:39,842 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:38:39,843 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:38:39,843 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:38:39,848 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:38:40,131 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:38:42,134 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:38:44,094 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:38:44,094 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:38:44,095 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:38:44,099 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:38:44,135 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:38:45,100 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:38:45,259 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:38:45,259 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:38:45,259 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:38:46,138 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:38:48,832 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:38:48,832 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:38:48,833 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:38:48,838 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:38:49,141 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:38:50,144 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:38:50,874 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:38:53,205 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:38:53,206 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:38:53,206 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:38:53,211 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:38:54,147 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:38:56,148 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:38:56,211 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 16:38:56,213 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:38:57,067 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:38:57,068 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:38:57,068 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:38:57,073 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:38:57,149 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:39:00,153 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:39:00,259 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:39:00,259 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:39:00,260 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:39:00,538 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:39:00,552 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:39:00,552 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:39:00,557 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:39:01,153 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:39:01,558 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:39:02,156 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:39:04,622 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:39:04,622 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:39:04,623 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:39:04,628 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:39:05,158 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:39:06,160 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:39:06,666 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:39:09,189 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:39:09,190 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:39:09,190 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:39:09,195 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:39:10,162 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:39:12,163 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:39:12,196 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:39:12,765 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:39:12,766 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:39:12,767 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:39:12,771 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:39:13,164 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:39:14,166 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:39:15,259 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:39:15,259 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:39:15,260 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:39:17,497 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:39:17,988 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:39:17,988 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:39:17,989 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:39:17,994 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:39:18,168 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:39:20,171 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:39:21,275 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:39:21,276 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:39:21,276 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:39:21,281 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:39:22,173 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:39:23,282 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:39:24,176 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:39:24,884 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:39:24,885 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:39:24,885 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:39:24,890 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:39:25,176 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:39:26,179 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:39:26,210 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 16:39:29,212 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:39:29,461 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:39:29,461 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:39:29,462 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:39:29,467 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:39:30,181 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:39:30,259 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:39:30,259 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:39:30,260 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:39:32,184 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:39:33,107 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:39:33,107 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:39:33,107 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:39:33,112 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:39:33,185 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:39:35,150 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:39:36,190 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:39:36,841 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:39:36,842 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:39:36,842 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:39:36,847 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:39:37,190 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:39:38,193 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:39:40,848 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:39:42,615 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:39:42,616 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:39:42,616 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:39:42,621 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:39:43,196 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:39:44,198 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:39:45,259 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:39:45,260 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:39:45,260 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:39:45,979 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:39:45,979 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:39:45,980 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:39:45,980 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:39:45,985 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:39:46,199 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:39:48,203 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:39:49,742 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:39:49,743 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:39:49,743 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:39:49,748 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:39:50,204 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:39:51,748 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:39:52,207 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:39:53,523 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:39:53,523 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:39:53,524 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:39:53,528 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:39:54,209 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:39:56,212 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:39:56,216 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 16:39:57,217 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:39:58,966 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:39:58,967 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:39:58,967 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:39:58,972 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:39:59,214 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:40:00,216 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:40:00,259 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:40:00,260 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:40:00,260 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:40:02,260 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:40:02,261 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:40:02,261 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:40:02,262 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:40:02,267 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:40:03,218 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:40:04,221 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:40:06,185 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:40:06,186 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:40:06,186 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:40:06,191 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:40:06,222 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:40:08,191 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:40:08,225 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:40:09,487 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:40:09,488 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:40:09,488 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:40:09,493 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:40:10,227 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:40:12,230 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:40:13,250 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:40:13,251 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:40:13,251 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:40:13,251 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:40:13,256 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:40:14,231 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:40:15,260 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:40:15,260 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:40:15,260 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:40:16,234 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:40:18,549 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:40:18,861 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:40:18,861 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:40:18,862 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:40:18,867 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:40:19,235 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:40:20,238 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:40:22,313 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:40:22,314 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:40:22,314 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:40:22,322 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:40:23,240 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:40:24,242 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:40:24,322 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:40:26,215 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 16:40:27,628 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:40:27,629 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:40:27,630 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:40:27,634 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:40:28,244 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:40:29,635 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:40:30,247 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:40:30,260 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:40:30,260 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:40:30,260 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:40:31,524 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:40:31,525 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:40:31,525 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:40:31,531 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:40:32,249 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:40:34,252 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:40:35,006 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:40:35,007 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:40:35,007 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:40:35,007 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:40:35,012 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:40:35,253 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:40:36,257 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:40:38,461 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:40:38,462 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:40:38,462 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:40:38,467 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:40:39,258 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:40:40,261 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:40:40,467 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:40:45,260 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:40:45,260 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:40:45,260 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:40:45,578 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:40:46,653 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:40:46,653 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:40:46,654 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:40:46,658 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:40:47,264 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:40:48,267 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:40:49,501 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:40:49,502 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:40:49,502 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:40:49,508 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:40:50,268 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:40:51,508 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:40:52,271 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:40:55,299 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:40:55,300 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:40:55,300 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:40:55,304 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:40:56,220 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 16:40:56,277 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:40:57,221 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:40:58,280 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:40:59,735 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:40:59,736 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:40:59,736 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:40:59,741 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:41:00,260 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:41:00,260 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:41:00,260 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:41:00,281 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:41:02,284 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:41:02,558 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:41:04,116 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:41:04,117 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:41:04,117 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:41:04,122 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:41:04,286 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:41:06,289 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:41:08,123 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:41:09,876 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:41:09,876 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:41:09,877 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:41:09,881 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:41:10,292 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:41:12,294 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:41:13,326 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:41:13,327 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:41:13,328 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:41:13,328 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:41:13,332 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:41:14,296 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:41:15,260 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:41:15,260 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:41:15,260 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:41:16,299 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:41:17,419 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:41:17,420 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:41:17,420 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:41:17,425 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:41:18,300 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:41:18,425 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:41:20,303 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:41:21,668 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:41:21,669 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:41:21,669 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:41:21,674 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:41:22,304 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:41:23,675 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:41:24,307 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:41:26,219 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 16:41:29,220 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:41:30,118 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:41:30,119 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:41:30,119 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:41:30,124 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:41:30,260 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:41:30,260 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:41:30,260 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:41:30,311 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:41:32,314 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:41:34,563 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:41:34,825 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:41:34,826 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:41:34,826 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:41:34,831 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:41:35,316 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:41:36,320 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:41:38,887 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:41:38,888 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:41:38,888 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:41:38,893 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:41:39,321 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:41:39,893 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:41:40,324 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:41:44,894 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:41:45,093 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:41:45,094 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:41:45,094 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:41:45,099 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:41:45,260 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:41:45,260 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:41:45,261 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:41:45,326 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:41:46,329 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:41:49,567 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:41:49,568 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:41:49,568 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:41:49,573 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:41:50,331 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:41:50,574 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:41:52,334 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:41:53,094 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:41:53,095 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:41:53,095 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:41:53,100 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:41:53,336 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:41:54,338 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:41:56,101 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:41:56,221 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 16:41:56,861 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:41:56,861 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:41:56,862 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:41:56,867 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:41:57,340 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:41:58,343 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:42:00,260 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:42:00,260 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:42:00,261 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:42:01,040 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:42:01,044 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:42:01,045 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:42:01,049 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:42:01,344 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:42:02,050 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:42:02,347 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:42:05,113 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:42:05,114 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:42:05,114 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:42:05,119 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:42:05,349 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:42:06,351 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:42:07,120 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:42:08,844 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:42:08,845 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:42:08,846 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:42:08,850 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:42:09,353 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:42:10,356 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:42:12,851 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:42:15,260 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:42:15,260 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:42:15,261 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:42:18,031 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:42:18,031 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:42:18,032 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:42:18,032 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:42:18,036 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:42:18,360 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:42:20,363 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:42:21,926 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:42:21,927 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:42:21,927 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:42:21,932 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:42:22,364 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:42:23,933 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:42:24,367 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:42:26,223 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 16:42:26,417 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:42:26,418 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:42:26,418 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:42:26,422 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:42:27,369 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:42:28,371 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:42:29,423 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:42:29,861 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:42:29,862 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:42:29,863 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:42:29,867 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:42:30,261 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:42:30,261 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:42:30,261 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:42:30,373 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:42:32,376 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:42:33,584 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:42:33,585 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:42:33,585 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:42:33,590 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:42:34,377 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:42:34,591 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:42:36,381 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:42:37,966 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:42:37,967 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:42:37,967 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:42:37,972 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:42:38,383 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:42:39,973 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:42:40,384 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:42:43,235 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:42:43,235 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:42:43,236 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:42:43,241 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:42:43,385 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:42:45,241 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:42:45,260 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:42:45,261 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:42:45,261 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:42:46,389 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:42:49,191 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:42:49,192 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:42:49,192 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:42:49,197 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:42:49,390 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:42:51,198 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:42:52,394 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:42:54,120 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:42:54,121 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:42:54,122 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:42:54,126 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:42:54,396 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:42:56,225 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 16:42:56,226 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:42:56,397 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:42:58,424 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:42:58,424 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:42:58,425 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:42:58,429 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:42:59,399 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:43:00,261 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:43:00,261 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:43:00,261 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:43:00,402 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:43:01,477 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:43:02,060 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:43:02,061 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:43:02,061 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:43:02,066 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:43:02,403 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:43:04,406 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:43:05,471 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:43:05,471 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:43:05,472 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:43:05,477 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:43:06,407 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:43:07,477 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:43:08,410 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:43:10,458 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:43:10,459 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:43:10,459 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:43:10,463 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:43:11,412 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:43:12,414 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:43:13,464 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:43:14,509 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:43:14,509 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:43:14,510 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:43:14,515 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:43:15,261 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:43:15,261 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:43:15,261 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:43:15,416 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:43:16,419 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:43:18,496 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:43:18,973 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:43:18,974 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:43:18,974 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:43:18,979 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:43:19,420 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:43:20,423 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:43:22,555 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:43:22,556 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:43:22,556 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:43:22,562 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:43:23,424 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:43:23,562 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:43:24,425 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:43:26,227 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 16:43:28,899 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:43:28,899 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:43:28,900 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:43:28,900 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:43:28,905 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:43:29,428 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:43:30,261 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:43:30,261 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:43:30,261 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:43:30,430 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:43:34,445 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:43:34,446 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:43:34,447 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:43:34,447 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:43:34,451 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:43:35,433 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:43:36,437 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:43:38,232 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:43:38,233 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:43:38,233 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:43:38,238 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:43:38,438 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:43:40,238 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:43:40,441 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:43:41,912 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:43:41,913 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:43:41,914 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:43:41,919 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:43:42,442 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:43:44,446 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:43:45,015 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:43:45,016 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:43:45,016 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:43:45,021 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:43:45,261 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:43:45,261 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:43:45,261 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:43:45,446 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:43:45,558 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:43:46,449 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:43:48,905 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:43:48,906 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:43:48,906 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:43:48,911 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:43:49,451 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:43:50,453 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:43:50,912 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:43:52,210 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:43:52,210 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:43:52,211 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:43:52,216 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:43:52,454 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:43:54,457 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:43:55,888 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:43:55,889 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:43:55,890 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:43:55,895 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:43:56,229 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 16:43:56,230 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:43:56,458 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:43:58,461 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:43:59,639 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:43:59,640 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:43:59,640 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:43:59,645 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:44:00,261 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:44:00,261 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:44:00,261 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:44:00,462 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:44:01,535 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:44:02,465 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:44:02,982 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:44:02,982 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:44:02,983 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:44:02,987 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:44:03,465 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:44:04,468 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:44:06,407 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:44:06,407 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:44:06,408 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:44:06,413 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:44:06,469 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:44:07,413 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:44:08,472 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:44:10,979 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:44:10,979 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:44:10,980 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:44:10,985 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:44:11,474 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:44:12,476 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:44:12,985 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:44:15,261 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:44:15,261 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:44:15,261 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:44:15,997 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:44:15,997 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:44:15,998 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:44:16,002 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:44:16,478 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:44:18,003 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:44:18,481 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:44:19,462 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:44:19,463 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:44:19,463 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:44:19,468 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:44:19,482 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:44:22,485 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:44:23,469 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:44:24,200 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:44:24,200 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:44:24,201 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:44:24,206 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:44:24,486 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:44:26,234 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 16:44:26,489 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:44:27,720 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:44:27,721 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:44:27,721 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:44:27,727 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:44:28,490 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:44:28,727 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:44:30,261 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:44:30,261 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:44:30,262 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:44:30,493 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:44:34,503 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:44:36,353 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:44:36,353 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:44:36,354 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:44:36,359 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:44:36,500 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:44:38,504 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:44:40,359 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:44:41,101 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:44:41,102 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:44:41,102 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:44:41,107 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:44:41,507 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:44:42,510 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:44:45,022 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:44:45,023 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:44:45,023 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:44:45,028 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:44:45,261 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:44:45,261 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:44:45,262 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:44:45,513 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:44:45,545 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:44:46,516 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:44:48,931 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:44:48,932 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:44:48,932 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:44:48,937 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:44:49,519 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:44:50,522 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:44:50,938 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:44:54,274 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:44:54,275 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:44:54,275 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:44:54,280 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:44:54,526 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:44:56,233 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 16:44:56,235 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:44:56,528 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:44:58,502 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:44:58,503 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:44:58,503 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:44:58,508 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:44:58,531 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:45:00,261 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:45:00,261 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:45:00,262 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:45:00,536 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:45:01,528 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:45:06,528 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:45:07,082 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:45:07,083 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:45:07,083 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:45:07,088 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:45:07,543 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:45:08,546 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:45:10,326 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:45:10,326 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:45:10,327 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:45:10,332 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:45:10,547 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:45:12,332 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:45:12,548 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:45:14,014 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:45:14,014 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:45:14,015 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:45:14,020 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:45:14,549 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:45:15,261 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:45:15,262 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:45:15,262 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:45:16,552 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:45:17,573 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:45:17,677 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:45:17,678 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:45:17,678 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:45:17,683 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:45:18,554 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:45:20,557 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:45:21,127 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:45:21,127 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:45:21,128 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:45:21,133 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:45:21,557 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:45:22,559 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:45:23,133 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:45:26,238 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 16:45:26,721 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:45:26,722 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:45:26,722 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:45:26,727 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:45:27,562 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:45:28,563 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:45:28,728 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:45:30,261 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:45:30,262 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:45:30,262 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:45:31,865 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:45:31,866 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:45:31,866 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:45:31,871 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:45:32,565 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:45:33,871 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:45:34,568 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:45:37,006 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:45:37,007 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:45:37,007 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:45:37,012 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:45:37,571 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:45:38,574 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:45:39,012 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:45:43,228 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:45:43,229 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:45:43,229 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:45:43,234 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:45:43,579 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:45:44,234 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:45:44,579 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:45:45,262 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:45:45,262 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:45:45,262 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:45:47,503 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:45:47,504 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:45:47,504 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:45:47,509 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:45:47,581 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:45:49,509 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:45:50,585 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:45:51,911 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:45:51,911 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:45:51,912 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:45:51,917 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:45:52,586 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:45:54,590 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:45:54,917 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:45:56,238 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 16:45:57,864 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:45:57,865 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:45:57,866 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:45:57,871 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:45:58,592 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:46:00,262 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:46:00,262 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:46:00,262 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:46:00,510 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:46:00,594 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:46:01,384 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:46:01,385 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:46:01,385 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:46:01,390 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:46:01,594 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:46:02,598 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:46:04,642 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:46:04,643 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:46:04,643 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:46:04,658 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:46:05,600 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:46:05,659 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:46:06,603 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:46:08,162 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:46:08,163 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:46:08,163 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:46:08,168 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:46:08,604 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:46:10,608 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:46:11,169 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:46:12,924 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:46:12,925 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:46:12,925 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:46:12,930 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:46:13,610 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:46:14,613 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:46:15,262 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:46:15,262 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:46:15,262 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:46:16,554 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:46:16,985 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:46:16,985 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:46:16,986 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:46:16,990 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:46:17,614 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:46:18,617 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:46:21,707 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:46:21,708 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:46:21,709 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:46:21,709 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:46:21,714 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:46:22,619 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:46:24,623 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:46:25,477 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:46:25,477 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:46:25,478 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:46:25,483 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:46:25,623 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:46:26,243 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 16:46:27,244 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:46:28,627 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:46:29,003 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:46:29,003 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:46:29,004 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:46:29,009 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:46:29,628 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:46:30,262 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:46:30,262 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:46:30,262 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:46:30,631 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:46:32,563 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:46:32,754 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:46:32,754 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:46:32,755 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:46:32,760 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:46:33,633 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:46:34,635 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:46:36,559 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:46:36,559 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:46:36,560 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:46:36,565 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:46:36,638 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:46:37,565 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:46:38,641 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:46:40,296 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:46:40,296 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:46:40,297 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:46:40,301 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:46:40,642 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:46:42,645 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:46:43,302 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:46:43,750 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:46:43,751 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:46:43,751 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:46:43,756 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:46:44,646 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:46:45,262 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:46:45,262 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:46:45,262 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:46:46,649 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:46:47,624 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:46:47,624 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:46:47,625 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:46:47,629 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:46:47,650 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:46:48,666 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:46:50,654 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:46:51,255 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:46:51,256 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:46:51,257 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:46:51,261 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:46:51,654 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:46:52,657 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:46:54,262 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:46:55,169 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:46:55,169 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:46:55,170 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:46:55,175 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:46:55,660 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:46:56,242 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 16:46:56,660 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:46:58,676 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:46:58,677 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:46:58,677 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:46:58,682 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:46:59,662 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:46:59,683 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:47:00,262 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:47:00,262 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:47:00,262 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:47:00,664 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:47:02,804 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:47:02,805 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:47:02,805 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:47:02,810 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:47:03,666 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:47:04,669 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:47:04,811 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:47:06,284 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:47:06,285 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:47:06,285 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:47:06,290 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:47:06,670 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:47:08,671 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:47:10,291 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:47:10,646 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:47:10,646 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:47:10,647 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:47:10,651 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:47:10,672 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:47:12,676 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:47:14,773 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:47:14,773 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:47:14,774 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:47:14,779 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:47:15,262 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:47:15,262 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:47:15,263 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:47:15,511 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:47:15,677 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:47:16,751 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:47:20,511 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:47:22,436 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:47:22,437 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:47:22,437 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:47:22,442 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:47:22,755 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:47:24,758 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:47:26,247 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 16:47:26,248 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:47:26,828 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:47:26,829 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:47:26,830 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:47:26,835 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:47:27,759 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:47:28,762 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:47:30,262 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:47:30,262 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:47:30,263 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:47:31,109 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:47:31,110 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:47:31,110 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:47:31,116 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:47:31,764 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:47:32,117 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:47:32,767 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:47:35,988 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:47:35,989 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:47:35,989 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:47:35,994 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:47:36,771 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:47:37,995 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:47:38,774 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:47:40,081 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:47:40,081 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:47:40,082 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:47:40,087 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:47:40,776 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:47:42,778 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:47:43,088 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:47:43,829 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:47:43,830 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:47:43,830 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:47:43,835 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:47:44,780 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:47:45,262 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:47:45,262 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:47:45,263 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:47:46,783 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:47:48,530 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:47:49,453 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:47:49,454 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:47:49,454 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:47:49,459 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:47:49,784 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:47:50,787 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:47:52,520 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:47:52,521 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:47:52,522 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:47:52,527 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:47:52,788 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:47:54,527 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:47:54,793 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:47:56,246 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 16:47:57,681 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:47:57,682 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:47:57,682 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:47:57,687 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:47:57,795 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:47:59,688 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:48:00,262 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:48:00,262 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:48:00,263 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:48:00,799 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:48:01,842 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:48:01,842 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:48:01,843 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:48:01,849 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:48:02,801 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:48:04,802 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:48:04,850 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:48:06,282 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:48:06,282 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:48:06,283 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:48:06,288 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:48:06,804 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:48:08,808 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:48:10,288 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:48:10,789 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:48:10,790 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:48:10,790 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:48:10,795 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:48:10,810 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:48:12,815 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:48:15,262 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:48:15,263 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:48:15,263 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:48:15,524 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:48:16,160 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:48:16,160 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:48:16,161 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:48:16,166 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:48:16,819 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:48:18,823 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:48:21,165 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:48:21,165 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:48:21,166 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:48:21,166 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:48:21,171 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:48:21,826 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:48:22,828 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:48:25,046 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:48:25,046 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:48:25,047 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:48:25,052 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:48:25,832 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:48:26,248 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 16:48:26,249 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:48:26,832 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:48:28,978 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:48:28,979 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:48:28,980 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:48:28,984 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:48:29,837 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:48:30,262 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:48:30,263 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:48:30,263 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:48:30,840 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:48:31,570 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:48:34,172 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:48:34,173 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:48:34,173 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:48:34,178 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:48:34,844 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:48:36,848 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:48:37,179 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:48:39,078 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:48:39,079 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:48:39,079 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:48:39,084 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:48:39,853 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:48:40,856 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:48:43,085 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:48:43,574 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:48:43,574 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:48:43,575 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:48:43,580 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:48:43,859 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:48:44,862 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:48:45,263 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:48:45,263 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:48:45,263 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:48:47,094 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:48:47,095 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:48:47,095 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:48:47,100 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:48:47,865 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:48:48,101 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:48:48,866 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:48:51,773 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:48:51,773 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:48:51,774 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:48:51,779 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:48:51,871 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:48:53,779 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:48:54,875 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:48:55,246 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:48:55,247 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:48:55,247 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:48:55,252 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:48:55,877 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:48:56,250 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 16:48:56,880 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:48:58,072 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:48:58,073 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:48:58,073 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:48:58,078 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:48:58,881 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:48:59,079 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:49:00,263 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:49:00,263 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:49:00,263 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:49:00,883 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:49:02,465 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:49:02,465 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:49:02,466 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:49:02,470 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:49:02,886 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:49:04,506 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:49:04,891 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:49:08,683 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:49:08,684 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:49:08,684 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:49:08,690 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:49:08,894 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:49:09,690 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:49:10,897 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:49:12,225 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:49:12,226 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:49:12,226 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:49:12,231 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:49:12,898 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:49:14,901 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:49:15,232 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:49:15,263 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:49:15,263 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:49:15,263 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:49:16,081 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:49:16,081 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:49:16,082 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:49:16,087 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:49:16,902 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:49:18,905 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:49:19,105 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:49:19,106 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:49:19,106 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:49:19,111 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:49:19,906 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:49:20,908 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:49:21,112 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:49:22,754 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:49:22,755 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:49:22,755 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:49:22,760 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:49:22,909 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:49:24,913 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:49:25,995 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:49:25,996 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:49:25,997 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:49:26,001 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:49:26,252 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 16:49:26,253 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:49:26,914 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:49:28,917 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:49:30,263 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:49:30,263 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:49:30,263 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:49:31,558 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:49:34,551 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:49:34,551 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:49:34,552 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:49:34,557 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:49:34,920 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:49:36,923 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:49:37,557 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:49:38,922 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:49:38,923 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:49:38,923 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:49:38,929 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:49:39,927 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:49:40,930 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:49:41,985 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:49:41,986 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:49:41,988 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:49:41,993 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:49:42,931 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:49:42,993 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:49:44,934 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:49:45,263 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:49:45,263 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:49:45,263 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:49:46,117 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:49:46,118 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:49:46,118 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:49:46,123 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:49:46,935 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:49:48,124 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:49:48,938 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:49:49,608 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:49:49,609 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:49:49,609 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:49:49,614 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:49:49,939 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:49:50,941 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:49:52,897 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:49:52,901 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:49:52,901 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:49:52,906 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:49:52,943 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:49:53,906 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:49:54,947 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:49:56,254 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 16:49:56,796 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:49:56,797 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:49:56,797 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:49:56,802 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:49:56,948 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:49:58,951 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:49:59,803 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:50:00,263 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:50:00,263 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:50:00,263 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:50:01,539 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:50:01,540 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:50:01,540 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:50:01,545 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:50:01,953 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:50:02,955 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:50:05,416 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:50:05,417 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:50:05,418 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:50:05,418 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:50:05,423 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:50:05,957 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:50:06,960 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:50:10,175 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:50:10,176 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:50:10,177 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:50:10,182 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:50:10,963 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:50:11,183 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:50:12,966 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:50:13,905 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:50:13,905 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:50:13,906 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:50:13,911 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:50:13,966 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:50:15,263 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:50:15,263 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:50:15,264 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:50:16,510 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:50:16,970 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:50:19,899 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:50:19,900 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:50:19,900 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:50:19,905 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:50:19,972 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:50:21,905 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:50:22,976 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:50:23,355 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:50:23,356 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:50:23,356 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:50:23,361 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:50:23,976 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:50:24,979 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:50:26,256 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 16:50:26,762 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:50:26,762 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:50:26,763 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:50:26,768 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:50:26,980 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:50:27,768 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:50:28,983 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:50:30,263 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:50:30,263 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:50:30,264 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:50:33,572 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:50:34,273 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:50:34,274 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:50:34,274 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:50:34,279 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:50:34,987 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:50:36,991 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:50:38,401 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:50:38,402 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:50:38,402 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:50:38,407 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:50:38,992 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:50:39,407 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:50:40,995 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:50:44,408 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:50:45,263 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:50:45,263 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:50:45,264 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:50:47,496 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:50:47,497 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:50:47,497 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:50:47,503 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:50:47,999 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:50:49,002 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:50:49,503 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:50:51,909 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:50:51,910 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:50:51,910 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:50:51,915 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:50:52,003 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:50:54,916 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:50:55,009 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:50:56,261 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 16:50:56,811 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:50:56,813 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:50:56,813 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:50:56,818 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:50:57,010 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:50:59,013 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:50:59,795 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:50:59,795 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:50:59,796 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:50:59,801 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:51:00,013 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:51:00,264 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:51:00,264 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:51:00,265 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:51:00,482 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:51:03,015 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:51:04,478 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:51:04,479 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:51:04,479 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:51:04,484 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:51:05,016 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:51:05,522 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:51:07,019 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:51:09,055 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:51:09,056 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:51:09,056 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:51:09,061 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:51:10,022 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:51:11,024 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:51:11,062 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:51:12,483 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:51:12,483 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:51:12,484 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:51:12,488 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:51:13,025 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:51:15,026 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:51:15,263 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:51:15,264 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:51:15,264 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:51:16,474 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:51:18,572 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:51:18,573 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:51:18,573 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:51:18,579 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:51:19,029 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:51:21,031 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:51:21,579 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:51:23,343 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:51:23,344 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:51:23,344 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:51:23,349 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:51:24,033 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:51:25,035 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:51:26,261 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 16:51:27,032 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:51:27,033 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:51:27,033 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:51:27,033 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:51:27,038 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:51:27,039 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:51:29,042 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:51:30,263 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:51:30,264 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:51:30,264 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:51:32,049 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:51:32,050 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:51:32,050 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:51:32,050 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:51:32,055 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:51:33,044 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:51:35,047 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:51:37,056 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:51:40,535 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:51:40,536 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:51:40,537 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:51:40,542 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:51:41,052 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:51:42,542 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:51:43,055 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:51:44,396 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:51:44,397 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:51:44,397 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:51:44,402 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:51:45,056 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:51:45,264 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:51:45,264 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:51:45,264 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:51:47,059 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:51:48,510 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:51:48,730 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:51:48,731 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:51:48,731 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:51:48,736 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:51:49,061 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:51:51,064 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:51:52,508 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:51:52,509 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:51:52,509 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:51:52,514 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:51:53,065 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:51:53,514 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:51:55,069 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:51:56,268 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 16:51:56,606 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:51:56,607 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:51:56,607 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:51:56,612 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:51:57,070 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:51:58,612 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:51:59,073 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:52:00,264 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:52:00,264 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:52:00,264 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:52:01,010 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:52:01,011 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:52:01,012 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:52:01,017 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:52:01,074 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:52:03,077 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:52:04,018 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:52:04,514 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:52:04,515 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:52:04,515 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:52:04,520 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:52:05,078 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:52:07,081 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:52:08,547 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:52:08,548 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:52:08,548 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:52:08,553 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:52:09,083 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:52:09,553 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:52:11,086 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:52:11,994 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:52:11,995 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:52:11,996 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:52:12,000 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:52:12,087 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:52:15,001 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:52:15,088 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:52:15,264 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:52:15,264 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:52:15,264 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:52:17,119 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:52:17,120 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:52:17,120 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:52:17,125 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:52:18,090 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:52:19,092 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:52:20,126 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:52:20,205 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:52:20,206 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:52:20,206 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:52:20,211 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:52:21,093 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:52:23,096 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:52:23,728 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:52:23,729 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:52:23,730 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:52:23,734 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:52:24,097 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:52:25,099 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:52:25,735 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:52:26,265 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 16:52:27,399 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:52:27,400 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:52:27,400 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:52:27,405 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:52:28,101 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:52:29,103 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:52:30,264 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:52:30,264 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:52:30,264 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:52:30,691 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:52:30,692 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:52:30,692 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:52:30,697 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:52:31,105 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:52:31,698 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:52:33,108 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:52:35,371 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:52:35,372 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:52:35,372 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:52:35,377 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:52:36,110 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:52:37,112 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:52:37,378 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:52:41,532 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:52:41,533 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:52:41,533 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:52:41,538 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:52:42,115 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:52:42,539 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:52:43,116 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:52:45,264 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:52:45,264 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:52:45,264 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:52:46,862 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:52:46,863 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:52:46,863 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:52:46,868 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:52:47,118 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:52:47,868 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:52:49,121 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:52:51,229 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:52:51,230 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:52:51,231 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:52:51,236 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:52:52,123 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:52:53,123 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:52:53,237 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:52:55,590 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:52:55,591 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:52:55,591 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:52:55,596 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:52:56,126 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:52:56,270 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 16:52:57,129 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:52:58,272 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:53:00,264 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:53:00,264 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:53:00,265 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:53:01,825 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:53:01,826 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:53:01,826 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:53:01,831 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:53:02,131 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:53:03,134 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:53:03,870 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:53:06,317 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:53:06,318 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:53:06,318 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:53:06,323 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:53:07,136 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:53:09,139 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:53:09,323 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:53:14,324 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:53:14,694 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:53:14,695 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:53:14,695 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:53:14,700 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:53:15,143 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:53:15,264 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:53:15,264 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:53:15,265 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:53:17,146 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:53:19,477 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:53:20,834 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:53:20,835 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:53:20,835 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:53:20,840 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:53:21,148 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:53:23,151 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:53:24,840 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:53:26,150 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:53:26,151 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:53:26,151 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:53:26,156 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:53:26,269 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 16:53:27,154 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:53:29,157 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:53:29,644 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:53:29,645 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:53:29,645 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:53:29,650 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:53:30,157 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:53:30,264 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:53:30,264 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:53:30,265 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:53:30,536 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:53:31,160 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:53:33,414 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:53:33,414 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:53:33,415 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:53:33,424 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:53:34,162 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:53:35,164 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:53:36,425 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:53:36,982 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:53:36,983 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:53:36,983 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:53:36,988 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:53:37,167 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:53:39,170 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:53:41,283 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:53:41,284 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:53:41,284 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:53:41,289 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:53:42,172 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:53:42,290 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:53:43,174 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:53:44,792 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:53:44,792 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:53:44,793 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:53:44,797 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:53:45,175 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:53:45,264 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:53:45,265 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:53:45,265 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:53:47,178 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:53:47,491 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:53:48,047 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:53:48,048 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:53:48,048 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:53:48,053 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:53:48,179 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:53:51,176 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:53:51,177 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:53:51,177 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:53:51,182 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:53:51,184 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:53:51,184 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:53:53,183 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:53:53,185 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:53:56,271 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 16:53:58,273 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:53:58,775 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:53:58,776 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:53:58,776 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:53:58,781 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:53:59,189 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:54:00,264 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:54:00,265 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:54:00,265 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:54:01,192 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:54:02,276 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:54:02,280 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:54:02,281 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:54:02,285 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:54:03,193 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:54:03,286 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:54:05,196 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:54:05,926 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:54:05,927 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:54:05,927 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:54:05,932 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:54:06,197 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:54:07,200 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:54:08,933 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:54:09,153 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:54:09,154 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:54:09,154 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:54:09,159 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:54:09,201 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:54:11,204 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:54:12,603 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:54:12,604 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:54:12,604 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:54:12,609 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:54:13,206 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:54:14,610 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:54:15,209 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:54:15,264 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:54:15,265 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:54:15,265 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:54:17,632 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:54:17,632 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:54:17,633 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:54:17,637 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:54:18,210 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:54:19,213 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:54:19,674 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:54:20,975 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:54:20,976 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:54:20,976 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:54:20,982 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:54:21,214 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:54:23,215 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:54:24,982 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:54:26,273 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 16:54:30,026 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:54:30,027 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:54:30,027 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:54:30,028 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:54:30,033 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:54:30,219 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:54:30,265 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:54:30,265 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:54:30,265 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:54:33,223 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:54:33,398 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:54:33,399 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:54:33,399 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:54:33,404 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:54:34,224 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:54:35,224 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:54:35,405 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:54:36,510 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:54:36,511 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:54:36,512 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:54:36,517 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:54:37,227 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:54:39,229 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:54:39,814 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:54:39,815 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:54:39,815 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:54:39,821 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:54:40,230 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:54:40,821 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:54:41,232 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:54:45,265 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:54:45,265 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:54:45,265 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:54:46,547 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:54:47,971 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:54:47,971 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:54:47,972 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:54:47,976 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:54:48,236 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:54:51,240 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:54:52,015 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:54:56,276 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 16:54:56,529 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:54:56,530 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:54:56,530 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:54:56,535 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:54:57,244 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:54:57,536 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:54:59,248 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:54:59,895 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:54:59,896 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:54:59,896 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:54:59,901 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:55:00,248 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:55:00,265 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:55:00,265 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:55:00,265 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:55:01,251 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:55:02,561 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:55:03,976 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:55:03,977 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:55:03,977 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:55:03,982 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:55:04,252 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:55:05,255 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:55:07,222 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:55:07,223 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:55:07,223 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:55:07,228 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:55:07,256 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:55:08,228 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:55:09,259 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:55:11,410 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:55:11,410 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:55:11,411 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:55:11,415 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:55:12,261 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:55:13,262 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:55:13,416 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:55:15,265 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:55:15,265 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:55:15,265 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:55:17,372 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:55:17,372 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:55:17,373 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:55:17,377 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:55:18,265 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:55:19,267 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:55:19,378 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:55:21,004 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:55:21,005 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:55:21,005 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:55:21,010 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:55:21,268 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:55:23,271 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:55:24,517 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:55:24,518 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:55:24,518 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:55:24,518 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:55:24,523 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:55:25,273 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:55:26,278 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 16:55:27,275 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:55:28,968 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:55:28,969 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:55:28,969 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:55:28,974 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:55:29,277 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:55:29,975 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:55:30,265 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:55:30,265 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:55:30,265 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:55:31,280 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:55:33,151 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:55:33,152 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:55:33,152 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:55:33,157 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:55:33,281 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:55:35,158 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:55:35,282 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:55:36,662 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:55:36,663 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:55:36,663 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:55:36,668 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:55:37,285 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:55:39,288 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:55:40,669 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:55:45,088 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:55:45,089 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:55:45,089 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:55:45,094 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:55:45,265 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:55:45,265 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:55:45,266 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:55:45,291 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:55:46,513 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:55:47,294 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:55:48,542 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:55:48,542 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:55:48,543 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:55:48,547 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:55:49,295 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:55:51,298 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:55:51,548 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:55:54,191 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:55:54,191 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:55:54,192 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:55:54,197 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:55:54,301 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:55:56,280 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 16:55:57,281 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:55:57,305 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:55:57,921 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:55:57,922 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:55:57,922 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:55:57,927 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:55:58,306 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:55:59,308 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:56:00,265 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:56:00,265 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:56:00,266 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:56:01,549 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:56:01,550 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:56:01,551 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:56:01,555 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:56:02,310 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:56:02,556 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:56:03,312 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:56:05,236 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:56:05,237 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:56:05,237 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:56:05,242 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:56:05,313 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:56:07,317 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:56:08,243 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:56:10,360 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:56:10,360 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:56:10,361 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:56:10,366 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:56:11,319 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:56:13,322 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:56:13,367 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:56:15,265 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:56:15,265 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:56:15,266 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:56:18,521 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:56:19,014 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:56:19,015 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:56:19,016 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:56:19,021 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:56:19,325 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:56:21,328 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:56:24,022 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:56:24,286 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:56:24,287 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:56:24,287 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:56:24,292 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:56:24,330 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:56:26,285 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 16:56:27,334 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:56:27,720 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:56:27,721 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:56:27,721 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:56:27,726 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:56:28,334 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:56:29,335 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:56:29,727 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:56:30,265 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:56:30,265 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:56:30,266 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:56:31,456 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:56:31,457 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:56:31,457 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:56:31,462 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:56:32,337 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:56:33,340 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:56:34,996 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:56:34,997 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:56:34,997 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:56:34,997 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:56:35,002 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:56:35,341 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:56:37,351 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:56:38,105 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:56:38,105 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:56:38,106 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:56:38,111 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:56:38,352 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:56:39,354 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:56:40,112 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:56:42,690 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:56:42,691 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:56:42,691 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:56:42,696 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:56:43,357 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:56:45,265 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:56:45,266 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:56:45,266 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:56:45,358 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:56:45,539 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:56:45,564 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:56:45,564 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:56:45,564 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:56:45,569 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:56:46,358 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:56:47,361 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:56:50,570 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:56:50,640 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:56:50,641 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:56:50,641 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:56:50,646 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:56:51,363 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:56:53,367 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:56:55,682 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:56:56,284 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 16:56:56,558 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:56:56,559 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:56:56,560 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:56:56,565 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:56:57,370 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:56:59,373 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:57:00,266 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:57:00,266 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:57:00,266 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:57:01,544 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:57:02,274 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:57:02,275 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:57:02,275 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:57:02,280 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:57:02,375 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:57:05,379 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:57:06,057 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:57:06,058 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:57:06,058 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:57:06,063 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:57:06,379 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:57:07,063 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:57:07,380 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:57:09,305 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:57:09,306 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:57:09,306 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:57:09,312 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:57:09,381 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:57:11,385 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:57:12,312 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:57:12,326 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:57:12,327 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:57:12,327 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:57:12,332 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:57:12,385 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:57:15,266 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:57:15,266 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:57:15,266 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:57:15,389 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:57:16,811 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:57:16,812 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:57:16,813 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:57:16,818 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:57:17,390 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:57:17,818 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:57:19,393 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:57:21,146 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:57:21,147 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:57:21,147 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:57:21,152 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:57:21,394 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:57:23,152 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:57:23,395 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:57:25,266 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:57:25,267 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:57:25,267 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:57:25,272 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:57:25,396 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:57:26,289 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 16:57:27,400 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:57:28,290 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:57:30,266 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:57:30,266 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:57:30,266 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:57:33,515 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:57:33,708 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:57:33,708 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:57:33,709 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:57:33,714 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:57:34,404 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:57:35,406 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:57:38,715 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:57:42,021 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:57:42,021 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:57:42,022 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:57:42,027 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:57:42,412 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:57:43,414 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:57:44,028 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:57:45,266 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:57:45,266 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:57:45,266 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:57:47,061 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:57:47,062 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:57:47,062 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:57:47,067 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:57:47,417 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:57:49,067 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:57:49,420 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:57:50,670 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:57:50,671 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:57:50,672 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:57:50,677 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:57:51,421 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:57:53,424 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:57:53,853 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:57:53,853 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:57:53,854 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:57:53,859 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:57:54,426 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:57:54,859 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:57:55,428 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:57:56,288 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 16:57:56,924 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:57:56,925 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:57:56,925 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:57:56,930 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:57:57,429 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:57:59,432 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:57:59,931 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:58:00,266 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:58:00,266 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:58:00,266 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:58:00,952 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:58:00,953 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:58:00,953 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:58:00,958 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:58:01,433 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:58:03,436 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:58:04,994 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:58:09,387 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:58:09,388 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:58:09,388 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:58:09,393 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:58:09,440 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:58:10,394 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:58:11,443 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:58:12,630 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:58:12,631 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:58:12,631 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:58:12,637 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:58:13,444 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:58:15,266 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:58:15,266 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:58:15,266 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:58:15,448 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:58:15,494 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:58:20,495 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:58:20,785 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:58:20,786 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:58:20,786 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:58:20,791 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:58:21,451 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:58:23,454 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:58:24,920 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:58:24,921 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:58:24,921 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:58:24,927 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:58:25,455 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:58:25,927 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:58:26,293 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 16:58:27,458 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:58:30,266 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:58:30,266 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:58:30,267 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:58:31,502 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:58:33,468 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:58:33,469 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:58:33,469 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:58:33,474 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:58:34,466 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:58:35,468 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:58:36,821 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:58:36,822 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:58:36,822 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:58:36,822 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:58:36,827 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:58:37,471 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:58:39,474 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:58:40,204 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:58:40,204 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:58:40,205 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:58:40,210 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:58:40,474 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:58:41,477 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:58:42,211 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:58:44,067 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:58:44,068 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:58:44,068 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:58:44,073 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:58:44,479 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:58:45,266 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:58:45,266 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:58:45,267 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:58:45,481 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:58:47,486 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:58:47,907 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:58:47,908 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:58:47,909 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:58:47,914 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:58:48,483 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:58:49,488 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:58:51,456 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:58:51,457 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:58:51,457 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:58:51,462 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:58:51,489 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:58:53,463 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:58:53,490 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:58:56,292 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 16:58:59,294 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:58:59,804 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:58:59,805 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:58:59,805 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:58:59,811 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:59:00,266 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:59:00,266 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:59:00,267 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:59:00,495 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:59:01,498 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:59:04,474 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:59:05,001 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:59:05,002 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:59:05,002 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:59:05,007 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:59:05,500 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:59:07,503 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:59:08,263 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:59:08,263 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:59:08,264 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:59:08,269 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:59:08,504 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:59:09,506 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:59:10,269 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:59:12,360 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:59:12,360 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:59:12,361 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:59:12,367 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:59:12,508 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:59:15,266 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:59:15,267 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:59:15,267 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:59:15,510 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:59:15,561 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:59:15,637 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:59:15,637 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:59:15,638 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:59:15,643 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:59:16,511 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:59:17,514 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:59:18,909 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:59:18,910 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:59:18,911 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:59:18,916 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:59:19,515 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:59:20,916 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:59:21,518 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:59:22,387 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:59:22,387 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:59:22,388 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:59:22,393 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:59:22,518 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:59:25,522 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:59:25,570 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:59:25,570 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:59:25,571 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:59:25,576 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:59:26,294 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 16:59:26,296 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:59:26,523 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:59:27,525 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:59:30,266 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:59:30,267 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:59:30,267 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:59:31,511 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:59:32,640 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:59:32,641 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:59:32,641 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:59:32,646 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:59:33,529 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:59:35,532 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:59:36,647 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:59:36,703 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:59:36,703 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:59:36,704 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:59:36,709 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:59:37,534 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:59:39,538 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:59:40,430 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:59:40,430 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:59:40,431 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:59:40,436 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:59:40,538 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:59:42,437 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:59:43,542 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:59:43,712 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:59:43,713 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:59:43,713 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:59:43,718 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:59:44,542 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:59:45,267 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:59:45,267 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:59:45,267 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:59:45,545 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:59:46,971 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:59:46,972 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:59:46,972 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:59:46,978 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:59:47,546 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:59:47,978 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:59:49,551 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:59:50,454 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:59:50,455 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:59:50,456 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:59:50,460 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:59:50,551 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:59:53,461 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:59:53,553 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:59:54,793 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:59:54,794 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:59:54,794 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:59:54,799 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:59:55,555 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 16:59:56,296 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 16:59:57,558 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 16:59:59,123 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:59:59,123 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 16:59:59,124 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:59:59,124 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:59:59,129 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:59:59,560 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:00:00,267 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:00:00,267 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:00:00,267 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:00:01,563 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:00:03,232 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:00:03,232 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:00:03,233 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:00:03,238 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:00:03,564 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:00:04,238 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:00:05,567 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:00:07,606 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:00:07,607 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:00:07,607 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:00:07,612 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:00:08,569 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:00:09,571 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:00:09,612 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:00:12,106 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:00:12,107 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:00:12,107 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:00:12,113 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:00:12,573 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:00:13,576 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:00:15,113 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:00:15,267 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:00:15,267 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:00:15,267 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:00:15,587 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:00:15,587 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:00:15,588 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:00:15,592 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:00:16,578 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:00:17,579 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:00:20,630 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:00:24,063 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:00:24,064 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:00:24,064 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:00:24,069 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:00:24,583 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:00:25,586 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:00:26,069 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:00:26,298 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 17:00:30,267 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:00:30,267 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:00:30,267 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:00:31,067 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:00:31,067 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:00:31,068 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:00:31,072 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:00:31,073 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:00:31,589 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:00:33,592 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:00:35,002 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:00:35,003 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:00:35,003 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:00:35,008 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:00:35,593 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:00:37,009 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:00:37,598 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:00:38,366 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:00:38,367 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:00:38,367 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:00:38,372 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:00:38,598 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:00:39,601 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:00:42,373 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:00:43,901 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:00:43,902 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:00:43,902 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:00:43,908 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:00:44,604 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:00:45,267 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:00:45,267 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:00:45,267 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:00:45,606 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:00:47,487 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:00:47,522 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:00:47,522 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:00:47,523 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:00:47,528 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:00:47,607 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:00:49,611 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:00:51,273 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:00:51,277 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:00:51,277 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:00:51,282 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:00:51,612 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:00:53,282 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:00:53,619 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:00:54,521 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:00:54,522 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:00:54,522 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:00:54,527 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:00:54,620 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:00:56,301 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 17:00:57,624 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:00:57,936 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:00:57,937 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:00:57,937 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:00:57,942 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:00:58,625 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:00:58,943 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:00:59,627 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:01:00,267 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:01:00,267 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:01:00,267 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:01:02,085 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:01:02,086 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:01:02,086 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:01:02,092 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:01:02,629 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:01:03,629 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:01:04,092 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:01:05,451 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:01:05,451 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:01:05,452 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:01:05,457 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:01:05,630 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:01:07,633 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:01:09,457 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:01:09,777 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:01:09,778 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:01:09,778 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:01:09,783 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:01:10,635 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:01:11,638 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:01:14,055 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:01:14,055 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:01:14,056 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:01:14,062 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:01:14,640 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:01:15,062 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:01:15,267 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:01:15,267 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:01:15,268 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:01:15,642 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:01:17,422 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:01:17,423 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:01:17,423 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:01:17,428 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:01:17,644 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:01:19,646 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:01:20,429 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:01:23,227 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:01:23,228 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:01:23,228 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:01:23,233 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:01:23,649 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:01:25,650 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:01:26,233 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:01:26,303 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 17:01:28,669 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:01:28,670 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:01:28,670 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:01:28,675 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:01:29,652 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:01:30,267 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:01:30,267 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:01:30,268 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:01:31,545 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:01:31,654 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:01:36,546 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:01:37,167 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:01:37,168 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:01:37,168 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:01:37,173 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:01:37,659 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:01:39,662 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:01:42,174 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:01:45,267 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:01:45,267 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:01:45,268 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:01:45,828 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:01:45,829 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:01:45,829 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:01:45,835 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:01:46,666 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:01:47,668 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:01:47,870 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:01:50,860 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:01:50,861 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:01:50,861 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:01:50,866 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:01:51,670 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:01:53,674 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:01:53,867 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:01:54,533 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:01:54,534 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:01:54,534 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:01:54,539 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:01:54,676 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:01:56,308 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 17:01:57,680 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:01:58,813 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:01:58,813 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:01:58,814 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:01:58,819 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:01:59,681 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:01:59,819 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:02:00,267 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:02:00,267 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:02:00,268 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:02:01,682 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:02:02,724 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:02:02,724 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:02:02,725 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:02:02,730 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:02:03,683 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:02:05,686 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:02:05,731 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:02:07,995 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:02:07,996 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:02:07,996 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:02:08,001 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:02:08,688 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:02:09,690 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:02:11,002 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:02:12,036 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:02:12,037 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:02:12,037 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:02:12,042 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:02:12,692 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:02:13,693 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:02:15,267 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:02:15,268 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:02:15,268 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:02:16,492 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:02:18,685 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:02:18,686 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:02:18,686 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:02:18,692 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:02:18,696 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:02:21,692 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:02:21,699 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:02:22,367 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:02:22,368 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:02:22,369 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:02:22,374 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:02:22,700 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:02:23,703 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:02:26,307 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 17:02:27,145 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:02:27,146 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:02:27,146 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:02:27,146 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:02:27,151 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:02:27,705 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:02:29,708 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:02:30,094 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:02:30,095 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:02:30,095 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:02:30,100 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:02:30,268 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:02:30,268 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:02:30,268 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:02:30,709 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:02:31,709 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:02:32,494 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:02:33,632 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:02:33,633 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:02:33,633 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:02:33,638 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:02:33,711 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:02:35,714 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:02:37,639 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:02:39,171 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:02:39,172 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:02:39,172 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:02:39,177 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:02:39,718 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:02:41,721 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:02:42,719 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:02:42,720 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:02:42,720 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:02:42,720 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:02:42,725 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:02:43,722 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:02:45,268 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:02:45,268 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:02:45,268 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:02:45,725 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:02:48,509 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:02:51,308 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:02:51,309 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:02:51,309 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:02:51,315 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:02:51,728 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:02:53,731 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:02:54,315 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:02:55,643 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:02:55,644 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:02:55,644 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:02:55,649 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:02:55,734 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:02:56,312 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 17:02:57,737 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:03:00,131 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:03:00,132 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:03:00,132 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:03:00,132 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:03:00,137 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:03:00,268 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:03:00,268 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:03:00,268 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:03:00,739 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:03:01,741 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:03:03,909 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:03:03,909 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:03:03,910 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:03:03,915 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:03:04,743 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:03:05,746 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:03:05,916 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:03:07,956 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:03:07,957 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:03:07,957 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:03:07,962 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:03:08,748 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:03:09,748 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:03:10,963 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:03:14,384 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:03:14,385 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:03:14,385 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:03:14,391 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:03:14,751 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:03:15,268 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:03:15,268 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:03:15,268 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:03:15,754 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:03:16,520 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:03:19,492 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:03:19,492 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:03:19,493 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:03:19,498 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:03:19,756 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:03:21,535 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:03:21,759 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:03:25,940 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:03:25,941 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:03:25,941 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:03:25,947 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:03:26,311 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 17:03:26,762 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:03:27,312 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:03:27,765 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:03:29,816 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:03:29,817 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:03:29,818 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:03:29,823 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:03:30,268 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:03:30,268 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:03:30,268 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:03:30,766 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:03:31,767 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:03:32,524 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:03:33,295 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:03:33,295 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:03:33,296 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:03:33,301 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:03:33,768 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:03:35,772 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:03:36,804 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:03:36,805 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:03:36,805 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:03:36,810 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:03:37,774 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:03:37,810 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:03:39,777 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:03:42,811 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:03:45,268 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:03:45,268 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:03:45,268 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:03:45,502 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:03:45,572 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:03:45,572 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:03:45,577 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:03:45,781 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:03:47,784 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:03:48,578 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:03:48,863 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:03:48,863 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:03:48,864 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:03:48,869 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:03:49,785 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:03:51,789 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:03:52,615 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:03:52,616 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:03:52,616 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:03:52,621 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:03:52,789 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:03:53,622 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:03:55,794 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:03:56,168 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:03:56,169 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:03:56,169 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:03:56,175 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:03:56,316 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 17:03:56,795 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:03:57,797 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:03:59,318 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:04:00,268 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:04:00,268 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:04:00,268 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:04:01,190 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:04:01,191 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:04:01,191 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:04:01,197 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:04:01,800 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:04:03,803 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:04:04,430 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:04:04,431 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:04:04,432 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:04:04,432 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:04:04,437 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:04:04,804 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:04:05,804 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:04:08,303 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:04:08,303 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:04:08,304 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:04:08,309 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:04:08,806 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:04:09,809 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:04:10,309 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:04:11,961 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:04:11,962 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:04:11,962 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:04:11,967 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:04:12,811 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:04:13,813 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:04:15,268 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:04:15,268 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:04:15,269 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:04:15,541 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:04:17,886 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:04:17,887 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:04:17,888 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:04:17,893 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:04:18,817 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:04:19,819 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:04:20,893 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:04:22,898 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:04:22,899 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:04:22,899 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:04:22,905 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:04:23,822 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:04:25,825 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:04:25,905 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:04:26,315 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 17:04:26,910 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:04:26,911 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:04:26,911 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:04:26,917 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:04:27,826 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:04:29,827 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:04:30,268 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:04:30,268 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:04:30,269 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:04:31,486 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:04:31,646 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:04:31,647 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:04:31,648 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:04:31,653 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:04:31,828 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:04:33,832 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:04:35,178 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:04:35,179 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:04:35,179 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:04:35,184 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:04:35,833 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:04:37,184 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:04:37,838 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:04:38,646 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:04:38,647 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:04:38,647 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:04:38,652 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:04:38,838 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:04:39,841 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:04:42,653 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:04:42,930 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:04:42,931 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:04:42,931 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:04:42,936 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:04:43,843 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:04:45,268 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:04:45,268 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:04:45,269 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:04:45,846 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:04:46,341 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:04:46,342 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:04:46,342 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:04:46,347 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:04:46,847 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:04:47,848 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:04:48,347 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:04:51,501 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:04:51,502 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:04:51,503 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:04:51,507 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:04:51,850 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:04:53,508 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:04:53,851 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:04:56,317 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 17:04:56,879 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:04:56,879 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:04:56,880 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:04:56,885 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:04:57,855 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:04:58,885 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:04:59,858 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:05:00,268 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:05:00,269 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:05:00,269 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:05:00,476 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:05:00,580 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:05:00,580 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:05:00,585 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:05:00,859 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:05:01,861 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:05:03,957 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:05:03,958 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:05:03,958 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:05:03,959 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:05:03,964 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:05:04,863 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:05:05,866 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:05:07,454 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:05:07,454 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:05:07,454 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:05:07,459 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:05:07,867 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:05:09,460 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:05:09,870 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:05:10,881 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:05:10,882 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:05:10,882 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:05:10,887 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:05:11,871 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:05:13,874 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:05:14,166 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:05:14,167 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:05:14,167 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:05:14,172 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:05:14,875 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:05:15,172 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:05:15,269 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:05:15,269 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:05:15,269 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:05:15,878 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:05:20,218 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:05:20,219 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:05:20,220 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:05:20,220 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:05:20,225 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:05:20,881 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:05:21,883 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:05:24,627 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:05:24,628 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:05:24,628 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:05:24,633 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:05:24,885 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:05:25,633 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:05:25,886 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:05:26,319 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 17:05:30,269 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:05:30,269 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:05:30,269 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:05:31,547 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:05:33,006 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:05:33,007 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:05:33,007 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:05:33,012 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:05:33,891 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:05:35,894 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:05:36,568 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:05:36,569 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:05:36,569 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:05:36,569 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:05:36,574 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:05:36,895 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:05:37,899 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:05:40,102 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:05:40,103 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:05:40,103 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:05:40,108 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:05:40,901 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:05:41,904 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:05:42,109 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:05:43,335 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:05:43,336 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:05:43,336 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:05:43,342 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:05:43,905 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:05:45,269 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:05:45,269 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:05:45,269 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:05:45,908 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:05:47,495 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:05:48,538 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:05:48,538 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:05:48,538 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:05:48,544 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:05:48,910 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:05:49,912 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:05:52,583 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:05:53,068 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:05:53,069 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:05:53,069 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:05:53,074 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:05:53,915 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:05:55,919 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:05:56,321 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 17:05:57,149 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:05:57,150 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:05:57,150 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:05:57,155 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:05:57,920 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:05:58,156 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:05:59,924 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:06:00,269 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:06:00,269 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:06:00,269 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:06:02,141 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:06:02,142 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:06:02,142 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:06:02,148 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:06:02,925 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:06:03,926 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:06:04,148 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:06:06,772 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:06:06,773 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:06:06,773 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:06:06,779 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:06:06,928 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:06:09,779 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:06:09,930 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:06:10,260 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:06:10,261 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:06:10,261 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:06:10,266 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:06:10,931 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:06:11,935 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:06:15,267 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:06:15,269 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:06:15,269 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:06:15,269 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:06:15,543 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:06:15,565 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:06:15,566 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:06:15,571 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:06:15,937 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:06:17,940 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:06:19,097 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:06:19,101 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:06:19,101 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:06:19,106 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:06:19,942 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:06:21,142 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:06:21,945 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:06:22,186 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:06:22,186 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:06:22,187 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:06:22,192 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:06:22,946 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:06:23,948 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:06:26,120 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:06:26,120 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:06:26,121 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:06:26,126 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:06:26,324 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 17:06:26,326 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:06:26,950 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:06:27,953 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:06:30,079 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:06:30,080 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:06:30,080 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:06:30,085 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:06:30,269 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:06:30,269 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:06:30,269 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:06:30,955 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:06:31,576 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:06:31,955 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:06:35,081 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:06:35,081 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:06:35,082 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:06:35,087 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:06:35,958 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:06:37,087 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:06:37,963 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:06:38,623 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:06:38,624 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:06:38,624 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:06:38,629 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:06:38,963 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:06:39,966 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:06:42,139 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:06:42,139 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:06:42,140 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:06:42,140 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:06:42,145 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:06:42,968 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:06:43,970 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:06:45,269 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:06:45,269 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:06:45,269 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:06:46,006 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:06:46,007 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:06:46,007 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:06:46,013 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:06:46,972 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:06:47,975 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:06:48,013 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:06:53,014 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:06:54,377 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:06:54,378 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:06:54,378 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:06:54,383 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:06:54,980 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:06:55,982 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:06:56,326 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 17:06:57,875 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:06:57,876 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:06:57,876 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:06:57,881 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:06:57,983 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:06:58,882 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:06:59,987 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:07:00,269 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:07:00,269 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:07:00,270 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:07:00,915 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:07:00,915 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:07:00,916 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:07:00,921 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:07:00,987 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:07:03,921 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:07:03,989 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:07:04,070 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:07:04,070 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:07:04,071 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:07:04,075 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:07:04,990 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:07:05,992 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:07:07,592 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:07:07,593 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:07:07,593 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:07:07,598 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:07:07,994 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:07:09,598 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:07:09,997 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:07:14,599 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:07:15,269 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:07:15,269 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:07:15,270 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:07:16,139 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:07:16,140 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:07:16,140 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:07:16,145 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:07:17,001 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:07:18,004 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:07:20,047 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:07:20,048 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:07:20,048 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:07:20,048 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:07:20,053 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:07:21,005 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:07:22,008 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:07:23,380 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:07:23,381 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:07:23,381 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:07:23,387 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:07:24,009 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:07:25,387 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:07:26,011 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:07:26,331 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 17:07:26,708 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:07:26,708 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:07:26,708 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:07:26,714 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:07:27,011 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:07:28,014 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:07:29,987 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:07:29,988 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:07:29,988 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:07:29,993 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:07:30,015 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:07:30,269 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:07:30,269 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:07:30,270 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:07:30,495 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:07:32,018 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:07:33,603 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:07:33,603 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:07:33,603 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:07:33,608 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:07:34,020 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:07:35,609 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:07:36,023 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:07:37,780 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:07:37,781 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:07:37,781 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:07:37,786 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:07:38,025 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:07:40,028 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:07:40,786 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:07:41,112 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:07:41,112 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:07:41,113 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:07:41,118 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:07:42,030 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:07:44,033 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:07:45,269 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:07:45,270 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:07:45,270 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:07:46,538 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:07:46,918 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:07:46,919 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:07:46,919 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:07:46,924 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:07:47,034 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:07:50,038 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:07:51,925 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:07:55,490 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:07:55,491 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:07:55,491 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:07:55,496 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:07:56,043 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:07:56,330 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 17:07:57,331 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:07:58,046 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:07:59,625 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:07:59,626 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:07:59,626 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:07:59,631 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:08:00,047 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:08:00,270 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:08:00,270 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:08:00,270 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:08:02,050 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:08:02,519 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:08:02,999 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:08:03,000 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:08:03,000 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:08:03,004 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:08:03,051 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:08:06,054 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:08:06,763 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:08:06,763 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:08:06,764 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:08:06,768 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:08:07,055 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:08:07,769 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:08:08,057 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:08:12,769 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:08:15,270 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:08:15,270 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:08:15,270 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:08:15,314 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:08:15,554 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:08:15,555 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:08:15,560 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:08:16,062 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:08:18,063 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:08:18,560 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:08:20,079 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:08:20,080 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:08:20,080 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:08:20,085 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:08:21,065 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:08:22,068 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:08:24,085 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:08:25,271 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:08:25,271 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:08:25,272 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:08:25,276 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:08:26,070 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:08:26,335 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 17:08:28,073 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:08:28,797 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:08:28,798 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:08:28,798 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:08:28,803 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:08:29,074 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:08:29,803 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:08:30,076 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:08:30,270 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:08:30,270 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:08:30,270 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:08:31,943 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:08:31,944 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:08:31,944 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:08:31,949 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:08:32,078 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:08:34,081 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:08:34,950 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:08:35,634 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:08:35,635 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:08:35,635 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:08:35,640 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:08:36,082 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:08:38,087 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:08:39,168 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:08:39,169 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:08:39,169 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:08:39,174 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:08:40,088 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:08:40,174 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:08:42,091 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:08:44,230 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:08:44,231 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:08:44,231 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:08:44,235 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:08:45,093 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:08:45,236 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:08:45,270 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:08:45,270 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:08:45,270 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:08:46,094 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:08:49,489 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:08:49,490 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:08:49,491 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:08:49,496 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:08:50,096 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:08:50,496 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:08:52,099 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:08:55,496 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:08:56,334 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 17:08:57,797 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:08:57,798 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:08:57,798 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:08:57,803 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:08:58,103 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:09:00,106 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:09:00,270 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:09:00,270 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:09:00,270 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:09:00,499 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:09:01,019 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:09:01,020 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:09:01,020 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:09:01,025 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:09:01,107 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:09:04,110 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:09:06,025 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:09:07,078 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:09:07,079 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:09:07,079 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:09:07,084 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:09:07,112 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:09:10,116 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:09:10,506 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:09:10,506 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:09:10,507 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:09:10,511 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:09:11,117 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:09:11,512 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:09:12,119 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:09:15,270 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:09:15,270 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:09:15,270 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:09:17,500 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:09:18,975 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:09:18,976 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:09:18,976 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:09:18,981 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:09:19,123 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:09:22,126 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:09:23,018 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:09:24,740 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:09:24,741 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:09:24,741 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:09:24,746 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:09:25,128 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:09:26,130 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:09:26,338 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 17:09:27,581 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:09:27,582 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:09:27,582 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:09:27,587 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:09:28,132 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:09:28,587 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:09:30,135 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:09:30,270 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:09:30,270 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:09:30,270 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:09:32,534 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:09:32,535 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:09:32,535 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:09:32,541 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:09:33,136 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:09:34,139 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:09:34,542 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:09:39,542 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:09:41,069 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:09:41,070 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:09:41,070 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:09:41,075 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:09:41,145 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:09:44,148 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:09:44,433 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:09:44,433 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:09:44,434 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:09:44,438 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:09:45,149 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:09:45,270 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:09:45,270 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:09:45,271 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:09:45,544 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:09:46,151 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:09:47,964 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:09:47,964 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:09:47,965 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:09:47,969 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:09:48,152 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:09:50,156 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:09:50,970 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:09:50,981 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:09:50,981 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:09:50,982 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:09:50,987 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:09:51,157 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:09:52,159 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:09:55,078 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:09:55,079 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:09:55,079 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:09:55,085 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:09:55,162 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:09:56,086 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:09:56,338 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 17:09:58,166 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:10:00,270 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:10:00,270 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:10:00,271 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:10:00,420 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:10:00,574 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:10:00,574 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:10:00,579 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:10:01,167 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:10:01,580 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:10:02,168 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:10:03,562 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:10:03,563 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:10:03,563 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:10:03,568 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:10:04,169 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:10:06,172 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:10:07,290 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:10:07,291 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:10:07,291 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:10:07,291 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:10:07,296 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:10:08,173 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:10:10,177 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:10:12,297 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:10:15,270 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:10:15,270 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:10:15,271 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:10:16,668 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:10:16,668 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:10:16,669 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:10:16,674 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:10:17,181 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:10:17,710 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:10:18,183 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:10:22,711 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:10:25,709 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:10:25,710 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:10:25,711 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:10:25,715 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:10:26,188 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:10:26,340 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 17:10:28,189 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:10:28,342 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:10:28,780 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:10:28,781 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:10:28,781 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:10:28,786 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:10:29,189 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:10:30,192 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:10:30,270 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:10:30,270 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:10:30,271 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:10:32,262 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:10:32,263 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:10:32,263 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:10:32,269 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:10:33,194 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:10:34,194 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:10:34,269 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:10:36,376 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:10:36,377 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:10:36,377 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:10:36,382 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:10:37,196 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:10:38,200 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:10:39,383 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:10:39,741 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:10:39,742 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:10:39,743 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:10:39,747 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:10:40,201 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:10:42,204 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:10:44,388 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:10:44,389 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:10:44,389 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:10:44,389 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:10:44,394 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:10:45,206 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:10:45,270 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:10:45,271 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:10:45,271 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:10:46,208 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:10:47,219 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:10:47,220 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:10:47,220 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:10:47,225 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:10:48,209 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:10:50,211 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:10:50,226 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:10:50,391 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:10:50,392 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:10:50,393 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:10:50,398 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:10:51,212 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:10:52,214 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:10:55,399 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:10:56,342 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 17:10:59,014 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:10:59,015 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:10:59,015 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:10:59,020 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:10:59,219 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:11:00,222 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:11:00,270 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:11:00,271 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:11:00,271 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:11:00,538 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:11:02,330 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:11:02,331 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:11:02,331 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:11:02,337 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:11:03,223 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:11:04,226 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:11:06,338 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:11:08,376 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:11:08,377 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:11:08,377 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:11:08,382 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:11:09,229 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:11:10,232 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:11:11,383 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:11:13,711 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:11:13,712 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:11:13,712 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:11:13,717 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:11:14,235 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:11:15,271 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:11:15,271 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:11:15,271 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:11:16,238 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:11:16,566 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:11:17,845 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:11:17,846 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:11:17,846 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:11:17,851 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:11:18,239 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:11:20,242 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:11:21,852 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:11:21,924 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:11:21,925 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:11:21,925 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:11:21,930 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:11:22,243 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:11:24,246 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:11:26,344 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 17:11:27,346 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:11:27,387 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:11:27,388 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:11:27,388 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:11:27,393 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:11:28,248 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:11:30,251 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:11:30,271 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:11:30,271 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:11:30,271 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:11:32,350 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:11:32,351 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:11:32,351 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:11:32,352 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:11:32,357 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:11:33,253 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:11:34,256 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:11:37,358 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:11:37,500 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:11:37,501 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:11:37,501 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:11:37,506 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:11:38,260 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:11:40,263 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:11:42,507 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:11:43,108 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:11:43,109 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:11:43,109 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:11:43,114 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:11:43,264 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:11:44,265 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:11:45,271 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:11:45,271 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:11:45,271 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:11:47,514 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:11:47,515 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:11:47,515 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:11:47,515 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:11:47,520 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:11:48,267 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:11:50,270 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:11:51,702 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:11:51,703 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:11:51,703 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:11:51,708 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:11:52,271 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:11:52,708 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:11:54,275 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:11:54,968 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:11:54,972 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:11:54,972 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:11:54,977 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:11:55,276 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:11:56,279 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:11:56,346 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 17:11:58,348 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:11:58,690 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:11:58,691 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:11:58,691 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:11:58,696 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:11:59,281 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:12:00,271 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:12:00,271 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:12:00,271 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:12:00,281 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:12:03,356 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:12:03,357 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:12:03,357 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:12:03,358 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:12:03,363 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:12:04,283 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:12:06,284 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:12:07,710 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:12:07,711 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:12:07,711 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:12:07,716 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:12:08,286 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:12:08,716 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:12:10,289 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:12:11,214 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:12:11,215 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:12:11,215 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:12:11,220 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:12:11,289 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:12:14,221 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:12:14,293 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:12:15,271 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:12:15,271 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:12:15,271 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:12:15,379 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:12:15,548 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:12:15,548 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:12:15,553 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:12:16,294 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:12:18,297 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:12:19,554 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:12:19,788 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:12:19,789 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:12:19,789 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:12:19,794 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:12:20,298 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:12:22,301 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:12:24,730 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:12:24,731 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:12:24,731 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:12:24,731 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:12:24,737 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:12:25,303 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:12:26,305 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:12:26,348 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 17:12:29,503 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:12:29,503 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:12:29,504 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:12:29,509 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:12:30,271 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:12:30,271 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:12:30,271 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:12:30,308 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:12:30,491 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:12:32,311 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:12:33,033 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:12:33,033 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:12:33,033 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:12:33,039 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:12:33,311 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:12:34,314 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:12:36,075 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:12:38,642 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:12:38,643 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:12:38,643 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:12:38,648 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:12:39,318 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:12:40,321 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:12:41,648 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:12:41,860 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:12:41,861 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:12:41,861 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:12:41,866 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:12:42,322 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:12:44,325 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:12:45,271 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:12:45,271 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:12:45,271 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:12:45,768 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:12:45,769 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:12:45,770 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:12:45,774 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:12:46,326 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:12:46,775 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:12:48,329 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:12:50,266 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:12:50,266 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:12:50,267 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:12:50,271 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:12:50,330 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:12:52,272 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:12:52,332 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:12:55,468 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:12:55,468 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:12:55,469 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:12:55,474 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:12:56,335 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:12:56,354 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 17:12:57,355 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:12:58,336 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:12:58,818 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:12:58,819 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:12:58,819 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:12:58,824 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:12:59,337 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:13:00,271 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:13:00,271 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:13:00,272 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:13:00,339 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:13:02,569 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:13:05,774 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:13:05,775 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:13:05,775 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:13:05,779 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:13:06,343 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:13:07,780 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:13:08,346 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:13:10,398 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:13:10,398 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:13:10,399 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:13:10,404 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:13:11,348 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:13:12,351 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:13:13,404 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:13:13,820 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:13:13,821 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:13:13,821 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:13:13,826 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:13:14,352 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:13:15,271 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:13:15,271 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:13:15,272 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:13:16,355 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:13:17,153 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:13:17,154 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:13:17,154 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:13:17,159 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:13:17,355 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:13:18,358 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:13:19,160 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:13:20,411 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:13:20,412 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:13:20,412 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:13:20,417 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:13:21,360 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:13:22,361 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:13:23,936 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:13:23,937 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:13:23,937 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:13:23,943 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:13:24,362 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:13:24,943 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:13:26,353 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 17:13:26,365 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:13:28,898 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:13:28,899 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:13:28,899 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:13:28,904 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:13:29,366 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:13:30,271 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:13:30,271 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:13:30,272 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:13:30,369 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:13:30,501 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:13:32,379 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:13:32,380 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:13:32,380 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:13:32,385 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:13:33,371 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:13:34,372 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:13:36,386 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:13:37,945 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:13:37,946 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:13:37,946 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:13:37,951 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:13:38,375 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:13:40,378 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:13:41,952 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:13:42,829 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:13:42,830 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:13:42,830 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:13:42,835 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:13:43,380 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:13:44,383 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:13:45,271 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:13:45,272 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:13:45,272 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:13:47,564 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:13:48,729 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:13:48,729 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:13:48,730 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:13:48,735 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:13:49,385 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:13:50,389 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:13:52,350 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:13:52,351 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:13:52,352 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:13:52,357 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:13:52,390 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:13:53,357 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:13:54,394 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:13:56,299 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:13:56,300 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:13:56,300 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:13:56,305 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:13:56,358 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 17:13:56,396 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:13:58,359 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:13:58,397 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:13:59,679 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:13:59,680 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:13:59,680 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:13:59,685 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:14:00,272 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:14:00,272 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:14:00,272 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:14:00,398 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:14:02,401 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:14:03,527 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:14:04,903 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:14:04,904 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:14:04,904 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:14:04,910 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:14:05,403 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:14:06,405 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:14:08,760 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:14:08,761 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:14:08,761 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:14:08,762 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:14:08,767 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:14:09,407 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:14:10,410 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:14:12,093 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:14:12,093 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:14:12,094 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:14:12,099 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:14:12,411 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:14:14,099 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:14:14,412 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:14:15,272 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:14:15,272 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:14:15,272 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:14:16,612 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:14:16,613 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:14:16,613 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:14:16,618 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:14:17,414 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:14:18,417 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:14:19,619 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:14:22,843 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:14:22,844 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:14:22,844 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:14:22,849 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:14:23,419 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:14:24,422 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:14:24,850 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:14:26,357 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 17:14:28,193 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:14:28,194 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:14:28,194 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:14:28,199 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:14:28,424 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:14:30,200 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:14:30,272 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:14:30,272 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:14:30,272 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:14:30,425 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:14:31,668 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:14:31,669 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:14:31,669 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:14:31,675 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:14:32,426 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:14:34,429 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:14:34,429 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:14:34,430 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:14:34,430 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:14:34,435 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:14:35,430 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:14:35,435 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:14:36,433 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:14:40,436 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:14:42,269 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:14:42,269 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:14:42,270 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:14:42,275 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:14:42,438 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:14:44,441 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:14:45,272 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:14:45,272 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:14:45,272 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:14:45,550 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:14:49,960 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:14:49,960 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:14:49,961 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:14:49,966 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:14:50,444 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:14:50,966 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:14:52,447 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:14:54,378 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:14:54,379 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:14:54,379 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:14:54,384 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:14:54,448 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:14:56,362 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 17:14:56,363 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:14:56,453 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:14:58,990 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:14:58,991 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:14:58,991 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:14:58,996 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:14:59,454 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:15:00,272 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:15:00,272 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:15:00,272 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:15:00,457 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:15:01,508 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:15:04,215 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:15:04,216 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:15:04,216 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:15:04,221 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:15:04,459 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:15:06,462 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:15:07,222 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:15:07,906 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:15:07,907 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:15:07,907 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:15:07,912 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:15:08,463 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:15:10,466 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:15:11,817 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:15:11,817 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:15:11,818 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:15:11,823 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:15:12,468 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:15:12,823 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:15:14,471 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:15:15,272 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:15:15,272 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:15:15,272 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:15:15,554 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:15:15,554 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:15:15,555 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:15:15,560 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:15:16,472 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:15:18,475 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:15:18,560 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:15:18,981 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:15:18,982 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:15:18,982 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:15:18,987 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:15:19,476 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:15:20,478 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:15:23,988 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:15:24,333 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:15:24,334 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:15:24,334 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:15:24,339 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:15:24,481 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:15:26,361 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 17:15:26,483 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:15:27,816 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:15:27,816 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:15:27,817 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:15:27,822 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:15:28,485 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:15:29,822 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:15:30,272 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:15:30,272 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:15:30,273 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:15:30,488 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:15:32,172 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:15:32,173 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:15:32,173 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:15:32,178 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:15:32,489 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:15:34,492 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:15:35,179 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:15:35,475 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:15:35,476 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:15:35,476 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:15:35,481 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:15:35,493 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:15:38,498 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:15:40,380 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:15:40,380 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:15:40,381 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:15:40,381 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:15:40,386 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:15:40,499 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:15:42,502 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:15:45,272 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:15:45,272 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:15:45,273 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:15:45,498 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:15:48,834 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:15:48,835 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:15:48,836 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:15:48,840 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:15:49,506 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:15:50,509 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:15:50,841 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:15:51,728 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:15:51,729 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:15:51,729 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:15:51,734 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:15:52,510 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:15:54,511 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:15:55,284 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:15:55,284 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:15:55,285 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:15:55,289 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:15:55,514 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:15:56,290 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:15:56,363 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 17:15:56,517 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:16:00,199 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:16:00,199 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:16:00,200 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:16:00,204 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:16:00,272 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:16:00,272 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:16:00,273 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:16:00,519 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:16:01,571 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:16:02,522 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:16:06,572 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:16:08,737 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:16:08,738 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:16:08,738 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:16:08,743 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:16:09,526 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:16:10,528 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:16:11,743 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:16:15,272 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:16:15,272 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:16:15,273 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:16:17,483 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:16:18,841 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:16:18,842 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:16:18,842 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:16:18,847 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:16:19,534 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:16:20,535 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:16:22,135 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:16:22,136 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:16:22,136 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:16:22,141 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:16:22,536 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:16:23,141 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:16:24,539 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:16:26,365 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 17:16:28,004 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:16:28,005 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:16:28,005 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:16:28,011 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:16:28,541 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:16:29,012 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:16:30,272 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:16:30,273 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:16:30,273 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:16:30,544 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:16:31,711 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:16:31,712 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:16:31,712 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:16:31,717 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:16:32,545 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:16:34,549 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:16:34,717 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:16:39,718 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:16:40,171 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:16:40,172 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:16:40,172 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:16:40,176 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:16:40,553 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:16:42,556 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:16:45,177 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:16:45,273 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:16:45,273 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:16:45,273 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:16:46,745 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:16:46,746 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:16:46,746 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:16:46,751 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:16:47,560 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:16:48,563 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:16:50,752 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:16:52,776 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:16:52,777 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:16:52,778 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:16:52,783 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:16:53,565 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:16:54,568 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:16:55,784 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:16:56,367 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 17:16:56,857 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:16:56,858 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:16:56,858 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:16:56,863 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:16:57,571 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:16:58,573 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:17:00,273 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:17:00,273 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:17:00,273 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:17:01,561 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:17:02,084 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:17:02,084 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:17:02,085 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:17:02,090 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:17:02,575 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:17:04,578 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:17:05,163 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:17:05,164 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:17:05,164 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:17:05,169 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:17:05,579 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:17:06,581 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:17:07,170 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:17:08,704 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:17:08,705 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:17:08,706 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:17:08,710 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:17:09,583 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:17:10,586 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:17:12,223 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:17:12,224 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:17:12,224 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:17:12,224 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:17:12,229 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:17:12,587 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:17:14,590 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:17:15,273 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:17:15,273 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:17:15,273 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:17:16,319 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:17:16,320 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:17:16,320 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:17:16,325 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:17:16,591 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:17:17,325 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:17:18,592 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:17:22,326 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:17:24,042 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:17:24,043 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:17:24,043 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:17:24,049 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:17:24,596 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:17:26,373 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 17:17:26,599 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:17:27,374 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:17:27,481 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:17:27,481 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:17:27,482 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:17:27,487 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:17:27,599 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:17:28,602 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:17:30,273 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:17:30,273 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:17:30,273 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:17:32,473 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:17:32,939 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:17:32,939 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:17:32,940 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:17:32,944 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:17:33,606 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:17:34,609 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:17:37,320 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:17:37,321 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:17:37,321 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:17:37,326 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:17:37,611 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:17:38,327 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:17:38,615 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:17:41,651 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:17:41,651 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:17:41,652 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:17:41,657 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:17:42,617 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:17:43,657 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:17:44,620 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:17:45,255 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:17:45,256 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:17:45,257 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:17:45,262 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:17:45,273 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:17:45,273 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:17:45,273 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:17:45,621 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:17:46,623 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:17:49,520 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:17:49,990 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:17:49,990 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:17:49,991 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:17:49,996 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:17:50,625 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:17:52,628 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:17:54,351 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:17:54,352 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:17:54,352 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:17:54,358 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:17:54,629 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:17:55,358 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:17:56,372 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 17:17:56,634 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:17:58,439 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:17:58,440 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:17:58,440 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:17:58,445 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:17:58,635 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:18:00,273 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:18:00,273 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:18:00,274 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:18:00,574 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:18:00,636 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:18:02,304 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:18:02,305 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:18:02,305 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:18:02,310 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:18:02,637 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:18:04,640 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:18:06,311 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:18:06,335 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:18:06,335 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:18:06,335 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:18:06,340 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:18:06,641 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:18:08,645 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:18:10,190 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:18:10,191 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:18:10,191 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:18:10,196 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:18:10,646 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:18:12,196 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:18:12,649 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:18:13,753 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:18:13,754 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:18:13,754 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:18:13,759 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:18:14,651 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:18:15,273 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:18:15,273 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:18:15,274 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:18:16,654 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:18:17,459 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:18:17,460 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:18:17,460 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:18:17,460 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:18:17,465 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:18:17,654 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:18:18,657 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:18:22,466 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:18:23,055 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:18:23,056 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:18:23,056 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:18:23,061 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:18:23,659 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:18:24,662 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:18:26,376 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 17:18:26,905 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:18:26,906 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:18:26,906 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:18:26,911 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:18:27,664 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:18:27,911 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:18:28,664 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:18:30,273 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:18:30,273 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:18:30,274 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:18:32,162 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:18:32,163 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:18:32,163 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:18:32,168 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:18:32,666 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:18:33,169 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:18:34,668 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:18:35,910 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:18:35,911 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:18:35,911 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:18:35,916 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:18:36,669 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:18:38,673 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:18:38,916 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:18:39,678 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:18:39,679 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:18:39,679 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:18:39,684 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:18:40,674 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:18:42,677 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:18:44,685 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:18:45,209 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:18:45,210 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:18:45,211 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:18:45,215 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:18:45,274 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:18:45,274 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:18:45,274 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:18:45,679 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:18:46,680 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:18:48,695 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:18:48,695 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:18:48,696 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:18:48,700 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:18:49,681 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:18:49,701 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:18:50,684 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:18:52,237 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:18:52,238 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:18:52,238 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:18:52,243 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:18:52,685 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:18:54,688 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:18:55,243 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:18:56,376 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 17:18:56,574 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:18:56,575 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:18:56,575 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:18:56,580 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:18:56,690 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:18:58,693 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:18:59,801 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:18:59,802 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:18:59,802 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:18:59,807 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:19:00,274 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:19:00,274 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:19:00,274 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:19:00,568 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:19:00,694 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:19:02,697 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:19:03,527 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:19:03,528 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:19:03,529 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:19:03,534 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:19:03,698 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:19:04,701 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:19:06,534 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:19:06,732 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:19:06,732 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:19:06,733 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:19:06,738 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:19:07,702 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:19:08,703 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:19:09,994 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:19:09,995 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:19:09,995 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:19:10,000 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:19:10,704 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:19:12,000 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:19:12,707 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:19:13,199 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:19:13,200 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:19:13,200 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:19:13,205 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:19:13,708 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:19:14,710 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:19:15,274 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:19:15,274 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:19:15,274 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:19:17,530 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:19:18,658 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:19:18,659 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:19:18,660 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:19:18,665 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:19:18,713 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:19:20,716 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:19:21,674 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:19:21,675 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:19:21,676 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:19:21,680 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:19:21,716 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:19:22,680 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:19:24,720 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:19:24,991 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:19:24,992 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:19:24,993 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:19:24,997 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:19:25,721 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:19:26,381 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 17:19:26,723 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:19:28,382 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:19:30,274 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:19:30,274 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:19:30,274 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:19:31,241 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:19:31,242 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:19:31,242 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:19:31,248 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:19:31,726 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:19:32,728 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:19:34,249 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:19:34,437 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:19:34,438 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:19:34,438 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:19:34,443 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:19:34,730 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:19:36,731 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:19:39,444 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:19:42,930 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:19:42,930 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:19:42,931 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:19:42,935 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:19:43,736 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:19:44,741 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:19:44,936 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:19:45,274 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:19:45,274 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:19:45,274 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:19:45,961 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:19:45,962 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:19:45,962 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:19:45,967 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:19:46,742 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:19:48,743 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:19:49,883 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:19:49,884 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:19:49,884 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:19:49,889 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:19:50,744 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:19:50,889 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:19:52,747 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:19:53,450 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:19:53,450 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:19:53,451 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:19:53,456 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:19:53,748 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:19:54,748 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:19:56,380 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 17:19:56,381 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:19:57,190 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:19:57,191 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:19:57,191 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:19:57,196 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:19:57,752 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:19:58,754 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:20:00,274 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:20:00,274 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:20:00,275 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:20:00,904 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:20:00,904 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:20:00,905 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:20:00,910 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:20:01,756 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:20:01,910 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:20:02,758 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:20:04,117 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:20:04,118 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:20:04,118 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:20:04,123 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:20:04,759 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:20:06,762 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:20:07,124 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:20:07,306 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:20:07,307 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:20:07,307 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:20:07,312 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:20:07,763 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:20:08,765 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:20:12,313 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:20:12,923 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:20:12,924 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:20:12,924 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:20:12,929 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:20:13,768 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:20:14,771 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:20:15,274 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:20:15,274 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:20:15,274 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:20:16,467 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:20:16,467 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:20:16,468 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:20:16,472 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:20:16,772 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:20:17,509 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:20:18,775 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:20:20,381 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:20:20,382 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:20:20,382 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:20:20,387 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:20:20,776 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:20:22,777 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:20:23,388 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:20:23,743 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:20:23,744 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:20:23,744 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:20:23,749 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:20:23,778 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:20:26,385 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 17:20:26,782 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:20:27,356 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:20:27,357 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:20:27,358 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:20:27,362 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:20:27,782 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:20:28,783 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:20:29,363 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:20:30,274 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:20:30,274 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:20:30,275 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:20:31,193 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:20:31,194 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:20:31,195 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:20:31,199 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:20:31,784 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:20:32,787 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:20:35,200 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:20:35,341 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:20:35,342 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:20:35,342 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:20:35,347 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:20:35,789 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:20:36,791 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:20:38,937 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:20:38,938 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:20:38,938 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:20:38,946 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:20:39,794 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:20:40,795 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:20:40,946 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:20:42,261 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:20:42,261 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:20:42,262 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:20:42,267 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:20:42,796 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:20:44,799 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:20:45,274 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:20:45,274 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:20:45,275 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:20:46,183 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:20:46,183 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:20:46,184 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:20:46,184 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:20:46,189 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:20:46,800 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:20:48,803 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:20:51,189 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:20:51,356 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:20:51,357 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:20:51,358 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:20:51,363 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:20:51,805 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:20:52,807 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:20:54,386 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:20:54,387 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:20:54,388 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:20:54,392 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:20:54,809 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:20:56,384 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 17:20:56,385 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:20:56,811 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:20:58,558 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:20:58,559 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:20:58,559 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:20:58,564 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:20:58,812 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:21:00,274 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:21:00,274 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:21:00,275 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:21:00,815 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:21:01,566 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:21:01,830 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:21:01,831 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:21:01,831 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:21:01,836 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:21:02,816 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:21:04,819 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:21:06,837 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:21:08,834 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:21:08,835 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:21:08,836 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:21:08,841 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:21:09,822 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:21:10,825 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:21:11,841 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:21:12,279 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:21:12,280 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:21:12,280 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:21:12,285 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:21:12,826 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:21:14,831 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:21:15,274 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:21:15,275 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:21:15,275 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:21:15,980 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:21:19,945 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:21:21,275 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:21:24,062 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:21:26,276 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:21:27,354 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:21:30,274 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:21:31,052 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:21:34,740 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:21:35,276 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:21:36,886 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:21:36,887 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:21:36,887 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:21:36,892 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:21:36,893 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:21:36,893 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:21:36,897 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:21:36,897 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:21:36,897 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:21:36,902 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:21:36,902 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 17:21:36,903 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:21:36,903 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:21:36,907 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:21:36,908 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:21:36,908 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:21:36,912 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:21:36,912 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:21:36,912 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:21:36,917 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:21:37,844 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:21:38,092 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:21:38,092 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:21:38,093 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:21:38,098 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:21:38,848 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:21:38,848 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:21:40,277 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:21:40,851 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:21:40,906 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:21:40,906 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:21:40,907 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:21:40,912 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:21:41,852 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:21:41,912 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:21:42,854 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:21:45,275 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:21:46,913 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:21:48,363 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:21:48,364 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:21:48,364 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:21:48,369 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:21:48,858 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:21:50,278 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:21:50,859 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:21:52,406 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:21:53,170 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:21:53,170 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:21:53,171 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:21:53,176 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:21:53,860 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:21:54,863 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:21:55,278 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:21:56,388 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 17:21:57,898 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:21:57,898 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:21:57,899 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:21:57,899 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:21:57,903 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:21:58,866 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:22:00,275 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:22:00,870 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:22:01,642 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:22:01,643 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:22:01,643 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:22:01,648 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:22:01,870 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:22:02,873 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:22:03,685 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:22:05,279 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:22:05,378 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:22:05,379 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:22:05,379 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:22:05,384 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:22:05,874 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:22:06,875 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:22:09,384 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:22:10,237 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:22:10,238 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:22:10,238 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:22:10,243 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:22:10,280 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:22:10,877 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:22:12,881 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:22:13,771 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:22:13,771 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:22:13,771 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:22:13,776 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:22:13,881 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:22:14,777 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:22:14,884 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:22:15,275 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:22:19,777 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:22:19,915 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:22:19,915 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:22:19,915 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:22:19,920 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:22:20,281 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:22:20,887 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:22:22,890 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:22:24,417 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:22:24,418 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:22:24,418 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:22:24,423 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:22:24,891 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:22:25,281 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:22:25,424 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:22:26,393 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 17:22:26,894 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:22:29,359 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:22:29,360 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:22:29,360 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:22:29,366 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:22:29,896 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:22:30,275 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:22:30,897 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:22:31,366 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:22:32,998 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:22:32,998 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:22:32,998 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:22:33,003 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:22:33,898 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:22:34,901 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:22:35,282 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:22:37,041 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:22:39,399 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:22:39,400 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:22:39,400 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:22:39,405 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:22:39,905 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:22:40,282 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:22:40,908 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:22:42,406 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:22:42,769 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:22:42,770 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:22:42,770 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:22:42,775 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:22:42,909 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:22:44,912 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:22:45,275 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:22:47,082 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:22:47,082 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:22:47,083 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:22:47,087 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:22:47,914 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:22:48,125 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:22:48,916 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:22:50,476 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:22:50,799 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:22:50,800 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:22:50,800 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:22:50,805 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:22:50,917 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:22:52,920 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:22:53,806 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:22:55,473 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:22:55,473 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:22:55,474 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:22:55,477 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:22:55,479 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:22:55,924 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:22:56,392 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 17:22:56,926 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:22:59,394 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:22:59,780 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:22:59,781 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:22:59,781 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:22:59,786 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:22:59,928 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:23:00,275 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:23:00,930 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:23:04,251 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:23:04,252 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:23:04,252 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:23:04,257 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:23:04,932 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:23:05,293 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:23:05,478 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:23:06,935 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:23:08,230 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:23:08,231 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:23:08,231 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:23:08,236 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:23:08,936 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:23:10,478 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:23:10,938 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:23:11,237 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:23:11,790 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:23:11,790 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:23:11,790 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:23:11,796 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:23:11,938 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:23:12,941 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:23:15,275 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:23:16,497 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:23:16,497 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:23:16,498 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:23:16,498 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:23:16,502 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:23:16,943 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:23:18,946 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:23:19,771 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:23:19,772 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:23:19,772 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:23:19,776 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:23:19,947 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:23:20,479 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:23:20,949 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:23:21,777 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:23:23,534 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:23:23,535 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:23:23,535 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:23:23,540 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:23:23,951 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:23:24,953 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:23:25,480 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:23:26,394 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 17:23:26,730 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:23:26,731 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:23:26,731 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:23:26,736 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:23:26,954 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:23:27,736 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:23:28,957 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:23:30,275 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:23:30,904 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:23:30,904 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:23:30,905 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:23:30,909 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:23:30,958 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:23:32,949 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:23:32,960 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:23:34,255 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:23:34,256 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:23:34,256 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:23:34,261 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:23:34,961 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:23:35,481 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:23:36,964 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:23:37,425 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:23:37,426 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:23:37,426 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:23:37,431 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:23:37,965 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:23:38,431 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:23:38,969 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:23:40,481 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:23:41,232 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:23:41,232 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:23:41,233 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:23:41,237 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:23:41,971 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:23:42,973 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:23:44,238 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:23:45,275 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:23:48,361 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:23:48,361 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:23:48,361 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:23:48,366 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:23:48,976 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:23:49,401 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:23:50,482 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:23:50,979 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:23:52,473 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:23:52,473 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:23:52,473 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:23:52,478 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:23:52,981 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:23:54,479 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:23:54,984 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:23:55,483 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:23:56,396 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 17:23:56,434 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:23:56,435 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:23:56,435 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:23:56,440 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:23:56,986 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:23:58,989 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:23:59,888 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:23:59,888 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:23:59,888 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:23:59,889 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:23:59,894 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:23:59,990 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:24:00,275 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:24:00,990 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:24:03,114 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:24:03,115 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:24:03,115 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:24:03,120 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:24:03,992 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:24:04,995 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:24:05,157 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:24:05,484 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:24:07,099 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:24:07,100 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:24:07,100 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:24:07,105 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:24:07,996 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:24:08,999 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:24:10,253 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:24:10,254 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:24:10,254 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:24:10,254 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:24:10,259 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:24:10,484 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:24:11,000 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:24:13,002 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:24:14,319 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:24:14,320 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:24:14,320 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:24:14,325 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:24:15,003 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:24:15,275 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:24:15,325 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:24:17,006 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:24:17,822 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:24:17,823 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:24:17,823 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:24:17,827 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:24:18,007 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:24:19,007 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:24:20,485 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:24:20,865 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:24:22,535 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:24:22,536 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:24:22,536 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:24:22,541 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:24:23,010 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:24:25,013 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:24:25,486 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:24:26,399 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 17:24:26,400 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:24:26,457 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:24:26,457 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:24:26,457 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:24:26,462 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:24:27,014 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:24:29,017 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:24:30,275 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:24:31,463 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:24:31,671 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:24:31,671 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:24:31,671 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:24:31,676 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:24:32,018 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:24:33,021 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:24:35,132 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:24:35,133 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:24:35,133 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:24:35,138 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:24:35,486 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:24:36,023 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:24:37,026 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:24:37,138 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:24:39,427 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:24:39,427 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:24:39,427 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:24:39,432 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:24:40,029 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:24:40,487 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:24:41,029 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:24:42,433 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:24:45,275 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:24:47,433 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:24:48,009 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:24:48,010 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:24:48,010 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:24:48,015 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:24:48,033 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:24:50,488 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:24:51,036 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:24:53,054 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:24:54,018 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:24:54,018 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:24:54,019 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:24:54,023 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:24:54,038 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:24:55,041 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:24:55,488 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:24:56,401 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 17:24:58,128 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:24:58,128 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:24:58,129 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:24:58,129 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:24:58,133 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:24:59,044 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:25:00,276 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:25:01,047 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:25:01,821 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:25:01,822 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:25:01,822 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:25:01,827 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:25:02,048 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:25:03,051 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:25:03,865 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:25:05,489 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:25:06,319 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:25:06,319 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:25:06,320 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:25:06,324 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:25:07,053 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:25:09,056 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:25:09,325 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:25:10,490 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:25:10,922 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:25:10,922 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:25:10,923 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:25:10,927 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:25:11,057 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:25:13,060 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:25:14,928 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:25:15,276 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:25:15,454 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:25:15,454 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:25:15,455 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:25:15,459 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:25:16,062 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:25:17,064 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:25:20,491 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:25:20,498 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:25:21,088 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:25:21,089 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:25:21,089 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:25:21,094 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:25:22,067 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:25:23,070 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:25:25,491 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:25:26,039 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:25:26,040 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:25:26,040 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:25:26,040 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:25:26,045 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:25:26,072 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:25:26,403 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 17:25:27,074 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:25:29,641 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:25:29,642 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:25:29,642 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:25:29,647 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:25:30,077 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:25:30,276 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:25:31,079 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:25:31,647 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:25:34,589 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:25:34,590 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:25:34,590 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:25:34,595 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:25:35,082 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:25:35,492 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:25:37,083 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:25:37,633 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:25:38,304 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:25:38,304 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:25:38,304 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:25:38,309 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:25:39,086 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:25:40,493 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:25:41,089 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:25:41,789 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:25:41,790 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:25:41,790 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:25:41,795 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:25:42,089 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:25:42,796 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:25:43,092 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:25:45,276 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:25:47,796 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:25:47,797 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:25:47,797 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:25:47,797 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:25:47,802 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:25:48,094 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:25:49,098 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:25:50,494 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:25:52,838 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:25:52,878 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:25:52,879 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:25:52,879 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:25:52,884 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:25:53,102 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:25:55,107 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:25:55,494 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:25:56,405 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 17:25:58,406 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:26:00,276 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:26:00,375 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:26:00,376 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:26:00,376 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:26:00,380 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:26:01,113 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:26:03,118 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:26:03,417 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:26:05,495 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:26:08,418 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:26:08,926 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:26:08,927 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:26:08,927 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:26:08,932 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:26:09,126 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:26:10,496 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:26:11,127 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:26:13,932 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:26:14,080 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:26:14,080 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:26:14,080 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:26:14,085 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:26:14,129 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:26:15,132 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:26:15,276 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:26:19,086 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:26:20,497 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:26:21,171 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:26:21,171 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:26:21,172 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:26:21,176 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:26:22,136 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:26:23,138 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:26:24,177 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:26:24,806 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:26:24,807 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:26:24,807 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:26:24,812 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:26:25,139 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:26:25,497 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:26:26,407 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 17:26:27,142 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:26:28,339 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:26:28,339 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:26:28,340 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:26:28,344 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:26:29,143 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:26:29,345 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:26:30,276 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:26:31,146 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:26:32,110 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:26:32,111 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:26:32,111 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:26:32,116 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:26:32,147 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:26:35,149 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:26:35,153 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:26:35,498 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:26:35,736 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:26:35,737 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:26:35,737 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:26:35,742 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:26:36,149 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:26:37,152 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:26:40,091 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:26:40,092 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:26:40,092 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:26:40,097 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:26:40,155 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:26:40,499 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:26:41,097 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:26:41,157 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:26:43,665 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:26:43,666 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:26:43,666 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:26:43,671 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:26:44,159 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:26:45,161 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:26:45,276 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:26:46,671 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:26:47,536 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:26:47,537 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:26:47,537 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:26:47,541 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:26:48,165 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:26:49,167 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:26:50,500 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:26:52,188 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:26:52,188 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:26:52,189 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:26:52,189 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:26:52,193 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:26:53,170 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:26:55,173 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:26:55,500 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:26:56,316 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:26:56,316 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:26:56,316 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:26:56,321 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:26:56,413 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 17:26:57,175 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:26:57,414 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:26:59,178 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:26:59,612 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:26:59,613 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:26:59,613 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:26:59,618 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:27:00,179 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:27:00,276 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:27:01,182 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:27:02,618 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:27:03,283 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:27:03,283 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:27:03,283 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:27:03,288 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:27:04,183 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:27:05,186 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:27:05,501 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:27:07,348 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:27:07,349 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:27:07,349 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:27:07,354 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:27:08,187 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:27:08,354 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:27:09,190 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:27:10,502 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:27:11,180 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:27:11,181 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:27:11,181 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:27:11,186 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:27:11,191 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:27:13,194 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:27:14,187 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:27:14,618 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:27:14,618 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:27:14,619 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:27:14,624 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:27:15,195 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:27:15,276 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:27:17,198 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:27:19,624 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:27:20,503 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:27:23,003 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:27:23,004 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:27:23,004 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:27:23,009 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:27:23,202 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:27:25,010 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:27:25,203 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:27:25,503 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:27:26,411 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 17:27:26,516 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:27:26,517 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:27:26,517 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:27:26,522 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:27:27,204 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:27:29,207 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:27:30,276 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:27:30,523 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:27:32,115 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:27:32,115 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:27:32,115 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:27:32,120 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:27:32,208 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:27:33,211 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:27:35,057 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:27:35,057 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:27:35,058 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:27:35,062 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:27:35,213 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:27:35,504 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:27:36,063 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:27:37,216 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:27:40,397 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:27:40,398 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:27:40,398 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:27:40,403 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:27:40,505 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:27:41,219 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:27:41,403 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:27:43,222 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:27:43,835 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:27:43,836 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:27:43,836 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:27:43,841 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:27:44,222 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:27:45,225 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:27:45,276 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:27:46,841 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:27:47,005 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:27:47,006 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:27:47,006 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:27:47,011 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:27:47,226 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:27:49,229 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:27:50,506 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:27:52,050 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:27:52,785 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:27:52,786 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:27:52,786 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:27:52,791 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:27:53,232 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:27:55,234 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:27:55,506 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:27:56,416 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 17:27:56,997 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:27:56,998 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:27:56,998 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:27:57,003 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:27:57,237 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:27:58,004 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:27:59,240 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:28:00,045 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:28:00,045 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:28:00,046 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:28:00,050 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:28:00,241 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:28:00,276 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:28:01,243 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:28:03,051 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:28:03,924 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:28:03,924 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:28:03,925 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:28:03,929 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:28:04,245 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:28:05,247 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:28:05,507 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:28:07,407 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:28:07,408 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:28:07,408 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:28:07,413 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:28:08,249 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:28:08,413 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:28:09,249 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:28:10,508 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:28:11,137 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:28:11,138 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:28:11,138 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:28:11,143 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:28:11,250 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:28:13,254 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:28:14,143 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:28:14,616 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:28:14,617 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:28:14,617 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:28:14,622 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:28:15,255 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:28:15,277 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:28:17,258 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:28:19,623 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:28:20,509 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:28:20,668 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:28:20,669 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:28:20,669 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:28:20,674 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:28:21,260 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:28:23,264 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:28:24,058 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:28:24,059 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:28:24,059 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:28:24,064 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:28:24,264 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:28:25,065 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:28:25,267 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:28:25,509 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:28:26,415 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 17:28:29,263 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:28:29,264 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:28:29,264 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:28:29,269 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:28:29,269 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:28:30,269 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:28:30,277 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:28:31,273 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:28:33,610 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:28:33,611 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:28:33,611 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:28:33,616 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:28:34,275 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:28:35,277 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:28:35,510 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:28:35,653 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:28:37,033 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:28:37,034 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:28:37,034 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:28:37,039 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:28:37,279 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:28:39,281 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:28:40,511 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:28:40,544 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:28:40,544 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:28:40,545 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:28:40,550 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:28:41,282 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:28:41,550 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:28:43,285 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:28:45,277 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:28:45,879 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:28:45,880 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:28:45,880 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:28:45,885 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:28:46,287 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:28:46,921 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:28:47,289 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:28:49,124 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:28:49,124 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:28:49,124 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:28:49,129 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:28:49,290 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:28:50,512 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:28:51,291 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:28:52,130 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:28:52,649 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:28:52,650 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:28:52,650 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:28:52,655 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:28:53,293 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:28:55,296 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:28:55,512 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:28:56,420 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 17:28:57,422 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:28:58,000 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:28:58,001 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:28:58,001 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:28:58,006 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:28:58,298 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:28:59,301 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:29:00,277 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:29:03,007 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:29:03,185 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:29:03,186 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:29:03,186 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:29:03,191 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:29:03,303 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:29:05,306 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:29:05,513 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:29:06,495 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:29:06,496 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:29:06,496 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:29:06,501 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:29:07,307 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:29:08,501 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:29:09,310 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:29:10,514 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:29:10,852 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:29:10,852 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:29:10,852 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:29:10,857 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:29:11,311 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:29:13,315 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:29:13,858 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:29:14,918 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:29:14,918 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:29:14,918 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:29:14,923 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:29:15,277 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:29:15,316 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:29:17,319 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:29:18,924 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:29:19,820 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:29:19,820 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:29:19,820 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:29:19,825 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:29:20,321 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:29:20,514 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:29:21,323 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:29:24,861 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:29:25,515 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:29:25,900 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:29:25,900 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:29:25,901 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:29:25,906 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:29:26,326 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:29:26,420 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 17:29:27,328 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:29:29,333 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:29:29,334 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:29:29,334 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:29:29,339 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:29:30,277 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:29:30,330 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:29:30,339 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:29:31,332 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:29:34,414 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:29:34,415 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:29:34,415 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:29:34,420 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:29:35,335 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:29:35,457 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:29:35,516 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:29:37,338 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:29:37,679 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:29:37,680 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:29:37,680 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:29:37,685 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:29:38,339 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:29:39,342 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:29:40,516 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:29:40,685 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:29:41,297 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:29:41,298 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:29:41,298 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:29:41,303 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:29:41,343 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:29:43,346 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:29:45,277 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:29:46,303 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:29:46,728 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:29:46,729 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:29:46,729 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:29:46,734 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:29:47,348 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:29:49,351 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:29:50,517 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:29:51,769 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:29:55,317 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:29:55,318 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:29:55,318 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:29:55,323 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:29:55,355 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:29:55,518 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:29:56,422 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 17:29:57,357 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:29:57,423 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:29:59,458 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:29:59,458 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:29:59,460 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:29:59,465 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:30:00,277 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:30:00,358 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:30:01,361 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:30:02,390 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:30:02,391 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:30:02,391 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:30:02,396 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:30:02,433 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:30:03,362 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:30:05,250 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:30:05,250 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:30:05,250 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:30:05,255 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:30:05,365 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:30:05,365 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:30:05,519 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:30:07,368 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:30:08,256 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:30:09,415 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:30:09,416 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:30:09,416 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:30:09,421 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:30:10,370 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:30:10,519 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:30:11,372 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:30:13,242 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:30:13,242 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:30:13,243 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:30:13,247 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:30:13,374 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:30:14,248 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:30:15,277 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:30:15,375 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:30:16,370 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:30:16,371 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:30:16,371 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:30:16,376 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:30:16,376 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:30:19,380 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:30:19,413 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:30:20,224 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:30:20,225 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:30:20,225 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:30:20,230 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:30:20,381 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:30:20,520 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:30:21,383 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:30:25,230 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:30:25,314 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:30:25,315 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:30:25,315 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:30:25,320 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:30:25,385 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:30:25,521 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:30:26,424 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 17:30:27,388 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:30:29,830 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:30:29,830 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:30:29,831 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:30:29,835 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:30:30,277 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:30:30,390 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:30:30,836 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:30:31,391 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:30:33,345 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:30:33,346 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:30:33,346 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:30:33,351 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:30:33,392 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:30:35,395 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:30:35,522 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:30:36,389 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:30:36,650 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:30:36,650 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:30:36,650 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:30:36,655 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:30:37,396 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:30:39,400 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:30:40,243 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:30:40,244 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:30:40,244 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:30:40,249 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:30:40,401 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:30:40,522 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:30:41,404 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:30:42,249 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:30:45,277 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:30:46,274 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:30:46,274 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:30:46,274 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:30:46,279 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:30:46,406 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:30:47,317 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:30:47,407 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:30:50,523 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:30:52,318 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:30:52,489 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:30:52,490 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:30:52,490 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:30:52,495 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:30:53,410 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:30:55,413 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:30:55,524 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:30:56,170 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:30:56,170 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:30:56,170 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:30:56,175 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:30:56,415 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:30:56,426 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 17:30:57,418 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:30:57,427 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:31:00,277 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:31:00,714 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:31:00,715 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:31:00,715 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:31:00,720 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:31:01,420 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:31:02,757 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:31:03,423 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:31:04,456 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:31:04,457 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:31:04,457 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:31:04,462 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:31:05,424 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:31:05,525 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:31:07,428 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:31:08,405 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:31:08,406 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:31:08,406 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:31:08,406 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:31:08,411 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:31:08,428 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:31:09,431 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:31:10,526 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:31:11,940 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:31:11,941 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:31:11,941 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:31:11,946 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:31:12,433 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:31:13,435 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:31:13,946 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:31:14,860 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:31:14,861 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:31:14,861 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:31:14,866 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:31:15,278 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:31:15,437 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:31:17,438 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:31:19,318 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:31:19,319 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:31:19,319 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:31:19,319 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:31:19,324 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:31:19,439 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:31:20,526 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:31:21,442 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:31:22,777 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:31:22,778 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:31:22,778 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:31:22,784 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:31:23,443 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:31:24,784 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:31:25,446 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:31:25,527 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:31:26,428 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 17:31:27,370 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:31:27,371 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:31:27,371 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:31:27,375 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:31:27,447 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:31:29,450 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:31:30,278 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:31:30,376 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:31:35,376 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:31:35,528 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:31:35,940 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:31:35,941 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:31:35,941 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:31:35,947 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:31:36,454 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:31:37,457 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:31:40,529 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:31:40,947 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:31:42,938 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:31:42,939 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:31:42,939 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:31:42,944 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:31:43,461 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:31:45,278 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:31:45,464 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:31:46,782 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:31:46,783 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:31:46,783 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:31:46,783 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:31:46,788 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:31:47,466 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:31:49,469 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:31:51,168 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:31:51,826 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:31:53,621 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:31:53,621 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:31:53,621 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:31:53,626 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:31:54,471 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:31:55,474 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:31:56,169 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:31:56,430 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 17:31:57,431 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:31:58,526 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:31:58,527 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:31:58,527 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:31:58,532 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:31:59,477 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:32:00,278 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:32:01,480 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:32:02,532 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:32:03,310 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:32:03,310 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:32:03,311 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:32:03,315 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:32:03,481 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:32:05,485 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:32:06,170 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:32:06,657 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:32:06,658 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:32:06,658 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:32:06,663 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:32:07,486 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:32:07,664 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:32:09,489 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:32:11,170 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:32:11,543 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:32:11,544 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:32:11,544 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:32:11,549 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:32:12,491 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:32:13,493 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:32:13,549 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:32:15,278 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:32:18,550 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:32:20,134 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:32:20,134 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:32:20,134 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:32:20,140 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:32:20,497 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:32:21,171 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:32:21,500 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:32:23,678 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:32:23,679 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:32:23,679 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:32:23,679 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:32:23,684 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:32:24,501 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:32:25,504 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:32:26,172 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:32:26,435 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 17:32:27,175 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:32:27,176 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:32:27,176 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:32:27,180 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:32:27,505 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:32:29,181 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:32:29,508 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:32:30,278 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:32:32,539 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:32:32,539 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:32:32,539 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:32:32,544 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:32:33,510 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:32:34,581 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:32:35,511 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:32:36,172 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:32:39,582 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:32:39,593 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:32:39,594 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:32:39,594 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:32:39,598 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:32:40,515 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:32:41,173 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:32:41,516 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:32:43,718 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:32:43,719 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:32:43,719 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:32:43,724 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:32:44,518 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:32:44,724 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:32:45,278 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:32:45,520 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:32:48,721 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:32:48,721 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:32:48,721 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:32:48,726 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:32:49,522 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:32:49,761 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:32:51,174 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:32:51,525 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:32:51,942 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:32:51,943 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:32:51,943 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:32:51,948 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:32:52,526 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:32:53,529 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:32:54,948 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:32:55,582 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:32:55,582 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:32:55,583 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:32:55,587 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:32:56,174 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:32:56,434 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 17:32:56,532 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:32:57,534 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:32:59,950 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:32:59,950 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:32:59,950 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:32:59,951 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:32:59,955 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:33:00,278 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:33:00,536 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:33:01,538 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:33:03,308 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:33:03,309 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:33:03,309 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:33:03,314 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:33:03,539 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:33:05,349 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:33:05,540 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:33:06,175 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:33:07,441 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:33:07,441 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:33:07,441 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:33:07,446 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:33:07,541 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:33:09,544 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:33:10,447 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:33:11,026 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:33:11,027 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:33:11,027 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:33:11,032 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:33:11,176 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:33:11,546 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:33:13,549 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:33:14,813 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:33:14,814 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:33:14,814 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:33:14,819 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:33:15,278 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:33:15,550 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:33:15,820 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:33:17,553 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:33:18,722 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:33:18,722 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:33:18,723 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:33:18,727 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:33:19,554 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:33:21,177 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:33:21,557 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:33:21,765 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:33:22,359 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:33:22,360 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:33:22,360 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:33:22,365 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:33:22,558 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:33:23,560 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:33:25,684 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:33:25,685 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:33:25,685 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:33:25,690 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:33:26,177 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:33:26,439 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 17:33:26,562 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:33:27,440 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:33:27,563 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:33:30,016 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:33:30,017 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:33:30,017 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:33:30,022 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:33:30,278 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:33:30,564 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:33:31,567 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:33:33,022 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:33:34,187 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:33:34,188 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:33:34,188 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:33:34,193 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:33:34,568 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:33:35,571 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:33:36,178 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:33:38,229 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:33:41,179 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:33:42,579 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:33:42,580 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:33:42,580 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:33:42,585 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:33:43,579 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:33:43,579 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:33:43,585 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:33:45,278 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:33:46,582 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:33:46,582 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:33:46,582 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:33:46,587 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:33:47,583 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:33:47,583 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:33:48,625 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:33:51,180 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:33:51,477 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:33:51,478 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:33:51,478 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:33:51,483 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:33:51,585 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:33:53,588 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:33:54,483 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:33:55,060 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:33:55,061 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:33:55,061 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:33:55,066 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:33:55,589 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:33:56,180 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:33:56,438 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 17:33:57,592 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:33:58,761 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:33:58,761 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:33:58,762 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:33:58,766 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:33:59,593 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:33:59,767 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:34:00,278 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:34:01,596 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:34:04,767 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:34:06,181 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:34:07,354 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:34:07,355 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:34:07,355 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:34:07,360 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:34:07,599 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:34:09,602 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:34:10,361 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:34:11,181 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:34:15,278 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:34:15,361 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:34:15,936 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:34:15,937 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:34:15,937 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:34:15,942 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:34:16,606 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:34:17,608 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:34:19,496 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:34:19,497 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:34:19,497 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:34:19,502 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:34:19,610 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:34:20,503 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:34:21,182 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:34:21,611 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:34:25,503 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:34:26,039 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:34:26,040 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:34:26,040 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:34:26,045 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:34:26,183 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:34:26,443 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 17:34:26,613 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:34:27,616 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:34:30,278 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:34:30,956 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:34:30,957 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:34:30,957 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:34:30,957 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:34:30,962 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:34:31,618 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:34:33,621 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:34:34,630 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:34:34,631 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:34:34,631 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:34:34,636 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:34:35,625 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:34:35,625 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:34:36,184 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:34:36,636 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:34:37,532 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:34:37,533 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:34:37,533 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:34:37,538 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:34:37,626 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:34:39,630 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:34:40,837 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:34:40,837 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:34:40,837 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:34:40,842 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:34:41,184 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:34:41,631 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:34:41,843 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:34:43,632 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:34:45,279 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:34:46,107 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:34:46,107 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:34:46,107 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:34:46,112 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:34:46,634 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:34:47,149 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:34:47,636 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:34:49,955 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:34:49,955 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:34:49,956 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:34:49,960 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:34:50,638 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:34:51,185 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:34:51,640 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:34:52,961 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:34:53,600 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:34:53,601 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:34:53,601 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:34:53,606 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:34:53,642 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:34:55,643 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:34:56,186 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:34:56,442 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 17:34:58,443 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:34:58,919 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:34:58,920 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:34:58,920 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:34:58,925 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:34:59,646 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:35:00,279 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:35:01,649 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:35:02,925 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:35:02,925 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:35:02,925 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:35:02,930 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:35:03,650 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:35:03,965 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:35:05,653 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:35:06,187 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:35:06,679 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:35:06,680 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:35:06,680 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:35:06,692 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:35:07,656 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:35:07,656 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:35:09,693 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:35:09,694 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:35:09,695 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:35:09,695 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:35:09,699 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:35:10,658 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:35:11,187 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:35:11,658 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:35:13,746 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:35:13,747 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:35:13,747 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:35:13,752 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:35:14,660 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:35:14,752 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:35:15,279 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:35:15,663 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:35:17,495 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:35:17,495 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:35:17,495 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:35:17,500 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:35:17,664 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:35:19,667 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:35:20,538 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:35:20,895 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:35:20,896 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:35:20,896 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:35:20,901 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:35:21,188 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:35:21,668 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:35:23,671 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:35:25,650 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:35:25,650 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:35:25,651 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:35:25,651 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:35:25,655 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:35:25,672 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:35:26,189 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:35:26,447 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 17:35:27,675 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:35:29,498 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:35:29,499 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:35:29,499 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:35:29,504 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:35:29,676 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:35:30,279 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:35:31,505 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:35:31,679 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:35:33,789 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:35:33,790 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:35:33,790 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:35:33,795 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:35:34,681 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:35:35,684 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:35:36,190 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:35:36,833 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:35:36,891 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:35:36,892 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:35:36,892 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:35:36,897 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:35:37,685 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:35:39,689 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:35:40,059 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:35:40,060 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:35:40,060 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:35:40,065 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:35:40,690 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:35:41,190 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:35:41,692 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:35:42,065 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:35:43,719 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:35:43,720 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:35:43,720 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:35:43,725 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:35:44,694 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:35:45,279 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:35:45,695 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:35:47,725 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:35:48,602 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:35:48,603 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:35:48,603 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:35:48,607 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:35:48,696 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:35:49,699 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:35:51,191 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:35:52,991 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:35:52,992 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:35:52,992 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:35:52,992 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:35:52,997 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:35:53,701 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:35:55,704 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:35:56,192 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:35:56,374 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:35:56,375 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:35:56,375 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:35:56,380 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:35:56,446 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 17:35:56,706 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:35:57,708 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:35:58,448 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:36:00,279 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:36:00,481 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:36:00,481 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:36:00,481 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:36:00,486 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:36:00,710 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:36:01,710 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:36:03,521 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:36:04,657 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:36:04,658 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:36:04,658 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:36:04,663 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:36:04,712 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:36:05,714 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:36:06,193 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:36:08,568 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:36:08,569 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:36:08,569 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:36:08,570 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:36:08,574 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:36:08,716 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:36:09,719 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:36:11,193 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:36:13,052 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:36:13,053 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:36:13,053 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:36:13,058 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:36:13,721 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:36:14,058 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:36:15,279 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:36:15,724 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:36:19,059 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:36:21,194 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:36:22,410 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:36:22,411 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:36:22,411 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:36:22,416 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:36:22,728 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:36:23,730 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:36:24,417 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:36:25,402 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:36:25,403 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:36:25,403 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:36:25,408 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:36:25,732 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:36:26,195 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:36:26,452 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 17:36:27,735 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:36:29,453 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:36:29,756 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:36:29,756 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:36:29,757 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:36:29,761 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:36:30,279 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:36:30,736 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:36:31,739 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:36:34,435 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:36:34,435 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:36:34,436 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:36:34,440 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:36:34,477 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:36:34,741 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:36:35,741 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:36:36,195 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:36:39,256 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:36:39,257 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:36:39,257 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:36:39,262 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:36:39,745 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:36:40,262 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:36:41,196 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:36:41,748 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:36:43,954 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:36:43,955 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:36:43,955 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:36:43,960 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:36:44,750 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:36:45,279 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:36:45,752 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:36:45,961 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:36:50,961 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:36:51,197 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:36:52,217 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:36:52,218 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:36:52,218 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:36:52,222 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:36:52,756 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:36:53,759 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:36:56,223 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:36:56,358 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:36:56,359 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:36:56,359 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:36:56,364 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:36:56,451 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 17:36:56,761 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:36:56,873 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:36:57,764 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:37:00,279 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:37:01,452 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:37:02,774 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:37:02,774 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:37:02,775 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:37:02,779 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:37:03,769 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:37:03,769 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:37:05,874 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:37:06,817 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:37:10,874 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:37:11,221 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:37:11,222 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:37:11,222 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:37:11,226 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:37:11,773 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:37:12,227 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:37:13,777 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:37:15,279 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:37:15,621 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:37:15,621 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:37:15,621 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:37:15,626 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:37:15,778 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:37:17,661 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:37:17,781 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:37:19,319 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:37:19,319 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:37:19,320 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:37:19,324 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:37:19,782 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:37:20,875 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:37:21,785 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:37:23,325 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:37:23,415 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:37:23,416 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:37:23,416 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:37:23,421 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:37:23,787 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:37:25,790 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:37:25,876 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:37:26,453 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 17:37:27,080 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:37:27,081 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:37:27,081 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:37:27,086 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:37:27,791 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:37:29,086 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:37:29,794 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:37:30,279 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:37:30,370 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:37:30,370 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:37:30,370 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:37:30,375 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:37:30,794 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:37:31,797 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:37:34,413 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:37:35,876 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:37:36,144 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:37:36,145 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:37:36,145 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:37:36,150 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:37:36,800 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:37:37,802 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:37:39,848 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:37:39,849 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:37:39,849 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:37:39,849 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:37:39,854 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:37:40,805 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:37:40,877 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:37:41,808 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:37:43,509 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:37:43,509 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:37:43,509 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:37:43,514 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:37:43,809 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:37:45,279 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:37:45,515 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:37:45,810 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:37:49,324 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:37:49,324 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:37:49,324 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:37:49,329 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:37:49,813 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:37:50,878 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:37:51,365 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:37:51,816 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:37:52,845 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:37:52,845 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:37:52,846 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:37:52,850 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:37:53,819 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:37:53,820 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:37:55,879 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:37:56,455 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 17:37:56,457 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:37:56,538 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:37:56,539 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:37:56,539 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:37:56,543 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:37:56,822 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:37:57,825 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:37:59,893 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:37:59,894 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:37:59,894 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:37:59,899 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:38:00,280 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:38:00,827 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:38:01,827 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:38:01,900 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:38:03,782 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:38:03,782 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:38:03,782 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:38:03,787 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:38:03,828 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:38:05,831 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:38:05,880 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:38:07,826 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:38:08,141 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:38:08,142 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:38:08,142 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:38:08,147 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:38:08,833 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:38:09,835 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:38:10,880 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:38:11,326 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:38:11,327 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:38:11,327 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:38:11,332 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:38:11,836 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:38:13,332 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:38:13,838 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:38:15,280 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:38:15,299 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:38:15,299 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:38:15,300 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:38:15,304 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:38:15,839 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:38:17,842 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:38:18,341 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:38:19,096 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:38:19,097 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:38:19,097 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:38:19,102 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:38:19,844 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:38:20,881 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:38:21,847 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:38:23,263 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:38:23,264 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:38:23,264 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:38:23,269 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:38:23,848 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:38:24,270 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:38:25,851 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:38:25,882 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:38:26,457 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 17:38:27,148 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:38:27,149 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:38:27,149 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:38:27,154 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:38:27,852 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:38:29,853 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:38:30,154 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:38:30,280 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:38:31,114 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:38:31,115 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:38:31,115 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:38:31,119 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:38:31,855 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:38:33,858 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:38:35,158 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:38:35,866 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:38:35,867 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:38:35,867 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:38:35,872 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:38:35,883 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:38:36,860 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:38:37,862 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:38:40,473 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:38:40,474 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:38:40,474 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:38:40,474 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:38:40,479 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:38:40,865 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:38:40,883 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:38:41,868 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:38:44,626 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:38:44,627 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:38:44,627 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:38:44,632 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:38:44,869 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:38:45,280 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:38:45,632 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:38:45,870 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:38:49,423 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:38:49,423 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:38:49,424 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:38:49,428 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:38:49,872 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:38:50,884 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:38:51,468 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:38:51,875 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:38:55,049 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:38:55,050 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:38:55,050 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:38:55,055 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:38:55,878 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:38:55,885 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:38:56,459 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 17:38:57,460 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:38:57,882 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:38:58,362 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:38:58,363 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:38:58,363 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:38:58,368 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:38:58,882 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:38:59,885 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:39:00,280 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:39:02,224 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:39:02,224 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:39:02,224 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:39:02,229 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:39:02,887 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:39:03,265 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:39:03,889 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:39:05,886 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:39:07,367 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:39:07,367 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:39:07,368 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:39:07,372 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:39:07,892 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:39:08,373 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:39:09,894 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:39:10,886 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:39:11,295 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:39:11,296 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:39:11,296 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:39:11,301 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:39:11,896 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:39:13,899 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:39:14,302 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:39:14,806 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:39:14,807 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:39:14,807 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:39:14,812 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:39:14,900 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:39:15,280 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:39:15,902 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:39:19,813 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:39:20,887 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:39:23,179 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:39:23,180 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:39:23,180 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:39:23,185 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:39:23,907 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:39:25,185 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:39:25,887 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:39:25,910 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:39:26,461 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 17:39:28,905 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:39:28,905 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:39:28,906 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:39:28,910 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:39:28,911 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:39:29,914 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:39:30,280 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:39:30,911 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:39:33,208 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:39:33,208 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:39:33,208 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:39:33,214 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:39:33,916 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:39:35,888 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:39:35,918 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:39:36,249 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:39:36,849 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:39:36,850 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:39:36,850 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:39:36,855 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:39:36,918 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:39:39,923 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:39:40,889 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:39:41,349 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:39:41,349 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:39:41,350 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:39:41,350 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:39:41,354 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:39:41,925 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:39:43,928 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:39:45,280 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:39:45,868 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:39:45,869 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:39:45,869 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:39:45,874 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:39:45,929 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:39:46,909 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:39:47,932 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:39:50,878 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:39:50,878 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:39:50,879 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:39:50,883 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:39:50,890 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:39:50,934 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:39:52,884 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:39:53,937 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:39:54,843 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:39:54,844 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:39:54,844 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:39:54,849 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:39:54,938 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:39:55,891 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:39:56,466 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 17:39:57,941 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:39:58,468 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:40:00,280 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:40:03,410 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:40:03,411 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:40:03,411 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:40:03,416 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:40:03,944 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:40:04,453 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:40:05,892 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:40:05,947 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:40:07,325 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:40:07,325 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:40:07,325 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:40:07,330 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:40:07,948 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:40:09,951 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:40:10,331 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:40:10,892 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:40:11,634 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:40:11,635 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:40:11,635 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:40:11,639 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:40:11,953 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:40:13,956 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:40:15,280 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:40:15,640 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:40:17,340 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:40:17,341 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:40:17,341 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:40:17,346 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:40:17,958 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:40:19,961 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:40:20,893 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:40:20,972 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:40:20,973 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:40:20,973 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:40:20,973 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:40:20,977 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:40:21,962 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:40:23,965 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:40:25,894 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:40:25,978 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:40:26,465 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 17:40:29,390 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:40:29,391 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:40:29,391 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:40:29,396 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:40:29,969 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:40:30,280 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:40:31,397 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:40:31,970 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:40:33,547 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:40:33,548 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:40:33,548 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:40:33,553 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:40:33,971 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:40:35,895 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:40:35,975 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:40:36,590 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:40:39,339 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:40:39,339 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:40:39,339 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:40:39,344 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:40:39,979 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:40:40,895 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:40:41,982 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:40:42,345 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:40:43,817 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:40:43,818 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:40:43,818 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:40:43,823 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:40:43,983 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:40:45,280 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:40:45,986 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:40:47,824 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:40:48,818 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:40:48,819 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:40:48,819 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:40:48,824 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:40:48,988 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:40:50,896 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:40:51,992 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:40:52,862 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:40:53,187 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:40:53,188 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:40:53,188 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:40:53,192 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:40:53,993 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:40:55,897 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:40:55,996 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:40:56,470 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 17:40:57,943 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:40:57,944 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:40:57,944 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:40:57,944 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:40:57,949 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:40:57,999 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:41:00,002 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:41:00,280 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:41:02,949 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:41:03,553 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:41:03,554 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:41:03,554 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:41:03,559 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:41:04,004 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:41:05,898 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:41:06,007 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:41:07,083 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:41:07,083 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:41:07,084 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:41:07,088 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:41:08,008 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:41:08,089 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:41:10,011 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:41:10,162 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:41:10,163 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:41:10,163 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:41:10,168 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:41:10,898 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:41:11,012 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:41:12,014 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:41:13,168 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:41:14,874 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:41:14,875 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:41:14,875 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:41:14,880 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:41:15,016 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:41:15,281 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:41:18,020 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:41:18,880 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:41:19,414 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:41:19,414 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:41:19,414 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:41:19,419 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:41:20,021 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:41:20,899 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:41:22,025 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:41:24,458 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:41:25,900 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:41:26,469 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 17:41:27,821 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:41:27,822 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:41:27,822 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:41:27,827 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:41:28,028 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:41:29,827 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:41:30,031 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:41:30,281 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:41:31,180 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:41:31,180 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:41:31,181 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:41:31,186 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:41:32,032 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:41:34,035 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:41:34,683 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:41:34,683 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:41:34,684 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:41:34,689 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:41:35,036 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:41:35,690 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:41:35,901 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:41:38,038 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:41:40,690 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:41:40,901 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:41:43,179 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:41:43,180 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:41:43,180 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:41:43,185 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:41:44,043 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:41:45,281 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:41:46,046 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:41:46,185 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:41:47,575 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:41:47,575 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:41:47,576 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:41:47,581 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:41:48,047 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:41:50,050 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:41:50,902 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:41:51,617 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:41:51,822 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:41:51,822 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:41:51,823 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:41:51,827 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:41:52,051 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:41:54,054 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:41:55,317 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:41:55,318 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:41:55,318 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:41:55,323 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:41:55,903 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:41:56,056 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:41:56,471 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 17:41:57,473 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:41:58,060 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:42:00,281 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:42:00,324 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:42:00,324 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:42:00,324 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:42:00,329 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:42:01,061 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:42:02,064 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:42:03,369 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:42:05,904 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:42:07,498 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:42:07,499 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:42:07,499 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:42:07,504 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:42:08,067 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:42:08,504 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:42:10,070 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:42:10,904 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:42:13,505 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:42:15,281 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:42:15,926 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:42:15,927 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:42:15,927 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:42:15,932 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:42:16,074 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:42:18,077 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:42:18,970 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:42:19,443 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:42:19,444 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:42:19,444 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:42:19,448 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:42:20,078 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:42:20,905 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:42:22,081 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:42:22,677 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:42:22,677 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:42:22,678 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:42:22,682 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:42:23,082 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:42:24,084 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:42:24,683 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:42:25,896 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:42:25,896 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:42:25,897 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:42:25,901 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:42:25,906 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:42:26,086 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:42:26,474 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 17:42:28,089 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:42:29,144 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:42:29,145 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:42:29,145 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:42:29,150 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:42:30,090 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:42:30,150 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:42:30,590 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:42:32,093 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:42:35,151 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:42:35,590 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:42:36,011 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:42:36,012 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:42:36,012 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:42:36,017 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:42:36,096 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:42:38,099 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:42:39,654 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:42:39,654 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:42:39,655 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:42:39,659 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:42:40,101 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:42:40,591 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:42:40,660 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:42:42,104 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:42:43,166 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:42:43,167 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:42:43,167 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:42:43,171 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:42:44,106 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:42:45,590 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:42:46,107 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:42:46,172 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:42:47,294 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:42:47,295 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:42:47,295 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:42:47,299 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:42:48,108 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:42:50,111 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:42:50,592 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:42:50,964 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:42:50,965 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:42:50,965 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:42:50,970 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:42:51,112 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:42:51,970 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:42:54,115 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:42:54,706 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:42:54,707 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:42:54,707 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:42:54,712 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:42:55,116 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:42:55,592 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:42:56,119 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:42:56,476 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 17:42:57,477 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:42:59,515 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:42:59,516 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:42:59,516 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:42:59,521 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:43:00,122 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:43:00,592 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:43:02,125 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:43:02,521 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:43:03,994 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:43:03,994 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:43:03,995 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:43:04,000 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:43:04,126 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:43:05,593 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:43:06,129 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:43:08,038 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:43:08,468 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:43:08,469 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:43:08,469 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:43:08,474 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:43:09,132 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:43:10,134 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:43:10,594 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:43:11,676 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:43:11,676 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:43:11,676 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:43:11,681 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:43:12,135 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:43:13,682 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:43:14,137 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:43:15,590 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:43:15,656 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:43:15,657 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:43:15,657 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:43:15,662 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:43:16,138 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:43:18,141 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:43:18,697 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:43:19,563 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:43:19,564 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:43:19,564 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:43:19,568 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:43:20,142 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:43:20,595 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:43:22,146 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:43:24,050 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:43:24,051 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:43:24,051 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:43:24,051 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:43:24,056 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:43:24,147 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:43:25,595 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:43:26,150 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:43:26,478 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 17:43:28,578 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:43:28,579 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:43:28,579 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:43:28,584 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:43:29,151 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:43:29,584 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:43:30,154 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:43:30,590 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:43:34,585 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:43:34,661 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:43:34,661 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:43:34,661 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:43:34,666 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:43:35,157 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:43:35,596 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:43:36,158 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:43:38,358 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:43:38,359 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:43:38,359 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:43:38,364 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:43:39,159 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:43:40,164 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:43:40,364 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:43:40,597 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:43:42,215 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:43:42,215 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:43:42,216 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:43:42,221 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:43:43,166 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:43:44,168 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:43:45,591 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:43:46,222 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:43:46,602 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:43:46,603 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:43:46,603 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:43:46,608 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:43:47,170 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:43:48,170 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:43:50,105 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:43:50,105 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:43:50,105 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:43:50,110 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:43:50,172 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:43:50,598 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:43:52,111 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:43:52,175 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:43:53,748 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:43:53,749 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:43:53,749 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:43:53,754 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:43:54,176 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:43:55,602 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:43:56,179 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:43:56,480 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 17:43:57,482 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:43:58,595 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:43:58,596 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:43:58,596 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:43:58,601 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:43:59,182 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:44:00,185 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:44:00,591 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:44:02,179 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:44:02,179 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:44:02,179 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:44:02,184 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:44:02,186 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:44:03,221 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:44:04,189 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:44:05,603 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:44:05,707 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:44:05,708 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:44:05,708 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:44:05,713 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:44:06,190 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:44:08,193 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:44:08,713 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:44:10,459 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:44:10,459 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:44:10,459 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:44:10,464 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:44:10,603 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:44:11,195 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:44:12,198 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:44:13,345 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:44:13,346 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:44:13,346 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:44:13,350 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:44:14,199 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:44:14,351 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:44:15,591 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:44:16,203 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:44:16,993 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:44:16,994 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:44:16,994 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:44:16,999 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:44:17,204 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:44:20,041 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:44:20,207 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:44:20,456 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:44:20,457 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:44:20,457 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:44:20,462 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:44:20,604 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:44:21,208 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:44:22,208 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:44:25,096 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:44:25,097 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:44:25,097 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:44:25,098 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:44:25,102 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:44:25,210 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:44:25,605 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:44:26,482 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 17:44:28,215 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:44:29,751 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:44:29,751 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:44:29,752 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:44:29,757 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:44:30,216 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:44:30,591 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:44:30,757 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:44:32,219 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:44:35,572 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:44:35,573 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:44:35,573 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:44:35,578 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:44:35,613 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:44:36,222 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:44:36,613 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:44:38,225 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:44:39,302 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:44:39,303 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:44:39,303 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:44:39,308 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:44:40,228 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:44:40,606 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:44:42,231 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:44:42,308 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:44:45,591 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:44:47,308 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:44:47,866 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:44:47,867 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:44:47,867 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:44:47,872 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:44:48,234 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:44:50,237 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:44:50,607 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:44:52,134 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:44:52,135 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:44:52,135 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:44:52,140 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:44:52,238 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:44:53,140 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:44:54,241 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:44:55,607 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:44:56,485 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 17:44:58,486 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:44:58,665 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:44:58,666 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:44:58,666 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:44:58,671 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:44:59,245 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:45:00,248 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:45:00,591 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:45:03,672 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:45:03,776 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:45:03,777 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:45:03,777 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:45:03,782 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:45:04,250 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:45:05,608 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:45:06,253 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:45:08,027 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:45:08,028 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:45:08,028 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:45:08,032 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:45:08,254 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:45:09,033 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:45:10,257 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:45:10,609 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:45:11,888 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:45:11,889 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:45:11,889 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:45:11,894 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:45:12,258 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:45:14,260 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:45:14,895 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:45:15,591 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:45:16,218 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:45:16,218 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:45:16,219 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:45:16,224 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:45:16,261 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:45:18,265 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:45:20,262 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:45:20,610 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:45:21,845 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:45:21,846 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:45:21,846 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:45:21,851 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:45:22,267 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:45:24,271 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:45:25,603 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:45:25,604 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:45:25,604 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:45:25,604 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:45:25,609 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:45:25,610 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:45:26,272 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:45:26,490 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 17:45:28,275 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:45:29,411 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:45:29,411 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:45:29,412 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:45:29,416 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:45:30,276 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:45:30,591 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:45:31,417 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:45:32,279 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:45:33,823 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:45:33,824 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:45:33,824 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:45:33,828 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:45:34,281 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:45:35,611 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:45:36,284 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:45:36,865 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:45:37,664 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:45:37,664 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:45:37,665 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:45:37,669 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:45:38,285 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:45:40,289 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:45:40,612 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:45:41,165 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:45:41,166 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:45:41,166 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:45:41,171 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:45:41,290 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:45:42,172 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:45:44,294 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:45:45,318 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:45:45,318 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:45:45,319 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:45:45,323 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:45:45,591 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:45:46,295 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:45:47,324 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:45:48,298 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:45:49,811 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:45:49,811 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:45:49,812 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:45:49,816 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:45:50,299 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:45:50,613 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:45:52,302 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:45:52,853 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:45:55,613 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:45:56,489 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 17:45:56,934 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:45:56,934 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:45:56,935 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:45:56,940 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:45:57,307 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:45:57,940 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:45:58,307 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:46:00,591 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:46:02,941 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:46:04,026 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:46:04,026 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:46:04,026 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:46:04,031 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:46:04,311 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:46:05,614 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:46:06,314 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:46:07,697 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:46:07,698 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:46:07,699 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:46:07,703 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:46:08,315 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:46:08,704 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:46:10,318 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:46:10,614 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:46:11,788 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:46:11,789 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:46:11,789 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:46:11,794 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:46:12,319 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:46:13,794 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:46:14,323 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:46:15,591 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:46:16,803 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:46:16,803 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:46:16,803 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:46:16,808 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:46:17,325 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:46:18,327 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:46:18,845 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:46:20,615 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:46:21,305 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:46:21,306 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:46:21,306 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:46:21,311 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:46:21,329 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:46:24,312 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:46:24,333 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:46:24,428 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:46:24,428 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:46:24,428 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:46:24,433 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:46:25,334 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:46:25,616 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:46:26,337 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:46:26,493 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 17:46:28,215 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:46:28,215 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:46:28,215 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:46:28,220 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:46:28,338 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:46:30,221 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:46:30,341 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:46:30,591 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:46:32,768 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:46:32,769 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:46:32,769 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:46:32,773 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:46:33,343 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:46:34,345 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:46:35,617 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:46:35,809 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:46:37,258 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:46:37,259 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:46:37,259 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:46:37,264 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:46:37,347 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:46:40,352 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:46:40,617 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:46:41,094 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:46:41,095 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:46:41,095 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:46:41,095 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:46:41,100 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:46:41,353 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:46:44,356 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:46:45,591 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:46:46,100 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:46:49,475 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:46:49,476 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:46:49,476 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:46:49,481 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:46:50,360 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:46:50,618 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:46:51,517 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:46:52,363 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:46:53,779 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:46:53,780 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:46:53,780 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:46:53,785 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:46:54,364 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:46:55,619 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:46:56,365 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:46:56,492 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 17:46:57,493 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:46:58,139 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:46:58,139 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:46:58,139 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:46:58,145 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:46:58,368 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:47:00,371 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:47:00,592 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:47:01,209 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:47:01,210 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:47:01,210 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:47:01,215 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:47:01,371 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:47:03,254 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:47:04,375 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:47:05,062 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:47:05,063 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:47:05,063 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:47:05,068 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:47:05,375 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:47:05,620 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:47:08,377 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:47:08,825 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:47:08,826 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:47:08,826 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:47:08,826 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:47:08,831 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:47:09,378 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:47:10,380 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:47:10,620 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:47:13,832 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:47:14,703 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:47:14,703 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:47:14,704 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:47:14,709 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:47:15,384 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:47:15,592 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:47:16,386 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:47:19,160 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:47:19,160 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:47:19,161 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:47:19,161 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:47:19,165 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:47:19,388 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:47:20,621 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:47:22,391 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:47:24,202 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:47:24,586 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:47:24,587 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:47:24,587 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:47:24,592 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:47:25,393 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:47:25,622 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:47:26,396 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:47:26,498 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 17:47:29,499 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:47:30,592 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:47:33,228 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:47:33,229 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:47:33,229 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:47:33,234 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:47:33,399 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:47:35,269 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:47:35,623 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:47:36,276 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:47:36,277 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:47:36,277 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:47:36,282 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:47:36,402 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:47:36,402 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:47:38,405 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:47:39,737 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:47:39,737 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:47:39,738 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:47:39,742 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:47:40,408 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:47:40,623 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:47:40,743 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:47:42,411 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:47:45,592 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:47:45,743 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:47:46,008 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:47:46,008 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:47:46,009 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:47:46,013 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:47:46,413 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:47:48,416 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:47:49,441 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:47:49,441 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:47:49,442 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:47:49,446 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:47:50,417 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:47:50,624 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:47:51,447 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:47:52,421 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:47:52,874 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:47:52,875 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:47:52,875 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:47:52,880 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:47:53,421 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:47:54,424 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:47:55,625 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:47:56,497 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 17:47:56,498 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:47:56,610 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:47:56,610 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:47:56,610 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:47:56,615 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:47:57,427 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:47:58,427 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:48:00,113 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:48:00,113 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:48:00,114 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:48:00,118 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:48:00,429 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:48:00,592 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:48:02,119 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:48:02,432 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:48:04,359 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:48:04,359 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:48:04,359 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:48:04,365 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:48:04,433 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:48:05,626 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:48:06,436 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:48:07,402 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:48:07,999 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:48:08,000 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:48:08,000 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:48:08,005 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:48:08,437 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:48:10,440 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:48:10,627 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:48:12,240 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:48:12,240 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:48:12,241 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:48:12,246 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:48:12,441 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:48:13,246 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:48:14,445 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:48:15,592 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:48:16,597 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:48:16,597 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:48:16,597 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:48:16,602 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:48:17,446 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:48:18,449 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:48:18,637 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:48:20,628 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:48:22,747 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:48:22,748 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:48:22,748 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:48:22,753 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:48:23,452 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:48:23,753 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:48:24,454 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:48:25,628 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:48:26,178 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:48:26,179 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:48:26,179 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:48:26,184 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:48:26,455 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:48:26,499 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 17:48:28,457 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:48:29,500 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:48:30,592 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:48:34,501 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:48:34,566 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:48:34,570 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:48:34,570 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:48:34,575 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:48:35,461 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:48:35,629 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:48:36,463 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:48:37,635 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:48:37,636 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:48:37,636 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:48:37,641 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:48:38,465 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:48:39,642 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:48:40,467 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:48:40,629 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:48:41,150 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:48:41,151 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:48:41,151 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:48:41,156 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:48:41,468 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:48:42,470 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:48:45,156 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:48:45,592 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:48:46,463 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:48:46,463 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:48:46,463 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:48:46,468 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:48:46,472 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:48:48,475 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:48:50,506 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:48:50,631 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:48:53,971 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:48:53,971 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:48:53,971 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:48:53,976 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:48:54,479 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:48:55,631 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:48:55,977 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:48:56,482 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:48:56,501 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 17:48:58,253 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:48:58,254 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:48:58,254 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:48:58,259 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:48:58,484 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:49:00,487 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:49:00,592 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:49:01,260 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:49:01,626 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:49:01,627 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:49:01,627 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:49:01,632 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:49:02,489 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:49:04,492 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:49:05,632 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:49:06,210 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:49:06,211 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:49:06,211 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:49:06,217 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:49:06,493 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:49:07,217 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:49:08,496 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:49:10,633 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:49:11,306 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:49:11,307 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:49:11,307 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:49:11,312 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:49:11,498 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:49:12,312 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:49:14,502 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:49:15,592 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:49:16,340 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:49:16,341 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:49:16,341 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:49:16,346 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:49:16,503 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:49:17,381 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:49:18,506 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:49:19,881 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:49:19,882 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:49:19,882 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:49:19,887 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:49:20,507 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:49:20,634 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:49:22,508 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:49:22,887 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:49:23,615 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:49:23,615 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:49:23,616 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:49:23,620 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:49:24,510 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:49:25,634 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:49:26,503 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 17:49:26,513 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:49:27,352 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:49:27,353 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:49:27,353 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:49:27,358 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:49:27,513 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:49:28,359 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:49:30,517 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:49:30,592 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:49:33,332 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:49:33,333 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:49:33,333 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:49:33,338 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:49:33,373 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:49:33,518 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:49:35,635 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:49:36,522 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:49:38,374 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:49:40,636 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:49:41,926 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:49:41,927 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:49:41,927 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:49:41,932 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:49:42,527 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:49:43,933 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:49:44,530 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:49:45,592 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:49:46,798 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:49:46,798 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:49:46,799 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:49:46,804 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:49:47,532 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:49:48,535 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:49:49,841 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:49:50,413 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:49:50,413 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:49:50,414 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:49:50,418 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:49:50,536 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:49:50,637 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:49:52,537 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:49:55,419 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:49:55,637 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:49:55,819 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:49:55,820 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:49:55,820 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:49:55,825 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:49:56,505 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 17:49:56,539 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:49:58,541 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:50:00,507 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:50:00,592 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:50:02,724 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:50:02,725 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:50:02,725 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:50:02,730 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:50:03,544 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:50:04,547 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:50:05,638 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:50:05,765 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:50:05,987 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:50:05,988 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:50:05,988 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:50:05,993 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:50:06,548 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:50:08,551 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:50:09,660 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:50:09,661 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:50:09,661 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:50:09,666 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:50:10,552 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:50:10,639 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:50:11,666 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:50:12,555 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:50:13,963 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:50:13,964 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:50:13,964 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:50:13,969 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:50:14,557 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:50:15,593 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:50:16,560 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:50:16,970 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:50:16,971 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:50:16,972 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:50:16,972 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:50:16,977 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:50:17,561 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:50:18,563 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:50:20,640 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:50:21,903 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:50:21,904 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:50:21,904 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:50:21,909 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:50:22,565 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:50:22,909 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:50:24,568 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:50:25,640 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:50:26,507 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 17:50:28,509 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:50:30,367 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:50:30,367 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:50:30,367 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:50:30,372 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:50:30,572 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:50:30,593 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:50:32,575 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:50:34,094 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:50:34,095 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:50:34,095 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:50:34,095 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:50:34,100 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:50:34,576 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:50:35,641 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:50:36,579 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:50:38,614 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:50:38,615 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:50:38,615 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:50:38,620 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:50:39,581 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:50:39,620 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:50:40,585 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:50:40,641 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:50:41,536 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:50:41,536 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:50:41,537 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:50:41,541 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:50:41,586 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:50:44,589 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:50:45,035 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:50:45,035 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:50:45,036 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:50:45,036 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:50:45,041 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:50:45,590 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:50:45,593 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:50:46,592 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:50:48,513 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:50:48,514 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:50:48,514 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:50:48,519 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:50:48,594 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:50:50,558 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:50:50,595 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:50:50,642 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:50:51,366 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:50:51,367 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:50:51,367 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:50:51,372 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:50:51,596 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:50:54,599 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:50:55,243 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:50:55,244 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:50:55,244 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:50:55,249 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:50:55,600 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:50:55,643 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:50:56,249 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:50:56,510 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 17:50:56,602 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:51:00,592 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:51:00,593 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:51:00,593 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:51:00,594 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:51:00,598 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:51:00,606 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:51:01,598 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:51:02,609 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:51:05,062 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:51:05,062 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:51:05,063 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:51:05,067 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:51:05,610 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:51:05,644 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:51:06,613 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:51:07,105 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:51:09,860 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:51:09,861 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:51:09,861 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:51:09,866 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:51:10,615 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:51:10,644 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:51:12,616 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:51:12,866 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:51:14,623 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:51:14,624 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:51:14,624 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:51:14,629 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:51:15,594 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:51:15,618 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:51:16,621 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:51:18,408 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:51:18,409 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:51:18,409 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:51:18,409 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:51:18,414 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:51:18,622 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:51:20,625 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:51:20,645 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:51:22,707 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:51:22,708 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:51:22,708 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:51:22,713 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:51:23,627 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:51:23,713 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:51:24,629 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:51:25,646 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:51:26,195 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:51:26,195 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:51:26,196 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:51:26,201 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:51:26,512 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 17:51:26,630 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:51:28,634 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:51:29,513 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:51:30,259 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:51:30,260 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:51:30,260 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:51:30,265 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:51:30,594 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:51:30,637 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:51:32,640 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:51:33,955 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:51:33,956 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:51:33,956 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:51:33,961 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:51:34,641 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:51:34,997 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:51:35,647 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:51:36,644 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:51:39,998 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:51:40,647 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:51:42,343 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:51:42,344 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:51:42,344 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:51:42,348 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:51:42,649 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:51:44,652 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:51:45,349 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:51:45,594 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:51:46,451 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:51:46,451 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:51:46,452 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:51:46,456 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:51:46,653 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:51:48,656 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:51:49,639 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:51:49,640 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:51:49,640 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:51:49,645 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:51:49,657 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:51:50,646 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:51:50,648 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:51:52,661 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:51:53,165 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:51:53,166 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:51:53,166 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:51:53,171 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:51:53,661 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:51:54,664 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:51:55,649 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:51:56,171 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:51:56,517 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 17:51:56,870 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:51:56,871 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:51:56,871 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:51:56,876 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:51:57,667 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:51:58,669 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:52:00,594 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:52:01,101 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:52:01,102 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:52:01,102 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:52:01,107 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:52:01,671 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:52:02,145 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:52:02,672 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:52:04,543 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:52:04,544 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:52:04,544 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:52:04,549 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:52:04,673 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:52:05,650 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:52:06,676 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:52:07,550 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:52:08,243 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:52:08,244 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:52:08,244 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:52:08,248 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:52:08,677 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:52:10,650 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:52:10,680 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:52:11,543 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:52:11,544 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:52:11,544 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:52:11,549 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:52:11,681 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:52:13,549 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:52:14,685 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:52:15,594 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:52:18,550 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:52:20,088 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:52:20,088 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:52:20,088 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:52:20,093 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:52:20,651 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:52:20,688 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:52:22,691 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:52:23,301 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:52:23,302 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:52:23,302 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:52:23,307 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:52:23,692 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:52:24,308 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:52:24,694 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:52:25,652 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:52:26,516 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 17:52:26,882 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:52:26,882 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:52:26,883 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:52:26,887 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:52:27,696 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:52:28,698 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:52:29,888 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:52:30,325 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:52:30,325 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:52:30,326 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:52:30,330 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:52:30,594 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:52:30,700 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:52:32,703 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:52:35,271 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:52:35,271 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:52:35,272 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:52:35,272 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:52:35,277 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:52:35,704 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:52:36,112 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:52:36,707 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:52:39,627 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:52:39,628 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:52:39,628 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:52:39,633 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:52:39,709 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:52:40,634 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:52:41,113 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:52:42,714 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:52:45,448 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:52:45,448 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:52:45,449 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:52:45,454 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:52:45,594 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:52:45,715 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:52:46,454 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:52:48,719 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:52:49,050 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:52:49,050 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:52:49,050 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:52:49,055 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:52:49,720 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:52:50,720 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:52:51,114 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:52:52,093 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:52:52,335 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:52:52,336 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:52:52,336 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:52:52,340 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:52:52,721 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:52:54,724 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:52:56,114 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:52:56,521 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 17:52:57,522 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:52:58,771 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:52:58,771 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:52:58,772 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:52:58,776 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:52:59,729 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:53:00,594 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:53:00,732 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:53:02,454 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:53:02,454 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:53:02,455 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:53:02,459 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:53:02,733 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:53:03,497 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:53:04,736 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:53:06,115 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:53:08,498 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:53:09,749 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:53:09,750 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:53:09,751 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:53:09,755 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:53:10,739 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:53:11,116 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:53:12,742 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:53:13,756 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:53:15,190 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:53:15,191 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:53:15,191 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:53:15,196 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:53:15,595 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:53:15,744 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:53:16,747 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:53:19,196 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:53:21,077 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:53:21,078 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:53:21,078 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:53:21,083 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:53:21,121 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:53:21,750 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:53:22,752 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:53:25,122 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:53:26,117 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:53:26,477 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:53:26,478 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:53:26,478 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:53:26,483 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:53:26,520 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 17:53:26,754 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:53:28,757 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:53:30,521 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:53:30,594 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:53:30,774 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:53:30,775 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:53:30,775 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:53:30,779 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:53:31,759 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:53:32,761 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:53:35,163 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:53:35,164 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:53:35,164 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:53:35,169 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:53:35,763 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:53:36,118 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:53:36,170 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:53:36,766 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:53:38,336 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:53:38,337 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:53:38,337 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:53:38,342 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:53:38,767 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:53:40,772 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:53:41,119 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:53:41,342 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:53:42,850 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:53:42,851 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:53:42,851 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:53:42,856 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:53:43,773 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:53:44,776 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:53:45,595 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:53:46,857 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:53:47,837 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:53:47,838 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:53:47,838 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:53:47,842 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:53:48,778 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:53:50,781 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:53:51,120 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:53:51,882 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:53:56,120 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:53:56,438 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:53:56,439 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:53:56,439 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:53:56,444 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:53:56,525 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 17:53:56,785 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:53:57,527 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:53:58,789 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:53:59,988 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:53:59,989 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:53:59,989 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:53:59,994 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:54:00,595 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:54:00,790 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:54:02,793 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:54:02,995 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:54:05,265 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:54:05,266 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:54:05,266 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:54:05,271 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:54:05,795 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:54:06,121 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:54:06,798 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:54:08,310 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:54:09,763 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:54:09,764 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:54:09,764 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:54:09,769 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:54:09,800 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:54:11,122 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:54:12,803 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:54:13,249 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:54:13,250 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:54:13,250 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:54:13,255 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:54:13,804 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:54:14,256 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:54:14,807 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:54:15,595 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:54:16,270 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:54:16,271 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:54:16,271 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:54:16,276 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:54:16,808 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:54:18,811 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:54:19,314 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:54:20,087 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:54:20,088 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:54:20,088 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:54:20,093 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:54:20,812 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:54:21,123 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:54:22,816 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:54:23,579 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:54:23,579 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:54:23,579 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:54:23,584 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:54:23,816 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:54:24,585 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:54:26,123 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:54:26,524 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 17:54:26,820 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:54:27,190 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:54:27,191 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:54:27,191 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:54:27,197 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:54:27,821 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:54:28,823 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:54:30,198 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:54:30,595 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:54:30,973 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:54:30,973 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:54:30,974 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:54:30,979 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:54:31,825 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:54:32,827 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:54:34,597 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:54:34,598 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:54:34,598 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:54:34,603 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:54:34,829 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:54:35,603 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:54:36,127 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:54:36,832 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:54:39,980 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:54:39,981 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:54:39,981 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:54:39,986 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:54:40,835 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:54:40,986 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:54:41,127 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:54:42,838 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:54:43,891 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:54:43,892 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:54:43,892 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:54:43,897 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:54:44,840 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:54:45,595 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:54:46,841 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:54:46,898 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:54:48,159 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:54:48,160 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:54:48,160 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:54:48,165 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:54:48,842 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:54:50,845 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:54:51,128 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:54:52,202 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:54:52,431 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:54:52,432 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:54:52,432 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:54:52,437 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:54:52,846 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:54:54,849 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:54:55,767 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:54:55,768 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:54:55,768 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:54:55,773 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:54:55,850 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:54:56,129 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:54:56,526 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 17:54:57,527 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:54:58,854 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:55:00,595 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:55:01,121 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:55:01,125 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:55:01,125 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:55:01,130 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:55:01,856 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:55:02,857 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:55:03,165 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:55:05,228 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:55:05,229 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:55:05,229 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:55:05,234 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:55:05,859 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:55:06,130 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:55:06,861 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:55:08,235 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:55:08,770 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:55:08,770 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:55:08,771 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:55:08,775 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:55:08,863 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:55:10,866 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:55:11,130 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:55:13,776 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:55:15,595 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:55:16,942 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:55:16,943 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:55:16,943 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:55:16,947 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:55:17,870 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:55:18,873 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:55:18,985 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:55:21,010 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:55:21,011 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:55:21,011 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:55:21,016 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:55:21,132 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:55:21,875 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:55:22,877 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:55:24,016 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:55:24,200 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:55:24,201 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:55:24,202 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:55:24,207 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:55:24,878 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:55:26,132 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:55:26,528 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 17:55:26,879 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:55:27,705 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:55:27,706 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:55:27,706 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:55:27,711 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:55:27,880 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:55:29,711 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:55:30,595 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:55:30,884 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:55:31,549 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:55:31,549 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:55:31,550 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:55:31,554 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:55:31,884 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:55:32,887 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:55:35,590 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:55:35,649 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:55:35,649 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:55:35,650 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:55:35,654 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:55:35,889 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:55:36,133 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:55:38,892 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:55:39,288 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:55:39,289 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:55:39,289 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:55:39,294 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:55:39,893 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:55:40,897 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:55:41,133 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:55:41,294 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:55:45,595 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:55:46,295 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:55:48,326 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:55:48,326 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:55:48,327 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:55:48,332 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:55:48,902 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:55:50,903 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:55:51,134 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:55:51,369 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:55:52,993 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:55:52,994 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:55:52,994 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:55:52,999 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:55:53,904 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:55:54,907 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:55:56,135 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:55:56,531 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 17:55:56,532 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:55:56,740 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:55:56,740 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:55:56,741 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:55:56,745 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:55:56,908 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:55:58,912 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:56:00,172 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:56:00,173 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:56:00,173 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:56:00,178 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:56:00,595 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:56:00,914 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:56:02,178 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:56:02,917 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:56:04,559 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:56:04,560 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:56:04,560 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:56:04,565 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:56:04,918 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:56:06,136 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:56:06,921 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:56:07,601 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:56:08,995 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:56:08,996 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:56:08,996 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:56:09,001 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:56:09,923 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:56:10,925 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:56:11,136 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:56:13,001 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:56:13,586 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:56:13,589 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:56:13,589 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:56:13,594 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:56:13,927 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:56:14,930 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:56:15,595 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:56:17,021 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:56:17,022 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:56:17,022 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:56:17,027 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:56:17,931 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:56:18,065 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:56:18,932 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:56:21,127 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:56:21,128 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:56:21,128 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:56:21,133 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:56:21,137 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:56:21,934 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:56:22,936 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:56:23,133 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:56:24,688 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:56:24,689 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:56:24,689 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:56:24,694 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:56:24,938 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:56:26,138 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:56:26,533 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 17:56:26,941 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:56:28,534 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:56:29,812 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:56:29,813 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:56:29,813 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:56:29,818 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:56:29,942 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:56:30,595 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:56:32,946 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:56:33,818 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:56:35,607 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:56:35,608 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:56:35,608 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:56:35,612 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:56:35,948 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:56:36,139 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:56:36,950 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:56:38,960 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:56:38,961 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:56:38,961 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:56:38,961 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:56:38,966 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:56:39,952 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:56:40,954 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:56:41,139 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:56:42,331 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:56:42,331 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:56:42,331 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:56:42,336 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:56:42,955 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:56:44,337 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:56:44,959 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:56:45,595 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:56:45,828 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:56:45,829 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:56:45,829 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:56:45,834 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:56:45,959 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:56:48,963 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:56:49,870 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:56:50,315 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:56:50,316 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:56:50,316 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:56:50,321 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:56:50,964 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:56:51,140 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:56:52,967 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:56:54,685 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:56:54,686 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:56:54,686 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:56:54,691 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:56:54,968 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:56:55,691 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:56:56,141 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:56:56,535 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 17:56:56,972 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:56:58,322 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:56:58,323 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:56:58,323 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:56:58,328 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:56:58,974 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:57:00,596 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:57:00,977 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:57:01,328 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:57:02,309 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:57:02,310 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:57:02,310 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:57:02,315 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:57:02,978 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:57:04,981 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:57:06,142 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:57:06,354 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:57:10,729 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:57:10,730 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:57:10,730 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:57:10,734 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:57:10,985 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:57:11,142 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:57:11,735 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:57:12,988 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:57:15,596 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:57:16,323 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:57:16,324 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:57:16,324 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:57:16,328 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:57:16,990 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:57:17,365 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:57:18,993 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:57:21,143 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:57:22,366 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:57:24,025 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:57:24,026 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:57:24,026 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:57:24,031 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:57:24,996 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:57:26,144 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:57:26,537 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 17:57:26,999 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:57:27,462 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:57:27,463 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:57:27,463 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:57:27,463 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:57:27,468 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:57:28,000 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:57:29,002 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:57:30,596 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:57:31,011 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:57:31,012 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:57:31,012 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:57:31,017 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:57:32,004 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:57:33,007 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:57:33,053 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:57:34,605 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:57:34,606 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:57:34,606 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:57:34,610 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:57:35,008 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:57:36,145 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:57:37,011 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:57:38,611 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:57:41,145 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:57:43,259 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:57:43,260 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:57:43,260 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:57:43,265 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:57:44,017 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:57:44,266 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:57:45,019 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:57:45,596 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:57:48,454 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:57:48,455 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:57:48,455 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:57:48,460 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:57:49,021 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:57:49,497 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:57:51,025 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:57:51,146 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:57:51,877 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:57:51,877 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:57:51,878 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:57:51,883 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:57:52,025 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:57:54,883 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:57:55,029 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:57:56,147 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:57:56,193 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:57:56,194 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:57:56,194 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:57:56,200 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:57:56,539 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 17:57:57,030 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:57:59,035 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:58:00,541 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:58:00,596 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:58:04,842 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:58:04,843 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:58:04,843 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:58:04,848 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:58:05,038 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:58:05,885 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:58:06,148 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:58:07,041 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:58:09,624 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:58:09,625 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:58:09,625 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:58:09,630 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:58:10,043 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:58:11,044 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:58:11,149 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:58:11,630 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:58:14,857 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:58:14,858 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:58:14,858 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:58:14,863 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:58:15,047 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:58:15,596 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:58:16,863 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:58:17,048 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:58:18,369 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:58:18,370 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:58:18,370 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:58:18,374 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:58:19,049 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:58:21,052 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:58:21,150 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:58:22,414 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:58:23,371 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:58:23,372 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:58:23,372 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:58:23,376 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:58:24,054 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:58:25,057 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:58:26,150 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:58:26,544 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 17:58:27,546 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:58:28,930 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:58:28,931 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:58:28,931 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:58:28,936 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:58:29,059 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:58:30,596 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:58:31,062 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:58:32,937 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:58:33,409 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:58:33,410 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:58:33,410 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:58:33,415 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:58:34,064 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:58:35,066 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:58:36,151 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:58:36,918 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:58:36,919 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:58:36,919 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:58:36,924 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:58:37,068 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:58:38,924 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:58:39,069 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:58:40,281 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:58:40,282 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:58:40,282 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:58:40,287 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:58:41,072 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:58:41,152 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:58:43,075 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:58:44,288 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:58:45,596 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:58:48,655 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:58:48,656 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:58:48,656 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:58:48,661 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:58:49,078 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:58:49,697 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:58:51,081 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:58:51,153 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:58:51,944 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:58:51,945 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:58:51,945 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:58:51,950 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:58:52,082 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:58:54,950 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:58:55,084 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:58:56,013 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:58:56,014 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:58:56,014 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:58:56,019 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:58:56,084 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:58:56,153 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:58:56,543 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 17:58:59,089 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:58:59,341 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:58:59,342 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:58:59,342 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:58:59,347 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:59:00,090 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:59:00,347 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:59:00,596 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:59:01,093 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:59:02,979 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:59:02,979 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:59:02,979 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:59:02,984 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:59:03,094 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:59:05,097 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:59:06,021 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:59:06,154 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:59:07,438 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:59:07,439 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:59:07,439 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:59:07,445 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:59:08,099 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:59:09,101 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:59:11,029 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:59:11,030 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:59:11,030 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:59:11,030 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:59:11,035 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:59:11,102 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:59:11,155 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:59:13,106 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:59:15,143 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:59:15,144 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:59:15,144 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:59:15,149 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:59:15,596 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:59:16,108 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:59:16,149 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:59:17,110 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:59:18,833 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:59:18,834 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:59:18,834 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:59:18,839 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:59:19,112 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:59:21,115 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:59:21,156 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:59:21,877 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:59:24,254 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:59:24,255 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:59:24,255 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:59:24,262 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:59:25,117 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:59:26,156 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:59:26,548 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 17:59:27,120 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:59:27,549 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:59:29,411 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:59:29,412 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:59:29,412 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:59:29,419 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:59:30,122 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:59:30,596 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:59:31,124 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:59:33,419 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:59:34,191 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:59:34,192 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:59:34,192 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:59:34,197 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:59:35,127 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:59:36,157 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:59:37,130 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:59:37,559 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:59:37,559 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:59:37,560 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:59:37,565 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:59:38,131 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:59:38,566 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:59:39,133 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:59:40,790 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:59:40,791 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:59:40,791 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:59:40,795 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:59:41,136 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:59:41,158 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:59:43,139 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:59:43,796 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:59:45,596 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:59:45,767 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:59:45,768 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:59:45,768 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:59:45,773 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:59:46,141 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:59:47,143 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:59:48,810 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:59:51,159 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:59:51,568 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:59:51,568 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:59:51,569 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:59:51,573 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:59:52,146 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:59:53,149 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 17:59:54,574 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:59:55,190 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:59:55,191 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 17:59:55,191 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:59:55,196 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:59:56,151 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 17:59:56,160 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 17:59:56,548 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 17:59:57,151 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:00:00,549 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:00:00,596 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:00:01,345 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:00:01,345 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:00:01,346 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:00:01,350 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:00:02,156 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:00:03,156 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:00:06,161 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:00:06,390 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:00:06,672 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:00:06,673 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:00:06,673 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:00:06,679 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:00:07,159 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:00:09,160 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:00:10,117 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:00:10,118 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:00:10,118 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:00:10,123 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:00:10,160 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:00:11,161 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:00:12,124 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:00:13,164 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:00:15,597 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:00:15,731 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:00:15,731 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:00:15,731 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:00:15,736 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:00:16,166 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:00:17,169 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:00:17,773 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:00:20,012 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:00:20,013 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:00:20,013 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:00:20,018 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:00:20,170 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:00:21,162 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:00:23,019 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:00:23,174 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:00:25,621 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:00:25,622 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:00:25,622 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:00:25,627 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:00:26,163 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:00:26,176 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:00:26,553 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 18:00:27,178 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:00:28,554 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:00:30,597 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:00:30,733 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:00:30,733 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:00:30,733 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:00:30,738 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:00:31,181 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:00:33,184 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:00:33,773 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:00:36,164 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:00:38,774 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:00:39,358 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:00:39,358 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:00:39,359 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:00:39,363 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:00:40,188 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:00:41,164 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:00:41,192 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:00:42,896 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:00:42,897 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:00:42,897 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:00:42,902 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:00:43,193 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:00:43,902 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:00:45,196 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:00:45,597 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:00:47,512 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:00:47,513 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:00:47,513 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:00:47,518 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:00:48,198 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:00:49,198 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:00:49,553 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:00:51,166 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:00:51,872 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:00:51,873 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:00:51,873 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:00:51,878 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:00:52,200 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:00:53,203 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:00:54,879 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:00:56,166 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:00:56,552 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 18:00:57,283 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:00:57,284 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:00:57,284 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:00:57,289 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:00:58,207 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:00:59,209 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:01:00,290 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:01:00,597 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:01:01,154 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:01:01,155 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:01:01,155 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:01:01,160 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:01:01,210 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:01:03,213 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:01:04,278 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:01:04,278 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:01:04,279 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:01:04,283 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:01:05,215 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:01:06,167 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:01:06,284 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:01:07,218 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:01:07,517 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:01:07,518 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:01:07,518 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:01:07,523 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:01:08,219 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:01:09,221 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:01:11,168 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:01:11,523 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:01:11,809 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:01:11,810 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:01:11,810 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:01:11,815 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:01:12,223 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:01:13,225 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:01:15,550 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:01:15,551 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:01:15,551 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:01:15,556 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:01:15,597 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:01:16,228 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:01:16,556 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:01:17,228 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:01:20,682 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:01:20,683 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:01:20,683 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:01:20,688 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:01:21,169 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:01:21,231 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:01:21,725 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:01:23,234 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:01:26,170 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:01:26,554 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 18:01:27,555 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:01:29,128 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:01:29,129 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:01:29,129 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:01:29,134 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:01:29,237 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:01:30,597 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:01:31,240 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:01:33,134 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:01:33,803 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:01:33,804 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:01:33,804 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:01:33,809 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:01:34,242 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:01:35,244 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:01:36,171 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:01:38,173 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:01:38,174 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:01:38,174 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:01:38,174 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:01:38,179 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:01:38,246 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:01:41,171 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:01:41,252 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:01:41,672 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:01:41,678 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:01:41,678 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:01:41,683 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:01:42,252 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:01:43,255 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:01:43,684 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:01:45,333 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:01:45,334 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:01:45,334 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:01:45,339 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:01:45,597 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:01:46,256 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:01:47,259 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:01:49,340 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:01:51,172 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:01:53,766 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:01:53,767 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:01:53,767 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:01:53,772 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:01:54,263 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:01:54,772 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:01:55,264 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:01:56,173 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:01:56,556 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 18:01:57,542 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:01:57,543 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:01:57,543 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:01:57,548 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:01:58,267 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:01:59,269 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:02:00,548 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:02:00,597 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:02:01,031 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:02:01,031 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:02:01,031 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:02:01,036 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:02:01,271 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:02:03,274 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:02:05,303 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:02:05,304 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:02:05,304 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:02:05,309 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:02:06,174 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:02:06,276 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:02:06,309 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:02:07,278 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:02:09,576 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:02:09,577 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:02:09,577 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:02:09,582 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:02:10,280 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:02:11,175 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:02:11,283 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:02:11,582 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:02:13,838 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:02:13,839 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:02:13,839 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:02:13,844 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:02:14,285 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:02:15,288 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:02:15,597 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:02:16,844 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:02:17,368 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:02:17,368 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:02:17,369 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:02:17,374 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:02:18,290 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:02:19,293 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:02:21,176 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:02:22,410 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:02:23,148 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:02:23,148 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:02:23,149 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:02:23,154 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:02:23,296 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:02:25,302 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:02:26,176 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:02:26,558 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 18:02:26,868 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:02:26,869 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:02:26,869 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:02:26,874 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:02:27,303 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:02:27,874 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:02:29,307 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:02:30,597 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:02:32,875 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:02:35,531 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:02:35,532 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:02:35,532 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:02:35,537 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:02:36,177 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:02:36,317 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:02:37,320 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:02:38,574 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:02:39,599 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:02:39,600 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:02:39,600 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:02:39,605 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:02:40,323 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:02:41,178 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:02:41,325 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:02:43,147 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:02:43,147 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:02:43,148 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:02:43,153 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:02:43,326 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:02:44,153 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:02:45,329 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:02:45,597 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:02:46,699 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:02:46,699 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:02:46,699 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:02:46,705 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:02:47,331 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:02:49,334 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:02:49,742 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:02:50,250 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:02:50,251 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:02:50,251 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:02:50,256 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:02:50,335 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:02:51,179 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:02:53,338 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:02:53,927 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:02:53,928 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:02:53,928 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:02:53,933 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:02:54,339 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:02:54,934 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:02:55,341 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:02:56,180 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:02:56,560 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 18:02:58,942 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:02:58,942 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:02:58,942 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:02:58,947 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:02:59,345 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:02:59,948 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:03:00,597 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:03:01,348 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:03:02,719 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:03:02,720 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:03:02,720 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:03:02,725 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:03:03,349 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:03:05,352 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:03:05,761 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:03:06,181 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:03:06,518 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:03:06,519 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:03:06,519 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:03:06,524 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:03:07,354 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:03:09,357 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:03:11,181 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:03:11,524 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:03:11,957 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:03:11,958 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:03:11,958 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:03:11,963 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:03:12,359 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:03:13,361 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:03:15,598 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:03:16,964 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:03:17,309 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:03:17,309 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:03:17,310 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:03:17,315 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:03:17,364 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:03:19,367 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:03:21,182 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:03:22,355 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:03:25,876 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:03:25,877 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:03:25,877 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:03:25,882 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:03:26,183 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:03:26,371 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:03:26,562 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 18:03:27,374 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:03:27,564 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:03:29,237 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:03:29,238 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:03:29,238 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:03:29,243 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:03:29,375 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:03:30,598 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:03:31,378 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:03:32,879 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:03:32,880 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:03:32,880 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:03:32,880 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:03:32,885 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:03:33,379 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:03:35,382 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:03:36,154 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:03:36,154 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:03:36,155 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:03:36,160 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:03:36,184 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:03:36,383 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:03:37,386 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:03:38,160 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:03:39,957 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:03:39,958 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:03:39,958 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:03:39,963 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:03:40,388 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:03:41,185 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:03:41,391 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:03:43,963 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:03:44,194 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:03:44,195 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:03:44,195 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:03:44,200 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:03:44,393 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:03:45,396 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:03:45,598 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:03:47,380 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:03:47,380 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:03:47,381 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:03:47,385 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:03:47,397 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:03:49,398 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:03:49,422 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:03:51,155 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:03:51,156 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:03:51,156 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:03:51,161 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:03:51,186 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:03:51,399 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:03:53,403 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:03:54,863 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:03:54,864 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:03:54,864 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:03:54,864 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:03:54,869 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:03:55,404 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:03:56,186 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:03:56,568 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 18:03:57,407 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:03:58,499 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:03:58,500 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:03:58,500 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:03:58,505 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:03:59,409 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:04:00,505 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:04:00,598 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:04:01,412 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:04:02,280 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:04:02,280 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:04:02,281 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:04:02,287 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:04:02,413 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:04:05,417 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:04:06,187 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:04:06,323 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:04:07,185 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:04:07,186 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:04:07,186 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:04:07,191 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:04:07,418 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:04:09,421 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:04:10,558 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:04:10,558 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:04:10,559 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:04:10,564 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:04:11,188 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:04:11,422 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:04:11,564 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:04:13,425 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:04:15,598 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:04:15,904 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:04:15,905 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:04:15,905 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:04:15,911 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:04:16,427 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:04:16,946 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:04:17,428 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:04:20,285 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:04:20,286 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:04:20,286 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:04:20,291 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:04:20,430 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:04:21,189 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:04:22,291 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:04:23,434 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:04:25,200 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:04:25,200 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:04:25,201 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:04:25,205 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:04:25,435 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:04:26,190 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:04:26,567 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 18:04:27,436 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:04:27,568 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:04:28,466 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:04:28,466 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:04:28,467 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:04:28,471 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:04:29,437 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:04:30,598 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:04:31,441 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:04:32,207 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:04:32,208 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:04:32,208 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:04:32,213 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:04:32,441 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:04:33,250 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:04:33,444 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:04:36,191 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:04:36,463 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:04:36,464 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:04:36,464 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:04:36,469 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:04:37,446 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:04:38,470 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:04:39,449 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:04:40,014 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:04:40,015 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:04:40,015 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:04:40,020 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:04:40,450 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:04:41,192 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:04:41,454 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:04:44,020 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:04:44,564 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:04:44,565 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:04:44,565 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:04:44,570 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:04:45,456 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:04:45,598 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:04:47,459 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:04:49,217 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:04:49,217 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:04:49,217 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:04:49,218 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:04:49,222 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:04:49,460 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:04:51,193 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:04:51,464 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:04:54,259 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:04:54,524 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:04:54,525 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:04:54,525 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:04:54,530 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:04:55,466 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:04:56,193 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:04:56,572 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 18:04:57,469 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:04:58,159 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:04:58,160 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:04:58,160 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:04:58,165 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:04:58,471 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:04:59,472 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:05:00,166 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:05:00,598 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:05:01,370 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:05:01,371 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:05:01,371 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:05:01,376 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:05:01,473 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:05:03,477 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:05:05,415 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:05:05,514 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:05:05,514 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:05:05,515 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:05:05,520 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:05:06,194 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:05:06,478 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:05:07,481 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:05:10,295 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:05:10,296 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:05:10,296 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:05:10,301 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:05:10,483 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:05:11,195 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:05:11,302 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:05:11,484 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:05:14,225 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:05:14,225 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:05:14,226 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:05:14,231 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:05:14,486 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:05:15,488 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:05:15,598 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:05:17,231 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:05:20,366 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:05:20,366 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:05:20,367 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:05:20,371 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:05:20,491 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:05:21,196 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:05:22,410 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:05:23,495 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:05:23,642 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:05:23,643 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:05:23,643 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:05:23,648 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:05:24,495 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:05:25,498 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:05:26,197 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:05:26,571 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 18:05:26,858 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:05:26,859 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:05:26,859 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:05:26,864 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:05:27,499 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:05:27,864 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:05:29,502 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:05:30,472 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:05:30,473 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:05:30,473 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:05:30,478 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:05:30,503 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:05:30,598 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:05:33,479 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:05:33,505 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:05:36,198 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:05:38,479 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:05:38,843 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:05:38,844 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:05:38,844 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:05:38,849 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:05:39,508 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:05:41,198 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:05:41,513 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:05:42,608 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:05:42,609 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:05:42,609 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:05:42,614 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:05:43,514 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:05:43,614 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:05:45,517 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:05:45,598 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:05:46,312 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:05:46,313 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:05:46,313 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:05:46,318 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:05:46,518 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:05:49,355 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:05:49,521 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:05:51,200 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:05:54,355 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:05:55,333 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:05:55,334 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:05:55,334 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:05:55,339 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:05:55,525 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:05:56,201 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:05:56,576 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 18:05:57,528 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:05:59,577 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:06:00,598 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:06:04,054 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:06:04,055 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:06:04,056 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:06:04,061 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:06:04,534 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:06:05,098 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:06:05,536 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:06:06,202 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:06:07,300 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:06:07,300 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:06:07,300 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:06:07,305 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:06:07,537 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:06:09,539 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:06:10,306 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:06:11,202 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:06:11,463 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:06:11,464 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:06:11,464 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:06:11,469 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:06:11,540 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:06:13,543 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:06:15,146 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:06:15,147 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:06:15,147 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:06:15,152 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:06:15,545 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:06:15,598 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:06:16,152 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:06:17,548 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:06:21,153 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:06:21,203 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:06:23,611 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:06:23,612 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:06:23,612 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:06:23,617 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:06:24,552 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:06:25,555 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:06:26,204 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:06:26,575 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 18:06:26,577 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:06:26,787 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:06:26,787 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:06:26,788 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:06:26,793 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:06:27,556 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:06:29,559 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:06:30,599 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:06:31,443 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:06:31,443 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:06:31,443 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:06:31,449 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:06:31,560 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:06:32,486 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:06:33,564 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:06:35,061 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:06:35,062 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:06:35,062 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:06:35,067 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:06:35,565 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:06:36,205 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:06:37,569 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:06:38,067 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:06:39,374 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:06:39,374 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:06:39,375 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:06:39,380 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:06:39,570 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:06:41,205 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:06:41,575 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:06:43,381 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:06:44,083 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:06:44,083 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:06:44,083 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:06:44,088 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:06:44,576 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:06:45,579 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:06:45,599 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:06:47,971 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:06:47,972 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:06:47,972 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:06:47,977 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:06:48,581 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:06:49,014 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:06:49,583 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:06:51,206 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:06:52,360 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:06:52,360 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:06:52,361 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:06:52,365 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:06:52,585 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:06:53,587 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:06:54,366 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:06:55,803 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:06:55,804 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:06:55,804 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:06:55,809 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:06:56,207 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:06:56,580 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 18:06:56,589 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:06:57,591 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:06:59,581 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:06:59,676 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:06:59,676 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:06:59,676 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:06:59,682 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:07:00,594 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:07:00,599 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:07:01,597 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:07:04,682 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:07:06,015 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:07:06,016 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:07:06,016 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:07:06,021 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:07:06,208 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:07:06,600 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:07:07,602 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:07:09,615 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:07:09,616 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:07:09,616 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:07:09,621 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:07:10,604 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:07:10,621 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:07:11,208 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:07:11,605 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:07:15,599 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:07:15,622 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:07:16,010 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:07:16,011 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:07:16,011 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:07:16,016 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:07:16,608 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:07:17,611 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:07:20,887 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:07:20,888 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:07:20,888 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:07:20,888 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:07:20,893 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:07:21,323 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:07:21,613 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:07:23,616 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:07:24,801 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:07:24,801 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:07:24,801 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:07:24,806 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:07:25,617 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:07:26,323 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:07:26,579 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 18:07:26,580 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:07:27,621 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:07:30,599 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:07:31,580 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:07:33,441 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:07:33,441 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:07:33,442 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:07:33,446 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:07:33,624 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:07:35,627 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:07:36,324 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:07:37,176 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:07:37,177 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:07:37,177 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:07:37,177 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:07:37,182 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:07:37,629 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:07:39,632 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:07:40,448 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:07:40,449 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:07:40,449 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:07:40,454 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:07:40,632 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:07:41,325 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:07:41,634 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:07:42,454 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:07:45,599 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:07:45,821 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:07:45,822 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:07:45,822 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:07:45,827 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:07:46,637 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:07:47,638 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:07:47,866 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:07:49,275 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:07:49,276 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:07:49,276 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:07:49,281 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:07:49,639 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:07:51,326 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:07:51,642 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:07:53,244 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:07:53,244 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:07:53,245 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:07:53,245 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:07:53,249 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:07:53,643 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:07:55,646 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:07:56,326 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:07:56,581 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 18:07:57,444 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:07:57,445 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:07:57,445 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:07:57,450 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:07:57,647 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:07:58,450 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:07:59,652 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:08:00,599 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:08:01,271 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:08:01,271 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:08:01,272 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:08:01,277 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:08:01,653 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:08:03,656 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:08:04,315 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:08:04,699 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:08:04,700 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:08:04,700 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:08:04,705 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:08:05,657 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:08:06,327 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:08:07,660 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:08:08,205 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:08:08,205 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:08:08,206 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:08:08,210 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:08:08,661 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:08:09,662 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:08:10,211 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:08:11,328 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:08:12,588 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:08:12,588 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:08:12,589 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:08:12,594 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:08:12,663 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:08:15,594 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:08:15,599 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:08:15,668 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:08:16,597 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:08:16,598 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:08:16,598 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:08:16,603 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:08:16,668 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:08:19,672 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:08:20,643 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:08:21,148 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:08:21,149 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:08:21,149 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:08:21,154 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:08:21,329 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:08:21,673 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:08:23,676 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:08:26,155 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:08:26,330 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:08:26,583 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 18:08:27,439 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:08:27,439 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:08:27,440 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:08:27,444 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:08:27,678 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:08:29,681 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:08:30,599 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:08:31,134 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:08:31,135 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:08:31,135 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:08:31,140 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:08:31,178 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:08:31,683 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:08:33,686 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:08:34,872 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:08:34,873 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:08:34,873 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:08:34,878 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:08:35,687 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:08:36,331 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:08:36,878 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:08:37,690 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:08:38,461 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:08:38,461 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:08:38,461 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:08:38,467 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:08:38,691 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:08:39,693 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:08:41,331 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:08:42,467 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:08:43,035 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:08:43,036 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:08:43,036 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:08:43,041 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:08:43,697 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:08:45,599 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:08:45,700 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:08:46,477 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:08:46,478 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:08:46,478 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:08:46,483 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:08:46,701 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:08:47,522 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:08:47,703 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:08:50,494 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:08:50,495 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:08:50,495 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:08:50,500 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:08:50,705 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:08:51,332 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:08:51,707 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:08:53,500 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:08:53,816 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:08:53,817 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:08:53,817 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:08:53,822 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:08:54,709 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:08:55,710 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:08:56,333 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:08:56,585 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 18:08:57,702 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:08:57,703 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:08:57,703 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:08:57,708 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:08:57,711 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:08:58,709 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:08:59,715 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:09:00,599 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:09:01,218 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:09:01,219 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:09:01,220 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:09:01,224 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:09:01,716 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:09:03,719 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:09:04,262 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:09:05,403 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:09:05,404 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:09:05,404 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:09:05,408 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:09:05,720 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:09:06,334 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:09:07,723 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:09:08,429 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:09:08,430 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:09:08,430 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:09:08,435 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:09:08,724 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:09:09,435 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:09:09,727 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:09:11,334 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:09:14,435 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:09:15,118 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:09:15,119 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:09:15,119 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:09:15,124 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:09:15,599 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:09:15,730 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:09:17,733 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:09:19,852 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:09:19,852 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:09:19,853 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:09:19,853 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:09:19,857 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:09:20,735 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:09:21,335 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:09:21,737 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:09:23,196 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:09:23,197 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:09:23,197 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:09:23,202 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:09:23,739 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:09:25,202 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:09:25,740 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:09:26,336 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:09:26,587 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 18:09:27,645 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:09:27,645 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:09:27,645 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:09:27,650 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:09:27,741 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:09:29,744 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:09:30,599 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:09:30,650 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:09:31,821 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:09:31,822 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:09:31,822 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:09:31,827 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:09:32,746 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:09:33,748 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:09:35,258 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:09:35,259 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:09:35,259 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:09:35,264 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:09:35,749 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:09:36,264 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:09:36,337 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:09:37,753 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:09:41,264 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:09:41,338 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:09:43,896 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:09:43,896 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:09:43,896 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:09:43,901 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:09:44,758 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:09:45,600 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:09:45,760 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:09:46,902 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:09:49,538 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:09:49,539 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:09:49,539 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:09:49,544 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:09:49,762 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:09:51,339 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:09:51,765 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:09:52,582 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:09:54,269 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:09:54,269 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:09:54,270 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:09:54,275 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:09:54,767 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:09:55,770 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:09:56,339 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:09:56,589 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 18:09:57,590 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:10:00,600 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:10:01,301 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:10:01,302 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:10:01,302 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:10:01,307 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:10:01,774 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:10:03,346 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:10:03,778 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:10:04,779 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:10:04,780 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:10:04,780 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:10:04,785 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:10:05,779 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:10:06,340 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:10:07,782 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:10:08,652 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:10:08,653 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:10:08,653 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:10:08,653 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:10:08,658 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:10:08,782 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:10:11,341 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:10:11,784 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:10:13,659 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:10:15,511 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:10:15,512 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:10:15,512 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:10:15,517 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:10:15,600 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:10:15,787 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:10:17,790 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:10:19,518 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:10:20,300 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:10:20,301 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:10:20,301 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:10:20,306 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:10:20,792 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:10:21,341 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:10:21,794 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:10:25,343 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:10:26,098 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:10:26,099 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:10:26,099 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:10:26,104 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:10:26,342 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:10:26,595 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 18:10:26,797 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:10:27,799 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:10:29,126 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:10:29,127 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:10:29,127 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:10:29,132 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:10:29,800 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:10:30,600 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:10:31,133 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:10:31,804 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:10:33,638 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:10:33,639 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:10:33,639 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:10:33,644 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:10:33,805 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:10:35,806 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:10:36,343 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:10:36,682 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:10:37,962 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:10:37,963 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:10:37,963 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:10:37,968 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:10:38,808 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:10:39,811 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:10:41,343 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:10:41,470 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:10:41,471 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:10:41,471 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:10:41,476 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:10:41,813 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:10:42,477 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:10:43,816 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:10:45,600 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:10:46,124 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:10:46,125 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:10:46,125 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:10:46,130 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:10:46,818 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:10:47,820 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:10:48,166 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:10:49,608 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:10:49,609 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:10:49,609 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:10:49,614 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:10:49,821 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:10:51,344 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:10:51,824 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:10:53,615 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:10:54,774 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:10:54,775 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:10:54,775 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:10:54,780 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:10:54,826 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:10:56,345 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:10:56,594 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 18:10:57,830 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:10:59,595 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:11:00,551 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:11:00,552 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:11:00,552 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:11:00,557 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:11:00,600 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:11:00,833 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:11:01,835 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:11:05,009 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:11:05,009 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:11:05,010 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:11:05,010 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:11:05,015 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:11:05,838 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:11:06,346 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:11:07,841 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:11:08,327 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:11:08,327 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:11:08,328 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:11:08,332 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:11:08,842 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:11:09,844 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:11:10,333 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:11:11,346 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:11:11,821 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:11:11,821 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:11:11,822 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:11:11,826 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:11:11,845 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:11:13,849 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:11:15,510 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:11:15,511 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:11:15,511 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:11:15,511 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:11:15,516 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:11:15,600 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:11:15,850 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:11:17,853 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:11:20,517 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:11:21,347 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:11:21,390 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:11:21,391 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:11:21,391 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:11:21,396 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:11:21,856 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:11:23,859 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:11:25,432 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:11:25,433 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:11:25,433 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:11:25,439 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:11:25,860 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:11:26,348 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:11:26,439 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:11:26,599 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 18:11:27,863 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:11:29,891 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:11:29,891 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:11:29,892 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:11:29,896 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:11:30,600 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:11:30,864 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:11:31,865 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:11:31,897 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:11:34,208 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:11:34,208 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:11:34,208 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:11:34,213 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:11:34,867 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:11:35,869 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:11:36,349 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:11:37,250 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:11:40,231 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:11:40,232 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:11:40,232 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:11:40,237 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:11:40,872 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:11:41,876 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:11:42,108 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:11:43,238 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:11:44,081 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:11:44,082 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:11:44,082 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:11:44,087 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:11:44,878 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:11:45,600 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:11:45,880 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:11:49,088 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:11:49,809 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:11:49,810 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:11:49,810 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:11:49,815 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:11:49,883 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:11:51,109 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:11:51,886 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:11:54,678 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:11:54,679 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:11:54,679 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:11:54,680 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:11:54,684 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:11:54,888 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:11:55,890 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:11:56,110 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:11:56,598 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 18:11:58,627 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:11:58,627 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:11:58,628 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:11:58,632 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:11:58,893 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:11:59,893 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:12:00,601 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:12:00,633 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:12:02,239 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:12:02,240 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:12:02,240 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:12:02,245 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:12:02,895 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:12:03,898 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:12:06,111 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:12:06,282 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:12:10,916 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:12:10,917 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:12:10,917 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:12:10,922 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:12:11,111 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:12:11,903 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:12:11,922 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:12:13,906 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:12:14,780 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:12:14,780 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:12:14,780 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:12:14,785 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:12:14,906 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:12:15,601 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:12:17,786 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:12:17,908 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:12:18,039 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:12:18,040 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:12:18,040 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:12:18,045 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:12:18,908 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:12:19,911 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:12:21,112 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:12:22,409 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:12:22,409 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:12:22,410 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:12:22,414 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:12:22,913 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:12:23,415 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:12:23,913 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:12:26,113 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:12:26,603 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 18:12:26,878 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:12:26,879 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:12:26,879 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:12:26,884 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:12:26,915 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:12:28,884 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:12:29,919 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:12:30,049 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:12:30,049 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:12:30,050 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:12:30,055 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:12:30,601 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:12:30,919 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:12:31,922 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:12:34,055 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:12:34,321 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:12:34,322 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:12:34,322 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:12:34,327 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:12:34,924 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:12:35,926 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:12:36,114 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:12:38,566 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:12:38,567 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:12:38,567 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:12:38,572 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:12:38,928 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:12:39,572 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:12:39,929 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:12:41,115 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:12:44,573 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:12:45,601 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:12:46,900 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:12:46,900 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:12:46,900 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:12:46,906 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:12:46,934 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:12:49,939 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:12:49,941 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:12:51,116 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:12:51,384 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:12:51,385 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:12:51,385 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:12:51,391 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:12:51,941 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:12:53,944 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:12:54,702 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:12:54,703 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:12:54,703 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:12:54,708 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:12:54,945 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:12:55,709 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:12:55,948 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:12:56,116 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:12:56,602 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 18:12:59,124 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:12:59,124 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:12:59,125 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:12:59,130 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:12:59,953 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:13:00,601 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:13:01,131 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:13:01,954 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:13:04,006 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:13:04,007 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:13:04,007 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:13:04,013 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:13:04,956 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:13:05,959 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:13:06,117 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:13:07,049 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:13:07,896 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:13:07,897 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:13:07,897 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:13:07,902 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:13:07,960 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:13:09,964 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:13:10,641 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:13:10,642 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:13:10,642 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:13:10,647 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:13:10,965 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:13:11,118 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:13:11,967 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:13:12,648 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:13:14,764 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:13:14,765 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:13:14,765 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:13:14,771 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:13:14,970 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:13:15,601 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:13:15,972 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:13:17,772 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:13:20,602 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:13:20,603 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:13:20,603 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:13:20,608 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:13:20,976 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:13:21,119 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:13:21,978 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:13:23,645 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:13:24,962 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:13:24,962 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:13:24,962 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:13:24,968 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:13:24,980 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:13:26,119 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:13:26,608 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 18:13:27,985 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:13:28,413 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:13:28,414 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:13:28,414 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:13:28,420 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:13:28,986 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:13:29,420 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:13:29,989 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:13:30,601 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:13:33,595 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:13:33,596 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:13:33,596 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:13:33,602 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:13:33,991 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:13:34,637 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:13:35,995 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:13:36,120 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:13:38,132 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:13:38,133 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:13:38,133 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:13:38,139 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:13:38,998 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:13:40,001 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:13:40,139 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:13:41,121 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:13:41,418 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:13:41,419 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:13:41,419 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:13:41,425 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:13:42,004 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:13:44,007 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:13:45,425 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:13:45,601 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:13:47,408 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:13:47,409 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:13:47,409 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:13:47,414 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:13:48,010 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:13:50,014 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:13:50,449 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:13:50,992 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:13:50,993 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:13:50,993 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:13:50,998 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:13:51,015 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:13:51,122 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:13:54,019 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:13:55,999 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:13:56,122 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:13:56,606 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 18:13:59,649 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:13:59,650 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:13:59,650 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:13:59,655 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:14:00,023 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:14:00,601 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:14:01,656 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:14:02,025 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:14:03,251 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:14:03,252 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:14:03,252 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:14:03,257 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:14:04,026 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:14:06,029 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:14:06,123 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:14:07,294 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:14:07,831 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:14:07,832 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:14:07,832 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:14:07,837 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:14:08,030 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:14:10,033 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:14:11,124 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:14:12,003 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:14:12,004 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:14:12,004 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:14:12,009 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:14:12,034 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:14:13,010 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:14:14,040 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:14:15,601 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:14:16,382 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:14:16,383 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:14:16,383 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:14:16,392 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:14:17,043 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:14:18,049 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:14:18,425 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:14:20,959 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:14:20,959 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:14:20,959 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:14:20,970 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:14:21,052 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:14:21,125 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:14:23,971 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:14:24,054 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:14:26,125 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:14:26,608 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 18:14:27,812 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:14:27,813 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:14:27,813 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:14:27,821 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:14:28,058 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:14:29,821 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:14:30,060 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:14:30,601 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:14:31,498 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:14:31,499 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:14:31,499 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:14:31,509 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:14:32,061 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:14:34,068 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:14:35,023 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:14:35,031 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:14:35,031 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:14:35,031 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:14:35,039 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:14:35,069 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:14:36,126 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:14:38,074 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:14:40,040 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:14:40,321 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:14:40,322 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:14:40,322 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:14:40,340 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:14:41,078 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:14:41,126 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:14:42,085 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:14:43,826 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:14:43,827 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:14:43,827 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:14:43,838 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:14:44,087 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:14:45,601 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:14:45,839 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:14:46,088 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:14:50,839 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:14:51,127 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:14:52,345 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:14:52,346 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:14:52,346 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:14:52,357 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:14:53,094 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:14:54,101 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:14:56,128 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:14:56,358 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:14:56,610 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 18:14:58,058 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:14:58,059 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:14:58,059 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:14:58,066 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:14:58,105 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:15:00,112 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:15:00,601 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:15:02,067 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:15:02,737 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:15:02,738 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:15:02,738 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:15:02,745 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:15:03,114 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:15:04,122 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:15:06,129 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:15:07,020 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:15:07,020 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:15:07,021 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:15:07,034 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:15:07,125 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:15:08,035 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:15:10,144 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:15:10,163 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:15:10,163 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:15:10,164 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:15:10,170 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:15:11,129 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:15:11,145 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:15:12,150 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:15:13,170 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:15:13,933 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:15:13,934 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:15:13,934 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:15:13,939 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:15:14,152 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:15:15,601 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:15:16,160 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:15:18,514 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:15:18,514 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:15:18,515 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:15:18,515 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:15:18,522 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:15:19,163 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:15:20,170 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:15:21,131 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:15:21,701 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:15:21,702 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:15:21,702 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:15:21,717 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:15:22,172 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:15:23,717 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:15:24,174 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:15:24,534 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:15:24,535 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:15:24,535 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:15:24,543 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:15:25,175 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:15:26,131 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:15:26,179 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:15:26,612 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 18:15:28,796 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:15:28,796 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:15:28,797 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:15:28,797 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:15:28,803 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:15:29,182 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:15:30,183 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:15:30,602 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:15:33,803 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:15:34,332 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:15:34,332 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:15:34,332 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:15:34,341 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:15:35,188 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:15:36,132 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:15:36,189 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:15:38,244 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:15:38,244 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:15:38,245 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:15:38,252 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:15:39,192 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:15:39,253 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:15:40,200 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:15:41,133 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:15:43,013 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:15:43,014 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:15:43,014 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:15:43,022 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:15:43,210 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:15:43,210 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:15:45,023 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:15:45,602 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:15:47,083 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:15:47,083 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:15:47,084 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:15:47,095 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:15:47,214 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:15:49,220 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:15:50,126 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:15:51,134 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:15:51,776 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:15:51,776 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:15:51,776 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:15:51,783 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:15:52,222 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:15:53,230 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:15:55,784 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:15:55,909 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:15:55,910 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:15:55,910 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:15:55,916 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:15:56,135 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:15:56,233 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:15:56,615 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 18:15:57,238 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:15:59,702 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:15:59,703 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:15:59,703 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:15:59,712 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:16:00,243 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:16:00,602 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:16:01,250 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:16:01,712 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:16:04,404 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:16:04,404 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:16:04,405 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:16:04,412 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:16:05,260 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:16:05,260 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:16:06,136 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:16:07,445 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:16:07,937 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:16:07,938 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:16:07,938 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:16:07,944 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:16:08,262 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:16:09,269 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:16:11,136 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:16:12,945 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:16:14,694 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:16:14,695 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:16:14,695 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:16:14,704 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:16:15,283 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:16:15,283 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:16:15,602 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:16:17,964 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:16:17,965 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:16:17,965 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:16:17,965 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:16:17,974 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:16:18,286 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:16:19,291 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:16:21,137 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:16:21,467 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:16:21,468 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:16:21,468 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:16:21,475 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:16:22,293 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:16:23,298 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:16:23,475 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:16:24,821 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:16:24,821 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:16:24,822 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:16:24,831 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:16:25,305 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:16:25,306 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:16:26,138 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:16:26,617 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 18:16:28,195 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:16:28,196 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:16:28,196 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:16:28,204 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:16:28,308 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:16:29,204 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:16:29,314 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:16:30,602 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:16:31,710 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:16:31,710 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:16:31,710 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:16:31,717 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:16:32,316 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:16:33,320 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:16:34,753 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:16:35,124 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:16:35,124 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:16:35,124 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:16:35,135 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:16:35,323 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:16:36,139 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:16:37,330 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:16:38,715 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:16:38,715 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:16:38,716 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:16:38,722 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:16:39,339 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:16:39,340 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:16:40,723 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:16:41,140 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:16:42,550 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:16:42,551 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:16:42,551 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:16:42,560 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:16:43,351 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:16:43,351 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:16:45,602 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:16:46,560 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:16:46,847 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:16:46,848 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:16:46,848 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:16:46,855 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:16:47,360 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:16:47,360 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:16:51,141 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:16:51,896 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:16:52,184 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:16:52,185 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:16:52,185 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:16:52,197 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:16:52,365 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:16:53,368 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:16:56,141 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:16:56,587 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:16:56,588 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:16:56,588 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:16:56,596 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:16:56,619 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 18:16:57,380 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:16:57,380 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:16:57,620 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:17:00,602 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:17:00,745 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:17:00,746 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:17:00,746 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:17:00,762 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:17:01,392 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:17:01,392 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:17:02,789 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:17:04,016 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:17:04,017 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:17:04,017 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:17:04,030 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:17:04,394 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:17:05,398 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:17:06,142 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:17:07,513 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:17:07,514 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:17:07,514 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:17:07,522 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:17:08,400 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:17:08,523 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:17:09,405 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:17:10,562 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:17:10,563 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:17:10,563 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:17:10,576 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:17:11,143 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:17:11,412 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:17:11,413 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:17:13,577 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:17:14,401 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:17:14,402 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:17:14,402 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:17:14,415 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:17:14,415 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:17:15,424 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:17:15,602 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:17:18,346 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:17:18,346 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:17:18,346 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:17:18,354 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:17:18,426 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:17:19,389 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:17:19,431 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:17:21,144 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:17:22,393 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:17:22,394 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:17:22,394 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:17:22,403 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:17:22,434 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:17:23,439 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:17:24,403 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:17:26,144 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:17:26,621 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 18:17:26,728 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:17:26,729 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:17:26,729 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:17:26,736 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:17:27,446 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:17:27,447 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:17:29,736 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:17:30,093 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:17:30,094 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:17:30,094 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:17:30,101 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:17:30,449 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:17:30,602 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:17:31,455 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:17:34,159 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:17:34,160 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:17:34,160 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:17:34,166 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:17:34,458 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:17:35,201 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:17:35,466 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:17:36,145 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:17:39,214 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:17:39,215 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:17:39,215 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:17:39,222 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:17:39,470 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:17:40,223 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:17:41,146 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:17:41,475 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:17:43,600 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:17:43,600 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:17:43,601 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:17:43,607 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:17:44,479 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:17:45,482 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:17:45,602 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:17:45,608 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:17:46,906 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:17:46,907 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:17:46,907 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:17:46,916 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:17:47,487 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:17:47,487 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:17:50,949 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:17:51,147 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:17:52,126 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:17:52,126 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:17:52,127 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:17:52,133 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:17:52,492 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:17:53,496 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:17:55,407 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:17:55,408 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:17:55,408 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:17:55,415 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:17:55,499 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:17:56,148 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:17:56,415 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:17:56,623 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 18:17:57,501 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:17:59,757 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:17:59,757 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:17:59,758 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:17:59,766 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:18:00,506 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:18:00,603 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:18:01,510 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:18:01,766 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:18:03,682 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:18:03,683 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:18:03,683 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:18:03,690 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:18:04,513 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:18:05,517 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:18:06,148 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:18:06,935 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:18:06,935 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:18:06,936 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:18:06,936 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:18:06,942 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:18:07,522 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:18:07,522 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:18:10,712 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:18:10,713 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:18:10,713 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:18:10,719 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:18:11,149 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:18:11,530 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:18:11,530 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:18:12,719 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:18:15,603 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:18:16,295 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:18:16,296 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:18:16,296 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:18:16,315 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:18:16,535 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:18:17,539 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:18:18,337 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:18:20,013 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:18:20,014 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:18:20,014 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:18:20,019 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:18:20,541 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:18:21,150 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:18:21,542 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:18:23,626 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:18:23,627 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:18:23,627 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:18:23,628 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:18:23,633 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:18:24,544 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:18:25,548 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:18:26,150 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:18:26,625 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 18:18:27,246 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:18:27,247 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:18:27,247 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:18:27,253 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:18:27,549 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:18:29,254 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:18:29,554 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:18:30,480 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:18:30,480 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:18:30,481 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:18:30,490 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:18:30,555 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:18:30,603 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:18:31,558 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:18:34,227 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:18:34,228 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:18:34,228 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:18:34,233 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:18:34,269 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:18:34,561 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:18:35,564 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:18:36,151 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:18:38,507 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:18:38,507 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:18:38,508 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:18:38,514 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:18:38,566 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:18:39,514 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:18:39,569 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:18:41,152 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:18:44,514 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:18:45,603 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:18:47,025 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:18:47,026 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:18:47,026 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:18:47,032 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:18:47,580 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:18:47,580 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:18:50,069 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:18:51,153 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:18:52,755 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:18:52,755 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:18:52,756 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:18:52,762 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:18:53,586 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:18:53,586 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:18:55,762 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:18:56,153 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:18:56,631 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 18:18:57,297 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:18:57,298 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:18:57,298 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:18:57,304 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:18:57,589 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:18:59,595 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:19:00,603 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:19:00,727 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:19:00,728 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:19:00,728 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:19:00,734 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:19:00,769 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:19:01,599 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:19:01,599 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:19:05,011 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:19:05,011 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:19:05,012 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:19:05,017 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:19:05,604 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:19:05,604 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:19:06,017 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:19:06,154 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:19:08,490 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:19:08,491 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:19:08,491 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:19:08,497 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:19:08,606 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:19:09,609 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:19:11,155 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:19:11,498 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:19:12,100 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:19:12,100 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:19:12,101 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:19:12,106 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:19:12,611 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:19:13,614 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:19:15,603 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:19:15,611 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:19:15,611 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:19:15,611 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:19:15,618 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:19:15,618 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:19:16,653 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:19:17,622 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:19:20,096 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:19:20,097 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:19:20,097 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:19:20,102 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:19:20,624 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:19:21,156 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:19:21,627 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:19:22,103 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:19:24,060 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:19:24,061 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:19:24,061 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:19:24,067 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:19:24,628 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:19:25,631 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:19:26,156 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:19:26,629 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 18:19:27,630 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:19:28,303 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:19:28,304 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:19:28,304 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:19:28,310 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:19:28,633 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:19:29,636 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:19:30,603 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:19:31,951 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:19:31,952 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:19:31,952 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:19:31,957 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:19:32,638 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:19:32,993 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:19:33,640 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:19:35,641 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:19:35,642 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:19:35,642 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:19:35,647 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:19:36,157 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:19:36,643 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:19:37,646 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:19:38,648 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:19:38,757 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:19:38,758 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:19:38,758 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:19:38,764 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:19:39,650 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:19:39,650 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:19:41,158 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:19:43,375 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:19:43,376 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:19:43,376 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:19:43,381 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:19:43,654 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:19:44,382 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:19:45,603 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:19:45,657 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:19:46,799 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:19:46,799 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:19:46,799 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:19:46,804 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:19:47,660 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:19:47,661 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:19:49,841 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:19:50,533 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:19:50,533 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:19:50,533 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:19:50,538 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:19:50,662 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:19:51,159 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:19:51,665 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:19:54,757 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:19:54,758 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:19:54,758 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:19:54,763 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:19:55,669 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:19:55,669 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:19:55,764 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:19:56,159 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:19:56,634 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 18:19:58,351 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:19:58,352 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:19:58,352 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:19:58,357 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:19:58,671 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:19:59,675 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:20:00,603 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:20:01,358 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:20:05,431 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:20:05,431 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:20:05,431 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:20:05,436 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:20:05,678 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:20:06,160 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:20:06,473 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:20:07,681 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:20:09,464 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:20:09,465 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:20:09,465 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:20:09,470 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:20:09,683 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:20:11,160 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:20:11,686 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:20:12,471 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:20:13,115 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:20:13,116 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:20:13,116 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:20:13,121 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:20:13,689 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:20:13,689 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:20:15,603 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:20:18,122 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:20:18,868 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:20:18,869 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:20:18,869 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:20:18,874 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:20:19,695 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:20:19,695 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:20:21,161 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:20:23,910 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:20:26,162 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:20:26,532 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:20:26,533 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:20:26,533 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:20:26,538 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:20:26,634 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 18:20:26,698 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:20:27,701 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:20:29,635 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:20:30,092 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:20:30,093 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:20:30,094 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:20:30,098 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:20:30,604 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:20:30,702 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:20:31,705 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:20:35,099 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:20:35,270 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:20:35,271 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:20:35,271 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:20:35,276 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:20:35,709 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:20:35,710 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:20:36,163 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:20:40,314 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:20:40,786 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:20:40,787 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:20:40,787 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:20:40,792 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:20:41,164 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:20:41,715 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:20:41,715 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:20:44,500 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:20:44,501 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:20:44,501 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:20:44,506 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:20:44,718 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:20:45,506 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:20:45,604 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:20:45,718 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:20:48,505 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:20:48,506 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:20:48,506 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:20:48,511 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:20:48,720 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:20:49,722 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:20:50,545 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:20:51,165 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:20:53,240 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:20:53,241 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:20:53,241 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:20:53,246 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:20:53,726 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:20:53,727 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:20:56,165 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:20:56,246 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:20:56,639 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 18:20:58,126 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:20:58,126 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:20:58,127 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:20:58,132 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:20:58,729 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:20:59,734 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:21:00,604 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:21:01,461 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:21:01,462 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:21:01,462 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:21:01,462 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:21:01,469 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:21:01,735 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:21:03,739 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:21:06,166 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:21:06,506 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:21:10,063 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:21:10,064 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:21:10,064 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:21:10,069 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:21:10,743 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:21:11,167 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:21:11,746 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:21:12,070 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:21:13,204 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:21:13,205 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:21:13,205 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:21:13,210 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:21:13,749 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:21:13,749 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:21:15,604 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:21:17,000 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:21:17,000 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:21:17,001 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:21:17,006 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:21:17,753 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:21:17,754 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:21:18,041 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:21:20,155 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:21:20,156 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:21:20,156 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:21:20,161 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:21:20,755 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:21:21,168 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:21:21,758 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:21:23,162 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:21:26,168 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:21:26,604 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:21:26,605 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:21:26,605 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:21:26,611 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:21:26,638 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 18:21:26,760 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:21:27,763 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:21:28,639 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:21:30,604 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:21:31,145 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:21:31,146 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:21:31,146 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:21:31,150 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:21:31,767 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:21:31,767 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:21:34,189 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:21:34,869 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:21:34,870 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:21:34,870 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:21:34,875 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:21:35,771 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:21:35,772 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:21:36,169 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:21:39,112 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:21:39,112 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:21:39,113 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:21:39,118 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:21:39,775 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:21:39,776 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:21:40,118 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:21:41,170 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:21:44,254 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:21:44,255 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:21:44,255 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:21:44,260 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:21:44,780 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:21:45,260 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:21:45,604 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:21:45,782 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:21:49,141 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:21:49,142 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:21:49,142 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:21:49,147 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:21:49,786 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:21:49,786 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:21:51,171 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:21:51,185 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:21:52,367 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:21:52,368 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:21:52,368 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:21:52,373 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:21:52,788 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:21:53,790 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:21:56,171 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:21:56,373 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:21:56,482 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:21:56,483 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:21:56,483 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:21:56,488 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:21:56,643 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 18:21:56,792 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:21:57,794 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:22:00,604 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:22:01,181 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:22:01,181 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:22:01,181 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:22:01,186 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:22:01,800 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:22:01,800 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:22:02,221 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:22:05,447 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:22:05,448 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:22:05,448 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:22:05,453 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:22:05,804 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:22:05,804 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:22:06,172 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:22:07,453 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:22:09,415 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:22:09,416 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:22:09,416 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:22:09,421 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:22:09,808 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:22:09,808 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:22:11,173 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:22:12,883 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:22:12,883 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:22:12,884 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:22:12,884 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:22:12,888 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:22:13,812 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:22:13,812 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:22:15,604 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:22:17,889 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:22:18,193 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:22:18,194 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:22:18,194 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:22:18,198 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:22:18,815 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:22:19,818 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:22:21,174 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:22:23,237 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:22:26,174 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:22:26,642 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 18:22:26,743 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:22:26,744 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:22:26,744 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:22:26,748 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:22:26,821 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:22:27,824 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:22:28,749 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:22:30,469 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:22:30,470 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:22:30,470 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:22:30,474 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:22:30,604 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:22:30,826 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:22:31,828 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:22:33,959 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:22:33,960 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:22:33,960 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:22:33,960 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:22:33,965 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:22:34,830 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:22:35,832 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:22:36,175 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:22:37,366 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:22:37,367 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:22:37,367 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:22:37,371 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:22:37,835 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:22:37,835 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:22:39,372 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:22:40,929 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:22:40,930 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:22:40,930 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:22:40,935 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:22:41,176 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:22:41,840 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:22:41,840 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:22:44,935 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:22:45,604 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:22:49,936 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:22:50,176 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:22:50,177 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:22:50,177 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:22:50,182 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:22:50,846 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:22:51,178 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:22:51,848 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:22:53,903 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:22:53,903 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:22:53,904 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:22:53,909 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:22:54,850 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:22:55,853 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:22:55,909 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:22:56,178 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:22:56,645 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 18:22:58,874 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:22:58,874 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:22:58,875 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:22:58,880 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:22:59,858 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:22:59,858 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:23:00,604 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:23:01,880 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:23:02,401 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:23:02,401 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:23:02,401 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:23:02,406 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:23:02,859 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:23:03,862 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:23:06,179 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:23:07,442 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:23:07,988 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:23:07,989 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:23:07,989 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:23:07,994 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:23:08,865 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:23:09,867 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:23:11,180 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:23:12,553 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:23:12,554 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:23:12,554 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:23:12,554 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:23:12,559 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:23:12,869 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:23:13,871 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:23:15,604 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:23:17,560 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:23:17,680 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:23:17,680 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:23:17,680 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:23:17,685 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:23:17,874 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:23:19,881 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:23:20,665 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:23:20,666 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:23:20,666 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:23:20,671 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:23:20,882 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:23:21,181 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:23:21,884 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:23:22,671 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:23:24,059 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:23:24,060 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:23:24,060 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:23:24,065 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:23:24,886 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:23:25,889 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:23:26,182 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:23:26,647 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 18:23:27,431 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:23:27,431 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:23:27,432 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:23:27,436 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:23:27,892 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:23:27,892 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:23:28,437 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:23:30,605 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:23:30,891 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:23:30,892 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:23:30,892 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:23:30,897 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:23:31,896 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:23:31,896 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:23:33,933 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:23:34,529 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:23:34,530 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:23:34,530 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:23:34,535 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:23:34,898 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:23:35,898 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:23:36,183 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:23:38,999 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:23:39,000 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:23:39,000 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:23:39,000 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:23:39,005 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:23:39,903 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:23:39,903 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:23:41,183 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:23:42,149 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:23:42,150 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:23:42,150 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:23:42,155 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:23:42,906 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:23:43,908 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:23:44,156 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:23:45,605 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:23:46,826 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:23:46,827 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:23:46,827 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:23:46,833 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:23:46,911 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:23:47,913 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:23:49,869 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:23:50,111 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:23:50,112 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:23:50,112 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:23:50,118 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:23:50,915 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:23:51,184 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:23:51,915 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:23:55,119 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:23:55,313 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:23:55,314 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:23:55,314 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:23:55,319 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:23:55,920 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:23:55,920 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:23:56,185 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:23:56,649 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 18:23:59,545 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:23:59,546 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:23:59,546 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:23:59,551 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:23:59,925 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:23:59,925 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:24:00,551 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:24:00,605 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:24:02,996 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:24:03,000 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:24:03,000 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:24:03,005 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:24:03,930 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:24:03,930 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:24:06,037 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:24:06,186 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:24:10,954 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:24:10,955 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:24:10,955 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:24:10,960 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:24:11,186 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:24:11,936 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:24:11,936 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:24:11,960 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:24:14,687 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:24:14,687 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:24:14,687 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:24:14,693 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:24:14,938 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:24:15,605 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:24:15,941 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:24:17,694 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:24:21,187 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:24:22,694 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:24:23,143 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:24:23,144 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:24:23,144 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:24:23,149 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:24:23,947 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:24:23,947 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:24:26,188 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:24:26,651 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 18:24:26,981 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:24:26,982 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:24:26,982 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:24:26,987 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:24:27,951 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:24:27,951 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:24:27,988 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:24:30,605 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:24:30,747 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:24:30,747 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:24:30,747 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:24:30,752 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:24:30,953 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:24:31,955 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:24:33,789 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:24:35,284 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:24:35,285 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:24:35,285 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:24:35,290 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:24:35,959 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:24:35,960 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:24:36,189 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:24:38,338 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:24:38,339 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:24:38,339 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:24:38,344 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:24:38,961 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:24:39,344 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:24:39,964 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:24:41,189 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:24:41,937 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:24:41,938 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:24:41,938 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:24:41,944 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:24:41,965 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:24:43,970 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:24:44,944 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:24:45,373 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:24:45,374 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:24:45,374 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:24:45,379 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:24:45,606 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:24:45,973 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:24:45,973 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:24:49,278 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:24:49,278 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:24:49,279 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:24:49,283 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:24:49,978 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:24:49,978 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:24:50,321 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:24:51,191 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:24:52,908 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:24:52,909 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:24:52,909 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:24:52,913 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:24:52,980 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:24:53,982 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:24:55,914 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:24:56,191 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:24:56,530 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:24:56,531 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:24:56,531 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:24:56,536 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:24:56,653 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 18:24:56,984 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:24:57,986 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:25:00,605 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:25:01,654 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:25:05,044 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:25:05,044 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:25:05,044 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:25:05,049 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:25:05,994 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:25:05,995 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:25:06,192 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:25:07,085 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:25:08,667 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:25:08,668 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:25:08,668 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:25:08,673 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:25:08,996 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:25:09,999 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:25:11,193 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:25:12,674 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:25:15,605 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:25:15,786 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:25:15,787 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:25:15,787 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:25:15,792 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:25:16,002 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:25:17,829 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:25:18,003 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:25:20,182 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:25:20,182 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:25:20,183 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:25:20,187 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:25:21,005 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:25:21,194 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:25:22,008 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:25:23,188 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:25:26,194 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:25:26,658 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 18:25:28,646 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:25:28,646 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:25:28,647 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:25:28,647 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:25:28,652 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:25:29,012 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:25:30,014 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:25:30,605 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:25:32,293 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:25:32,293 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:25:32,294 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:25:32,299 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:25:33,016 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:25:34,018 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:25:34,334 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:25:36,195 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:25:37,407 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:25:37,407 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:25:37,408 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:25:37,413 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:25:38,023 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:25:38,023 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:25:39,414 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:25:41,196 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:25:41,330 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:25:41,330 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:25:41,331 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:25:41,335 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:25:42,027 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:25:42,027 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:25:45,336 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:25:45,534 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:25:45,534 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:25:45,535 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:25:45,540 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:25:45,605 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:25:46,038 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:25:46,038 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:25:50,540 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:25:50,977 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:25:50,978 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:25:50,978 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:25:50,983 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:25:51,041 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:25:51,197 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:25:52,043 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:25:55,482 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:25:55,483 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:25:55,483 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:25:55,488 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:25:56,047 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:25:56,048 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:25:56,198 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:25:56,488 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:25:56,657 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 18:25:58,999 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:25:58,999 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:25:59,000 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:25:59,005 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:25:59,049 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:26:00,053 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:26:00,606 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:26:02,005 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:26:03,312 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:26:03,313 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:26:03,313 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:26:03,318 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:26:04,057 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:26:04,058 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:26:06,199 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:26:06,572 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:26:06,573 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:26:06,573 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:26:06,578 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:26:07,059 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:26:07,578 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:26:08,062 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:26:11,698 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:26:12,216 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:26:12,217 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:26:12,217 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:26:12,222 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:26:13,065 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:26:13,222 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:26:14,067 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:26:15,606 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:26:17,432 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:26:17,432 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:26:17,433 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:26:17,438 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:26:18,072 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:26:18,072 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:26:18,473 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:26:20,699 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:26:20,772 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:26:20,773 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:26:20,773 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:26:20,778 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:26:21,074 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:26:22,076 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:26:23,779 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:26:24,268 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:26:24,269 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:26:24,269 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:26:24,274 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:26:25,078 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:26:25,700 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:26:26,080 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:26:26,662 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 18:26:28,295 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:26:28,296 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:26:28,296 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:26:28,301 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:26:29,082 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:26:29,302 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:26:30,085 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:26:30,606 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:26:32,675 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:26:32,675 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:26:32,676 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:26:32,681 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:26:33,086 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:26:34,089 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:26:34,717 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:26:35,701 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:26:38,318 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:26:38,319 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:26:38,319 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:26:38,324 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:26:39,093 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:26:40,095 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:26:40,324 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:26:40,701 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:26:45,325 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:26:45,606 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:26:46,798 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:26:46,799 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:26:46,799 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:26:46,804 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:26:47,100 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:26:48,102 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:26:50,319 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:26:50,319 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:26:50,320 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:26:50,325 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:26:50,325 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:26:50,702 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:26:51,104 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:26:52,107 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:26:54,460 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:26:54,461 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:26:54,462 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:26:54,467 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:26:55,109 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:26:55,467 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:26:55,703 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:26:56,111 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:26:56,661 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 18:26:58,196 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:26:58,197 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:26:58,197 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:26:58,202 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:26:59,113 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:27:00,116 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:27:00,606 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:27:01,202 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:27:01,562 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:27:01,563 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:27:01,563 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:27:01,567 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:27:02,120 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:27:02,120 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:27:05,291 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:27:05,292 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:27:05,292 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:27:05,297 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:27:05,704 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:27:06,124 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:27:06,124 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:27:06,298 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:27:08,854 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:27:08,855 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:27:08,856 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:27:08,861 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:27:09,126 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:27:10,128 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:27:10,704 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:27:11,861 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:27:12,523 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:27:12,523 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:27:12,524 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:27:12,529 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:27:13,130 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:27:14,132 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:27:15,606 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:27:16,738 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:27:16,739 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:27:16,739 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:27:16,744 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:27:17,135 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:27:17,781 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:27:18,137 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:27:20,705 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:27:22,782 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:27:25,266 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:27:25,267 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:27:25,268 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:27:25,273 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:27:25,706 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:27:26,143 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:27:26,143 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:27:26,666 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 18:27:28,667 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:27:29,011 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:27:29,011 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:27:29,012 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:27:29,016 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:27:29,145 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:27:30,147 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:27:30,606 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:27:32,156 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:27:32,157 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:27:32,157 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:27:32,162 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:27:33,149 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:27:34,151 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:27:34,197 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:27:35,707 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:27:38,503 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:27:38,503 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:27:38,504 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:27:38,508 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:27:39,155 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:27:39,509 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:27:40,157 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:27:40,707 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:27:44,509 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:27:45,606 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:27:46,760 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:27:46,761 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:27:46,761 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:27:46,766 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:27:47,162 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:27:48,165 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:27:49,802 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:27:50,708 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:27:51,856 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:27:51,857 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:27:51,857 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:27:51,862 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:27:52,169 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:27:52,169 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:27:54,862 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:27:55,175 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:27:55,176 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:27:55,176 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:27:55,181 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:27:55,709 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:27:56,173 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:27:56,173 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:27:56,665 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 18:27:59,035 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:27:59,036 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:27:59,036 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:27:59,041 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:27:59,174 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:28:00,041 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:28:00,176 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:28:00,606 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:28:02,773 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:28:02,773 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:28:02,774 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:28:02,778 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:28:03,178 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:28:04,180 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:28:05,710 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:28:05,814 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:28:06,141 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:28:06,142 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:28:06,142 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:28:06,147 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:28:06,181 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:28:08,184 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:28:10,106 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:28:10,106 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:28:10,107 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:28:10,112 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:28:10,185 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:28:10,710 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:28:11,112 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:28:12,188 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:28:15,606 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:28:16,113 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:28:18,408 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:28:18,409 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:28:18,409 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:28:18,414 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:28:19,193 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:28:20,195 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:28:20,711 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:28:21,450 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:28:22,178 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:28:22,179 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:28:22,179 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:28:22,183 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:28:22,196 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:28:24,199 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:28:25,712 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:28:25,901 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:28:25,902 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:28:25,902 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:28:25,907 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:28:26,203 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:28:26,203 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:28:26,670 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 18:28:26,672 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:28:29,346 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:28:29,347 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:28:29,347 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:28:29,352 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:28:30,207 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:28:30,207 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:28:30,607 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:28:32,353 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:28:35,311 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:28:35,312 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:28:35,312 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:28:35,316 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:28:35,713 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:28:36,212 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:28:36,212 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:28:37,354 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:28:39,589 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:28:39,590 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:28:39,590 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:28:39,595 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:28:40,216 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:28:40,217 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:28:40,713 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:28:42,596 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:28:43,262 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:28:43,263 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:28:43,263 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:28:43,268 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:28:44,222 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:28:44,222 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:28:45,607 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:28:47,009 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:28:47,010 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:28:47,010 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:28:47,015 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:28:47,224 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:28:48,053 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:28:48,226 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:28:50,714 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:28:51,697 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:28:51,698 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:28:51,698 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:28:51,703 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:28:52,228 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:28:52,229 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:28:53,704 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:28:55,715 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:28:56,021 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:28:56,022 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:28:56,022 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:28:56,027 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:28:56,231 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:28:56,670 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 18:28:58,234 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:28:59,671 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:29:00,607 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:29:01,119 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:29:01,120 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:29:01,120 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:29:01,125 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:29:01,236 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:29:02,239 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:29:04,609 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:29:04,610 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:29:04,610 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:29:04,615 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:29:05,241 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:29:05,615 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:29:05,716 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:29:06,243 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:29:08,263 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:29:08,263 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:29:08,264 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:29:08,269 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:29:09,245 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:29:10,247 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:29:10,717 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:29:11,270 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:29:12,401 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:29:12,402 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:29:12,402 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:29:12,407 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:29:13,249 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:29:14,251 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:29:15,607 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:29:15,946 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:29:15,946 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:29:15,947 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:29:15,951 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:29:16,253 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:29:16,989 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:29:18,256 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:29:20,417 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:29:20,418 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:29:20,418 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:29:20,423 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:29:20,718 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:29:21,258 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:29:22,261 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:29:22,424 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:29:24,306 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:29:24,306 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:29:24,307 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:29:24,312 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:29:25,262 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:29:25,718 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:29:26,265 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:29:26,675 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 18:29:27,676 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:29:28,277 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:29:28,278 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:29:28,278 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:29:28,283 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:29:29,267 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:29:30,269 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:29:30,607 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:29:31,894 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:29:31,895 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:29:31,895 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:29:31,900 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:29:32,272 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:29:32,272 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:29:32,937 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:29:35,719 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:29:37,100 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:29:37,101 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:29:37,101 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:29:37,106 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:29:37,276 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:29:38,106 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:29:38,278 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:29:40,305 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:29:40,305 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:29:40,306 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:29:40,310 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:29:40,720 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:29:41,280 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:29:42,281 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:29:43,311 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:29:44,307 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:29:44,308 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:29:44,308 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:29:44,313 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:29:45,284 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:29:45,607 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:29:46,286 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:29:48,314 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:29:50,721 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:29:50,851 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:29:50,852 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:29:50,852 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:29:50,857 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:29:51,289 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:29:52,292 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:29:53,858 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:29:55,216 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:29:55,216 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:29:55,217 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:29:55,222 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:29:55,293 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:29:55,721 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:29:56,296 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:29:56,674 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 18:29:58,914 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:29:58,915 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:29:58,915 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:29:58,915 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:29:58,920 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:29:59,297 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:30:00,301 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:30:00,607 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:30:03,127 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:30:03,128 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:30:03,128 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:30:03,133 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:30:03,303 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:30:04,170 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:30:04,305 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:30:05,722 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:30:08,223 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:30:08,224 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:30:08,224 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:30:08,229 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:30:08,308 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:30:09,229 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:30:10,311 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:30:10,723 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:30:12,122 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:30:12,123 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:30:12,124 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:30:12,129 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:30:12,312 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:30:14,315 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:30:15,129 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:30:15,607 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:30:15,717 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:30:15,718 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:30:15,718 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:30:15,724 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:30:16,319 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:30:16,319 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:30:20,477 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:30:20,478 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:30:20,478 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:30:20,478 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:30:20,483 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:30:20,724 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:30:21,322 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:30:22,324 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:30:24,492 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:30:24,493 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:30:24,493 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:30:24,498 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:30:25,326 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:30:25,498 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:30:25,724 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:30:26,328 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:30:26,679 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 18:30:27,812 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:30:27,813 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:30:27,813 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:30:27,818 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:30:28,331 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:30:28,332 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:30:30,607 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:30:30,819 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:30:32,089 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:30:32,090 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:30:32,090 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:30:32,095 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:30:32,334 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:30:34,337 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:30:35,037 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:30:35,038 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:30:35,038 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:30:35,043 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:30:35,338 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:30:35,726 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:30:36,044 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:30:36,340 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:30:38,630 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:30:38,630 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:30:38,631 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:30:38,635 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:30:39,342 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:30:40,345 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:30:40,726 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:30:41,636 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:30:42,586 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:30:42,587 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:30:42,587 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:30:42,592 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:30:43,348 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:30:44,351 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:30:45,607 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:30:46,606 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:30:46,607 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:30:46,607 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:30:46,612 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:30:46,650 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:30:47,352 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:30:48,355 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:30:49,926 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:30:49,927 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:30:49,927 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:30:49,932 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:30:50,358 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:30:50,358 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:30:50,727 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:30:51,932 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:30:55,728 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:30:56,321 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:30:56,322 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:30:56,322 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:30:56,327 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:30:56,361 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:30:56,678 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 18:30:57,679 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:30:58,364 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:30:59,875 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:30:59,876 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:30:59,877 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:30:59,881 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:31:00,369 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:31:00,369 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:31:00,608 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:31:02,882 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:31:03,994 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:31:03,995 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:31:03,995 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:31:04,000 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:31:04,373 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:31:04,373 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:31:05,729 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:31:07,357 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:31:07,358 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:31:07,359 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:31:07,363 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:31:07,375 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:31:08,364 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:31:08,375 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:31:10,729 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:31:13,139 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:31:13,140 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:31:13,140 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:31:13,145 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:31:13,378 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:31:14,146 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:31:14,381 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:31:15,608 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:31:19,146 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:31:20,730 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:31:21,773 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:31:21,774 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:31:21,774 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:31:21,779 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:31:22,388 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:31:22,388 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:31:24,779 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:31:24,781 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:31:24,781 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:31:24,781 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:31:24,786 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:31:25,389 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:31:25,731 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:31:26,392 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:31:26,680 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 18:31:30,377 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:31:30,378 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:31:30,379 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:31:30,379 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:31:30,384 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:31:30,394 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:31:30,608 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:31:32,397 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:31:35,385 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:31:35,496 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:31:35,497 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:31:35,497 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:31:35,502 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:31:35,732 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:31:36,402 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:31:36,402 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:31:40,539 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:31:40,732 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:31:42,425 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:31:42,425 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:31:42,426 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:31:42,430 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:31:43,407 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:31:44,410 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:31:45,279 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:31:45,280 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:31:45,280 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:31:45,285 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:31:45,410 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:31:45,608 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:31:46,285 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:31:46,413 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:31:48,529 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:31:48,530 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:31:48,530 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:31:48,535 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:31:49,414 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:31:50,417 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:31:50,733 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:31:51,575 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:31:51,815 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:31:51,816 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:31:51,817 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:31:51,821 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:31:52,420 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:31:52,420 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:31:55,253 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:31:55,254 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:31:55,254 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:31:55,259 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:31:55,422 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:31:55,734 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:31:56,424 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:31:56,682 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 18:31:56,684 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:31:59,607 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:31:59,608 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:31:59,608 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:31:59,613 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:32:00,428 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:32:00,428 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:32:00,608 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:32:02,614 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:32:05,735 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:32:07,614 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:32:07,834 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:32:07,834 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:32:07,835 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:32:07,840 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:32:08,434 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:32:08,434 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:32:10,736 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:32:11,947 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:32:11,948 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:32:11,948 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:32:11,953 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:32:12,438 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:32:12,439 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:32:12,954 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:32:15,608 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:32:15,755 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:32:15,756 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:32:15,756 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:32:15,761 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:32:16,443 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:32:16,443 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:32:18,799 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:32:19,823 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:32:19,823 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:32:19,824 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:32:19,829 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:32:20,447 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:32:20,448 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:32:20,737 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:32:23,829 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:32:23,985 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:32:23,986 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:32:23,987 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:32:23,991 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:32:24,452 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:32:24,452 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:32:25,738 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:32:26,684 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 18:32:29,009 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:32:29,010 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:32:29,010 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:32:29,010 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:32:29,015 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:32:29,455 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:32:30,457 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:32:30,608 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:32:32,262 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:32:32,262 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:32:32,263 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:32:32,268 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:32:32,458 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:32:34,306 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:32:34,461 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:32:35,739 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:32:35,842 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:32:35,843 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:32:35,843 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:32:35,848 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:32:36,465 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:32:36,465 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:32:39,849 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:32:40,621 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:32:40,622 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:32:40,623 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:32:40,627 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:32:40,739 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:32:41,468 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:32:42,471 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:32:45,608 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:32:45,628 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:32:47,458 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:32:47,459 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:32:47,459 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:32:47,464 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:32:47,474 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:32:48,477 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:32:50,701 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:32:50,702 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:32:50,702 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:32:50,703 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:32:50,707 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:32:50,741 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:32:51,479 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:32:52,481 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:32:54,542 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:32:54,546 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:32:54,546 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:32:54,551 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:32:55,483 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:32:55,741 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:32:56,486 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:32:56,552 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:32:56,686 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 18:33:00,608 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:33:00,615 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:33:00,615 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:33:00,616 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:33:00,621 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:33:01,490 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:33:01,658 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:33:02,492 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:33:05,477 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:33:05,478 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:33:05,478 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:33:05,483 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:33:05,494 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:33:05,742 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:33:06,496 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:33:07,484 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:33:09,829 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:33:09,830 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:33:09,831 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:33:09,835 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:33:10,501 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:33:10,501 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:33:10,743 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:33:12,836 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:33:13,329 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:33:13,330 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:33:13,331 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:33:13,335 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:33:13,502 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:33:14,505 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:33:15,608 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:33:18,336 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:33:20,744 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:33:21,848 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:33:21,849 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:33:21,849 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:33:21,854 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:33:22,512 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:33:22,512 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:33:23,854 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:33:25,744 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:33:26,003 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:33:26,004 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:33:26,004 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:33:26,009 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:33:26,516 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:33:26,516 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:33:26,689 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 18:33:29,417 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:33:29,417 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:33:29,418 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:33:29,418 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:33:29,423 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:33:29,518 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:33:30,520 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:33:30,608 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:33:33,251 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:33:33,252 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:33:33,252 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:33:33,257 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:33:33,522 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:33:34,525 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:33:35,295 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:33:35,745 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:33:37,021 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:33:37,022 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:33:37,022 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:33:37,027 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:33:37,526 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:33:38,529 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:33:40,746 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:33:41,002 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:33:41,003 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:33:41,003 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:33:41,003 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:33:41,008 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:33:41,531 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:33:42,533 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:33:44,363 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:33:44,363 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:33:44,364 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:33:44,369 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:33:44,536 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:33:45,608 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:33:46,369 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:33:46,539 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:33:48,787 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:33:48,788 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:33:48,788 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:33:48,793 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:33:49,540 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:33:50,543 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:33:50,747 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:33:51,831 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:33:52,542 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:33:52,543 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:33:52,543 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:33:52,548 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:33:53,545 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:33:54,548 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:33:55,747 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:33:56,269 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:33:56,270 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:33:56,270 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:33:56,275 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:33:56,549 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:33:56,700 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 18:33:57,701 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:33:58,552 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:34:00,609 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:34:00,616 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:34:00,616 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:34:00,617 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:34:00,622 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:34:01,555 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:34:02,557 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:34:03,659 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:34:03,683 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:34:03,684 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:34:03,684 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:34:03,689 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:34:04,560 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:34:04,561 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:34:05,748 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:34:08,690 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:34:09,146 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:34:09,147 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:34:09,148 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:34:09,153 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:34:09,563 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:34:10,566 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:34:10,749 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:34:14,154 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:34:15,609 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:34:16,208 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:34:16,208 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:34:16,208 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:34:16,213 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:34:16,569 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:34:18,573 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:34:19,251 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:34:19,759 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:34:19,760 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:34:19,760 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:34:19,765 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:34:20,576 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:34:20,576 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:34:20,750 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:34:23,911 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:34:23,912 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:34:23,912 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:34:23,917 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:34:24,578 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:34:24,918 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:34:25,750 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:34:26,581 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:34:26,693 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 18:34:27,415 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:34:27,415 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:34:27,416 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:34:27,421 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:34:27,582 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:34:28,584 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:34:30,422 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:34:30,609 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:34:32,415 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:34:32,416 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:34:32,416 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:34:32,421 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:34:32,587 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:34:34,590 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:34:35,459 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:34:35,751 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:34:36,572 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:34:36,572 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:34:36,573 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:34:36,578 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:34:36,591 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:34:38,596 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:34:40,578 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:34:40,752 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:34:41,504 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:34:41,505 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:34:41,505 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:34:41,510 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:34:41,598 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:34:42,601 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:34:45,609 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:34:45,824 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:34:45,825 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:34:45,825 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:34:45,826 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:34:45,831 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:34:46,605 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:34:48,608 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:34:50,472 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:34:50,473 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:34:50,473 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:34:50,478 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:34:50,609 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:34:50,753 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:34:51,478 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:34:52,612 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:34:55,753 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:34:56,225 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:34:56,226 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:34:56,227 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:34:56,231 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:34:56,614 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:34:56,699 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 18:34:56,701 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:34:58,617 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:34:59,934 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:34:59,934 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:34:59,935 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:34:59,939 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:35:00,609 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:35:00,620 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:35:01,940 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:35:02,623 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:35:03,409 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:35:03,410 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:35:03,410 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:35:03,415 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:35:03,623 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:35:04,626 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:35:05,754 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:35:07,455 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:35:07,884 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:35:07,885 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:35:07,886 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:35:07,890 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:35:08,628 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:35:10,631 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:35:10,756 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:35:12,321 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:35:12,321 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:35:12,322 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:35:12,326 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:35:12,633 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:35:13,327 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:35:14,636 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:35:15,609 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:35:15,611 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:35:15,612 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:35:15,612 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:35:15,617 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:35:15,636 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:35:16,639 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:35:18,655 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:35:19,356 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:35:19,357 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:35:19,357 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:35:19,362 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:35:19,641 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:35:20,643 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:35:20,757 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:35:24,363 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:35:24,674 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:35:24,675 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:35:24,675 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:35:24,680 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:35:25,646 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:35:25,758 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:35:26,649 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:35:26,698 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 18:35:28,129 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:35:28,130 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:35:28,131 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:35:28,136 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:35:28,650 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:35:30,136 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:35:30,609 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:35:30,653 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:35:32,618 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:35:32,619 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:35:32,619 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:35:32,624 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:35:32,654 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:35:34,657 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:35:35,663 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:35:36,118 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:35:36,830 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:35:36,831 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:35:36,831 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:35:36,836 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:35:37,659 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:35:38,665 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:35:40,079 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:35:40,080 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:35:40,080 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:35:40,085 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:35:40,666 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:35:41,085 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:35:41,119 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:35:42,669 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:35:45,609 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:35:46,086 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:35:47,100 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:35:47,101 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:35:47,101 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:35:47,106 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:35:47,674 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:35:48,676 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:35:51,120 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:35:51,143 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:35:51,360 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:35:51,361 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:35:51,361 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:35:51,366 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:35:51,678 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:35:52,680 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:35:56,120 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:35:56,367 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:35:56,703 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 18:35:59,705 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:35:59,706 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:35:59,706 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:35:59,710 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:36:00,609 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:36:00,688 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:36:00,688 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:36:01,711 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:36:04,024 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:36:04,024 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:36:04,025 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:36:04,030 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:36:04,690 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:36:06,121 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:36:06,693 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:36:07,067 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:36:07,672 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:36:07,672 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:36:07,673 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:36:07,678 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:36:07,694 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:36:08,696 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:36:10,995 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:36:10,996 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:36:10,997 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:36:11,001 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:36:11,122 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:36:11,698 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:36:12,700 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:36:13,002 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:36:15,064 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:36:15,065 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:36:15,066 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:36:15,070 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:36:15,609 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:36:15,702 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:36:16,705 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:36:18,071 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:36:20,087 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:36:20,088 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:36:20,088 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:36:20,093 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:36:20,707 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:36:21,123 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:36:22,710 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:36:23,131 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:36:23,736 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:36:23,737 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:36:23,737 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:36:23,742 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:36:24,713 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:36:24,714 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:36:26,123 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:36:26,702 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 18:36:28,058 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:36:28,058 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:36:28,059 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:36:28,064 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:36:28,716 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:36:29,064 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:36:30,609 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:36:30,719 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:36:31,271 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:36:31,272 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:36:31,272 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:36:31,277 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:36:31,719 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:36:32,722 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:36:34,315 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:36:36,124 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:36:39,315 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:36:39,838 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:36:39,839 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:36:39,840 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:36:39,844 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:36:40,728 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:36:40,729 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:36:41,125 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:36:43,917 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:36:43,918 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:36:43,918 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:36:43,923 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:36:44,734 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:36:44,734 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:36:44,924 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:36:45,610 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:36:47,208 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:36:47,209 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:36:47,209 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:36:47,214 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:36:47,735 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:36:48,738 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:36:50,251 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:36:51,126 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:36:51,473 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:36:51,474 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:36:51,475 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:36:51,479 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:36:51,740 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:36:52,742 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:36:55,480 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:36:56,126 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:36:56,197 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:36:56,198 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:36:56,198 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:36:56,203 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:36:56,707 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 18:36:56,744 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:36:58,748 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:36:59,944 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:36:59,945 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:36:59,945 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:36:59,950 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:37:00,610 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:37:00,752 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:37:00,752 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:37:00,950 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:37:03,285 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:37:03,286 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:37:03,286 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:37:03,291 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:37:03,754 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:37:04,756 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:37:06,127 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:37:06,330 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:37:06,624 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:37:06,625 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:37:06,625 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:37:06,630 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:37:06,758 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:37:08,760 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:37:11,128 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:37:11,262 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:37:11,263 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:37:11,263 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:37:11,268 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:37:11,762 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:37:12,269 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:37:12,765 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:37:14,504 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:37:14,505 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:37:14,505 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:37:14,510 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:37:14,766 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:37:15,610 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:37:16,769 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:37:17,511 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:37:18,579 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:37:18,580 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:37:18,580 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:37:18,585 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:37:18,771 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:37:20,774 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:37:21,129 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:37:22,623 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:37:25,662 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:37:25,663 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:37:25,663 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:37:25,668 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:37:25,777 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:37:26,129 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:37:26,706 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 18:37:26,779 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:37:27,707 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:37:29,348 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:37:29,349 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:37:29,349 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:37:29,354 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:37:29,781 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:37:30,610 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:37:30,783 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:37:33,355 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:37:33,446 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:37:33,447 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:37:33,447 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:37:33,452 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:37:33,785 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:37:34,788 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:37:36,130 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:37:38,491 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:37:40,011 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:37:40,012 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:37:40,012 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:37:40,017 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:37:40,794 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:37:40,794 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:37:41,131 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:37:43,525 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:37:43,525 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:37:43,526 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:37:43,526 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:37:43,531 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:37:43,797 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:37:44,799 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:37:45,610 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:37:48,532 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:37:51,132 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:37:52,011 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:37:52,012 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:37:52,013 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:37:52,017 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:37:52,806 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:37:52,806 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:37:54,018 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:37:55,767 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:37:55,768 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:37:55,768 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:37:55,773 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:37:55,807 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:37:56,132 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:37:56,708 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 18:37:56,810 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:37:59,710 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:38:00,325 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:38:00,326 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:38:00,327 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:38:00,331 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:38:00,610 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:38:00,814 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:38:02,817 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:38:04,187 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:38:04,188 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:38:04,188 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:38:04,193 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:38:04,818 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:38:05,230 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:38:06,133 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:38:06,822 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:38:07,615 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:38:07,615 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:38:07,616 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:38:07,621 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:38:07,822 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:38:08,825 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:38:10,621 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:38:11,134 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:38:12,619 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:38:12,620 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:38:12,620 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:38:12,625 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:38:12,827 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:38:14,830 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:38:15,610 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:38:15,626 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:38:16,110 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:38:16,111 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:38:16,111 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:38:16,116 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:38:16,832 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:38:18,838 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:38:19,401 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:38:19,401 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:38:19,402 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:38:19,407 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:38:19,838 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:38:20,841 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:38:21,135 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:38:21,407 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:38:23,858 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:38:23,859 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:38:23,859 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:38:23,864 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:38:24,845 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:38:24,845 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:38:26,136 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:38:26,710 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 18:38:26,712 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:38:28,097 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:38:28,098 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:38:28,098 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:38:28,103 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:38:28,850 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:38:28,850 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:38:30,610 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:38:32,104 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:38:33,019 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:38:33,020 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:38:33,020 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:38:33,026 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:38:33,853 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:38:34,855 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:38:36,137 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:38:36,742 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:38:36,743 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:38:36,743 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:38:36,748 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:38:36,856 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:38:37,749 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:38:38,860 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:38:41,137 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:38:41,351 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:38:41,352 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:38:41,352 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:38:41,357 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:38:41,862 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:38:42,862 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:38:43,358 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:38:45,060 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:38:45,061 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:38:45,061 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:38:45,066 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:38:45,610 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:38:45,865 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:38:46,868 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:38:49,066 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:38:51,138 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:38:53,210 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:38:53,211 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:38:53,211 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:38:53,216 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:38:53,872 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:38:54,216 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:38:54,874 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:38:56,139 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:38:56,678 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:38:56,679 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:38:56,679 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:38:56,684 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:38:56,713 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 18:38:56,876 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:38:58,879 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:38:59,714 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:39:00,186 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:39:00,187 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:39:00,187 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:39:00,192 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:39:00,610 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:39:00,881 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:39:02,884 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:39:04,450 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:39:04,451 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:39:04,451 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:39:04,457 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:39:04,886 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:39:05,494 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:39:06,140 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:39:06,889 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:39:07,737 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:39:07,738 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:39:07,738 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:39:07,742 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:39:07,890 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:39:08,892 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:39:10,743 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:39:11,141 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:39:15,610 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:39:15,743 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:39:16,006 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:39:16,006 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:39:16,006 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:39:16,011 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:39:16,899 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:39:16,900 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:39:20,110 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:39:20,110 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:39:20,111 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:39:20,116 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:39:20,904 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:39:20,904 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:39:21,116 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:39:21,142 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:39:23,403 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:39:23,404 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:39:23,404 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:39:23,409 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:39:23,905 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:39:24,908 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:39:26,142 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:39:26,409 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:39:26,715 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 18:39:27,088 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:39:27,089 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:39:27,089 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:39:27,094 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:39:27,910 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:39:28,913 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:39:30,328 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:39:30,329 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:39:30,329 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:39:30,334 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:39:30,611 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:39:30,914 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:39:32,334 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:39:32,917 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:39:35,604 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:39:35,605 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:39:35,605 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:39:35,610 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:39:35,919 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:39:36,143 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:39:36,922 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:39:37,646 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:39:41,144 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:39:41,160 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:39:41,161 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:39:41,161 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:39:41,166 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:39:41,925 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:39:42,928 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:39:43,167 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:39:44,811 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:39:44,812 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:39:44,812 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:39:44,817 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:39:44,930 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:39:45,611 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:39:46,933 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:39:48,545 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:39:48,546 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:39:48,546 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:39:48,546 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:39:48,551 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:39:48,934 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:39:50,937 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:39:51,145 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:39:53,438 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:39:53,438 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:39:53,439 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:39:53,443 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:39:53,939 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:39:54,444 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:39:54,942 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:39:56,145 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:39:56,717 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 18:39:57,921 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:39:57,922 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:39:57,922 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:39:57,927 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:39:57,943 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:39:58,946 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:39:59,927 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:40:00,611 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:40:02,572 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:40:02,572 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:40:02,572 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:40:02,577 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:40:02,950 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:40:04,953 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:40:05,615 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:40:06,146 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:40:08,066 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:40:08,066 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:40:08,067 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:40:08,072 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:40:08,957 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:40:08,957 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:40:11,073 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:40:11,147 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:40:13,390 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:40:13,391 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:40:13,391 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:40:13,396 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:40:13,959 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:40:14,962 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:40:15,611 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:40:16,396 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:40:16,517 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:40:16,518 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:40:16,518 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:40:16,523 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:40:16,964 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:40:18,967 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:40:19,897 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:40:19,897 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:40:19,898 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:40:19,902 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:40:19,967 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:40:20,970 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:40:21,148 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:40:21,903 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:40:23,247 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:40:23,248 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:40:23,248 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:40:23,253 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:40:23,971 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:40:24,974 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:40:26,148 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:40:26,719 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 18:40:26,891 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:40:26,891 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:40:26,892 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:40:26,896 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:40:26,975 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:40:27,897 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:40:28,979 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:40:30,611 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:40:31,115 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:40:31,116 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:40:31,116 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:40:31,121 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:40:31,980 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:40:32,983 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:40:33,158 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:40:35,763 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:40:36,385 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:40:36,386 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:40:36,386 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:40:36,391 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:40:36,985 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:40:38,391 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:40:38,988 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:40:40,228 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:40:40,228 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:40:40,229 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:40:40,233 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:40:40,763 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:40:40,992 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:40:40,992 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:40:43,662 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:40:43,663 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:40:43,663 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:40:43,663 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:40:43,671 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:40:43,995 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:40:44,998 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:40:45,611 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:40:47,736 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:40:47,736 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:40:47,736 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:40:47,741 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:40:47,999 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:40:48,778 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:40:49,000 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:40:50,764 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:40:51,573 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:40:51,574 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:40:51,574 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:40:51,579 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:40:52,001 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:40:53,004 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:40:54,579 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:40:54,702 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:40:54,703 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:40:54,703 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:40:54,708 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:40:55,005 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:40:55,765 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:40:56,721 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 18:40:57,008 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:40:58,259 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:40:58,260 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:40:58,260 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:40:58,265 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:40:59,011 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:40:59,011 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:41:00,265 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:41:00,611 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:41:02,631 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:41:02,632 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:41:02,632 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:41:02,637 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:41:03,015 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:41:05,018 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:41:05,675 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:41:05,765 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:41:10,675 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:41:10,766 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:41:10,986 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:41:10,987 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:41:10,987 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:41:10,993 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:41:11,021 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:41:13,024 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:41:14,762 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:41:14,763 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:41:14,763 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:41:14,767 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:41:15,025 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:41:15,611 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:41:15,768 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:41:17,029 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:41:18,141 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:41:18,141 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:41:18,141 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:41:18,146 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:41:19,032 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:41:19,032 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:41:20,767 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:41:21,182 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:41:22,178 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:41:22,179 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:41:22,179 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:41:22,184 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:41:23,036 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:41:23,037 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:41:25,767 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:41:26,008 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:41:26,009 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:41:26,009 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:41:26,014 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:41:26,038 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:41:26,723 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 18:41:26,725 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:41:27,042 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:41:30,229 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:41:30,230 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:41:30,230 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:41:30,235 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:41:30,611 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:41:31,046 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:41:31,046 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:41:32,236 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:41:35,115 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:41:35,115 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:41:35,116 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:41:35,120 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:41:35,768 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:41:36,049 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:41:37,051 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:41:38,159 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:41:39,496 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:41:39,497 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:41:39,497 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:41:39,502 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:41:40,053 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:41:40,769 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:41:41,054 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:41:43,502 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:41:45,456 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:41:45,457 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:41:45,457 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:41:45,463 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:41:45,612 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:41:46,058 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:41:47,060 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:41:48,759 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:41:48,760 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:41:48,760 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:41:48,760 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:41:48,765 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:41:49,062 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:41:50,770 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:41:51,065 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:41:52,871 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:41:52,872 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:41:52,872 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:41:52,877 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:41:53,066 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:41:53,878 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:41:55,069 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:41:55,770 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:41:56,726 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 18:41:57,042 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:41:57,043 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:41:57,043 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:41:57,048 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:41:57,071 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:41:59,048 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:41:59,074 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:42:00,612 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:42:00,715 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:42:00,715 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:42:00,716 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:42:00,721 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:42:01,076 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:42:03,079 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:42:04,759 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:42:04,837 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:42:04,837 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:42:04,837 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:42:04,842 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:42:05,080 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:42:05,771 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:42:07,083 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:42:08,958 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:42:08,959 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:42:08,959 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:42:08,964 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:42:09,084 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:42:09,964 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:42:10,772 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:42:11,088 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:42:12,291 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:42:12,291 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:42:12,292 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:42:12,296 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:42:13,091 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:42:13,091 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:42:15,297 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:42:15,612 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:42:17,752 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:42:17,752 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:42:17,752 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:42:17,757 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:42:18,094 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:42:19,096 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:42:20,773 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:42:20,794 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:42:21,674 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:42:21,675 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:42:21,675 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:42:21,680 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:42:22,098 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:42:23,101 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:42:25,752 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:42:25,753 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:42:25,753 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:42:25,758 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:42:25,773 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:42:26,102 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:42:26,728 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 18:42:26,729 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:42:27,105 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:42:29,201 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:42:29,201 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:42:29,202 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:42:29,207 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:42:30,106 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:42:30,612 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:42:31,109 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:42:32,207 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:42:34,518 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:42:34,519 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:42:34,519 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:42:34,524 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:42:35,111 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:42:35,774 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:42:37,114 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:42:37,562 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:42:39,415 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:42:39,415 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:42:39,416 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:42:39,420 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:42:40,116 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:42:40,774 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:42:41,118 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:42:43,114 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:42:43,115 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:42:43,115 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:42:43,116 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:42:43,120 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:42:43,121 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:42:45,125 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:42:45,612 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:42:48,121 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:42:50,775 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:42:51,476 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:42:51,476 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:42:51,477 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:42:51,481 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:42:52,129 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:42:53,132 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:42:53,482 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:42:55,100 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:42:55,100 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:42:55,101 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:42:55,106 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:42:55,133 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:42:55,776 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:42:56,730 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 18:42:57,136 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:42:58,731 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:42:59,265 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:42:59,265 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:42:59,266 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:42:59,271 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:43:00,138 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:43:00,612 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:43:01,142 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:43:04,271 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:43:05,346 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:43:05,347 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:43:05,347 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:43:05,352 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:43:05,777 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:43:06,144 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:43:07,147 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:43:09,391 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:43:10,207 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:43:10,208 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:43:10,208 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:43:10,213 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:43:10,777 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:43:11,151 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:43:11,151 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:43:15,213 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:43:15,319 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:43:15,320 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:43:15,320 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:43:15,325 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:43:15,612 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:43:16,154 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:43:17,157 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:43:19,434 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:43:19,434 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:43:19,434 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:43:19,439 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:43:20,158 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:43:20,478 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:43:20,778 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:43:21,161 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:43:22,889 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:43:22,890 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:43:22,890 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:43:22,895 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:43:23,162 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:43:25,165 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:43:25,779 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:43:25,895 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:43:26,299 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:43:26,300 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:43:26,300 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:43:26,305 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:43:26,735 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 18:43:27,168 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:43:27,169 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:43:30,612 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:43:30,799 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:43:30,799 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:43:30,799 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:43:30,805 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:43:31,171 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:43:31,842 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:43:33,174 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:43:35,740 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:43:35,741 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:43:35,741 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:43:35,746 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:43:35,780 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:43:36,176 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:43:37,178 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:43:37,746 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:43:39,612 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:43:39,613 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:43:39,613 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:43:39,618 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:43:40,180 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:43:40,780 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:43:41,183 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:43:42,743 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:43:42,744 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:43:42,744 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:43:42,749 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:43:42,750 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:43:43,184 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:43:45,188 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:43:45,612 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:43:46,359 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:43:46,360 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:43:46,360 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:43:46,364 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:43:47,191 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:43:47,192 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:43:48,402 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:43:50,103 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:43:50,104 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:43:50,104 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:43:50,109 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:43:50,193 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:43:50,781 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:43:51,195 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:43:53,466 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:43:53,467 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:43:53,467 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:43:53,468 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:43:53,472 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:43:54,197 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:43:55,200 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:43:55,782 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:43:56,734 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 18:43:57,084 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:43:57,084 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:43:57,085 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:43:57,090 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:43:57,201 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:43:59,090 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:43:59,206 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:44:00,612 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:44:00,728 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:44:00,728 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:44:00,729 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:44:00,733 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:44:01,209 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:44:03,212 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:44:03,833 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:44:03,834 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:44:03,834 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:44:03,839 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:44:04,212 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:44:04,840 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:44:05,215 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:44:05,783 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:44:07,058 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:44:07,059 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:44:07,059 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:44:07,064 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:44:07,216 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:44:09,219 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:44:10,065 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:44:10,533 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:44:10,534 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:44:10,534 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:44:10,539 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:44:10,783 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:44:11,221 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:44:13,225 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:44:14,858 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:44:14,859 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:44:14,859 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:44:14,864 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:44:15,226 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:44:15,613 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:44:15,865 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:44:17,230 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:44:18,805 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:44:18,806 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:44:18,806 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:44:18,811 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:44:19,231 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:44:20,784 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:44:21,234 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:44:21,850 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:44:21,924 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:44:21,925 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:44:21,925 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:44:21,930 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:44:22,235 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:44:23,237 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:44:25,574 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:44:25,574 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:44:25,575 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:44:25,580 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:44:25,785 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:44:26,239 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:44:26,739 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 18:44:27,242 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:44:27,740 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:44:30,033 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:44:30,034 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:44:30,034 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:44:30,039 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:44:30,244 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:44:30,613 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:44:31,246 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:44:33,039 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:44:35,786 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:44:36,391 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:44:36,392 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:44:36,392 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:44:36,397 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:44:37,252 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:44:37,252 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:44:38,398 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:44:39,611 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:44:39,612 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:44:39,612 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:44:39,617 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:44:40,254 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:44:40,786 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:44:41,257 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:44:43,002 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:44:43,003 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:44:43,003 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:44:43,008 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:44:43,258 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:44:44,008 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:44:45,264 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:44:45,613 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:44:46,506 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:44:46,506 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:44:46,506 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:44:46,515 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:44:47,268 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:44:47,268 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:44:49,551 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:44:49,890 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:44:49,891 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:44:49,891 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:44:49,896 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:44:50,269 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:44:50,787 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:44:51,272 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:44:54,348 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:44:54,349 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:44:54,349 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:44:54,354 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:44:55,276 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:44:55,276 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:44:55,354 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:44:55,788 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:44:56,738 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 18:44:57,873 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:44:57,874 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:44:57,874 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:44:57,878 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:44:58,278 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:44:59,281 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:45:00,613 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:45:00,879 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:45:01,108 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:45:01,109 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:45:01,109 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:45:01,114 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:45:01,283 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:45:03,287 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:45:04,422 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:45:04,423 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:45:04,423 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:45:04,428 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:45:05,290 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:45:05,290 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:45:05,789 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:45:06,429 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:45:09,387 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:45:09,388 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:45:09,388 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:45:09,393 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:45:10,293 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:45:10,789 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:45:11,295 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:45:12,394 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:45:12,688 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:45:12,689 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:45:12,689 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:45:12,694 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:45:13,296 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:45:15,299 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:45:15,613 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:45:17,694 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:45:20,790 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:45:22,695 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:45:24,715 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:45:24,716 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:45:24,716 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:45:24,721 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:45:25,305 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:45:25,791 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:45:26,744 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 18:45:27,308 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:45:27,745 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:45:28,258 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:45:28,258 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:45:28,259 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:45:28,264 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:45:28,309 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:45:29,312 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:45:30,613 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:45:31,804 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:45:31,804 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:45:31,805 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:45:31,810 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:45:32,313 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:45:32,845 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:45:33,316 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:45:35,495 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:45:35,496 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:45:35,496 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:45:35,501 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:45:35,792 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:45:36,318 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:45:37,320 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:45:38,502 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:45:39,550 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:45:39,551 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:45:39,551 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:45:39,556 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:45:40,324 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:45:40,792 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:45:41,326 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:45:43,200 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:45:43,201 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:45:43,201 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:45:43,206 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:45:43,327 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:45:44,206 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:45:45,332 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:45:45,613 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:45:46,641 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:45:46,641 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:45:46,642 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:45:46,647 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:45:47,333 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:45:49,336 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:45:49,682 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:45:50,299 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:45:50,300 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:45:50,300 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:45:50,305 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:45:50,337 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:45:50,793 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:45:51,339 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:45:54,573 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:45:54,574 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:45:54,574 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:45:54,579 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:45:55,343 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:45:55,344 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:45:55,579 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:45:55,794 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:45:56,742 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 18:45:58,505 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:45:58,506 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:45:58,506 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:45:58,515 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:45:59,348 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:45:59,348 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:46:00,613 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:46:01,516 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:46:02,814 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:46:02,815 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:46:02,815 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:46:02,820 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:46:03,351 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:46:05,355 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:46:05,795 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:46:06,858 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:46:10,795 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:46:11,206 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:46:11,207 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:46:11,207 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:46:11,212 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:46:11,358 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:46:12,213 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:46:13,361 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:46:15,321 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:46:15,322 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:46:15,322 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:46:15,327 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:46:15,363 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:46:15,613 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:46:17,328 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:46:17,366 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:46:20,146 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:46:20,147 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:46:20,147 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:46:20,152 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:46:20,368 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:46:20,796 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:46:21,371 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:46:23,190 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:46:23,729 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:46:23,730 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:46:23,730 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:46:23,735 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:46:24,373 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:46:25,375 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:46:25,797 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:46:26,748 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 18:46:28,749 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:46:30,613 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:46:30,795 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:46:30,795 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:46:30,795 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:46:30,801 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:46:31,378 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:46:33,381 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:46:33,837 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:46:34,526 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:46:34,527 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:46:34,527 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:46:34,531 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:46:35,385 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:46:35,385 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:46:35,798 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:46:39,532 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:46:40,768 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:46:40,769 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:46:40,769 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:46:40,774 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:46:40,798 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:46:41,388 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:46:43,392 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:46:44,278 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:46:44,278 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:46:44,279 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:46:44,288 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:46:44,393 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:46:45,288 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:46:45,396 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:46:45,613 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:46:50,289 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:46:50,799 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:46:52,670 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:46:52,671 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:46:52,671 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:46:52,675 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:46:53,403 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:46:53,403 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:46:55,676 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:46:55,800 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:46:56,747 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 18:46:56,969 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:46:56,969 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:46:56,969 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:46:56,974 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:46:57,405 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:46:59,408 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:47:00,613 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:47:00,645 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:47:00,646 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:47:00,646 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:47:00,651 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:47:00,689 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:47:01,412 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:47:01,413 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:47:04,740 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:47:04,741 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:47:04,741 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:47:04,746 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:47:05,415 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:47:05,747 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:47:05,801 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:47:07,418 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:47:09,386 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:47:09,387 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:47:09,387 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:47:09,392 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:47:09,419 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:47:10,802 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:47:11,392 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:47:11,422 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:47:14,190 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:47:14,191 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:47:14,191 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:47:14,196 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:47:14,424 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:47:15,427 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:47:15,614 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:47:17,197 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:47:17,302 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:47:17,303 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:47:17,303 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:47:17,308 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:47:17,428 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:47:19,429 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:47:20,803 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:47:21,411 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:47:21,412 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:47:21,412 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:47:21,417 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:47:21,430 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:47:22,418 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:47:23,433 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:47:25,803 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:47:26,600 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:47:26,600 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:47:26,601 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:47:26,606 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:47:26,752 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 18:47:27,439 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:47:27,440 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:47:27,754 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:47:30,614 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:47:32,327 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:47:32,328 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:47:32,328 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:47:32,333 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:47:32,442 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:47:33,369 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:47:33,445 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:47:35,804 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:47:38,370 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:47:38,783 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:47:38,784 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:47:38,784 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:47:38,789 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:47:39,448 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:47:40,805 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:47:41,452 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:47:43,790 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:47:44,233 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:47:44,234 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:47:44,235 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:47:44,240 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:47:44,455 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:47:45,457 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:47:45,614 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:47:47,756 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:47:47,757 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:47:47,757 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:47:47,762 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:47:48,459 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:47:48,797 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:47:49,462 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:47:50,806 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:47:51,210 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:47:51,211 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:47:51,211 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:47:51,215 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:47:51,463 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:47:53,466 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:47:54,216 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:47:54,760 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:47:54,761 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:47:54,762 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:47:54,767 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:47:55,469 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:47:55,469 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:47:55,806 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:47:56,751 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 18:47:59,753 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:48:00,031 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:48:00,032 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:48:00,032 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:48:00,037 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:48:00,472 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:48:00,614 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:48:01,476 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:48:03,938 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:48:03,939 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:48:03,939 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:48:03,944 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:48:04,477 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:48:04,981 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:48:05,480 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:48:05,807 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:48:09,951 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:48:09,951 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:48:09,952 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:48:09,957 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:48:10,482 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:48:10,807 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:48:10,958 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:48:11,485 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:48:15,154 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:48:15,155 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:48:15,156 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:48:15,160 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:48:15,487 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:48:15,614 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:48:16,161 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:48:17,491 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:48:19,900 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:48:19,901 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:48:19,901 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:48:19,906 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:48:20,493 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:48:20,808 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:48:21,494 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:48:21,941 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:48:23,300 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:48:23,301 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:48:23,301 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:48:23,306 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:48:23,495 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:48:25,498 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:48:25,809 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:48:26,753 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 18:48:27,754 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:48:28,873 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:48:28,874 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:48:28,874 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:48:28,879 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:48:29,500 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:48:30,614 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:48:31,503 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:48:32,880 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:48:33,366 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:48:33,366 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:48:33,367 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:48:33,372 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:48:33,504 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:48:35,507 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:48:35,810 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:48:36,832 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:48:36,833 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:48:36,833 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:48:36,839 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:48:37,508 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:48:38,839 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:48:39,511 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:48:40,752 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:48:40,753 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:48:40,753 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:48:40,758 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:48:40,810 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:48:41,515 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:48:41,515 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:48:43,887 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:48:43,888 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:48:43,888 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:48:43,888 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:48:43,893 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:48:44,518 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:48:45,521 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:48:45,614 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:48:48,894 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:48:49,709 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:48:49,709 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:48:49,709 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:48:49,714 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:48:50,523 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:48:50,811 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:48:51,526 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:48:52,812 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:48:52,813 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:48:52,813 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:48:52,818 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:48:53,529 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:48:53,530 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:48:54,819 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:48:55,812 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:48:56,341 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:48:56,342 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:48:56,342 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:48:56,347 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:48:56,531 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:48:56,755 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 18:48:57,534 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:48:59,711 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:48:59,712 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:48:59,712 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:48:59,717 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:49:00,535 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:49:00,614 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:49:00,717 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:49:01,539 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:49:04,429 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:49:04,429 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:49:04,430 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:49:04,435 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:49:04,541 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:49:05,543 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:49:05,813 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:49:06,469 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:49:10,813 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:49:11,470 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:49:13,190 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:49:13,191 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:49:13,191 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:49:13,197 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:49:13,547 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:49:15,551 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:49:15,614 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:49:17,197 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:49:17,284 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:49:17,285 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:49:17,285 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:49:17,289 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:49:17,552 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:49:19,555 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:49:20,720 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:49:20,720 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:49:20,721 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:49:20,726 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:49:20,814 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:49:21,558 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:49:21,558 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:49:22,726 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:49:25,815 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:49:26,448 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:49:26,449 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:49:26,450 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:49:26,455 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:49:26,561 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:49:26,757 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 18:49:27,563 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:49:27,759 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:49:30,511 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:49:30,512 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:49:30,512 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:49:30,517 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:49:30,565 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:49:30,615 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:49:31,567 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:49:33,517 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:49:33,858 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:49:33,858 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:49:33,859 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:49:33,864 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:49:34,569 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:49:35,572 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:49:35,816 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:49:38,901 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:49:40,816 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:49:42,277 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:49:42,278 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:49:42,278 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:49:42,283 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:49:42,576 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:49:43,578 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:49:44,283 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:49:45,165 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:49:45,166 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:49:45,166 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:49:45,171 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:49:45,581 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:49:45,615 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:49:47,584 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:49:49,228 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:49:49,228 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:49:49,229 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:49:49,233 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:49:49,585 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:49:50,269 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:49:50,817 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:49:51,589 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:49:54,146 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:49:54,147 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:49:54,147 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:49:54,152 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:49:54,590 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:49:55,593 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:49:56,153 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:49:56,537 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:49:56,762 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 18:49:58,787 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:49:58,788 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:49:58,788 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:49:58,793 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:49:59,597 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:49:59,597 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:50:00,615 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:50:01,794 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:50:02,422 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:50:02,422 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:50:02,422 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:50:02,428 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:50:02,600 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:50:03,603 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:50:05,662 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:50:05,663 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:50:05,664 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:50:05,669 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:50:06,538 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:50:06,605 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:50:07,607 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:50:07,670 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:50:10,697 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:50:10,698 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:50:10,698 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:50:10,703 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:50:11,539 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:50:11,612 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:50:11,612 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:50:12,703 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:50:15,606 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:50:15,607 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:50:15,607 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:50:15,612 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:50:15,614 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:50:15,615 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:50:17,618 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:50:18,612 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:50:19,744 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:50:19,745 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:50:19,745 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:50:19,750 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:50:20,621 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:50:21,540 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:50:21,625 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:50:23,785 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:50:24,069 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:50:24,070 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:50:24,070 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:50:24,075 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:50:24,627 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:50:25,629 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:50:26,541 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:50:26,761 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 18:50:28,600 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:50:28,601 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:50:28,601 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:50:28,606 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:50:28,631 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:50:29,607 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:50:29,633 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:50:30,615 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:50:34,453 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:50:34,453 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:50:34,453 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:50:34,458 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:50:34,636 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:50:35,493 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:50:35,639 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:50:36,542 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:50:38,202 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:50:38,202 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:50:38,203 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:50:38,208 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:50:38,641 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:50:39,643 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:50:41,209 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:50:41,542 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:50:43,229 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:50:43,230 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:50:43,231 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:50:43,236 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:50:43,646 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:50:45,615 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:50:45,650 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:50:46,236 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:50:47,790 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:50:47,791 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:50:47,791 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:50:47,796 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:50:48,652 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:50:49,655 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:50:51,544 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:50:51,834 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:50:53,853 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:50:53,854 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:50:53,854 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:50:53,859 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:50:54,657 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:50:55,660 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:50:56,544 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:50:56,766 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 18:50:57,071 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:50:57,072 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:50:57,072 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:50:57,072 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:50:57,077 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:50:57,661 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:50:59,664 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:51:00,323 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:51:00,324 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:51:00,325 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:51:00,329 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:51:00,615 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:51:00,665 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:51:01,668 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:51:02,330 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:51:03,506 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:51:03,507 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:51:03,507 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:51:03,512 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:51:03,670 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:51:05,673 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:51:06,545 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:51:06,579 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:51:06,579 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:51:06,580 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:51:06,584 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:51:06,673 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:51:07,585 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:51:07,676 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:51:09,852 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:51:09,853 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:51:09,853 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:51:09,858 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:51:10,678 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:51:11,546 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:51:11,680 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:51:12,859 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:51:15,615 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:51:17,860 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:51:18,777 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:51:18,778 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:51:18,778 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:51:18,783 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:51:19,687 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:51:19,687 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:51:21,547 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:51:23,822 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:51:24,397 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:51:24,398 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:51:24,398 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:51:24,403 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:51:24,690 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:51:25,692 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:51:26,548 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:51:26,766 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 18:51:27,756 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:51:27,756 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:51:27,757 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:51:27,761 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:51:28,694 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:51:29,697 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:51:29,762 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:51:30,615 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:51:30,990 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:51:30,991 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:51:30,991 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:51:30,996 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:51:31,700 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:51:31,700 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:51:34,266 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:51:34,267 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:51:34,267 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:51:34,272 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:51:34,702 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:51:35,272 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:51:35,704 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:51:36,549 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:51:38,304 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:51:38,305 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:51:38,305 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:51:38,310 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:51:38,706 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:51:39,708 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:51:40,311 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:51:41,549 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:51:42,232 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:51:42,233 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:51:42,233 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:51:42,238 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:51:42,711 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:51:43,713 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:51:45,615 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:51:45,911 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:51:45,911 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:51:45,911 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:51:45,912 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:51:45,916 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:51:46,716 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:51:47,719 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:51:49,361 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:51:49,362 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:51:49,362 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:51:49,367 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:51:49,720 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:51:51,367 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:51:51,550 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:51:51,723 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:51:54,129 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:51:54,129 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:51:54,130 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:51:54,134 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:51:54,725 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:51:55,727 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:51:56,551 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:51:56,771 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 18:51:56,772 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:51:58,406 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:51:58,407 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:51:58,407 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:51:58,412 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:51:58,729 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:51:59,732 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:52:00,615 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:52:02,413 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:52:02,431 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:52:02,431 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:52:02,432 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:52:02,436 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:52:02,735 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:52:03,738 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:52:06,013 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:52:06,013 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:52:06,014 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:52:06,018 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:52:06,552 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:52:06,739 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:52:07,740 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:52:08,019 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:52:10,348 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:52:10,349 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:52:10,349 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:52:10,354 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:52:10,742 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:52:11,553 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:52:11,744 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:52:13,355 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:52:15,128 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:52:15,128 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:52:15,129 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:52:15,133 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:52:15,615 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:52:15,747 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:52:17,750 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:52:19,036 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:52:19,037 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:52:19,037 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:52:19,037 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:52:19,042 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:52:19,753 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:52:19,753 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:52:21,553 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:52:23,348 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:52:23,348 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:52:23,349 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:52:23,353 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:52:23,756 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:52:24,354 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:52:25,759 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:52:26,554 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:52:26,770 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 18:52:26,860 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:52:26,860 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:52:26,860 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:52:26,865 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:52:27,762 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:52:27,762 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:52:29,866 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:52:30,165 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:52:30,166 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:52:30,166 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:52:30,171 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:52:30,616 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:52:30,764 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:52:31,766 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:52:34,422 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:52:34,423 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:52:34,423 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:52:34,428 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:52:34,768 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:52:35,466 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:52:35,770 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:52:36,555 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:52:37,843 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:52:37,843 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:52:37,844 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:52:37,848 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:52:38,772 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:52:39,774 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:52:40,849 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:52:41,396 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:52:41,396 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:52:41,397 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:52:41,402 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:52:41,556 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:52:41,776 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:52:43,786 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:52:45,437 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:52:45,438 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:52:45,438 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:52:45,443 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:52:45,616 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:52:45,788 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:52:46,443 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:52:47,792 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:52:49,953 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:52:49,953 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:52:49,953 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:52:49,958 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:52:50,793 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:52:51,557 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:52:51,794 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:52:51,994 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:52:55,682 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:52:55,682 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:52:55,683 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:52:55,687 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:52:55,796 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:52:56,557 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:52:56,775 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 18:52:57,776 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:52:57,799 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:52:58,938 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:52:58,939 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:52:58,939 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:52:58,944 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:52:59,803 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:52:59,803 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:53:00,616 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:53:02,945 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:53:04,228 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:53:04,228 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:53:04,228 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:53:04,233 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:53:04,807 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:53:05,809 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:53:06,558 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:53:07,806 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:53:07,807 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:53:07,807 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:53:07,813 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:53:07,813 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:53:08,814 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:53:09,817 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:53:11,559 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:53:12,168 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:53:12,169 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:53:12,169 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:53:12,174 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:53:12,818 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:53:13,821 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:53:14,174 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:53:15,527 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:53:15,528 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:53:15,528 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:53:15,533 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:53:15,616 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:53:15,822 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:53:17,825 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:53:19,048 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:53:19,049 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:53:19,049 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:53:19,053 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:53:19,828 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:53:19,829 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:53:20,090 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:53:21,560 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:53:22,881 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:53:22,881 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:53:22,882 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:53:22,886 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:53:23,832 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:53:23,833 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:53:25,887 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:53:26,560 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:53:26,774 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 18:53:26,975 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:53:26,976 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:53:26,976 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:53:26,981 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:53:27,837 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:53:27,837 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:53:30,519 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:53:30,520 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:53:30,520 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:53:30,525 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:53:30,616 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:53:30,838 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:53:31,525 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:53:31,841 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:53:34,401 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:53:34,402 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:53:34,402 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:53:34,407 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:53:34,843 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:53:35,845 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:53:36,561 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:53:37,446 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:53:41,562 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:53:41,639 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:53:41,640 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:53:41,640 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:53:41,645 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:53:41,849 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:53:42,645 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:53:43,852 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:53:45,616 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:53:47,646 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:53:48,553 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:53:48,554 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:53:48,554 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:53:48,559 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:53:48,857 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:53:49,859 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:53:51,563 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:53:52,337 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:53:52,338 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:53:52,338 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:53:52,343 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:53:52,861 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:53:53,343 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:53:53,863 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:53:55,268 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:53:55,269 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:53:55,269 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:53:55,274 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:53:55,865 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:53:56,563 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:53:56,779 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 18:53:57,868 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:53:58,781 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:54:00,616 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:54:01,075 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:54:01,076 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:54:01,076 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:54:01,081 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:54:01,874 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:54:01,874 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:54:04,119 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:54:06,564 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:54:07,490 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:54:07,490 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:54:07,491 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:54:07,495 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:54:07,878 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:54:09,496 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:54:09,881 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:54:11,565 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:54:12,086 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:54:12,087 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:54:12,087 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:54:12,091 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:54:12,882 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:54:13,885 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:54:15,092 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:54:15,616 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:54:15,973 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:54:15,973 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:54:15,973 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:54:15,978 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:54:16,887 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:54:17,889 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:54:20,348 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:54:20,349 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:54:20,349 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:54:20,350 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:54:20,353 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:54:20,891 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:54:21,566 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:54:21,894 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:54:23,196 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:54:23,197 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:54:23,198 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:54:23,202 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:54:23,897 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:54:23,897 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:54:26,203 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:54:26,566 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:54:26,778 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 18:54:28,145 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:54:28,146 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:54:28,146 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:54:28,151 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:54:28,900 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:54:29,902 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:54:30,616 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:54:32,152 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:54:32,660 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:54:32,661 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:54:32,661 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:54:32,666 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:54:32,904 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:54:33,906 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:54:36,567 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:54:37,054 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:54:37,055 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:54:37,055 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:54:37,060 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:54:37,910 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:54:37,911 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:54:38,061 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:54:40,504 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:54:40,505 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:54:40,505 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:54:40,510 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:54:40,913 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:54:41,568 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:54:41,915 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:54:43,510 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:54:45,426 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:54:45,426 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:54:45,427 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:54:45,431 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:54:45,616 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:54:45,919 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:54:47,922 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:54:49,432 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:54:51,569 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:54:53,939 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:54:53,940 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:54:53,940 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:54:53,946 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:54:54,926 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:54:54,946 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:54:55,928 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:54:56,569 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:54:56,783 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 18:54:58,516 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:54:58,517 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:54:58,518 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:54:58,522 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:54:58,930 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:54:59,933 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:55:00,523 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:55:00,616 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:55:05,523 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:55:06,570 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:55:07,108 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:55:07,109 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:55:07,109 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:55:07,114 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:55:07,944 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:55:07,944 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:55:10,394 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:55:10,395 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:55:10,395 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:55:10,400 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:55:10,947 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:55:11,400 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:55:11,571 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:55:11,948 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:55:14,892 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:55:14,893 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:55:14,893 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:55:14,898 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:55:14,952 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:55:15,617 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:55:15,955 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:55:16,899 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:55:18,815 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:55:18,815 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:55:18,816 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:55:18,820 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:55:18,958 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:55:19,960 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:55:21,572 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:55:22,766 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:55:22,767 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:55:22,767 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:55:22,767 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:55:22,772 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:55:22,962 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:55:23,965 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:55:26,572 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:55:26,782 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 18:55:27,527 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:55:27,528 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:55:27,528 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:55:27,533 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:55:27,967 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:55:28,534 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:55:29,970 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:55:30,540 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:55:30,541 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:55:30,541 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:55:30,546 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:55:30,617 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:55:30,971 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:55:31,972 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:55:33,547 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:55:35,838 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:55:35,838 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:55:35,838 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:55:35,843 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:55:35,974 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:55:36,573 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:55:37,977 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:55:38,883 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:55:41,574 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:55:41,772 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:55:41,773 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:55:41,773 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:55:41,778 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:55:41,980 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:55:43,983 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:55:44,778 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:55:45,215 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:55:45,216 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:55:45,217 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:55:45,221 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:55:45,617 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:55:45,987 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:55:45,987 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:55:50,222 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:55:51,575 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:55:53,849 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:55:53,850 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:55:53,850 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:55:53,855 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:55:53,992 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:55:55,856 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:55:55,995 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:55:56,575 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:55:56,784 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 18:55:58,181 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:55:58,182 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:55:58,182 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:55:58,186 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:55:58,996 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:55:59,999 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:56:00,617 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:56:01,187 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:56:01,312 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:56:01,313 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:56:01,313 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:56:01,318 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:56:02,001 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:56:04,004 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:56:06,261 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:56:06,262 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:56:06,263 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:56:06,263 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:56:06,267 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:56:06,577 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:56:07,006 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:56:08,009 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:56:09,765 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:56:09,765 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:56:09,766 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:56:09,770 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:56:10,010 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:56:11,577 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:56:11,771 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:56:12,013 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:56:15,617 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:56:16,771 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:56:18,331 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:56:18,332 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:56:18,332 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:56:18,337 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:56:19,017 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:56:20,020 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:56:21,578 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:56:22,375 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:56:24,153 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:56:24,154 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:56:24,154 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:56:24,159 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:56:25,022 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:56:26,025 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:56:26,579 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:56:26,786 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 18:56:27,188 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:56:27,189 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:56:27,189 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:56:27,193 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:56:28,028 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:56:28,028 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:56:28,194 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:56:30,617 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:56:32,426 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:56:32,427 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:56:32,427 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:56:32,432 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:56:33,031 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:56:33,470 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:56:34,034 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:56:36,500 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:56:36,501 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:56:36,501 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:56:36,506 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:56:36,580 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:56:37,035 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:56:38,038 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:56:38,506 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:56:41,580 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:56:41,743 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:56:41,744 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:56:41,744 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:56:41,748 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:56:42,040 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:56:43,749 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:56:44,043 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:56:45,617 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:56:48,749 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:56:50,276 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:56:50,277 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:56:50,277 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:56:50,282 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:56:51,049 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:56:51,581 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:56:52,051 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:56:54,319 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:56:54,322 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:56:54,323 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:56:54,323 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:56:54,328 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:56:55,053 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:56:56,056 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:56:56,582 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:56:56,788 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 18:56:57,601 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:56:57,602 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:56:57,602 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:56:57,607 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:56:58,057 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:56:59,608 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:57:00,060 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:57:00,617 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:57:01,462 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:57:01,463 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:57:01,463 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:57:01,468 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:57:02,063 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:57:04,066 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:57:04,587 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:57:04,588 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:57:04,588 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:57:04,593 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:57:05,066 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:57:05,594 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:57:06,069 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:57:06,583 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:57:08,190 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:57:08,191 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:57:08,192 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:57:08,196 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:57:09,071 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:57:10,075 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:57:11,197 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:57:11,477 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:57:11,478 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:57:11,479 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:57:11,483 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:57:11,583 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:57:12,076 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:57:14,079 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:57:15,172 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:57:15,173 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:57:15,173 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:57:15,178 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:57:15,617 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:57:16,082 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:57:16,083 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:57:17,179 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:57:18,957 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:57:18,961 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:57:18,961 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:57:18,966 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:57:19,085 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:57:20,087 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:57:21,584 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:57:22,855 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:57:22,856 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:57:22,856 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:57:22,857 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:57:22,862 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:57:23,089 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:57:24,091 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:57:26,204 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:57:26,204 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:57:26,205 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:57:26,209 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:57:26,585 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:57:26,790 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 18:57:27,093 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:57:28,094 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:57:28,792 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:57:30,617 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:57:32,673 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:57:32,673 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:57:32,674 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:57:32,679 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:57:33,097 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:57:34,099 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:57:34,715 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:57:36,586 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:57:37,233 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:57:37,234 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:57:37,234 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:57:37,239 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:57:38,103 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:57:38,104 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:57:40,240 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:57:41,586 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:57:41,753 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:57:41,754 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:57:41,754 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:57:41,759 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:57:42,106 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:57:44,109 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:57:45,422 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:57:45,422 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:57:45,423 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:57:45,423 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:57:45,427 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:57:45,618 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:57:46,114 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:57:46,114 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:57:49,313 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:57:49,313 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:57:49,314 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:57:49,318 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:57:50,118 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:57:50,118 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:57:51,355 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:57:51,587 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:57:52,889 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:57:52,889 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:57:52,890 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:57:52,895 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:57:53,119 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:57:54,122 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:57:56,138 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:57:56,139 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:57:56,139 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:57:56,144 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:57:56,588 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:57:56,793 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 18:57:56,795 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:57:57,124 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:57:58,126 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:58:00,390 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:58:00,391 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:58:00,391 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:58:00,396 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:58:00,618 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:58:01,128 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:58:02,132 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:58:02,396 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:58:04,071 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:58:04,072 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:58:04,072 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:58:04,077 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:58:04,133 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:58:06,136 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:58:06,589 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:58:07,695 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:58:07,695 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:58:07,696 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:58:07,696 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:58:07,701 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:58:08,137 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:58:10,140 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:58:11,589 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:58:12,702 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:58:13,145 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:58:13,146 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:58:13,146 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:58:13,152 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:58:14,144 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:58:14,144 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:58:15,618 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:58:17,003 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:58:17,003 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:58:17,004 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:58:17,008 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:58:17,146 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:58:18,047 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:58:18,148 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:58:20,250 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:58:20,251 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:58:20,251 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:58:20,256 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:58:21,150 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:58:21,590 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:58:22,153 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:58:23,257 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:58:24,770 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:58:24,771 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:58:24,771 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:58:24,776 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:58:25,154 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:58:26,157 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:58:26,591 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:58:26,795 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 18:58:28,796 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:58:30,273 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:58:30,274 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:58:30,274 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:58:30,278 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:58:30,618 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:58:31,159 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:58:32,162 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:58:34,279 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:58:35,176 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:58:35,177 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:58:35,177 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:58:35,182 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:58:36,166 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:58:36,166 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:58:36,592 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:58:38,836 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:58:38,837 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:58:38,837 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:58:38,842 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:58:39,168 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:58:39,843 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:58:40,170 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:58:41,592 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:58:42,517 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:58:42,518 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:58:42,518 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:58:42,523 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:58:43,172 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:58:44,175 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:58:45,524 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:58:45,618 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:58:46,048 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:58:46,049 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:58:46,049 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:58:46,054 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:58:46,177 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:58:48,180 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:58:51,091 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:58:51,432 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:58:51,433 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:58:51,434 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:58:51,438 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:58:51,593 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:58:52,184 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:58:52,184 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:58:54,697 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:58:54,699 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:58:54,699 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:58:54,703 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:58:55,186 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:58:56,188 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:58:56,594 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:58:56,704 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:58:56,797 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 18:58:58,248 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:58:58,249 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:58:58,249 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:58:58,254 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:58:59,190 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:59:00,191 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:59:00,618 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:59:02,255 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:59:02,629 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:59:02,630 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:59:02,630 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:59:02,635 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:59:03,194 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:59:04,196 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:59:05,989 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:59:05,990 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:59:05,990 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:59:05,995 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:59:06,197 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:59:06,595 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:59:07,996 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:59:08,201 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:59:09,733 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:59:09,734 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:59:09,734 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:59:09,739 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:59:10,202 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:59:11,596 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:59:12,205 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:59:13,740 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:59:14,057 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:59:14,058 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:59:14,058 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:59:14,063 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:59:14,206 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:59:15,618 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:59:16,209 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:59:17,479 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:59:17,480 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:59:17,480 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:59:17,485 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:59:18,213 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:59:18,213 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:59:19,523 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:59:21,610 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:59:22,704 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:59:22,705 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:59:22,705 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:59:22,710 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:59:23,215 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:59:24,218 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:59:24,711 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:59:26,610 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:59:26,799 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 18:59:27,409 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:59:27,410 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:59:27,410 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:59:27,415 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:59:28,222 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:59:28,222 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:59:30,416 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:59:30,618 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:59:32,087 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:59:32,088 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:59:32,088 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:59:32,093 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:59:32,224 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:59:34,227 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:59:36,131 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:59:36,611 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:59:40,630 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:59:40,631 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:59:40,631 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:59:40,636 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:59:41,232 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:59:41,612 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:59:41,636 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:59:42,234 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:59:45,618 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:59:46,403 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:59:46,404 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:59:46,404 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:59:46,409 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:59:47,239 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:59:47,447 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:59:48,241 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:59:49,613 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:59:49,614 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:59:49,614 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:59:49,619 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:59:50,244 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 18:59:51,613 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:59:52,247 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 18:59:52,620 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:59:56,613 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 18:59:56,801 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 18:59:57,803 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:59:58,089 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:59:58,090 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 18:59:58,091 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:59:58,095 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:59:58,250 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:00:00,253 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:00:00,618 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:00:02,475 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:00:02,476 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:00:02,476 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:00:02,481 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:00:03,256 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:00:03,518 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:00:04,259 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:00:05,586 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:00:05,587 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:00:05,588 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:00:05,592 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:00:06,262 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:00:06,262 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:00:06,614 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:00:08,593 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:00:09,712 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:00:09,713 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:00:09,713 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:00:09,718 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:00:10,264 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:00:11,615 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:00:12,267 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:00:13,719 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:00:13,841 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:00:13,842 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:00:13,842 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:00:13,847 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:00:14,268 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:00:15,618 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:00:16,271 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:00:18,209 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:00:18,210 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:00:18,210 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:00:18,215 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:00:18,272 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:00:19,255 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:00:20,275 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:00:21,616 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:00:22,905 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:00:22,906 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:00:22,907 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:00:22,912 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:00:23,277 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:00:24,280 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:00:24,912 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:00:26,617 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:00:26,658 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:00:26,659 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:00:26,659 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:00:26,664 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:00:26,806 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 19:00:27,281 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:00:28,284 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:00:30,618 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:00:30,808 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:00:33,351 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:00:33,352 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:00:33,352 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:00:33,357 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:00:34,289 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:00:34,289 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:00:36,395 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:00:36,542 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:00:36,543 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:00:36,543 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:00:36,548 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:00:36,618 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:00:37,291 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:00:38,294 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:00:39,765 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:00:39,766 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:00:39,766 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:00:39,771 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:00:40,295 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:00:41,619 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:00:41,772 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:00:42,298 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:00:45,618 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:00:46,772 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:00:48,199 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:00:48,200 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:00:48,200 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:00:48,205 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:00:48,302 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:00:50,306 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:00:50,620 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:00:52,243 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:00:53,512 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:00:53,513 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:00:53,513 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:00:53,518 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:00:54,310 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:00:54,310 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:00:55,620 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:00:56,806 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 19:00:56,957 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:00:56,958 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:00:56,958 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:00:56,963 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:00:57,311 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:00:57,964 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:00:58,314 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:01:00,619 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:01:00,674 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:01:00,675 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:01:00,675 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:01:00,680 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:01:01,316 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:01:02,319 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:01:03,719 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:01:04,682 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:01:04,683 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:01:04,683 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:01:04,688 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:01:05,321 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:01:05,621 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:01:06,323 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:01:08,465 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:01:08,466 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:01:08,466 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:01:08,471 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:01:09,325 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:01:09,471 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:01:10,328 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:01:10,622 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:01:13,375 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:01:13,376 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:01:13,376 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:01:13,381 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:01:14,332 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:01:14,332 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:01:15,382 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:01:15,619 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:01:18,050 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:01:18,051 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:01:18,051 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:01:18,056 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:01:18,334 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:01:20,337 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:01:20,623 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:01:21,095 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:01:21,265 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:01:21,266 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:01:21,267 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:01:21,271 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:01:21,338 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:01:22,340 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:01:24,656 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:01:24,657 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:01:24,658 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:01:24,662 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:01:25,342 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:01:25,623 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:01:26,344 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:01:26,663 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:01:26,811 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 19:01:28,177 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:01:28,178 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:01:28,178 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:01:28,183 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:01:28,346 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:01:30,349 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:01:30,619 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:01:31,975 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:01:31,976 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:01:31,976 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:01:31,976 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:01:31,981 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:01:32,350 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:01:34,353 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:01:35,624 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:01:36,452 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:01:36,453 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:01:36,454 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:01:36,458 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:01:37,355 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:01:37,459 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:01:38,357 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:01:39,939 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:01:39,939 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:01:39,940 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:01:39,944 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:01:40,359 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:01:40,625 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:01:42,362 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:01:42,945 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:01:43,579 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:01:43,580 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:01:43,580 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:01:43,585 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:01:44,365 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:01:44,365 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:01:45,619 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:01:47,006 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:01:47,006 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:01:47,007 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:01:47,011 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:01:47,368 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:01:48,050 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:01:48,371 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:01:50,626 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:01:53,051 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:01:53,139 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:01:53,139 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:01:53,140 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:01:53,145 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:01:53,374 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:01:54,376 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:01:55,626 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:01:56,809 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 19:01:57,458 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:01:57,459 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:01:57,459 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:01:57,464 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:01:58,379 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:01:58,379 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:01:58,465 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:02:00,619 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:02:01,620 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:02:01,620 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:02:01,621 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:02:01,626 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:02:02,384 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:02:02,385 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:02:03,662 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:02:05,186 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:02:05,186 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:02:05,187 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:02:05,191 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:02:05,386 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:02:05,627 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:02:06,388 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:02:09,192 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:02:10,404 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:02:10,405 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:02:10,406 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:02:10,411 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:02:10,628 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:02:11,391 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:02:12,394 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:02:14,365 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:02:14,366 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:02:14,366 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:02:14,366 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:02:14,371 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:02:14,395 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:02:15,619 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:02:16,398 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:02:17,640 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:02:17,640 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:02:17,641 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:02:17,645 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:02:18,401 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:02:18,402 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:02:19,682 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:02:20,629 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:02:21,207 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:02:21,208 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:02:21,208 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:02:21,213 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:02:21,403 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:02:22,406 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:02:24,882 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:02:24,882 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:02:24,883 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:02:24,883 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:02:24,887 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:02:25,407 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:02:25,629 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:02:26,410 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:02:26,814 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 19:02:28,018 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:02:28,019 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:02:28,019 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:02:28,024 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:02:28,411 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:02:30,025 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:02:30,414 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:02:30,619 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:02:31,508 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:02:31,509 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:02:31,509 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:02:31,514 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:02:32,417 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:02:32,417 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:02:34,773 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:02:34,774 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:02:34,774 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:02:34,779 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:02:35,419 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:02:35,630 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:02:35,779 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:02:36,421 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:02:38,498 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:02:38,499 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:02:38,499 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:02:38,504 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:02:39,423 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:02:40,425 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:02:40,631 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:02:41,505 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:02:45,619 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:02:46,505 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:02:47,032 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:02:47,033 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:02:47,033 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:02:47,038 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:02:47,431 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:02:48,434 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:02:50,632 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:02:50,902 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:02:50,903 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:02:50,903 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:02:50,907 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:02:51,436 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:02:51,908 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:02:52,438 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:02:55,060 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:02:55,061 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:02:55,061 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:02:55,066 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:02:55,440 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:02:55,632 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:02:56,443 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:02:56,813 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 19:02:57,815 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:02:58,288 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:02:58,289 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:02:58,290 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:02:58,294 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:02:58,444 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:03:00,447 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:03:00,619 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:03:01,572 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:03:01,573 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:03:01,573 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:03:01,578 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:03:02,451 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:03:02,451 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:03:03,615 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:03:05,311 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:03:05,312 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:03:05,313 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:03:05,318 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:03:05,453 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:03:05,633 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:03:06,455 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:03:09,319 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:03:10,601 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:03:10,602 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:03:10,603 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:03:10,607 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:03:10,634 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:03:11,458 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:03:12,461 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:03:13,948 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:03:13,949 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:03:13,950 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:03:13,954 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:03:14,462 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:03:14,955 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:03:15,619 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:03:16,465 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:03:18,011 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:03:18,012 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:03:18,012 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:03:18,017 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:03:18,466 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:03:20,055 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:03:20,467 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:03:20,635 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:03:21,587 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:03:21,588 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:03:21,588 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:03:21,593 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:03:22,471 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:03:22,471 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:03:24,830 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:03:24,831 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:03:24,831 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:03:24,837 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:03:25,472 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:03:25,635 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:03:25,837 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:03:26,475 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:03:26,819 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 19:03:29,918 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:03:29,919 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:03:29,919 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:03:29,924 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:03:30,477 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:03:30,620 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:03:30,924 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:03:32,480 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:03:33,648 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:03:33,649 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:03:33,649 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:03:33,654 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:03:34,483 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:03:34,484 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:03:35,637 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:03:36,691 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:03:37,283 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:03:37,284 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:03:37,285 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:03:37,290 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:03:37,485 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:03:38,488 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:03:40,637 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:03:42,290 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:03:43,643 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:03:43,644 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:03:43,644 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:03:43,649 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:03:44,493 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:03:44,494 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:03:45,620 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:03:47,075 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:03:47,075 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:03:47,076 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:03:47,081 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:03:47,496 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:03:48,118 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:03:48,499 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:03:50,638 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:03:51,137 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:03:51,138 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:03:51,138 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:03:51,143 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:03:51,501 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:03:52,504 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:03:53,144 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:03:54,933 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:03:54,934 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:03:54,934 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:03:54,939 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:03:55,506 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:03:55,639 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:03:56,508 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:03:56,817 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 19:03:58,819 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:04:00,482 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:04:00,483 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:04:00,484 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:04:00,489 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:04:00,510 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:04:00,620 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:04:02,515 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:04:04,398 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:04:04,399 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:04:04,399 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:04:04,399 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:04:04,404 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:04:04,516 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:04:05,640 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:04:06,519 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:04:08,380 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:04:08,380 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:04:08,381 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:04:08,386 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:04:08,520 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:04:10,387 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:04:10,524 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:04:11,047 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:04:12,560 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:04:12,561 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:04:12,561 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:04:12,566 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:04:13,525 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:04:14,528 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:04:15,567 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:04:15,620 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:04:15,921 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:04:15,921 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:04:15,922 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:04:15,926 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:04:16,529 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:04:18,532 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:04:19,227 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:04:19,228 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:04:19,228 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:04:19,234 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:04:19,533 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:04:20,536 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:04:21,048 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:04:21,234 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:04:24,820 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:04:24,820 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:04:24,821 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:04:24,826 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:04:25,539 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:04:26,049 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:04:26,539 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:04:26,823 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 19:04:26,824 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:04:28,146 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:04:28,147 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:04:28,147 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:04:28,152 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:04:28,540 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:04:30,543 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:04:30,620 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:04:31,426 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:04:31,426 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:04:31,427 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:04:31,432 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:04:31,544 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:04:32,470 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:04:32,547 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:04:36,050 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:04:37,471 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:04:40,582 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:04:40,583 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:04:40,583 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:04:40,588 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:04:41,051 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:04:41,553 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:04:42,555 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:04:42,588 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:04:43,439 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:04:43,440 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:04:43,440 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:04:43,445 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:04:43,556 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:04:44,558 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:04:45,620 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:04:47,057 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:04:47,058 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:04:47,058 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:04:47,063 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:04:47,562 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:04:48,102 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:04:48,562 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:04:50,353 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:04:50,354 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:04:50,354 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:04:50,359 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:04:50,563 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:04:51,052 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:04:52,566 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:04:53,359 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:04:53,905 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:04:53,906 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:04:53,906 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:04:53,911 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:04:54,569 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:04:54,569 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:04:56,052 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:04:56,822 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 19:04:58,270 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:04:58,271 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:04:58,271 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:04:58,276 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:04:58,572 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:04:59,276 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:05:00,575 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:05:00,620 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:05:02,229 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:05:02,230 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:05:02,230 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:05:02,235 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:05:02,577 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:05:04,578 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:05:05,275 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:05:05,480 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:05:05,481 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:05:05,481 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:05:05,486 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:05:05,579 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:05:06,053 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:05:06,582 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:05:10,487 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:05:11,054 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:05:14,132 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:05:14,133 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:05:14,133 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:05:14,138 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:05:14,588 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:05:15,620 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:05:16,139 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:05:16,591 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:05:19,476 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:05:19,477 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:05:19,477 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:05:19,482 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:05:19,593 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:05:20,595 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:05:21,055 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:05:21,518 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:05:23,005 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:05:23,006 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:05:23,006 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:05:23,011 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:05:23,597 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:05:24,599 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:05:26,055 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:05:26,764 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:05:26,764 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:05:26,765 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:05:26,765 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:05:26,770 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:05:26,827 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 19:05:27,601 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:05:28,604 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:05:30,620 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:05:30,999 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:05:31,000 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:05:31,000 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:05:31,005 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:05:31,605 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:05:32,042 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:05:32,608 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:05:34,747 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:05:34,748 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:05:34,748 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:05:34,753 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:05:35,609 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:05:36,056 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:05:36,613 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:05:37,585 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:05:37,586 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:05:37,586 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:05:37,587 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:05:37,591 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:05:37,614 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:05:38,616 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:05:41,057 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:05:41,311 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:05:41,311 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:05:41,312 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:05:41,317 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:05:41,619 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:05:42,621 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:05:43,317 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:05:45,064 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:05:45,065 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:05:45,065 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:05:45,070 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:05:45,621 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:05:45,624 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:05:46,626 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:05:49,071 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:05:51,058 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:05:51,102 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:05:51,103 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:05:51,103 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:05:51,109 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:05:51,629 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:05:52,632 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:05:54,109 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:05:54,546 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:05:54,547 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:05:54,547 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:05:54,552 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:05:54,633 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:05:56,058 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:05:56,636 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:05:56,826 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 19:05:58,509 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:05:58,510 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:05:58,510 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:05:58,515 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:05:58,637 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:05:59,516 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:06:00,621 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:06:00,640 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:06:02,061 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:06:02,062 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:06:02,062 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:06:02,067 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:06:02,643 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:06:04,646 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:06:05,107 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:06:06,059 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:06:06,364 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:06:06,365 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:06:06,365 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:06:06,370 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:06:06,647 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:06:08,650 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:06:09,926 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:06:09,927 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:06:09,927 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:06:09,932 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:06:10,654 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:06:10,654 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:06:10,933 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:06:11,060 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:06:13,640 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:06:13,641 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:06:13,642 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:06:13,647 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:06:13,655 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:06:14,658 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:06:15,621 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:06:16,647 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:06:21,061 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:06:21,648 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:06:22,185 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:06:22,185 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:06:22,186 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:06:22,191 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:06:22,663 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:06:24,666 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:06:25,201 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:06:25,201 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:06:25,202 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:06:25,207 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:06:25,666 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:06:26,061 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:06:26,667 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:06:26,828 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 19:06:26,829 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:06:29,719 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:06:29,720 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:06:29,720 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:06:29,725 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:06:30,621 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:06:30,671 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:06:30,671 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:06:32,726 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:06:36,062 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:06:36,110 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:06:36,111 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:06:36,111 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:06:36,116 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:06:36,674 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:06:38,116 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:06:38,677 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:06:39,266 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:06:39,267 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:06:39,268 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:06:39,273 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:06:39,678 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:06:40,680 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:06:41,063 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:06:43,008 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:06:43,009 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:06:43,009 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:06:43,014 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:06:43,682 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:06:44,014 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:06:44,685 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:06:45,621 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:06:49,015 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:06:49,430 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:06:49,431 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:06:49,431 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:06:49,436 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:06:49,689 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:06:50,691 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:06:51,064 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:06:53,935 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:06:53,936 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:06:53,936 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:06:53,941 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:06:54,696 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:06:54,696 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:06:54,942 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:06:56,064 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:06:56,830 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 19:06:57,499 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:06:57,500 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:06:57,500 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:06:57,505 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:06:57,697 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:06:58,700 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:07:00,506 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:07:00,621 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:07:01,855 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:07:01,856 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:07:01,856 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:07:01,861 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:07:02,705 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:07:02,705 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:07:05,899 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:07:06,065 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:07:06,113 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:07:06,114 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:07:06,114 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:07:06,119 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:07:06,707 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:07:08,710 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:07:09,012 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:07:09,013 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:07:09,014 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:07:09,019 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:07:09,711 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:07:10,713 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:07:11,019 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:07:11,066 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:07:12,294 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:07:12,295 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:07:12,295 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:07:12,301 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:07:12,715 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:07:14,718 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:07:15,621 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:07:16,301 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:07:16,621 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:07:16,622 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:07:16,622 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:07:16,627 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:07:16,719 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:07:18,722 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:07:19,817 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:07:19,818 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:07:19,818 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:07:19,823 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:07:20,725 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:07:20,725 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:07:21,067 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:07:21,824 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:07:24,075 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:07:24,076 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:07:24,076 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:07:24,080 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:07:24,729 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:07:24,730 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:07:26,068 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:07:26,832 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 19:07:26,834 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:07:30,516 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:07:30,517 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:07:30,517 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:07:30,522 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:07:30,621 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:07:30,733 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:07:32,523 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:07:32,736 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:07:36,069 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:07:36,474 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:07:36,475 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:07:36,475 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:07:36,480 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:07:36,738 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:07:38,481 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:07:38,741 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:07:39,955 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:07:39,956 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:07:39,956 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:07:39,961 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:07:40,744 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:07:40,744 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:07:41,069 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:07:43,206 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:07:43,207 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:07:43,207 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:07:43,212 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:07:43,746 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:07:44,213 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:07:44,748 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:07:45,621 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:07:48,327 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:07:48,328 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:07:48,328 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:07:48,333 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:07:48,752 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:07:49,370 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:07:50,755 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:07:51,070 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:07:51,823 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:07:51,824 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:07:51,824 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:07:51,829 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:07:52,758 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:07:52,758 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:07:54,830 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:07:56,071 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:07:56,835 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 19:07:59,836 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:08:00,262 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:08:00,263 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:08:00,263 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:08:00,267 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:08:00,621 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:08:00,762 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:08:02,767 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:08:04,945 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:08:04,945 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:08:04,945 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:08:04,946 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:08:04,950 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:08:05,768 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:08:06,074 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:08:06,771 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:08:09,530 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:08:09,531 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:08:09,531 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:08:09,536 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:08:09,773 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:08:10,536 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:08:10,775 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:08:11,072 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:08:15,537 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:08:15,621 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:08:15,939 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:08:15,940 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:08:15,940 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:08:15,945 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:08:16,780 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:08:16,780 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:08:20,983 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:08:21,073 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:08:21,552 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:08:21,552 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:08:21,553 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:08:21,558 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:08:21,783 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:08:22,785 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:08:25,477 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:08:25,478 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:08:25,478 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:08:25,483 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:08:25,787 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:08:26,074 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:08:26,483 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:08:26,788 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:08:26,836 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 19:08:30,595 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:08:30,596 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:08:30,596 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:08:30,601 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:08:30,622 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:08:30,790 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:08:31,601 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:08:32,793 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:08:35,335 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:08:35,336 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:08:35,337 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:08:35,341 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:08:35,795 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:08:36,075 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:08:36,797 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:08:37,379 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:08:38,829 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:08:38,830 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:08:38,830 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:08:38,835 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:08:39,799 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:08:40,801 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:08:41,075 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:08:42,836 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:08:43,040 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:08:43,041 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:08:43,041 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:08:43,046 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:08:43,804 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:08:44,806 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:08:45,622 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:08:47,373 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:08:47,374 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:08:47,374 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:08:47,379 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:08:47,809 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:08:48,418 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:08:48,812 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:08:51,076 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:08:51,436 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:08:51,437 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:08:51,437 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:08:51,442 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:08:51,813 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:08:52,816 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:08:53,443 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:08:55,420 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:08:55,421 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:08:55,421 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:08:55,427 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:08:55,817 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:08:56,077 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:08:56,820 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:08:56,841 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 19:08:58,843 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:09:00,622 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:09:03,843 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:09:03,919 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:09:03,919 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:09:03,920 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:09:03,924 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:09:04,827 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:09:04,828 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:09:06,078 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:09:08,389 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:09:08,390 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:09:08,390 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:09:08,395 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:09:08,829 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:09:09,395 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:09:10,832 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:09:11,138 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:09:13,531 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:09:13,531 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:09:13,532 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:09:13,536 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:09:13,834 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:09:14,537 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:09:14,834 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:09:15,622 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:09:18,294 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:09:18,295 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:09:18,295 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:09:18,300 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:09:18,837 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:09:20,338 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:09:20,840 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:09:21,139 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:09:25,116 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:09:25,117 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:09:25,117 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:09:25,122 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:09:25,843 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:09:26,122 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:09:26,139 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:09:26,841 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 19:09:26,847 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:09:29,266 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:09:29,266 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:09:29,267 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:09:29,272 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:09:29,849 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:09:30,622 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:09:30,851 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:09:31,273 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:09:33,887 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:09:33,888 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:09:33,888 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:09:33,893 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:09:34,855 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:09:34,856 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:09:36,140 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:09:36,931 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:09:39,178 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:09:39,179 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:09:39,179 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:09:39,184 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:09:39,858 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:09:40,861 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:09:41,141 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:09:42,184 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:09:42,955 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:09:42,956 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:09:42,956 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:09:42,961 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:09:43,863 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:09:44,865 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:09:45,622 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:09:46,761 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:09:46,762 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:09:46,762 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:09:46,767 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:09:46,867 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:09:47,806 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:09:48,870 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:09:49,661 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:09:49,662 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:09:49,662 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:09:49,667 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:09:49,871 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:09:50,873 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:09:51,142 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:09:53,667 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:09:55,207 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:09:55,207 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:09:55,208 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:09:55,212 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:09:55,876 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:09:56,142 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:09:56,848 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 19:09:56,879 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:09:58,850 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:09:59,686 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:09:59,686 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:09:59,687 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:09:59,691 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:09:59,882 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:10:00,622 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:10:00,884 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:10:03,203 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:10:03,204 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:10:03,204 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:10:03,209 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:10:03,887 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:10:04,246 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:10:04,890 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:10:06,143 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:10:06,339 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:10:06,340 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:10:06,340 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:10:06,345 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:10:06,891 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:10:08,894 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:10:09,346 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:10:09,696 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:10:09,697 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:10:09,697 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:10:09,702 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:10:09,895 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:10:10,897 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:10:11,144 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:10:12,951 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:10:12,951 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:10:12,952 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:10:12,957 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:10:13,899 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:10:14,902 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:10:14,957 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:10:15,622 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:10:19,958 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:10:21,047 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:10:21,047 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:10:21,047 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:10:21,052 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:10:21,145 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:10:21,906 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:10:22,908 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:10:25,091 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:10:26,145 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:10:26,845 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 19:10:29,391 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:10:29,392 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:10:29,392 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:10:29,397 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:10:29,913 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:10:30,397 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:10:30,622 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:10:30,915 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:10:35,398 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:10:36,146 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:10:37,692 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:10:37,693 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:10:37,693 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:10:37,698 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:10:37,919 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:10:38,922 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:10:40,699 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:10:41,146 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:10:41,185 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:10:41,186 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:10:41,186 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:10:41,191 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:10:41,924 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:10:42,926 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:10:45,622 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:10:46,192 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:10:46,969 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:10:46,969 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:10:46,970 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:10:46,974 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:10:47,930 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:10:48,933 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:10:51,147 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:10:51,741 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:10:51,742 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:10:51,742 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:10:51,742 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:10:51,748 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:10:51,934 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:10:52,937 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:10:56,122 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:10:56,123 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:10:56,123 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:10:56,128 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:10:56,148 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:10:56,850 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 19:10:56,851 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:10:56,939 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:10:56,939 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:10:59,874 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:10:59,874 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:10:59,875 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:10:59,880 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:10:59,940 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:11:00,622 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:11:00,943 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:11:01,880 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:11:04,459 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:11:04,459 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:11:04,460 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:11:04,464 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:11:04,947 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:11:06,149 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:11:06,950 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:11:07,502 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:11:08,379 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:11:08,380 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:11:08,380 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:11:08,385 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:11:08,953 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:11:08,953 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:11:11,150 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:11:13,386 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:11:13,822 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:11:13,823 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:11:13,824 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:11:13,829 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:11:13,956 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:11:14,958 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:11:15,622 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:11:18,277 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:11:18,278 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:11:18,278 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:11:18,283 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:11:18,963 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:11:18,963 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:11:19,321 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:11:21,150 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:11:23,501 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:11:23,502 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:11:23,502 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:11:23,507 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:11:23,966 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:11:24,508 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:11:24,968 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:11:26,151 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:11:26,849 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 19:11:27,999 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:11:28,000 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:11:28,000 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:11:28,005 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:11:28,970 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:11:28,971 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:11:30,006 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:11:30,622 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:11:32,675 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:11:32,675 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:11:32,676 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:11:32,681 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:11:32,973 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:11:34,976 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:11:35,718 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:11:36,152 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:11:38,018 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:11:38,019 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:11:38,019 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:11:38,024 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:11:38,980 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:11:38,980 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:11:41,024 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:11:41,153 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:11:41,296 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:11:41,297 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:11:41,298 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:11:41,303 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:11:41,982 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:11:42,985 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:11:45,622 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:11:45,889 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:11:45,889 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:11:45,890 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:11:45,894 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:11:45,987 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:11:46,929 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:11:46,990 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:11:50,625 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:11:50,626 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:11:50,626 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:11:50,631 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:11:50,992 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:11:51,154 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:11:52,632 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:11:52,995 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:11:55,032 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:11:55,033 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:11:55,033 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:11:55,038 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:11:55,997 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:11:56,154 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:11:56,855 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 19:11:56,999 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:11:57,856 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:11:58,762 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:11:58,763 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:11:58,763 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:11:58,769 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:11:59,000 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:12:00,623 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:12:01,003 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:12:02,006 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:12:02,007 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:12:02,007 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:12:02,012 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:12:03,006 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:12:03,006 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:12:03,049 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:12:06,155 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:12:06,585 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:12:06,585 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:12:06,586 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:12:06,590 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:12:07,008 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:12:08,591 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:12:09,011 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:12:10,370 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:12:10,371 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:12:10,371 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:12:10,376 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:12:11,014 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:12:11,014 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:12:11,156 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:12:14,376 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:12:14,464 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:12:14,464 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:12:14,464 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:12:14,469 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:12:15,018 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:12:15,019 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:12:15,623 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:12:19,133 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:12:19,133 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:12:19,133 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:12:19,138 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:12:20,021 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:12:20,173 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:12:21,024 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:12:21,157 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:12:25,174 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:12:26,158 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:12:26,854 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 19:12:27,652 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:12:27,653 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:12:27,653 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:12:27,658 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:12:28,028 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:12:29,030 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:12:30,623 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:12:30,658 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:12:30,730 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:12:30,731 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:12:30,731 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:12:30,736 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:12:31,031 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:12:33,034 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:12:34,204 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:12:34,205 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:12:34,205 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:12:34,210 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:12:35,037 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:12:35,038 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:12:36,159 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:12:36,210 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:12:38,274 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:12:38,275 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:12:38,275 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:12:38,280 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:12:39,041 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:12:39,042 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:12:41,160 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:12:41,281 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:12:45,623 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:12:46,281 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:12:46,780 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:12:46,781 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:12:46,781 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:12:46,786 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:12:47,047 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:12:49,051 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:12:51,161 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:12:51,822 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:12:52,306 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:12:52,307 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:12:52,307 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:12:52,312 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:12:53,055 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:12:53,055 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:12:56,161 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:12:56,398 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:12:56,399 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:12:56,399 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:12:56,404 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:12:56,856 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 19:12:56,857 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:12:57,059 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:12:57,059 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:13:00,623 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:13:00,770 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:13:00,773 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:13:00,773 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:13:00,778 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:13:01,061 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:13:02,813 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:13:03,065 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:13:06,162 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:13:07,814 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:13:09,366 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:13:09,367 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:13:09,367 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:13:09,372 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:13:10,069 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:13:11,072 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:13:11,163 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:13:13,373 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:13:13,674 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:13:13,675 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:13:13,676 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:13:13,680 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:13:14,073 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:13:15,076 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:13:15,623 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:13:18,609 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:13:18,610 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:13:18,610 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:13:18,610 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:13:18,615 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:13:19,079 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:13:21,082 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:13:21,164 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:13:22,451 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:13:22,452 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:13:22,453 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:13:22,457 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:13:23,085 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:13:23,085 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:13:24,458 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:13:25,598 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:13:25,599 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:13:25,599 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:13:25,603 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:13:26,086 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:13:26,164 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:13:26,858 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 19:13:27,089 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:13:29,048 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:13:29,049 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:13:29,049 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:13:29,054 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:13:29,090 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:13:30,054 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:13:30,623 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:13:31,093 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:13:32,745 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:13:32,746 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:13:32,746 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:13:32,751 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:13:33,094 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:13:35,096 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:13:35,789 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:13:36,165 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:13:38,310 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:13:38,310 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:13:38,310 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:13:38,315 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:13:39,100 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:13:39,100 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:13:41,166 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:13:41,315 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:13:41,895 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:13:41,896 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:13:41,896 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:13:41,901 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:13:42,102 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:13:43,104 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:13:45,354 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:13:45,354 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:13:45,355 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:13:45,359 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:13:45,623 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:13:46,107 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:13:46,360 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:13:47,110 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:13:48,965 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:13:48,966 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:13:48,966 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:13:48,971 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:13:49,111 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:13:51,114 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:13:51,167 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:13:52,009 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:13:52,910 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:13:52,911 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:13:52,911 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:13:52,916 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:13:53,115 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:13:55,118 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:13:56,167 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:13:56,860 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 19:13:57,861 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:13:59,356 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:13:59,357 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:13:59,357 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:13:59,362 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:14:00,121 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:14:00,623 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:14:01,123 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:14:03,008 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:14:03,009 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:14:03,009 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:14:03,009 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:14:03,014 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:14:03,125 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:14:05,128 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:14:06,168 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:14:06,250 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:14:06,251 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:14:06,251 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:14:06,255 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:14:07,132 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:14:07,132 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:14:08,256 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:14:11,410 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:14:13,256 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:14:15,623 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:14:16,164 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:14:16,165 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:14:16,165 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:14:16,170 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:14:17,139 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:14:17,139 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:14:19,205 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:14:21,411 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:14:21,872 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:14:21,873 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:14:21,873 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:14:21,878 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:14:22,142 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:14:23,145 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:14:24,879 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:14:25,143 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:14:25,143 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:14:25,144 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:14:25,148 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:14:25,149 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:14:26,412 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:14:26,862 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 19:14:27,152 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:14:30,366 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:14:30,367 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:14:30,367 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:14:30,367 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:14:30,372 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:14:30,623 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:14:31,156 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:14:31,156 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:14:33,884 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:14:33,885 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:14:33,885 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:14:33,891 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:14:34,158 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:14:35,160 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:14:35,925 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:14:36,413 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:14:37,995 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:14:37,996 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:14:37,996 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:14:38,001 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:14:38,162 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:14:39,162 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:14:41,001 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:14:41,414 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:14:41,654 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:14:41,655 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:14:41,655 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:14:41,660 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:14:42,165 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:14:43,167 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:14:45,623 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:14:46,490 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:14:46,491 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:14:46,491 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:14:46,492 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:14:46,496 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:14:47,172 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:14:47,173 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:14:50,388 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:14:50,389 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:14:50,389 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:14:50,394 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:14:51,175 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:14:51,175 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:14:51,415 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:14:52,395 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:14:54,348 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:14:54,349 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:14:54,349 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:14:54,354 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:14:55,179 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:14:55,179 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:14:56,415 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:14:56,864 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 19:14:57,780 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:14:57,781 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:14:57,781 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:14:57,781 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:14:57,786 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:14:58,181 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:14:59,184 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:15:00,624 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:15:00,996 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:15:00,996 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:15:00,996 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:15:01,002 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:15:01,185 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:15:03,037 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:15:03,189 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:15:04,309 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:15:04,310 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:15:04,310 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:15:04,315 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:15:05,192 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:15:05,192 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:15:06,416 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:15:08,316 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:15:09,921 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:15:09,922 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:15:09,923 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:15:09,928 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:15:10,195 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:15:11,197 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:15:11,417 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:15:13,928 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:15:15,218 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:15:15,219 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:15:15,219 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:15:15,224 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:15:15,624 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:15:16,200 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:15:17,202 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:15:19,225 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:15:19,784 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:15:19,785 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:15:19,785 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:15:19,790 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:15:20,204 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:15:21,207 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:15:21,418 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:15:24,423 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:15:24,424 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:15:24,424 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:15:24,424 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:15:24,430 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:15:25,211 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:15:25,211 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:15:26,418 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:15:26,866 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 19:15:29,124 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:15:29,125 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:15:29,125 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:15:29,130 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:15:29,213 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:15:30,131 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:15:30,624 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:15:31,216 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:15:32,453 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:15:32,454 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:15:32,454 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:15:32,459 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:15:33,220 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:15:33,220 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:15:35,493 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:15:36,419 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:15:37,004 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:15:37,005 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:15:37,005 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:15:37,010 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:15:37,222 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:15:39,225 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:15:41,011 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:15:41,420 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:15:42,292 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:15:42,293 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:15:42,293 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:15:42,298 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:15:43,229 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:15:43,229 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:15:45,421 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:15:45,421 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:15:45,421 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:15:45,426 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:15:45,624 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:15:46,232 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:15:46,426 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:15:47,235 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:15:48,452 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:15:48,453 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:15:48,453 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:15:48,458 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:15:49,238 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:15:49,238 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:15:51,421 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:15:51,493 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:15:54,386 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:15:54,386 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:15:54,387 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:15:54,391 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:15:55,243 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:15:55,243 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:15:56,421 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:15:56,868 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 19:15:56,870 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:15:58,153 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:15:58,153 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:15:58,154 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:15:58,158 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:15:58,244 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:15:59,247 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:16:00,624 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:16:01,286 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:16:01,287 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:16:01,287 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:16:01,292 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:16:02,249 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:16:02,329 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:16:03,252 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:16:04,140 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:16:04,141 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:16:04,141 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:16:04,146 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:16:04,253 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:16:05,255 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:16:06,423 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:16:08,018 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:16:08,019 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:16:08,019 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:16:08,019 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:16:08,024 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:16:08,257 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:16:09,259 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:16:11,423 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:16:11,846 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:16:11,847 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:16:11,847 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:16:11,852 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:16:12,261 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:16:13,261 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:16:13,853 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:16:15,624 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:16:16,306 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:16:16,307 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:16:16,307 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:16:16,312 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:16:17,266 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:16:17,266 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:16:19,349 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:16:20,742 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:16:20,743 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:16:20,744 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:16:20,748 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:16:21,270 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:16:21,270 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:16:21,424 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:16:24,206 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:16:24,207 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:16:24,207 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:16:24,212 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:16:24,272 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:16:25,212 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:16:25,274 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:16:26,425 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:16:26,870 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 19:16:27,741 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:16:27,742 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:16:27,742 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:16:27,747 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:16:28,276 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:16:29,278 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:16:30,624 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:16:30,748 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:16:31,470 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:16:31,471 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:16:31,471 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:16:31,476 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:16:32,280 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:16:33,282 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:16:35,515 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:16:35,516 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:16:35,516 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:16:35,521 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:16:36,284 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:16:36,426 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:16:36,521 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:16:37,287 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:16:41,426 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:16:41,522 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:16:42,530 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:16:42,531 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:16:42,531 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:16:42,536 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:16:43,292 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:16:43,292 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:16:45,624 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:16:46,537 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:16:46,779 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:16:46,779 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:16:46,780 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:16:46,784 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:16:47,296 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:16:49,299 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:16:51,427 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:16:51,822 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:16:55,258 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:16:55,259 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:16:55,259 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:16:55,264 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:16:55,302 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:16:56,428 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:16:56,872 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 19:16:56,874 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:16:57,305 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:16:59,334 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:16:59,334 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:16:59,335 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:16:59,339 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:17:00,307 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:17:00,624 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:17:01,309 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:17:02,340 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:17:03,614 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:17:03,615 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:17:03,615 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:17:03,620 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:17:04,312 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:17:05,314 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:17:06,429 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:17:07,645 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:17:07,646 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:17:07,646 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:17:07,646 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:17:07,650 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:17:08,316 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:17:09,319 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:17:11,429 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:17:12,651 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:17:13,701 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:17:13,701 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:17:13,702 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:17:13,707 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:17:14,321 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:17:15,324 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:17:15,624 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:17:17,060 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:17:17,060 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:17:17,061 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:17:17,065 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:17:17,325 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:17:18,101 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:17:19,328 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:17:21,181 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:17:21,182 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:17:21,182 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:17:21,187 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:17:21,329 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:17:21,430 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:17:23,188 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:17:23,331 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:17:25,122 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:17:25,123 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:17:25,123 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:17:25,128 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:17:25,332 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:17:26,431 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:17:26,877 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 19:17:27,335 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:17:28,879 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:17:29,267 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:17:29,268 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:17:29,268 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:17:29,273 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:17:29,336 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:17:30,624 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:17:31,339 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:17:32,265 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:17:32,266 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:17:32,266 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:17:32,271 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:17:32,339 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:17:33,342 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:17:34,309 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:17:36,432 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:17:39,274 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:17:39,275 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:17:39,275 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:17:39,280 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:17:39,345 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:17:40,280 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:17:41,348 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:17:41,433 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:17:42,829 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:17:42,830 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:17:42,831 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:17:42,835 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:17:43,352 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:17:43,352 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:17:45,625 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:17:45,836 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:17:46,266 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:17:46,266 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:17:46,266 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:17:46,271 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:17:46,354 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:17:47,357 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:17:49,761 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:17:49,761 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:17:49,761 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:17:49,766 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:17:50,358 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:17:51,361 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:17:51,434 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:17:51,767 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:17:54,478 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:17:54,478 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:17:54,479 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:17:54,483 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:17:55,365 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:17:55,365 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:17:56,434 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:17:56,877 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 19:17:56,878 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:17:58,608 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:17:58,609 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:17:58,609 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:17:58,613 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:17:59,369 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:17:59,369 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:18:00,625 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:18:02,041 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:18:02,041 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:18:02,041 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:18:02,041 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:18:02,046 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:18:02,371 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:18:03,374 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:18:05,466 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:18:05,467 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:18:05,467 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:18:05,472 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:18:06,376 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:18:06,435 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:18:07,377 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:18:07,472 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:18:09,008 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:18:09,009 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:18:09,009 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:18:09,014 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:18:09,378 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:18:11,381 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:18:11,436 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:18:13,014 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:18:15,625 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:18:17,423 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:18:17,424 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:18:17,424 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:18:17,429 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:18:18,385 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:18:18,465 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:18:19,388 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:18:21,437 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:18:22,789 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:18:22,790 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:18:22,790 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:18:22,795 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:18:23,392 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:18:23,392 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:18:23,796 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:18:26,438 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:18:26,882 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 19:18:27,886 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:18:27,887 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:18:27,887 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:18:27,896 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:18:28,395 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:18:28,896 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:18:29,397 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:18:30,625 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:18:31,228 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:18:31,228 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:18:31,229 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:18:31,234 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:18:31,399 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:18:33,402 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:18:34,269 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:18:35,385 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:18:35,385 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:18:35,386 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:18:35,390 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:18:35,403 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:18:36,439 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:18:37,406 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:18:39,391 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:18:41,439 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:18:43,671 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:18:43,672 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:18:43,672 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:18:43,677 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:18:44,410 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:18:44,678 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:18:45,413 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:18:45,625 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:18:47,765 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:18:47,766 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:18:47,766 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:18:47,771 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:18:48,415 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:18:49,418 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:18:49,809 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:18:51,440 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:18:52,864 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:18:52,864 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:18:52,865 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:18:52,870 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:18:53,422 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:18:53,422 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:18:54,871 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:18:56,441 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:18:56,881 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 19:18:57,902 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:18:57,903 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:18:57,903 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:18:57,908 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:18:58,425 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:18:59,427 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:18:59,908 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:19:00,625 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:19:04,909 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:19:06,304 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:19:06,305 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:19:06,305 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:19:06,310 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:19:06,433 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:19:06,442 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:19:07,435 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:19:10,346 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:19:11,443 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:19:14,753 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:19:14,754 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:19:14,754 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:19:14,758 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:19:15,441 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:19:15,442 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:19:15,625 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:19:15,759 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:19:18,601 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:19:18,602 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:19:18,602 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:19:18,607 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:19:19,446 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:19:19,446 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:19:21,444 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:19:21,645 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:19:21,799 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:19:21,800 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:19:21,800 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:19:21,805 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:19:22,448 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:19:23,450 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:19:25,169 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:19:25,170 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:19:25,170 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:19:25,175 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:19:25,451 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:19:26,444 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:19:26,886 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 19:19:26,887 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:19:27,454 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:19:28,858 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:19:28,858 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:19:28,859 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:19:28,863 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:19:29,457 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:19:29,457 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:19:30,625 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:19:32,864 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:19:33,878 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:19:33,879 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:19:33,879 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:19:33,884 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:19:34,460 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:19:35,462 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:19:36,445 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:19:37,914 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:19:37,915 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:19:37,915 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:19:37,915 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:19:37,920 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:19:38,464 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:19:39,466 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:19:41,446 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:19:42,921 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:19:45,625 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:19:45,674 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:19:45,674 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:19:45,674 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:19:45,679 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:19:46,472 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:19:47,474 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:19:48,717 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:19:48,965 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:19:48,966 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:19:48,966 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:19:48,971 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:19:49,477 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:19:49,477 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:19:51,447 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:19:52,518 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:19:52,518 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:19:52,519 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:19:52,523 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:19:53,481 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:19:53,481 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:19:54,524 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:19:56,112 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:19:56,113 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:19:56,113 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:19:56,118 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:19:56,447 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:19:56,483 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:19:56,885 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 19:19:57,485 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:19:59,721 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:19:59,722 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:19:59,722 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:19:59,722 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:19:59,727 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:20:00,487 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:20:00,625 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:20:01,489 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:20:04,727 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:20:06,037 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:20:06,037 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:20:06,037 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:20:06,042 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:20:06,448 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:20:06,493 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:20:07,495 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:20:09,490 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:20:09,490 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:20:09,490 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:20:09,495 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:20:09,497 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:20:10,495 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:20:11,449 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:20:11,500 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:20:15,496 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:20:15,625 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:20:17,267 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:20:17,268 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:20:17,268 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:20:17,273 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:20:17,505 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:20:19,508 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:20:21,310 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:20:21,450 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:20:21,821 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:20:21,822 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:20:21,822 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:20:21,827 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:20:22,510 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:20:23,512 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:20:25,310 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:20:25,311 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:20:25,311 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:20:25,316 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:20:25,513 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:20:26,316 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:20:26,450 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:20:26,891 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 19:20:27,516 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:20:28,817 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:20:28,818 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:20:28,818 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:20:28,823 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:20:29,519 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:20:29,519 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:20:30,625 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:20:31,823 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:20:31,907 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:20:31,908 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:20:31,908 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:20:31,912 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:20:32,521 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:20:33,523 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:20:36,451 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:20:36,950 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:20:37,234 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:20:37,235 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:20:37,235 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:20:37,240 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:20:37,525 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:20:39,529 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:20:41,452 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:20:42,240 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:20:45,625 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:20:45,732 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:20:45,732 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:20:45,732 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:20:45,737 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:20:46,534 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:20:47,535 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:20:47,773 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:20:51,453 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:20:52,774 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:20:53,615 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:20:53,616 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:20:53,616 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:20:53,621 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:20:54,538 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:20:55,541 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:20:56,454 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:20:56,889 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 19:20:57,150 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:20:57,151 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:20:57,151 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:20:57,156 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:20:57,542 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:20:58,157 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:20:59,545 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:21:00,626 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:21:01,641 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:21:01,642 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:21:01,642 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:21:01,647 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:21:02,547 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:21:03,549 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:21:03,685 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:21:06,045 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:21:06,046 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:21:06,046 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:21:06,051 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:21:06,455 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:21:06,550 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:21:07,553 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:21:09,052 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:21:09,718 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:21:09,719 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:21:09,719 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:21:09,725 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:21:10,555 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:21:11,455 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:21:11,557 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:21:13,415 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:21:13,416 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:21:13,416 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:21:13,421 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:21:13,558 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:21:14,421 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:21:15,561 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:21:15,626 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:21:18,720 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:21:18,720 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:21:18,720 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:21:18,725 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:21:19,564 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:21:19,564 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:21:19,761 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:21:21,457 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:21:22,474 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:21:22,474 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:21:22,475 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:21:22,479 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:21:22,565 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:21:23,568 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:21:25,480 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:21:25,913 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:21:25,914 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:21:25,914 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:21:25,919 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:21:26,457 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:21:26,570 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:21:26,892 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 19:21:27,572 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:21:29,566 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:21:29,566 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:21:29,567 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:21:29,571 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:21:29,573 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:21:30,572 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:21:30,626 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:21:31,576 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:21:33,925 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:21:33,926 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:21:33,926 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:21:33,931 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:21:34,578 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:21:35,581 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:21:35,969 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:21:36,458 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:21:37,978 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:21:37,979 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:21:37,979 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:21:37,984 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:21:38,582 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:21:39,585 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:21:40,985 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:21:41,459 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:21:43,716 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:21:43,720 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:21:43,720 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:21:43,725 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:21:44,588 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:21:45,590 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:21:45,626 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:21:46,726 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:21:47,334 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:21:47,334 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:21:47,335 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:21:47,339 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:21:47,593 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:21:49,595 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:21:51,460 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:21:52,378 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:21:55,692 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:21:55,693 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:21:55,693 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:21:55,698 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:21:56,461 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:21:56,599 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:21:56,894 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 19:21:57,602 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:21:57,895 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:21:59,243 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:21:59,244 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:21:59,244 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:21:59,249 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:21:59,603 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:22:00,626 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:22:01,606 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:22:02,410 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:22:02,411 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:22:02,411 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:22:02,416 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:22:02,607 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:22:03,453 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:22:03,611 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:22:05,535 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:22:05,536 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:22:05,536 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:22:05,541 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:22:05,612 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:22:06,462 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:22:07,615 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:22:08,542 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:22:09,062 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:22:09,063 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:22:09,063 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:22:09,068 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:22:09,618 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:22:09,618 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:22:11,462 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:22:14,068 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:22:14,224 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:22:14,225 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:22:14,225 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:22:14,230 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:22:14,621 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:22:15,623 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:22:15,626 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:22:18,200 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:22:18,201 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:22:18,201 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:22:18,206 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:22:18,625 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:22:19,241 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:22:19,628 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:22:21,463 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:22:23,476 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:22:23,477 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:22:23,477 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:22:23,482 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:22:23,630 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:22:24,482 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:22:25,633 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:22:26,464 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:22:26,896 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 19:22:27,001 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:22:27,002 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:22:27,002 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:22:27,007 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:22:27,636 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:22:27,636 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:22:30,007 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:22:30,626 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:22:31,378 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:22:31,378 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:22:31,379 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:22:31,383 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:22:31,638 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:22:33,641 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:22:34,647 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:22:34,647 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:22:34,647 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:22:34,652 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:22:35,642 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:22:35,642 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:22:35,653 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:22:36,465 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:22:39,537 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:22:39,538 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:22:39,538 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:22:39,543 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:22:39,644 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:22:41,466 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:22:41,543 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:22:41,647 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:22:44,796 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:22:44,797 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:22:44,797 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:22:44,802 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:22:45,626 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:22:45,652 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:22:45,652 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:22:46,802 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:22:51,466 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:22:51,803 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:22:53,368 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:22:53,369 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:22:53,369 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:22:53,375 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:22:53,657 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:22:55,661 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:22:56,467 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:22:56,570 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:22:56,571 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:22:56,571 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:22:56,576 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:22:56,661 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:22:56,898 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 19:22:56,899 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:22:57,664 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:23:00,626 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:23:01,262 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:23:01,263 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:23:01,263 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:23:01,268 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:23:01,666 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:23:02,305 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:23:03,671 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:23:05,221 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:23:05,222 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:23:05,222 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:23:05,227 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:23:05,674 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:23:05,674 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:23:06,468 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:23:08,227 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:23:08,720 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:23:08,721 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:23:08,721 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:23:08,726 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:23:09,678 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:23:09,678 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:23:11,469 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:23:11,965 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:23:11,966 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:23:11,966 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:23:11,971 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:23:12,680 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:23:13,682 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:23:13,971 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:23:15,626 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:23:17,148 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:23:17,149 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:23:17,149 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:23:17,154 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:23:17,686 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:23:17,686 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:23:19,189 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:23:21,320 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:23:21,321 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:23:21,321 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:23:21,326 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:23:21,470 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:23:21,689 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:23:23,692 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:23:24,326 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:23:25,079 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:23:25,079 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:23:25,080 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:23:25,084 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:23:25,695 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:23:25,695 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:23:26,471 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:23:26,900 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 19:23:29,035 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:23:29,036 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:23:29,036 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:23:29,041 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:23:29,699 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:23:29,700 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:23:30,041 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:23:30,626 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:23:33,145 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:23:33,146 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:23:33,146 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:23:33,151 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:23:33,703 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:23:33,704 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:23:35,189 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:23:36,472 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:23:39,059 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:23:39,059 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:23:39,060 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:23:39,065 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:23:39,709 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:23:39,709 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:23:41,065 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:23:41,472 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:23:42,569 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:23:42,570 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:23:42,570 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:23:42,575 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:23:42,711 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:23:43,714 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:23:45,626 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:23:46,576 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:23:47,259 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:23:47,260 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:23:47,260 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:23:47,264 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:23:47,717 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:23:49,720 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:23:51,473 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:23:52,302 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:23:55,132 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:23:55,133 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:23:55,133 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:23:55,138 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:23:55,725 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:23:55,726 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:23:56,474 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:23:56,906 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 19:23:57,907 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:24:00,627 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:24:02,754 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:24:02,755 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:24:02,755 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:24:02,760 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:24:03,733 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:24:03,733 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:24:03,797 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:24:06,377 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:24:06,378 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:24:06,378 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:24:06,383 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:24:06,475 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:24:06,735 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:24:07,737 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:24:09,384 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:24:11,475 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:24:11,594 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:24:11,595 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:24:11,595 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:24:11,600 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:24:11,740 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:24:13,743 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:24:14,600 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:24:15,627 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:24:17,119 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:24:17,120 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:24:17,120 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:24:17,124 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:24:17,747 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:24:17,747 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:24:20,161 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:24:20,675 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:24:20,676 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:24:20,676 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:24:20,681 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:24:20,749 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:24:21,476 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:24:21,751 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:24:24,335 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:24:24,336 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:24:24,336 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:24:24,341 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:24:24,753 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:24:25,341 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:24:25,756 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:24:26,477 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:24:26,905 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 19:24:30,080 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:24:30,081 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:24:30,081 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:24:30,086 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:24:30,627 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:24:30,759 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:24:31,086 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:24:31,761 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:24:33,998 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:24:33,998 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:24:33,998 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:24:34,003 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:24:34,763 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:24:35,766 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:24:36,478 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:24:37,041 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:24:37,564 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:24:37,565 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:24:37,565 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:24:37,570 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:24:37,768 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:24:39,771 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:24:41,737 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:24:42,417 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:24:42,418 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:24:42,418 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:24:42,418 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:24:42,423 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:24:42,773 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:24:43,775 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:24:45,627 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:24:45,747 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:24:45,747 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:24:45,748 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:24:45,752 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:24:45,776 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:24:47,781 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:24:47,789 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:24:50,738 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:24:50,950 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:24:50,951 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:24:50,951 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:24:50,956 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:24:51,785 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:24:51,785 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:24:52,957 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:24:55,738 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:24:56,910 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 19:24:57,616 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:24:57,617 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:24:57,617 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:24:57,622 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:24:57,788 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:24:58,622 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:24:59,791 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:25:00,627 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:25:01,189 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:25:01,190 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:25:01,190 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:25:01,195 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:25:01,792 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:25:03,795 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:25:04,233 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:25:05,739 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:25:06,093 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:25:06,094 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:25:06,094 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:25:06,099 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:25:06,796 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:25:07,799 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:25:09,702 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:25:09,703 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:25:09,703 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:25:09,703 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:25:09,708 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:25:09,800 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:25:10,740 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:25:11,803 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:25:13,715 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:25:13,716 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:25:13,716 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:25:13,722 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:25:13,805 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:25:14,722 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:25:15,627 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:25:15,808 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:25:16,993 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:25:16,993 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:25:16,993 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:25:16,998 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:25:17,809 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:25:19,812 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:25:20,034 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:25:20,741 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:25:21,700 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:25:21,701 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:25:21,701 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:25:21,706 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:25:21,813 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:25:23,816 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:25:25,423 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:25:25,424 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:25:25,424 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:25:25,424 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:25:25,429 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:25:25,742 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:25:25,817 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:25:26,910 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 19:25:27,821 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:25:29,039 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:25:29,039 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:25:29,039 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:25:29,044 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:25:29,822 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:25:30,627 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:25:31,045 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:25:31,825 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:25:32,803 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:25:32,804 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:25:32,804 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:25:32,808 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:25:32,825 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:25:33,828 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:25:35,743 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:25:36,846 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:25:39,467 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:25:39,468 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:25:39,468 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:25:39,473 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:25:39,831 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:25:40,744 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:25:41,834 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:25:42,473 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:25:43,197 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:25:43,198 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:25:43,198 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:25:43,203 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:25:43,836 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:25:45,627 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:25:45,839 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:25:48,204 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:25:49,199 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:25:49,199 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:25:49,199 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:25:49,204 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:25:49,842 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:25:50,745 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:25:51,845 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:25:53,243 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:25:53,880 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:25:53,881 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:25:53,881 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:25:53,886 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:25:54,847 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:25:55,745 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:25:55,849 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:25:56,915 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 19:25:58,609 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:25:58,610 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:25:58,610 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:25:58,610 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:25:58,615 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:25:58,851 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:25:59,854 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:26:00,627 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:26:03,615 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:26:05,746 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:26:06,874 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:26:06,875 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:26:06,875 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:26:06,880 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:26:07,861 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:26:07,862 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:26:08,881 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:26:10,747 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:26:10,978 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:26:10,979 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:26:10,979 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:26:10,984 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:26:11,864 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:26:13,867 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:26:13,984 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:26:14,286 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:26:14,286 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:26:14,287 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:26:14,292 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:26:14,868 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:26:15,627 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:26:15,870 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:26:18,370 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:26:18,370 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:26:18,371 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:26:18,376 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:26:18,872 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:26:19,414 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:26:19,875 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:26:20,748 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:26:22,968 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:26:22,969 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:26:22,969 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:26:22,974 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:26:23,877 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:26:23,877 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:26:24,974 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:26:25,748 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:26:26,400 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:26:26,401 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:26:26,401 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:26:26,406 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:26:26,879 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:26:26,915 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 19:26:27,882 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:26:29,638 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:26:29,639 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:26:29,639 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:26:29,645 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:26:29,883 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:26:30,627 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:26:30,645 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:26:31,886 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:26:33,221 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:26:33,221 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:26:33,222 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:26:33,226 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:26:33,887 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:26:35,749 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:26:35,890 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:26:36,263 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:26:38,041 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:26:38,042 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:26:38,042 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:26:38,047 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:26:38,892 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:26:39,895 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:26:40,750 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:26:41,534 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:26:41,535 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:26:41,535 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:26:41,535 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:26:41,540 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:26:41,896 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:26:43,899 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:26:45,226 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:26:45,226 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:26:45,227 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:26:45,231 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:26:45,628 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:26:45,901 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:26:47,232 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:26:47,905 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:26:50,751 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:26:51,021 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:26:51,022 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:26:51,022 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:26:51,027 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:26:51,907 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:26:53,027 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:26:53,910 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:26:55,408 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:26:55,409 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:26:55,409 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:26:55,414 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:26:55,751 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:26:55,911 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:26:56,920 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 19:26:57,914 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:26:58,921 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:26:59,278 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:26:59,278 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:26:59,279 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:26:59,284 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:26:59,916 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:27:00,628 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:27:01,919 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:27:03,634 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:27:03,635 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:27:03,635 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:27:03,640 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:27:03,921 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:27:04,678 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:27:05,752 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:27:05,924 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:27:08,359 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:27:08,360 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:27:08,360 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:27:08,364 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:27:08,926 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:27:09,929 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:27:10,365 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:27:10,753 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:27:13,323 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:27:13,324 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:27:13,324 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:27:13,329 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:27:13,931 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:27:15,628 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:27:15,934 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:27:16,329 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:27:18,932 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:27:18,933 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:27:18,933 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:27:18,938 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:27:18,938 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:27:19,941 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:27:20,754 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:27:21,974 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:27:25,755 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:27:26,919 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 19:27:27,499 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:27:27,500 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:27:27,500 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:27:27,500 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:27:27,505 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:27:27,946 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:27:29,949 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:27:30,628 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:27:31,046 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:27:31,047 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:27:31,047 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:27:31,052 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:27:31,950 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:27:33,090 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:27:33,953 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:27:34,342 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:27:34,343 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:27:34,343 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:27:34,348 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:27:34,954 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:27:35,756 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:27:35,957 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:27:38,349 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:27:40,756 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:27:40,869 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:27:40,870 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:27:40,870 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:27:40,875 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:27:40,960 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:27:41,963 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:27:43,876 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:27:44,516 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:27:44,517 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:27:44,517 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:27:44,522 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:27:44,965 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:27:45,628 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:27:45,967 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:27:48,167 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:27:48,167 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:27:48,168 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:27:48,172 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:27:48,970 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:27:49,210 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:27:49,973 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:27:50,757 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:27:51,868 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:27:51,869 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:27:51,869 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:27:51,874 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:27:51,974 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:27:53,977 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:27:54,874 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:27:54,958 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:27:54,959 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:27:54,959 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:27:54,964 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:27:54,978 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:27:55,758 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:27:55,980 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:27:56,924 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 19:27:58,693 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:27:58,694 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:27:58,694 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:27:58,699 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:27:58,982 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:27:59,984 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:28:00,628 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:28:00,699 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:28:02,052 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:28:02,052 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:28:02,053 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:28:02,057 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:28:02,986 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:28:03,990 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:28:05,759 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:28:06,095 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:28:07,022 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:28:07,023 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:28:07,023 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:28:07,028 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:28:07,994 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:28:07,994 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:28:10,760 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:28:11,765 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:28:11,766 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:28:11,766 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:28:11,766 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:28:11,771 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:28:11,996 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:28:13,999 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:28:15,628 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:28:15,709 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:28:15,709 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:28:15,710 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:28:15,714 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:28:16,000 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:28:17,750 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:28:18,003 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:28:19,485 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:28:19,486 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:28:19,487 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:28:19,492 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:28:20,005 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:28:20,761 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:28:22,008 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:28:23,250 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:28:23,250 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:28:23,251 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:28:23,251 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:28:23,255 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:28:24,009 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:28:25,761 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:28:26,012 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:28:26,719 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:28:26,720 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:28:26,720 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:28:26,725 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:28:26,923 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 19:28:27,013 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:28:28,015 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:28:28,924 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:28:30,628 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:28:31,369 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:28:31,369 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:28:31,370 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:28:31,375 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:28:32,017 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:28:34,021 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:28:34,390 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:28:34,391 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:28:34,391 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:28:34,391 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:28:34,396 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:28:35,021 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:28:35,763 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:28:36,024 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:28:39,397 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:28:40,763 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:28:42,990 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:28:42,991 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:28:42,991 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:28:42,996 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:28:43,027 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:28:44,030 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:28:44,996 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:28:45,629 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:28:46,720 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:28:46,721 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:28:46,721 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:28:46,726 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:28:47,033 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:28:48,036 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:28:49,964 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:28:49,965 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:28:49,965 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:28:49,970 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:28:50,037 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:28:50,765 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:28:50,970 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:28:52,038 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:28:54,923 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:28:54,924 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:28:54,924 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:28:54,929 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:28:55,039 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:28:55,765 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:28:56,042 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:28:56,925 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 19:28:56,926 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:28:58,475 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:28:58,475 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:28:58,475 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:28:58,481 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:28:59,044 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:29:00,047 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:29:00,628 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:29:02,480 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:29:02,480 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:29:02,480 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:29:02,481 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:29:02,485 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:29:03,048 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:29:04,052 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:29:05,766 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:29:06,728 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:29:06,729 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:29:06,729 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:29:06,734 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:29:07,054 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:29:07,734 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:29:08,056 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:29:10,767 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:29:10,768 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:29:10,769 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:29:10,769 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:29:10,774 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:29:11,058 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:29:12,060 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:29:12,774 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:29:15,628 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:29:16,225 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:29:16,226 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:29:16,226 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:29:16,231 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:29:17,063 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:29:18,065 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:29:18,270 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:29:19,718 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:29:19,719 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:29:19,719 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:29:19,724 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:29:20,067 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:29:20,768 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:29:22,070 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:29:23,725 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:29:24,642 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:29:24,642 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:29:24,643 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:29:24,648 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:29:25,072 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:29:25,769 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:29:26,074 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:29:26,927 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 19:29:28,151 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:29:28,152 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:29:28,152 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:29:28,157 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:29:29,076 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:29:29,158 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:29:30,078 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:29:30,628 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:29:31,517 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:29:31,518 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:29:31,518 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:29:31,523 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:29:32,080 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:29:34,083 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:29:34,562 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:29:35,675 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:29:35,676 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:29:35,676 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:29:35,681 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:29:36,084 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:29:36,188 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:29:38,087 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:29:39,682 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:29:40,712 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:29:40,713 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:29:40,713 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:29:40,718 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:29:41,089 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:29:41,189 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:29:42,091 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:29:44,719 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:29:45,628 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:29:46,109 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:29:46,109 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:29:46,110 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:29:46,114 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:29:47,097 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:29:48,100 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:29:50,120 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:29:50,121 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:29:50,121 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:29:50,122 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:29:50,127 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:29:51,101 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:29:51,190 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:29:52,104 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:29:55,127 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:29:56,069 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:29:56,070 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:29:56,070 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:29:56,075 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:29:56,106 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:29:56,191 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:29:56,930 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 19:29:58,109 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:30:00,629 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:30:00,858 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:30:00,859 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:30:00,859 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:30:00,859 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:30:00,864 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:30:01,111 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:30:02,113 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:30:05,901 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:30:05,902 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:30:05,902 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:30:05,902 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:30:05,907 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:30:06,117 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:30:06,192 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:30:08,120 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:30:09,568 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:30:09,569 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:30:09,569 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:30:09,575 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:30:10,122 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:30:11,192 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:30:11,575 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:30:12,125 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:30:12,923 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:30:12,924 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:30:12,924 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:30:12,929 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:30:13,125 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:30:14,128 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:30:15,629 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:30:16,116 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:30:16,117 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:30:16,117 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:30:16,122 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:30:16,129 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:30:17,158 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:30:18,131 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:30:21,193 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:30:22,159 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:30:24,500 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:30:24,501 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:30:24,501 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:30:24,506 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:30:25,135 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:30:26,137 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:30:26,194 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:30:26,932 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 19:30:27,754 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:30:27,754 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:30:27,755 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:30:27,755 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:30:27,760 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:30:28,139 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:30:30,142 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:30:30,629 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:30:31,659 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:30:31,660 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:30:31,660 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:30:31,665 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:30:32,143 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:30:33,702 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:30:34,146 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:30:36,195 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:30:37,014 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:30:37,014 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:30:37,015 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:30:37,020 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:30:37,148 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:30:38,150 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:30:39,020 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:30:41,196 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:30:41,392 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:30:41,393 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:30:41,393 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:30:41,398 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:30:42,153 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:30:44,156 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:30:44,399 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:30:45,629 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:30:47,238 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:30:47,239 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:30:47,239 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:30:47,243 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:30:48,162 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:30:48,162 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:30:50,282 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:30:51,197 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:30:51,790 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:30:51,791 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:30:51,791 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:30:51,796 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:30:52,165 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:30:54,168 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:30:55,796 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:30:56,197 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:30:56,934 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 19:30:58,727 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:30:58,728 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:30:58,728 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:30:58,733 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:30:59,171 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:31:00,174 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:31:00,629 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:31:01,733 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:31:02,363 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:31:02,364 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:31:02,364 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:31:02,369 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:31:03,175 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:31:04,179 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:31:06,179 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:31:06,180 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:31:06,180 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:31:06,185 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:31:06,198 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:31:07,181 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:31:07,185 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:31:08,183 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:31:10,087 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:31:10,088 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:31:10,088 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:31:10,094 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:31:10,185 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:31:11,199 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:31:12,188 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:31:13,095 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:31:15,629 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:31:18,095 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:31:18,685 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:31:18,685 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:31:18,686 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:31:18,691 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:31:19,192 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:31:20,195 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:31:21,200 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:31:23,338 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:31:23,338 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:31:23,339 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:31:23,339 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:31:23,343 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:31:24,197 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:31:26,200 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:31:26,201 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:31:26,794 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:31:26,794 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:31:26,795 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:31:26,799 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:31:26,937 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 19:31:27,201 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:31:28,203 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:31:28,938 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:31:30,629 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:31:31,480 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:31:31,481 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:31:31,481 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:31:31,486 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:31:32,206 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:31:34,207 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:31:34,523 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:31:35,404 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:31:35,405 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:31:35,405 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:31:35,412 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:31:36,202 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:31:36,208 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:31:38,213 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:31:38,938 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:31:38,939 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:31:38,939 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:31:38,944 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:31:39,213 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:31:39,944 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:31:40,216 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:31:41,202 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:31:42,712 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:31:42,713 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:31:42,713 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:31:42,718 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:31:43,218 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:31:44,220 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:31:45,629 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:31:45,718 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:31:46,268 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:31:46,269 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:31:46,269 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:31:46,274 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:31:47,223 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:31:48,226 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:31:49,798 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:31:49,798 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:31:49,799 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:31:49,804 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:31:50,227 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:31:50,804 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:31:51,203 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:31:52,230 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:31:53,140 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:31:53,141 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:31:53,141 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:31:53,146 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:31:53,231 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:31:54,233 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:31:56,147 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:31:56,204 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:31:56,661 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:31:56,662 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:31:56,662 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:31:56,667 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:31:56,939 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 19:31:57,235 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:31:58,238 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:32:00,128 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:32:00,129 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:32:00,129 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:32:00,134 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:32:00,239 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:32:00,629 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:32:02,135 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:32:02,242 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:32:06,205 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:32:07,135 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:32:09,004 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:32:09,005 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:32:09,005 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:32:09,011 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:32:09,247 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:32:10,250 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:32:11,206 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:32:12,295 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:32:12,296 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:32:12,296 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:32:12,296 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:32:12,301 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:32:13,251 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:32:14,254 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:32:15,629 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:32:16,395 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:32:16,396 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:32:16,396 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:32:16,400 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:32:17,256 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:32:17,438 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:32:18,258 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:32:20,133 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:32:20,134 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:32:20,134 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:32:20,139 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:32:20,259 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:32:21,207 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:32:22,262 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:32:23,140 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:32:23,597 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:32:23,597 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:32:23,598 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:32:23,602 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:32:24,264 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:32:26,207 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:32:26,267 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:32:26,941 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 19:32:27,563 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:32:27,564 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:32:27,564 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:32:27,569 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:32:28,268 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:32:28,569 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:32:30,271 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:32:30,598 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:32:30,599 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:32:30,599 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:32:30,604 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:32:30,629 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:32:31,272 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:32:32,275 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:32:33,604 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:32:35,346 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:32:35,346 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:32:35,346 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:32:35,351 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:32:36,208 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:32:36,279 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:32:36,279 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:32:39,390 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:32:41,209 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:32:43,778 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:32:43,779 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:32:43,779 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:32:43,784 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:32:44,284 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:32:44,785 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:32:45,630 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:32:46,287 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:32:49,510 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:32:49,510 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:32:49,511 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:32:49,515 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:32:50,290 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:32:50,554 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:32:51,210 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:32:52,293 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:32:53,373 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:32:53,374 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:32:53,374 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:32:53,379 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:32:54,296 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:32:54,297 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:32:56,211 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:32:56,380 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:32:56,710 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:32:56,710 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:32:56,711 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:32:56,716 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:32:56,943 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 19:32:57,298 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:32:58,301 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:33:00,630 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:33:01,945 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:33:02,229 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:33:02,229 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:33:02,229 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:33:02,235 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:33:02,303 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:33:04,307 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:33:05,766 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:33:05,767 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:33:05,767 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:33:05,772 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:33:06,212 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:33:06,309 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:33:07,773 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:33:08,312 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:33:08,776 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:33:08,777 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:33:08,777 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:33:08,782 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:33:09,312 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:33:10,315 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:33:11,213 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:33:12,783 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:33:15,630 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:33:17,116 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:33:17,117 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:33:17,117 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:33:17,122 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:33:17,318 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:33:18,158 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:33:18,321 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:33:20,143 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:33:20,144 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:33:20,144 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:33:20,149 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:33:20,322 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:33:21,214 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:33:22,325 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:33:24,150 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:33:25,056 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:33:25,056 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:33:25,057 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:33:25,062 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:33:25,327 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:33:26,214 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:33:26,328 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:33:26,948 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 19:33:29,949 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:33:30,630 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:33:33,465 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:33:33,466 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:33:33,466 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:33:33,471 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:33:34,332 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:33:35,510 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:33:36,215 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:33:36,335 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:33:38,880 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:33:38,881 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:33:38,881 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:33:38,886 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:33:39,337 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:33:40,340 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:33:40,887 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:33:41,216 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:33:42,365 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:33:42,366 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:33:42,366 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:33:42,371 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:33:43,342 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:33:44,342 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:33:45,630 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:33:46,372 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:33:46,451 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:33:46,451 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:33:46,452 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:33:46,456 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:33:47,345 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:33:48,348 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:33:51,217 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:33:51,495 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:33:53,815 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:33:53,816 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:33:53,816 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:33:53,821 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:33:54,351 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:33:56,217 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:33:56,354 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:33:56,822 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:33:56,947 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 19:34:00,630 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:34:01,948 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:34:02,082 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:34:02,082 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:34:02,082 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:34:02,088 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:34:02,358 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:34:04,362 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:34:06,218 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:34:07,126 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:34:07,472 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:34:07,473 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:34:07,473 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:34:07,478 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:34:08,366 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:34:08,366 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:34:10,912 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:34:10,912 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:34:10,913 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:34:10,918 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:34:11,219 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:34:11,368 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:34:12,370 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:34:12,918 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:34:15,630 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:34:16,858 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:34:16,858 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:34:16,859 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:34:16,864 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:34:17,373 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:34:18,374 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:34:18,902 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:34:19,987 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:34:19,987 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:34:19,987 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:34:19,992 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:34:20,375 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:34:21,220 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:34:22,378 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:34:23,055 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:34:23,056 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:34:23,056 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:34:23,061 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:34:23,379 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:34:24,061 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:34:24,382 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:34:26,221 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:34:26,953 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 19:34:28,209 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:34:28,210 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:34:28,210 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:34:28,215 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:34:28,384 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:34:29,215 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:34:30,389 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:34:30,630 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:34:33,674 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:34:33,674 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:34:33,674 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:34:33,680 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:34:34,394 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:34:34,717 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:34:36,222 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:34:36,398 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:34:39,718 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:34:40,046 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:34:40,047 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:34:40,047 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:34:40,052 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:34:40,402 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:34:41,222 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:34:42,407 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:34:44,020 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:34:44,021 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:34:44,021 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:34:44,026 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:34:44,409 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:34:45,026 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:34:45,630 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:34:46,413 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:34:50,027 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:34:51,223 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:34:52,466 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:34:52,467 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:34:52,467 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:34:52,472 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:34:53,419 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:34:54,423 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:34:55,473 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:34:56,237 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:34:56,951 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 19:35:00,630 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:35:00,953 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:35:01,877 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:35:01,878 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:35:01,878 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:35:01,883 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:35:02,426 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:35:04,430 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:35:06,238 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:35:06,570 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:35:06,571 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:35:06,571 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:35:06,571 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:35:06,576 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:35:07,432 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:35:08,434 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:35:10,431 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:35:10,431 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:35:10,432 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:35:10,436 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:35:10,437 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:35:11,238 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:35:12,437 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:35:12,440 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:35:14,974 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:35:14,975 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:35:14,976 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:35:14,981 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:35:15,442 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:35:15,630 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:35:16,444 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:35:17,981 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:35:19,033 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:35:19,033 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:35:19,033 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:35:19,038 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:35:19,446 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:35:20,447 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:35:21,240 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:35:23,074 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:35:24,377 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:35:24,377 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:35:24,378 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:35:24,383 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:35:24,449 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:35:26,240 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:35:26,452 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:35:26,954 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 19:35:27,542 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:35:27,543 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:35:27,543 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:35:27,547 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:35:28,455 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:35:28,456 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:35:28,548 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:35:30,630 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:35:33,548 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:35:35,052 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:35:35,053 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:35:35,053 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:35:35,059 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:35:35,459 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:35:36,241 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:35:36,461 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:35:39,094 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:35:39,980 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:35:39,980 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:35:39,981 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:35:39,985 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:35:40,464 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:35:41,242 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:35:42,467 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:35:43,772 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:35:43,773 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:35:43,773 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:35:43,778 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:35:44,468 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:35:44,778 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:35:45,630 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:35:46,471 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:35:46,828 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:35:46,829 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:35:46,829 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:35:46,834 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:35:47,474 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:35:48,476 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:35:49,869 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:35:50,096 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:35:50,097 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:35:50,097 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:35:50,102 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:35:50,477 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:35:51,243 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:35:52,481 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:35:55,102 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:35:55,960 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:35:55,961 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:35:55,961 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:35:55,966 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:35:56,244 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:35:56,483 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:35:56,956 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 19:35:58,486 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:36:00,630 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:36:00,957 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:36:01,691 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:36:01,691 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:36:01,692 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:36:01,696 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:36:02,488 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:36:04,492 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:36:05,427 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:36:05,428 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:36:05,428 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:36:05,433 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:36:05,493 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:36:06,245 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:36:06,433 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:36:06,495 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:36:10,842 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:36:10,843 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:36:10,843 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:36:10,848 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:36:11,246 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:36:11,498 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:36:11,849 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:36:12,501 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:36:15,601 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:36:15,601 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:36:15,601 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:36:15,606 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:36:15,631 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:36:16,503 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:36:16,503 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:36:17,607 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:36:21,247 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:36:21,745 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:36:21,746 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:36:21,746 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:36:21,751 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:36:22,507 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:36:22,751 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:36:24,510 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:36:25,978 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:36:25,979 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:36:25,979 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:36:25,984 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:36:26,247 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:36:26,511 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:36:26,958 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 19:36:27,959 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:36:28,514 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:36:29,918 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:36:29,919 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:36:29,919 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:36:29,924 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:36:30,515 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:36:30,631 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:36:32,518 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:36:33,925 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:36:36,248 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:36:38,387 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:36:38,388 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:36:38,388 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:36:38,393 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:36:38,522 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:36:39,394 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:36:40,532 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:36:41,249 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:36:43,301 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:36:43,301 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:36:43,302 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:36:43,306 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:36:43,535 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:36:44,537 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:36:45,307 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:36:45,631 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:36:47,826 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:36:47,827 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:36:47,827 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:36:47,832 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:36:48,540 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:36:50,542 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:36:50,869 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:36:51,250 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:36:52,079 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:36:52,080 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:36:52,081 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:36:52,085 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:36:52,543 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:36:54,546 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:36:56,086 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:36:56,251 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:36:56,636 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:36:56,636 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:36:56,637 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:36:56,642 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:36:56,960 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 19:36:57,547 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:36:58,550 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:37:00,631 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:37:00,738 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:37:00,738 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:37:00,739 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:37:00,743 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:37:01,551 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:37:01,781 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:37:02,554 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:37:04,066 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:37:04,067 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:37:04,067 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:37:04,072 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:37:04,556 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:37:06,252 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:37:06,559 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:37:07,073 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:37:10,309 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:37:10,310 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:37:10,310 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:37:10,315 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:37:10,561 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:37:11,253 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:37:12,316 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:37:12,562 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:37:14,791 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:37:14,792 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:37:14,792 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:37:14,797 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:37:15,564 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:37:15,631 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:37:16,566 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:37:17,798 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:37:19,265 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:37:19,265 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:37:19,266 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:37:19,270 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:37:19,568 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:37:20,571 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:37:21,254 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:37:22,945 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:37:22,945 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:37:22,946 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:37:22,946 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:37:22,951 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:37:23,573 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:37:24,575 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:37:26,254 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:37:26,963 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 19:37:27,431 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:37:27,432 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:37:27,432 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:37:27,437 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:37:27,577 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:37:28,437 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:37:28,579 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:37:30,631 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:37:31,317 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:37:31,318 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:37:31,318 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:37:31,323 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:37:31,581 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:37:32,583 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:37:34,361 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:37:36,002 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:37:36,002 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:37:36,002 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:37:36,008 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:37:36,256 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:37:36,586 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:37:38,589 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:37:40,008 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:37:40,761 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:37:40,761 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:37:40,762 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:37:40,767 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:37:41,256 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:37:41,591 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:37:42,594 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:37:44,822 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:37:44,823 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:37:44,823 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:37:44,828 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:37:45,596 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:37:45,631 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:37:45,828 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:37:46,597 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:37:48,542 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:37:48,542 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:37:48,543 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:37:48,547 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:37:48,599 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:37:50,602 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:37:51,257 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:37:51,588 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:37:54,283 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:37:54,284 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:37:54,284 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:37:54,289 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:37:54,604 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:37:56,258 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:37:56,608 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:37:56,965 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 19:37:56,966 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:37:57,642 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:37:57,643 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:37:57,643 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:37:57,647 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:37:58,611 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:37:58,611 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:38:00,631 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:38:01,268 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:38:01,269 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:38:01,269 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:38:01,274 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:38:01,612 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:38:02,309 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:38:02,615 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:38:04,720 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:38:04,721 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:38:04,721 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:38:04,729 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:38:05,618 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:38:06,259 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:38:06,620 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:38:07,729 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:38:08,691 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:38:08,692 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:38:08,692 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:38:08,697 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:38:09,622 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:38:10,625 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:38:11,260 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:38:12,429 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:38:12,430 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:38:12,430 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:38:12,436 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:38:12,626 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:38:13,436 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:38:14,629 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:38:15,631 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:38:16,500 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:38:16,500 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:38:16,500 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:38:16,505 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:38:16,630 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:38:18,541 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:38:18,633 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:38:20,033 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:38:20,034 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:38:20,034 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:38:20,039 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:38:20,635 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:38:21,261 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:38:22,638 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:38:24,040 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:38:24,439 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:38:24,440 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:38:24,440 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:38:24,446 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:38:24,639 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:38:26,262 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:38:26,642 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:38:26,967 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 19:38:29,348 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:38:29,349 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:38:29,349 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:38:29,349 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:38:29,354 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:38:29,644 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:38:30,631 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:38:30,646 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:38:34,032 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:38:34,033 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:38:34,033 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:38:34,038 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:38:34,649 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:38:35,073 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:38:36,262 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:38:36,652 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:38:39,176 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:38:39,177 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:38:39,177 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:38:39,182 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:38:39,653 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:38:40,182 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:38:40,656 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:38:41,263 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:38:42,627 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:38:42,628 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:38:42,628 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:38:42,633 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:38:42,657 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:38:44,661 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:38:45,632 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:38:45,634 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:38:46,166 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:38:46,166 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:38:46,166 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:38:46,171 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:38:46,662 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:38:48,666 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:38:49,653 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:38:49,654 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:38:49,654 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:38:49,659 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:38:49,667 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:38:50,660 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:38:50,669 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:38:51,264 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:38:55,169 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:38:55,170 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:38:55,170 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:38:55,175 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:38:55,672 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:38:56,175 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:38:56,265 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:38:56,673 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:38:56,970 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 19:38:59,556 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:38:59,557 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:38:59,557 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:38:59,562 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:38:59,675 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:39:00,632 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:39:00,677 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:39:01,562 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:39:02,810 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:39:02,810 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:39:02,811 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:39:02,815 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:39:03,679 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:39:04,683 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:39:06,266 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:39:06,854 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:39:06,864 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:39:06,864 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:39:06,865 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:39:06,869 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:39:07,685 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:39:08,687 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:39:11,266 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:39:11,787 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:39:11,787 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:39:11,788 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:39:11,793 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:39:12,691 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:39:12,691 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:39:12,793 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:39:15,632 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:39:15,999 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:39:16,000 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:39:16,000 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:39:16,005 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:39:16,693 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:39:18,041 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:39:18,696 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:39:20,794 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:39:20,795 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:39:20,795 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:39:20,800 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:39:21,267 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:39:21,699 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:39:22,701 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:39:23,800 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:39:25,937 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:39:25,938 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:39:25,938 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:39:25,943 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:39:26,704 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:39:26,823 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:39:26,972 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 19:39:28,707 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:39:28,973 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:39:30,346 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:39:30,346 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:39:30,347 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:39:30,351 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:39:30,632 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:39:30,708 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:39:32,711 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:39:34,240 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:39:34,240 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:39:34,241 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:39:34,241 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:39:34,246 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:39:34,712 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:39:35,824 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:39:36,715 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:39:37,735 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:39:37,736 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:39:37,736 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:39:37,741 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:39:38,718 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:39:38,718 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:39:39,741 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:39:40,688 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:39:40,689 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:39:40,689 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:39:40,694 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:39:40,719 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:39:40,824 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:39:42,722 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:39:44,050 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:39:44,051 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:39:44,051 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:39:44,056 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:39:44,724 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:39:45,056 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:39:45,632 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:39:46,727 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:39:47,572 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:39:47,573 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:39:47,573 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:39:47,578 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:39:47,729 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:39:48,732 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:39:50,614 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:39:50,826 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:39:51,204 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:39:51,204 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:39:51,205 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:39:51,210 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:39:51,733 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:39:52,736 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:39:55,463 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:39:55,463 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:39:55,463 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:39:55,468 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:39:55,737 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:39:55,826 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:39:56,469 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:39:56,740 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:39:56,974 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 19:39:59,238 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:39:59,239 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:39:59,239 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:39:59,244 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:39:59,742 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:40:00,632 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:40:00,744 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:40:02,245 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:40:03,619 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:40:03,620 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:40:03,620 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:40:03,624 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:40:03,746 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:40:04,750 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:40:05,827 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:40:07,662 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:40:08,913 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:40:08,913 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:40:08,914 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:40:08,919 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:40:09,753 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:40:10,755 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:40:10,828 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:40:12,403 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:40:12,403 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:40:12,404 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:40:12,409 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:40:12,756 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:40:13,409 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:40:14,759 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:40:15,632 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:40:15,668 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:40:15,669 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:40:15,669 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:40:15,673 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:40:15,760 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:40:16,762 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:40:18,709 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:40:19,308 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:40:19,309 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:40:19,309 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:40:19,314 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:40:19,764 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:40:20,767 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:40:20,829 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:40:22,851 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:40:22,851 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:40:22,851 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:40:22,856 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:40:23,769 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:40:23,857 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:40:24,771 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:40:25,830 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:40:25,962 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:40:25,963 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:40:25,963 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:40:25,968 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:40:26,772 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:40:26,976 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 19:40:28,776 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:40:28,977 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:40:30,622 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:40:30,623 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:40:30,623 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:40:30,628 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:40:30,632 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:40:30,777 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:40:32,780 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:40:34,629 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:40:35,831 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:40:39,009 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:40:39,009 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:40:39,010 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:40:39,014 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:40:39,784 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:40:40,015 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:40:40,786 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:40:40,831 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:40:42,944 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:40:42,945 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:40:42,945 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:40:42,950 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:40:43,789 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:40:44,791 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:40:45,632 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:40:45,951 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:40:46,058 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:40:46,058 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:40:46,058 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:40:46,063 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:40:46,792 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:40:48,797 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:40:50,832 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:40:51,102 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:40:54,445 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:40:54,446 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:40:54,446 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:40:54,451 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:40:54,800 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:40:55,833 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:40:56,451 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:40:56,804 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:40:56,981 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 19:40:59,223 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:40:59,224 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:40:59,224 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:40:59,229 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:40:59,806 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:41:00,632 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:41:00,808 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:41:02,161 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:41:02,161 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:41:02,162 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:41:02,162 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:41:02,167 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:41:02,810 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:41:04,814 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:41:05,834 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:41:06,140 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:41:06,141 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:41:06,141 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:41:06,146 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:41:06,815 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:41:08,146 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:41:08,818 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:41:09,563 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:41:09,564 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:41:09,564 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:41:09,569 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:41:09,819 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:41:10,821 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:41:10,834 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:41:12,870 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:41:12,871 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:41:12,871 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:41:12,876 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:41:13,823 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:41:13,876 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:41:14,826 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:41:15,632 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:41:18,877 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:41:20,651 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:41:20,651 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:41:20,652 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:41:20,656 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:41:20,829 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:41:20,835 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:41:22,832 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:41:24,694 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:41:24,909 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:41:24,910 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:41:24,910 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:41:24,915 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:41:25,834 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:41:25,836 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:41:26,836 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:41:26,980 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 19:41:29,283 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:41:29,284 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:41:29,284 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:41:29,289 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:41:29,838 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:41:30,289 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:41:30,633 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:41:30,841 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:41:33,127 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:41:33,127 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:41:33,127 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:41:33,132 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:41:33,842 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:41:34,845 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:41:35,837 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:41:36,169 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:41:37,219 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:41:37,220 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:41:37,220 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:41:37,225 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:41:37,847 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:41:38,850 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:41:40,838 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:41:41,226 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:41:41,600 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:41:41,600 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:41:41,600 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:41:41,605 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:41:41,852 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:41:42,854 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:41:44,902 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:41:44,902 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:41:44,903 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:41:44,907 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:41:45,633 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:41:45,856 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:41:46,858 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:41:46,908 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:41:48,559 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:41:48,559 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:41:48,560 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:41:48,564 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:41:48,861 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:41:50,839 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:41:50,864 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:41:52,602 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:41:52,924 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:41:52,924 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:41:52,925 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:41:52,930 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:41:53,866 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:41:54,869 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:41:55,839 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:41:56,205 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:41:56,205 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:41:56,205 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:41:56,210 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:41:56,870 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:41:56,986 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 19:41:57,987 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:41:58,873 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:41:59,446 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:41:59,447 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:41:59,447 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:41:59,452 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:41:59,874 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:42:00,633 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:42:00,876 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:42:02,915 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:42:02,916 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:42:02,916 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:42:02,921 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:42:03,878 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:42:03,957 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:42:04,882 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:42:05,840 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:42:06,517 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:42:06,518 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:42:06,518 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:42:06,523 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:42:06,883 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:42:08,886 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:42:09,523 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:42:10,841 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:42:11,896 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:42:11,897 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:42:11,897 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:42:11,902 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:42:12,891 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:42:12,891 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:42:14,902 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:42:15,633 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:42:16,435 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:42:16,436 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:42:16,436 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:42:16,440 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:42:16,893 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:42:18,896 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:42:20,363 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:42:20,363 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:42:20,364 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:42:20,364 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:42:20,369 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:42:20,842 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:42:20,897 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:42:22,900 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:42:23,578 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:42:23,579 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:42:23,579 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:42:23,584 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:42:23,901 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:42:24,904 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:42:25,585 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:42:25,843 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:42:26,985 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 19:42:27,416 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:42:27,417 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:42:27,417 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:42:27,422 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:42:27,905 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:42:28,908 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:42:30,633 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:42:30,954 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:42:30,955 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:42:30,955 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:42:30,955 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:42:30,960 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:42:31,910 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:42:32,912 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:42:35,844 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:42:35,998 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:42:39,364 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:42:39,365 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:42:39,365 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:42:39,370 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:42:39,916 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:42:40,844 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:42:40,919 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:42:41,371 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:42:43,845 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:42:43,845 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:42:43,846 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:42:43,850 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:42:43,921 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:42:44,922 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:42:45,633 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:42:46,851 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:42:47,721 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:42:47,722 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:42:47,722 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:42:47,727 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:42:47,925 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:42:48,927 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:42:50,845 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:42:52,766 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:42:53,255 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:42:53,256 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:42:53,256 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:42:53,261 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:42:53,930 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:42:54,932 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:42:55,846 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:42:56,990 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 19:42:57,347 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:42:57,348 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:42:57,348 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:42:57,353 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:42:57,936 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:42:58,353 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:42:58,938 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:43:00,633 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:43:03,354 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:43:05,511 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:43:05,512 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:43:05,512 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:43:05,517 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:43:05,847 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:43:05,943 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:43:06,946 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:43:08,553 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:43:09,022 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:43:09,023 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:43:09,023 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:43:09,028 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:43:09,947 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:43:10,848 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:43:10,950 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:43:14,029 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:43:14,273 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:43:14,274 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:43:14,274 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:43:14,280 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:43:14,952 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:43:15,633 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:43:16,955 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:43:19,281 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:43:20,217 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:43:20,218 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:43:20,218 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:43:20,223 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:43:20,849 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:43:20,958 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:43:22,961 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:43:23,925 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:43:23,925 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:43:23,926 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:43:23,931 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:43:23,962 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:43:24,931 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:43:24,962 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:43:25,849 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:43:26,989 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 19:43:29,708 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:43:29,709 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:43:29,709 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:43:29,714 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:43:29,965 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:43:30,633 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:43:30,714 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:43:30,968 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:43:33,805 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:43:33,806 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:43:33,806 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:43:33,811 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:43:33,970 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:43:34,972 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:43:35,849 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:43:35,850 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:43:37,627 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:43:37,627 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:43:37,628 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:43:37,633 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:43:37,974 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:43:38,977 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:43:40,851 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:43:41,633 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:43:43,146 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:43:43,147 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:43:43,147 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:43:43,152 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:43:43,980 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:43:44,982 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:43:45,633 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:43:46,486 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:43:46,487 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:43:46,487 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:43:46,492 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:43:46,983 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:43:47,529 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:43:48,987 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:43:49,564 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:43:49,565 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:43:49,565 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:43:49,570 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:43:49,988 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:43:50,852 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:43:50,990 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:43:52,571 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:43:53,882 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:43:53,882 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:43:53,883 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:43:53,887 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:43:53,992 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:43:54,994 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:43:55,853 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:43:56,995 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 19:43:57,996 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:43:58,759 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:43:58,759 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:43:58,760 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:43:58,764 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:43:58,998 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:44:00,633 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:44:01,000 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:44:02,983 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:44:02,984 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:44:02,984 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:44:02,989 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:44:03,002 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:44:03,025 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:44:05,006 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:44:05,854 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:44:07,262 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:44:07,263 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:44:07,263 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:44:07,269 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:44:08,008 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:44:08,269 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:44:09,010 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:44:10,789 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:44:10,789 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:44:10,790 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:44:10,794 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:44:10,854 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:44:11,011 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:44:13,014 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:44:13,795 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:44:14,744 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:44:14,745 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:44:14,745 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:44:14,750 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:44:15,015 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:44:15,634 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:44:17,018 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:44:18,876 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:44:18,877 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:44:18,877 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:44:18,877 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:44:18,882 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:44:19,019 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:44:21,023 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:44:21,286 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:44:23,653 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:44:23,654 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:44:23,654 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:44:23,659 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:44:24,025 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:44:24,659 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:44:25,027 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:44:26,286 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:44:26,994 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 19:44:29,995 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:44:30,634 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:44:32,073 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:44:32,074 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:44:32,074 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:44:32,079 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:44:33,033 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:44:33,033 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:44:35,117 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:44:35,877 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:44:35,878 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:44:35,878 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:44:35,883 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:44:36,035 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:44:36,287 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:44:37,037 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:44:40,616 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:44:40,617 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:44:40,617 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:44:40,617 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:44:40,622 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:44:41,040 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:44:41,288 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:44:43,043 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:44:44,101 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:44:44,102 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:44:44,102 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:44:44,107 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:44:45,046 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:44:45,046 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:44:45,634 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:44:46,107 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:44:50,508 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:44:50,508 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:44:50,508 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:44:50,513 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:44:51,051 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:44:51,289 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:44:51,549 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:44:53,054 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:44:53,600 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:44:53,601 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:44:53,601 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:44:53,606 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:44:54,054 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:44:55,057 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:44:56,289 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:44:56,606 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:44:56,999 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 19:44:57,278 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:44:57,279 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:44:57,279 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:44:57,283 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:44:58,058 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:44:59,061 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:45:00,634 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:45:01,669 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:45:01,669 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:45:01,669 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:45:01,670 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:45:01,674 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:45:02,063 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:45:03,065 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:45:06,290 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:45:06,710 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:45:06,813 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:45:06,813 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:45:06,813 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:45:06,818 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:45:07,069 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:45:09,071 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:45:10,255 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:45:10,255 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:45:10,256 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:45:10,260 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:45:11,074 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:45:11,075 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:45:11,291 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:45:12,261 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:45:13,702 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:45:13,702 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:45:13,702 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:45:13,708 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:45:14,076 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:45:15,079 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:45:15,634 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:45:17,005 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:45:17,005 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:45:17,006 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:45:17,010 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:45:17,080 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:45:18,045 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:45:19,081 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:45:20,434 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:45:20,435 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:45:20,435 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:45:20,440 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:45:21,083 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:45:21,292 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:45:23,086 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:45:23,440 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:45:24,072 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:45:24,073 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:45:24,073 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:45:24,083 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:45:24,087 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:45:25,089 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:45:26,293 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:45:26,998 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 19:45:28,597 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:45:28,598 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:45:28,598 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:45:28,598 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:45:28,603 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:45:29,091 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:45:30,634 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:45:31,094 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:45:31,856 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:45:31,857 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:45:31,857 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:45:31,861 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:45:32,095 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:45:33,097 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:45:33,897 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:45:35,326 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:45:35,327 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:45:35,327 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:45:35,332 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:45:36,099 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:45:36,294 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:45:37,101 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:45:38,999 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:45:39,000 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:45:39,000 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:45:39,001 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:45:39,005 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:45:39,102 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:45:41,105 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:45:41,294 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:45:42,872 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:45:42,873 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:45:42,873 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:45:42,878 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:45:43,107 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:45:44,879 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:45:45,110 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:45:45,634 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:45:46,179 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:45:46,179 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:45:46,179 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:45:46,184 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:45:47,118 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:45:47,118 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:45:50,222 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:45:50,918 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:45:50,919 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:45:50,919 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:45:50,924 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:45:51,121 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:45:51,295 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:45:53,124 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:45:54,232 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:45:54,233 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:45:54,233 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:45:54,238 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:45:55,127 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:45:55,128 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:45:55,239 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:45:56,296 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:45:57,003 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 19:46:00,634 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:46:01,005 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:46:01,827 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:46:01,828 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:46:01,828 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:46:01,833 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:46:02,131 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:46:03,134 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:46:04,996 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:46:04,997 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:46:04,997 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:46:05,002 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:46:05,136 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:46:06,297 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:46:07,003 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:46:07,139 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:46:10,039 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:46:10,039 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:46:10,040 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:46:10,044 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:46:10,140 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:46:11,143 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:46:11,298 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:46:12,045 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:46:14,120 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:46:14,120 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:46:14,121 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:46:14,125 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:46:14,145 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:46:15,147 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:46:15,634 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:46:17,126 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:46:18,151 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:46:18,152 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:46:18,152 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:46:18,157 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:46:19,151 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:46:19,151 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:46:21,276 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:46:21,277 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:46:21,277 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:46:21,282 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:46:21,299 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:46:22,153 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:46:22,282 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:46:23,155 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:46:25,228 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:46:25,229 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:46:25,229 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:46:25,235 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:46:26,157 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:46:26,299 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:46:27,002 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 19:46:27,159 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:46:28,003 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:46:29,543 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:46:29,544 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:46:29,544 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:46:29,549 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:46:30,161 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:46:30,634 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:46:31,163 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:46:32,891 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:46:32,891 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:46:32,892 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:46:32,897 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:46:33,165 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:46:33,933 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:46:35,168 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:46:36,300 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:46:37,425 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:46:37,426 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:46:37,426 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:46:37,431 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:46:38,169 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:46:39,170 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:46:39,431 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:46:41,138 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:46:41,139 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:46:41,139 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:46:41,144 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:46:41,171 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:46:41,301 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:46:43,174 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:46:45,026 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:46:45,027 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:46:45,027 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:46:45,027 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:46:45,032 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:46:45,175 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:46:45,634 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:46:47,178 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:46:49,008 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:46:49,009 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:46:49,009 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:46:49,014 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:46:49,181 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:46:50,049 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:46:51,184 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:46:51,302 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:46:52,962 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:46:52,963 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:46:52,963 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:46:52,968 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:46:53,185 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:46:55,188 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:46:55,969 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:46:56,303 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:46:56,500 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:46:56,501 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:46:56,501 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:46:56,506 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:46:57,008 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 19:46:57,189 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:46:59,192 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:47:00,635 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:47:01,009 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:47:04,261 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:47:04,262 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:47:04,262 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:47:04,267 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:47:05,198 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:47:05,198 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:47:06,304 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:47:06,305 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:47:11,304 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:47:11,306 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:47:13,673 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:47:13,674 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:47:13,674 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:47:13,679 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:47:14,203 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:47:15,206 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:47:15,635 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:47:16,680 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:47:18,151 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:47:18,152 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:47:18,152 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:47:18,157 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:47:18,207 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:47:19,210 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:47:21,305 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:47:22,193 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:47:22,931 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:47:22,931 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:47:22,932 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:47:22,937 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:47:23,212 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:47:25,216 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:47:26,306 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:47:27,007 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 19:47:27,127 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:47:27,128 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:47:27,128 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:47:27,132 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:47:27,217 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:47:28,133 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:47:29,220 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:47:30,635 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:47:31,035 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:47:31,036 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:47:31,036 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:47:31,041 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:47:31,221 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:47:33,224 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:47:34,077 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:47:35,709 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:47:35,710 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:47:35,710 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:47:35,715 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:47:36,226 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:47:36,307 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:47:37,228 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:47:39,195 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:47:39,196 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:47:39,196 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:47:39,196 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:47:39,201 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:47:39,229 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:47:41,232 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:47:41,308 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:47:42,574 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:47:42,574 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:47:42,575 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:47:42,579 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:47:43,233 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:47:44,580 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:47:45,237 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:47:45,635 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:47:45,851 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:47:45,851 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:47:45,852 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:47:45,856 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:47:46,237 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:47:47,240 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:47:49,644 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:47:49,644 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:47:49,645 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:47:49,645 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:47:49,649 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:47:50,243 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:47:51,246 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:47:51,309 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:47:53,228 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:47:53,229 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:47:53,229 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:47:53,234 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:47:53,247 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:47:55,235 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:47:55,250 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:47:56,309 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:47:56,546 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:47:56,547 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:47:56,547 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:47:56,552 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:47:57,009 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 19:47:57,251 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:47:59,254 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:48:00,635 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:48:01,010 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:48:02,697 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:48:02,698 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:48:02,698 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:48:02,703 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:48:03,256 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:48:05,261 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:48:06,123 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:48:06,127 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:48:06,127 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:48:06,127 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:48:06,132 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:48:06,261 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:48:06,310 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:48:07,264 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:48:11,033 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:48:11,034 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:48:11,034 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:48:11,039 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:48:11,266 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:48:11,311 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:48:12,040 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:48:13,269 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:48:15,635 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:48:17,040 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:48:18,860 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:48:18,860 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:48:18,860 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:48:18,865 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:48:19,272 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:48:21,276 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:48:21,312 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:48:22,486 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:48:22,487 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:48:22,487 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:48:22,487 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:48:22,492 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:48:23,279 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:48:23,279 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:48:26,313 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:48:27,011 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 19:48:28,013 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:48:30,635 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:48:31,584 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:48:31,584 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:48:31,584 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:48:31,589 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:48:32,286 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:48:33,286 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:48:33,625 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:48:35,661 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:48:35,662 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:48:35,662 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:48:35,667 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:48:36,288 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:48:36,314 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:48:37,290 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:48:38,667 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:48:39,966 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:48:39,967 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:48:39,967 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:48:39,972 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:48:40,292 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:48:41,294 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:48:41,314 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:48:43,575 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:48:43,576 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:48:43,577 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:48:43,581 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:48:44,297 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:48:44,582 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:48:45,299 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:48:45,635 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:48:47,444 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:48:47,445 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:48:47,445 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:48:47,449 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:48:48,303 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:48:49,305 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:48:50,485 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:48:51,315 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:48:53,878 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:48:53,879 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:48:53,879 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:48:53,884 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:48:54,308 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:48:55,310 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:48:55,884 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:48:56,316 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:48:57,014 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 19:48:59,408 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:48:59,409 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:48:59,409 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:48:59,414 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:49:00,313 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:49:00,635 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:49:01,314 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:49:01,414 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:49:03,127 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:49:03,128 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:49:03,128 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:49:03,133 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:49:03,315 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:49:05,319 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:49:06,317 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:49:07,170 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:49:08,532 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:49:08,533 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:49:08,533 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:49:08,538 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:49:09,323 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:49:09,323 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:49:11,317 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:49:12,539 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:49:12,691 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:49:12,691 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:49:12,692 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:49:12,696 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:49:13,326 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:49:15,329 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:49:15,635 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:49:17,022 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:49:17,022 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:49:17,022 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:49:17,027 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:49:17,330 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:49:18,065 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:49:19,333 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:49:21,318 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:49:21,794 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:49:21,798 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:49:21,798 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:49:21,803 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:49:22,335 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:49:23,336 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:49:23,803 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:49:25,237 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:49:25,238 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:49:25,238 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:49:25,244 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:49:25,337 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:49:26,878 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:49:27,016 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 19:49:27,340 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:49:29,017 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:49:29,600 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:49:29,600 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:49:29,601 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:49:29,605 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:49:30,342 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:49:30,635 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:49:31,345 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:49:33,658 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:49:33,659 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:49:33,659 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:49:33,664 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:49:34,346 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:49:34,701 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:49:35,349 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:49:35,879 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:49:36,814 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:49:36,814 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:49:36,815 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:49:36,819 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:49:37,350 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:49:39,353 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:49:39,820 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:49:40,880 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:49:41,563 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:49:41,564 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:49:41,564 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:49:41,569 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:49:42,355 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:49:43,358 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:49:45,570 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:49:45,635 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:49:46,139 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:49:46,139 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:49:46,139 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:49:46,144 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:49:46,360 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:49:47,362 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:49:49,605 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:49:49,606 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:49:49,606 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:49:49,611 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:49:50,365 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:49:50,611 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:49:50,881 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:49:51,368 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:49:54,510 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:49:54,511 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:49:54,511 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:49:54,516 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:49:55,373 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:49:55,373 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:49:55,882 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:49:56,516 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:49:57,018 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 19:49:57,878 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:49:57,878 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:49:57,879 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:49:57,883 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:49:58,374 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:49:59,377 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:50:00,635 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:50:01,884 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:50:05,771 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:50:05,771 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:50:05,771 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:50:05,776 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:50:05,883 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:50:06,382 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:50:07,383 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:50:07,813 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:50:09,901 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:50:09,901 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:50:09,902 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:50:09,906 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:50:10,385 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:50:10,883 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:50:11,387 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:50:12,907 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:50:15,636 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:50:17,907 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:50:18,120 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:50:18,121 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:50:18,121 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:50:18,126 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:50:18,392 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:50:19,394 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:50:20,884 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:50:21,641 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:50:21,642 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:50:21,642 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:50:21,647 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:50:22,397 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:50:23,397 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:50:23,648 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:50:25,051 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:50:25,052 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:50:25,052 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:50:25,057 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:50:25,398 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:50:25,885 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:50:27,020 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 19:50:27,401 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:50:28,705 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:50:28,706 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:50:28,706 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:50:28,706 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:50:28,712 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:50:29,403 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:50:30,636 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:50:31,406 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:50:33,713 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:50:34,022 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:50:34,023 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:50:34,023 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:50:34,027 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:50:34,407 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:50:35,410 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:50:35,886 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:50:37,615 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:50:37,616 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:50:37,616 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:50:37,621 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:50:38,412 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:50:39,413 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:50:39,622 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:50:40,887 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:50:43,769 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:50:43,770 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:50:43,770 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:50:43,774 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:50:44,416 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:50:44,775 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:50:45,419 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:50:45,636 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:50:47,649 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:50:47,650 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:50:47,650 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:50:47,655 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:50:48,422 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:50:49,424 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:50:50,693 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:50:50,888 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:50:51,434 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:50:51,435 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:50:51,435 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:50:51,440 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:50:52,426 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:50:53,429 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:50:55,107 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:50:55,108 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:50:55,108 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:50:55,113 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:50:55,430 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:50:55,888 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:50:56,113 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:50:57,022 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 19:50:57,433 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:50:59,038 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:50:59,038 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:50:59,039 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:50:59,043 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:50:59,434 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:51:00,636 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:51:01,435 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:51:02,044 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:51:02,880 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:51:02,880 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:51:02,881 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:51:02,885 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:51:03,437 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:51:05,441 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:51:05,890 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:51:07,895 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:51:07,896 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:51:07,896 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:51:07,896 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:51:07,901 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:51:08,443 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:51:09,445 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:51:10,891 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:51:11,702 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:51:11,703 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:51:11,703 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:51:11,708 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:51:12,447 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:51:13,447 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:51:13,708 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:51:15,636 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:51:18,709 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:51:19,726 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:51:19,727 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:51:19,727 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:51:19,732 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:51:20,452 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:51:20,892 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:51:21,455 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:51:23,116 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:51:23,117 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:51:23,117 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:51:23,122 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:51:23,456 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:51:24,122 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:51:25,459 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:51:25,893 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:51:26,777 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:51:26,778 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:51:26,778 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:51:26,783 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:51:27,028 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 19:51:27,461 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:51:29,462 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:51:30,030 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:51:30,636 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:51:31,801 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:51:31,801 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:51:31,802 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:51:31,807 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:51:32,463 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:51:33,466 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:51:35,846 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:51:35,894 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:51:36,190 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:51:36,191 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:51:36,191 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:51:36,196 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:51:36,468 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:51:37,470 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:51:40,099 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:51:40,100 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:51:40,100 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:51:40,105 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:51:40,472 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:51:40,894 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:51:41,106 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:51:41,474 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:51:43,420 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:51:43,420 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:51:43,421 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:51:43,426 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:51:43,475 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:51:45,480 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:51:45,636 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:51:46,426 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:51:47,432 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:51:47,433 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:51:47,433 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:51:47,438 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:51:47,481 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:51:49,485 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:51:50,895 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:51:51,474 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:51:52,283 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:51:52,284 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:51:52,284 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:51:52,289 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:51:52,487 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:51:53,489 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:51:55,896 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:51:57,027 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 19:51:57,028 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:51:58,317 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:51:58,318 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:51:58,318 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:51:58,323 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:51:58,492 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:51:59,495 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:52:00,636 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:52:02,324 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:52:05,897 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:52:06,773 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:52:06,774 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:52:06,774 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:52:06,779 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:52:07,502 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:52:07,503 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:52:07,779 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:52:09,972 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:52:09,973 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:52:09,973 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:52:09,978 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:52:10,504 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:52:10,898 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:52:11,505 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:52:12,979 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:52:13,723 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:52:13,724 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:52:13,724 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:52:13,729 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:52:14,507 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:52:15,509 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:52:15,636 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:52:17,487 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:52:17,487 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:52:17,487 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:52:17,492 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:52:17,510 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:52:18,530 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:52:19,513 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:52:20,899 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:52:22,182 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:52:22,183 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:52:22,183 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:52:22,188 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:52:22,516 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:52:23,518 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:52:24,188 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:52:25,855 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:52:25,855 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:52:25,856 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:52:25,860 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:52:25,899 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:52:26,520 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:52:27,032 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 19:52:27,522 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:52:30,033 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:52:30,636 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:52:34,393 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:52:34,394 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:52:34,394 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:52:34,399 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:52:34,526 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:52:35,438 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:52:35,527 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:52:35,900 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:52:38,057 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:52:38,058 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:52:38,058 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:52:38,063 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:52:38,529 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:52:39,531 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:52:40,901 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:52:41,064 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:52:41,569 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:52:41,570 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:52:41,570 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:52:41,575 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:52:42,533 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:52:43,536 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:52:45,636 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:52:46,576 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:52:50,173 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:52:50,174 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:52:50,174 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:52:50,179 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:52:50,542 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:52:50,902 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:52:51,545 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:52:52,218 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:52:55,027 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:52:55,027 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:52:55,027 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:52:55,032 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:52:55,547 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:52:55,903 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:52:57,031 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 19:52:57,548 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:52:58,032 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:52:59,430 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:52:59,431 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:52:59,431 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:52:59,436 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:52:59,549 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:53:00,636 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:53:01,552 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:53:03,437 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:53:03,902 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:53:03,902 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:53:03,902 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:53:03,907 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:53:04,554 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:53:05,558 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:53:05,904 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:53:07,968 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:53:07,968 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:53:07,969 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:53:07,974 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:53:08,560 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:53:08,974 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:53:09,562 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:53:10,904 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:53:12,361 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:53:12,362 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:53:12,362 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:53:12,367 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:53:12,564 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:53:13,566 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:53:14,368 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:53:15,637 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:53:19,368 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:53:20,905 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:53:21,487 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:53:21,488 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:53:21,488 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:53:21,493 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:53:21,571 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:53:23,574 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:53:24,494 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:53:25,787 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:53:25,788 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:53:25,788 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:53:25,793 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:53:25,906 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:53:26,575 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:53:27,036 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 19:53:27,578 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:53:29,891 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:53:29,891 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:53:29,891 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:53:29,892 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:53:29,896 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:53:30,580 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:53:30,637 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:53:31,582 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:53:34,707 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:53:34,708 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:53:34,708 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:53:34,713 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:53:35,586 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:53:35,586 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:53:35,750 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:53:35,907 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:53:39,037 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:53:39,038 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:53:39,038 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:53:39,042 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:53:39,589 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:53:40,907 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:53:41,043 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:53:41,592 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:53:42,250 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:53:42,251 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:53:42,251 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:53:42,256 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:53:42,592 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:53:43,595 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:53:45,637 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:53:46,256 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:53:46,989 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:53:46,990 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:53:46,990 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:53:46,995 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:53:47,598 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:53:49,602 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:53:50,903 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:53:50,904 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:53:50,904 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:53:50,908 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:53:50,909 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:53:51,604 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:53:51,604 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:53:51,909 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:53:55,400 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:53:55,401 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:53:55,401 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:53:55,405 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:53:55,606 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:53:55,909 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:53:57,036 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 19:53:57,037 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:53:57,609 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:54:00,165 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:54:00,166 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:54:00,166 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:54:00,170 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:54:00,611 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:54:00,637 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:54:01,613 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:54:02,171 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:54:04,829 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:54:04,830 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:54:04,830 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:54:04,835 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:54:05,619 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:54:05,619 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:54:05,910 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:54:07,874 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:54:09,863 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:54:09,864 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:54:09,864 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:54:09,869 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:54:10,622 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:54:10,911 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:54:11,624 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:54:13,870 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:54:14,113 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:54:14,114 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:54:14,114 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:54:14,119 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:54:14,627 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:54:15,630 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:54:15,637 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:54:18,377 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:54:18,378 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:54:18,378 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:54:18,390 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:54:18,632 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:54:19,422 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:54:19,634 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:54:20,912 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:54:21,878 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:54:21,879 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:54:21,879 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:54:21,884 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:54:22,636 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:54:23,637 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:54:24,884 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:54:25,912 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:54:27,038 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 19:54:27,706 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:54:27,710 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:54:27,710 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:54:27,715 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:54:28,640 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:54:29,643 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:54:30,637 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:54:30,715 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:54:31,823 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:54:31,823 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:54:31,824 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:54:31,828 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:54:32,644 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:54:33,647 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:54:35,867 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:54:35,913 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:54:37,606 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:54:37,607 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:54:37,608 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:54:37,613 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:54:37,649 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:54:39,652 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:54:40,914 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:54:41,295 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:54:41,296 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:54:41,296 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:54:41,296 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:54:41,301 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:54:41,653 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:54:43,656 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:54:45,069 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:54:45,070 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:54:45,070 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:54:45,075 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:54:45,637 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:54:45,658 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:54:47,075 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:54:47,661 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:54:48,423 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:54:48,423 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:54:48,423 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:54:48,428 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:54:48,663 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:54:49,665 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:54:50,915 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:54:51,943 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:54:51,944 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:54:51,944 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:54:51,949 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:54:52,667 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:54:52,950 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:54:53,670 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:54:55,544 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:54:55,545 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:54:55,545 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:54:55,550 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:54:55,671 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:54:55,916 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:54:57,040 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 19:54:57,674 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:54:58,041 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:54:59,161 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:54:59,162 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:54:59,162 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:54:59,167 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:54:59,675 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:55:00,637 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:55:01,678 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:55:02,393 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:55:02,393 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:55:02,394 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:55:02,399 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:55:02,679 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:55:03,434 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:55:03,682 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:55:05,917 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:55:06,430 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:55:06,431 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:55:06,431 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:55:06,436 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:55:06,685 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:55:07,687 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:55:08,436 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:55:10,917 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:55:11,696 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:55:11,696 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:55:11,697 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:55:11,702 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:55:12,690 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:55:13,691 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:55:13,703 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:55:15,637 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:55:18,014 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:55:18,015 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:55:18,015 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:55:18,019 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:55:18,694 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:55:19,058 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:55:19,696 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:55:20,918 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:55:21,687 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:55:21,687 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:55:21,687 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:55:21,692 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:55:21,697 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:55:23,701 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:55:24,693 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:55:25,564 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:55:25,565 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:55:25,565 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:55:25,570 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:55:25,702 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:55:25,919 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:55:27,042 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 19:55:27,705 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:55:30,044 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:55:30,637 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:55:30,774 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:55:30,778 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:55:30,778 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:55:30,783 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:55:31,709 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:55:31,709 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:55:34,446 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:55:34,447 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:55:34,447 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:55:34,452 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:55:34,711 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:55:35,452 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:55:35,713 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:55:35,920 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:55:40,453 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:55:40,921 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:55:43,609 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:55:43,610 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:55:43,610 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:55:43,615 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:55:43,718 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:55:45,616 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:55:45,637 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:55:45,720 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:55:46,473 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:55:46,474 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:55:46,474 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:55:46,479 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:55:46,721 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:55:47,723 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:55:50,922 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:55:50,996 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:55:50,996 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:55:50,997 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:55:50,997 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:55:51,001 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:55:51,729 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:55:51,729 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:55:54,540 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:55:54,540 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:55:54,541 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:55:54,545 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:55:54,731 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:55:55,733 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:55:55,922 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:55:56,546 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:55:57,045 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 19:55:58,975 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:55:58,975 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:55:58,975 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:55:58,980 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:55:59,737 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:55:59,738 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:56:00,638 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:56:01,981 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:56:02,049 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:56:02,050 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:56:02,050 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:56:02,055 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:56:02,739 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:56:03,741 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:56:05,923 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:56:05,927 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:56:05,928 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:56:05,928 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:56:05,933 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:56:06,745 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:56:07,747 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:56:07,933 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:56:10,149 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:56:10,150 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:56:10,150 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:56:10,155 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:56:10,749 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:56:10,924 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:56:11,751 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:56:13,155 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:56:15,638 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:56:15,800 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:56:15,801 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:56:15,801 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:56:15,806 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:56:16,754 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:56:17,757 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:56:18,842 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:56:19,127 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:56:19,128 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:56:19,128 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:56:19,133 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:56:19,758 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:56:20,925 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:56:21,762 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:56:22,690 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:56:22,691 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:56:22,691 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:56:22,696 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:56:22,762 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:56:23,765 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:56:24,696 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:56:25,926 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:56:26,178 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:56:26,179 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:56:26,179 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:56:26,183 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:56:26,766 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:56:27,047 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 19:56:27,769 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:56:30,048 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:56:30,573 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:56:30,574 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:56:30,574 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:56:30,579 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:56:30,638 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:56:30,771 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:56:31,773 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:56:34,173 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:56:34,173 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:56:34,173 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:56:34,179 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:56:34,775 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:56:35,214 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:56:35,776 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:56:35,927 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:56:37,867 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:56:37,868 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:56:37,868 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:56:37,873 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:56:38,778 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:56:39,785 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:56:40,874 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:56:40,927 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:56:41,395 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:56:41,396 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:56:41,396 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:56:41,401 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:56:41,786 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:56:43,789 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:56:45,455 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:56:45,456 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:56:45,456 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:56:45,461 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:56:45,638 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:56:45,791 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:56:46,462 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:56:47,794 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:56:48,961 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:56:48,962 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:56:48,962 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:56:48,967 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:56:49,798 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:56:49,799 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:56:50,928 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:56:52,006 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:56:52,445 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:56:52,445 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:56:52,446 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:56:52,451 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:56:52,800 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:56:53,803 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:56:55,929 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:56:57,049 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 19:56:57,050 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:56:58,309 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:56:58,309 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:56:58,309 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:56:58,314 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:56:58,806 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:56:59,808 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:57:00,638 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:57:02,315 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:57:02,882 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:57:02,883 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:57:02,883 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:57:02,887 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:57:03,810 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:57:03,810 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:57:05,930 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:57:06,388 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:57:06,389 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:57:06,390 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:57:06,394 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:57:06,813 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:57:07,395 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:57:07,816 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:57:10,931 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:57:12,395 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:57:12,590 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:57:12,591 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:57:12,591 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:57:12,596 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:57:12,818 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:57:13,821 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:57:15,638 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:57:17,596 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:57:17,964 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:57:17,964 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:57:17,965 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:57:17,970 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:57:18,825 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:57:19,827 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:57:20,932 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:57:21,290 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:57:21,291 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:57:21,291 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:57:21,297 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:57:21,830 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:57:23,297 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:57:23,833 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:57:25,421 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:57:25,422 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:57:25,422 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:57:25,427 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:57:25,834 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:57:25,932 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:57:27,051 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 19:57:27,837 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:57:28,669 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:57:28,670 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:57:28,670 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:57:28,670 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:57:28,675 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:57:28,838 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:57:29,840 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:57:30,638 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:57:33,170 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:57:33,171 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:57:33,171 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:57:33,176 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:57:33,843 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:57:34,217 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:57:35,846 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:57:35,933 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:57:36,813 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:57:36,814 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:57:36,814 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:57:36,818 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:57:36,846 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:57:37,849 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:57:39,819 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:57:40,204 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:57:40,205 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:57:40,205 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:57:40,210 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:57:40,850 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:57:40,934 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:57:41,853 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:57:45,211 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:57:45,638 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:57:48,835 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:57:48,836 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:57:48,836 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:57:48,842 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:57:48,857 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:57:49,860 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:57:50,878 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:57:50,935 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:57:53,256 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:57:53,257 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:57:53,257 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:57:53,261 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:57:53,862 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:57:55,865 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:57:55,936 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:57:56,262 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:57:57,053 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 19:57:57,316 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:57:57,316 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:57:57,316 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:57:57,321 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:57:57,867 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:57:59,870 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:58:00,638 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:58:01,322 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:58:01,995 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:58:01,995 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:58:01,996 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:58:02,000 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:58:02,872 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:58:03,874 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:58:05,937 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:58:07,038 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:58:07,144 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:58:07,145 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:58:07,145 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:58:07,150 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:58:07,879 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:58:07,880 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:58:10,600 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:58:10,601 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:58:10,601 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:58:10,606 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:58:10,881 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:58:10,937 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:58:11,884 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:58:12,606 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:58:15,638 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:58:16,160 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:58:16,161 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:58:16,161 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:58:16,166 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:58:16,887 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:58:17,887 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:58:18,202 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:58:20,938 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:58:21,973 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:58:21,974 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:58:21,974 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:58:21,979 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:58:22,891 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:58:23,894 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:58:23,980 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:58:25,939 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:58:26,265 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:58:26,265 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:58:26,266 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:58:26,270 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:58:26,895 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:58:27,056 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 19:58:27,898 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:58:29,057 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:58:30,638 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:58:31,004 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:58:31,005 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:58:31,005 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:58:31,010 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:58:31,902 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:58:31,902 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:58:35,047 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:58:35,100 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:58:35,101 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:58:35,101 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:58:35,106 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:58:35,906 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:58:35,906 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:58:35,940 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:58:40,107 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:58:40,674 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:58:40,675 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:58:40,675 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:58:40,680 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:58:40,909 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:58:40,941 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:58:41,912 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:58:44,373 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:58:44,374 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:58:44,374 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:58:44,378 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:58:44,914 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:58:45,379 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:58:45,638 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:58:45,915 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:58:47,483 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:58:47,483 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:58:47,483 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:58:47,488 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:58:47,916 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:58:49,920 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:58:50,526 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:58:50,942 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:58:54,077 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:58:54,078 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:58:54,078 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:58:54,083 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:58:54,923 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:58:55,925 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:58:55,942 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:58:56,084 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:58:57,058 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 19:58:57,180 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:58:57,181 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:58:57,181 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:58:57,186 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:58:57,928 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:58:57,929 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:59:00,638 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:59:01,187 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:59:02,050 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:59:02,051 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:59:02,051 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:59:02,056 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:59:02,931 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:59:03,934 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:59:06,174 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:59:06,174 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:59:06,175 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:59:06,179 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:59:06,524 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:59:06,937 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:59:07,180 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:59:07,939 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:59:10,096 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:59:10,096 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:59:10,097 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:59:10,102 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:59:10,941 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:59:11,525 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:59:11,944 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:59:13,102 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:59:14,397 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:59:14,398 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:59:14,398 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:59:14,403 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:59:14,945 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:59:15,639 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:59:15,948 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:59:18,403 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:59:21,526 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:59:23,040 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:59:23,041 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:59:23,041 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:59:23,045 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:59:23,953 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:59:23,953 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:59:24,046 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:59:26,526 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:59:27,060 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 19:59:28,645 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:59:28,646 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:59:28,646 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:59:28,650 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:59:28,956 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:59:29,651 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:59:29,959 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:59:30,639 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:59:33,358 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:59:33,358 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:59:33,358 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:59:33,363 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:59:33,961 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:59:35,402 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:59:35,964 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:59:36,527 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:59:36,991 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:59:36,992 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:59:36,992 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:59:36,997 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:59:37,967 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:59:37,968 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:59:40,998 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:59:41,470 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:59:41,470 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:59:41,471 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:59:41,475 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:59:41,528 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:59:41,970 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:59:43,973 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:59:45,639 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:59:46,476 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:59:50,128 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:59:50,129 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:59:50,129 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:59:50,134 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:59:50,978 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:59:51,529 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:59:51,979 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:59:52,170 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:59:54,507 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:59:54,508 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:59:54,508 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:59:54,513 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:59:54,981 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:59:55,983 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 19:59:56,529 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 19:59:57,065 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 19:59:58,044 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:59:58,045 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 19:59:58,046 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:59:58,046 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:59:58,050 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:59:58,985 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 19:59:59,987 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:00:00,639 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:00:02,514 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:00:02,514 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:00:02,515 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:00:02,519 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:00:02,990 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:00:03,558 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:00:03,992 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:00:06,530 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:00:08,559 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:00:11,203 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:00:11,204 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:00:11,204 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:00:11,209 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:00:11,531 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:00:12,000 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:00:12,000 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:00:14,209 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:00:15,639 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:00:17,086 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:00:17,086 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:00:17,087 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:00:17,092 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:00:18,005 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:00:18,006 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:00:20,130 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:00:20,392 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:00:20,393 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:00:20,393 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:00:20,398 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:00:21,007 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:00:21,532 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:00:22,010 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:00:24,144 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:00:24,145 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:00:24,145 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:00:24,150 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:00:25,012 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:00:25,150 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:00:26,015 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:00:26,533 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:00:27,064 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 20:00:29,008 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:00:29,009 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:00:29,009 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:00:29,015 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:00:29,016 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:00:30,019 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:00:30,639 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:00:31,015 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:00:32,457 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:00:32,458 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:00:32,458 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:00:32,463 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:00:33,021 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:00:34,024 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:00:36,132 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:00:36,133 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:00:36,134 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:00:36,134 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:00:36,139 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:00:36,534 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:00:37,026 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:00:38,028 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:00:39,678 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:00:39,678 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:00:39,679 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:00:39,683 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:00:40,029 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:00:41,535 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:00:41,684 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:00:42,032 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:00:43,410 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:00:43,411 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:00:43,411 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:00:43,416 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:00:44,034 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:00:45,639 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:00:46,037 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:00:46,892 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:00:46,892 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:00:46,893 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:00:46,893 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:00:46,898 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:00:47,038 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:00:48,040 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:00:51,033 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:00:51,034 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:00:51,034 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:00:51,039 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:00:51,043 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:00:51,536 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:00:52,039 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:00:52,046 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:00:54,366 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:00:54,367 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:00:54,367 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:00:54,372 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:00:55,048 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:00:56,050 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:00:56,536 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:00:57,070 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 20:00:57,071 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:00:57,702 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:00:57,702 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:00:57,702 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:00:57,707 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:00:58,051 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:01:00,054 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:01:00,639 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:01:00,922 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:01:00,922 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:01:00,922 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:01:00,927 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:01:01,055 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:01:02,057 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:01:02,966 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:01:05,252 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:01:05,253 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:01:05,253 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:01:05,257 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:01:06,063 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:01:06,063 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:01:06,537 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:01:08,258 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:01:11,112 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:01:11,113 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:01:11,113 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:01:11,118 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:01:11,538 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:01:12,069 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:01:12,069 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:01:14,118 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:01:15,631 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:01:15,631 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:01:15,632 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:01:15,636 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:01:15,639 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:01:16,071 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:01:18,074 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:01:19,383 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:01:19,384 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:01:19,384 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:01:19,384 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:01:19,389 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:01:20,077 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:01:20,078 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:01:21,539 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:01:23,518 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:01:23,519 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:01:23,519 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:01:23,524 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:01:24,080 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:01:24,525 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:01:26,083 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:01:26,540 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:01:27,069 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 20:01:28,945 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:01:28,946 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:01:28,946 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:01:28,951 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:01:29,085 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:01:29,951 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:01:30,087 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:01:30,639 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:01:33,517 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:01:33,517 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:01:33,518 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:01:33,522 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:01:34,090 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:01:35,558 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:01:36,093 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:01:36,541 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:01:37,450 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:01:37,450 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:01:37,451 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:01:37,455 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:01:38,096 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:01:38,097 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:01:41,456 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:01:41,511 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:01:41,512 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:01:41,512 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:01:41,517 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:01:41,541 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:01:42,099 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:01:44,102 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:01:45,639 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:01:46,518 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:01:50,115 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:01:50,116 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:01:50,116 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:01:50,120 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:01:51,108 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:01:51,542 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:01:52,110 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:01:52,158 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:01:55,974 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:01:55,975 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:01:55,975 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:01:55,980 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:01:56,113 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:01:56,543 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:01:57,074 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 20:01:58,075 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:01:58,116 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:01:59,194 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:01:59,195 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:01:59,195 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:01:59,200 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:02:00,119 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:02:00,119 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:02:00,639 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:02:03,201 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:02:05,033 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:02:05,034 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:02:05,034 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:02:05,039 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:02:05,122 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:02:06,126 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:02:06,544 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:02:09,079 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:02:09,370 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:02:09,371 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:02:09,371 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:02:09,376 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:02:10,130 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:02:10,130 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:02:11,544 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:02:12,903 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:02:12,903 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:02:12,904 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:02:12,909 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:02:13,132 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:02:14,135 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:02:14,909 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:02:15,640 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:02:19,910 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:02:21,545 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:02:21,789 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:02:21,790 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:02:21,790 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:02:21,795 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:02:22,140 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:02:24,143 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:02:25,796 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:02:26,253 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:02:26,254 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:02:26,254 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:02:26,259 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:02:26,546 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:02:27,073 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 20:02:27,145 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:02:28,147 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:02:30,595 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:02:30,596 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:02:30,596 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:02:30,601 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:02:30,640 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:02:31,149 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:02:31,602 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:02:32,152 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:02:35,962 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:02:35,962 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:02:35,963 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:02:35,968 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:02:36,154 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:02:36,547 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:02:37,006 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:02:38,157 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:02:39,850 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:02:39,851 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:02:39,851 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:02:39,856 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:02:40,158 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:02:41,548 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:02:42,161 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:02:42,856 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:02:43,551 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:02:43,552 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:02:43,552 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:02:43,557 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:02:44,162 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:02:45,640 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:02:46,166 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:02:48,558 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:02:49,127 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:02:49,128 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:02:49,128 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:02:49,134 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:02:49,168 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:02:50,171 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:02:51,549 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:02:53,515 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:02:53,516 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:02:53,516 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:02:53,521 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:02:54,175 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:02:54,175 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:02:54,522 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:02:56,549 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:02:57,079 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 20:02:57,199 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:02:57,200 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:02:57,200 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:02:57,205 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:02:58,179 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:02:58,180 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:03:00,206 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:03:00,640 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:03:01,684 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:03:01,685 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:03:01,685 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:03:01,691 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:03:02,182 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:03:04,185 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:03:05,625 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:03:05,626 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:03:05,626 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:03:05,626 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:03:05,631 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:03:06,187 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:03:06,550 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:03:08,190 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:03:10,420 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:03:10,421 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:03:10,421 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:03:10,426 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:03:11,192 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:03:11,426 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:03:11,551 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:03:12,195 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:03:15,640 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:03:15,740 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:03:15,740 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:03:15,740 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:03:15,745 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:03:16,197 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:03:16,782 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:03:18,200 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:03:18,979 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:03:18,980 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:03:18,980 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:03:18,986 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:03:19,201 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:03:20,203 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:03:21,552 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:03:21,986 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:03:22,472 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:03:22,473 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:03:22,473 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:03:22,478 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:03:23,206 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:03:24,208 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:03:26,047 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:03:26,048 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:03:26,048 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:03:26,053 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:03:26,209 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:03:26,553 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:03:27,053 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:03:27,078 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 20:03:28,213 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:03:30,117 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:03:30,118 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:03:30,118 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:03:30,123 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:03:30,214 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:03:30,640 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:03:32,124 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:03:32,215 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:03:33,598 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:03:33,598 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:03:33,598 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:03:33,603 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:03:34,217 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:03:36,220 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:03:36,554 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:03:37,643 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:03:38,091 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:03:38,092 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:03:38,092 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:03:38,097 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:03:38,221 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:03:40,224 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:03:41,446 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:03:41,447 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:03:41,447 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:03:41,452 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:03:41,554 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:03:42,227 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:03:42,227 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:03:43,453 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:03:44,806 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:03:44,807 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:03:44,807 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:03:44,811 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:03:45,229 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:03:45,640 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:03:46,232 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:03:48,812 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:03:49,735 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:03:49,736 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:03:49,736 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:03:49,741 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:03:50,236 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:03:51,556 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:03:52,239 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:03:54,779 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:03:56,556 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:03:57,083 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 20:03:58,233 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:03:58,234 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:03:58,234 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:03:58,239 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:03:58,242 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:04:00,239 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:04:00,244 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:04:00,640 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:04:02,276 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:04:02,277 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:04:02,277 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:04:02,282 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:04:03,245 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:04:04,248 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:04:05,319 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:04:06,122 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:04:06,123 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:04:06,123 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:04:06,128 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:04:06,250 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:04:06,557 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:04:08,254 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:04:11,128 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:04:11,508 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:04:11,508 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:04:11,509 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:04:11,514 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:04:11,558 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:04:12,258 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:04:12,258 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:04:15,317 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:04:15,317 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:04:15,318 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:04:15,323 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:04:15,640 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:04:16,262 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:04:16,262 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:04:16,323 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:04:20,546 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:04:20,546 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:04:20,546 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:04:20,551 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:04:21,265 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:04:21,559 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:04:21,590 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:04:22,268 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:04:26,444 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:04:26,445 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:04:26,445 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:04:26,450 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:04:26,652 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:04:27,082 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 20:04:27,083 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:04:27,271 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:04:28,274 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:04:30,640 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:04:30,732 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:04:30,732 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:04:30,733 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:04:30,737 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:04:31,276 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:04:32,278 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:04:32,774 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:04:34,256 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:04:34,257 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:04:34,257 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:04:34,263 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:04:34,279 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:04:35,653 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:04:36,282 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:04:37,723 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:04:37,724 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:04:37,724 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:04:37,729 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:04:38,284 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:04:38,729 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:04:40,287 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:04:40,653 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:04:42,644 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:04:42,645 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:04:42,645 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:04:42,650 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:04:43,289 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:04:44,291 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:04:44,650 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:04:45,640 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:04:47,811 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:04:47,811 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:04:47,812 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:04:47,816 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:04:48,294 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:04:49,854 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:04:50,298 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:04:50,654 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:04:53,524 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:04:53,524 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:04:53,525 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:04:53,530 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:04:54,303 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:04:54,303 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:04:55,530 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:04:55,655 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:04:56,900 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:04:56,901 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:04:56,901 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:04:56,906 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:04:57,088 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 20:04:57,305 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:04:58,307 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:05:00,283 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:05:00,283 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:05:00,284 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:05:00,288 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:05:00,308 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:05:00,640 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:05:01,289 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:05:02,311 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:05:03,454 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:05:03,454 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:05:03,454 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:05:03,460 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:05:04,315 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:05:04,315 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:05:05,656 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:05:06,498 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:05:07,026 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:05:07,027 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:05:07,027 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:05:07,032 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:05:07,318 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:05:08,320 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:05:10,657 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:05:10,756 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:05:10,756 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:05:10,757 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:05:10,762 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:05:11,322 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:05:11,762 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:05:12,323 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:05:14,314 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:05:14,315 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:05:14,315 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:05:14,321 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:05:14,324 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:05:15,641 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:05:16,327 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:05:17,321 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:05:18,865 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:05:18,866 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:05:18,866 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:05:18,871 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:05:19,329 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:05:20,332 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:05:20,658 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:05:22,103 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:05:22,104 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:05:22,104 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:05:22,109 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:05:22,333 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:05:23,109 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:05:24,334 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:05:25,398 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:05:25,399 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:05:25,399 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:05:25,404 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:05:25,658 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:05:26,337 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:05:26,337 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:05:27,087 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 20:05:28,935 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:05:28,935 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:05:28,936 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:05:28,936 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:05:28,941 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:05:29,339 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:05:30,341 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:05:30,641 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:05:33,941 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:05:34,745 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:05:34,745 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:05:34,746 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:05:34,751 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:05:35,344 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:05:35,660 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:05:36,347 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:05:38,837 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:05:38,838 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:05:38,838 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:05:38,844 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:05:39,348 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:05:39,844 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:05:40,351 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:05:40,660 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:05:43,171 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:05:43,172 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:05:43,172 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:05:43,177 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:05:43,353 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:05:44,356 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:05:45,178 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:05:45,641 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:05:46,594 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:05:46,594 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:05:46,595 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:05:46,599 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:05:47,358 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:05:48,360 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:05:50,639 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:05:50,661 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:05:51,114 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:05:51,116 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:05:51,116 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:05:51,121 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:05:51,364 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:05:52,366 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:05:55,204 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:05:55,204 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:05:55,205 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:05:55,209 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:05:55,368 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:05:55,662 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:05:56,210 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:05:56,371 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:05:57,089 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 20:05:58,930 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:05:58,931 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:05:58,931 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:05:58,936 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:05:59,373 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:06:00,375 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:06:00,641 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:06:01,937 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:06:05,635 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:06:05,636 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:06:05,636 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:06:05,641 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:06:05,678 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:06:06,383 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:06:06,383 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:06:07,678 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:06:08,755 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:06:08,756 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:06:08,756 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:06:08,761 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:06:09,385 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:06:10,387 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:06:10,663 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:06:12,762 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:06:14,573 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:06:14,573 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:06:14,574 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:06:14,578 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:06:15,390 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:06:15,641 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:06:16,393 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:06:18,184 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:06:18,184 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:06:18,184 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:06:18,184 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:06:18,189 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:06:18,394 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:06:20,398 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:06:20,665 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:06:21,647 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:06:21,647 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:06:21,647 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:06:21,652 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:06:22,401 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:06:22,402 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:06:23,653 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:06:25,665 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:06:27,091 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 20:06:29,092 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:06:30,231 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:06:30,232 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:06:30,233 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:06:30,237 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:06:30,406 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:06:30,641 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:06:32,409 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:06:34,238 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:06:34,510 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:06:34,511 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:06:34,511 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:06:34,517 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:06:35,411 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:06:35,666 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:06:36,414 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:06:39,059 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:06:39,060 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:06:39,060 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:06:39,065 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:06:39,416 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:06:40,065 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:06:40,418 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:06:40,667 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:06:44,013 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:06:44,014 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:06:44,014 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:06:44,019 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:06:44,421 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:06:45,641 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:06:46,020 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:06:46,424 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:06:47,563 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:06:47,563 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:06:47,563 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:06:47,569 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:06:48,427 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:06:48,428 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:06:50,668 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:06:51,575 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:06:51,575 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:06:51,575 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:06:51,576 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:06:51,580 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:06:52,433 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:06:52,433 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:06:55,668 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:06:56,264 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:06:56,264 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:06:56,265 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:06:56,269 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:06:56,436 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:06:57,093 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 20:06:57,094 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:06:58,439 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:07:00,210 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:07:00,211 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:07:00,211 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:07:00,217 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:07:00,440 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:07:00,641 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:07:02,217 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:07:02,441 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:07:04,270 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:07:04,270 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:07:04,271 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:07:04,276 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:07:04,442 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:07:05,669 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:07:06,447 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:07:07,314 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:07:08,603 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:07:08,604 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:07:08,604 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:07:08,609 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:07:09,449 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:07:10,451 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:07:10,670 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:07:12,609 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:07:13,361 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:07:13,362 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:07:13,362 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:07:13,367 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:07:13,453 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:07:14,456 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:07:15,641 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:07:17,605 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:07:17,606 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:07:17,606 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:07:17,611 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:07:17,612 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:07:18,460 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:07:18,460 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:07:20,671 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:07:20,957 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:07:20,958 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:07:20,958 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:07:20,964 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:07:21,462 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:07:22,465 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:07:22,964 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:07:25,365 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:07:25,365 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:07:25,366 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:07:25,370 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:07:25,467 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:07:25,671 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:07:26,469 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:07:27,095 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 20:07:28,096 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:07:29,616 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:07:29,617 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:07:29,617 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:07:29,622 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:07:30,474 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:07:30,474 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:07:30,641 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:07:33,598 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:07:33,599 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:07:33,599 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:07:33,599 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:07:33,604 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:07:34,478 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:07:34,478 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:07:35,672 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:07:37,597 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:07:37,598 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:07:37,599 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:07:37,604 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:07:38,483 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:07:38,483 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:07:38,605 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:07:40,673 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:07:41,431 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:07:41,432 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:07:41,432 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:07:41,437 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:07:41,484 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:07:42,487 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:07:44,438 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:07:44,781 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:07:44,782 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:07:44,782 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:07:44,787 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:07:45,489 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:07:45,641 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:07:46,492 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:07:48,563 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:07:48,564 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:07:48,564 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:07:48,569 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:07:49,495 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:07:49,606 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:07:50,497 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:07:50,674 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:07:54,607 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:07:55,674 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:07:56,905 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:07:56,906 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:07:56,906 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:07:56,911 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:07:57,098 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 20:07:57,502 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:07:58,504 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:08:00,099 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:08:00,216 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:08:00,216 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:08:00,216 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:08:00,221 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:08:00,506 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:08:00,642 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:08:02,509 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:08:04,903 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:08:04,904 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:08:04,904 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:08:04,909 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:08:05,511 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:08:05,675 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:08:05,946 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:08:06,515 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:08:08,492 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:08:08,492 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:08:08,493 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:08:08,498 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:08:08,516 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:08:10,519 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:08:10,676 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:08:11,498 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:08:13,117 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:08:13,121 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:08:13,121 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:08:13,126 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:08:13,521 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:08:14,524 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:08:15,642 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:08:17,126 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:08:20,677 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:08:21,902 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:08:21,903 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:08:21,903 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:08:21,908 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:08:22,529 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:08:22,529 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:08:22,909 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:08:25,678 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:08:25,755 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:08:25,756 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:08:25,756 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:08:25,761 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:08:26,533 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:08:26,533 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:08:27,100 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 20:08:28,101 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:08:29,254 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:08:29,255 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:08:29,255 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:08:29,260 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:08:29,535 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:08:30,537 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:08:30,642 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:08:32,767 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:08:32,768 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:08:32,768 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:08:32,772 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:08:33,539 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:08:33,810 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:08:34,542 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:08:35,679 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:08:36,074 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:08:36,075 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:08:36,075 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:08:36,080 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:08:36,543 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:08:38,546 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:08:39,081 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:08:39,797 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:08:39,798 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:08:39,798 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:08:39,803 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:08:40,550 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:08:40,550 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:08:40,679 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:08:43,096 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:08:43,097 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:08:43,097 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:08:43,102 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:08:43,551 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:08:44,102 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:08:44,552 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:08:45,642 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:08:46,433 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:08:46,434 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:08:46,434 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:08:46,439 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:08:46,553 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:08:48,556 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:08:49,479 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:08:50,022 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:08:50,023 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:08:50,023 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:08:50,028 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:08:50,559 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:08:50,680 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:08:52,562 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:08:53,940 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:08:53,941 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:08:53,941 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:08:53,946 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:08:54,565 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:08:54,565 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:08:54,947 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:08:55,681 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:08:57,102 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 20:08:58,302 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:08:58,303 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:08:58,303 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:08:58,308 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:08:58,567 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:09:00,308 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:09:00,571 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:09:00,642 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:09:02,429 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:09:02,430 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:09:02,430 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:09:02,435 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:09:02,572 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:09:04,575 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:09:05,475 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:09:05,682 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:09:06,200 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:09:06,200 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:09:06,201 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:09:06,205 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:09:06,577 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:09:08,580 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:09:10,785 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:09:10,786 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:09:10,786 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:09:10,786 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:09:10,791 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:09:10,894 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:09:11,582 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:09:12,585 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:09:14,524 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:09:14,527 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:09:14,528 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:09:14,533 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:09:14,587 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:09:15,642 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:09:16,534 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:09:16,590 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:09:20,490 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:09:20,491 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:09:20,491 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:09:20,496 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:09:20,592 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:09:20,895 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:09:21,534 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:09:22,596 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:09:23,785 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:09:23,786 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:09:23,786 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:09:23,791 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:09:24,599 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:09:24,599 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:09:25,896 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:09:26,791 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:09:27,105 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 20:09:27,694 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:09:27,694 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:09:27,695 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:09:27,699 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:09:28,604 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:09:28,604 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:09:30,642 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:09:31,668 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:09:31,668 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:09:31,668 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:09:31,673 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:09:32,608 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:09:32,608 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:09:32,710 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:09:35,781 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:09:35,781 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:09:35,782 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:09:35,787 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:09:35,897 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:09:36,613 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:09:36,614 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:09:37,787 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:09:40,880 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:09:40,881 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:09:40,881 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:09:40,886 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:09:40,898 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:09:41,616 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:09:42,619 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:09:42,886 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:09:44,822 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:09:44,823 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:09:44,823 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:09:44,829 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:09:45,621 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:09:45,642 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:09:46,624 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:09:48,830 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:09:50,899 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:09:53,369 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:09:53,370 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:09:53,370 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:09:53,375 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:09:53,630 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:09:54,375 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:09:54,632 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:09:55,899 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:09:57,107 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 20:09:58,059 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:09:58,060 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:09:58,060 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:09:58,065 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:09:58,636 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:09:58,637 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:10:00,066 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:10:00,642 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:10:02,954 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:10:02,954 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:10:02,954 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:10:02,959 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:10:03,639 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:10:04,642 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:10:05,900 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:10:05,997 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:10:06,410 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:10:06,411 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:10:06,411 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:10:06,416 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:10:06,644 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:10:08,647 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:10:09,843 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:10:09,844 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:10:09,844 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:10:09,849 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:10:10,651 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:10:10,651 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:10:10,901 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:10:11,849 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:10:14,214 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:10:14,215 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:10:14,215 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:10:14,220 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:10:14,653 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:10:15,642 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:10:16,656 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:10:17,220 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:10:20,902 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:10:22,221 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:10:22,649 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:10:22,650 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:10:22,650 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:10:22,655 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:10:22,659 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:10:24,663 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:10:25,902 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:10:27,109 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 20:10:28,110 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:10:28,796 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:10:28,796 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:10:28,797 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:10:28,801 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:10:29,666 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:10:30,642 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:10:30,668 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:10:32,246 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:10:32,247 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:10:32,247 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:10:32,251 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:10:32,670 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:10:33,289 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:10:34,673 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:10:35,903 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:10:35,923 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:10:35,924 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:10:35,924 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:10:35,929 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:10:36,676 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:10:36,676 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:10:38,930 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:10:40,059 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:10:40,059 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:10:40,059 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:10:40,064 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:10:40,680 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:10:40,680 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:10:40,904 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:10:44,065 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:10:45,642 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:10:45,907 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:10:45,908 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:10:45,908 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:10:45,913 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:10:46,686 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:10:46,686 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:10:49,657 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:10:49,658 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:10:49,658 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:10:49,658 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:10:49,664 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:10:49,688 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:10:50,690 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:10:50,905 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:10:52,838 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:10:52,839 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:10:52,839 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:10:52,844 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:10:53,692 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:10:54,692 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:10:54,844 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:10:55,906 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:10:57,111 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 20:11:00,113 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:11:00,642 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:11:04,217 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:11:04,218 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:11:04,218 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:11:04,223 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:11:04,699 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:11:05,261 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:11:05,907 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:11:06,703 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:11:09,553 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:11:09,554 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:11:09,554 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:11:09,559 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:11:09,704 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:11:10,559 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:11:10,705 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:11:10,907 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:11:13,762 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:11:13,762 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:11:13,762 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:11:13,767 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:11:14,709 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:11:14,710 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:11:15,643 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:11:15,768 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:11:17,606 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:11:17,607 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:11:17,607 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:11:17,612 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:11:17,711 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:11:18,714 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:11:20,908 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:11:21,650 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:11:22,607 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:11:22,608 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:11:22,608 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:11:22,613 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:11:22,716 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:11:24,719 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:11:25,909 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:11:25,941 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:11:25,942 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:11:25,942 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:11:25,947 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:11:26,723 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:11:26,723 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:11:26,947 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:11:27,116 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 20:11:29,019 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:11:29,019 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:11:29,020 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:11:29,025 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:11:29,724 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:11:30,643 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:11:30,727 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:11:32,025 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:11:34,973 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:11:34,974 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:11:34,974 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:11:34,979 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:11:35,730 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:11:35,910 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:11:36,732 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:11:38,017 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:11:39,058 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:11:39,059 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:11:39,059 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:11:39,064 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:11:39,734 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:11:40,737 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:11:40,911 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:11:42,845 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:11:42,846 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:11:42,846 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:11:42,851 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:11:43,739 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:11:43,851 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:11:44,742 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:11:45,643 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:11:48,247 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:11:48,247 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:11:48,248 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:11:48,252 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:11:48,744 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:11:49,289 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:11:50,749 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:11:50,912 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:11:52,964 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:11:52,965 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:11:52,966 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:11:52,971 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:11:53,750 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:11:54,751 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:11:54,971 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:11:55,912 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:11:56,547 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:11:56,547 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:11:56,548 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:11:56,552 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:11:56,752 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:11:57,115 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 20:11:58,756 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:12:00,117 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:12:00,643 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:12:00,801 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:12:00,801 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:12:00,801 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:12:00,806 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:12:01,757 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:12:02,760 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:12:04,533 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:12:04,533 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:12:04,534 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:12:04,539 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:12:04,761 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:12:05,539 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:12:05,913 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:12:06,768 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:12:10,539 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:12:10,914 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:12:13,108 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:12:13,109 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:12:13,109 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:12:13,114 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:12:13,772 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:12:14,774 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:12:15,643 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:12:16,115 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:12:16,698 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:12:16,699 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:12:16,699 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:12:16,704 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:12:16,775 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:12:18,778 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:12:20,505 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:12:20,505 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:12:20,506 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:12:20,512 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:12:20,780 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:12:20,915 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:12:21,512 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:12:22,783 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:12:24,385 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:12:24,386 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:12:24,386 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:12:24,391 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:12:24,784 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:12:25,916 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:12:26,785 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:12:27,121 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 20:12:27,122 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:12:27,834 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:12:27,835 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:12:27,835 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:12:27,839 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:12:28,789 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:12:28,789 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:12:30,643 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:12:31,330 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:12:31,331 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:12:31,331 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:12:31,336 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:12:31,791 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:12:32,373 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:12:32,793 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:12:35,112 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:12:35,112 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:12:35,112 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:12:35,117 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:12:35,795 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:12:35,917 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:12:36,797 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:12:38,118 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:12:38,795 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:12:38,796 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:12:38,796 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:12:38,801 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:12:38,801 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:12:40,804 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:12:40,917 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:12:43,802 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:12:44,252 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:12:44,253 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:12:44,253 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:12:44,257 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:12:44,807 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:12:45,643 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:12:46,810 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:12:48,578 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:12:48,578 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:12:48,579 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:12:48,583 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:12:48,811 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:12:49,621 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:12:50,816 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:12:50,918 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:12:54,622 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:12:55,428 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:12:55,428 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:12:55,429 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:12:55,434 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:12:55,819 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:12:55,919 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:12:56,821 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:12:57,120 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 20:12:58,702 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:12:58,702 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:12:58,703 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:12:58,707 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:12:58,823 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:12:59,708 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:13:00,643 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:13:00,824 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:13:04,020 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:13:04,020 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:13:04,020 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:13:04,025 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:13:04,826 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:13:05,061 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:13:05,920 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:13:06,831 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:13:08,702 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:13:08,703 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:13:08,703 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:13:08,708 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:13:08,832 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:13:10,709 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:13:10,835 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:13:10,920 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:13:12,426 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:13:12,427 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:13:12,427 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:13:12,432 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:13:12,836 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:13:14,840 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:13:15,643 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:13:16,346 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:13:16,347 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:13:16,347 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:13:16,347 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:13:16,352 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:13:16,841 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:13:18,844 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:13:20,921 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:13:21,390 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:13:24,867 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:13:24,868 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:13:24,868 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:13:24,873 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:13:25,848 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:13:25,922 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:13:26,851 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:13:26,874 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:13:27,125 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 20:13:30,643 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:13:32,127 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:13:34,039 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:13:34,040 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:13:34,040 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:13:34,045 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:13:34,855 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:13:35,923 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:13:36,859 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:13:38,082 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:13:39,870 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:13:39,871 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:13:39,871 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:13:39,876 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:13:40,861 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:13:40,924 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:13:42,864 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:13:43,877 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:13:44,266 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:13:44,266 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:13:44,266 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:13:44,271 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:13:44,865 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:13:45,643 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:13:46,869 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:13:47,926 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:13:47,927 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:13:47,927 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:13:47,932 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:13:48,870 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:13:48,969 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:13:50,875 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:13:51,321 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:13:52,586 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:13:52,586 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:13:52,587 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:13:52,591 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:13:52,876 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:13:54,592 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:13:54,879 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:13:56,321 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:13:56,336 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:13:56,337 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:13:56,338 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:13:56,343 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:13:56,880 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:13:57,125 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 20:13:58,884 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:14:00,126 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:14:00,430 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:14:00,431 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:14:00,431 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:14:00,436 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:14:00,643 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:14:00,885 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:14:02,888 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:14:04,093 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:14:04,094 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:14:04,094 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:14:04,099 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:14:04,889 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:14:05,137 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:14:06,322 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:14:06,894 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:14:10,138 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:14:10,666 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:14:10,667 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:14:10,667 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:14:10,672 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:14:10,896 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:14:11,323 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:14:12,899 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:14:14,190 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:14:14,191 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:14:14,191 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:14:14,196 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:14:14,900 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:14:15,196 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:14:15,643 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:14:16,904 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:14:17,838 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:14:17,838 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:14:17,838 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:14:17,844 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:14:17,904 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:14:20,881 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:14:20,908 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:14:20,953 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:14:20,954 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:14:20,954 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:14:20,959 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:14:21,324 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:14:21,909 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:14:22,912 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:14:25,959 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:14:26,325 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:14:27,130 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 20:14:29,214 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:14:29,214 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:14:29,215 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:14:29,219 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:14:29,916 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:14:30,644 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:14:30,918 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:14:31,220 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:14:33,365 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:14:33,365 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:14:33,366 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:14:33,371 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:14:33,920 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:14:34,922 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:14:36,326 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:14:36,409 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:14:41,326 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:14:41,410 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:14:41,581 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:14:41,582 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:14:41,582 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:14:41,587 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:14:41,927 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:14:42,930 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:14:45,049 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:14:45,050 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:14:45,050 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:14:45,055 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:14:45,644 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:14:45,932 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:14:46,932 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:14:47,056 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:14:49,443 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:14:49,444 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:14:49,444 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:14:49,449 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:14:49,935 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:14:50,938 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:14:51,327 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:14:52,485 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:14:53,386 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:14:53,387 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:14:53,387 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:14:53,392 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:14:53,940 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:14:54,942 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:14:56,328 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:14:56,632 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:14:56,633 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:14:56,633 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:14:56,638 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:14:56,943 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:14:57,129 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 20:14:58,130 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:14:58,946 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:15:00,399 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:15:00,399 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:15:00,400 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:15:00,405 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:15:00,644 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:15:00,947 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:15:02,950 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:15:03,405 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:15:04,130 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:15:04,131 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:15:04,131 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:15:04,136 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:15:04,951 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:15:06,329 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:15:06,956 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:15:08,188 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:15:08,189 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:15:08,189 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:15:08,194 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:15:08,957 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:15:09,194 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:15:10,964 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:15:11,330 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:15:11,912 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:15:11,913 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:15:11,913 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:15:11,918 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:15:11,964 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:15:12,967 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:15:14,918 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:15:15,644 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:15:16,768 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:15:16,769 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:15:16,769 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:15:16,774 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:15:16,969 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:15:18,972 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:15:20,018 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:15:20,018 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:15:20,019 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:15:20,019 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:15:20,023 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:15:20,974 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:15:21,331 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:15:22,977 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:15:25,024 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:15:26,331 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:15:27,132 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 20:15:28,540 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:15:28,541 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:15:28,541 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:15:28,546 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:15:28,981 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:15:30,546 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:15:30,644 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:15:30,984 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:15:33,011 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:15:33,011 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:15:33,011 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:15:33,016 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:15:33,986 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:15:34,988 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:15:36,053 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:15:36,332 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:15:36,439 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:15:36,440 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:15:36,440 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:15:36,445 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:15:36,990 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:15:39,077 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:15:40,043 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:15:40,044 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:15:40,044 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:15:40,049 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:15:40,077 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:15:41,333 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:15:42,050 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:15:43,081 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:15:43,539 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:15:43,540 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:15:43,540 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:15:43,545 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:15:44,082 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:15:45,084 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:15:45,644 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:15:46,857 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:15:46,858 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:15:46,858 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:15:46,863 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:15:47,086 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:15:47,901 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:15:49,087 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:15:51,334 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:15:52,902 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:15:54,954 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:15:54,955 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:15:54,956 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:15:54,961 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:15:55,092 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:15:56,335 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:15:57,095 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:15:57,134 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 20:15:58,135 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:15:58,299 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:15:58,300 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:15:58,300 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:15:58,305 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:15:59,096 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:16:00,644 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:16:01,099 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:16:03,305 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:16:05,588 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:16:05,589 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:16:05,589 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:16:05,594 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:16:06,102 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:16:06,336 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:16:07,106 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:16:08,629 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:16:10,068 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:16:10,069 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:16:10,069 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:16:10,074 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:16:10,108 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:16:11,336 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:16:13,111 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:16:14,074 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:16:14,292 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:16:14,293 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:16:14,293 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:16:14,298 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:16:15,113 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:16:15,644 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:16:17,114 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:16:18,247 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:16:18,248 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:16:18,248 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:16:18,253 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:16:19,115 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:16:19,289 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:16:21,119 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:16:21,337 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:16:21,773 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:16:21,774 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:16:21,774 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:16:21,779 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:16:22,119 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:16:23,122 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:16:24,780 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:16:25,647 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:16:25,647 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:16:25,648 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:16:25,652 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:16:26,124 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:16:26,338 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:16:27,127 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:16:27,136 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 20:16:29,412 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:16:29,413 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:16:29,413 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:16:29,418 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:16:30,129 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:16:30,418 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:16:30,644 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:16:31,131 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:16:33,803 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:16:33,804 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:16:33,804 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:16:33,809 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:16:34,133 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:16:35,134 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:16:35,849 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:16:36,339 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:16:37,299 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:16:37,299 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:16:37,300 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:16:37,304 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:16:38,135 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:16:39,138 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:16:40,882 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:16:40,882 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:16:40,883 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:16:40,883 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:16:40,888 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:16:41,139 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:16:41,340 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:16:43,142 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:16:44,602 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:16:44,602 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:16:44,603 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:16:44,608 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:16:45,144 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:16:45,645 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:16:46,608 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:16:47,147 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:16:47,872 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:16:47,873 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:16:47,873 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:16:47,878 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:16:48,148 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:16:49,150 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:16:51,341 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:16:51,914 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:16:52,809 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:16:52,810 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:16:52,810 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:16:52,815 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:16:53,154 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:16:55,157 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:16:56,342 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:16:57,138 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 20:16:57,139 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:17:00,645 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:17:01,354 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:17:01,354 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:17:01,354 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:17:01,361 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:17:02,161 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:17:02,397 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:17:03,163 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:17:05,906 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:17:05,907 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:17:05,907 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:17:05,912 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:17:06,165 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:17:06,342 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:17:07,169 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:17:07,913 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:17:10,822 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:17:10,823 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:17:10,823 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:17:10,828 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:17:11,171 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:17:11,343 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:17:13,173 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:17:13,828 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:17:14,765 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:17:14,766 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:17:14,766 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:17:14,770 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:17:15,174 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:17:15,645 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:17:17,177 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:17:19,771 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:17:19,889 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:17:19,889 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:17:19,889 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:17:19,896 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:17:20,179 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:17:21,181 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:17:21,344 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:17:24,930 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:17:25,628 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:17:25,629 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:17:25,630 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:17:25,635 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:17:26,185 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:17:26,345 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:17:27,140 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 20:17:27,187 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:17:30,142 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:17:30,645 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:17:33,797 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:17:33,798 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:17:33,798 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:17:33,803 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:17:34,191 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:17:35,194 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:17:35,841 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:17:36,346 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:17:38,779 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:17:38,779 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:17:38,780 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:17:38,784 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:17:39,196 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:17:41,198 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:17:41,346 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:17:41,785 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:17:42,555 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:17:42,555 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:17:42,555 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:17:42,560 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:17:43,199 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:17:45,202 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:17:45,645 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:17:47,561 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:17:48,352 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:17:48,358 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:17:48,358 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:17:48,363 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:17:49,205 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:17:51,209 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:17:51,347 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:17:52,035 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:17:52,035 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:17:52,036 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:17:52,040 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:17:52,209 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:17:53,041 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:17:53,210 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:17:55,800 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:17:55,800 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:17:55,801 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:17:55,805 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:17:56,212 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:17:56,348 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:17:57,142 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 20:17:57,214 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:17:58,143 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:17:59,905 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:17:59,906 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:17:59,906 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:17:59,911 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:18:00,216 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:18:00,645 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:18:01,218 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:18:03,912 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:18:06,349 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:18:07,486 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:18:07,487 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:18:07,487 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:18:07,492 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:18:08,224 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:18:09,224 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:18:09,493 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:18:11,345 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:18:11,346 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:18:11,346 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:18:11,349 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:18:11,351 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:18:12,226 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:18:13,229 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:18:15,351 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:18:15,627 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:18:15,628 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:18:15,628 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:18:15,633 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:18:15,645 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:18:16,231 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:18:17,233 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:18:20,168 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:18:20,168 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:18:20,169 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:18:20,173 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:18:20,235 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:18:21,209 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:18:21,350 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:18:23,239 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:18:23,382 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:18:23,383 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:18:23,383 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:18:23,388 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:18:24,239 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:18:25,242 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:18:26,351 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:18:26,388 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:18:27,144 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 20:18:30,645 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:18:31,965 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:18:31,966 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:18:31,966 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:18:31,966 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:18:31,971 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:18:32,246 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:18:33,249 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:18:35,330 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:18:35,331 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:18:35,331 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:18:35,336 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:18:36,250 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:18:36,352 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:18:37,251 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:18:37,337 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:18:41,015 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:18:41,016 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:18:41,016 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:18:41,022 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:18:41,253 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:18:41,353 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:18:43,022 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:18:43,256 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:18:45,645 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:18:45,679 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:18:45,680 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:18:45,680 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:18:45,685 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:18:46,258 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:18:47,261 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:18:48,722 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:18:49,742 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:18:49,742 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:18:49,743 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:18:49,748 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:18:50,264 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:18:51,266 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:18:51,591 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:18:53,270 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:18:53,271 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:18:53,271 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:18:53,276 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:18:54,268 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:18:54,276 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:18:55,270 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:18:56,592 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:18:57,149 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 20:18:58,594 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:18:58,597 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:18:58,598 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:18:58,602 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:18:59,272 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:18:59,603 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:19:00,645 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:19:01,275 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:19:01,870 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:19:01,871 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:19:01,871 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:19:01,876 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:19:02,276 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:19:03,279 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:19:04,913 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:19:05,348 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:19:05,349 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:19:05,349 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:19:05,354 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:19:06,280 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:19:06,593 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:19:07,284 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:19:10,355 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:19:11,593 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:19:13,969 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:19:13,970 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:19:13,970 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:19:13,975 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:19:14,288 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:19:15,291 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:19:15,645 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:19:15,975 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:19:18,321 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:19:18,322 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:19:18,322 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:19:18,327 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:19:19,293 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:19:21,295 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:19:21,365 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:19:21,594 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:19:22,255 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:19:22,256 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:19:22,256 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:19:22,261 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:19:22,295 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:19:25,299 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:19:26,595 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:19:27,148 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 20:19:27,150 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:19:30,645 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:19:30,825 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:19:30,825 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:19:30,825 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:19:30,830 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:19:31,303 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:19:32,865 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:19:33,306 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:19:34,369 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:19:34,370 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:19:34,370 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:19:34,375 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:19:35,308 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:19:36,596 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:19:37,313 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:19:37,687 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:19:37,688 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:19:37,688 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:19:37,693 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:19:38,313 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:19:38,694 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:19:39,316 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:19:41,460 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:19:41,461 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:19:41,461 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:19:41,466 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:19:41,597 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:19:42,319 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:19:43,321 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:19:44,467 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:19:45,646 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:19:47,332 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:19:47,332 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:19:47,332 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:19:47,337 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:19:48,327 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:19:49,330 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:19:50,374 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:19:51,580 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:19:51,580 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:19:51,581 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:19:51,585 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:19:51,598 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:19:52,333 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:19:53,335 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:19:54,988 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:19:54,989 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:19:54,990 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:19:54,994 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:19:55,337 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:19:55,995 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:19:56,598 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:19:57,154 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 20:19:57,340 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:19:58,291 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:19:58,292 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:19:58,292 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:19:58,297 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:19:58,343 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:20:00,646 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:20:01,297 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:20:01,346 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:20:01,567 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:20:01,568 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:20:01,568 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:20:01,573 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:20:02,348 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:20:03,351 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:20:05,067 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:20:05,067 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:20:05,068 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:20:05,073 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:20:05,352 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:20:06,599 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:20:07,073 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:20:07,358 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:20:10,833 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:20:10,833 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:20:10,834 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:20:10,840 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:20:11,361 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:20:11,600 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:20:12,840 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:20:13,366 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:20:13,591 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:20:13,591 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:20:13,591 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:20:13,596 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:20:14,366 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:20:15,369 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:20:15,646 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:20:18,371 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:20:18,372 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:20:18,372 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:20:18,372 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:20:18,377 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:20:19,374 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:20:21,377 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:20:21,601 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:20:23,414 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:20:26,601 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:20:26,989 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:20:26,989 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:20:26,989 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:20:26,994 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:20:27,153 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 20:20:27,384 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:20:29,154 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:20:29,389 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:20:30,281 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:20:30,282 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:20:30,282 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:20:30,287 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:20:30,390 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:20:30,646 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:20:31,392 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:20:33,262 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:20:33,262 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:20:33,263 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:20:33,268 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:20:33,394 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:20:34,305 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:20:35,399 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:20:36,602 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:20:39,146 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:20:39,147 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:20:39,147 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:20:39,151 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:20:39,402 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:20:40,152 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:20:41,407 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:20:41,603 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:20:42,902 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:20:42,903 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:20:42,903 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:20:42,908 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:20:43,408 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:20:45,413 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:20:45,646 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:20:45,908 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:20:49,659 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:20:49,659 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:20:49,660 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:20:49,664 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:20:50,420 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:20:51,423 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:20:51,604 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:20:51,701 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:20:53,201 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:20:53,201 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:20:53,202 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:20:53,206 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:20:53,424 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:20:55,427 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:20:56,604 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:20:56,619 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:20:56,620 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:20:56,620 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:20:56,625 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:20:57,158 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 20:20:57,159 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:20:57,429 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:20:59,434 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:21:00,063 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:21:00,064 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:21:00,064 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:21:00,069 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:21:00,435 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:21:00,646 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:21:01,437 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:21:03,069 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:21:06,606 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:21:06,741 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:21:06,741 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:21:06,741 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:21:06,746 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:21:07,445 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:21:08,747 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:21:09,449 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:21:10,579 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:21:10,579 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:21:10,580 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:21:10,585 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:21:11,450 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:21:11,606 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:21:13,455 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:21:14,585 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:21:15,646 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:21:16,642 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:21:16,643 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:21:16,643 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:21:16,647 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:21:17,458 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:21:19,463 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:21:19,685 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:21:21,607 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:21:22,106 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:21:22,107 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:21:22,107 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:21:22,112 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:21:22,466 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:21:23,469 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:21:25,113 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:21:26,537 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:21:26,538 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:21:26,538 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:21:26,543 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:21:26,608 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:21:27,158 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 20:21:27,473 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:21:29,478 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:21:30,159 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:21:30,505 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:21:30,506 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:21:30,506 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:21:30,511 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:21:30,646 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:21:31,479 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:21:33,484 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:21:34,244 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:21:34,244 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:21:34,245 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:21:34,249 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:21:34,485 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:21:35,285 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:21:35,487 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:21:36,609 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:21:38,531 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:21:38,532 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:21:38,532 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:21:38,538 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:21:39,491 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:21:40,538 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:21:41,496 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:21:41,610 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:21:43,643 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:21:43,643 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:21:43,644 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:21:43,648 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:21:44,499 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:21:45,502 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:21:45,646 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:21:45,649 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:21:47,140 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:21:47,141 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:21:47,141 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:21:47,146 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:21:47,504 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:21:49,508 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:21:50,443 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:21:50,444 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:21:50,444 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:21:50,449 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:21:50,509 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:21:51,450 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:21:51,611 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:21:53,460 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:21:53,461 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:21:53,461 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:21:53,466 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:21:53,514 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:21:53,514 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:21:55,517 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:21:56,466 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:21:56,611 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:21:57,163 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 20:21:59,013 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:21:59,014 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:21:59,014 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:21:59,019 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:21:59,521 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:22:00,646 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:22:01,526 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:22:02,020 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:22:02,741 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:22:02,742 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:22:02,742 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:22:02,747 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:22:03,527 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:22:05,532 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:22:06,426 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:22:06,427 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:22:06,427 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:22:06,432 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:22:06,533 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:22:06,612 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:22:07,433 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:22:07,535 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:22:11,577 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:22:11,578 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:22:11,578 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:22:11,583 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:22:11,613 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:22:12,541 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:22:12,583 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:22:13,544 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:22:14,770 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:22:14,771 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:22:14,771 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:22:14,776 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:22:15,545 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:22:15,646 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:22:17,550 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:22:17,776 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:22:18,941 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:22:18,942 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:22:18,942 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:22:18,947 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:22:19,551 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:22:21,556 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:22:21,614 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:22:22,986 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:22:23,820 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:22:23,821 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:22:23,821 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:22:23,826 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:22:24,559 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:22:25,561 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:22:26,615 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:22:27,036 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:22:27,037 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:22:27,037 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:22:27,041 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:22:27,163 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 20:22:27,563 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:22:28,164 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:22:29,568 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:22:30,646 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:22:32,016 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:22:32,017 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:22:32,017 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:22:32,022 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:22:32,571 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:22:33,574 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:22:34,057 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:22:36,616 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:22:36,860 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:22:36,861 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:22:36,861 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:22:36,866 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:22:37,578 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:22:39,582 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:22:39,867 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:22:40,540 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:22:40,541 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:22:40,541 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:22:40,546 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:22:40,583 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:22:41,617 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:22:43,588 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:22:44,497 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:22:44,498 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:22:44,498 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:22:44,503 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:22:44,589 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:22:45,504 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:22:45,646 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:22:47,595 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:22:48,036 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:22:48,037 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:22:48,037 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:22:48,042 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:22:48,595 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:22:49,598 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:22:51,077 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:22:51,158 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:22:51,158 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:22:51,158 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:22:51,163 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:22:51,600 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:22:51,618 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:22:53,602 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:22:55,910 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:22:55,911 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:22:55,911 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:22:55,917 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:22:56,605 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:22:56,618 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:22:56,917 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:22:57,170 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 20:22:57,608 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:22:59,763 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:22:59,763 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:22:59,764 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:22:59,769 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:23:00,609 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:23:00,647 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:23:01,612 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:23:02,764 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:23:02,765 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:23:02,765 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:23:02,765 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:23:02,770 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:23:03,613 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:23:05,616 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:23:06,494 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:23:06,495 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:23:06,495 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:23:06,499 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:23:06,617 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:23:06,619 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:23:07,622 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:23:08,500 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:23:09,998 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:23:09,998 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:23:09,999 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:23:10,003 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:23:10,624 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:23:11,620 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:23:11,626 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:23:14,004 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:23:15,647 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:23:15,737 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:23:15,738 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:23:15,738 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:23:15,742 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:23:16,629 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:23:17,631 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:23:18,834 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:23:18,835 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:23:18,835 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:23:18,840 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:23:19,632 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:23:19,840 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:23:21,621 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:23:21,635 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:23:22,601 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:23:22,602 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:23:22,602 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:23:22,607 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:23:22,636 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:23:25,608 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:23:25,640 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:23:26,621 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:23:27,168 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 20:23:28,977 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:23:28,978 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:23:28,978 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:23:28,983 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:23:29,642 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:23:30,647 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:23:30,983 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:23:31,646 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:23:33,084 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:23:33,085 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:23:33,085 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:23:33,090 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:23:33,647 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:23:35,650 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:23:36,125 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:23:36,622 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:23:37,554 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:23:37,555 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:23:37,555 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:23:37,560 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:23:37,651 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:23:39,654 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:23:41,496 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:23:41,497 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:23:41,498 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:23:41,498 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:23:41,503 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:23:41,623 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:23:41,655 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:23:43,658 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:23:45,647 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:23:45,796 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:23:45,797 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:23:45,797 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:23:45,802 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:23:46,660 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:23:46,837 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:23:47,663 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:23:49,017 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:23:49,018 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:23:49,018 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:23:49,025 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:23:49,664 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:23:51,624 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:23:51,669 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:23:52,026 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:23:54,367 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:23:54,367 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:23:54,368 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:23:54,372 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:23:54,670 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:23:55,673 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:23:56,625 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:23:57,172 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 20:23:57,174 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:23:57,478 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:23:57,478 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:23:57,479 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:23:57,484 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:23:57,674 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:23:59,677 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:24:00,647 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:24:02,484 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:24:05,378 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:24:05,378 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:24:05,378 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:24:05,383 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:24:05,681 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:24:06,626 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:24:07,683 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:24:08,421 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:24:09,927 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:24:09,928 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:24:09,928 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:24:09,933 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:24:10,685 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:24:11,626 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:24:11,688 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:24:13,606 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:24:13,606 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:24:13,607 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:24:13,607 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:24:13,611 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:24:13,689 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:24:15,647 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:24:15,692 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:24:16,842 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:24:16,842 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:24:16,843 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:24:16,847 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:24:17,693 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:24:18,885 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:24:19,696 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:24:21,627 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:24:23,886 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:24:24,056 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:24:24,057 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:24:24,057 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:24:24,062 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:24:24,700 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:24:25,702 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:24:26,628 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:24:27,172 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 20:24:28,292 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:24:28,293 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:24:28,293 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:24:28,298 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:24:28,704 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:24:29,299 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:24:29,705 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:24:30,647 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:24:31,850 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:24:31,850 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:24:31,850 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:24:31,855 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:24:32,706 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:24:33,709 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:24:34,893 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:24:35,185 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:24:35,186 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:24:35,186 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:24:35,191 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:24:35,710 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:24:36,629 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:24:37,713 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:24:39,172 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:24:39,173 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:24:39,173 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:24:39,177 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:24:39,714 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:24:40,178 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:24:41,630 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:24:41,717 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:24:42,689 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:24:42,689 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:24:42,690 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:24:42,694 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:24:42,718 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:24:45,647 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:24:45,695 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:24:45,719 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:24:45,899 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:24:45,900 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:24:45,900 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:24:45,905 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:24:46,720 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:24:47,723 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:24:50,588 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:24:50,589 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:24:50,589 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:24:50,594 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:24:50,726 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:24:51,594 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:24:51,631 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:24:51,728 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:24:53,890 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:24:53,891 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:24:53,891 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:24:53,896 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:24:54,730 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:24:55,733 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:24:56,631 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:24:56,750 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:24:56,750 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:24:56,751 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:24:56,751 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:24:56,755 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:24:57,177 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 20:24:57,734 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:24:59,737 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:25:00,647 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:25:01,985 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:25:01,986 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:25:01,986 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:25:01,986 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:25:01,991 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:25:02,739 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:25:03,741 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:25:05,544 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:25:05,545 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:25:05,545 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:25:05,550 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:25:05,742 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:25:06,632 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:25:07,550 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:25:07,745 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:25:10,342 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:25:10,342 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:25:10,342 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:25:10,347 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:25:10,746 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:25:11,633 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:25:11,749 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:25:13,348 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:25:15,054 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:25:15,054 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:25:15,055 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:25:15,059 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:25:15,647 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:25:15,751 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:25:17,755 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:25:19,060 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:25:19,524 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:25:19,524 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:25:19,524 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:25:19,529 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:25:19,756 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:25:21,634 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:25:21,759 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:25:23,142 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:25:23,143 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:25:23,143 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:25:23,148 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:25:23,761 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:25:24,148 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:25:25,764 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:25:26,635 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:25:26,842 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:25:26,843 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:25:26,843 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:25:26,848 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:25:27,177 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 20:25:27,765 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:25:29,178 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:25:29,766 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:25:30,647 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:25:32,019 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:25:32,020 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:25:32,020 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:25:32,025 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:25:32,768 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:25:33,771 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:25:35,062 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:25:36,636 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:25:37,741 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:25:37,742 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:25:37,742 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:25:37,747 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:25:37,773 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:25:39,776 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:25:40,747 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:25:41,636 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:25:43,292 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:25:43,293 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:25:43,294 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:25:43,298 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:25:43,780 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:25:45,647 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:25:45,783 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:25:46,299 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:25:47,427 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:25:47,428 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:25:47,428 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:25:47,433 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:25:47,785 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:25:49,787 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:25:51,470 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:25:51,637 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:25:53,419 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:25:53,420 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:25:53,420 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:25:53,425 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:25:53,791 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:25:55,794 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:25:56,638 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:25:57,181 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 20:25:57,182 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:26:00,371 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:26:00,372 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:26:00,372 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:26:00,377 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:26:00,648 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:26:00,797 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:26:01,797 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:26:02,378 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:26:04,779 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:26:04,779 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:26:04,779 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:26:04,785 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:26:04,799 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:26:06,639 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:26:07,804 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:26:07,822 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:26:08,585 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:26:08,586 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:26:08,586 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:26:08,591 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:26:08,804 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:26:09,807 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:26:11,621 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:26:11,621 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:26:11,621 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:26:11,626 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:26:11,640 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:26:11,808 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:26:13,627 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:26:13,811 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:26:15,556 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:26:15,557 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:26:15,557 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:26:15,562 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:26:15,647 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:26:15,812 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:26:17,815 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:26:19,563 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:26:21,213 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:26:21,213 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:26:21,214 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:26:21,218 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:26:21,641 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:26:21,817 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:26:23,821 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:26:24,957 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:26:24,958 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:26:24,958 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:26:24,958 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:26:24,963 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:26:25,822 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:26:26,642 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:26:27,180 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 20:26:27,825 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:26:28,747 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:26:28,748 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:26:28,748 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:26:28,753 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:26:28,825 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:26:30,648 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:26:30,753 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:26:31,829 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:26:33,008 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:26:33,008 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:26:33,008 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:26:33,013 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:26:33,830 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:26:35,831 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:26:36,049 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:26:36,643 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:26:37,585 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:26:37,586 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:26:37,586 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:26:37,591 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:26:37,832 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:26:39,836 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:26:41,591 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:26:41,644 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:26:41,667 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:26:41,668 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:26:41,668 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:26:41,672 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:26:41,837 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:26:43,840 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:26:45,338 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:26:45,338 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:26:45,338 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:26:45,343 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:26:45,648 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:26:45,841 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:26:47,344 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:26:47,844 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:26:49,378 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:26:49,379 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:26:49,379 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:26:49,384 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:26:49,846 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:26:51,645 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:26:51,850 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:26:52,421 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:26:53,151 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:26:53,152 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:26:53,152 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:26:53,157 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:26:53,851 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:26:55,854 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:26:56,633 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:26:56,634 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:26:56,634 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:26:56,639 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:26:56,645 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:26:56,854 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:26:57,186 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 20:26:57,855 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:26:58,187 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:27:00,648 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:27:01,980 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:27:01,981 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:27:01,981 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:27:01,986 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:27:02,858 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:27:03,860 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:27:04,021 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:27:05,709 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:27:05,710 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:27:05,710 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:27:05,714 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:27:05,861 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:27:06,646 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:27:07,866 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:27:09,446 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:27:09,447 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:27:09,447 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:27:09,447 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:27:09,452 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:27:09,867 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:27:11,647 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:27:11,870 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:27:13,598 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:27:13,599 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:27:13,599 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:27:13,604 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:27:13,871 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:27:14,604 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:27:15,648 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:27:15,875 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:27:17,442 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:27:17,442 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:27:17,443 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:27:17,447 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:27:17,876 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:27:19,879 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:27:20,485 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:27:21,648 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:27:22,514 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:27:22,514 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:27:22,515 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:27:22,519 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:27:22,881 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:27:23,883 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:27:25,520 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:27:25,702 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:27:25,703 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:27:25,703 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:27:25,708 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:27:25,884 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:27:26,649 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:27:27,185 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 20:27:27,887 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:27:30,212 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:27:30,213 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:27:30,213 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:27:30,218 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:27:30,648 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:27:30,889 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:27:31,218 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:27:31,892 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:27:33,587 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:27:33,587 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:27:33,588 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:27:33,593 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:27:33,893 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:27:35,650 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:27:35,896 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:27:36,630 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:27:37,835 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:27:37,836 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:27:37,836 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:27:37,841 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:27:37,897 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:27:39,900 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:27:40,650 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:27:41,842 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:27:42,080 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:27:42,081 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:27:42,081 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:27:42,086 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:27:42,902 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:27:43,904 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:27:45,648 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:27:47,087 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:27:47,166 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:27:47,167 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:27:47,167 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:27:47,172 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:27:47,907 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:27:49,910 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:27:50,621 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:27:50,622 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:27:50,622 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:27:50,626 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:27:50,651 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:27:50,912 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:27:51,914 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:27:52,627 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:27:54,529 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:27:54,530 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:27:54,530 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:27:54,535 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:27:54,916 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:27:55,652 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:27:55,918 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:27:57,190 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 20:27:57,995 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:27:57,996 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:27:57,996 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:27:57,996 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:27:58,002 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:27:58,920 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:27:59,923 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:28:00,648 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:28:03,003 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:28:04,558 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:28:04,559 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:28:04,559 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:28:04,564 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:28:04,926 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:28:05,653 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:28:05,928 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:28:08,237 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:28:08,238 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:28:08,238 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:28:08,238 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:28:08,243 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:28:08,931 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:28:09,932 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:28:11,244 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:28:13,243 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:28:15,648 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:28:17,285 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:28:17,285 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:28:17,286 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:28:17,290 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:28:17,936 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:28:18,329 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:28:19,939 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:28:20,807 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:28:20,808 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:28:20,808 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:28:20,813 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:28:20,940 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:28:21,245 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:28:21,943 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:28:23,813 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:28:24,905 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:28:24,906 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:28:24,906 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:28:24,911 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:28:24,945 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:28:26,246 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:28:27,190 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 20:28:27,948 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:28:29,191 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:28:29,495 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:28:29,495 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:28:29,495 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:28:29,500 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:28:29,950 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:28:30,648 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:28:31,951 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:28:34,077 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:28:34,078 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:28:34,078 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:28:34,083 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:28:34,953 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:28:35,121 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:28:35,955 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:28:36,247 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:28:40,122 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:28:41,247 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:28:42,549 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:28:42,550 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:28:42,550 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:28:42,555 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:28:42,959 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:28:43,962 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:28:45,555 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:28:45,648 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:28:45,939 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:28:45,940 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:28:45,940 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:28:45,944 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:28:45,963 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:28:47,966 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:28:49,680 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:28:49,681 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:28:49,681 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:28:49,686 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:28:49,967 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:28:50,686 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:28:51,249 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:28:51,971 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:28:53,304 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:28:53,305 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:28:53,305 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:28:53,310 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:28:53,972 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:28:55,973 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:28:56,249 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:28:56,310 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:28:56,547 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:28:56,548 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:28:56,548 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:28:56,553 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:28:56,974 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:28:57,192 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 20:28:57,977 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:29:00,648 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:29:02,193 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:29:03,563 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:29:03,564 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:29:03,564 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:29:03,568 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:29:03,980 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:29:05,983 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:29:06,250 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:29:07,605 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:29:07,645 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:29:07,645 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:29:07,646 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:29:07,650 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:29:07,985 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:29:09,988 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:29:11,251 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:29:11,805 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:29:11,806 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:29:11,806 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:29:11,811 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:29:11,990 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:29:12,811 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:29:13,993 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:29:15,649 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:29:17,812 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:29:20,370 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:29:20,370 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:29:20,370 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:29:20,375 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:29:20,997 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:29:21,252 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:29:21,999 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:29:23,413 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:29:24,071 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:29:24,072 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:29:24,072 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:29:24,077 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:29:25,001 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:29:26,002 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:29:26,252 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:29:27,194 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 20:29:27,306 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:29:27,307 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:29:27,307 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:29:27,311 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:29:28,003 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:29:29,312 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:29:30,006 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:29:30,649 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:29:30,729 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:29:30,729 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:29:30,729 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:29:30,734 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:29:31,007 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:29:32,009 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:29:34,770 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:29:36,254 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:29:37,569 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:29:37,570 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:29:37,570 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:29:37,576 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:29:38,013 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:29:40,016 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:29:40,576 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:29:41,254 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:29:41,642 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:29:41,643 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:29:41,643 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:29:41,648 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:29:42,017 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:29:44,020 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:29:45,239 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:29:45,240 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:29:45,240 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:29:45,245 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:29:45,649 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:29:46,021 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:29:46,246 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:29:48,025 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:29:50,642 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:29:50,642 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:29:50,642 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:29:50,647 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:29:51,028 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:29:51,255 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:29:51,685 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:29:52,030 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:29:54,368 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:29:54,369 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:29:54,369 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:29:54,374 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:29:55,032 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:29:56,032 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:29:56,256 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:29:57,195 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 20:29:57,196 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:29:58,115 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:29:58,116 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:29:58,116 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:29:58,121 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:29:59,034 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:30:00,037 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:30:00,649 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:30:01,006 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:30:01,006 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:30:01,007 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:30:01,011 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:30:01,037 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:30:03,049 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:30:04,041 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:30:04,481 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:30:04,481 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:30:04,481 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:30:04,487 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:30:05,042 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:30:06,044 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:30:06,257 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:30:08,487 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:30:08,678 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:30:08,679 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:30:08,679 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:30:08,684 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:30:09,047 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:30:10,050 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:30:11,258 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:30:12,446 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:30:12,446 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:30:12,447 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:30:12,451 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:30:13,051 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:30:14,052 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:30:14,452 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:30:15,649 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:30:16,684 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:30:16,684 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:30:16,684 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:30:16,689 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:30:17,054 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:30:18,056 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:30:19,725 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:30:20,892 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:30:20,893 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:30:20,893 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:30:20,898 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:30:21,058 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:30:21,259 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:30:22,061 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:30:24,899 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:30:24,976 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:30:24,977 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:30:24,977 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:30:24,982 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:30:25,063 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:30:26,065 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:30:26,259 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:30:27,200 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 20:30:29,158 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:30:29,158 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:30:29,159 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:30:29,163 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:30:30,068 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:30:30,164 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:30:30,649 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:30:32,071 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:30:33,886 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:30:33,887 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:30:33,887 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:30:33,892 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:30:34,072 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:30:35,929 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:30:36,075 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:30:36,261 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:30:37,157 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:30:37,158 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:30:37,158 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:30:37,163 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:30:38,076 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:30:40,079 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:30:41,163 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:30:41,261 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:30:43,819 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:30:43,820 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:30:43,820 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:30:43,825 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:30:44,081 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:30:45,649 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:30:46,084 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:30:46,826 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:30:47,502 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:30:47,503 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:30:47,503 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:30:47,507 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:30:48,086 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:30:50,089 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:30:51,263 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:30:52,546 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:30:56,060 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:30:56,060 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:30:56,061 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:30:56,066 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:30:56,094 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:30:56,264 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:30:57,199 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 20:30:58,095 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:30:58,200 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:30:59,999 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:31:00,000 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:31:00,000 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:31:00,005 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:31:00,096 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:31:00,649 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:31:02,099 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:31:03,142 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:31:03,142 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:31:03,143 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:31:03,147 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:31:04,100 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:31:04,185 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:31:06,103 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:31:06,265 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:31:08,494 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:31:08,494 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:31:08,495 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:31:08,499 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:31:09,106 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:31:09,500 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:31:10,109 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:31:11,265 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:31:12,092 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:31:12,093 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:31:12,093 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:31:12,097 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:31:12,110 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:31:14,113 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:31:15,098 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:31:15,649 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:31:16,582 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:31:16,582 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:31:16,582 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:31:16,587 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:31:17,115 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:31:18,117 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:31:20,259 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:31:20,260 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:31:20,260 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:31:20,260 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:31:20,264 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:31:21,119 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:31:21,266 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:31:22,122 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:31:24,216 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:31:24,217 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:31:24,217 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:31:24,222 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:31:25,124 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:31:26,126 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:31:26,222 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:31:26,267 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:31:27,204 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 20:31:27,744 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:31:27,745 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:31:27,745 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:31:27,750 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:31:28,127 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:31:30,130 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:31:30,649 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:31:31,750 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:31:32,069 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:31:32,070 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:31:32,070 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:31:32,074 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:31:32,131 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:31:34,134 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:31:35,823 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:31:35,823 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:31:35,824 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:31:35,828 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:31:36,136 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:31:36,268 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:31:36,829 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:31:38,138 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:31:41,269 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:31:41,829 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:31:43,781 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:31:43,782 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:31:43,782 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:31:43,787 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:31:44,142 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:31:45,649 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:31:46,145 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:31:47,521 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:31:47,522 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:31:47,522 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:31:47,522 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:31:47,527 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:31:48,147 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:31:50,150 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:31:51,270 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:31:51,567 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:31:51,567 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:31:51,568 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:31:51,572 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:31:52,152 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:31:52,573 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:31:54,155 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:31:55,178 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:31:55,178 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:31:55,179 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:31:55,183 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:31:56,157 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:31:56,271 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:31:57,204 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 20:31:58,160 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:31:58,205 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:31:59,673 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:31:59,673 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:31:59,674 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:31:59,678 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:32:00,161 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:32:00,649 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:32:02,164 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:32:03,679 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:32:04,532 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:32:04,533 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:32:04,533 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:32:04,538 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:32:05,166 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:32:06,168 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:32:06,272 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:32:07,860 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:32:07,861 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:32:07,861 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:32:07,866 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:32:08,171 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:32:08,866 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:32:10,174 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:32:11,272 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:32:13,867 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:32:15,649 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:32:16,509 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:32:16,510 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:32:16,510 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:32:16,515 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:32:17,178 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:32:18,180 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:32:19,554 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:32:21,273 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:32:23,331 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:32:23,332 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:32:23,332 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:32:23,337 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:32:24,184 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:32:25,337 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:32:26,187 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:32:26,274 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:32:27,209 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 20:32:27,704 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:32:27,705 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:32:27,705 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:32:27,710 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:32:28,188 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:32:30,192 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:32:30,650 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:32:30,710 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:32:31,206 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:32:31,206 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:32:31,207 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:32:31,211 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:32:32,193 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:32:34,196 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:32:35,199 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:32:35,200 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:32:35,201 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:32:35,205 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:32:36,197 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:32:36,206 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:32:36,275 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:32:38,200 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:32:38,554 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:32:38,555 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:32:38,555 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:32:38,560 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:32:39,201 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:32:40,204 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:32:41,276 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:32:41,560 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:32:41,890 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:32:41,891 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:32:41,891 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:32:41,896 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:32:42,205 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:32:44,208 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:32:45,147 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:32:45,148 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:32:45,148 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:32:45,152 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:32:45,209 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:32:45,650 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:32:46,211 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:32:47,153 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:32:49,764 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:32:49,765 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:32:49,765 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:32:49,770 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:32:50,214 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:32:51,277 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:32:52,218 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:32:52,806 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:32:54,921 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:32:54,922 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:32:54,922 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:32:54,926 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:32:55,220 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:32:56,223 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:32:56,681 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:32:57,208 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 20:32:58,209 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:33:00,441 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:33:00,442 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:33:00,442 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:33:00,447 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:33:00,650 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:33:01,225 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:33:02,228 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:33:03,448 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:33:05,682 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:33:06,196 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:33:06,197 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:33:06,197 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:33:06,202 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:33:06,230 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:33:08,234 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:33:09,203 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:33:09,878 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:33:09,879 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:33:09,879 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:33:09,883 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:33:10,236 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:33:10,682 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:33:12,239 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:33:14,884 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:33:15,650 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:33:15,954 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:33:15,955 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:33:15,955 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:33:15,960 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:33:16,241 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:33:18,244 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:33:19,397 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:33:19,397 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:33:19,398 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:33:19,402 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:33:20,245 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:33:20,402 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:33:20,683 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:33:22,248 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:33:25,403 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:33:25,684 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:33:26,219 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:33:26,219 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:33:26,220 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:33:26,224 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:33:26,251 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:33:27,211 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 20:33:28,254 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:33:30,650 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:33:31,212 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:33:34,593 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:33:34,594 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:33:34,594 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:33:34,599 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:33:35,258 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:33:35,685 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:33:36,259 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:33:36,640 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:33:37,799 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:33:37,800 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:33:37,800 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:33:37,805 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:33:38,260 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:33:40,263 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:33:40,685 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:33:41,805 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:33:44,304 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:33:44,304 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:33:44,305 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:33:44,310 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:33:45,266 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:33:45,650 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:33:46,268 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:33:47,310 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:33:48,987 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:33:48,988 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:33:48,988 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:33:48,993 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:33:49,271 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:33:50,273 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:33:50,686 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:33:53,030 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:33:53,086 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:33:53,087 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:33:53,088 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:33:53,092 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:33:53,276 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:33:54,278 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:33:55,687 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:33:57,213 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 20:33:58,214 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:33:59,263 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:33:59,264 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:33:59,264 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:33:59,269 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:33:59,282 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:34:00,284 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:34:00,650 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:34:03,270 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:34:04,349 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:34:04,350 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:34:04,350 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:34:04,355 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:34:05,287 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:34:05,688 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:34:06,290 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:34:08,279 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:34:08,280 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:34:08,280 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:34:08,280 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:34:08,285 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:34:08,293 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:34:10,295 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:34:10,689 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:34:11,781 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:34:11,782 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:34:11,782 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:34:11,787 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:34:12,296 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:34:13,788 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:34:14,299 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:34:15,148 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:34:15,149 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:34:15,149 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:34:15,153 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:34:15,300 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:34:15,650 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:34:16,302 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:34:18,482 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:34:18,483 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:34:18,483 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:34:18,488 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:34:19,304 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:34:19,525 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:34:20,306 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:34:20,690 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:34:22,841 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:34:22,842 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:34:22,842 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:34:22,847 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:34:23,308 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:34:24,311 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:34:24,848 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:34:25,690 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:34:25,741 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:34:25,742 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:34:25,742 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:34:25,747 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:34:26,312 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:34:27,215 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 20:34:28,315 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:34:28,853 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:34:28,854 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:34:28,854 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:34:28,859 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:34:29,316 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:34:29,859 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:34:30,318 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:34:30,650 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:34:33,618 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:34:33,619 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:34:33,619 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:34:33,624 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:34:34,321 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:34:35,664 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:34:35,691 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:34:36,324 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:34:36,956 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:34:36,956 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:34:36,957 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:34:36,961 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:34:37,324 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:34:38,327 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:34:40,692 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:34:40,962 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:34:41,238 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:34:41,238 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:34:41,239 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:34:41,243 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:34:41,329 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:34:42,331 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:34:45,650 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:34:46,244 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:34:46,621 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:34:46,622 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:34:46,622 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:34:46,627 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:34:47,335 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:34:48,337 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:34:50,586 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:34:50,586 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:34:50,586 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:34:50,591 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:34:50,693 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:34:51,340 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:34:51,592 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:34:52,343 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:34:54,024 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:34:54,025 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:34:54,025 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:34:54,031 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:34:54,344 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:34:55,694 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:34:56,347 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:34:57,031 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:34:57,217 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 20:35:00,650 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:35:02,219 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:35:02,590 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:35:02,590 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:35:02,591 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:35:02,595 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:35:03,351 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:35:04,353 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:35:05,695 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:35:06,238 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:35:06,239 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:35:06,239 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:35:06,243 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:35:06,354 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:35:07,244 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:35:08,359 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:35:10,695 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:35:11,263 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:35:11,264 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:35:11,264 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:35:11,269 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:35:11,361 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:35:12,269 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:35:12,361 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:35:15,197 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:35:15,198 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:35:15,198 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:35:15,203 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:35:15,363 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:35:15,650 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:35:16,365 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:35:18,203 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:35:19,972 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:35:19,973 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:35:19,973 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:35:19,978 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:35:20,368 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:35:20,696 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:35:22,371 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:35:23,403 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:35:23,404 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:35:23,404 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:35:23,404 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:35:23,409 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:35:24,372 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:35:25,697 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:35:26,375 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:35:27,220 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 20:35:29,020 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:35:29,021 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:35:29,021 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:35:29,021 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:35:29,026 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:35:29,377 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:35:30,380 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:35:30,651 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:35:33,998 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:35:33,998 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:35:33,998 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:35:34,003 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:35:34,041 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:35:34,382 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:35:35,698 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:35:36,385 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:35:39,023 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:35:39,024 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:35:39,024 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:35:39,029 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:35:39,387 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:35:40,029 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:35:40,389 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:35:40,699 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:35:41,892 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:35:41,893 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:35:41,893 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:35:41,898 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:35:42,391 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:35:44,394 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:35:45,651 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:35:45,853 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:35:45,854 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:35:45,854 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:35:45,854 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:35:45,859 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:35:46,395 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:35:48,398 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:35:50,614 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:35:50,615 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:35:50,615 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:35:50,620 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:35:50,700 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:35:51,401 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:35:51,620 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:35:52,404 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:35:54,646 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:35:54,647 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:35:54,647 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:35:54,652 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:35:55,406 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:35:55,700 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:35:56,408 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:35:56,652 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:35:57,222 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 20:35:57,895 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:35:57,896 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:35:57,896 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:35:57,901 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:35:58,409 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:36:00,412 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:36:00,651 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:36:01,017 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:36:01,018 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:36:01,018 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:36:01,023 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:36:01,413 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:36:02,061 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:36:02,414 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:36:05,554 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:36:05,554 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:36:05,555 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:36:05,559 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:36:05,701 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:36:06,416 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:36:07,560 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:36:08,420 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:36:08,991 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:36:08,992 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:36:08,992 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:36:08,996 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:36:09,421 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:36:10,423 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:36:10,702 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:36:12,997 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:36:13,447 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:36:13,447 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:36:13,448 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:36:13,452 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:36:14,426 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:36:15,651 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:36:16,429 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:36:17,717 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:36:17,717 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:36:17,717 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:36:17,722 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:36:18,430 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:36:18,757 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:36:20,433 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:36:20,703 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:36:22,371 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:36:22,372 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:36:22,372 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:36:22,377 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:36:22,434 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:36:24,378 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:36:24,438 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:36:25,703 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:36:27,071 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:36:27,072 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:36:27,072 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:36:27,077 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:36:27,224 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 20:36:27,440 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:36:28,442 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:36:30,226 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:36:30,651 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:36:31,799 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:36:31,800 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:36:31,800 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:36:31,804 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:36:32,444 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:36:34,447 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:36:35,705 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:36:35,842 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:36:35,893 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:36:35,893 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:36:35,894 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:36:35,898 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:36:36,449 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:36:38,452 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:36:40,705 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:36:40,899 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:36:41,894 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:36:41,895 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:36:41,895 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:36:41,899 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:36:42,454 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:36:44,457 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:36:45,354 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:36:45,355 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:36:45,355 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:36:45,360 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:36:45,458 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:36:45,651 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:36:46,361 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:36:46,460 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:36:48,958 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:36:48,959 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:36:48,959 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:36:48,963 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:36:49,462 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:36:50,465 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:36:50,706 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:36:52,002 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:36:52,782 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:36:52,783 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:36:52,783 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:36:52,788 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:36:53,468 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:36:54,469 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:36:55,707 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:36:57,226 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 20:36:57,228 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:36:57,860 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:36:57,861 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:36:57,861 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:36:57,865 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:36:58,471 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:37:00,474 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:37:00,651 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:37:01,339 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:37:01,340 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:37:01,340 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:37:01,344 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:37:01,475 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:37:02,381 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:37:02,477 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:37:04,796 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:37:04,797 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:37:04,797 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:37:04,802 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:37:05,479 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:37:05,708 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:37:06,481 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:37:07,803 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:37:08,975 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:37:08,976 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:37:08,976 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:37:08,981 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:37:09,485 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:37:10,488 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:37:10,708 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:37:12,981 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:37:13,352 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:37:13,353 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:37:13,353 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:37:13,358 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:37:13,490 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:37:14,492 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:37:15,651 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:37:17,298 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:37:17,298 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:37:17,298 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:37:17,303 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:37:17,494 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:37:18,341 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:37:18,497 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:37:20,709 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:37:20,737 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:37:20,738 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:37:20,738 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:37:20,743 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:37:21,498 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:37:22,499 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:37:23,743 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:37:23,979 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:37:23,980 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:37:23,980 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:37:23,985 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:37:24,500 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:37:25,710 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:37:26,504 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:37:27,228 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 20:37:27,718 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:37:27,719 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:37:27,719 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:37:27,724 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:37:28,505 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:37:29,724 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:37:30,508 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:37:30,651 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:37:31,202 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:37:31,202 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:37:31,202 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:37:31,207 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:37:31,509 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:37:32,511 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:37:35,073 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:37:35,074 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:37:35,074 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:37:35,074 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:37:35,079 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:37:35,513 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:37:35,711 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:37:36,516 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:37:39,211 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:37:39,212 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:37:39,212 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:37:39,217 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:37:39,517 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:37:40,217 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:37:40,518 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:37:40,712 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:37:43,634 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:37:43,634 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:37:43,635 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:37:43,639 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:37:44,520 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:37:45,640 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:37:45,652 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:37:46,523 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:37:47,983 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:37:47,984 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:37:47,984 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:37:47,989 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:37:48,525 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:37:50,528 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:37:50,713 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:37:51,029 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:37:54,415 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:37:54,415 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:37:54,416 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:37:54,420 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:37:54,532 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:37:55,713 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:37:56,421 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:37:56,533 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:37:57,231 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 20:37:59,249 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:37:59,249 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:37:59,250 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:37:59,254 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:37:59,535 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:38:00,537 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:38:00,652 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:38:02,255 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:38:02,548 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:38:02,549 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:38:02,549 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:38:02,553 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:38:03,539 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:38:04,541 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:38:05,714 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:38:05,861 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:38:05,862 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:38:05,862 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:38:05,866 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:38:06,543 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:38:07,867 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:38:08,547 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:38:10,144 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:38:10,145 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:38:10,145 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:38:10,150 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:38:10,548 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:38:10,715 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:38:12,551 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:38:13,150 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:38:15,071 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:38:15,072 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:38:15,072 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:38:15,077 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:38:15,553 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:38:15,652 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:38:16,555 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:38:19,078 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:38:20,716 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:38:23,379 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:38:23,380 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:38:23,380 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:38:23,385 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:38:23,559 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:38:24,385 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:38:24,560 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:38:25,717 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:38:26,893 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:38:26,894 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:38:26,894 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:38:26,899 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:38:27,233 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 20:38:27,562 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:38:28,564 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:38:30,234 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:38:30,643 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:38:30,644 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:38:30,644 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:38:30,649 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:38:30,652 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:38:31,566 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:38:32,569 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:38:35,649 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:38:35,718 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:38:39,338 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:38:39,339 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:38:39,339 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:38:39,344 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:38:39,573 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:38:40,575 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:38:40,718 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:38:41,344 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:38:42,821 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:38:42,822 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:38:42,822 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:38:42,826 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:38:43,577 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:38:44,579 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:38:45,652 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:38:46,324 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:38:46,325 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:38:46,325 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:38:46,330 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:38:46,366 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:38:46,581 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:38:48,584 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:38:49,612 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:38:49,612 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:38:49,613 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:38:49,617 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:38:50,587 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:38:50,588 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:38:50,720 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:38:51,618 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:38:54,533 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:38:54,534 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:38:54,534 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:38:54,539 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:38:54,591 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:38:55,720 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:38:56,594 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:38:57,235 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 20:38:57,237 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:38:58,131 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:38:58,132 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:38:58,132 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:38:58,137 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:38:58,595 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:39:00,598 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:39:00,652 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:39:01,686 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:39:01,686 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:39:01,686 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:39:01,691 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:39:02,599 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:39:02,731 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:39:04,602 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:39:05,721 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:39:05,834 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:39:05,834 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:39:05,835 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:39:05,839 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:39:06,604 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:39:07,840 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:39:08,608 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:39:09,521 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:39:09,522 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:39:09,522 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:39:09,527 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:39:09,609 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:39:10,611 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:39:10,722 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:39:12,814 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:39:12,814 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:39:12,815 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:39:12,819 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:39:13,613 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:39:13,820 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:39:14,615 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:39:15,652 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:39:16,219 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:39:16,220 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:39:16,220 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:39:16,224 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:39:16,617 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:39:18,620 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:39:19,263 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:39:20,723 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:39:22,919 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:39:22,920 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:39:22,920 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:39:22,925 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:39:23,622 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:39:24,625 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:39:24,926 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:39:25,724 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:39:26,459 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:39:26,460 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:39:26,460 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:39:26,465 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:39:26,627 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:39:27,237 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 20:39:28,630 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:39:29,716 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:39:29,717 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:39:29,717 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:39:29,722 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:39:30,631 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:39:30,652 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:39:30,723 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:39:32,634 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:39:33,505 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:39:33,505 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:39:33,506 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:39:33,510 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:39:33,634 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:39:34,637 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:39:35,725 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:39:36,547 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:39:36,958 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:39:36,959 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:39:36,959 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:39:36,964 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:39:37,639 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:39:38,641 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:39:40,725 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:39:41,965 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:39:42,491 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:39:42,492 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:39:42,492 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:39:42,497 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:39:42,643 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:39:44,647 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:39:45,652 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:39:46,226 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:39:46,226 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:39:46,227 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:39:46,231 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:39:46,648 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:39:47,270 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:39:48,651 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:39:50,056 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:39:50,057 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:39:50,057 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:39:50,062 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:39:50,653 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:39:50,726 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:39:52,655 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:39:53,063 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:39:54,991 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:39:54,992 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:39:54,992 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:39:54,997 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:39:55,657 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:39:55,727 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:39:56,659 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:39:57,240 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 20:39:58,066 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:39:58,067 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:39:58,067 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:39:58,067 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:39:58,071 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:39:58,661 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:40:00,652 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:40:00,664 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:40:02,823 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:40:02,823 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:40:02,823 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:40:02,828 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:40:03,666 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:40:03,866 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:40:04,668 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:40:05,728 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:40:08,867 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:40:10,728 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:40:11,266 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:40:11,267 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:40:11,267 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:40:11,272 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:40:11,673 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:40:12,676 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:40:14,272 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:40:14,916 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:40:14,917 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:40:14,917 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:40:14,922 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:40:15,652 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:40:15,678 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:40:16,680 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:40:18,653 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:40:18,653 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:40:18,653 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:40:18,658 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:40:18,681 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:40:19,695 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:40:20,684 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:40:20,729 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:40:23,414 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:40:23,414 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:40:23,415 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:40:23,419 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:40:23,686 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:40:24,689 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:40:25,420 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:40:25,730 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:40:27,242 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 20:40:28,695 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:40:28,696 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:40:28,696 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:40:28,700 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:40:29,692 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:40:30,652 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:40:30,693 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:40:30,701 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:40:33,086 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:40:33,086 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:40:33,086 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:40:33,091 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:40:33,694 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:40:34,697 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:40:35,731 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:40:36,131 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:40:36,716 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:40:36,717 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:40:36,717 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:40:36,722 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:40:37,698 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:40:38,701 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:40:40,732 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:40:41,723 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:40:41,769 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:40:41,769 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:40:41,769 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:40:41,774 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:40:42,703 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:40:44,707 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:40:45,652 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:40:45,702 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:40:45,703 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:40:45,703 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:40:45,708 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:40:45,708 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:40:46,711 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:40:46,746 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:40:49,282 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:40:49,282 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:40:49,283 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:40:49,287 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:40:49,713 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:40:50,715 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:40:50,733 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:40:52,288 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:40:52,614 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:40:52,615 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:40:52,615 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:40:52,619 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:40:52,718 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:40:54,719 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:40:55,733 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:40:57,247 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 20:40:57,731 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:40:57,732 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:40:57,732 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:40:57,732 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:40:57,736 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:40:58,723 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:40:58,723 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:41:00,653 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:41:01,404 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:41:01,405 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:41:01,405 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:41:01,410 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:41:01,725 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:41:02,728 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:41:03,446 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:41:05,697 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:41:05,697 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:41:05,698 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:41:05,702 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:41:05,729 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:41:05,734 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:41:06,732 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:41:08,703 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:41:09,560 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:41:09,560 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:41:09,561 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:41:09,566 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:41:09,735 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:41:10,735 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:41:10,737 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:41:14,566 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:41:15,653 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:41:19,567 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:41:20,146 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:41:20,147 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:41:20,147 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:41:20,152 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:41:20,736 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:41:20,743 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:41:22,746 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:41:24,112 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:41:24,113 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:41:24,113 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:41:24,118 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:41:24,748 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:41:25,118 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:41:25,737 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:41:26,751 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:41:27,030 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:41:27,031 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:41:27,031 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:41:27,036 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:41:27,246 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 20:41:27,752 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:41:28,754 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:41:30,248 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:41:30,634 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:41:30,635 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:41:30,635 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:41:30,640 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:41:30,653 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:41:30,755 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:41:32,759 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:41:35,641 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:41:35,738 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:41:36,038 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:41:36,039 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:41:36,039 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:41:36,044 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:41:36,761 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:41:38,764 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:41:40,738 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:41:41,045 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:41:44,560 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:41:44,561 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:41:44,561 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:41:44,566 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:41:44,767 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:41:45,653 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:41:46,566 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:41:46,770 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:41:48,088 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:41:48,088 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:41:48,089 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:41:48,093 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:41:48,772 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:41:50,739 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:41:50,775 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:41:52,131 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:41:52,184 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:41:52,185 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:41:52,185 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:41:52,190 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:41:52,777 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:41:54,781 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:41:55,586 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:41:55,587 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:41:55,587 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:41:55,592 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:41:55,740 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:41:55,781 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:41:56,784 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:41:57,251 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 20:41:57,252 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:41:59,509 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:41:59,510 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:41:59,510 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:41:59,516 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:41:59,785 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:42:00,653 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:42:00,786 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:42:02,516 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:42:03,175 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:42:03,175 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:42:03,175 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:42:03,180 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:42:03,788 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:42:04,790 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:42:05,741 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:42:06,669 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:42:06,670 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:42:06,670 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:42:06,675 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:42:06,792 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:42:07,675 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:42:08,796 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:42:10,345 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:42:10,346 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:42:10,346 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:42:10,351 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:42:10,742 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:42:10,797 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:42:12,801 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:42:13,351 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:42:13,940 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:42:13,941 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:42:13,941 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:42:13,946 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:42:14,802 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:42:15,653 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:42:16,805 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:42:18,386 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:42:18,387 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:42:18,387 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:42:18,387 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:42:18,392 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:42:18,806 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:42:20,743 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:42:20,809 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:42:23,431 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:42:23,608 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:42:23,608 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:42:23,609 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:42:23,614 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:42:23,811 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:42:24,814 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:42:25,743 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:42:27,150 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:42:27,150 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:42:27,151 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:42:27,156 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:42:27,250 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 20:42:27,816 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:42:28,819 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:42:29,252 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:42:30,653 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:42:32,154 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:42:32,155 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:42:32,155 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:42:32,160 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:42:32,821 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:42:34,822 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:42:35,198 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:42:35,744 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:42:38,140 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:42:38,141 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:42:38,141 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:42:38,145 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:42:38,825 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:42:40,745 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:42:40,828 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:42:41,146 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:42:41,592 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:42:41,593 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:42:41,593 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:42:41,597 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:42:41,828 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:42:42,831 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:42:45,653 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:42:46,598 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:42:49,883 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:42:49,884 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:42:49,884 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:42:49,889 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:42:50,746 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:42:50,836 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:42:51,926 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:42:52,840 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:42:53,671 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:42:53,672 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:42:53,672 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:42:53,677 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:42:53,840 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:42:54,843 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:42:55,746 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:42:57,255 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 20:42:57,256 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:42:57,277 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:42:57,278 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:42:57,278 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:42:57,283 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:42:57,845 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:42:58,847 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:43:00,653 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:43:02,284 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:43:03,656 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:43:03,656 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:43:03,657 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:43:03,661 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:43:03,850 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:43:04,853 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:43:05,748 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:43:07,698 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:43:07,820 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:43:07,821 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:43:07,821 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:43:07,826 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:43:07,855 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:43:08,856 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:43:10,748 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:43:12,079 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:43:12,080 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:43:12,080 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:43:12,084 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:43:12,859 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:43:13,085 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:43:14,862 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:43:15,653 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:43:17,476 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:43:17,476 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:43:17,477 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:43:17,481 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:43:17,864 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:43:18,518 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:43:18,866 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:43:20,945 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:43:22,231 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:43:22,231 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:43:22,232 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:43:22,236 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:43:22,869 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:43:24,237 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:43:24,872 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:43:25,946 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:43:26,293 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:43:26,293 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:43:26,294 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:43:26,298 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:43:26,874 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:43:27,254 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 20:43:28,877 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:43:29,255 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:43:30,653 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:43:34,256 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:43:34,714 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:43:34,714 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:43:34,715 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:43:34,719 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:43:34,880 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:43:35,947 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:43:36,883 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:43:39,366 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:43:39,366 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:43:39,367 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:43:39,367 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:43:39,371 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:43:39,885 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:43:40,888 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:43:40,948 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:43:42,280 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:43:42,281 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:43:42,281 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:43:42,285 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:43:42,889 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:43:44,892 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:43:45,286 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:43:45,654 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:43:45,708 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:43:45,709 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:43:45,709 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:43:45,713 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:43:45,893 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:43:46,895 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:43:49,589 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:43:49,589 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:43:49,590 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:43:49,595 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:43:49,897 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:43:50,595 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:43:50,898 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:43:50,949 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:43:53,779 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:43:53,779 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:43:53,780 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:43:53,784 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:43:53,901 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:43:54,903 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:43:55,785 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:43:55,949 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:43:57,259 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 20:43:57,441 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:43:57,441 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:43:57,442 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:43:57,447 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:43:57,905 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:43:58,908 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:44:00,654 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:44:01,126 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:44:01,126 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:44:01,127 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:44:01,127 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:44:01,131 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:44:01,909 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:44:02,912 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:44:04,444 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:44:04,444 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:44:04,445 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:44:04,450 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:44:04,913 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:44:05,951 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:44:06,450 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:44:06,914 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:44:09,957 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:44:09,958 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:44:09,958 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:44:09,963 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:44:10,919 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:44:10,920 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:44:10,951 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:44:11,963 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:44:15,654 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:44:15,684 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:44:15,684 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:44:15,684 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:44:15,689 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:44:15,922 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:44:16,925 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:44:17,726 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:44:20,062 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:44:20,062 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:44:20,063 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:44:20,067 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:44:20,927 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:44:20,953 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:44:22,929 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:44:23,068 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:44:23,951 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:44:23,952 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:44:23,952 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:44:23,957 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:44:24,932 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:44:24,932 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:44:25,953 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:44:27,259 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 20:44:27,742 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:44:27,742 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:44:27,742 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:44:27,747 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:44:27,934 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:44:28,747 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:44:28,937 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:44:30,654 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:44:30,828 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:44:30,829 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:44:30,829 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:44:30,833 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:44:30,938 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:44:32,941 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:44:33,870 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:44:34,368 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:44:34,369 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:44:34,369 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:44:34,374 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:44:34,942 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:44:35,954 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:44:36,945 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:44:38,003 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:44:38,004 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:44:38,004 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:44:38,009 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:44:38,946 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:44:38,947 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:44:39,009 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:44:40,955 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:44:44,010 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:44:45,654 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:44:46,553 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:44:46,554 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:44:46,554 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:44:46,559 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:44:46,951 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:44:48,954 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:44:49,598 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:44:49,852 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:44:49,852 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:44:49,853 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:44:49,857 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:44:49,955 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:44:50,956 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:44:50,958 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:44:54,858 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:44:54,960 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:44:54,961 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:44:54,961 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:44:54,966 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:44:55,957 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:44:55,962 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:44:56,964 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:44:57,264 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 20:44:58,350 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:44:58,351 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:44:58,351 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:44:58,355 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:44:58,966 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:45:00,356 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:45:00,654 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:45:00,969 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:45:03,015 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:45:03,015 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:45:03,015 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:45:03,020 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:45:03,970 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:45:04,973 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:45:05,958 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:45:06,058 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:45:07,227 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:45:07,228 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:45:07,228 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:45:07,233 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:45:07,975 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:45:08,979 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:45:10,959 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:45:11,035 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:45:11,035 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:45:11,036 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:45:11,040 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:45:11,981 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:45:12,041 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:45:12,983 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:45:15,622 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:45:15,623 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:45:15,623 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:45:15,628 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:45:15,654 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:45:15,985 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:45:16,987 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:45:17,628 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:45:19,866 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:45:19,866 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:45:19,867 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:45:19,871 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:45:19,989 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:45:20,960 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:45:20,992 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:45:22,910 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:45:23,223 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:45:23,223 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:45:23,224 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:45:23,228 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:45:23,994 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:45:24,997 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:45:25,961 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:45:26,581 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:45:26,581 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:45:26,582 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:45:26,586 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:45:26,998 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:45:27,263 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 20:45:28,264 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:45:29,001 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:45:30,188 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:45:30,189 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:45:30,189 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:45:30,194 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:45:30,654 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:45:31,002 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:45:33,005 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:45:34,195 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:45:34,262 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:45:34,262 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:45:34,262 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:45:34,267 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:45:35,006 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:45:35,962 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:45:37,009 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:45:38,358 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:45:38,359 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:45:38,359 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:45:38,364 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:45:39,011 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:45:39,364 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:45:40,963 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:45:41,014 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:45:42,486 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:45:42,486 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:45:42,487 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:45:42,491 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:45:43,015 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:45:44,492 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:45:45,016 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:45:45,654 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:45:45,779 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:45:45,779 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:45:45,779 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:45:45,784 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:45:46,017 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:45:47,019 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:45:49,822 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:45:50,964 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:45:52,441 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:45:52,442 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:45:52,442 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:45:52,447 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:45:53,024 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:45:55,026 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:45:55,447 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:45:55,964 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:45:56,364 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:45:56,364 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:45:56,365 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:45:56,370 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:45:57,027 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:45:57,268 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 20:45:59,030 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:46:00,654 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:46:00,933 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:46:00,933 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:46:00,934 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:46:00,934 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:46:00,938 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:46:01,031 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:46:03,034 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:46:04,221 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:46:04,222 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:46:04,222 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:46:04,227 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:46:05,036 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:46:05,965 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:46:06,227 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:46:07,039 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:46:10,966 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:46:11,228 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:46:12,729 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:46:12,730 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:46:12,730 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:46:12,736 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:46:13,044 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:46:15,047 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:46:15,654 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:46:16,279 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:46:16,280 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:46:16,280 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:46:16,280 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:46:16,285 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:46:17,048 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:46:19,051 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:46:19,168 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:46:19,169 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:46:19,169 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:46:19,174 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:46:20,052 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:46:20,967 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:46:21,055 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:46:22,175 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:46:23,467 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:46:23,467 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:46:23,468 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:46:23,473 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:46:24,057 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:46:25,060 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:46:25,968 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:46:27,267 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 20:46:27,268 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:46:27,379 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:46:27,380 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:46:27,380 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:46:27,385 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:46:28,061 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:46:29,064 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:46:30,654 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:46:32,153 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:46:32,153 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:46:32,153 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:46:32,159 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:46:33,066 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:46:33,194 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:46:35,069 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:46:35,482 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:46:35,483 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:46:35,483 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:46:35,488 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:46:35,969 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:46:36,070 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:46:37,072 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:46:38,488 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:46:39,212 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:46:39,212 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:46:39,212 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:46:39,217 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:46:40,074 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:46:40,969 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:46:41,077 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:46:42,905 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:46:42,906 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:46:42,906 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:46:42,911 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:46:43,078 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:46:43,911 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:46:45,081 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:46:45,654 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:46:46,335 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:46:46,336 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:46:46,336 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:46:46,341 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:46:47,082 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:46:49,086 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:46:49,378 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:46:50,579 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:46:50,580 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:46:50,580 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:46:50,585 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:46:50,970 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:46:51,087 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:46:53,091 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:46:54,233 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:46:54,233 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:46:54,234 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:46:54,239 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:46:55,093 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:46:55,239 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:46:55,971 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:46:57,096 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:46:57,272 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 20:46:57,741 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:46:57,741 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:46:57,742 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:46:57,747 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:46:58,096 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:46:59,099 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:47:00,655 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:47:00,747 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:47:01,910 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:47:01,910 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:47:01,910 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:47:01,915 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:47:02,101 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:47:03,103 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:47:05,155 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:47:05,156 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:47:05,156 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:47:05,161 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:47:05,972 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:47:06,105 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:47:06,162 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:47:07,107 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:47:10,801 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:47:10,802 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:47:10,802 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:47:10,807 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:47:10,973 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:47:11,111 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:47:11,808 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:47:13,114 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:47:13,858 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:47:13,859 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:47:13,859 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:47:13,865 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:47:14,114 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:47:15,117 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:47:15,655 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:47:16,865 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:47:18,104 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:47:18,104 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:47:18,104 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:47:18,109 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:47:18,119 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:47:19,121 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:47:20,974 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:47:22,146 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:47:22,502 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:47:22,502 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:47:22,503 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:47:22,508 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:47:23,123 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:47:25,127 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:47:25,975 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:47:26,457 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:47:26,458 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:47:26,458 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:47:26,463 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:47:27,128 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:47:27,271 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 20:47:27,272 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:47:29,131 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:47:30,655 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:47:31,950 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:47:31,951 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:47:31,951 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:47:31,956 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:47:32,133 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:47:32,994 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:47:33,135 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:47:35,345 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:47:35,346 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:47:35,346 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:47:35,351 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:47:35,680 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:47:36,137 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:47:37,139 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:47:38,352 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:47:40,340 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:47:40,341 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:47:40,341 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:47:40,346 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:47:40,680 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:47:41,142 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:47:43,145 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:47:44,347 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:47:45,058 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:47:45,059 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:47:45,059 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:47:45,065 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:47:45,146 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:47:45,655 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:47:47,149 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:47:49,252 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:47:49,252 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:47:49,253 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:47:49,258 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:47:50,151 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:47:50,294 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:47:50,681 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:47:51,154 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:47:55,295 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:47:55,682 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:47:57,276 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 20:47:57,537 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:47:57,538 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:47:57,538 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:47:57,543 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:47:58,159 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:47:59,161 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:48:00,543 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:48:00,655 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:48:05,544 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:48:05,683 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:48:05,813 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:48:05,814 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:48:05,814 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:48:05,819 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:48:06,165 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:48:07,168 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:48:09,381 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:48:09,381 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:48:09,382 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:48:09,386 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:48:10,171 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:48:10,683 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:48:11,171 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:48:11,387 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:48:12,740 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:48:12,740 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:48:12,741 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:48:12,746 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:48:13,172 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:48:15,176 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:48:15,655 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:48:16,647 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:48:16,647 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:48:16,648 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:48:16,648 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:48:16,653 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:48:17,177 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:48:19,180 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:48:20,685 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:48:21,637 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:48:21,638 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:48:21,638 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:48:21,643 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:48:22,182 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:48:22,644 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:48:23,185 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:48:25,685 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:48:27,187 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:48:27,188 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:48:27,188 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:48:27,193 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:48:27,275 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 20:48:28,189 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:48:28,277 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:48:29,191 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:48:30,655 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:48:33,277 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:48:35,651 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:48:35,652 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:48:35,652 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:48:35,657 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:48:35,694 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:48:36,195 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:48:37,198 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:48:38,695 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:48:40,306 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:48:40,307 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:48:40,307 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:48:40,312 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:48:40,687 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:48:41,202 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:48:41,202 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:48:44,312 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:48:44,626 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:48:44,627 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:48:44,627 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:48:44,632 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:48:45,204 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:48:45,655 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:48:47,207 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:48:47,969 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:48:47,969 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:48:47,969 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:48:47,975 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:48:48,208 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:48:49,211 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:48:50,010 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:48:50,688 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:48:51,505 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:48:51,506 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:48:51,506 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:48:51,511 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:48:52,214 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:48:53,217 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:48:55,457 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:48:55,457 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:48:55,458 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:48:55,458 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:48:55,463 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:48:55,689 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:48:56,219 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:48:57,221 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:48:57,278 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 20:48:59,304 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:48:59,308 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:48:59,308 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:48:59,313 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:49:00,223 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:49:00,655 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:49:01,224 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:49:01,313 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:49:02,750 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:49:02,751 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:49:02,751 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:49:02,756 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:49:03,225 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:49:05,228 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:49:05,690 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:49:06,259 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:49:06,260 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:49:06,260 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:49:06,265 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:49:07,231 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:49:07,231 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:49:07,266 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:49:09,537 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:49:09,538 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:49:09,538 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:49:09,543 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:49:10,234 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:49:10,690 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:49:11,236 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:49:12,544 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:49:13,789 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:49:13,790 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:49:13,790 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:49:13,795 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:49:14,238 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:49:15,241 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:49:15,655 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:49:17,242 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:49:17,243 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:49:17,243 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:49:17,248 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:49:18,242 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:49:18,286 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:49:19,245 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:49:20,691 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:49:21,529 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:49:21,529 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:49:21,530 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:49:21,534 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:49:22,247 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:49:23,249 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:49:23,535 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:49:25,585 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:49:25,586 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:49:25,586 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:49:25,591 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:49:25,692 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:49:26,252 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:49:27,255 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:49:27,280 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 20:49:29,275 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:49:29,276 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:49:29,276 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:49:29,276 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:49:29,281 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:49:30,257 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:49:30,655 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:49:31,259 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:49:32,970 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:49:32,971 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:49:32,971 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:49:32,976 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:49:33,260 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:49:35,014 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:49:35,263 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:49:35,693 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:49:35,811 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:49:35,812 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:49:35,812 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:49:35,817 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:49:36,264 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:49:37,267 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:49:39,745 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:49:39,746 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:49:39,746 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:49:39,751 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:49:40,268 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:49:40,694 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:49:40,751 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:49:41,271 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:49:44,817 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:49:44,817 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:49:44,818 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:49:44,822 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:49:45,273 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:49:45,655 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:49:45,823 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:49:47,276 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:49:50,312 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:49:50,313 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:49:50,313 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:49:50,318 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:49:50,695 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:49:51,281 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:49:51,281 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:49:51,354 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:49:54,935 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:49:54,936 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:49:54,936 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:49:54,941 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:49:55,284 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:49:55,695 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:49:56,941 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:49:57,282 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 20:49:57,287 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:49:59,133 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:49:59,134 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:49:59,134 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:49:59,139 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:49:59,289 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:50:00,655 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:50:01,290 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:50:02,139 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:50:02,973 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:50:02,974 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:50:02,974 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:50:02,979 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:50:03,291 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:50:05,294 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:50:05,696 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:50:08,021 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:50:10,697 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:50:11,643 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:50:11,643 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:50:11,644 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:50:11,648 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:50:12,299 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:50:13,300 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:50:13,649 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:50:15,157 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:50:15,157 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:50:15,158 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:50:15,163 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:50:15,301 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:50:15,656 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:50:17,305 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:50:19,163 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:50:20,698 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:50:21,215 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:50:21,216 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:50:21,216 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:50:21,222 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:50:21,307 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:50:23,310 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:50:24,222 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:50:24,231 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:50:24,231 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:50:24,232 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:50:24,237 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:50:24,311 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:50:25,314 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:50:25,698 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:50:27,284 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 20:50:27,946 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:50:27,947 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:50:27,947 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:50:27,952 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:50:28,316 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:50:29,319 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:50:29,953 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:50:30,656 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:50:31,226 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:50:31,227 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:50:31,227 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:50:31,232 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:50:31,320 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:50:33,323 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:50:34,687 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:50:34,688 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:50:34,688 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:50:34,693 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:50:35,324 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:50:35,693 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:50:35,699 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:50:37,327 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:50:40,694 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:50:40,700 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:50:43,758 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:50:43,758 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:50:43,759 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:50:43,763 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:50:44,331 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:50:45,333 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:50:45,656 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:50:45,764 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:50:48,337 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:50:48,337 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:50:48,338 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:50:48,342 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:50:49,338 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:50:49,338 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:50:50,701 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:50:51,378 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:50:52,600 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:50:52,601 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:50:52,601 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:50:52,606 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:50:53,341 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:50:55,344 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:50:55,701 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:50:56,545 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:50:56,546 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:50:56,546 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:50:56,546 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:50:56,551 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:50:57,286 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 20:50:57,345 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:50:59,349 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:51:00,440 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:51:00,440 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:51:00,441 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:51:00,446 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:51:00,656 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:51:01,352 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:51:01,352 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:51:02,446 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:51:05,702 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:51:06,829 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:51:06,829 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:51:06,830 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:51:06,835 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:51:07,355 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:51:07,835 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:51:09,359 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:51:10,305 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:51:10,306 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:51:10,306 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:51:10,311 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:51:10,360 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:51:10,703 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:51:11,362 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:51:13,311 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:51:14,179 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:51:14,180 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:51:14,180 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:51:14,185 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:51:14,364 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:51:15,367 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:51:15,656 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:51:17,819 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:51:17,820 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:51:17,820 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:51:17,825 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:51:18,368 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:51:18,862 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:51:19,371 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:51:20,704 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:51:22,727 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:51:22,727 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:51:22,728 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:51:22,733 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:51:23,373 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:51:24,734 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:51:25,377 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:51:25,704 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:51:27,288 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 20:51:30,289 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:51:30,656 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:51:30,894 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:51:30,895 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:51:30,895 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:51:30,900 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:51:31,380 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:51:33,383 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:51:34,598 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:51:34,599 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:51:34,599 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:51:34,604 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:51:35,384 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:51:35,604 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:51:35,705 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:51:37,387 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:51:39,489 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:51:39,489 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:51:39,490 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:51:39,494 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:51:40,389 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:51:40,706 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:51:41,390 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:51:41,495 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:51:44,675 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:51:44,676 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:51:44,676 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:51:44,681 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:51:45,392 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:51:45,656 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:51:46,681 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:51:47,395 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:51:50,707 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:51:51,682 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:51:52,005 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:51:52,006 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:51:52,006 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:51:52,011 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:51:52,400 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:51:53,403 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:51:55,707 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:51:55,986 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:51:55,987 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:51:55,987 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:51:55,993 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:51:56,404 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:51:56,993 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:51:57,290 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 20:51:57,405 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:51:59,725 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:51:59,726 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:51:59,726 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:51:59,731 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:52:00,407 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:52:00,656 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:52:01,409 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:52:02,731 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:52:03,143 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:52:03,144 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:52:03,144 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:52:03,148 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:52:03,411 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:52:05,414 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:52:05,708 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:52:07,570 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:52:07,571 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:52:07,571 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:52:07,576 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:52:08,415 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:52:08,576 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:52:09,419 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:52:10,709 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:52:11,951 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:52:11,952 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:52:11,952 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:52:11,957 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:52:12,421 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:52:13,423 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:52:13,957 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:52:15,598 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:52:15,598 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:52:15,599 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:52:15,603 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:52:15,656 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:52:16,425 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:52:17,428 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:52:18,707 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:52:18,708 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:52:18,708 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:52:18,713 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:52:19,429 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:52:19,753 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:52:20,710 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:52:21,432 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:52:22,505 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:52:22,506 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:52:22,506 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:52:22,511 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:52:23,435 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:52:23,435 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:52:25,511 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:52:25,711 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:52:26,257 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:52:26,257 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:52:26,258 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:52:26,262 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:52:26,437 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:52:27,292 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 20:52:27,440 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:52:30,614 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:52:30,614 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:52:30,615 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:52:30,615 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:52:30,619 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:52:30,656 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:52:31,442 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:52:33,445 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:52:34,661 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:52:34,661 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:52:34,662 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:52:34,667 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:52:35,446 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:52:35,702 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:52:35,712 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:52:37,449 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:52:38,037 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:52:38,038 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:52:38,038 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:52:38,043 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:52:38,450 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:52:39,453 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:52:40,712 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:52:41,044 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:52:42,424 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:52:42,425 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:52:42,425 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:52:42,430 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:52:42,455 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:52:43,458 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:52:45,656 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:52:46,431 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:52:47,960 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:52:47,960 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:52:47,960 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:52:47,965 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:52:48,461 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:52:49,463 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:52:50,713 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:52:52,002 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:52:54,536 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:52:54,537 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:52:54,538 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:52:54,542 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:52:55,471 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:52:55,471 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:52:56,216 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:52:57,294 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 20:52:57,296 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:53:00,657 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:53:02,296 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:53:03,740 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:53:03,741 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:53:03,741 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:53:03,746 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:53:04,476 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:53:05,479 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:53:06,217 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:53:07,782 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:53:11,217 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:53:12,295 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:53:12,295 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:53:12,296 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:53:12,301 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:53:12,484 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:53:13,301 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:53:13,487 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:53:15,657 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:53:15,841 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:53:15,841 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:53:15,841 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:53:15,846 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:53:16,489 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:53:17,492 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:53:18,669 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:53:18,670 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:53:18,670 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:53:18,670 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:53:18,675 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:53:19,494 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:53:21,218 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:53:21,497 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:53:23,675 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:53:26,219 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:53:26,968 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:53:26,969 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:53:26,969 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:53:26,974 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:53:27,300 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 20:53:27,501 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:53:29,302 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:53:29,504 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:53:30,657 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:53:30,916 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:53:30,917 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:53:30,917 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:53:30,922 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:53:31,505 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:53:33,508 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:53:34,872 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:53:34,873 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:53:34,873 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:53:34,873 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:53:34,878 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:53:35,510 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:53:36,220 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:53:37,513 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:53:38,298 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:53:38,299 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:53:38,299 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:53:38,304 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:53:38,513 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:53:39,516 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:53:40,304 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:53:41,221 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:53:45,305 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:53:45,657 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:53:46,705 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:53:46,705 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:53:46,706 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:53:46,711 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:53:47,521 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:53:49,524 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:53:50,637 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:53:50,638 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:53:50,638 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:53:50,638 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:53:50,643 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:53:51,222 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:53:51,527 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:53:51,528 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:53:53,871 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:53:53,871 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:53:53,871 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:53:53,876 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:53:54,530 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:53:55,533 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:53:55,877 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:53:56,222 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:53:57,298 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 20:53:58,149 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:53:58,149 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:53:58,150 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:53:58,155 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:53:58,535 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:53:59,537 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:54:00,657 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:54:01,156 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:54:02,197 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:54:02,198 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:54:02,198 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:54:02,203 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:54:02,539 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:54:03,541 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:54:05,878 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:54:05,878 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:54:05,879 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:54:05,884 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:54:06,223 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:54:06,543 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:54:06,884 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:54:07,546 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:54:11,224 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:54:11,884 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:54:14,166 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:54:14,167 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:54:14,167 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:54:14,172 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:54:14,551 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:54:15,554 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:54:15,657 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:54:17,173 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:54:19,498 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:54:19,499 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:54:19,499 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:54:19,504 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:54:19,556 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:54:21,225 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:54:21,559 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:54:22,542 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:54:23,561 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:54:23,561 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:54:23,562 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:54:23,567 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:54:24,561 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:54:25,564 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:54:26,225 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:54:27,303 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 20:54:28,304 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:54:30,657 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:54:32,100 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:54:32,101 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:54:32,101 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:54:32,106 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:54:32,568 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:54:33,568 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:54:34,142 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:54:36,226 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:54:36,642 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:54:36,642 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:54:36,642 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:54:36,647 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:54:37,572 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:54:37,573 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:54:39,648 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:54:40,615 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:54:40,616 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:54:40,616 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:54:40,621 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:54:41,227 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:54:41,576 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:54:41,577 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:54:44,560 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:54:44,561 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:54:44,561 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:54:44,566 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:54:44,578 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:54:45,566 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:54:45,579 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:54:45,657 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:54:48,095 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:54:48,096 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:54:48,096 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:54:48,101 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:54:48,581 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:54:49,584 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:54:51,138 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:54:51,228 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:54:51,599 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:54:51,600 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:54:51,600 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:54:51,605 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:54:52,587 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:54:53,589 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:54:55,375 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:54:55,376 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:54:55,376 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:54:55,381 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:54:55,590 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:54:56,228 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:54:56,381 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:54:57,302 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 20:54:57,593 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:55:00,225 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:55:00,225 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:55:00,226 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:55:00,230 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:55:00,595 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:55:00,657 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:55:01,596 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:55:02,231 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:55:04,066 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:55:04,067 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:55:04,067 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:55:04,072 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:55:04,598 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:55:05,600 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:55:06,230 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:55:08,111 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:55:09,808 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:55:09,809 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:55:09,809 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:55:09,814 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:55:10,605 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:55:11,230 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:55:11,607 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:55:13,814 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:55:15,657 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:55:18,784 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:55:18,785 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:55:18,785 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:55:18,790 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:55:18,826 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:55:19,612 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:55:21,231 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:55:21,615 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:55:22,227 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:55:22,227 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:55:22,228 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:55:22,233 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:55:22,615 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:55:23,618 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:55:24,233 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:55:25,948 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:55:25,949 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:55:25,949 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:55:25,954 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:55:26,232 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:55:26,620 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:55:27,307 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 20:55:27,623 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:55:29,309 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:55:29,865 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:55:29,865 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:55:29,866 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:55:29,871 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:55:30,624 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:55:30,657 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:55:31,625 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:55:33,548 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:55:33,548 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:55:33,549 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:55:33,553 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:55:33,626 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:55:34,590 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:55:35,630 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:55:36,233 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:55:39,591 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:55:40,902 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:55:40,903 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:55:40,903 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:55:40,908 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:55:41,233 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:55:41,633 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:55:43,636 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:55:44,908 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:55:45,641 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:55:45,642 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:55:45,642 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:55:45,647 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:55:45,657 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:55:46,638 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:55:47,641 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:55:49,574 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:55:49,574 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:55:49,575 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:55:49,580 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:55:49,642 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:55:50,618 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:55:51,234 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:55:51,645 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:55:53,233 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:55:53,233 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:55:53,233 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:55:53,239 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:55:53,648 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:55:55,651 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:55:56,235 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:55:56,239 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:55:57,306 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 20:55:57,355 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:55:57,356 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:55:57,356 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:55:57,361 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:55:57,652 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:55:59,655 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:56:00,657 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:56:01,362 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:56:01,804 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:56:01,805 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:56:01,805 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:56:01,810 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:56:02,657 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:56:03,660 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:56:05,530 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:56:05,531 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:56:05,531 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:56:05,536 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:56:05,661 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:56:06,236 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:56:06,536 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:56:07,662 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:56:09,445 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:56:09,446 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:56:09,446 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:56:09,451 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:56:09,664 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:56:11,237 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:56:11,668 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:56:12,452 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:56:13,611 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:56:13,611 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:56:13,612 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:56:13,616 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:56:13,669 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:56:15,657 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:56:15,672 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:56:17,617 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:56:21,238 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:56:21,896 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:56:21,897 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:56:21,897 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:56:21,902 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:56:22,676 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:56:22,902 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:56:23,678 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:56:26,238 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:56:27,311 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 20:56:28,312 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:56:30,323 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:56:30,324 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:56:30,325 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:56:30,329 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:56:30,658 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:56:30,683 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:56:31,686 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:56:33,330 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:56:34,038 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:56:34,038 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:56:34,038 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:56:34,043 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:56:34,687 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:56:35,690 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:56:36,239 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:56:39,083 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:56:39,357 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:56:39,358 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:56:39,358 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:56:39,364 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:56:39,692 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:56:41,240 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:56:41,695 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:56:42,827 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:56:42,828 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:56:42,828 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:56:42,833 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:56:43,698 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:56:43,699 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:56:44,834 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:56:45,658 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:56:46,609 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:56:46,610 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:56:46,610 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:56:46,615 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:56:46,700 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:56:47,703 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:56:50,654 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:56:51,241 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:56:51,393 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:56:51,394 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:56:51,394 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:56:51,399 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:56:51,706 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:56:53,710 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:56:54,784 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:56:54,785 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:56:54,785 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:56:54,790 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:56:55,713 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:56:55,713 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:56:55,790 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:56:56,241 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:56:57,311 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 20:57:00,305 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:57:00,306 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:57:00,306 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:57:00,311 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:57:00,658 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:57:00,716 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:57:01,312 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:57:01,717 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:57:04,384 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:57:04,384 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:57:04,385 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:57:04,389 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:57:04,718 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:57:05,721 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:57:06,242 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:57:06,426 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:57:07,905 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:57:07,906 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:57:07,906 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:57:07,911 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:57:08,723 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:57:09,726 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:57:11,243 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:57:11,912 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:57:12,754 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:57:12,755 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:57:12,755 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:57:12,761 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:57:13,731 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:57:13,731 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:57:15,658 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:57:17,523 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:57:17,524 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:57:17,524 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:57:17,524 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:57:17,528 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:57:17,733 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:57:19,736 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:57:21,244 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:57:22,567 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:57:23,412 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:57:23,413 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:57:23,413 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:57:23,418 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:57:23,738 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:57:25,742 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:57:26,244 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:57:27,143 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:57:27,144 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:57:27,145 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:57:27,150 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:57:27,316 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 20:57:27,743 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:57:28,317 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:57:29,747 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:57:30,658 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:57:30,752 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:57:30,753 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:57:30,753 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:57:30,758 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:57:31,750 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:57:31,750 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:57:33,794 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:57:34,275 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:57:34,276 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:57:34,276 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:57:34,281 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:57:34,751 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:57:35,754 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:57:36,245 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:57:38,863 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:57:38,864 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:57:38,864 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:57:38,864 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:57:38,869 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:57:39,758 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:57:39,758 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:57:41,382 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:57:42,940 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:57:42,941 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:57:42,941 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:57:42,946 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:57:43,760 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:57:43,947 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:57:45,658 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:57:45,763 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:57:46,609 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:57:46,610 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:57:46,610 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:57:46,615 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:57:46,764 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:57:47,766 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:57:49,655 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:57:50,102 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:57:50,103 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:57:50,103 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:57:50,108 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:57:50,768 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:57:51,383 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:57:51,771 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:57:53,443 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:57:53,443 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:57:53,443 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:57:53,448 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:57:53,773 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:57:55,449 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:57:55,776 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:57:56,383 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:57:57,315 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 20:57:57,771 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:57:57,772 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:57:57,772 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:57:57,776 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:57:57,777 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:57:59,780 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:58:00,658 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:58:00,777 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:58:05,778 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:58:06,031 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:58:06,032 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:58:06,032 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:58:06,037 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:58:06,384 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:58:06,784 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:58:07,787 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:58:09,128 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:58:09,129 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:58:09,129 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:58:09,134 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:58:09,789 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:58:11,135 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:58:11,385 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:58:11,792 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:58:12,170 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:58:12,171 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:58:12,171 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:58:12,176 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:58:12,792 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:58:13,796 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:58:15,658 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:58:16,177 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:58:17,781 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:58:17,782 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:58:17,782 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:58:17,787 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:58:17,798 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:58:19,801 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:58:21,386 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:58:21,831 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:58:22,338 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:58:22,338 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:58:22,339 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:58:22,343 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:58:22,803 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:58:23,805 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:58:26,377 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:58:26,378 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:58:26,378 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:58:26,383 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:58:26,386 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:58:26,808 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:58:27,320 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 20:58:27,321 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:58:27,810 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:58:30,658 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:58:30,948 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:58:30,948 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:58:30,949 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:58:30,953 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:58:31,815 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:58:31,815 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:58:32,990 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:58:34,903 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:58:34,904 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:58:34,904 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:58:34,909 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:58:35,819 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:58:35,819 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:58:36,387 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:58:38,909 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:58:39,675 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:58:39,676 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:58:39,676 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:58:39,681 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:58:39,821 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:58:41,388 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:58:41,824 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:58:44,681 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:58:45,037 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:58:45,038 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:58:45,039 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:58:45,044 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:58:45,658 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:58:45,826 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:58:47,829 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:58:50,045 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:58:51,389 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:58:53,458 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:58:53,459 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:58:53,459 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:58:53,464 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:58:53,834 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:58:55,464 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:58:55,835 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:58:56,389 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:58:57,319 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 20:58:59,330 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:58:59,330 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:58:59,331 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:58:59,335 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:58:59,838 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:59:00,658 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:59:01,336 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:59:01,840 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:59:02,999 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:59:03,000 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:59:03,000 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:59:03,005 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:59:03,842 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:59:05,843 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:59:06,390 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:59:07,043 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:59:07,851 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:59:07,852 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:59:07,852 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:59:07,857 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:59:08,845 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:59:09,848 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:59:11,391 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:59:11,851 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:59:11,852 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:59:11,852 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:59:11,857 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:59:12,850 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:59:12,858 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:59:13,852 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:59:15,658 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:59:16,875 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:59:16,876 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:59:16,876 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:59:16,881 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:59:17,856 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:59:17,857 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:59:17,918 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:59:20,673 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:59:20,673 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:59:20,673 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:59:20,678 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:59:20,858 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:59:21,392 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:59:21,860 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:59:23,679 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:59:26,154 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:59:26,154 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:59:26,155 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:59:26,160 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:59:26,392 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:59:26,864 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:59:27,321 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 20:59:27,866 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:59:29,322 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:59:30,658 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:59:34,322 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:59:34,637 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:59:34,637 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:59:34,637 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:59:34,642 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:59:34,870 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:59:35,872 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:59:36,393 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:59:38,171 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:59:38,172 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:59:38,172 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:59:38,176 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:59:38,874 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:59:39,875 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:59:40,177 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:59:41,394 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:59:41,751 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:59:41,752 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:59:41,752 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:59:41,757 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:59:41,876 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:59:43,879 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:59:44,664 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:59:44,665 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:59:44,665 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:59:44,670 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:59:44,880 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:59:45,659 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:59:45,670 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:59:45,882 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:59:48,669 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:59:48,670 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 20:59:48,670 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:59:48,676 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:59:48,884 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 20:59:49,887 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 20:59:50,715 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:59:51,395 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:59:55,715 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:59:56,395 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 20:59:57,323 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 21:00:00,659 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:00:01,324 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:00:06,324 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:00:06,396 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 21:00:11,325 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:00:11,550 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 21:00:16,094 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:00:16,325 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:00:21,326 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:00:26,326 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:00:27,325 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 21:00:29,686 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 21:00:30,958 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:00:31,327 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:00:36,080 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 21:00:36,327 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:00:41,328 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:00:46,328 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:00:51,328 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:00:54,793 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 21:00:54,793 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 21:00:54,793 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:00:56,329 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:00:57,327 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 21:00:59,794 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 21:01:02,328 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:01:04,794 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 21:01:07,329 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:01:09,793 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:01:12,329 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:01:14,795 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 21:01:17,330 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:01:19,795 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 21:01:22,330 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:01:24,793 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:01:27,330 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:01:27,332 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 21:01:29,796 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 21:01:32,334 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:01:34,796 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 21:01:37,334 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:01:39,793 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:01:42,334 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:01:44,797 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 21:01:47,335 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:01:49,798 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 21:01:52,335 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:01:54,793 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:01:57,331 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 21:01:58,332 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:01:59,798 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 21:02:03,333 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:02:04,799 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 21:02:08,333 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:02:09,794 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:02:13,333 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:02:14,800 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 21:02:18,334 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:02:19,800 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 21:02:23,334 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:02:24,794 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:02:27,333 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 21:02:28,334 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:02:29,801 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 21:02:33,335 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:02:34,801 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 21:02:38,335 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:02:39,794 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:02:43,335 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:02:44,802 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 21:02:48,336 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:02:49,802 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 21:02:53,336 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:02:54,794 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:02:57,337 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 21:02:58,339 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:02:59,803 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 21:03:03,339 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:03:04,804 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 21:03:08,339 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:03:09,794 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:03:13,340 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:03:14,804 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 21:03:18,340 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:03:19,805 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 21:03:23,340 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:03:24,794 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:03:27,337 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 21:03:29,338 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:03:29,806 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 21:03:34,338 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:03:34,806 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 21:03:39,339 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:03:39,794 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:03:44,339 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:03:44,807 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 21:03:49,339 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:03:49,807 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 21:03:54,340 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:03:54,794 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:03:57,339 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 21:03:59,340 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:03:59,808 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 21:04:04,341 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:04:04,808 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: keepalive +2024-03-12 21:04:09,341 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:04:09,794 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:04:14,091 DEBUG SenderThread:452226 [sender.py:send():382] send: telemetry +2024-03-12 21:04:14,091 DEBUG SenderThread:452226 [sender.py:send():382] send: exit +2024-03-12 21:04:14,091 INFO SenderThread:452226 [sender.py:send_exit():589] handling exit code: 0 +2024-03-12 21:04:14,091 INFO SenderThread:452226 [sender.py:send_exit():591] handling runtime: 23718 +2024-03-12 21:04:14,096 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:04:14,096 INFO SenderThread:452226 [sender.py:send_exit():597] send defer +2024-03-12 21:04:14,096 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: defer +2024-03-12 21:04:14,096 INFO HandlerThread:452226 [handler.py:handle_request_defer():172] handle defer: 0 +2024-03-12 21:04:14,096 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: defer +2024-03-12 21:04:14,096 INFO SenderThread:452226 [sender.py:send_request_defer():613] handle sender defer: 0 +2024-03-12 21:04:14,097 INFO SenderThread:452226 [sender.py:transition_state():617] send defer: 1 +2024-03-12 21:04:14,097 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: defer +2024-03-12 21:04:14,097 INFO HandlerThread:452226 [handler.py:handle_request_defer():172] handle defer: 1 +2024-03-12 21:04:14,097 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: defer +2024-03-12 21:04:14,097 INFO SenderThread:452226 [sender.py:send_request_defer():613] handle sender defer: 1 +2024-03-12 21:04:14,097 INFO SenderThread:452226 [sender.py:transition_state():617] send defer: 2 +2024-03-12 21:04:14,097 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: defer +2024-03-12 21:04:14,097 INFO HandlerThread:452226 [handler.py:handle_request_defer():172] handle defer: 2 +2024-03-12 21:04:14,097 INFO HandlerThread:452226 [system_monitor.py:finish():203] Stopping system monitor +2024-03-12 21:04:14,097 DEBUG SystemMonitor:452226 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2024-03-12 21:04:14,097 DEBUG SystemMonitor:452226 [system_monitor.py:_start():183] Publishing last batch of metrics +2024-03-12 21:04:14,100 INFO HandlerThread:452226 [interfaces.py:finish():202] Joined cpu monitor +2024-03-12 21:04:14,100 INFO HandlerThread:452226 [interfaces.py:finish():202] Joined disk monitor +2024-03-12 21:04:14,736 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 21:04:15,667 INFO HandlerThread:452226 [interfaces.py:finish():202] Joined gpu monitor +2024-03-12 21:04:15,667 INFO HandlerThread:452226 [interfaces.py:finish():202] Joined memory monitor +2024-03-12 21:04:15,667 INFO HandlerThread:452226 [interfaces.py:finish():202] Joined network monitor +2024-03-12 21:04:15,667 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: poll_exit +2024-03-12 21:04:15,667 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:04:15,667 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: defer +2024-03-12 21:04:15,668 INFO SenderThread:452226 [sender.py:send_request_defer():613] handle sender defer: 2 +2024-03-12 21:04:15,668 INFO SenderThread:452226 [sender.py:transition_state():617] send defer: 3 +2024-03-12 21:04:15,668 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: defer +2024-03-12 21:04:15,668 INFO HandlerThread:452226 [handler.py:handle_request_defer():172] handle defer: 3 +2024-03-12 21:04:15,671 DEBUG SenderThread:452226 [sender.py:send():382] send: stats +2024-03-12 21:04:15,672 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: poll_exit +2024-03-12 21:04:15,672 DEBUG SenderThread:452226 [sender.py:send():382] send: history +2024-03-12 21:04:15,672 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:04:15,677 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:04:15,677 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: defer +2024-03-12 21:04:15,677 INFO SenderThread:452226 [sender.py:send_request_defer():613] handle sender defer: 3 +2024-03-12 21:04:15,677 INFO SenderThread:452226 [sender.py:transition_state():617] send defer: 4 +2024-03-12 21:04:15,677 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: defer +2024-03-12 21:04:15,677 INFO HandlerThread:452226 [handler.py:handle_request_defer():172] handle defer: 4 +2024-03-12 21:04:15,677 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: defer +2024-03-12 21:04:15,678 INFO SenderThread:452226 [sender.py:send_request_defer():613] handle sender defer: 4 +2024-03-12 21:04:15,678 INFO SenderThread:452226 [sender.py:transition_state():617] send defer: 5 +2024-03-12 21:04:15,678 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: defer +2024-03-12 21:04:15,678 INFO HandlerThread:452226 [handler.py:handle_request_defer():172] handle defer: 5 +2024-03-12 21:04:15,678 DEBUG SenderThread:452226 [sender.py:send():382] send: summary +2024-03-12 21:04:15,683 INFO SenderThread:452226 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:04:15,683 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: defer +2024-03-12 21:04:15,683 INFO SenderThread:452226 [sender.py:send_request_defer():613] handle sender defer: 5 +2024-03-12 21:04:15,683 INFO SenderThread:452226 [sender.py:transition_state():617] send defer: 6 +2024-03-12 21:04:15,683 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: defer +2024-03-12 21:04:15,683 INFO HandlerThread:452226 [handler.py:handle_request_defer():172] handle defer: 6 +2024-03-12 21:04:15,683 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: defer +2024-03-12 21:04:15,683 INFO SenderThread:452226 [sender.py:send_request_defer():613] handle sender defer: 6 +2024-03-12 21:04:15,685 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:04:15,739 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 21:04:15,739 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 21:04:16,059 INFO SenderThread:452226 [sender.py:transition_state():617] send defer: 7 +2024-03-12 21:04:16,060 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: defer +2024-03-12 21:04:16,060 INFO HandlerThread:452226 [handler.py:handle_request_defer():172] handle defer: 7 +2024-03-12 21:04:16,060 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: defer +2024-03-12 21:04:16,060 INFO SenderThread:452226 [sender.py:send_request_defer():613] handle sender defer: 7 +2024-03-12 21:04:16,067 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: poll_exit +2024-03-12 21:04:16,741 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 21:04:16,741 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/config.yaml +2024-03-12 21:04:17,399 INFO SenderThread:452226 [sender.py:transition_state():617] send defer: 8 +2024-03-12 21:04:17,399 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: poll_exit +2024-03-12 21:04:17,399 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: defer +2024-03-12 21:04:17,399 INFO HandlerThread:452226 [handler.py:handle_request_defer():172] handle defer: 8 +2024-03-12 21:04:17,400 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: defer +2024-03-12 21:04:17,400 INFO SenderThread:452226 [sender.py:send_request_defer():613] handle sender defer: 8 +2024-03-12 21:04:17,400 INFO SenderThread:452226 [job_builder.py:build():296] Attempting to build job artifact +2024-03-12 21:04:17,401 INFO SenderThread:452226 [job_builder.py:_get_source_type():426] is repo sourced job +2024-03-12 21:04:17,491 INFO SenderThread:452226 [job_builder.py:build():402] adding wandb-job metadata file +2024-03-12 21:04:17,523 INFO SenderThread:452226 [sender.py:transition_state():617] send defer: 9 +2024-03-12 21:04:17,523 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: defer +2024-03-12 21:04:17,523 INFO HandlerThread:452226 [handler.py:handle_request_defer():172] handle defer: 9 +2024-03-12 21:04:17,523 DEBUG SenderThread:452226 [sender.py:send():382] send: artifact +2024-03-12 21:04:17,742 INFO Thread-12 :452226 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 21:04:18,068 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: poll_exit +2024-03-12 21:04:18,894 INFO wandb-upload_1:452226 [upload_job.py:push():86] Skipped uploading /home/lilei/.local/share/wandb/artifacts/staging/tmpezq3c_ub +2024-03-12 21:04:19,386 INFO wandb-upload_0:452226 [upload_job.py:push():89] Uploaded file /home/lilei/.local/share/wandb/artifacts/staging/tmp7rp7uu2c +2024-03-12 21:04:20,955 INFO SenderThread:452226 [sender.py:send_artifact():1494] sent artifact job-https___github.com_TRI-ML_prismatic-vlms.git_scripts_pretrain.py - {'id': 'QXJ0aWZhY3Q6NzUzNTUzNzEx', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjE0MzcwMzk1OA==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6NzUzMTIzODUw', 'versionIndex': 6}}} +2024-03-12 21:04:20,955 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: defer +2024-03-12 21:04:20,955 INFO SenderThread:452226 [sender.py:send_request_defer():613] handle sender defer: 9 +2024-03-12 21:04:20,955 INFO SenderThread:452226 [dir_watcher.py:finish():358] shutting down directory watcher +2024-03-12 21:04:20,955 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:04:21,743 INFO SenderThread:452226 [dir_watcher.py:finish():388] scan: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files +2024-03-12 21:04:21,743 INFO SenderThread:452226 [dir_watcher.py:finish():402] scan save: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/config.yaml config.yaml +2024-03-12 21:04:21,744 INFO SenderThread:452226 [dir_watcher.py:finish():402] scan save: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/conda-environment.yaml conda-environment.yaml +2024-03-12 21:04:21,744 INFO SenderThread:452226 [dir_watcher.py:finish():402] scan save: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-metadata.json wandb-metadata.json +2024-03-12 21:04:21,744 INFO SenderThread:452226 [dir_watcher.py:finish():402] scan save: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/requirements.txt requirements.txt +2024-03-12 21:04:21,744 INFO SenderThread:452226 [dir_watcher.py:finish():402] scan save: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log output.log +2024-03-12 21:04:21,744 INFO SenderThread:452226 [dir_watcher.py:finish():402] scan save: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json wandb-summary.json +2024-03-12 21:04:21,744 INFO SenderThread:452226 [sender.py:transition_state():617] send defer: 10 +2024-03-12 21:04:21,744 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: poll_exit +2024-03-12 21:04:21,744 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: defer +2024-03-12 21:04:21,747 INFO HandlerThread:452226 [handler.py:handle_request_defer():172] handle defer: 10 +2024-03-12 21:04:21,752 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: defer +2024-03-12 21:04:21,753 INFO SenderThread:452226 [sender.py:send_request_defer():613] handle sender defer: 10 +2024-03-12 21:04:21,753 INFO SenderThread:452226 [file_pusher.py:finish():172] shutting down file pusher +2024-03-12 21:04:22,068 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: poll_exit +2024-03-12 21:04:22,068 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: poll_exit +2024-03-12 21:04:22,284 INFO wandb-upload_3:452226 [upload_job.py:push():131] Uploaded file runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/requirements.txt +2024-03-12 21:04:22,567 INFO wandb-upload_0:452226 [upload_job.py:push():131] Uploaded file runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/wandb-summary.json +2024-03-12 21:04:22,588 INFO wandb-upload_1:452226 [upload_job.py:push():131] Uploaded file runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/conda-environment.yaml +2024-03-12 21:04:22,603 INFO wandb-upload_2:452226 [upload_job.py:push():131] Uploaded file runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/config.yaml +2024-03-12 21:04:22,650 INFO wandb-upload_4:452226 [upload_job.py:push():131] Uploaded file runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/files/output.log +2024-03-12 21:04:22,851 INFO Thread-11 (_thread_body):452226 [sender.py:transition_state():617] send defer: 11 +2024-03-12 21:04:22,851 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: defer +2024-03-12 21:04:22,851 INFO HandlerThread:452226 [handler.py:handle_request_defer():172] handle defer: 11 +2024-03-12 21:04:22,851 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: defer +2024-03-12 21:04:22,851 INFO SenderThread:452226 [sender.py:send_request_defer():613] handle sender defer: 11 +2024-03-12 21:04:22,851 INFO SenderThread:452226 [file_pusher.py:join():178] waiting for file pusher +2024-03-12 21:04:22,852 INFO SenderThread:452226 [sender.py:transition_state():617] send defer: 12 +2024-03-12 21:04:22,853 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: defer +2024-03-12 21:04:22,853 INFO HandlerThread:452226 [handler.py:handle_request_defer():172] handle defer: 12 +2024-03-12 21:04:22,853 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: defer +2024-03-12 21:04:22,853 INFO SenderThread:452226 [sender.py:send_request_defer():613] handle sender defer: 12 +2024-03-12 21:04:22,853 INFO SenderThread:452226 [file_stream.py:finish():595] file stream finish called +2024-03-12 21:04:23,068 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: poll_exit +2024-03-12 21:04:23,505 INFO SenderThread:452226 [file_stream.py:finish():599] file stream finish is done +2024-03-12 21:04:23,505 INFO SenderThread:452226 [sender.py:transition_state():617] send defer: 13 +2024-03-12 21:04:23,505 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: poll_exit +2024-03-12 21:04:23,505 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: defer +2024-03-12 21:04:23,506 INFO HandlerThread:452226 [handler.py:handle_request_defer():172] handle defer: 13 +2024-03-12 21:04:23,506 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: defer +2024-03-12 21:04:23,506 INFO SenderThread:452226 [sender.py:send_request_defer():613] handle sender defer: 13 +2024-03-12 21:04:23,506 INFO SenderThread:452226 [sender.py:transition_state():617] send defer: 14 +2024-03-12 21:04:23,506 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: defer +2024-03-12 21:04:23,506 INFO HandlerThread:452226 [handler.py:handle_request_defer():172] handle defer: 14 +2024-03-12 21:04:23,506 DEBUG SenderThread:452226 [sender.py:send():382] send: final +2024-03-12 21:04:23,506 DEBUG SenderThread:452226 [sender.py:send():382] send: footer +2024-03-12 21:04:23,507 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: defer +2024-03-12 21:04:23,507 INFO SenderThread:452226 [sender.py:send_request_defer():613] handle sender defer: 14 +2024-03-12 21:04:23,507 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: poll_exit +2024-03-12 21:04:23,507 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: poll_exit +2024-03-12 21:04:23,507 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:04:23,508 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: server_info +2024-03-12 21:04:23,508 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: server_info +2024-03-12 21:04:23,550 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: get_summary +2024-03-12 21:04:23,551 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: sampled_history +2024-03-12 21:04:23,552 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: job_info +2024-03-12 21:04:23,699 DEBUG SenderThread:452226 [sender.py:send_request():409] send_request: job_info +2024-03-12 21:04:23,699 DEBUG HandlerThread:452226 [handler.py:handle_request():146] handle_request: shutdown +2024-03-12 21:04:23,699 INFO HandlerThread:452226 [handler.py:finish():869] shutting down handler +2024-03-12 21:04:24,552 INFO WriterThread:452226 [datastore.py:close():296] close: runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/run-srovbvi2.wandb +2024-03-12 21:04:24,699 INFO SenderThread:452226 [sender.py:finish():1572] shutting down sender +2024-03-12 21:04:24,699 INFO SenderThread:452226 [file_pusher.py:finish():172] shutting down file pusher +2024-03-12 21:04:24,699 INFO SenderThread:452226 [file_pusher.py:join():178] waiting for file pusher diff --git a/qformer2_32/wandb/run-20240312_142855-srovbvi2/logs/debug.log b/qformer2_32/wandb/run-20240312_142855-srovbvi2/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..ff8477fdccdd88c531c6a5fbc3fd22aaffd40692 --- /dev/null +++ b/qformer2_32/wandb/run-20240312_142855-srovbvi2/logs/debug.log @@ -0,0 +1,34 @@ +2024-03-12 14:28:55,053 INFO MainThread:451919 [wandb_setup.py:_flush():76] Current SDK version is 0.16.3 +2024-03-12 14:28:55,053 INFO MainThread:451919 [wandb_setup.py:_flush():76] Configure stats pid to 451919 +2024-03-12 14:28:55,053 INFO MainThread:451919 [wandb_setup.py:_flush():76] Loading settings from /home/lilei/.config/wandb/settings +2024-03-12 14:28:55,053 INFO MainThread:451919 [wandb_setup.py:_flush():76] Loading settings from /home/lilei/prismatic-vlms/wandb/settings +2024-03-12 14:28:55,053 INFO MainThread:451919 [wandb_setup.py:_flush():76] Loading settings from environment variables: {} +2024-03-12 14:28:55,053 INFO MainThread:451919 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False} +2024-03-12 14:28:55,053 INFO MainThread:451919 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'scripts/pretrain.py', 'program_abspath': '/home/lilei/prismatic-vlms/scripts/pretrain.py', 'program': '/home/lilei/prismatic-vlms/scripts/pretrain.py'} +2024-03-12 14:28:55,053 INFO MainThread:451919 [wandb_init.py:_log_setup():526] Logging user logs to runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/logs/debug.log +2024-03-12 14:28:55,053 INFO MainThread:451919 [wandb_init.py:_log_setup():527] Logging internal logs to runs/qformer2_32/wandb/run-20240312_142855-srovbvi2/logs/debug-internal.log +2024-03-12 14:28:55,054 INFO MainThread:451919 [wandb_init.py:init():566] calling init triggers +2024-03-12 14:28:55,054 INFO MainThread:451919 [wandb_init.py:init():573] wandb.init called with sweep_config: {} +config: {'model': {'type': 'one-stage+7b', 'model_id': 'one-stage+7b_qformer2_32', 'arch_specifier': 'qformer2_32', 'vision_backbone_id': 'clip-vit-l-336px', 'llm_backbone_id': 'vicuna-v15-7b', 'image_resize_strategy': 'letterbox', 'llm_max_length': 2048, 'align_epochs': 1, 'align_max_steps': None, 'align_global_batch_size': 256, 'align_per_device_batch_size': 16, 'align_learning_rate': 0.001, 'align_weight_decay': 0.0, 'align_max_grad_norm': 1.0, 'align_lr_scheduler_type': 'linear-warmup+cosine-decay', 'align_warmup_ratio': 0.03, 'align_train_strategy': 'fsdp-shard-grad-op', 'finetune_epochs': 1, 'finetune_max_steps': None, 'finetune_global_batch_size': 128, 'finetune_per_device_batch_size': 16, 'finetune_learning_rate': 2e-05, 'finetune_weight_decay': 0.1, 'finetune_max_grad_norm': 1.0, 'finetune_lr_scheduler_type': 'linear-warmup+cosine-decay', 'finetune_warmup_ratio': 0.03, 'finetune_train_strategy': 'fsdp-full-shard', 'enable_gradient_checkpointing': True, 'enable_mixed_precision_training': True, 'reduce_in_full_precision': False}, 'dataset': {'type': 'llava-v15', 'dataset_id': 'llava-v15', 'align_stage_components': ['download/llava-laion-cc-sbu-558k/chat.json', 'download/llava-laion-cc-sbu-558k'], 'finetune_stage_components': ['download/llava-v1.5-instruct/llava_v1_5_mix665k.json', 'download/llava-v1.5-instruct'], 'dataset_root_dir': 'data'}, 'stage': 'finetune', 'pretrained_checkpoint': None, 'run_id': 'qformer2_32', 'run_root_dir': 'runs', 'seed': 7, 'hf_token': '.hf_token', 'trackers': ['jsonl', 'wandb'], 'wandb_project': 'hf-vlms', 'wandb_entity': 'lilei_stones', 'max_length': 4096} +2024-03-12 14:28:55,054 INFO MainThread:451919 [wandb_init.py:init():616] starting backend +2024-03-12 14:28:55,054 INFO MainThread:451919 [wandb_init.py:init():620] setting up manager +2024-03-12 14:28:55,061 INFO MainThread:451919 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2024-03-12 14:28:55,064 INFO MainThread:451919 [wandb_init.py:init():628] backend started and connected +2024-03-12 14:28:55,078 INFO MainThread:451919 [wandb_init.py:init():720] updated telemetry +2024-03-12 14:28:55,097 INFO MainThread:451919 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout +2024-03-12 14:28:55,589 INFO MainThread:451919 [wandb_run.py:_on_init():2262] communicating current version +2024-03-12 14:28:55,635 INFO MainThread:451919 [wandb_run.py:_on_init():2271] got version response upgrade_message: "wandb version 0.16.4 is available! To upgrade, please run:\n $ pip install wandb --upgrade" + +2024-03-12 14:28:55,635 INFO MainThread:451919 [wandb_init.py:init():804] starting run threads in backend +2024-03-12 14:29:00,142 INFO MainThread:451919 [wandb_run.py:_console_start():2241] atexit reg +2024-03-12 14:29:00,143 INFO MainThread:451919 [wandb_run.py:_redirect():2096] redirect: wrap_raw +2024-03-12 14:29:00,143 INFO MainThread:451919 [wandb_run.py:_redirect():2161] Wrapping output streams. +2024-03-12 14:29:00,143 INFO MainThread:451919 [wandb_run.py:_redirect():2186] Redirects installed. +2024-03-12 14:29:00,144 INFO MainThread:451919 [wandb_init.py:init():847] run started, returning control to user process +2024-03-12 21:04:14,065 INFO MainThread:451919 [wandb_run.py:_finish():1970] finishing run lilei_stones/hf-vlms/srovbvi2 +2024-03-12 21:04:14,066 INFO MainThread:451919 [wandb_run.py:_atexit_cleanup():2210] got exitcode: 0 +2024-03-12 21:04:14,066 INFO MainThread:451919 [wandb_run.py:_restore():2193] restore +2024-03-12 21:04:14,066 INFO MainThread:451919 [wandb_run.py:_restore():2199] restore done +2024-03-12 21:04:24,701 INFO MainThread:451919 [wandb_run.py:_footer_history_summary_info():3866] rendering history +2024-03-12 21:04:24,701 INFO MainThread:451919 [wandb_run.py:_footer_history_summary_info():3898] rendering summary +2024-03-12 21:04:24,711 INFO MainThread:451919 [wandb_run.py:_footer_sync_info():3825] logging synced files diff --git a/qformer2_32/wandb/run-20240312_142855-srovbvi2/run-srovbvi2.wandb b/qformer2_32/wandb/run-20240312_142855-srovbvi2/run-srovbvi2.wandb new file mode 100644 index 0000000000000000000000000000000000000000..0d275e8cad99d65f6f14726dc6489ecad73b14ed --- /dev/null +++ b/qformer2_32/wandb/run-20240312_142855-srovbvi2/run-srovbvi2.wandb @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6e59e203125155c4eb5742464b2ab1cf73335116163c12b8b6e50ee1085ca198 +size 11896221 diff --git a/qformer2_64/checkpoints/latest-checkpoint.pt b/qformer2_64/checkpoints/latest-checkpoint.pt new file mode 100644 index 0000000000000000000000000000000000000000..5dbba77d01bc92ef3fe92f75e2e6e1071231829a --- /dev/null +++ b/qformer2_64/checkpoints/latest-checkpoint.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7522871138debb6048c8e78fa90be9138724dab2a46625991945b49a21841a54 +size 28821360822 diff --git a/qformer2_64/config.json b/qformer2_64/config.json new file mode 100644 index 0000000000000000000000000000000000000000..551a9bca12c504b9f3553867b7952fdb5b003180 --- /dev/null +++ b/qformer2_64/config.json @@ -0,0 +1,60 @@ +{ + "dataset": { + "align_stage_components": [ + "download/llava-laion-cc-sbu-558k/chat.json", + "download/llava-laion-cc-sbu-558k" + ], + "dataset_id": "llava-v15", + "dataset_root_dir": "data", + "finetune_stage_components": [ + "download/llava-v1.5-instruct/llava_v1_5_mix665k.json", + "download/llava-v1.5-instruct" + ], + "type": "llava-v15" + }, + "hf_token": ".hf_token", + "max_length": 4096, + "model": { + "align_epochs": 1, + "align_global_batch_size": 256, + "align_learning_rate": 0.001, + "align_lr_scheduler_type": "linear-warmup+cosine-decay", + "align_max_grad_norm": 1.0, + "align_max_steps": null, + "align_per_device_batch_size": 16, + "align_train_strategy": "fsdp-shard-grad-op", + "align_warmup_ratio": 0.03, + "align_weight_decay": 0.0, + "arch_specifier": "qformer2_64", + "enable_gradient_checkpointing": true, + "enable_mixed_precision_training": true, + "finetune_epochs": 1, + "finetune_global_batch_size": 128, + "finetune_learning_rate": 2e-05, + "finetune_lr_scheduler_type": "linear-warmup+cosine-decay", + "finetune_max_grad_norm": 1.0, + "finetune_max_steps": null, + "finetune_per_device_batch_size": 16, + "finetune_train_strategy": "fsdp-full-shard", + "finetune_warmup_ratio": 0.03, + "finetune_weight_decay": 0.1, + "image_resize_strategy": "letterbox", + "llm_backbone_id": "vicuna-v15-7b", + "llm_max_length": 2048, + "model_id": "one-stage+7b_qformer2_64", + "reduce_in_full_precision": false, + "type": "one-stage+7b", + "vision_backbone_id": "clip-vit-l-336px" + }, + "pretrained_checkpoint": null, + "run_id": "qformer2_64", + "run_root_dir": "runs", + "seed": 7, + "stage": "finetune", + "trackers": [ + "jsonl", + "wandb" + ], + "wandb_entity": "lilei_stones", + "wandb_project": "hf-vlms" +} \ No newline at end of file diff --git a/qformer2_64/config.yaml b/qformer2_64/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c68ee1009949146b91a476909f232ef48e77e9c6 --- /dev/null +++ b/qformer2_64/config.yaml @@ -0,0 +1,53 @@ +dataset: + align_stage_components: + - download/llava-laion-cc-sbu-558k/chat.json + - download/llava-laion-cc-sbu-558k + dataset_id: llava-v15 + dataset_root_dir: data + finetune_stage_components: + - download/llava-v1.5-instruct/llava_v1_5_mix665k.json + - download/llava-v1.5-instruct + type: llava-v15 +hf_token: .hf_token +max_length: 4096 +model: + align_epochs: 1 + align_global_batch_size: 256 + align_learning_rate: 0.001 + align_lr_scheduler_type: linear-warmup+cosine-decay + align_max_grad_norm: 1.0 + align_max_steps: null + align_per_device_batch_size: 16 + align_train_strategy: fsdp-shard-grad-op + align_warmup_ratio: 0.03 + align_weight_decay: 0.0 + arch_specifier: qformer2_64 + enable_gradient_checkpointing: true + enable_mixed_precision_training: true + finetune_epochs: 1 + finetune_global_batch_size: 128 + finetune_learning_rate: 2.0e-05 + finetune_lr_scheduler_type: linear-warmup+cosine-decay + finetune_max_grad_norm: 1.0 + finetune_max_steps: null + finetune_per_device_batch_size: 16 + finetune_train_strategy: fsdp-full-shard + finetune_warmup_ratio: 0.03 + finetune_weight_decay: 0.1 + image_resize_strategy: letterbox + llm_backbone_id: vicuna-v15-7b + llm_max_length: 2048 + model_id: one-stage+7b_qformer2_64 + reduce_in_full_precision: false + type: one-stage+7b + vision_backbone_id: clip-vit-l-336px +pretrained_checkpoint: null +run_id: qformer2_64 +run_root_dir: runs +seed: 7 +stage: finetune +trackers: +- jsonl +- wandb +wandb_entity: lilei_stones +wandb_project: hf-vlms diff --git a/qformer2_64/qformer2_64.jsonl b/qformer2_64/qformer2_64.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..5af777646cda8708ae05204a3ab1ad26e1ba4eb0 --- /dev/null +++ b/qformer2_64/qformer2_64.jsonl @@ -0,0 +1,5198 @@ +{"Finetune/Learning Rate": 1.2903225806451614e-07, "Finetune/Loss": 6.740506649017334, "Finetune/Loss (Raw)": 6.740506649017334, "Finetune/Step": 1, "Finetune/Step Time": 45.637909173965454} +{"Finetune/Learning Rate": 2.580645161290323e-07, "Finetune/Loss": 7.130492210388184, "Finetune/Loss (Raw)": 7.520477294921875, "Finetune/Step": 2, "Finetune/Step Time": 30.290448427200317} +{"Finetune/Learning Rate": 3.870967741935484e-07, "Finetune/Loss": 6.177045822143555, "Finetune/Loss (Raw)": 4.270153999328613, "Finetune/Step": 3, "Finetune/Step Time": 22.83147398630778} +{"Finetune/Learning Rate": 5.161290322580646e-07, "Finetune/Loss": 5.228446960449219, "Finetune/Loss (Raw)": 2.382648468017578, "Finetune/Step": 4, "Finetune/Step Time": 19.08970808982849} +{"Finetune/Learning Rate": 6.451612903225807e-07, "Finetune/Loss": 4.492379188537598, "Finetune/Loss (Raw)": 1.5481072664260864, "Finetune/Step": 5, "Finetune/Step Time": 17.255047941207884} +{"Finetune/Learning Rate": 7.741935483870968e-07, "Finetune/Loss": 3.980281352996826, "Finetune/Loss (Raw)": 1.4197927713394165, "Finetune/Step": 6, "Finetune/Step Time": 15.946140249570211} +{"Finetune/Learning Rate": 9.032258064516129e-07, "Finetune/Loss": 3.587404251098633, "Finetune/Loss (Raw)": 1.2301398515701294, "Finetune/Step": 7, "Finetune/Step Time": 14.93239436830793} +{"Finetune/Learning Rate": 1.0322580645161291e-06, "Finetune/Loss": 3.3234381675720215, "Finetune/Loss (Raw)": 1.4756768941879272, "Finetune/Step": 8, "Finetune/Step Time": 14.567561328411102} +{"Finetune/Learning Rate": 1.1612903225806454e-06, "Finetune/Loss": 3.0789787769317627, "Finetune/Loss (Raw)": 1.1233036518096924, "Finetune/Step": 9, "Finetune/Step Time": 13.97048020362854} +{"Finetune/Learning Rate": 1.2903225806451614e-06, "Finetune/Loss": 2.924769163131714, "Finetune/Loss (Raw)": 1.5368849039077759, "Finetune/Step": 10, "Finetune/Step Time": 14.041224884986878} +{"Finetune/Learning Rate": 1.4193548387096776e-06, "Finetune/Loss": 2.78460693359375, "Finetune/Loss (Raw)": 1.3829843997955322, "Finetune/Step": 11, "Finetune/Step Time": 13.537046735936945} +{"Finetune/Learning Rate": 1.5483870967741937e-06, "Finetune/Loss": 2.6571621894836426, "Finetune/Loss (Raw)": 1.2552679777145386, "Finetune/Step": 12, "Finetune/Step Time": 13.143624663352966} +{"Finetune/Learning Rate": 1.67741935483871e-06, "Finetune/Loss": 2.5418455600738525, "Finetune/Loss (Raw)": 1.158048152923584, "Finetune/Step": 13, "Finetune/Step Time": 12.819474403674786} +{"Finetune/Learning Rate": 1.8064516129032258e-06, "Finetune/Loss": 2.4584782123565674, "Finetune/Loss (Raw)": 1.3747031688690186, "Finetune/Step": 14, "Finetune/Step Time": 12.157394749777657} +{"Finetune/Learning Rate": 1.935483870967742e-06, "Finetune/Loss": 2.387556791305542, "Finetune/Loss (Raw)": 1.3946555852890015, "Finetune/Step": 15, "Finetune/Step Time": 11.681590572992961} +{"Finetune/Learning Rate": 2.0645161290322582e-06, "Finetune/Loss": 2.300328493118286, "Finetune/Loss (Raw)": 0.9919017553329468, "Finetune/Step": 16, "Finetune/Step Time": 11.274429619312286} +{"Finetune/Learning Rate": 2.1935483870967745e-06, "Finetune/Loss": 2.2338011264801025, "Finetune/Loss (Raw)": 1.1693660020828247, "Finetune/Step": 17, "Finetune/Step Time": 10.848207754247328} +{"Finetune/Learning Rate": 2.3225806451612907e-06, "Finetune/Loss": 2.166102647781372, "Finetune/Loss (Raw)": 1.0152249336242676, "Finetune/Step": 18, "Finetune/Step Time": 10.500811166233486} +{"Finetune/Learning Rate": 2.4516129032258066e-06, "Finetune/Loss": 2.1137030124664307, "Finetune/Loss (Raw)": 1.1705130338668823, "Finetune/Step": 19, "Finetune/Step Time": 10.13584556077656} +{"Finetune/Learning Rate": 2.580645161290323e-06, "Finetune/Loss": 2.0647904872894287, "Finetune/Loss (Raw)": 1.1354496479034424, "Finetune/Step": 20, "Finetune/Step Time": 9.79388976097107} +{"Finetune/Learning Rate": 2.709677419354839e-06, "Finetune/Loss": 2.0211639404296875, "Finetune/Loss (Raw)": 1.148635983467102, "Finetune/Step": 21, "Finetune/Step Time": 9.57558248156593} +{"Finetune/Learning Rate": 2.8387096774193553e-06, "Finetune/Loss": 1.9778484106063843, "Finetune/Loss (Raw)": 1.0682216882705688, "Finetune/Step": 22, "Finetune/Step Time": 9.322806585918773} +{"Finetune/Learning Rate": 2.967741935483871e-06, "Finetune/Loss": 1.9019570350646973, "Finetune/Loss (Raw)": 0.23234698176383972, "Finetune/Step": 23, "Finetune/Step Time": 9.277220694915108} +{"Finetune/Learning Rate": 3.0967741935483874e-06, "Finetune/Loss": 1.8724935054779053, "Finetune/Loss (Raw)": 1.194828987121582, "Finetune/Step": 24, "Finetune/Step Time": 9.030942757924398} +{"Finetune/Learning Rate": 3.225806451612903e-06, "Finetune/Loss": 1.8399709463119507, "Finetune/Loss (Raw)": 1.059431552886963, "Finetune/Step": 25, "Finetune/Step Time": 8.83101812362671} +{"Finetune/Learning Rate": 3.35483870967742e-06, "Finetune/Loss": 1.7795519828796387, "Finetune/Loss (Raw)": 0.26907670497894287, "Finetune/Step": 26, "Finetune/Step Time": 8.820758425272428} +{"Finetune/Learning Rate": 3.4838709677419357e-06, "Finetune/Loss": 1.7597602605819702, "Finetune/Loss (Raw)": 1.2451775074005127, "Finetune/Step": 27, "Finetune/Step Time": 8.611406600033796} +{"Finetune/Learning Rate": 3.6129032258064515e-06, "Finetune/Loss": 1.736609935760498, "Finetune/Loss (Raw)": 1.1115473508834839, "Finetune/Step": 28, "Finetune/Step Time": 8.425897470542363} +{"Finetune/Learning Rate": 3.741935483870968e-06, "Finetune/Loss": 1.7192699909210205, "Finetune/Loss (Raw)": 1.2337546348571777, "Finetune/Step": 29, "Finetune/Step Time": 8.305763976327304} +{"Finetune/Learning Rate": 3.870967741935484e-06, "Finetune/Loss": 1.6968419551849365, "Finetune/Loss (Raw)": 1.0464239120483398, "Finetune/Step": 30, "Finetune/Step Time": 8.195267192522685} +{"Finetune/Learning Rate": 4.000000000000001e-06, "Finetune/Loss": 1.6812738180160522, "Finetune/Loss (Raw)": 1.2142385244369507, "Finetune/Step": 31, "Finetune/Step Time": 8.065578845239454} +{"Finetune/Learning Rate": 4.1290322580645165e-06, "Finetune/Loss": 1.6620643138885498, "Finetune/Loss (Raw)": 1.0665631294250488, "Finetune/Step": 32, "Finetune/Step Time": 8.000292882323265} +{"Finetune/Learning Rate": 4.258064516129032e-06, "Finetune/Loss": 1.6406835317611694, "Finetune/Loss (Raw)": 0.9564988017082214, "Finetune/Step": 33, "Finetune/Step Time": 7.8919878005981445} +{"Finetune/Learning Rate": 4.387096774193549e-06, "Finetune/Loss": 1.622251272201538, "Finetune/Loss (Raw)": 1.0139873027801514, "Finetune/Step": 34, "Finetune/Step Time": 7.773475275320165} +{"Finetune/Learning Rate": 4.516129032258065e-06, "Finetune/Loss": 1.6130871772766113, "Finetune/Loss (Raw)": 1.301509141921997, "Finetune/Step": 35, "Finetune/Step Time": 7.672030769075666} +{"Finetune/Learning Rate": 4.6451612903225815e-06, "Finetune/Loss": 1.5997711420059204, "Finetune/Loss (Raw)": 1.1337120532989502, "Finetune/Step": 36, "Finetune/Step Time": 7.609988285435571} +{"Finetune/Learning Rate": 4.774193548387097e-06, "Finetune/Loss": 1.5904830694198608, "Finetune/Loss (Raw)": 1.2561100721359253, "Finetune/Step": 37, "Finetune/Step Time": 7.592343195064648} +{"Finetune/Learning Rate": 4.903225806451613e-06, "Finetune/Loss": 1.578101634979248, "Finetune/Loss (Raw)": 1.1199873685836792, "Finetune/Step": 38, "Finetune/Step Time": 7.483469404672322} +{"Finetune/Learning Rate": 5.032258064516129e-06, "Finetune/Loss": 1.5670866966247559, "Finetune/Loss (Raw)": 1.148518681526184, "Finetune/Step": 39, "Finetune/Step Time": 7.378321953308888} +{"Finetune/Learning Rate": 5.161290322580646e-06, "Finetune/Loss": 1.556158185005188, "Finetune/Loss (Raw)": 1.1299474239349365, "Finetune/Step": 40, "Finetune/Step Time": 7.328795850276947} +{"Finetune/Learning Rate": 5.290322580645162e-06, "Finetune/Loss": 1.5486433506011963, "Finetune/Loss (Raw)": 1.24805748462677, "Finetune/Step": 41, "Finetune/Step Time": 7.220859196127915} +{"Finetune/Learning Rate": 5.419354838709678e-06, "Finetune/Loss": 1.5409232378005981, "Finetune/Loss (Raw)": 1.2243900299072266, "Finetune/Step": 42, "Finetune/Step Time": 7.13087580885206} +{"Finetune/Learning Rate": 5.548387096774194e-06, "Finetune/Loss": 1.5317556858062744, "Finetune/Loss (Raw)": 1.1467207670211792, "Finetune/Step": 43, "Finetune/Step Time": 7.079466498175333} +{"Finetune/Learning Rate": 5.677419354838711e-06, "Finetune/Loss": 1.5236215591430664, "Finetune/Loss (Raw)": 1.1738508939743042, "Finetune/Step": 44, "Finetune/Step Time": 7.021694763140245} +{"Finetune/Learning Rate": 5.806451612903226e-06, "Finetune/Loss": 1.5129292011260986, "Finetune/Loss (Raw)": 1.0424643754959106, "Finetune/Step": 45, "Finetune/Step Time": 7.035771279864841} +{"Finetune/Learning Rate": 5.935483870967742e-06, "Finetune/Loss": 1.4866268634796143, "Finetune/Loss (Raw)": 0.3030252754688263, "Finetune/Step": 46, "Finetune/Step Time": 7.092548282250114} +{"Finetune/Learning Rate": 6.064516129032259e-06, "Finetune/Loss": 1.479301929473877, "Finetune/Loss (Raw)": 1.1423566341400146, "Finetune/Step": 47, "Finetune/Step Time": 7.039839551803913} +{"Finetune/Learning Rate": 6.193548387096775e-06, "Finetune/Loss": 1.471928596496582, "Finetune/Loss (Raw)": 1.125378131866455, "Finetune/Step": 48, "Finetune/Step Time": 7.005871877074242} +{"Finetune/Learning Rate": 6.3225806451612906e-06, "Finetune/Loss": 1.4655020236968994, "Finetune/Loss (Raw)": 1.157037377357483, "Finetune/Step": 49, "Finetune/Step Time": 6.94290136804386} +{"Finetune/Learning Rate": 6.451612903225806e-06, "Finetune/Loss": 1.4567904472351074, "Finetune/Loss (Raw)": 1.0299230813980103, "Finetune/Step": 50, "Finetune/Step Time": 6.885427765846252} +{"Finetune/Learning Rate": 6.580645161290323e-06, "Finetune/Loss": 1.4506640434265137, "Finetune/Loss (Raw)": 1.1443333625793457, "Finetune/Step": 51, "Finetune/Step Time": 6.823516046299654} +{"Finetune/Learning Rate": 6.70967741935484e-06, "Finetune/Loss": 1.445857048034668, "Finetune/Loss (Raw)": 1.2007019519805908, "Finetune/Step": 52, "Finetune/Step Time": 6.780800654337956} +{"Finetune/Learning Rate": 6.838709677419355e-06, "Finetune/Loss": 1.435886025428772, "Finetune/Loss (Raw)": 0.9173924922943115, "Finetune/Step": 53, "Finetune/Step Time": 6.761494856960368} +{"Finetune/Learning Rate": 6.967741935483871e-06, "Finetune/Loss": 1.4266468286514282, "Finetune/Loss (Raw)": 0.9369668364524841, "Finetune/Step": 54, "Finetune/Step Time": 6.72423474876969} +{"Finetune/Learning Rate": 7.096774193548388e-06, "Finetune/Loss": 1.4218355417251587, "Finetune/Loss (Raw)": 1.1620333194732666, "Finetune/Step": 55, "Finetune/Step Time": 6.694853154095736} +{"Finetune/Learning Rate": 7.225806451612903e-06, "Finetune/Loss": 1.4163625240325928, "Finetune/Loss (Raw)": 1.115346074104309, "Finetune/Step": 56, "Finetune/Step Time": 6.67612829378673} +{"Finetune/Learning Rate": 7.35483870967742e-06, "Finetune/Loss": 1.4094172716140747, "Finetune/Loss (Raw)": 1.0204827785491943, "Finetune/Step": 57, "Finetune/Step Time": 6.625022126917253} +{"Finetune/Learning Rate": 7.483870967741936e-06, "Finetune/Loss": 1.4035745859146118, "Finetune/Loss (Raw)": 1.0705405473709106, "Finetune/Step": 58, "Finetune/Step Time": 6.624152298631339} +{"Finetune/Learning Rate": 7.612903225806451e-06, "Finetune/Loss": 1.398373007774353, "Finetune/Loss (Raw)": 1.0966792106628418, "Finetune/Step": 59, "Finetune/Step Time": 6.604634773933281} +{"Finetune/Learning Rate": 7.741935483870968e-06, "Finetune/Loss": 1.393256664276123, "Finetune/Loss (Raw)": 1.0913946628570557, "Finetune/Step": 60, "Finetune/Step Time": 6.554155977567037} +{"Finetune/Learning Rate": 7.870967741935484e-06, "Finetune/Loss": 1.389204740524292, "Finetune/Loss (Raw)": 1.1460932493209839, "Finetune/Step": 61, "Finetune/Step Time": 6.513587830496616} +{"Finetune/Learning Rate": 8.000000000000001e-06, "Finetune/Loss": 1.386220932006836, "Finetune/Loss (Raw)": 1.2042102813720703, "Finetune/Step": 62, "Finetune/Step Time": 6.465929804309722} +{"Finetune/Learning Rate": 8.129032258064517e-06, "Finetune/Loss": 1.383418083190918, "Finetune/Loss (Raw)": 1.209634780883789, "Finetune/Step": 63, "Finetune/Step Time": 6.421461718423026} +{"Finetune/Learning Rate": 8.258064516129033e-06, "Finetune/Loss": 1.3784972429275513, "Finetune/Loss (Raw)": 1.0684893131256104, "Finetune/Step": 64, "Finetune/Step Time": 6.434808816760778} +{"Finetune/Learning Rate": 8.387096774193549e-06, "Finetune/Loss": 1.3734605312347412, "Finetune/Loss (Raw)": 1.0511101484298706, "Finetune/Step": 65, "Finetune/Step Time": 6.40679182272691} +{"Finetune/Learning Rate": 8.516129032258065e-06, "Finetune/Loss": 1.3678996562957764, "Finetune/Loss (Raw)": 1.006440281867981, "Finetune/Step": 66, "Finetune/Step Time": 6.371936382669391} +{"Finetune/Learning Rate": 8.64516129032258e-06, "Finetune/Loss": 1.360949993133545, "Finetune/Loss (Raw)": 0.9022802114486694, "Finetune/Step": 67, "Finetune/Step Time": 6.326915826370467} +{"Finetune/Learning Rate": 8.774193548387098e-06, "Finetune/Loss": 1.3575927019119263, "Finetune/Loss (Raw)": 1.1326535940170288, "Finetune/Step": 68, "Finetune/Step Time": 6.304561709656435} +{"Finetune/Learning Rate": 8.903225806451614e-06, "Finetune/Loss": 1.35464346408844, "Finetune/Loss (Raw)": 1.154083490371704, "Finetune/Step": 69, "Finetune/Step Time": 6.261792525001194} +{"Finetune/Learning Rate": 9.03225806451613e-06, "Finetune/Loss": 1.3523420095443726, "Finetune/Loss (Raw)": 1.1935442686080933, "Finetune/Step": 70, "Finetune/Step Time": 6.223735867227827} +{"Finetune/Learning Rate": 9.161290322580645e-06, "Finetune/Loss": 1.3464049100875854, "Finetune/Loss (Raw)": 0.9308093786239624, "Finetune/Step": 71, "Finetune/Step Time": 6.1888287638274715} +{"Finetune/Learning Rate": 9.290322580645163e-06, "Finetune/Loss": 1.3419435024261475, "Finetune/Loss (Raw)": 1.025182843208313, "Finetune/Step": 72, "Finetune/Step Time": 6.147949566443761} +{"Finetune/Learning Rate": 9.419354838709677e-06, "Finetune/Loss": 1.3362091779708862, "Finetune/Loss (Raw)": 0.9233396649360657, "Finetune/Step": 73, "Finetune/Step Time": 6.113127757425177} +{"Finetune/Learning Rate": 9.548387096774195e-06, "Finetune/Loss": 1.3308109045028687, "Finetune/Loss (Raw)": 0.9367338418960571, "Finetune/Step": 74, "Finetune/Step Time": 6.0795425982088656} +{"Finetune/Learning Rate": 9.67741935483871e-06, "Finetune/Loss": 1.324045181274414, "Finetune/Loss (Raw)": 0.8233900666236877, "Finetune/Step": 75, "Finetune/Step Time": 6.058277816772461} +{"Finetune/Learning Rate": 9.806451612903226e-06, "Finetune/Loss": 1.3194559812545776, "Finetune/Loss (Raw)": 0.9752599000930786, "Finetune/Step": 76, "Finetune/Step Time": 6.0351978289453605} +{"Finetune/Learning Rate": 9.935483870967742e-06, "Finetune/Loss": 1.3167619705200195, "Finetune/Loss (Raw)": 1.1120136976242065, "Finetune/Step": 77, "Finetune/Step Time": 6.000121153794326} +{"Finetune/Learning Rate": 1.0064516129032258e-05, "Finetune/Loss": 1.3155264854431152, "Finetune/Loss (Raw)": 1.2203913927078247, "Finetune/Step": 78, "Finetune/Step Time": 5.965270821864788} +{"Finetune/Learning Rate": 1.0193548387096774e-05, "Finetune/Loss": 1.3139179944992065, "Finetune/Loss (Raw)": 1.1884640455245972, "Finetune/Step": 79, "Finetune/Step Time": 5.935640126843996} +{"Finetune/Learning Rate": 1.0322580645161291e-05, "Finetune/Loss": 1.309757113456726, "Finetune/Loss (Raw)": 0.981040358543396, "Finetune/Step": 80, "Finetune/Step Time": 5.927529808878899} +{"Finetune/Learning Rate": 1.0451612903225807e-05, "Finetune/Loss": 1.3078051805496216, "Finetune/Loss (Raw)": 1.1516587734222412, "Finetune/Step": 81, "Finetune/Step Time": 5.911235129391706} +{"Finetune/Learning Rate": 1.0580645161290325e-05, "Finetune/Loss": 1.3053146600723267, "Finetune/Loss (Raw)": 1.1035823822021484, "Finetune/Step": 82, "Finetune/Step Time": 5.881767618946913} +{"Finetune/Learning Rate": 1.070967741935484e-05, "Finetune/Loss": 1.3011362552642822, "Finetune/Loss (Raw)": 0.9584990739822388, "Finetune/Step": 83, "Finetune/Step Time": 5.876982798059303} +{"Finetune/Learning Rate": 1.0838709677419356e-05, "Finetune/Loss": 1.2998065948486328, "Finetune/Loss (Raw)": 1.1894465684890747, "Finetune/Step": 84, "Finetune/Step Time": 5.866310744058518} +{"Finetune/Learning Rate": 1.096774193548387e-05, "Finetune/Loss": 1.2968662977218628, "Finetune/Loss (Raw)": 1.0498822927474976, "Finetune/Step": 85, "Finetune/Step Time": 5.840345155491549} +{"Finetune/Learning Rate": 1.1096774193548388e-05, "Finetune/Loss": 1.2858830690383911, "Finetune/Loss (Raw)": 0.35231178998947144, "Finetune/Step": 86, "Finetune/Step Time": 5.878855949224428} +{"Finetune/Learning Rate": 1.1225806451612904e-05, "Finetune/Loss": 1.2829763889312744, "Finetune/Loss (Raw)": 1.0329951047897339, "Finetune/Step": 87, "Finetune/Step Time": 5.853001517811041} +{"Finetune/Learning Rate": 1.1354838709677421e-05, "Finetune/Loss": 1.282379388809204, "Finetune/Loss (Raw)": 1.2304350137710571, "Finetune/Step": 88, "Finetune/Step Time": 5.849482910199598} +{"Finetune/Learning Rate": 1.1483870967741937e-05, "Finetune/Loss": 1.2800523042678833, "Finetune/Loss (Raw)": 1.0752754211425781, "Finetune/Step": 89, "Finetune/Step Time": 5.824963623218322} +{"Finetune/Learning Rate": 1.1612903225806453e-05, "Finetune/Loss": 1.278481364250183, "Finetune/Loss (Raw)": 1.1386640071868896, "Finetune/Step": 90, "Finetune/Step Time": 5.824918288654751} +{"Finetune/Learning Rate": 1.1741935483870967e-05, "Finetune/Loss": 1.2773171663284302, "Finetune/Loss (Raw)": 1.1725488901138306, "Finetune/Step": 91, "Finetune/Step Time": 5.818063956040603} +{"Finetune/Learning Rate": 1.1870967741935484e-05, "Finetune/Loss": 1.276535153388977, "Finetune/Loss (Raw)": 1.2053649425506592, "Finetune/Step": 92, "Finetune/Step Time": 5.796049200970193} +{"Finetune/Learning Rate": 1.2e-05, "Finetune/Loss": 1.2727659940719604, "Finetune/Loss (Raw)": 0.9260108470916748, "Finetune/Step": 93, "Finetune/Step Time": 5.7772496259340675} +{"Finetune/Learning Rate": 1.2129032258064518e-05, "Finetune/Loss": 1.2715810537338257, "Finetune/Loss (Raw)": 1.1613856554031372, "Finetune/Step": 94, "Finetune/Step Time": 5.7511739122106675} +{"Finetune/Learning Rate": 1.2258064516129034e-05, "Finetune/Loss": 1.2697815895080566, "Finetune/Loss (Raw)": 1.1006176471710205, "Finetune/Step": 95, "Finetune/Step Time": 5.729556440052233} +{"Finetune/Learning Rate": 1.238709677419355e-05, "Finetune/Loss": 1.2678842544555664, "Finetune/Loss (Raw)": 1.087639570236206, "Finetune/Step": 96, "Finetune/Step Time": 5.722432404756546} +{"Finetune/Learning Rate": 1.2516129032258067e-05, "Finetune/Loss": 1.2653940916061401, "Finetune/Loss (Raw)": 1.0263490676879883, "Finetune/Step": 97, "Finetune/Step Time": 5.708090946846402} +{"Finetune/Learning Rate": 1.2645161290322581e-05, "Finetune/Loss": 1.2639930248260498, "Finetune/Loss (Raw)": 1.128089189529419, "Finetune/Step": 98, "Finetune/Step Time": 5.68705432269038} +{"Finetune/Learning Rate": 1.2774193548387097e-05, "Finetune/Loss": 1.2534873485565186, "Finetune/Loss (Raw)": 0.22393497824668884, "Finetune/Step": 99, "Finetune/Step Time": 5.71361154498476} +{"Finetune/Learning Rate": 1.2903225806451613e-05, "Finetune/Loss": 1.2531479597091675, "Finetune/Loss (Raw)": 1.2195407152175903, "Finetune/Step": 100, "Finetune/Step Time": 5.697112045288086} +{"Finetune/Learning Rate": 1.303225806451613e-05, "Finetune/Loss": 1.25111722946167, "Finetune/Loss (Raw)": 1.0480461120605469, "Finetune/Step": 101, "Finetune/Step Time": 5.690300075134428} +{"Finetune/Learning Rate": 1.3161290322580646e-05, "Finetune/Loss": 1.2507612705230713, "Finetune/Loss (Raw)": 1.214806079864502, "Finetune/Step": 102, "Finetune/Step Time": 5.664814355326634} +{"Finetune/Learning Rate": 1.3290322580645164e-05, "Finetune/Loss": 1.2414809465408325, "Finetune/Loss (Raw)": 0.2948918640613556, "Finetune/Step": 103, "Finetune/Step Time": 5.691633805487919} +{"Finetune/Learning Rate": 1.341935483870968e-05, "Finetune/Loss": 1.2406681776046753, "Finetune/Loss (Raw)": 1.1569435596466064, "Finetune/Step": 104, "Finetune/Step Time": 5.6699789212300225} +{"Finetune/Learning Rate": 1.3548387096774194e-05, "Finetune/Loss": 1.2390116453170776, "Finetune/Loss (Raw)": 1.0667470693588257, "Finetune/Step": 105, "Finetune/Step Time": 5.644979372478667} +{"Finetune/Learning Rate": 1.367741935483871e-05, "Finetune/Loss": 1.2373876571655273, "Finetune/Loss (Raw)": 1.0668615102767944, "Finetune/Step": 106, "Finetune/Step Time": 5.6243015932586955} +{"Finetune/Learning Rate": 1.3806451612903227e-05, "Finetune/Loss": 1.236431360244751, "Finetune/Loss (Raw)": 1.1350748538970947, "Finetune/Step": 107, "Finetune/Step Time": 5.604803464123022} +{"Finetune/Learning Rate": 1.3935483870967743e-05, "Finetune/Loss": 1.234405755996704, "Finetune/Loss (Raw)": 1.0176657438278198, "Finetune/Step": 108, "Finetune/Step Time": 5.588523897859785} +{"Finetune/Learning Rate": 1.406451612903226e-05, "Finetune/Loss": 1.2335604429244995, "Finetune/Loss (Raw)": 1.1422792673110962, "Finetune/Step": 109, "Finetune/Step Time": 5.5699842610490435} +{"Finetune/Learning Rate": 1.4193548387096776e-05, "Finetune/Loss": 1.2319424152374268, "Finetune/Loss (Raw)": 1.055557370185852, "Finetune/Step": 110, "Finetune/Step Time": 5.5501091610301625} +{"Finetune/Learning Rate": 1.4322580645161292e-05, "Finetune/Loss": 1.231001853942871, "Finetune/Loss (Raw)": 1.1275498867034912, "Finetune/Step": 111, "Finetune/Step Time": 5.548301769806458} +{"Finetune/Learning Rate": 1.4451612903225806e-05, "Finetune/Loss": 1.2297015190124512, "Finetune/Loss (Raw)": 1.0853464603424072, "Finetune/Step": 112, "Finetune/Step Time": 5.548482796975544} +{"Finetune/Learning Rate": 1.4580645161290324e-05, "Finetune/Loss": 1.2288343906402588, "Finetune/Loss (Raw)": 1.1317296028137207, "Finetune/Step": 113, "Finetune/Step Time": 5.544324509865414} +{"Finetune/Learning Rate": 1.470967741935484e-05, "Finetune/Loss": 1.2202603816986084, "Finetune/Loss (Raw)": 0.25137919187545776, "Finetune/Step": 114, "Finetune/Step Time": 5.576123929860299} +{"Finetune/Learning Rate": 1.4838709677419357e-05, "Finetune/Loss": 1.2191557884216309, "Finetune/Loss (Raw)": 1.0932573080062866, "Finetune/Step": 115, "Finetune/Step Time": 5.565237986523172} +{"Finetune/Learning Rate": 1.4967741935483873e-05, "Finetune/Loss": 1.2166136503219604, "Finetune/Loss (Raw)": 0.9242540597915649, "Finetune/Step": 116, "Finetune/Step Time": 5.569979992406122} +{"Finetune/Learning Rate": 1.5096774193548389e-05, "Finetune/Loss": 1.215721607208252, "Finetune/Loss (Raw)": 1.1122411489486694, "Finetune/Step": 117, "Finetune/Step Time": 5.558004236628866} +{"Finetune/Learning Rate": 1.5225806451612903e-05, "Finetune/Loss": 1.2151060104370117, "Finetune/Loss (Raw)": 1.1430740356445312, "Finetune/Step": 118, "Finetune/Step Time": 5.5408269591250665} +{"Finetune/Learning Rate": 1.535483870967742e-05, "Finetune/Loss": 1.2136744260787964, "Finetune/Loss (Raw)": 1.0447508096694946, "Finetune/Step": 119, "Finetune/Step Time": 5.525563268100514} +{"Finetune/Learning Rate": 1.5483870967741936e-05, "Finetune/Loss": 1.2126318216323853, "Finetune/Loss (Raw)": 1.0885584354400635, "Finetune/Step": 120, "Finetune/Step Time": 5.514750160773596} +{"Finetune/Learning Rate": 1.5612903225806454e-05, "Finetune/Loss": 1.2111012935638428, "Finetune/Loss (Raw)": 1.02744460105896, "Finetune/Step": 121, "Finetune/Step Time": 5.522292468173445} +{"Finetune/Learning Rate": 1.5741935483870968e-05, "Finetune/Loss": 1.2090823650360107, "Finetune/Loss (Raw)": 0.9648056030273438, "Finetune/Step": 122, "Finetune/Step Time": 5.506973454209625} +{"Finetune/Learning Rate": 1.5870967741935485e-05, "Finetune/Loss": 1.2075276374816895, "Finetune/Loss (Raw)": 1.0178526639938354, "Finetune/Step": 123, "Finetune/Step Time": 5.49116984615481} +{"Finetune/Learning Rate": 1.6000000000000003e-05, "Finetune/Loss": 1.2069021463394165, "Finetune/Loss (Raw)": 1.1299526691436768, "Finetune/Step": 124, "Finetune/Step Time": 5.477988566121748} +{"Finetune/Learning Rate": 1.6129032258064517e-05, "Finetune/Loss": 1.205344557762146, "Finetune/Loss (Raw)": 1.012195110321045, "Finetune/Step": 125, "Finetune/Step Time": 5.465393581390381} +{"Finetune/Learning Rate": 1.6258064516129034e-05, "Finetune/Loss": 1.204545259475708, "Finetune/Loss (Raw)": 1.1046247482299805, "Finetune/Step": 126, "Finetune/Step Time": 5.471628187194703} +{"Finetune/Learning Rate": 1.638709677419355e-05, "Finetune/Loss": 1.2028354406356812, "Finetune/Loss (Raw)": 0.9874247908592224, "Finetune/Step": 127, "Finetune/Step Time": 5.455671753470353} +{"Finetune/Learning Rate": 1.6516129032258066e-05, "Finetune/Loss": 1.2022559642791748, "Finetune/Loss (Raw)": 1.1286554336547852, "Finetune/Step": 128, "Finetune/Step Time": 5.44280038587749} +{"Finetune/Learning Rate": 1.6645161290322583e-05, "Finetune/Loss": 1.158613920211792, "Finetune/Loss (Raw)": 1.1543198823928833, "Finetune/Step": 129, "Finetune/Step Time": 5.114425595849752} +{"Finetune/Learning Rate": 1.6774193548387098e-05, "Finetune/Loss": 1.10882568359375, "Finetune/Loss (Raw)": 1.1475852727890015, "Finetune/Step": 130, "Finetune/Step Time": 5.03460842743516} +{"Finetune/Learning Rate": 1.6903225806451615e-05, "Finetune/Loss": 1.084061622619629, "Finetune/Loss (Raw)": 1.1003485918045044, "Finetune/Step": 131, "Finetune/Step Time": 4.9987093806266785} +{"Finetune/Learning Rate": 1.703225806451613e-05, "Finetune/Loss": 1.074139952659607, "Finetune/Loss (Raw)": 1.1126877069473267, "Finetune/Step": 132, "Finetune/Step Time": 4.964915616437793} +{"Finetune/Learning Rate": 1.7161290322580647e-05, "Finetune/Loss": 1.0690827369689941, "Finetune/Loss (Raw)": 0.9007738828659058, "Finetune/Step": 133, "Finetune/Step Time": 4.932442391291261} +{"Finetune/Learning Rate": 1.729032258064516e-05, "Finetune/Loss": 1.0676672458648682, "Finetune/Loss (Raw)": 1.238617181777954, "Finetune/Step": 134, "Finetune/Step Time": 4.887015663087368} +{"Finetune/Learning Rate": 1.741935483870968e-05, "Finetune/Loss": 1.0658490657806396, "Finetune/Loss (Raw)": 0.9974083304405212, "Finetune/Step": 135, "Finetune/Step Time": 4.8509705401957035} +{"Finetune/Learning Rate": 1.7548387096774196e-05, "Finetune/Loss": 1.0628048181533813, "Finetune/Loss (Raw)": 1.086000680923462, "Finetune/Step": 136, "Finetune/Step Time": 4.784127442166209} +{"Finetune/Learning Rate": 1.7677419354838713e-05, "Finetune/Loss": 1.061639666557312, "Finetune/Loss (Raw)": 0.9741758704185486, "Finetune/Step": 137, "Finetune/Step Time": 4.746829722076654} +{"Finetune/Learning Rate": 1.7806451612903228e-05, "Finetune/Loss": 1.0583853721618652, "Finetune/Loss (Raw)": 1.1203413009643555, "Finetune/Step": 138, "Finetune/Step Time": 4.655083172023296} +{"Finetune/Learning Rate": 1.7935483870967742e-05, "Finetune/Loss": 1.0547130107879639, "Finetune/Loss (Raw)": 0.912915825843811, "Finetune/Step": 139, "Finetune/Step Time": 4.619588626548648} +{"Finetune/Learning Rate": 1.806451612903226e-05, "Finetune/Loss": 1.0549575090408325, "Finetune/Loss (Raw)": 1.2865662574768066, "Finetune/Step": 140, "Finetune/Step Time": 4.575374078005552} +{"Finetune/Learning Rate": 1.8193548387096777e-05, "Finetune/Loss": 1.0540112257003784, "Finetune/Loss (Raw)": 1.036921501159668, "Finetune/Step": 141, "Finetune/Step Time": 4.529773645102978} +{"Finetune/Learning Rate": 1.832258064516129e-05, "Finetune/Loss": 1.0513262748718262, "Finetune/Loss (Raw)": 1.0310300588607788, "Finetune/Step": 142, "Finetune/Step Time": 4.570404564961791} +{"Finetune/Learning Rate": 1.845161290322581e-05, "Finetune/Loss": 1.0479798316955566, "Finetune/Loss (Raw)": 0.966313898563385, "Finetune/Step": 143, "Finetune/Step Time": 4.584715077653527} +{"Finetune/Learning Rate": 1.8580645161290326e-05, "Finetune/Loss": 1.0431864261627197, "Finetune/Loss (Raw)": 0.37834131717681885, "Finetune/Step": 144, "Finetune/Step Time": 4.616658648476005} +{"Finetune/Learning Rate": 1.870967741935484e-05, "Finetune/Loss": 1.036764144897461, "Finetune/Loss (Raw)": 0.3473151624202728, "Finetune/Step": 145, "Finetune/Step Time": 4.651331640779972} +{"Finetune/Learning Rate": 1.8838709677419354e-05, "Finetune/Loss": 1.03643000125885, "Finetune/Loss (Raw)": 0.9724552631378174, "Finetune/Step": 146, "Finetune/Step Time": 4.641591601073742} +{"Finetune/Learning Rate": 1.896774193548387e-05, "Finetune/Loss": 1.0343470573425293, "Finetune/Loss (Raw)": 0.9039024710655212, "Finetune/Step": 147, "Finetune/Step Time": 4.647853011265397} +{"Finetune/Learning Rate": 1.909677419354839e-05, "Finetune/Loss": 1.0339064598083496, "Finetune/Loss (Raw)": 1.0790505409240723, "Finetune/Step": 148, "Finetune/Step Time": 4.650787990540266} +{"Finetune/Learning Rate": 1.9225806451612907e-05, "Finetune/Loss": 1.0337228775024414, "Finetune/Loss (Raw)": 1.1251198053359985, "Finetune/Step": 149, "Finetune/Step Time": 4.639905925840139} +{"Finetune/Learning Rate": 1.935483870967742e-05, "Finetune/Loss": 1.0342425107955933, "Finetune/Loss (Raw)": 1.1347495317459106, "Finetune/Step": 150, "Finetune/Step Time": 4.644051382318139} +{"Finetune/Learning Rate": 1.948387096774194e-05, "Finetune/Loss": 1.0409806966781616, "Finetune/Loss (Raw)": 1.0948299169540405, "Finetune/Step": 151, "Finetune/Step Time": 4.611733144149184} +{"Finetune/Learning Rate": 1.9612903225806452e-05, "Finetune/Loss": 1.0399353504180908, "Finetune/Loss (Raw)": 1.0610160827636719, "Finetune/Step": 152, "Finetune/Step Time": 4.618103677406907} +{"Finetune/Learning Rate": 1.974193548387097e-05, "Finetune/Loss": 1.0407058000564575, "Finetune/Loss (Raw)": 1.1580605506896973, "Finetune/Step": 153, "Finetune/Step Time": 4.627540621906519} +{"Finetune/Learning Rate": 1.9870967741935484e-05, "Finetune/Loss": 1.047510027885437, "Finetune/Loss (Raw)": 1.1400202512741089, "Finetune/Step": 154, "Finetune/Step Time": 4.589904580265284} +{"Finetune/Learning Rate": 2e-05, "Finetune/Loss": 1.0472736358642578, "Finetune/Loss (Raw)": 1.2149157524108887, "Finetune/Step": 155, "Finetune/Step Time": 4.597227048128843} +{"Finetune/Learning Rate": 1.9999998059597617e-05, "Finetune/Loss": 1.0409049987792969, "Finetune/Loss (Raw)": 0.29635679721832275, "Finetune/Step": 156, "Finetune/Step Time": 4.637802354991436} +{"Finetune/Learning Rate": 1.9999992238391222e-05, "Finetune/Loss": 1.0397251844406128, "Finetune/Loss (Raw)": 1.082746982574463, "Finetune/Step": 157, "Finetune/Step Time": 4.6277841832488775} +{"Finetune/Learning Rate": 1.999998253638307e-05, "Finetune/Loss": 1.0402569770812988, "Finetune/Loss (Raw)": 1.114481806755066, "Finetune/Step": 158, "Finetune/Step Time": 4.61867018789053} +{"Finetune/Learning Rate": 1.999996895357693e-05, "Finetune/Loss": 1.0391911268234253, "Finetune/Loss (Raw)": 1.077811360359192, "Finetune/Step": 159, "Finetune/Step Time": 4.611649435013533} +{"Finetune/Learning Rate": 1.9999951489978074e-05, "Finetune/Loss": 1.0388445854187012, "Finetune/Loss (Raw)": 1.0222136974334717, "Finetune/Step": 160, "Finetune/Step Time": 4.600859630852938} +{"Finetune/Learning Rate": 1.9999930145593275e-05, "Finetune/Loss": 1.0394656658172607, "Finetune/Loss (Raw)": 1.03599214553833, "Finetune/Step": 161, "Finetune/Step Time": 4.596788929775357} +{"Finetune/Learning Rate": 1.9999904920430818e-05, "Finetune/Loss": 1.0414409637451172, "Finetune/Loss (Raw)": 1.2668232917785645, "Finetune/Step": 162, "Finetune/Step Time": 4.595451274886727} +{"Finetune/Learning Rate": 1.9999875814500492e-05, "Finetune/Loss": 1.0397367477416992, "Finetune/Loss (Raw)": 1.0833882093429565, "Finetune/Step": 163, "Finetune/Step Time": 4.595777194947004} +{"Finetune/Learning Rate": 1.99998428278136e-05, "Finetune/Loss": 1.0384502410888672, "Finetune/Loss (Raw)": 0.9690340757369995, "Finetune/Step": 164, "Finetune/Step Time": 4.593812391161919} +{"Finetune/Learning Rate": 1.9999805960382928e-05, "Finetune/Loss": 1.0368566513061523, "Finetune/Loss (Raw)": 1.0521295070648193, "Finetune/Step": 165, "Finetune/Step Time": 4.570873290300369} +{"Finetune/Learning Rate": 1.99997652122228e-05, "Finetune/Loss": 1.035910964012146, "Finetune/Loss (Raw)": 0.9989431500434875, "Finetune/Step": 166, "Finetune/Step Time": 4.572966642677784} +{"Finetune/Learning Rate": 1.9999720583349015e-05, "Finetune/Loss": 1.035932183265686, "Finetune/Loss (Raw)": 1.1512176990509033, "Finetune/Step": 167, "Finetune/Step Time": 4.575073713436723} +{"Finetune/Learning Rate": 1.9999672073778905e-05, "Finetune/Loss": 1.034859538078308, "Finetune/Loss (Raw)": 0.992656946182251, "Finetune/Step": 168, "Finetune/Step Time": 4.562955135479569} +{"Finetune/Learning Rate": 1.999961968353129e-05, "Finetune/Loss": 1.032726764678955, "Finetune/Loss (Raw)": 0.9750699996948242, "Finetune/Step": 169, "Finetune/Step Time": 4.567888841032982} +{"Finetune/Learning Rate": 1.9999563412626503e-05, "Finetune/Loss": 1.026004433631897, "Finetune/Loss (Raw)": 0.36392855644226074, "Finetune/Step": 170, "Finetune/Step Time": 4.604769550263882} +{"Finetune/Learning Rate": 1.9999503261086378e-05, "Finetune/Loss": 1.0249371528625488, "Finetune/Loss (Raw)": 1.010097861289978, "Finetune/Step": 171, "Finetune/Step Time": 4.602203082293272} +{"Finetune/Learning Rate": 1.999943922893426e-05, "Finetune/Loss": 1.0235185623168945, "Finetune/Loss (Raw)": 0.9922769069671631, "Finetune/Step": 172, "Finetune/Step Time": 4.605427468195558} +{"Finetune/Learning Rate": 1.9999371316195005e-05, "Finetune/Loss": 1.0247899293899536, "Finetune/Loss (Raw)": 1.2052056789398193, "Finetune/Step": 173, "Finetune/Step Time": 4.573010217398405} +{"Finetune/Learning Rate": 1.9999299522894957e-05, "Finetune/Loss": 1.0294528007507324, "Finetune/Loss (Raw)": 0.8998709321022034, "Finetune/Step": 174, "Finetune/Step Time": 4.530708020552993} +{"Finetune/Learning Rate": 1.9999223849061987e-05, "Finetune/Loss": 1.0298510789871216, "Finetune/Loss (Raw)": 1.193324089050293, "Finetune/Step": 175, "Finetune/Step Time": 4.522363310679793} +{"Finetune/Learning Rate": 1.999914429472546e-05, "Finetune/Loss": 1.0238490104675293, "Finetune/Loss (Raw)": 0.35712072253227234, "Finetune/Step": 176, "Finetune/Step Time": 4.5459155309945345} +{"Finetune/Learning Rate": 1.9999060859916247e-05, "Finetune/Loss": 1.0224802494049072, "Finetune/Loss (Raw)": 0.9818476438522339, "Finetune/Step": 177, "Finetune/Step Time": 4.548427876085043} +{"Finetune/Learning Rate": 1.9998973544666734e-05, "Finetune/Loss": 1.0238778591156006, "Finetune/Loss (Raw)": 1.2087918519973755, "Finetune/Step": 178, "Finetune/Step Time": 4.550448462367058} +{"Finetune/Learning Rate": 1.9998882349010798e-05, "Finetune/Loss": 1.0220410823822021, "Finetune/Loss (Raw)": 0.9092491269111633, "Finetune/Step": 179, "Finetune/Step Time": 4.554273657500744} +{"Finetune/Learning Rate": 1.999878727298384e-05, "Finetune/Loss": 1.0210354328155518, "Finetune/Loss (Raw)": 1.0719687938690186, "Finetune/Step": 180, "Finetune/Step Time": 4.562775084748864} +{"Finetune/Learning Rate": 1.999868831662274e-05, "Finetune/Loss": 1.021850347518921, "Finetune/Loss (Raw)": 1.0217143297195435, "Finetune/Step": 181, "Finetune/Step Time": 4.548512542620301} +{"Finetune/Learning Rate": 1.9998585479965918e-05, "Finetune/Loss": 1.0228168964385986, "Finetune/Loss (Raw)": 1.0606683492660522, "Finetune/Step": 182, "Finetune/Step Time": 4.5542462933808565} +{"Finetune/Learning Rate": 1.999847876305328e-05, "Finetune/Loss": 1.021714687347412, "Finetune/Loss (Raw)": 1.0209524631500244, "Finetune/Step": 183, "Finetune/Step Time": 4.537108760327101} +{"Finetune/Learning Rate": 1.9998368165926237e-05, "Finetune/Loss": 1.0221121311187744, "Finetune/Loss (Raw)": 1.1662236452102661, "Finetune/Step": 184, "Finetune/Step Time": 4.523662956431508} +{"Finetune/Learning Rate": 1.9998253688627706e-05, "Finetune/Loss": 1.0214201211929321, "Finetune/Loss (Raw)": 0.9319155812263489, "Finetune/Step": 185, "Finetune/Step Time": 4.521645264700055} +{"Finetune/Learning Rate": 1.9998135331202124e-05, "Finetune/Loss": 1.0210280418395996, "Finetune/Loss (Raw)": 1.020334005355835, "Finetune/Step": 186, "Finetune/Step Time": 4.512157242745161} +{"Finetune/Learning Rate": 1.9998013093695413e-05, "Finetune/Loss": 1.0203948020935059, "Finetune/Loss (Raw)": 1.015630841255188, "Finetune/Step": 187, "Finetune/Step Time": 4.498199205845594} +{"Finetune/Learning Rate": 1.9997886976155018e-05, "Finetune/Loss": 1.0207834243774414, "Finetune/Loss (Raw)": 1.1411386728286743, "Finetune/Step": 188, "Finetune/Step Time": 4.49702307395637} +{"Finetune/Learning Rate": 1.9997756978629876e-05, "Finetune/Loss": 1.0197356939315796, "Finetune/Loss (Raw)": 1.0119946002960205, "Finetune/Step": 189, "Finetune/Step Time": 4.50800621882081} +{"Finetune/Learning Rate": 1.999762310117044e-05, "Finetune/Loss": 1.018724799156189, "Finetune/Loss (Raw)": 1.0748118162155151, "Finetune/Step": 190, "Finetune/Step Time": 4.512805327773094} +{"Finetune/Learning Rate": 1.999748534382867e-05, "Finetune/Loss": 1.0177745819091797, "Finetune/Loss (Raw)": 1.0880024433135986, "Finetune/Step": 191, "Finetune/Step Time": 4.518325442448258} +{"Finetune/Learning Rate": 1.9997343706658023e-05, "Finetune/Loss": 1.0177855491638184, "Finetune/Loss (Raw)": 1.0698949098587036, "Finetune/Step": 192, "Finetune/Step Time": 4.498178236186504} +{"Finetune/Learning Rate": 1.9997198189713462e-05, "Finetune/Loss": 1.0170009136199951, "Finetune/Loss (Raw)": 0.9506906270980835, "Finetune/Step": 193, "Finetune/Step Time": 4.491069132462144} +{"Finetune/Learning Rate": 1.9997048793051463e-05, "Finetune/Loss": 1.0174405574798584, "Finetune/Loss (Raw)": 1.0626883506774902, "Finetune/Step": 194, "Finetune/Step Time": 4.489913007244468} +{"Finetune/Learning Rate": 1.9996895516730005e-05, "Finetune/Loss": 1.0183141231536865, "Finetune/Loss (Raw)": 1.014114260673523, "Finetune/Step": 195, "Finetune/Step Time": 4.496919460594654} +{"Finetune/Learning Rate": 1.9996738360808566e-05, "Finetune/Loss": 1.0170905590057373, "Finetune/Loss (Raw)": 0.9760422706604004, "Finetune/Step": 196, "Finetune/Step Time": 4.493245668709278} +{"Finetune/Learning Rate": 1.999657732534814e-05, "Finetune/Loss": 1.0163613557815552, "Finetune/Loss (Raw)": 1.0607374906539917, "Finetune/Step": 197, "Finetune/Step Time": 4.499959724023938} +{"Finetune/Learning Rate": 1.9996412410411224e-05, "Finetune/Loss": 1.0150103569030762, "Finetune/Loss (Raw)": 1.0206031799316406, "Finetune/Step": 198, "Finetune/Step Time": 4.499562839046121} +{"Finetune/Learning Rate": 1.9996243616061812e-05, "Finetune/Loss": 1.0155932903289795, "Finetune/Loss (Raw)": 1.0054466724395752, "Finetune/Step": 199, "Finetune/Step Time": 4.503756718710065} +{"Finetune/Learning Rate": 1.9996070942365416e-05, "Finetune/Loss": 1.0157132148742676, "Finetune/Loss (Raw)": 1.04052734375, "Finetune/Step": 200, "Finetune/Step Time": 4.5082277823239565} +{"Finetune/Learning Rate": 1.9995894389389038e-05, "Finetune/Loss": 1.017399787902832, "Finetune/Loss (Raw)": 1.1392183303833008, "Finetune/Step": 201, "Finetune/Step Time": 4.51273282058537} +{"Finetune/Learning Rate": 1.9995713957201208e-05, "Finetune/Loss": 1.0189006328582764, "Finetune/Loss (Raw)": 1.1288392543792725, "Finetune/Step": 202, "Finetune/Step Time": 4.516080975532532} +{"Finetune/Learning Rate": 1.9995529645871938e-05, "Finetune/Loss": 1.0208643674850464, "Finetune/Loss (Raw)": 1.074748158454895, "Finetune/Step": 203, "Finetune/Step Time": 4.510480770841241} +{"Finetune/Learning Rate": 1.999534145547276e-05, "Finetune/Loss": 1.0215784311294556, "Finetune/Loss (Raw)": 1.0666553974151611, "Finetune/Step": 204, "Finetune/Step Time": 4.505377035588026} +{"Finetune/Learning Rate": 1.99951493860767e-05, "Finetune/Loss": 1.015415072441101, "Finetune/Loss (Raw)": 0.32311326265335083, "Finetune/Step": 205, "Finetune/Step Time": 4.554673565551639} +{"Finetune/Learning Rate": 1.9994953437758305e-05, "Finetune/Loss": 1.0138720273971558, "Finetune/Loss (Raw)": 1.0228774547576904, "Finetune/Step": 206, "Finetune/Step Time": 4.563721219077706} +{"Finetune/Learning Rate": 1.999475361059362e-05, "Finetune/Loss": 1.013939619064331, "Finetune/Loss (Raw)": 1.1971150636672974, "Finetune/Step": 207, "Finetune/Step Time": 4.586714768782258} +{"Finetune/Learning Rate": 1.9994549904660187e-05, "Finetune/Loss": 1.0145988464355469, "Finetune/Loss (Raw)": 1.0654168128967285, "Finetune/Step": 208, "Finetune/Step Time": 4.571760894730687} +{"Finetune/Learning Rate": 1.9994342320037065e-05, "Finetune/Loss": 1.013577938079834, "Finetune/Loss (Raw)": 1.0209988355636597, "Finetune/Step": 209, "Finetune/Step Time": 4.565913632512093} +{"Finetune/Learning Rate": 1.9994130856804808e-05, "Finetune/Loss": 1.007476806640625, "Finetune/Loss (Raw)": 0.3226258158683777, "Finetune/Step": 210, "Finetune/Step Time": 4.602872045710683} +{"Finetune/Learning Rate": 1.9993915515045485e-05, "Finetune/Loss": 1.0077567100524902, "Finetune/Loss (Raw)": 0.9943254590034485, "Finetune/Step": 211, "Finetune/Step Time": 4.59582925401628} +{"Finetune/Learning Rate": 1.999369629484267e-05, "Finetune/Loss": 1.0069125890731812, "Finetune/Loss (Raw)": 1.081405758857727, "Finetune/Step": 212, "Finetune/Step Time": 4.593586752191186} +{"Finetune/Learning Rate": 1.999347319628143e-05, "Finetune/Loss": 1.008042573928833, "Finetune/Loss (Raw)": 1.1945152282714844, "Finetune/Step": 213, "Finetune/Step Time": 4.596132971346378} +{"Finetune/Learning Rate": 1.9993246219448347e-05, "Finetune/Loss": 1.0130454301834106, "Finetune/Loss (Raw)": 0.9926812052726746, "Finetune/Step": 214, "Finetune/Step Time": 4.571128377690911} +{"Finetune/Learning Rate": 1.999301536443151e-05, "Finetune/Loss": 1.0124399662017822, "Finetune/Loss (Raw)": 0.955499529838562, "Finetune/Step": 215, "Finetune/Step Time": 4.57095636613667} +{"Finetune/Learning Rate": 1.9992780631320508e-05, "Finetune/Loss": 1.0102424621582031, "Finetune/Loss (Raw)": 0.949148952960968, "Finetune/Step": 216, "Finetune/Step Time": 4.569616304710507} +{"Finetune/Learning Rate": 1.9992542020206432e-05, "Finetune/Loss": 1.0106356143951416, "Finetune/Loss (Raw)": 1.125590443611145, "Finetune/Step": 217, "Finetune/Step Time": 4.5675379149615765} +{"Finetune/Learning Rate": 1.9992299531181892e-05, "Finetune/Loss": 1.0101344585418701, "Finetune/Loss (Raw)": 1.0745155811309814, "Finetune/Step": 218, "Finetune/Step Time": 4.554756557568908} +{"Finetune/Learning Rate": 1.999205316434098e-05, "Finetune/Loss": 1.0104854106903076, "Finetune/Loss (Raw)": 1.2174748182296753, "Finetune/Step": 219, "Finetune/Step Time": 4.542293617501855} +{"Finetune/Learning Rate": 1.9991802919779318e-05, "Finetune/Loss": 1.0087028741836548, "Finetune/Loss (Raw)": 0.9772035479545593, "Finetune/Step": 220, "Finetune/Step Time": 4.539776204153895} +{"Finetune/Learning Rate": 1.9991548797594017e-05, "Finetune/Loss": 1.0101895332336426, "Finetune/Loss (Raw)": 1.11631178855896, "Finetune/Step": 221, "Finetune/Step Time": 4.542665064334869} +{"Finetune/Learning Rate": 1.9991290797883693e-05, "Finetune/Loss": 1.0086309909820557, "Finetune/Loss (Raw)": 0.9618861079216003, "Finetune/Step": 222, "Finetune/Step Time": 4.548701491206884} +{"Finetune/Learning Rate": 1.9991028920748475e-05, "Finetune/Loss": 1.0078400373458862, "Finetune/Loss (Raw)": 0.9993644952774048, "Finetune/Step": 223, "Finetune/Step Time": 4.549547657370567} +{"Finetune/Learning Rate": 1.9990763166289993e-05, "Finetune/Loss": 1.0021497011184692, "Finetune/Loss (Raw)": 0.35927775502204895, "Finetune/Step": 224, "Finetune/Step Time": 4.576397553086281} +{"Finetune/Learning Rate": 1.9990493534611376e-05, "Finetune/Loss": 1.0024324655532837, "Finetune/Loss (Raw)": 1.0625509023666382, "Finetune/Step": 225, "Finetune/Step Time": 4.5761475414037704} +{"Finetune/Learning Rate": 1.9990220025817268e-05, "Finetune/Loss": 1.002922773361206, "Finetune/Loss (Raw)": 1.1908568143844604, "Finetune/Step": 226, "Finetune/Step Time": 4.5736775659024715} +{"Finetune/Learning Rate": 1.998994264001381e-05, "Finetune/Loss": 1.0105178356170654, "Finetune/Loss (Raw)": 1.1960840225219727, "Finetune/Step": 227, "Finetune/Step Time": 4.535719383507967} +{"Finetune/Learning Rate": 1.9989661377308654e-05, "Finetune/Loss": 1.0100321769714355, "Finetune/Loss (Raw)": 1.1573814153671265, "Finetune/Step": 228, "Finetune/Step Time": 4.5306048206985} +{"Finetune/Learning Rate": 1.9989376237810944e-05, "Finetune/Loss": 1.0099128484725952, "Finetune/Loss (Raw)": 1.0327786207199097, "Finetune/Step": 229, "Finetune/Step Time": 4.5238048285245895} +{"Finetune/Learning Rate": 1.9989087221631344e-05, "Finetune/Loss": 1.009498119354248, "Finetune/Loss (Raw)": 1.1617205142974854, "Finetune/Step": 230, "Finetune/Step Time": 4.526171075180173} +{"Finetune/Learning Rate": 1.9988794328882017e-05, "Finetune/Loss": 1.0151829719543457, "Finetune/Loss (Raw)": 1.0225409269332886, "Finetune/Step": 231, "Finetune/Step Time": 4.4891118835657835} +{"Finetune/Learning Rate": 1.9988497559676622e-05, "Finetune/Loss": 1.0145050287246704, "Finetune/Loss (Raw)": 1.0701771974563599, "Finetune/Step": 232, "Finetune/Step Time": 4.492566486820579} +{"Finetune/Learning Rate": 1.9988196914130332e-05, "Finetune/Loss": 1.0086069107055664, "Finetune/Loss (Raw)": 0.3117772936820984, "Finetune/Step": 233, "Finetune/Step Time": 4.534357413649559} +{"Finetune/Learning Rate": 1.9987892392359827e-05, "Finetune/Loss": 1.00943922996521, "Finetune/Loss (Raw)": 1.1734124422073364, "Finetune/Step": 234, "Finetune/Step Time": 4.553564807400107} +{"Finetune/Learning Rate": 1.998758399448328e-05, "Finetune/Loss": 1.0100274085998535, "Finetune/Loss (Raw)": 1.2103439569473267, "Finetune/Step": 235, "Finetune/Step Time": 4.5942709520459175} +{"Finetune/Learning Rate": 1.9987271720620376e-05, "Finetune/Loss": 1.010122537612915, "Finetune/Loss (Raw)": 1.0298494100570679, "Finetune/Step": 236, "Finetune/Step Time": 4.594401204958558} +{"Finetune/Learning Rate": 1.9986955570892302e-05, "Finetune/Loss": 1.0097874402999878, "Finetune/Loss (Raw)": 1.0993943214416504, "Finetune/Step": 237, "Finetune/Step Time": 4.627811199054122} +{"Finetune/Learning Rate": 1.998663554542175e-05, "Finetune/Loss": 1.0094904899597168, "Finetune/Loss (Raw)": 1.0175495147705078, "Finetune/Step": 238, "Finetune/Step Time": 4.631700538098812} +{"Finetune/Learning Rate": 1.9986311644332913e-05, "Finetune/Loss": 1.0077290534973145, "Finetune/Loss (Raw)": 0.9020845890045166, "Finetune/Step": 239, "Finetune/Step Time": 4.615732628852129} +{"Finetune/Learning Rate": 1.9985983867751497e-05, "Finetune/Loss": 1.0067979097366333, "Finetune/Loss (Raw)": 0.9661610126495361, "Finetune/Step": 240, "Finetune/Step Time": 4.602748626843095} +{"Finetune/Learning Rate": 1.99856522158047e-05, "Finetune/Loss": 1.0066330432891846, "Finetune/Loss (Raw)": 1.1106324195861816, "Finetune/Step": 241, "Finetune/Step Time": 4.590191902592778} +{"Finetune/Learning Rate": 1.998531668862123e-05, "Finetune/Loss": 1.013014554977417, "Finetune/Loss (Raw)": 1.0682060718536377, "Finetune/Step": 242, "Finetune/Step Time": 4.555257447063923} +{"Finetune/Learning Rate": 1.9984977286331302e-05, "Finetune/Loss": 1.012923002243042, "Finetune/Loss (Raw)": 1.0815385580062866, "Finetune/Step": 243, "Finetune/Step Time": 4.570976044982672} +{"Finetune/Learning Rate": 1.998463400906663e-05, "Finetune/Loss": 1.0129451751708984, "Finetune/Loss (Raw)": 0.9270982146263123, "Finetune/Step": 244, "Finetune/Step Time": 4.549740336835384} +{"Finetune/Learning Rate": 1.998428685696043e-05, "Finetune/Loss": 1.013741374015808, "Finetune/Loss (Raw)": 1.214155673980713, "Finetune/Step": 245, "Finetune/Step Time": 4.547742089256644} +{"Finetune/Learning Rate": 1.998393583014743e-05, "Finetune/Loss": 1.0140455961227417, "Finetune/Loss (Raw)": 1.182008981704712, "Finetune/Step": 246, "Finetune/Step Time": 4.5568173583596945} +{"Finetune/Learning Rate": 1.9983580928763852e-05, "Finetune/Loss": 1.0142433643341064, "Finetune/Loss (Raw)": 1.0700550079345703, "Finetune/Step": 247, "Finetune/Step Time": 4.56282209046185} +{"Finetune/Learning Rate": 1.998322215294743e-05, "Finetune/Loss": 1.0136206150054932, "Finetune/Loss (Raw)": 1.0088599920272827, "Finetune/Step": 248, "Finetune/Step Time": 4.57155747897923} +{"Finetune/Learning Rate": 1.9982859502837395e-05, "Finetune/Loss": 1.0136089324951172, "Finetune/Loss (Raw)": 1.0259335041046143, "Finetune/Step": 249, "Finetune/Step Time": 4.547649584710598} +{"Finetune/Learning Rate": 1.998249297857449e-05, "Finetune/Loss": 1.014555811882019, "Finetune/Loss (Raw)": 1.086026668548584, "Finetune/Step": 250, "Finetune/Step Time": 4.546235591173172} +{"Finetune/Learning Rate": 1.9982122580300947e-05, "Finetune/Loss": 1.0155781507492065, "Finetune/Loss (Raw)": 1.1486974954605103, "Finetune/Step": 251, "Finetune/Step Time": 4.561026586219668} +{"Finetune/Learning Rate": 1.9981748308160517e-05, "Finetune/Loss": 1.0149346590042114, "Finetune/Loss (Raw)": 1.0475877523422241, "Finetune/Step": 252, "Finetune/Step Time": 4.565427049994469} +{"Finetune/Learning Rate": 1.9981370162298444e-05, "Finetune/Loss": 1.0155460834503174, "Finetune/Loss (Raw)": 1.0904762744903564, "Finetune/Step": 253, "Finetune/Step Time": 4.56028632633388} +{"Finetune/Learning Rate": 1.9980988142861486e-05, "Finetune/Loss": 1.0145899057388306, "Finetune/Loss (Raw)": 0.9822152256965637, "Finetune/Step": 254, "Finetune/Step Time": 4.540229711681604} +{"Finetune/Learning Rate": 1.998060224999789e-05, "Finetune/Loss": 1.0093801021575928, "Finetune/Loss (Raw)": 0.32056698203086853, "Finetune/Step": 255, "Finetune/Step Time": 4.578153744339943} +{"Finetune/Learning Rate": 1.9980212483857412e-05, "Finetune/Loss": 1.0080487728118896, "Finetune/Loss (Raw)": 0.9582496881484985, "Finetune/Step": 256, "Finetune/Step Time": 4.579166324809194} +{"Finetune/Learning Rate": 1.9979818844591318e-05, "Finetune/Loss": 1.0082266330718994, "Finetune/Loss (Raw)": 1.1770793199539185, "Finetune/Step": 257, "Finetune/Step Time": 4.578819919377565} +{"Finetune/Learning Rate": 1.9979421332352372e-05, "Finetune/Loss": 1.0080344676971436, "Finetune/Loss (Raw)": 1.122994065284729, "Finetune/Step": 258, "Finetune/Step Time": 4.578695051372051} +{"Finetune/Learning Rate": 1.9979019947294836e-05, "Finetune/Loss": 1.00798761844635, "Finetune/Loss (Raw)": 1.0943481922149658, "Finetune/Step": 259, "Finetune/Step Time": 4.581001292914152} +{"Finetune/Learning Rate": 1.9978614689574486e-05, "Finetune/Loss": 1.0080997943878174, "Finetune/Loss (Raw)": 1.1270477771759033, "Finetune/Step": 260, "Finetune/Step Time": 4.600784866139293} +{"Finetune/Learning Rate": 1.9978205559348586e-05, "Finetune/Loss": 1.0079567432403564, "Finetune/Loss (Raw)": 0.8824716806411743, "Finetune/Step": 261, "Finetune/Step Time": 4.583650950342417} +{"Finetune/Learning Rate": 1.997779255677592e-05, "Finetune/Loss": 1.0049326419830322, "Finetune/Loss (Raw)": 0.8515284657478333, "Finetune/Step": 262, "Finetune/Step Time": 4.594895713031292} +{"Finetune/Learning Rate": 1.997737568201676e-05, "Finetune/Loss": 1.0052313804626465, "Finetune/Loss (Raw)": 1.0356314182281494, "Finetune/Step": 263, "Finetune/Step Time": 4.592633802443743} +{"Finetune/Learning Rate": 1.9976954935232892e-05, "Finetune/Loss": 1.004503607749939, "Finetune/Loss (Raw)": 0.992848813533783, "Finetune/Step": 264, "Finetune/Step Time": 4.6070533860474825} +{"Finetune/Learning Rate": 1.99765303165876e-05, "Finetune/Loss": 1.0047075748443604, "Finetune/Loss (Raw)": 1.0003037452697754, "Finetune/Step": 265, "Finetune/Step Time": 4.60334562882781} +{"Finetune/Learning Rate": 1.997610182624566e-05, "Finetune/Loss": 1.005605936050415, "Finetune/Loss (Raw)": 1.2353266477584839, "Finetune/Step": 266, "Finetune/Step Time": 4.614594645798206} +{"Finetune/Learning Rate": 1.9975669464373377e-05, "Finetune/Loss": 1.0055675506591797, "Finetune/Loss (Raw)": 0.9080079197883606, "Finetune/Step": 267, "Finetune/Step Time": 4.61229613609612} +{"Finetune/Learning Rate": 1.9975233231138525e-05, "Finetune/Loss": 1.0042413473129272, "Finetune/Loss (Raw)": 1.1168034076690674, "Finetune/Step": 268, "Finetune/Step Time": 4.615876249969006} +{"Finetune/Learning Rate": 1.997479312671041e-05, "Finetune/Loss": 1.0043418407440186, "Finetune/Loss (Raw)": 1.0498006343841553, "Finetune/Step": 269, "Finetune/Step Time": 4.621683379635215} +{"Finetune/Learning Rate": 1.9974349151259824e-05, "Finetune/Loss": 1.0045855045318604, "Finetune/Loss (Raw)": 1.0621974468231201, "Finetune/Step": 270, "Finetune/Step Time": 4.586319554597139} +{"Finetune/Learning Rate": 1.9973901304959066e-05, "Finetune/Loss": 1.005086064338684, "Finetune/Loss (Raw)": 1.030389666557312, "Finetune/Step": 271, "Finetune/Step Time": 4.577094728127122} +{"Finetune/Learning Rate": 1.9973449587981935e-05, "Finetune/Loss": 1.0109089612960815, "Finetune/Loss (Raw)": 1.1236644983291626, "Finetune/Step": 272, "Finetune/Step Time": 4.5367224756628275} +{"Finetune/Learning Rate": 1.9972994000503734e-05, "Finetune/Loss": 1.0164541006088257, "Finetune/Loss (Raw)": 1.0571043491363525, "Finetune/Step": 273, "Finetune/Step Time": 4.501729311421514} +{"Finetune/Learning Rate": 1.997253454270127e-05, "Finetune/Loss": 1.01750910282135, "Finetune/Loss (Raw)": 1.1074968576431274, "Finetune/Step": 274, "Finetune/Step Time": 4.508002396672964} +{"Finetune/Learning Rate": 1.9972071214752843e-05, "Finetune/Loss": 1.0184428691864014, "Finetune/Loss (Raw)": 1.0234307050704956, "Finetune/Step": 275, "Finetune/Step Time": 4.505257165059447} +{"Finetune/Learning Rate": 1.9971604016838264e-05, "Finetune/Loss": 1.0165460109710693, "Finetune/Loss (Raw)": 0.8362321257591248, "Finetune/Step": 276, "Finetune/Step Time": 4.538071401417255} +{"Finetune/Learning Rate": 1.997113294913885e-05, "Finetune/Loss": 1.0157749652862549, "Finetune/Loss (Raw)": 1.0264317989349365, "Finetune/Step": 277, "Finetune/Step Time": 4.537178095430136} +{"Finetune/Learning Rate": 1.9970658011837404e-05, "Finetune/Loss": 1.0150647163391113, "Finetune/Loss (Raw)": 1.0438501834869385, "Finetune/Step": 278, "Finetune/Step Time": 4.5299879796803} +{"Finetune/Learning Rate": 1.997017920511825e-05, "Finetune/Loss": 1.0160672664642334, "Finetune/Loss (Raw)": 1.2231348752975464, "Finetune/Step": 279, "Finetune/Step Time": 4.536207091063261} +{"Finetune/Learning Rate": 1.9969696529167197e-05, "Finetune/Loss": 1.0155209302902222, "Finetune/Loss (Raw)": 0.9910892248153687, "Finetune/Step": 280, "Finetune/Step Time": 4.534767592325807} +{"Finetune/Learning Rate": 1.996920998417156e-05, "Finetune/Loss": 1.0149626731872559, "Finetune/Loss (Raw)": 1.086607813835144, "Finetune/Step": 281, "Finetune/Step Time": 4.520283645018935} +{"Finetune/Learning Rate": 1.996871957032016e-05, "Finetune/Loss": 1.014064073562622, "Finetune/Loss (Raw)": 1.0250142812728882, "Finetune/Step": 282, "Finetune/Step Time": 4.519358741119504} +{"Finetune/Learning Rate": 1.996822528780332e-05, "Finetune/Loss": 1.013519287109375, "Finetune/Loss (Raw)": 1.1451728343963623, "Finetune/Step": 283, "Finetune/Step Time": 4.517896842211485} +{"Finetune/Learning Rate": 1.996772713681286e-05, "Finetune/Loss": 1.0202527046203613, "Finetune/Loss (Raw)": 1.1582285165786743, "Finetune/Step": 284, "Finetune/Step Time": 4.477337460964918} +{"Finetune/Learning Rate": 1.9967225117542097e-05, "Finetune/Loss": 1.019712209701538, "Finetune/Loss (Raw)": 1.0135751962661743, "Finetune/Step": 285, "Finetune/Step Time": 4.494909480214119} +{"Finetune/Learning Rate": 1.9966719230185864e-05, "Finetune/Loss": 1.0194779634475708, "Finetune/Loss (Raw)": 1.08449387550354, "Finetune/Step": 286, "Finetune/Step Time": 4.509892512112856} +{"Finetune/Learning Rate": 1.9966209474940477e-05, "Finetune/Loss": 1.019526481628418, "Finetune/Loss (Raw)": 1.084007740020752, "Finetune/Step": 287, "Finetune/Step Time": 4.516937801614404} +{"Finetune/Learning Rate": 1.996569585200377e-05, "Finetune/Loss": 1.0186643600463867, "Finetune/Loss (Raw)": 0.9118630290031433, "Finetune/Step": 288, "Finetune/Step Time": 4.525022691115737} +{"Finetune/Learning Rate": 1.9965178361575068e-05, "Finetune/Loss": 1.0193889141082764, "Finetune/Loss (Raw)": 1.1287364959716797, "Finetune/Step": 289, "Finetune/Step Time": 4.52967350371182} +{"Finetune/Learning Rate": 1.9964657003855197e-05, "Finetune/Loss": 1.0170382261276245, "Finetune/Loss (Raw)": 0.9659417271614075, "Finetune/Step": 290, "Finetune/Step Time": 4.540792426094413} +{"Finetune/Learning Rate": 1.9964131779046484e-05, "Finetune/Loss": 1.0173689126968384, "Finetune/Loss (Raw)": 1.1257191896438599, "Finetune/Step": 291, "Finetune/Step Time": 4.534586068242788} +{"Finetune/Learning Rate": 1.996360268735276e-05, "Finetune/Loss": 1.0184741020202637, "Finetune/Loss (Raw)": 1.1105046272277832, "Finetune/Step": 292, "Finetune/Step Time": 4.5230528470128775} +{"Finetune/Learning Rate": 1.9963069728979357e-05, "Finetune/Loss": 1.01925790309906, "Finetune/Loss (Raw)": 1.152453064918518, "Finetune/Step": 293, "Finetune/Step Time": 4.518988503143191} +{"Finetune/Learning Rate": 1.9962532904133107e-05, "Finetune/Loss": 1.0174134969711304, "Finetune/Loss (Raw)": 0.7628597021102905, "Finetune/Step": 294, "Finetune/Step Time": 4.524928001686931} +{"Finetune/Learning Rate": 1.996199221302234e-05, "Finetune/Loss": 1.0169405937194824, "Finetune/Loss (Raw)": 1.0906836986541748, "Finetune/Step": 295, "Finetune/Step Time": 4.530988773331046} +{"Finetune/Learning Rate": 1.9961447655856883e-05, "Finetune/Loss": 1.017155647277832, "Finetune/Loss (Raw)": 1.0201716423034668, "Finetune/Step": 296, "Finetune/Step Time": 4.536088950932026} +{"Finetune/Learning Rate": 1.9960899232848074e-05, "Finetune/Loss": 1.018424391746521, "Finetune/Loss (Raw)": 1.1374733448028564, "Finetune/Step": 297, "Finetune/Step Time": 4.539106730371714} +{"Finetune/Learning Rate": 1.9960346944208743e-05, "Finetune/Loss": 1.0226253271102905, "Finetune/Loss (Raw)": 0.9016613960266113, "Finetune/Step": 298, "Finetune/Step Time": 4.527229201048613} +{"Finetune/Learning Rate": 1.995979079015322e-05, "Finetune/Loss": 1.0171682834625244, "Finetune/Loss (Raw)": 0.311594158411026, "Finetune/Step": 299, "Finetune/Step Time": 4.56217940710485} +{"Finetune/Learning Rate": 1.995923077089734e-05, "Finetune/Loss": 1.0122857093811035, "Finetune/Loss (Raw)": 0.3672974109649658, "Finetune/Step": 300, "Finetune/Step Time": 4.588397037237883} +{"Finetune/Learning Rate": 1.9958666886658436e-05, "Finetune/Loss": 1.0059924125671387, "Finetune/Loss (Raw)": 0.39966481924057007, "Finetune/Step": 301, "Finetune/Step Time": 4.626881822943687} +{"Finetune/Learning Rate": 1.995809913765534e-05, "Finetune/Loss": 1.0059711933135986, "Finetune/Loss (Raw)": 0.8971643447875977, "Finetune/Step": 302, "Finetune/Step Time": 4.635348008945584} +{"Finetune/Learning Rate": 1.9957527524108386e-05, "Finetune/Loss": 0.9991788864135742, "Finetune/Loss (Raw)": 0.32390066981315613, "Finetune/Step": 303, "Finetune/Step Time": 4.672680541872978} +{"Finetune/Learning Rate": 1.99569520462394e-05, "Finetune/Loss": 1.004968285560608, "Finetune/Loss (Raw)": 1.0981649160385132, "Finetune/Step": 304, "Finetune/Step Time": 4.6363013461232185} +{"Finetune/Learning Rate": 1.995637270427172e-05, "Finetune/Loss": 1.0055257081985474, "Finetune/Loss (Raw)": 1.0532022714614868, "Finetune/Step": 305, "Finetune/Step Time": 4.634259013459086} +{"Finetune/Learning Rate": 1.9955789498430178e-05, "Finetune/Loss": 1.0034925937652588, "Finetune/Loss (Raw)": 0.9485588073730469, "Finetune/Step": 306, "Finetune/Step Time": 4.640709929168224} +{"Finetune/Learning Rate": 1.99552024289411e-05, "Finetune/Loss": 1.0055028200149536, "Finetune/Loss (Raw)": 1.1665501594543457, "Finetune/Step": 307, "Finetune/Step Time": 4.653996217995882} +{"Finetune/Learning Rate": 1.995461149603232e-05, "Finetune/Loss": 1.0053448677062988, "Finetune/Loss (Raw)": 1.051745057106018, "Finetune/Step": 308, "Finetune/Step Time": 4.643085993826389} +{"Finetune/Learning Rate": 1.9954016699933163e-05, "Finetune/Loss": 1.0063979625701904, "Finetune/Loss (Raw)": 1.1565252542495728, "Finetune/Step": 309, "Finetune/Step Time": 4.652153812348843} +{"Finetune/Learning Rate": 1.995341804087446e-05, "Finetune/Loss": 1.006040096282959, "Finetune/Loss (Raw)": 1.0148476362228394, "Finetune/Step": 310, "Finetune/Step Time": 4.647049473598599} +{"Finetune/Learning Rate": 1.9952815519088544e-05, "Finetune/Loss": 1.0061922073364258, "Finetune/Loss (Raw)": 1.0404318571090698, "Finetune/Step": 311, "Finetune/Step Time": 4.65421044267714} +{"Finetune/Learning Rate": 1.9952209134809234e-05, "Finetune/Loss": 1.0047824382781982, "Finetune/Loss (Raw)": 0.9857705235481262, "Finetune/Step": 312, "Finetune/Step Time": 4.652664076536894} +{"Finetune/Learning Rate": 1.995159888827186e-05, "Finetune/Loss": 1.0062389373779297, "Finetune/Loss (Raw)": 1.1183524131774902, "Finetune/Step": 313, "Finetune/Step Time": 4.660468773916364} +{"Finetune/Learning Rate": 1.9950984779713244e-05, "Finetune/Loss": 1.0075335502624512, "Finetune/Loss (Raw)": 1.1860344409942627, "Finetune/Step": 314, "Finetune/Step Time": 4.644284680485725} +{"Finetune/Learning Rate": 1.9950366809371713e-05, "Finetune/Loss": 1.0073466300964355, "Finetune/Loss (Raw)": 0.991701066493988, "Finetune/Step": 315, "Finetune/Step Time": 4.650088433176279} +{"Finetune/Learning Rate": 1.9949744977487087e-05, "Finetune/Loss": 1.0013984441757202, "Finetune/Loss (Raw)": 0.37977567315101624, "Finetune/Step": 316, "Finetune/Step Time": 4.688576383516192} +{"Finetune/Learning Rate": 1.994911928430069e-05, "Finetune/Loss": 1.0010833740234375, "Finetune/Loss (Raw)": 0.9716652631759644, "Finetune/Step": 317, "Finetune/Step Time": 4.684453869238496} +{"Finetune/Learning Rate": 1.994848973005534e-05, "Finetune/Loss": 1.0010164976119995, "Finetune/Loss (Raw)": 1.0662596225738525, "Finetune/Step": 318, "Finetune/Step Time": 4.67794936709106} +{"Finetune/Learning Rate": 1.994785631499535e-05, "Finetune/Loss": 1.0002987384796143, "Finetune/Loss (Raw)": 0.9961317777633667, "Finetune/Step": 319, "Finetune/Step Time": 4.690813183784485} +{"Finetune/Learning Rate": 1.994721903936654e-05, "Finetune/Loss": 0.9993689656257629, "Finetune/Loss (Raw)": 0.9508721232414246, "Finetune/Step": 320, "Finetune/Step Time": 4.686071507632732} +{"Finetune/Learning Rate": 1.9946577903416224e-05, "Finetune/Loss": 0.9945271015167236, "Finetune/Loss (Raw)": 0.3309485614299774, "Finetune/Step": 321, "Finetune/Step Time": 4.7238090839236975} +{"Finetune/Learning Rate": 1.994593290739322e-05, "Finetune/Loss": 0.9940148591995239, "Finetune/Loss (Raw)": 0.9971112608909607, "Finetune/Step": 322, "Finetune/Step Time": 4.724529949948192} +{"Finetune/Learning Rate": 1.9945284051547823e-05, "Finetune/Loss": 0.9944635629653931, "Finetune/Loss (Raw)": 1.0715458393096924, "Finetune/Step": 323, "Finetune/Step Time": 4.718496846035123} +{"Finetune/Learning Rate": 1.9944631336131856e-05, "Finetune/Loss": 0.9955021142959595, "Finetune/Loss (Raw)": 1.1089659929275513, "Finetune/Step": 324, "Finetune/Step Time": 4.717718908563256} +{"Finetune/Learning Rate": 1.994397476139862e-05, "Finetune/Loss": 0.9954122304916382, "Finetune/Loss (Raw)": 1.049248218536377, "Finetune/Step": 325, "Finetune/Step Time": 4.7146583404392} +{"Finetune/Learning Rate": 1.994331432760292e-05, "Finetune/Loss": 0.9969874620437622, "Finetune/Loss (Raw)": 1.2222305536270142, "Finetune/Step": 326, "Finetune/Step Time": 4.7148778438568115} +{"Finetune/Learning Rate": 1.9942650035001053e-05, "Finetune/Loss": 0.9971222281455994, "Finetune/Loss (Raw)": 1.0226961374282837, "Finetune/Step": 327, "Finetune/Step Time": 4.709117893129587} +{"Finetune/Learning Rate": 1.994198188385082e-05, "Finetune/Loss": 0.997407853603363, "Finetune/Loss (Raw)": 1.0770790576934814, "Finetune/Step": 328, "Finetune/Step Time": 4.716018952429295} +{"Finetune/Learning Rate": 1.9941309874411525e-05, "Finetune/Loss": 0.9959278106689453, "Finetune/Loss (Raw)": 0.9497897624969482, "Finetune/Step": 329, "Finetune/Step Time": 4.732343001291156} +{"Finetune/Learning Rate": 1.9940634006943952e-05, "Finetune/Loss": 0.9939672946929932, "Finetune/Loss (Raw)": 0.8778783082962036, "Finetune/Step": 330, "Finetune/Step Time": 4.736475579440594} +{"Finetune/Learning Rate": 1.9939954281710395e-05, "Finetune/Loss": 0.9938756823539734, "Finetune/Loss (Raw)": 1.0630284547805786, "Finetune/Step": 331, "Finetune/Step Time": 4.7407186683267355} +{"Finetune/Learning Rate": 1.9939270698974643e-05, "Finetune/Loss": 0.9920224547386169, "Finetune/Loss (Raw)": 0.8294389247894287, "Finetune/Step": 332, "Finetune/Step Time": 4.75689641572535} +{"Finetune/Learning Rate": 1.9938583259001983e-05, "Finetune/Loss": 0.9976166486740112, "Finetune/Loss (Raw)": 1.0391690731048584, "Finetune/Step": 333, "Finetune/Step Time": 4.714690830558538} +{"Finetune/Learning Rate": 1.993789196205919e-05, "Finetune/Loss": 0.9974453449249268, "Finetune/Loss (Raw)": 1.0009543895721436, "Finetune/Step": 334, "Finetune/Step Time": 4.709797969087958} +{"Finetune/Learning Rate": 1.9937196808414552e-05, "Finetune/Loss": 0.9960177540779114, "Finetune/Loss (Raw)": 1.014389991760254, "Finetune/Step": 335, "Finetune/Step Time": 4.697416001930833} +{"Finetune/Learning Rate": 1.993649779833784e-05, "Finetune/Loss": 0.9963245391845703, "Finetune/Loss (Raw)": 1.104671835899353, "Finetune/Step": 336, "Finetune/Step Time": 4.70059104077518} +{"Finetune/Learning Rate": 1.993579493210032e-05, "Finetune/Loss": 0.9979891777038574, "Finetune/Loss (Raw)": 1.2340844869613647, "Finetune/Step": 337, "Finetune/Step Time": 4.696303386241198} +{"Finetune/Learning Rate": 1.9935088209974774e-05, "Finetune/Loss": 1.0032517910003662, "Finetune/Loss (Raw)": 0.9962405562400818, "Finetune/Step": 338, "Finetune/Step Time": 4.6680702064186335} +{"Finetune/Learning Rate": 1.9934377632235455e-05, "Finetune/Loss": 1.0030517578125, "Finetune/Loss (Raw)": 0.9687097668647766, "Finetune/Step": 339, "Finetune/Step Time": 4.670983895659447} +{"Finetune/Learning Rate": 1.9933663199158135e-05, "Finetune/Loss": 1.001753807067871, "Finetune/Loss (Raw)": 0.9152708053588867, "Finetune/Step": 340, "Finetune/Step Time": 4.668941969051957} +{"Finetune/Learning Rate": 1.993294491102006e-05, "Finetune/Loss": 1.0004489421844482, "Finetune/Loss (Raw)": 1.0274937152862549, "Finetune/Step": 341, "Finetune/Step Time": 4.670263063162565} +{"Finetune/Learning Rate": 1.9932222768099992e-05, "Finetune/Loss": 1.0012646913528442, "Finetune/Loss (Raw)": 1.0971012115478516, "Finetune/Step": 342, "Finetune/Step Time": 4.651760747656226} +{"Finetune/Learning Rate": 1.9931496770678178e-05, "Finetune/Loss": 1.0018310546875, "Finetune/Loss (Raw)": 1.0279892683029175, "Finetune/Step": 343, "Finetune/Step Time": 4.649871904402971} +{"Finetune/Learning Rate": 1.993076691903636e-05, "Finetune/Loss": 1.0029656887054443, "Finetune/Loss (Raw)": 1.0943902730941772, "Finetune/Step": 344, "Finetune/Step Time": 4.634983351454139} +{"Finetune/Learning Rate": 1.993003321345779e-05, "Finetune/Loss": 1.002983570098877, "Finetune/Loss (Raw)": 1.1278743743896484, "Finetune/Step": 345, "Finetune/Step Time": 4.637558318674564} +{"Finetune/Learning Rate": 1.9929295654227193e-05, "Finetune/Loss": 1.0034542083740234, "Finetune/Loss (Raw)": 1.1347472667694092, "Finetune/Step": 346, "Finetune/Step Time": 4.6441800147295} +{"Finetune/Learning Rate": 1.9928554241630804e-05, "Finetune/Loss": 1.0030066967010498, "Finetune/Loss (Raw)": 1.1601982116699219, "Finetune/Step": 347, "Finetune/Step Time": 4.647732922807336} +{"Finetune/Learning Rate": 1.9927808975956354e-05, "Finetune/Loss": 1.0038073062896729, "Finetune/Loss (Raw)": 1.0796730518341064, "Finetune/Step": 348, "Finetune/Step Time": 4.667606757953763} +{"Finetune/Learning Rate": 1.9927059857493068e-05, "Finetune/Loss": 1.0041226148605347, "Finetune/Loss (Raw)": 1.1566787958145142, "Finetune/Step": 349, "Finetune/Step Time": 4.67649363912642} +{"Finetune/Learning Rate": 1.9926306886531658e-05, "Finetune/Loss": 1.00438392162323, "Finetune/Loss (Raw)": 0.9953445196151733, "Finetune/Step": 350, "Finetune/Step Time": 4.691054753959179} +{"Finetune/Learning Rate": 1.9925550063364337e-05, "Finetune/Loss": 1.004406213760376, "Finetune/Loss (Raw)": 1.0022025108337402, "Finetune/Step": 351, "Finetune/Step Time": 4.704778075218201} +{"Finetune/Learning Rate": 1.992478938828482e-05, "Finetune/Loss": 1.0094361305236816, "Finetune/Loss (Raw)": 1.003112554550171, "Finetune/Step": 352, "Finetune/Step Time": 4.676857780665159} +{"Finetune/Learning Rate": 1.9924024861588307e-05, "Finetune/Loss": 1.0087826251983643, "Finetune/Loss (Raw)": 0.9788995981216431, "Finetune/Step": 353, "Finetune/Step Time": 4.676025442779064} +{"Finetune/Learning Rate": 1.9923256483571495e-05, "Finetune/Loss": 1.0066198110580444, "Finetune/Loss (Raw)": 0.9140263199806213, "Finetune/Step": 354, "Finetune/Step Time": 4.682865599170327} +{"Finetune/Learning Rate": 1.9922484254532578e-05, "Finetune/Loss": 1.0056777000427246, "Finetune/Loss (Raw)": 1.0754767656326294, "Finetune/Step": 355, "Finetune/Step Time": 4.692568654194474} +{"Finetune/Learning Rate": 1.9921708174771237e-05, "Finetune/Loss": 1.0038204193115234, "Finetune/Loss (Raw)": 0.9196684956550598, "Finetune/Step": 356, "Finetune/Step Time": 4.699942581355572} +{"Finetune/Learning Rate": 1.9920928244588662e-05, "Finetune/Loss": 1.0030255317687988, "Finetune/Loss (Raw)": 0.9310289621353149, "Finetune/Step": 357, "Finetune/Step Time": 4.71392497420311} +{"Finetune/Learning Rate": 1.9920144464287527e-05, "Finetune/Loss": 0.9968636631965637, "Finetune/Loss (Raw)": 0.3730010986328125, "Finetune/Step": 358, "Finetune/Step Time": 4.751334298402071} +{"Finetune/Learning Rate": 1.9919356834171996e-05, "Finetune/Loss": 0.995526134967804, "Finetune/Loss (Raw)": 0.8513343930244446, "Finetune/Step": 359, "Finetune/Step Time": 4.752534972503781} +{"Finetune/Learning Rate": 1.9918565354547738e-05, "Finetune/Loss": 0.9950728416442871, "Finetune/Loss (Raw)": 1.0121625661849976, "Finetune/Step": 360, "Finetune/Step Time": 4.752241199836135} +{"Finetune/Learning Rate": 1.991777002572191e-05, "Finetune/Loss": 0.9996438026428223, "Finetune/Loss (Raw)": 0.8968548774719238, "Finetune/Step": 361, "Finetune/Step Time": 4.714404858648777} +{"Finetune/Learning Rate": 1.9916970848003166e-05, "Finetune/Loss": 0.9981157779693604, "Finetune/Loss (Raw)": 0.9778240323066711, "Finetune/Step": 362, "Finetune/Step Time": 4.7069680746644735} +{"Finetune/Learning Rate": 1.9916167821701645e-05, "Finetune/Loss": 0.9970975518226624, "Finetune/Loss (Raw)": 1.0800048112869263, "Finetune/Step": 363, "Finetune/Step Time": 4.665769021958113} +{"Finetune/Learning Rate": 1.991536094712899e-05, "Finetune/Loss": 0.9976738691329956, "Finetune/Loss (Raw)": 1.1036224365234375, "Finetune/Step": 364, "Finetune/Step Time": 4.664530515670776} +{"Finetune/Learning Rate": 1.991455022459833e-05, "Finetune/Loss": 0.9978484511375427, "Finetune/Loss (Raw)": 1.1217433214187622, "Finetune/Step": 365, "Finetune/Step Time": 4.633891047909856} +{"Finetune/Learning Rate": 1.9913735654424297e-05, "Finetune/Loss": 0.9970041513442993, "Finetune/Loss (Raw)": 0.9094763398170471, "Finetune/Step": 366, "Finetune/Step Time": 4.639113845303655} +{"Finetune/Learning Rate": 1.9912917236923003e-05, "Finetune/Loss": 0.9978334903717041, "Finetune/Loss (Raw)": 1.008235216140747, "Finetune/Step": 367, "Finetune/Step Time": 4.644546799361706} +{"Finetune/Learning Rate": 1.9912094972412067e-05, "Finetune/Loss": 0.998329758644104, "Finetune/Loss (Raw)": 1.0296909809112549, "Finetune/Step": 368, "Finetune/Step Time": 4.650440299883485} +{"Finetune/Learning Rate": 1.9911268861210584e-05, "Finetune/Loss": 0.9974302053451538, "Finetune/Loss (Raw)": 0.9954864978790283, "Finetune/Step": 369, "Finetune/Step Time": 4.6559744868427515} +{"Finetune/Learning Rate": 1.991043890363916e-05, "Finetune/Loss": 0.9981905817985535, "Finetune/Loss (Raw)": 1.1655434370040894, "Finetune/Step": 370, "Finetune/Step Time": 4.649600880220532} +{"Finetune/Learning Rate": 1.9909605100019883e-05, "Finetune/Loss": 0.9988195896148682, "Finetune/Loss (Raw)": 1.1620386838912964, "Finetune/Step": 371, "Finetune/Step Time": 4.628907132893801} +{"Finetune/Learning Rate": 1.9908767450676336e-05, "Finetune/Loss": 0.9998209476470947, "Finetune/Loss (Raw)": 1.055284023284912, "Finetune/Step": 372, "Finetune/Step Time": 4.638110514730215} +{"Finetune/Learning Rate": 1.9907925955933593e-05, "Finetune/Loss": 0.9938123226165771, "Finetune/Loss (Raw)": 0.4450480043888092, "Finetune/Step": 373, "Finetune/Step Time": 4.673972174525261} +{"Finetune/Learning Rate": 1.9907080616118223e-05, "Finetune/Loss": 0.9874331951141357, "Finetune/Loss (Raw)": 0.3654768466949463, "Finetune/Step": 374, "Finetune/Step Time": 4.702702829614282} +{"Finetune/Learning Rate": 1.9906231431558283e-05, "Finetune/Loss": 0.9877372980117798, "Finetune/Loss (Raw)": 1.1089873313903809, "Finetune/Step": 375, "Finetune/Step Time": 4.699658442288637} +{"Finetune/Learning Rate": 1.990537840258333e-05, "Finetune/Loss": 0.9869126081466675, "Finetune/Loss (Raw)": 0.9033029079437256, "Finetune/Step": 376, "Finetune/Step Time": 4.699548603966832} +{"Finetune/Learning Rate": 1.9904521529524402e-05, "Finetune/Loss": 0.9865322709083557, "Finetune/Loss (Raw)": 0.9772448539733887, "Finetune/Step": 377, "Finetune/Step Time": 4.705313749611378} +{"Finetune/Learning Rate": 1.990366081271404e-05, "Finetune/Loss": 0.9875891804695129, "Finetune/Loss (Raw)": 1.221311092376709, "Finetune/Step": 378, "Finetune/Step Time": 4.704427802935243} +{"Finetune/Learning Rate": 1.990279625248627e-05, "Finetune/Loss": 0.9876683354377747, "Finetune/Loss (Raw)": 1.158820629119873, "Finetune/Step": 379, "Finetune/Step Time": 4.690908171236515} +{"Finetune/Learning Rate": 1.990192784917661e-05, "Finetune/Loss": 0.9876291751861572, "Finetune/Loss (Raw)": 1.0425835847854614, "Finetune/Step": 380, "Finetune/Step Time": 4.683487510308623} +{"Finetune/Learning Rate": 1.9901055603122067e-05, "Finetune/Loss": 0.9874115586280823, "Finetune/Loss (Raw)": 1.0626238584518433, "Finetune/Step": 381, "Finetune/Step Time": 4.691826246678829} +{"Finetune/Learning Rate": 1.9900179514661148e-05, "Finetune/Loss": 0.9870277643203735, "Finetune/Loss (Raw)": 0.9330905675888062, "Finetune/Step": 382, "Finetune/Step Time": 4.736408449709415} +{"Finetune/Learning Rate": 1.9899299584133848e-05, "Finetune/Loss": 0.9918358325958252, "Finetune/Loss (Raw)": 0.9359959363937378, "Finetune/Step": 383, "Finetune/Step Time": 4.731825165450573} +{"Finetune/Learning Rate": 1.9898415811881643e-05, "Finetune/Loss": 0.9881234169006348, "Finetune/Loss (Raw)": 0.48305800557136536, "Finetune/Step": 384, "Finetune/Step Time": 4.765254057943821} +{"Finetune/Learning Rate": 1.989752819824751e-05, "Finetune/Loss": 0.9868654608726501, "Finetune/Loss (Raw)": 1.016066074371338, "Finetune/Step": 385, "Finetune/Step Time": 4.76825556717813} +{"Finetune/Learning Rate": 1.989663674357592e-05, "Finetune/Loss": 0.9860222935676575, "Finetune/Loss (Raw)": 1.0150682926177979, "Finetune/Step": 386, "Finetune/Step Time": 4.767452692613006} +{"Finetune/Learning Rate": 1.9895741448212822e-05, "Finetune/Loss": 0.9843175411224365, "Finetune/Loss (Raw)": 0.8761275410652161, "Finetune/Step": 387, "Finetune/Step Time": 4.769118823111057} +{"Finetune/Learning Rate": 1.9894842312505673e-05, "Finetune/Loss": 0.9829919338226318, "Finetune/Loss (Raw)": 0.957381546497345, "Finetune/Step": 388, "Finetune/Step Time": 4.753183117136359} +{"Finetune/Learning Rate": 1.9893939336803396e-05, "Finetune/Loss": 0.9834784269332886, "Finetune/Loss (Raw)": 0.9447392225265503, "Finetune/Step": 389, "Finetune/Step Time": 4.752778731286526} +{"Finetune/Learning Rate": 1.989303252145643e-05, "Finetune/Loss": 0.983434796333313, "Finetune/Loss (Raw)": 0.845938503742218, "Finetune/Step": 390, "Finetune/Step Time": 4.7392415180802345} +{"Finetune/Learning Rate": 1.9892121866816682e-05, "Finetune/Loss": 0.9831911325454712, "Finetune/Loss (Raw)": 1.004453182220459, "Finetune/Step": 391, "Finetune/Step Time": 4.735091783106327} +{"Finetune/Learning Rate": 1.989120737323757e-05, "Finetune/Loss": 0.9830639362335205, "Finetune/Loss (Raw)": 0.9765645861625671, "Finetune/Step": 392, "Finetune/Step Time": 4.7317197024822235} +{"Finetune/Learning Rate": 1.9890289041073984e-05, "Finetune/Loss": 0.9819312691688538, "Finetune/Loss (Raw)": 0.8553181290626526, "Finetune/Step": 393, "Finetune/Step Time": 4.7438826728612185} +{"Finetune/Learning Rate": 1.988936687068231e-05, "Finetune/Loss": 0.9786296486854553, "Finetune/Loss (Raw)": 0.8127208352088928, "Finetune/Step": 394, "Finetune/Step Time": 4.759285632520914} +{"Finetune/Learning Rate": 1.9888440862420433e-05, "Finetune/Loss": 0.9809796810150146, "Finetune/Loss (Raw)": 1.208804726600647, "Finetune/Step": 395, "Finetune/Step Time": 4.757895777001977} +{"Finetune/Learning Rate": 1.988751101664771e-05, "Finetune/Loss": 0.9809160232543945, "Finetune/Loss (Raw)": 1.1086642742156982, "Finetune/Step": 396, "Finetune/Step Time": 4.758929960429668} +{"Finetune/Learning Rate": 1.9886577333724995e-05, "Finetune/Loss": 0.9793571829795837, "Finetune/Loss (Raw)": 0.8502705693244934, "Finetune/Step": 397, "Finetune/Step Time": 4.774667229503393} +{"Finetune/Learning Rate": 1.9885639814014642e-05, "Finetune/Loss": 0.9802888631820679, "Finetune/Loss (Raw)": 1.1814543008804321, "Finetune/Step": 398, "Finetune/Step Time": 4.767894858494401} +{"Finetune/Learning Rate": 1.9884698457880476e-05, "Finetune/Loss": 0.9795111417770386, "Finetune/Loss (Raw)": 0.9308422207832336, "Finetune/Step": 399, "Finetune/Step Time": 4.7557156179100275} +{"Finetune/Learning Rate": 1.988375326568782e-05, "Finetune/Loss": 0.9781611561775208, "Finetune/Loss (Raw)": 0.9508621692657471, "Finetune/Step": 400, "Finetune/Step Time": 4.75304545275867} +{"Finetune/Learning Rate": 1.9882804237803487e-05, "Finetune/Loss": 0.9786326885223389, "Finetune/Loss (Raw)": 1.1174627542495728, "Finetune/Step": 401, "Finetune/Step Time": 4.756057396531105} +{"Finetune/Learning Rate": 1.9881851374595774e-05, "Finetune/Loss": 0.977879524230957, "Finetune/Loss (Raw)": 1.01108717918396, "Finetune/Step": 402, "Finetune/Step Time": 4.767908334732056} +{"Finetune/Learning Rate": 1.988089467643447e-05, "Finetune/Loss": 0.9775320887565613, "Finetune/Loss (Raw)": 0.9789569973945618, "Finetune/Step": 403, "Finetune/Step Time": 4.7673177141696215} +{"Finetune/Learning Rate": 1.9879934143690848e-05, "Finetune/Loss": 0.9794331789016724, "Finetune/Loss (Raw)": 1.0795780420303345, "Finetune/Step": 404, "Finetune/Step Time": 4.734752016142011} +{"Finetune/Learning Rate": 1.9878969776737675e-05, "Finetune/Loss": 0.9771725535392761, "Finetune/Loss (Raw)": 0.7370717525482178, "Finetune/Step": 405, "Finetune/Step Time": 4.746978189796209} +{"Finetune/Learning Rate": 1.9878001575949205e-05, "Finetune/Loss": 0.9727799296379089, "Finetune/Loss (Raw)": 0.48158907890319824, "Finetune/Step": 406, "Finetune/Step Time": 4.784900201484561} +{"Finetune/Learning Rate": 1.9877029541701174e-05, "Finetune/Loss": 0.9665206074714661, "Finetune/Loss (Raw)": 0.42194247245788574, "Finetune/Step": 407, "Finetune/Step Time": 4.810828685760498} +{"Finetune/Learning Rate": 1.9876053674370806e-05, "Finetune/Loss": 0.966524600982666, "Finetune/Loss (Raw)": 0.9915977716445923, "Finetune/Step": 408, "Finetune/Step Time": 4.8104507979005575} +{"Finetune/Learning Rate": 1.9875073974336827e-05, "Finetune/Loss": 0.9673796892166138, "Finetune/Loss (Raw)": 1.1960569620132446, "Finetune/Step": 409, "Finetune/Step Time": 4.811509940773249} +{"Finetune/Learning Rate": 1.9874090441979433e-05, "Finetune/Loss": 0.9689633250236511, "Finetune/Loss (Raw)": 1.2277204990386963, "Finetune/Step": 410, "Finetune/Step Time": 4.809674061834812} +{"Finetune/Learning Rate": 1.987310307768031e-05, "Finetune/Loss": 0.9678162336349487, "Finetune/Loss (Raw)": 0.9983434677124023, "Finetune/Step": 411, "Finetune/Step Time": 4.812483754009008} +{"Finetune/Learning Rate": 1.987211188182264e-05, "Finetune/Loss": 0.9616880416870117, "Finetune/Loss (Raw)": 0.37383130192756653, "Finetune/Step": 412, "Finetune/Step Time": 4.8583688121289015} +{"Finetune/Learning Rate": 1.9871116854791085e-05, "Finetune/Loss": 0.962573230266571, "Finetune/Loss (Raw)": 1.1268715858459473, "Finetune/Step": 413, "Finetune/Step Time": 4.8361413311213255} +{"Finetune/Learning Rate": 1.9870117996971797e-05, "Finetune/Loss": 0.9614906907081604, "Finetune/Loss (Raw)": 0.9459274411201477, "Finetune/Step": 414, "Finetune/Step Time": 4.820988016203046} +{"Finetune/Learning Rate": 1.986911530875241e-05, "Finetune/Loss": 0.9613046646118164, "Finetune/Loss (Raw)": 1.0601937770843506, "Finetune/Step": 415, "Finetune/Step Time": 4.8177909180521965} +{"Finetune/Learning Rate": 1.9868108790522052e-05, "Finetune/Loss": 0.9629817008972168, "Finetune/Loss (Raw)": 1.1265316009521484, "Finetune/Step": 416, "Finetune/Step Time": 4.803091112524271} +{"Finetune/Learning Rate": 1.986709844267133e-05, "Finetune/Loss": 0.962386965751648, "Finetune/Loss (Raw)": 1.0526098012924194, "Finetune/Step": 417, "Finetune/Step Time": 4.797722518444061} +{"Finetune/Learning Rate": 1.9866084265592343e-05, "Finetune/Loss": 0.963454008102417, "Finetune/Loss (Raw)": 1.1025148630142212, "Finetune/Step": 418, "Finetune/Step Time": 4.785215953364968} +{"Finetune/Learning Rate": 1.986506625967867e-05, "Finetune/Loss": 0.9622788429260254, "Finetune/Loss (Raw)": 0.9752982258796692, "Finetune/Step": 419, "Finetune/Step Time": 4.803128661587834} +{"Finetune/Learning Rate": 1.9864044425325382e-05, "Finetune/Loss": 0.956450343132019, "Finetune/Loss (Raw)": 0.36446309089660645, "Finetune/Step": 420, "Finetune/Step Time": 4.839543247595429} +{"Finetune/Learning Rate": 1.986301876292903e-05, "Finetune/Loss": 0.9555586576461792, "Finetune/Loss (Raw)": 1.0383199453353882, "Finetune/Step": 421, "Finetune/Step Time": 4.839481862261891} +{"Finetune/Learning Rate": 1.986198927288766e-05, "Finetune/Loss": 0.9574769735336304, "Finetune/Loss (Raw)": 1.0083930492401123, "Finetune/Step": 422, "Finetune/Step Time": 4.845326563343406} +{"Finetune/Learning Rate": 1.9860955955600784e-05, "Finetune/Loss": 0.9578074216842651, "Finetune/Loss (Raw)": 1.1329929828643799, "Finetune/Step": 423, "Finetune/Step Time": 4.838840678334236} +{"Finetune/Learning Rate": 1.9859918811469424e-05, "Finetune/Loss": 0.9579905867576599, "Finetune/Loss (Raw)": 1.043610692024231, "Finetune/Step": 424, "Finetune/Step Time": 4.838128007948399} +{"Finetune/Learning Rate": 1.9858877840896076e-05, "Finetune/Loss": 0.9550327062606812, "Finetune/Loss (Raw)": 0.7588688731193542, "Finetune/Step": 425, "Finetune/Step Time": 4.873731696978211} +{"Finetune/Learning Rate": 1.9857833044284714e-05, "Finetune/Loss": 0.9559582471847534, "Finetune/Loss (Raw)": 1.0201244354248047, "Finetune/Step": 426, "Finetune/Step Time": 4.852777445688844} +{"Finetune/Learning Rate": 1.9856784422040802e-05, "Finetune/Loss": 0.9612162113189697, "Finetune/Loss (Raw)": 0.9846182465553284, "Finetune/Step": 427, "Finetune/Step Time": 4.817809049040079} +{"Finetune/Learning Rate": 1.98557319745713e-05, "Finetune/Loss": 0.9671454429626465, "Finetune/Loss (Raw)": 1.1262437105178833, "Finetune/Step": 428, "Finetune/Step Time": 4.780086550861597} +{"Finetune/Learning Rate": 1.985467570228463e-05, "Finetune/Loss": 0.9724566340446472, "Finetune/Loss (Raw)": 1.0794899463653564, "Finetune/Step": 429, "Finetune/Step Time": 4.746336329728365} +{"Finetune/Learning Rate": 1.985361560559072e-05, "Finetune/Loss": 0.9725048542022705, "Finetune/Loss (Raw)": 0.9033389091491699, "Finetune/Step": 430, "Finetune/Step Time": 4.740021156147122} +{"Finetune/Learning Rate": 1.9852551684900964e-05, "Finetune/Loss": 0.9784858822822571, "Finetune/Loss (Raw)": 1.0894726514816284, "Finetune/Step": 431, "Finetune/Step Time": 4.711604302749038} +{"Finetune/Learning Rate": 1.9851483940628256e-05, "Finetune/Loss": 0.9772968292236328, "Finetune/Loss (Raw)": 0.9459683299064636, "Finetune/Step": 432, "Finetune/Step Time": 4.717869307845831} +{"Finetune/Learning Rate": 1.9850412373186967e-05, "Finetune/Loss": 0.9762715101242065, "Finetune/Loss (Raw)": 0.9219560027122498, "Finetune/Step": 433, "Finetune/Step Time": 4.715491956099868} +{"Finetune/Learning Rate": 1.9849336982992948e-05, "Finetune/Loss": 0.976559042930603, "Finetune/Loss (Raw)": 0.9853684306144714, "Finetune/Step": 434, "Finetune/Step Time": 4.710652077570558} +{"Finetune/Learning Rate": 1.984825777046354e-05, "Finetune/Loss": 0.9740049839019775, "Finetune/Loss (Raw)": 0.8396263122558594, "Finetune/Step": 435, "Finetune/Step Time": 4.690372601151466} +{"Finetune/Learning Rate": 1.9847174736017557e-05, "Finetune/Loss": 0.9730483889579773, "Finetune/Loss (Raw)": 0.9293033480644226, "Finetune/Step": 436, "Finetune/Step Time": 4.687070459127426} +{"Finetune/Learning Rate": 1.9846087880075314e-05, "Finetune/Loss": 0.9720602631568909, "Finetune/Loss (Raw)": 1.030046820640564, "Finetune/Step": 437, "Finetune/Step Time": 4.689046842977405} +{"Finetune/Learning Rate": 1.984499720305859e-05, "Finetune/Loss": 0.9716610312461853, "Finetune/Loss (Raw)": 0.9637392163276672, "Finetune/Step": 438, "Finetune/Step Time": 4.685659071430564} +{"Finetune/Learning Rate": 1.984390270539066e-05, "Finetune/Loss": 0.9711794853210449, "Finetune/Loss (Raw)": 0.978797197341919, "Finetune/Step": 439, "Finetune/Step Time": 4.690504506230354} +{"Finetune/Learning Rate": 1.9842804387496276e-05, "Finetune/Loss": 0.9719220995903015, "Finetune/Loss (Raw)": 1.0808273553848267, "Finetune/Step": 440, "Finetune/Step Time": 4.697838576510549} +{"Finetune/Learning Rate": 1.9841702249801677e-05, "Finetune/Loss": 0.9714006185531616, "Finetune/Loss (Raw)": 1.0516088008880615, "Finetune/Step": 441, "Finetune/Step Time": 4.695298332720995} +{"Finetune/Learning Rate": 1.9840596292734573e-05, "Finetune/Loss": 0.9655816555023193, "Finetune/Loss (Raw)": 0.4412027895450592, "Finetune/Step": 442, "Finetune/Step Time": 4.734777765348554} +{"Finetune/Learning Rate": 1.9839486516724173e-05, "Finetune/Loss": 0.9662111401557922, "Finetune/Loss (Raw)": 1.0722771883010864, "Finetune/Step": 443, "Finetune/Step Time": 4.739244922995567} +{"Finetune/Learning Rate": 1.983837292220115e-05, "Finetune/Loss": 0.97214674949646, "Finetune/Loss (Raw)": 1.1395326852798462, "Finetune/Step": 444, "Finetune/Step Time": 4.701834095641971} +{"Finetune/Learning Rate": 1.983725550959768e-05, "Finetune/Loss": 0.9725075960159302, "Finetune/Loss (Raw)": 1.0178526639938354, "Finetune/Step": 445, "Finetune/Step Time": 4.692224849015474} +{"Finetune/Learning Rate": 1.98361342793474e-05, "Finetune/Loss": 0.9729277491569519, "Finetune/Loss (Raw)": 1.1200361251831055, "Finetune/Step": 446, "Finetune/Step Time": 4.70065738633275} +{"Finetune/Learning Rate": 1.9835009231885444e-05, "Finetune/Loss": 0.9732428789138794, "Finetune/Loss (Raw)": 1.0364675521850586, "Finetune/Step": 447, "Finetune/Step Time": 4.68526435829699} +{"Finetune/Learning Rate": 1.9833880367648412e-05, "Finetune/Loss": 0.9732987880706787, "Finetune/Loss (Raw)": 0.9580375552177429, "Finetune/Step": 448, "Finetune/Step Time": 4.685785012319684} +{"Finetune/Learning Rate": 1.9832747687074403e-05, "Finetune/Loss": 0.9796243906021118, "Finetune/Loss (Raw)": 1.1406155824661255, "Finetune/Step": 449, "Finetune/Step Time": 4.663150135427713} +{"Finetune/Learning Rate": 1.9831611190602984e-05, "Finetune/Loss": 0.9792490005493164, "Finetune/Loss (Raw)": 0.949065089225769, "Finetune/Step": 450, "Finetune/Step Time": 4.661758994683623} +{"Finetune/Learning Rate": 1.983047087867521e-05, "Finetune/Loss": 0.9785137176513672, "Finetune/Loss (Raw)": 0.9774335622787476, "Finetune/Step": 451, "Finetune/Step Time": 4.666396252810955} +{"Finetune/Learning Rate": 1.9829326751733607e-05, "Finetune/Loss": 0.9776983261108398, "Finetune/Loss (Raw)": 1.004590392112732, "Finetune/Step": 452, "Finetune/Step Time": 4.659320453181863} +{"Finetune/Learning Rate": 1.9828178810222197e-05, "Finetune/Loss": 0.9790646433830261, "Finetune/Loss (Raw)": 1.224137306213379, "Finetune/Step": 453, "Finetune/Step Time": 4.654561273753643} +{"Finetune/Learning Rate": 1.9827027054586468e-05, "Finetune/Loss": 0.9772756099700928, "Finetune/Loss (Raw)": 0.9932324886322021, "Finetune/Step": 454, "Finetune/Step Time": 4.669331964105368} +{"Finetune/Learning Rate": 1.9825871485273395e-05, "Finetune/Loss": 0.9778133630752563, "Finetune/Loss (Raw)": 1.0915273427963257, "Finetune/Step": 455, "Finetune/Step Time": 4.667543625459075} +{"Finetune/Learning Rate": 1.9824712102731435e-05, "Finetune/Loss": 0.9720972776412964, "Finetune/Loss (Raw)": 0.3454276919364929, "Finetune/Step": 456, "Finetune/Step Time": 4.693019676953554} +{"Finetune/Learning Rate": 1.982354890741052e-05, "Finetune/Loss": 0.9718612432479858, "Finetune/Loss (Raw)": 0.9195786118507385, "Finetune/Step": 457, "Finetune/Step Time": 4.673550423234701} +{"Finetune/Learning Rate": 1.982238189976206e-05, "Finetune/Loss": 0.9745362997055054, "Finetune/Loss (Raw)": 1.2202872037887573, "Finetune/Step": 458, "Finetune/Step Time": 4.670633882284164} +{"Finetune/Learning Rate": 1.9821211080238953e-05, "Finetune/Loss": 0.9740573167800903, "Finetune/Loss (Raw)": 1.0017122030258179, "Finetune/Step": 459, "Finetune/Step Time": 4.6732491645962} +{"Finetune/Learning Rate": 1.9820036449295564e-05, "Finetune/Loss": 0.9742229580879211, "Finetune/Loss (Raw)": 0.8506401777267456, "Finetune/Step": 460, "Finetune/Step Time": 4.672430692240596} +{"Finetune/Learning Rate": 1.9818858007387753e-05, "Finetune/Loss": 0.9738165140151978, "Finetune/Loss (Raw)": 0.9871447086334229, "Finetune/Step": 461, "Finetune/Step Time": 4.671459659934044} +{"Finetune/Learning Rate": 1.9817675754972846e-05, "Finetune/Loss": 0.9737545847892761, "Finetune/Loss (Raw)": 0.9930305480957031, "Finetune/Step": 462, "Finetune/Step Time": 4.670742152258754} +{"Finetune/Learning Rate": 1.9816489692509654e-05, "Finetune/Loss": 0.9751641750335693, "Finetune/Loss (Raw)": 1.1948190927505493, "Finetune/Step": 463, "Finetune/Step Time": 4.664525413885713} +{"Finetune/Learning Rate": 1.9815299820458458e-05, "Finetune/Loss": 0.9688163995742798, "Finetune/Loss (Raw)": 0.29215988516807556, "Finetune/Step": 464, "Finetune/Step Time": 4.70152641274035} +{"Finetune/Learning Rate": 1.9814106139281033e-05, "Finetune/Loss": 0.96736079454422, "Finetune/Loss (Raw)": 1.0477622747421265, "Finetune/Step": 465, "Finetune/Step Time": 4.709085362032056} +{"Finetune/Learning Rate": 1.9812908649440618e-05, "Finetune/Loss": 0.9669516682624817, "Finetune/Loss (Raw)": 0.9438737034797668, "Finetune/Step": 466, "Finetune/Step Time": 4.71386275999248} +{"Finetune/Learning Rate": 1.981170735140194e-05, "Finetune/Loss": 0.9678870439529419, "Finetune/Loss (Raw)": 1.0884277820587158, "Finetune/Step": 467, "Finetune/Step Time": 4.713784079998732} +{"Finetune/Learning Rate": 1.9810502245631194e-05, "Finetune/Loss": 0.9672961235046387, "Finetune/Loss (Raw)": 0.8396448493003845, "Finetune/Step": 468, "Finetune/Step Time": 4.725884560495615} +{"Finetune/Learning Rate": 1.980929333259606e-05, "Finetune/Loss": 0.962041974067688, "Finetune/Loss (Raw)": 0.35495492815971375, "Finetune/Step": 469, "Finetune/Step Time": 4.759303633123636} +{"Finetune/Learning Rate": 1.9808080612765694e-05, "Finetune/Loss": 0.9611780643463135, "Finetune/Loss (Raw)": 0.9865295886993408, "Finetune/Step": 470, "Finetune/Step Time": 4.7584390211850405} +{"Finetune/Learning Rate": 1.980686408661073e-05, "Finetune/Loss": 0.9610748291015625, "Finetune/Loss (Raw)": 1.0147662162780762, "Finetune/Step": 471, "Finetune/Step Time": 4.774006305262446} +{"Finetune/Learning Rate": 1.9805643754603275e-05, "Finetune/Loss": 0.9614274501800537, "Finetune/Loss (Raw)": 1.1395363807678223, "Finetune/Step": 472, "Finetune/Step Time": 4.775329856202006} +{"Finetune/Learning Rate": 1.9804419617216922e-05, "Finetune/Loss": 0.9598170518875122, "Finetune/Loss (Raw)": 0.9217385053634644, "Finetune/Step": 473, "Finetune/Step Time": 4.773753708228469} +{"Finetune/Learning Rate": 1.9803191674926727e-05, "Finetune/Loss": 0.9591996669769287, "Finetune/Loss (Raw)": 1.055725336074829, "Finetune/Step": 474, "Finetune/Step Time": 4.770042819902301} +{"Finetune/Learning Rate": 1.9801959928209236e-05, "Finetune/Loss": 0.95848548412323, "Finetune/Loss (Raw)": 1.0687717199325562, "Finetune/Step": 475, "Finetune/Step Time": 4.76884837821126} +{"Finetune/Learning Rate": 1.9800724377542463e-05, "Finetune/Loss": 0.9581185579299927, "Finetune/Loss (Raw)": 1.0327173471450806, "Finetune/Step": 476, "Finetune/Step Time": 4.753122495487332} +{"Finetune/Learning Rate": 1.9799485023405904e-05, "Finetune/Loss": 0.9577175974845886, "Finetune/Loss (Raw)": 1.1053466796875, "Finetune/Step": 477, "Finetune/Step Time": 4.758947901427746} +{"Finetune/Learning Rate": 1.9798241866280523e-05, "Finetune/Loss": 0.9575424194335938, "Finetune/Loss (Raw)": 0.9729229211807251, "Finetune/Step": 478, "Finetune/Step Time": 4.735204745084047} +{"Finetune/Learning Rate": 1.979699490664877e-05, "Finetune/Loss": 0.9577834010124207, "Finetune/Loss (Raw)": 1.0330510139465332, "Finetune/Step": 479, "Finetune/Step Time": 4.7197630032896996} +{"Finetune/Learning Rate": 1.9795744144994562e-05, "Finetune/Loss": 0.9584793448448181, "Finetune/Loss (Raw)": 1.0921988487243652, "Finetune/Step": 480, "Finetune/Step Time": 4.711779072880745} +{"Finetune/Learning Rate": 1.97944895818033e-05, "Finetune/Loss": 0.9588642120361328, "Finetune/Loss (Raw)": 1.0281580686569214, "Finetune/Step": 481, "Finetune/Step Time": 4.716670162975788} +{"Finetune/Learning Rate": 1.979323121756185e-05, "Finetune/Loss": 0.9599443674087524, "Finetune/Loss (Raw)": 1.0522853136062622, "Finetune/Step": 482, "Finetune/Step Time": 4.725592451170087} +{"Finetune/Learning Rate": 1.9791969052758563e-05, "Finetune/Loss": 0.9596417546272278, "Finetune/Loss (Raw)": 1.0367344617843628, "Finetune/Step": 483, "Finetune/Step Time": 4.736031098291278} +{"Finetune/Learning Rate": 1.9790703087883258e-05, "Finetune/Loss": 0.9600440263748169, "Finetune/Loss (Raw)": 0.971174418926239, "Finetune/Step": 484, "Finetune/Step Time": 4.740253929048777} +{"Finetune/Learning Rate": 1.9789433323427233e-05, "Finetune/Loss": 0.9594694375991821, "Finetune/Loss (Raw)": 0.8574660420417786, "Finetune/Step": 485, "Finetune/Step Time": 4.7441319692879915} +{"Finetune/Learning Rate": 1.9788159759883258e-05, "Finetune/Loss": 0.9643124341964722, "Finetune/Loss (Raw)": 0.9929150342941284, "Finetune/Step": 486, "Finetune/Step Time": 4.719521913677454} +{"Finetune/Learning Rate": 1.9786882397745573e-05, "Finetune/Loss": 0.9658269882202148, "Finetune/Loss (Raw)": 1.0451991558074951, "Finetune/Step": 487, "Finetune/Step Time": 4.718661965802312} +{"Finetune/Learning Rate": 1.9785601237509907e-05, "Finetune/Loss": 0.9660845994949341, "Finetune/Loss (Raw)": 1.045134425163269, "Finetune/Step": 488, "Finetune/Step Time": 4.716134060174227} +{"Finetune/Learning Rate": 1.978431627967345e-05, "Finetune/Loss": 0.9664956331253052, "Finetune/Loss (Raw)": 0.9494609236717224, "Finetune/Step": 489, "Finetune/Step Time": 4.720558805391192} +{"Finetune/Learning Rate": 1.9783027524734865e-05, "Finetune/Loss": 0.9662120342254639, "Finetune/Loss (Raw)": 0.9415223002433777, "Finetune/Step": 490, "Finetune/Step Time": 4.726735124364495} +{"Finetune/Learning Rate": 1.9781734973194294e-05, "Finetune/Loss": 0.9630747437477112, "Finetune/Loss (Raw)": 0.678432822227478, "Finetune/Step": 491, "Finetune/Step Time": 4.75487994402647} +{"Finetune/Learning Rate": 1.9780438625553352e-05, "Finetune/Loss": 0.9620062112808228, "Finetune/Loss (Raw)": 0.9668583869934082, "Finetune/Step": 492, "Finetune/Step Time": 4.7602841425687075} +{"Finetune/Learning Rate": 1.9779138482315128e-05, "Finetune/Loss": 0.9616798162460327, "Finetune/Loss (Raw)": 1.0799555778503418, "Finetune/Step": 493, "Finetune/Step Time": 4.762781713157892} +{"Finetune/Learning Rate": 1.977783454398418e-05, "Finetune/Loss": 0.9622213244438171, "Finetune/Loss (Raw)": 0.9787920117378235, "Finetune/Step": 494, "Finetune/Step Time": 4.7737836968153715} +{"Finetune/Learning Rate": 1.9776526811066543e-05, "Finetune/Loss": 0.9623243808746338, "Finetune/Loss (Raw)": 1.021429419517517, "Finetune/Step": 495, "Finetune/Step Time": 4.773844465613365} +{"Finetune/Learning Rate": 1.977521528406972e-05, "Finetune/Loss": 0.9624230861663818, "Finetune/Loss (Raw)": 1.0423299074172974, "Finetune/Step": 496, "Finetune/Step Time": 4.781488576903939} +{"Finetune/Learning Rate": 1.9773899963502693e-05, "Finetune/Loss": 0.9622742533683777, "Finetune/Loss (Raw)": 0.9764289259910583, "Finetune/Step": 497, "Finetune/Step Time": 4.777954516932368} +{"Finetune/Learning Rate": 1.9772580849875905e-05, "Finetune/Loss": 0.9613751173019409, "Finetune/Loss (Raw)": 1.0504555702209473, "Finetune/Step": 498, "Finetune/Step Time": 4.78180480375886} +{"Finetune/Learning Rate": 1.9771257943701284e-05, "Finetune/Loss": 0.9592690467834473, "Finetune/Loss (Raw)": 0.8924586772918701, "Finetune/Step": 499, "Finetune/Step Time": 4.784306090325117} +{"Finetune/Learning Rate": 1.9769931245492225e-05, "Finetune/Loss": 0.9588130712509155, "Finetune/Loss (Raw)": 0.9969215989112854, "Finetune/Step": 500, "Finetune/Step Time": 4.794041644781828} +{"Finetune/Learning Rate": 1.976860075576359e-05, "Finetune/Loss": 0.9616115689277649, "Finetune/Loss (Raw)": 0.8032532334327698, "Finetune/Step": 501, "Finetune/Step Time": 4.766765719279647} +{"Finetune/Learning Rate": 1.9767266475031716e-05, "Finetune/Loss": 0.9657809734344482, "Finetune/Loss (Raw)": 0.8991626501083374, "Finetune/Step": 502, "Finetune/Step Time": 4.747144686058164} +{"Finetune/Learning Rate": 1.9765928403814416e-05, "Finetune/Loss": 0.9652304649353027, "Finetune/Loss (Raw)": 1.0385277271270752, "Finetune/Step": 503, "Finetune/Step Time": 4.757778838276863} +{"Finetune/Learning Rate": 1.9764586542630965e-05, "Finetune/Loss": 0.9665128588676453, "Finetune/Loss (Raw)": 1.0674430131912231, "Finetune/Step": 504, "Finetune/Step Time": 4.747497653588653} +{"Finetune/Learning Rate": 1.9763240892002113e-05, "Finetune/Loss": 0.966797947883606, "Finetune/Loss (Raw)": 1.0137337446212769, "Finetune/Step": 505, "Finetune/Step Time": 4.749178262427449} +{"Finetune/Learning Rate": 1.9761891452450082e-05, "Finetune/Loss": 0.9637453556060791, "Finetune/Loss (Raw)": 0.8305879831314087, "Finetune/Step": 506, "Finetune/Step Time": 4.761546457186341} +{"Finetune/Learning Rate": 1.9760538224498563e-05, "Finetune/Loss": 0.9626966714859009, "Finetune/Loss (Raw)": 1.0245858430862427, "Finetune/Step": 507, "Finetune/Step Time": 4.757178710773587} +{"Finetune/Learning Rate": 1.975918120867272e-05, "Finetune/Loss": 0.9623442888259888, "Finetune/Loss (Raw)": 0.9974743127822876, "Finetune/Step": 508, "Finetune/Step Time": 4.757154071703553} +{"Finetune/Learning Rate": 1.975782040549918e-05, "Finetune/Loss": 0.9616513252258301, "Finetune/Loss (Raw)": 0.9739278554916382, "Finetune/Step": 509, "Finetune/Step Time": 4.760241437703371} +{"Finetune/Learning Rate": 1.9756455815506045e-05, "Finetune/Loss": 0.9616258144378662, "Finetune/Loss (Raw)": 0.9298239350318909, "Finetune/Step": 510, "Finetune/Step Time": 4.7286707647144794} +{"Finetune/Learning Rate": 1.9755087439222888e-05, "Finetune/Loss": 0.962952733039856, "Finetune/Loss (Raw)": 1.1058433055877686, "Finetune/Step": 511, "Finetune/Step Time": 4.699719112366438} +{"Finetune/Learning Rate": 1.975371527718075e-05, "Finetune/Loss": 0.967234194278717, "Finetune/Loss (Raw)": 1.0310813188552856, "Finetune/Step": 512, "Finetune/Step Time": 4.6644726768136024} +{"Finetune/Learning Rate": 1.9752339329912137e-05, "Finetune/Loss": 0.9676167964935303, "Finetune/Loss (Raw)": 1.0650407075881958, "Finetune/Step": 513, "Finetune/Step Time": 4.663137353956699} +{"Finetune/Learning Rate": 1.9750959597951026e-05, "Finetune/Loss": 0.9682844877243042, "Finetune/Loss (Raw)": 1.100537657737732, "Finetune/Step": 514, "Finetune/Step Time": 4.676446629688144} +{"Finetune/Learning Rate": 1.9749576081832865e-05, "Finetune/Loss": 0.9645946025848389, "Finetune/Loss (Raw)": 0.40381550788879395, "Finetune/Step": 515, "Finetune/Step Time": 4.7125694286078215} +{"Finetune/Learning Rate": 1.9748188782094576e-05, "Finetune/Loss": 0.9656153321266174, "Finetune/Loss (Raw)": 1.0880342721939087, "Finetune/Step": 516, "Finetune/Step Time": 4.715594317764044} +{"Finetune/Learning Rate": 1.9746797699274535e-05, "Finetune/Loss": 0.9664110541343689, "Finetune/Loss (Raw)": 1.0465915203094482, "Finetune/Step": 517, "Finetune/Step Time": 4.714347714558244} +{"Finetune/Learning Rate": 1.9745402833912598e-05, "Finetune/Loss": 0.9681227207183838, "Finetune/Loss (Raw)": 1.0650320053100586, "Finetune/Step": 518, "Finetune/Step Time": 4.7168693989515305} +{"Finetune/Learning Rate": 1.9744004186550086e-05, "Finetune/Loss": 0.9686368703842163, "Finetune/Loss (Raw)": 1.0702636241912842, "Finetune/Step": 519, "Finetune/Step Time": 4.731814123690128} +{"Finetune/Learning Rate": 1.974260175772978e-05, "Finetune/Loss": 0.9689364433288574, "Finetune/Loss (Raw)": 1.0149140357971191, "Finetune/Step": 520, "Finetune/Step Time": 4.7313321847468615} +{"Finetune/Learning Rate": 1.9741195547995944e-05, "Finetune/Loss": 0.9701578617095947, "Finetune/Loss (Raw)": 1.0116541385650635, "Finetune/Step": 521, "Finetune/Step Time": 4.722692811861634} +{"Finetune/Learning Rate": 1.9739785557894296e-05, "Finetune/Loss": 0.9716302156448364, "Finetune/Loss (Raw)": 1.0011813640594482, "Finetune/Step": 522, "Finetune/Step Time": 4.704418556764722} +{"Finetune/Learning Rate": 1.9738371787972025e-05, "Finetune/Loss": 0.9692936539649963, "Finetune/Loss (Raw)": 0.909726619720459, "Finetune/Step": 523, "Finetune/Step Time": 4.711732637137175} +{"Finetune/Learning Rate": 1.9736954238777793e-05, "Finetune/Loss": 0.9685983061790466, "Finetune/Loss (Raw)": 1.0196599960327148, "Finetune/Step": 524, "Finetune/Step Time": 4.713263867422938} +{"Finetune/Learning Rate": 1.9735532910861713e-05, "Finetune/Loss": 0.9699684977531433, "Finetune/Loss (Raw)": 1.025658369064331, "Finetune/Step": 525, "Finetune/Step Time": 4.696689043194056} +{"Finetune/Learning Rate": 1.9734107804775385e-05, "Finetune/Loss": 0.9680606126785278, "Finetune/Loss (Raw)": 0.9372448921203613, "Finetune/Step": 526, "Finetune/Step Time": 4.709602694958448} +{"Finetune/Learning Rate": 1.9732678921071858e-05, "Finetune/Loss": 0.9683787822723389, "Finetune/Loss (Raw)": 0.9715640544891357, "Finetune/Step": 527, "Finetune/Step Time": 4.703516675159335} +{"Finetune/Learning Rate": 1.9731246260305657e-05, "Finetune/Loss": 0.9691333174705505, "Finetune/Loss (Raw)": 1.047443151473999, "Finetune/Step": 528, "Finetune/Step Time": 4.698693471029401} +{"Finetune/Learning Rate": 1.9729809823032767e-05, "Finetune/Loss": 0.969010055065155, "Finetune/Loss (Raw)": 1.1016898155212402, "Finetune/Step": 529, "Finetune/Step Time": 4.698311580345035} +{"Finetune/Learning Rate": 1.9728369609810648e-05, "Finetune/Loss": 0.9697024822235107, "Finetune/Loss (Raw)": 1.0997092723846436, "Finetune/Step": 530, "Finetune/Step Time": 4.68253867700696} +{"Finetune/Learning Rate": 1.972692562119821e-05, "Finetune/Loss": 0.9711229801177979, "Finetune/Loss (Raw)": 1.1607892513275146, "Finetune/Step": 531, "Finetune/Step Time": 4.679269207641482} +{"Finetune/Learning Rate": 1.9725477857755842e-05, "Finetune/Loss": 0.9703500270843506, "Finetune/Loss (Raw)": 0.9806386828422546, "Finetune/Step": 532, "Finetune/Step Time": 4.676926048472524} +{"Finetune/Learning Rate": 1.9724026320045393e-05, "Finetune/Loss": 0.9720085859298706, "Finetune/Loss (Raw)": 0.9493693709373474, "Finetune/Step": 533, "Finetune/Step Time": 4.673340391367674} +{"Finetune/Learning Rate": 1.9722571008630172e-05, "Finetune/Loss": 0.975569486618042, "Finetune/Loss (Raw)": 0.9373771548271179, "Finetune/Step": 534, "Finetune/Step Time": 4.6431555058807135} +{"Finetune/Learning Rate": 1.9721111924074958e-05, "Finetune/Loss": 0.9795567393302917, "Finetune/Loss (Raw)": 0.9323188066482544, "Finetune/Step": 535, "Finetune/Step Time": 4.617655308917165} +{"Finetune/Learning Rate": 1.9719649066946e-05, "Finetune/Loss": 0.9782302379608154, "Finetune/Loss (Raw)": 0.8218017816543579, "Finetune/Step": 536, "Finetune/Step Time": 4.640076901763678} +{"Finetune/Learning Rate": 1.9718182437810996e-05, "Finetune/Loss": 0.9772170782089233, "Finetune/Loss (Raw)": 1.0663697719573975, "Finetune/Step": 537, "Finetune/Step Time": 4.641240861266851} +{"Finetune/Learning Rate": 1.971671203723912e-05, "Finetune/Loss": 0.9756035804748535, "Finetune/Loss (Raw)": 1.0211973190307617, "Finetune/Step": 538, "Finetune/Step Time": 4.650405513122678} +{"Finetune/Learning Rate": 1.9715237865801007e-05, "Finetune/Loss": 0.9763978719711304, "Finetune/Loss (Raw)": 1.1000083684921265, "Finetune/Step": 539, "Finetune/Step Time": 4.6441178526729345} +{"Finetune/Learning Rate": 1.971375992406875e-05, "Finetune/Loss": 0.9810978174209595, "Finetune/Loss (Raw)": 0.975420355796814, "Finetune/Step": 540, "Finetune/Step Time": 4.598024696111679} +{"Finetune/Learning Rate": 1.9712278212615913e-05, "Finetune/Loss": 0.9801548719406128, "Finetune/Loss (Raw)": 1.0061815977096558, "Finetune/Step": 541, "Finetune/Step Time": 4.605398550629616} +{"Finetune/Learning Rate": 1.9710792732017517e-05, "Finetune/Loss": 0.9807659983634949, "Finetune/Loss (Raw)": 1.0241506099700928, "Finetune/Step": 542, "Finetune/Step Time": 4.6048815082758665} +{"Finetune/Learning Rate": 1.9709303482850047e-05, "Finetune/Loss": 0.9812909960746765, "Finetune/Loss (Raw)": 1.127393126487732, "Finetune/Step": 543, "Finetune/Step Time": 4.604817667976022} +{"Finetune/Learning Rate": 1.9707810465691456e-05, "Finetune/Loss": 0.9809201955795288, "Finetune/Loss (Raw)": 1.079061508178711, "Finetune/Step": 544, "Finetune/Step Time": 4.605611048638821} +{"Finetune/Learning Rate": 1.970631368112115e-05, "Finetune/Loss": 0.9804342985153198, "Finetune/Loss (Raw)": 0.9904230833053589, "Finetune/Step": 545, "Finetune/Step Time": 4.603361126035452} +{"Finetune/Learning Rate": 1.970481312972001e-05, "Finetune/Loss": 0.9791917204856873, "Finetune/Loss (Raw)": 0.9434646368026733, "Finetune/Step": 546, "Finetune/Step Time": 4.6048538479954} +{"Finetune/Learning Rate": 1.9703308812070358e-05, "Finetune/Loss": 0.9799706935882568, "Finetune/Loss (Raw)": 1.075002670288086, "Finetune/Step": 547, "Finetune/Step Time": 4.595526661723852} +{"Finetune/Learning Rate": 1.9701800728756e-05, "Finetune/Loss": 0.9846267700195312, "Finetune/Loss (Raw)": 0.9604498744010925, "Finetune/Step": 548, "Finetune/Step Time": 4.5730988550931215} +{"Finetune/Learning Rate": 1.970028888036219e-05, "Finetune/Loss": 0.9831340312957764, "Finetune/Loss (Raw)": 0.8472448587417603, "Finetune/Step": 549, "Finetune/Step Time": 4.58184565231204} +{"Finetune/Learning Rate": 1.9698773267475652e-05, "Finetune/Loss": 0.9830753803253174, "Finetune/Loss (Raw)": 1.0008885860443115, "Finetune/Step": 550, "Finetune/Step Time": 4.564013222232461} +{"Finetune/Learning Rate": 1.9697253890684556e-05, "Finetune/Loss": 0.9818559288978577, "Finetune/Loss (Raw)": 0.9769056439399719, "Finetune/Step": 551, "Finetune/Step Time": 4.5649653766304255} +{"Finetune/Learning Rate": 1.969573075057855e-05, "Finetune/Loss": 0.9822736382484436, "Finetune/Loss (Raw)": 1.0970754623413086, "Finetune/Step": 552, "Finetune/Step Time": 4.560520131140947} +{"Finetune/Learning Rate": 1.9694203847748734e-05, "Finetune/Loss": 0.9823274612426758, "Finetune/Loss (Raw)": 0.7657548785209656, "Finetune/Step": 553, "Finetune/Step Time": 4.524073213338852} +{"Finetune/Learning Rate": 1.969267318278767e-05, "Finetune/Loss": 0.9833172559738159, "Finetune/Loss (Raw)": 1.146824598312378, "Finetune/Step": 554, "Finetune/Step Time": 4.522012136876583} +{"Finetune/Learning Rate": 1.969113875628937e-05, "Finetune/Loss": 0.9830412268638611, "Finetune/Loss (Raw)": 0.9492829442024231, "Finetune/Step": 555, "Finetune/Step Time": 4.53623959608376} +{"Finetune/Learning Rate": 1.9689600568849328e-05, "Finetune/Loss": 0.982153058052063, "Finetune/Loss (Raw)": 1.0125561952590942, "Finetune/Step": 556, "Finetune/Step Time": 4.548340750858188} +{"Finetune/Learning Rate": 1.968805862106448e-05, "Finetune/Loss": 0.9812151789665222, "Finetune/Loss (Raw)": 0.959438681602478, "Finetune/Step": 557, "Finetune/Step Time": 4.5448948964476585} +{"Finetune/Learning Rate": 1.968651291353322e-05, "Finetune/Loss": 0.9761313199996948, "Finetune/Loss (Raw)": 0.25260719656944275, "Finetune/Step": 558, "Finetune/Step Time": 4.575861854478717} +{"Finetune/Learning Rate": 1.968496344685541e-05, "Finetune/Loss": 0.9752858877182007, "Finetune/Loss (Raw)": 0.9812548160552979, "Finetune/Step": 559, "Finetune/Step Time": 4.569635331630707} +{"Finetune/Learning Rate": 1.9683410221632374e-05, "Finetune/Loss": 0.9745334982872009, "Finetune/Loss (Raw)": 0.8496654033660889, "Finetune/Step": 560, "Finetune/Step Time": 4.564856702461839} +{"Finetune/Learning Rate": 1.9681853238466883e-05, "Finetune/Loss": 0.9742834568023682, "Finetune/Loss (Raw)": 0.8899539113044739, "Finetune/Step": 561, "Finetune/Step Time": 4.565893270075321} +{"Finetune/Learning Rate": 1.9680292497963172e-05, "Finetune/Loss": 0.9739622473716736, "Finetune/Loss (Raw)": 0.9442514181137085, "Finetune/Step": 562, "Finetune/Step Time": 4.563440980389714} +{"Finetune/Learning Rate": 1.9678728000726936e-05, "Finetune/Loss": 0.9756118059158325, "Finetune/Loss (Raw)": 1.0507655143737793, "Finetune/Step": 563, "Finetune/Step Time": 4.5697770938277245} +{"Finetune/Learning Rate": 1.967715974736532e-05, "Finetune/Loss": 0.9764399528503418, "Finetune/Loss (Raw)": 1.0353176593780518, "Finetune/Step": 564, "Finetune/Step Time": 4.579470170661807} +{"Finetune/Learning Rate": 1.9675587738486935e-05, "Finetune/Loss": 0.9758459329605103, "Finetune/Loss (Raw)": 0.9540099501609802, "Finetune/Step": 565, "Finetune/Step Time": 4.583596037700772} +{"Finetune/Learning Rate": 1.9674011974701855e-05, "Finetune/Loss": 0.9767451286315918, "Finetune/Loss (Raw)": 1.0788335800170898, "Finetune/Step": 566, "Finetune/Step Time": 4.584771130234003} +{"Finetune/Learning Rate": 1.9672432456621597e-05, "Finetune/Loss": 0.9769234657287598, "Finetune/Loss (Raw)": 1.0016205310821533, "Finetune/Step": 567, "Finetune/Step Time": 4.5873256009072065} +{"Finetune/Learning Rate": 1.9670849184859136e-05, "Finetune/Loss": 0.9707246422767639, "Finetune/Loss (Raw)": 0.28738269209861755, "Finetune/Step": 568, "Finetune/Step Time": 4.61481318436563} +{"Finetune/Learning Rate": 1.9669262160028916e-05, "Finetune/Loss": 0.9697853326797485, "Finetune/Loss (Raw)": 0.9313806891441345, "Finetune/Step": 569, "Finetune/Step Time": 4.620792116969824} +{"Finetune/Learning Rate": 1.966767138274683e-05, "Finetune/Loss": 0.9742923378944397, "Finetune/Loss (Raw)": 1.0180974006652832, "Finetune/Step": 570, "Finetune/Step Time": 4.587513307109475} +{"Finetune/Learning Rate": 1.966607685363022e-05, "Finetune/Loss": 0.9733697175979614, "Finetune/Loss (Raw)": 0.954170823097229, "Finetune/Step": 571, "Finetune/Step Time": 4.593107841908932} +{"Finetune/Learning Rate": 1.96644785732979e-05, "Finetune/Loss": 0.9725865721702576, "Finetune/Loss (Raw)": 1.0392943620681763, "Finetune/Step": 572, "Finetune/Step Time": 4.598005399107933} +{"Finetune/Learning Rate": 1.966287654237013e-05, "Finetune/Loss": 0.9730104207992554, "Finetune/Loss (Raw)": 1.0721030235290527, "Finetune/Step": 573, "Finetune/Step Time": 4.602341836318374} +{"Finetune/Learning Rate": 1.9661270761468625e-05, "Finetune/Loss": 0.972830057144165, "Finetune/Loss (Raw)": 1.09695303440094, "Finetune/Step": 574, "Finetune/Step Time": 4.594409948214889} +{"Finetune/Learning Rate": 1.965966123121656e-05, "Finetune/Loss": 0.9736294746398926, "Finetune/Loss (Raw)": 1.1387981176376343, "Finetune/Step": 575, "Finetune/Step Time": 4.613015104085207} +{"Finetune/Learning Rate": 1.965804795223856e-05, "Finetune/Loss": 0.9738788604736328, "Finetune/Loss (Raw)": 0.9899585247039795, "Finetune/Step": 576, "Finetune/Step Time": 4.61411808617413} +{"Finetune/Learning Rate": 1.9656430925160704e-05, "Finetune/Loss": 0.9727368354797363, "Finetune/Loss (Raw)": 0.9944337606430054, "Finetune/Step": 577, "Finetune/Step Time": 4.598167784512043} +{"Finetune/Learning Rate": 1.9654810150610536e-05, "Finetune/Loss": 0.9727858304977417, "Finetune/Loss (Raw)": 0.9553295969963074, "Finetune/Step": 578, "Finetune/Step Time": 4.594380855560303} +{"Finetune/Learning Rate": 1.9653185629217038e-05, "Finetune/Loss": 0.97183758020401, "Finetune/Loss (Raw)": 0.8560634255409241, "Finetune/Step": 579, "Finetune/Step Time": 4.607322700321674} +{"Finetune/Learning Rate": 1.9651557361610663e-05, "Finetune/Loss": 0.9721304178237915, "Finetune/Loss (Raw)": 1.0420725345611572, "Finetune/Step": 580, "Finetune/Step Time": 4.609296061098576} +{"Finetune/Learning Rate": 1.9649925348423308e-05, "Finetune/Loss": 0.9714644551277161, "Finetune/Loss (Raw)": 1.1388956308364868, "Finetune/Step": 581, "Finetune/Step Time": 4.6188410352915525} +{"Finetune/Learning Rate": 1.964828959028832e-05, "Finetune/Loss": 0.9718928337097168, "Finetune/Loss (Raw)": 1.0480667352676392, "Finetune/Step": 582, "Finetune/Step Time": 4.619987018406391} +{"Finetune/Learning Rate": 1.964665008784051e-05, "Finetune/Loss": 0.9704049825668335, "Finetune/Loss (Raw)": 0.9010768532752991, "Finetune/Step": 583, "Finetune/Step Time": 4.626225788146257} +{"Finetune/Learning Rate": 1.9645006841716137e-05, "Finetune/Loss": 0.9761609435081482, "Finetune/Loss (Raw)": 1.0821950435638428, "Finetune/Step": 584, "Finetune/Step Time": 4.599262870848179} +{"Finetune/Learning Rate": 1.9643359852552912e-05, "Finetune/Loss": 0.9774355292320251, "Finetune/Loss (Raw)": 1.0827312469482422, "Finetune/Step": 585, "Finetune/Step Time": 4.600904483348131} +{"Finetune/Learning Rate": 1.9641709120989996e-05, "Finetune/Loss": 0.9706098437309265, "Finetune/Loss (Raw)": 0.3465914726257324, "Finetune/Step": 586, "Finetune/Step Time": 4.639045557007194} +{"Finetune/Learning Rate": 1.9640054647668015e-05, "Finetune/Loss": 0.9720854759216309, "Finetune/Loss (Raw)": 1.1905966997146606, "Finetune/Step": 587, "Finetune/Step Time": 4.630680341273546} +{"Finetune/Learning Rate": 1.9638396433229025e-05, "Finetune/Loss": 0.9732354879379272, "Finetune/Loss (Raw)": 0.9978324174880981, "Finetune/Step": 588, "Finetune/Step Time": 4.639105034992099} +{"Finetune/Learning Rate": 1.963673447831656e-05, "Finetune/Loss": 0.9740153551101685, "Finetune/Loss (Raw)": 1.0869742631912231, "Finetune/Step": 589, "Finetune/Step Time": 4.634305778890848} +{"Finetune/Learning Rate": 1.9635068783575577e-05, "Finetune/Loss": 0.9747902154922485, "Finetune/Loss (Raw)": 1.0922034978866577, "Finetune/Step": 590, "Finetune/Step Time": 4.63505051471293} +{"Finetune/Learning Rate": 1.9633399349652512e-05, "Finetune/Loss": 0.972874641418457, "Finetune/Loss (Raw)": 0.9496417045593262, "Finetune/Step": 591, "Finetune/Step Time": 4.637366196140647} +{"Finetune/Learning Rate": 1.9631726177195235e-05, "Finetune/Loss": 0.9781873822212219, "Finetune/Loss (Raw)": 0.972183883190155, "Finetune/Step": 592, "Finetune/Step Time": 4.598077615723014} +{"Finetune/Learning Rate": 1.9630049266853076e-05, "Finetune/Loss": 0.9778211116790771, "Finetune/Loss (Raw)": 1.0008771419525146, "Finetune/Step": 593, "Finetune/Step Time": 4.612350290641189} +{"Finetune/Learning Rate": 1.96283686192768e-05, "Finetune/Loss": 0.9783557653427124, "Finetune/Loss (Raw)": 1.0123097896575928, "Finetune/Step": 594, "Finetune/Step Time": 4.60011026263237} +{"Finetune/Learning Rate": 1.9626684235118646e-05, "Finetune/Loss": 0.9777733087539673, "Finetune/Loss (Raw)": 1.0138840675354004, "Finetune/Step": 595, "Finetune/Step Time": 4.589985219761729} +{"Finetune/Learning Rate": 1.9624996115032287e-05, "Finetune/Loss": 0.9785443544387817, "Finetune/Loss (Raw)": 0.9383267164230347, "Finetune/Step": 596, "Finetune/Step Time": 4.57251944206655} +{"Finetune/Learning Rate": 1.9623304259672844e-05, "Finetune/Loss": 0.9826416969299316, "Finetune/Loss (Raw)": 0.8794192671775818, "Finetune/Step": 597, "Finetune/Step Time": 4.537764418870211} +{"Finetune/Learning Rate": 1.96216086696969e-05, "Finetune/Loss": 0.9827806949615479, "Finetune/Loss (Raw)": 1.0043190717697144, "Finetune/Step": 598, "Finetune/Step Time": 4.560274686664343} +{"Finetune/Learning Rate": 1.9619909345762474e-05, "Finetune/Loss": 0.982831597328186, "Finetune/Loss (Raw)": 1.0212862491607666, "Finetune/Step": 599, "Finetune/Step Time": 4.541893050074577} +{"Finetune/Learning Rate": 1.961820628852905e-05, "Finetune/Loss": 0.9825482368469238, "Finetune/Loss (Raw)": 1.1032646894454956, "Finetune/Step": 600, "Finetune/Step Time": 4.546209499239922} +{"Finetune/Learning Rate": 1.9616499498657537e-05, "Finetune/Loss": 0.9829741716384888, "Finetune/Loss (Raw)": 0.976256787776947, "Finetune/Step": 601, "Finetune/Step Time": 4.5694773104041815} +{"Finetune/Learning Rate": 1.961478897681032e-05, "Finetune/Loss": 0.9831066131591797, "Finetune/Loss (Raw)": 1.0726696252822876, "Finetune/Step": 602, "Finetune/Step Time": 4.5624112617224455} +{"Finetune/Learning Rate": 1.9613074723651214e-05, "Finetune/Loss": 0.9820656776428223, "Finetune/Loss (Raw)": 0.9355400204658508, "Finetune/Step": 603, "Finetune/Step Time": 4.5674969255924225} +{"Finetune/Learning Rate": 1.9611356739845484e-05, "Finetune/Loss": 0.9823484420776367, "Finetune/Loss (Raw)": 1.0689107179641724, "Finetune/Step": 604, "Finetune/Step Time": 4.566837469115853} +{"Finetune/Learning Rate": 1.9609635026059855e-05, "Finetune/Loss": 0.9811028242111206, "Finetune/Loss (Raw)": 0.9459115266799927, "Finetune/Step": 605, "Finetune/Step Time": 4.561749022454023} +{"Finetune/Learning Rate": 1.9607909582962478e-05, "Finetune/Loss": 0.9807502627372742, "Finetune/Loss (Raw)": 0.9277896285057068, "Finetune/Step": 606, "Finetune/Step Time": 4.579211749136448} +{"Finetune/Learning Rate": 1.9606180411222974e-05, "Finetune/Loss": 0.9790964126586914, "Finetune/Loss (Raw)": 0.8213624954223633, "Finetune/Step": 607, "Finetune/Step Time": 4.60919588804245} +{"Finetune/Learning Rate": 1.9604447511512396e-05, "Finetune/Loss": 0.9791184663772583, "Finetune/Loss (Raw)": 1.0950181484222412, "Finetune/Step": 608, "Finetune/Step Time": 4.60635332390666} +{"Finetune/Learning Rate": 1.960271088450325e-05, "Finetune/Loss": 0.9782552719116211, "Finetune/Loss (Raw)": 0.9176665544509888, "Finetune/Step": 609, "Finetune/Step Time": 4.601177891716361} +{"Finetune/Learning Rate": 1.960097053086949e-05, "Finetune/Loss": 0.9776159524917603, "Finetune/Loss (Raw)": 0.9704551100730896, "Finetune/Step": 610, "Finetune/Step Time": 4.591771524399519} +{"Finetune/Learning Rate": 1.9599226451286504e-05, "Finetune/Loss": 0.9776183366775513, "Finetune/Loss (Raw)": 1.0370419025421143, "Finetune/Step": 611, "Finetune/Step Time": 4.569913158193231} +{"Finetune/Learning Rate": 1.9597478646431147e-05, "Finetune/Loss": 0.9794878959655762, "Finetune/Loss (Raw)": 1.2104803323745728, "Finetune/Step": 612, "Finetune/Step Time": 4.5670484732836485} +{"Finetune/Learning Rate": 1.95957271169817e-05, "Finetune/Loss": 0.9798524975776672, "Finetune/Loss (Raw)": 0.9041332006454468, "Finetune/Step": 613, "Finetune/Step Time": 4.559436036273837} +{"Finetune/Learning Rate": 1.95939718636179e-05, "Finetune/Loss": 0.9798139333724976, "Finetune/Loss (Raw)": 0.9879738092422485, "Finetune/Step": 614, "Finetune/Step Time": 4.545023497194052} +{"Finetune/Learning Rate": 1.9592212887020923e-05, "Finetune/Loss": 0.9797375798225403, "Finetune/Loss (Raw)": 1.0354300737380981, "Finetune/Step": 615, "Finetune/Step Time": 4.551876548677683} +{"Finetune/Learning Rate": 1.9590450187873403e-05, "Finetune/Loss": 0.9794000387191772, "Finetune/Loss (Raw)": 1.0019292831420898, "Finetune/Step": 616, "Finetune/Step Time": 4.553787715733051} +{"Finetune/Learning Rate": 1.95886837668594e-05, "Finetune/Loss": 0.9798035621643066, "Finetune/Loss (Raw)": 1.0011051893234253, "Finetune/Step": 617, "Finetune/Step Time": 4.545811116695404} +{"Finetune/Learning Rate": 1.958691362466443e-05, "Finetune/Loss": 0.9806316494941711, "Finetune/Loss (Raw)": 1.0475163459777832, "Finetune/Step": 618, "Finetune/Step Time": 4.539866933599114} +{"Finetune/Learning Rate": 1.958513976197545e-05, "Finetune/Loss": 0.9825612306594849, "Finetune/Loss (Raw)": 0.9254215955734253, "Finetune/Step": 619, "Finetune/Step Time": 4.5347979832440615} +{"Finetune/Learning Rate": 1.9583362179480866e-05, "Finetune/Loss": 0.9831002354621887, "Finetune/Loss (Raw)": 1.0358541011810303, "Finetune/Step": 620, "Finetune/Step Time": 4.525668490678072} +{"Finetune/Learning Rate": 1.9581580877870514e-05, "Finetune/Loss": 0.9826486110687256, "Finetune/Loss (Raw)": 1.0221515893936157, "Finetune/Step": 621, "Finetune/Step Time": 4.54456964507699} +{"Finetune/Learning Rate": 1.9579795857835692e-05, "Finetune/Loss": 0.9825767278671265, "Finetune/Loss (Raw)": 0.9695882797241211, "Finetune/Step": 622, "Finetune/Step Time": 4.532162105664611} +{"Finetune/Learning Rate": 1.9578007120069123e-05, "Finetune/Loss": 0.9829517602920532, "Finetune/Loss (Raw)": 1.0694276094436646, "Finetune/Step": 623, "Finetune/Step Time": 4.533723693341017} +{"Finetune/Learning Rate": 1.957621466526499e-05, "Finetune/Loss": 0.9826014041900635, "Finetune/Loss (Raw)": 0.9974939227104187, "Finetune/Step": 624, "Finetune/Step Time": 4.519918091595173} +{"Finetune/Learning Rate": 1.9574418494118905e-05, "Finetune/Loss": 0.9832277894020081, "Finetune/Loss (Raw)": 1.0566051006317139, "Finetune/Step": 625, "Finetune/Step Time": 4.520969791337848} +{"Finetune/Learning Rate": 1.9572618607327928e-05, "Finetune/Loss": 0.9833536744117737, "Finetune/Loss (Raw)": 1.066564679145813, "Finetune/Step": 626, "Finetune/Step Time": 4.512919634580612} +{"Finetune/Learning Rate": 1.9570815005590556e-05, "Finetune/Loss": 0.984013557434082, "Finetune/Loss (Raw)": 0.9769264459609985, "Finetune/Step": 627, "Finetune/Step Time": 4.514840837568045} +{"Finetune/Learning Rate": 1.9569007689606735e-05, "Finetune/Loss": 0.9839674234390259, "Finetune/Loss (Raw)": 0.991017758846283, "Finetune/Step": 628, "Finetune/Step Time": 4.4996095933020115} +{"Finetune/Learning Rate": 1.9567196660077848e-05, "Finetune/Loss": 0.9857389330863953, "Finetune/Loss (Raw)": 1.0300068855285645, "Finetune/Step": 629, "Finetune/Step Time": 4.489998463541269} +{"Finetune/Learning Rate": 1.9565381917706726e-05, "Finetune/Loss": 0.9876063466072083, "Finetune/Loss (Raw)": 1.1381886005401611, "Finetune/Step": 630, "Finetune/Step Time": 4.477398520335555} +{"Finetune/Learning Rate": 1.9563563463197625e-05, "Finetune/Loss": 0.9866282343864441, "Finetune/Loss (Raw)": 0.9133291840553284, "Finetune/Step": 631, "Finetune/Step Time": 4.468782680109143} +{"Finetune/Learning Rate": 1.9561741297256257e-05, "Finetune/Loss": 0.9874423742294312, "Finetune/Loss (Raw)": 1.1716499328613281, "Finetune/Step": 632, "Finetune/Step Time": 4.467393059283495} +{"Finetune/Learning Rate": 1.955991542058977e-05, "Finetune/Loss": 0.9894452095031738, "Finetune/Loss (Raw)": 1.2701045274734497, "Finetune/Step": 633, "Finetune/Step Time": 4.4624236430972815} +{"Finetune/Learning Rate": 1.9558085833906753e-05, "Finetune/Loss": 0.9904053211212158, "Finetune/Loss (Raw)": 0.9534739851951599, "Finetune/Step": 634, "Finetune/Step Time": 4.458615595474839} +{"Finetune/Learning Rate": 1.9556252537917228e-05, "Finetune/Loss": 0.9902727603912354, "Finetune/Loss (Raw)": 1.007621169090271, "Finetune/Step": 635, "Finetune/Step Time": 4.462983638048172} +{"Finetune/Learning Rate": 1.955441553333266e-05, "Finetune/Loss": 0.9909147024154663, "Finetune/Loss (Raw)": 1.0796501636505127, "Finetune/Step": 636, "Finetune/Step Time": 4.487686859443784} +{"Finetune/Learning Rate": 1.955257482086596e-05, "Finetune/Loss": 0.9919179677963257, "Finetune/Loss (Raw)": 1.1023458242416382, "Finetune/Step": 637, "Finetune/Step Time": 4.477209052070975} +{"Finetune/Learning Rate": 1.955073040123147e-05, "Finetune/Loss": 0.9932750463485718, "Finetune/Loss (Raw)": 1.1035308837890625, "Finetune/Step": 638, "Finetune/Step Time": 4.4707007966935635} +{"Finetune/Learning Rate": 1.9548882275144974e-05, "Finetune/Loss": 0.9926205277442932, "Finetune/Loss (Raw)": 1.022058367729187, "Finetune/Step": 639, "Finetune/Step Time": 4.467272076755762} +{"Finetune/Learning Rate": 1.9547030443323695e-05, "Finetune/Loss": 0.9924411773681641, "Finetune/Loss (Raw)": 1.0081210136413574, "Finetune/Step": 640, "Finetune/Step Time": 4.467407839372754} +{"Finetune/Learning Rate": 1.9545174906486287e-05, "Finetune/Loss": 0.9927111864089966, "Finetune/Loss (Raw)": 1.0996148586273193, "Finetune/Step": 641, "Finetune/Step Time": 4.479958133772016} +{"Finetune/Learning Rate": 1.9543315665352855e-05, "Finetune/Loss": 0.9867966771125793, "Finetune/Loss (Raw)": 0.34347060322761536, "Finetune/Step": 642, "Finetune/Step Time": 4.4958548452705145} +{"Finetune/Learning Rate": 1.954145272064493e-05, "Finetune/Loss": 0.9914662837982178, "Finetune/Loss (Raw)": 1.0015300512313843, "Finetune/Step": 643, "Finetune/Step Time": 4.458262974396348} +{"Finetune/Learning Rate": 1.9539586073085482e-05, "Finetune/Loss": 0.9911355376243591, "Finetune/Loss (Raw)": 1.0456931591033936, "Finetune/Step": 644, "Finetune/Step Time": 4.4575175400823355} +{"Finetune/Learning Rate": 1.9537715723398926e-05, "Finetune/Loss": 0.991430938243866, "Finetune/Loss (Raw)": 1.0844076871871948, "Finetune/Step": 645, "Finetune/Step Time": 4.456216195598245} +{"Finetune/Learning Rate": 1.9535841672311106e-05, "Finetune/Loss": 0.9911201000213623, "Finetune/Loss (Raw)": 1.025241494178772, "Finetune/Step": 646, "Finetune/Step Time": 4.466513141989708} +{"Finetune/Learning Rate": 1.9533963920549307e-05, "Finetune/Loss": 0.99018794298172, "Finetune/Loss (Raw)": 0.9509456157684326, "Finetune/Step": 647, "Finetune/Step Time": 4.466742167249322} +{"Finetune/Learning Rate": 1.9532082468842243e-05, "Finetune/Loss": 0.99034583568573, "Finetune/Loss (Raw)": 1.0351219177246094, "Finetune/Step": 648, "Finetune/Step Time": 4.471145210787654} +{"Finetune/Learning Rate": 1.953019731792007e-05, "Finetune/Loss": 0.990512490272522, "Finetune/Loss (Raw)": 1.0329877138137817, "Finetune/Step": 649, "Finetune/Step Time": 4.466399015858769} +{"Finetune/Learning Rate": 1.9528308468514383e-05, "Finetune/Loss": 0.9904140830039978, "Finetune/Loss (Raw)": 0.9885858297348022, "Finetune/Step": 650, "Finetune/Step Time": 4.4798890091478825} +{"Finetune/Learning Rate": 1.9526415921358202e-05, "Finetune/Loss": 0.9913803339004517, "Finetune/Loss (Raw)": 1.0334045886993408, "Finetune/Step": 651, "Finetune/Step Time": 4.487895095720887} +{"Finetune/Learning Rate": 1.952451967718599e-05, "Finetune/Loss": 0.9912559986114502, "Finetune/Loss (Raw)": 1.003747582435608, "Finetune/Step": 652, "Finetune/Step Time": 4.492000067606568} +{"Finetune/Learning Rate": 1.952261973673364e-05, "Finetune/Loss": 0.9904301762580872, "Finetune/Loss (Raw)": 0.9199613332748413, "Finetune/Step": 653, "Finetune/Step Time": 4.494188556447625} +{"Finetune/Learning Rate": 1.9520716100738487e-05, "Finetune/Loss": 0.9925559163093567, "Finetune/Loss (Raw)": 1.2093379497528076, "Finetune/Step": 654, "Finetune/Step Time": 4.485639240592718} +{"Finetune/Learning Rate": 1.9518808769939287e-05, "Finetune/Loss": 0.9927152395248413, "Finetune/Loss (Raw)": 0.9919500350952148, "Finetune/Step": 655, "Finetune/Step Time": 4.495390998199582} +{"Finetune/Learning Rate": 1.9516897745076243e-05, "Finetune/Loss": 0.9921188354492188, "Finetune/Loss (Raw)": 0.971106767654419, "Finetune/Step": 656, "Finetune/Step Time": 4.498716663569212} +{"Finetune/Learning Rate": 1.9514983026890988e-05, "Finetune/Loss": 0.9910215735435486, "Finetune/Loss (Raw)": 0.9612414836883545, "Finetune/Step": 657, "Finetune/Step Time": 4.518990581855178} +{"Finetune/Learning Rate": 1.9513064616126588e-05, "Finetune/Loss": 0.9901426434516907, "Finetune/Loss (Raw)": 0.9872032403945923, "Finetune/Step": 658, "Finetune/Step Time": 4.52054657228291} +{"Finetune/Learning Rate": 1.951114251352753e-05, "Finetune/Loss": 0.9873878955841064, "Finetune/Loss (Raw)": 0.8081808686256409, "Finetune/Step": 659, "Finetune/Step Time": 4.542292587459087} +{"Finetune/Learning Rate": 1.950921671983976e-05, "Finetune/Loss": 0.9885700941085815, "Finetune/Loss (Raw)": 1.1319645643234253, "Finetune/Step": 660, "Finetune/Step Time": 4.547164490446448} +{"Finetune/Learning Rate": 1.9507287235810625e-05, "Finetune/Loss": 0.9890748262405396, "Finetune/Loss (Raw)": 1.0139727592468262, "Finetune/Step": 661, "Finetune/Step Time": 4.539553904905915} +{"Finetune/Learning Rate": 1.9505354062188934e-05, "Finetune/Loss": 0.9900268316268921, "Finetune/Loss (Raw)": 1.0592358112335205, "Finetune/Step": 662, "Finetune/Step Time": 4.52861519344151} +{"Finetune/Learning Rate": 1.9503417199724904e-05, "Finetune/Loss": 0.9914512038230896, "Finetune/Loss (Raw)": 1.1146280765533447, "Finetune/Step": 663, "Finetune/Step Time": 4.517187975347042} +{"Finetune/Learning Rate": 1.95014766491702e-05, "Finetune/Loss": 0.993037760257721, "Finetune/Loss (Raw)": 1.024886131286621, "Finetune/Step": 664, "Finetune/Step Time": 4.504788666963577} +{"Finetune/Learning Rate": 1.9499532411277907e-05, "Finetune/Loss": 0.9937927722930908, "Finetune/Loss (Raw)": 1.1630100011825562, "Finetune/Step": 665, "Finetune/Step Time": 4.513101898133755} +{"Finetune/Learning Rate": 1.9497584486802548e-05, "Finetune/Loss": 0.9949824810028076, "Finetune/Loss (Raw)": 1.173478364944458, "Finetune/Step": 666, "Finetune/Step Time": 4.505217786878347} +{"Finetune/Learning Rate": 1.9495632876500076e-05, "Finetune/Loss": 0.9951825141906738, "Finetune/Loss (Raw)": 1.1256183385849, "Finetune/Step": 667, "Finetune/Step Time": 4.518965855240822} +{"Finetune/Learning Rate": 1.949367758112787e-05, "Finetune/Loss": 0.9943182468414307, "Finetune/Loss (Raw)": 0.8647872805595398, "Finetune/Step": 668, "Finetune/Step Time": 4.524415729567409} +{"Finetune/Learning Rate": 1.949171860144474e-05, "Finetune/Loss": 0.9945637583732605, "Finetune/Loss (Raw)": 1.0376163721084595, "Finetune/Step": 669, "Finetune/Step Time": 4.518472891300917} +{"Finetune/Learning Rate": 1.9489755938210936e-05, "Finetune/Loss": 0.9937392473220825, "Finetune/Loss (Raw)": 0.9186127185821533, "Finetune/Step": 670, "Finetune/Step Time": 4.5206726975739} +{"Finetune/Learning Rate": 1.9487789592188123e-05, "Finetune/Loss": 0.9926857948303223, "Finetune/Loss (Raw)": 0.9925450682640076, "Finetune/Step": 671, "Finetune/Step Time": 4.519433714449406} +{"Finetune/Learning Rate": 1.9485819564139404e-05, "Finetune/Loss": 0.9925628900527954, "Finetune/Loss (Raw)": 1.0633329153060913, "Finetune/Step": 672, "Finetune/Step Time": 4.516327127814293} +{"Finetune/Learning Rate": 1.9483845854829304e-05, "Finetune/Loss": 0.9935882091522217, "Finetune/Loss (Raw)": 1.1216635704040527, "Finetune/Step": 673, "Finetune/Step Time": 4.517190046608448} +{"Finetune/Learning Rate": 1.9481868465023788e-05, "Finetune/Loss": 0.9949844479560852, "Finetune/Loss (Raw)": 1.1221868991851807, "Finetune/Step": 674, "Finetune/Step Time": 4.517476413398981} +{"Finetune/Learning Rate": 1.9479887395490234e-05, "Finetune/Loss": 0.9949333667755127, "Finetune/Loss (Raw)": 1.0684540271759033, "Finetune/Step": 675, "Finetune/Step Time": 4.5195666663348675} +{"Finetune/Learning Rate": 1.9477902646997464e-05, "Finetune/Loss": 0.9955332279205322, "Finetune/Loss (Raw)": 1.0372315645217896, "Finetune/Step": 676, "Finetune/Step Time": 4.513991821557283} +{"Finetune/Learning Rate": 1.947591422031572e-05, "Finetune/Loss": 0.9961456060409546, "Finetune/Loss (Raw)": 0.9256330728530884, "Finetune/Step": 677, "Finetune/Step Time": 4.54658298753202} +{"Finetune/Learning Rate": 1.9473922116216664e-05, "Finetune/Loss": 0.997199535369873, "Finetune/Loss (Raw)": 1.1357899904251099, "Finetune/Step": 678, "Finetune/Step Time": 4.558606743812561} +{"Finetune/Learning Rate": 1.9471926335473396e-05, "Finetune/Loss": 0.9985238313674927, "Finetune/Loss (Raw)": 1.1464171409606934, "Finetune/Step": 679, "Finetune/Step Time": 4.563803400844336} +{"Finetune/Learning Rate": 1.9469926878860447e-05, "Finetune/Loss": 0.9975119233131409, "Finetune/Loss (Raw)": 0.9675543308258057, "Finetune/Step": 680, "Finetune/Step Time": 4.562269555404782} +{"Finetune/Learning Rate": 1.9467923747153756e-05, "Finetune/Loss": 0.9999305605888367, "Finetune/Loss (Raw)": 1.0753364562988281, "Finetune/Step": 681, "Finetune/Step Time": 4.560752939432859} +{"Finetune/Learning Rate": 1.9465916941130705e-05, "Finetune/Loss": 0.9994468688964844, "Finetune/Loss (Raw)": 1.084917426109314, "Finetune/Step": 682, "Finetune/Step Time": 4.5577327366918325} +{"Finetune/Learning Rate": 1.9463906461570098e-05, "Finetune/Loss": 1.0007373094558716, "Finetune/Loss (Raw)": 1.114456295967102, "Finetune/Step": 683, "Finetune/Step Time": 4.5387098509818316} +{"Finetune/Learning Rate": 1.9461892309252158e-05, "Finetune/Loss": 1.000335693359375, "Finetune/Loss (Raw)": 0.9611591100692749, "Finetune/Step": 684, "Finetune/Step Time": 4.527772478759289} +{"Finetune/Learning Rate": 1.945987448495854e-05, "Finetune/Loss": 0.9988176822662354, "Finetune/Loss (Raw)": 0.7651252746582031, "Finetune/Step": 685, "Finetune/Step Time": 4.543173192068934} +{"Finetune/Learning Rate": 1.945785298947233e-05, "Finetune/Loss": 1.00417160987854, "Finetune/Loss (Raw)": 0.937903106212616, "Finetune/Step": 686, "Finetune/Step Time": 4.519119227305055} +{"Finetune/Learning Rate": 1.9455827823578017e-05, "Finetune/Loss": 1.0041210651397705, "Finetune/Loss (Raw)": 0.9747819304466248, "Finetune/Step": 687, "Finetune/Step Time": 4.5159494038671255} +{"Finetune/Learning Rate": 1.9453798988061535e-05, "Finetune/Loss": 1.0054452419281006, "Finetune/Loss (Raw)": 1.0191807746887207, "Finetune/Step": 688, "Finetune/Step Time": 4.514368364587426} +{"Finetune/Learning Rate": 1.9451766483710236e-05, "Finetune/Loss": 1.004238486289978, "Finetune/Loss (Raw)": 0.7354812622070312, "Finetune/Step": 689, "Finetune/Step Time": 4.527403257787228} +{"Finetune/Learning Rate": 1.94497303113129e-05, "Finetune/Loss": 1.0034496784210205, "Finetune/Loss (Raw)": 0.8432803153991699, "Finetune/Step": 690, "Finetune/Step Time": 4.574573505669832} +{"Finetune/Learning Rate": 1.9447690471659715e-05, "Finetune/Loss": 1.0035350322723389, "Finetune/Loss (Raw)": 1.0616786479949951, "Finetune/Step": 691, "Finetune/Step Time": 4.570643898099661} +{"Finetune/Learning Rate": 1.944564696554231e-05, "Finetune/Loss": 1.0027698278427124, "Finetune/Loss (Raw)": 0.9373772144317627, "Finetune/Step": 692, "Finetune/Step Time": 4.593129046261311} +{"Finetune/Learning Rate": 1.944359979375373e-05, "Finetune/Loss": 1.003006935119629, "Finetune/Loss (Raw)": 0.9843683242797852, "Finetune/Step": 693, "Finetune/Step Time": 4.5866290461272} +{"Finetune/Learning Rate": 1.9441548957088438e-05, "Finetune/Loss": 0.9977220296859741, "Finetune/Loss (Raw)": 0.4023670554161072, "Finetune/Step": 694, "Finetune/Step Time": 4.617104306817055} +{"Finetune/Learning Rate": 1.9439494456342327e-05, "Finetune/Loss": 0.9966799020767212, "Finetune/Loss (Raw)": 0.8682277798652649, "Finetune/Step": 695, "Finetune/Step Time": 4.628966126590967} +{"Finetune/Learning Rate": 1.943743629231271e-05, "Finetune/Loss": 1.003127098083496, "Finetune/Loss (Raw)": 1.1126224994659424, "Finetune/Step": 696, "Finetune/Step Time": 4.596561038866639} +{"Finetune/Learning Rate": 1.9435374465798318e-05, "Finetune/Loss": 1.0032017230987549, "Finetune/Loss (Raw)": 0.9409331679344177, "Finetune/Step": 697, "Finetune/Step Time": 4.609501503407955} +{"Finetune/Learning Rate": 1.9433308977599306e-05, "Finetune/Loss": 1.004570484161377, "Finetune/Loss (Raw)": 1.1932989358901978, "Finetune/Step": 698, "Finetune/Step Time": 4.602869542315602} +{"Finetune/Learning Rate": 1.943123982851725e-05, "Finetune/Loss": 1.0065441131591797, "Finetune/Loss (Raw)": 1.2068004608154297, "Finetune/Step": 699, "Finetune/Step Time": 4.599775267764926} +{"Finetune/Learning Rate": 1.9429167019355143e-05, "Finetune/Loss": 1.006244421005249, "Finetune/Loss (Raw)": 1.000925064086914, "Finetune/Step": 700, "Finetune/Step Time": 4.613780722022057} +{"Finetune/Learning Rate": 1.9427090550917407e-05, "Finetune/Loss": 1.0003173351287842, "Finetune/Loss (Raw)": 0.3134348690509796, "Finetune/Step": 701, "Finetune/Step Time": 4.646660901606083} +{"Finetune/Learning Rate": 1.9425010424009876e-05, "Finetune/Loss": 0.9993451833724976, "Finetune/Loss (Raw)": 0.9725192189216614, "Finetune/Step": 702, "Finetune/Step Time": 4.659643109887838} +{"Finetune/Learning Rate": 1.9422926639439806e-05, "Finetune/Loss": 0.9973201751708984, "Finetune/Loss (Raw)": 0.8795992732048035, "Finetune/Step": 703, "Finetune/Step Time": 4.649587204679847} +{"Finetune/Learning Rate": 1.9420839198015878e-05, "Finetune/Loss": 0.9973418712615967, "Finetune/Loss (Raw)": 0.9927338361740112, "Finetune/Step": 704, "Finetune/Step Time": 4.666921213269234} +{"Finetune/Learning Rate": 1.941874810054818e-05, "Finetune/Loss": 0.9975411891937256, "Finetune/Loss (Raw)": 1.0199460983276367, "Finetune/Step": 705, "Finetune/Step Time": 4.6665430925786495} +{"Finetune/Learning Rate": 1.941665334784823e-05, "Finetune/Loss": 0.996230959892273, "Finetune/Loss (Raw)": 0.7876227498054504, "Finetune/Step": 706, "Finetune/Step Time": 4.669074481353164} +{"Finetune/Learning Rate": 1.9414554940728963e-05, "Finetune/Loss": 0.9966330528259277, "Finetune/Loss (Raw)": 0.9075238704681396, "Finetune/Step": 707, "Finetune/Step Time": 4.654019713401794} +{"Finetune/Learning Rate": 1.9412452880004728e-05, "Finetune/Loss": 0.9976063966751099, "Finetune/Loss (Raw)": 1.166662573814392, "Finetune/Step": 708, "Finetune/Step Time": 4.66512019187212} +{"Finetune/Learning Rate": 1.941034716649129e-05, "Finetune/Loss": 0.9962956309318542, "Finetune/Loss (Raw)": 0.9711237549781799, "Finetune/Step": 709, "Finetune/Step Time": 4.666234955191612} +{"Finetune/Learning Rate": 1.9408237801005837e-05, "Finetune/Loss": 0.9950164556503296, "Finetune/Loss (Raw)": 0.8843385577201843, "Finetune/Step": 710, "Finetune/Step Time": 4.649623284116387} +{"Finetune/Learning Rate": 1.9406124784366976e-05, "Finetune/Loss": 0.996440052986145, "Finetune/Loss (Raw)": 1.0832926034927368, "Finetune/Step": 711, "Finetune/Step Time": 4.66301385499537} +{"Finetune/Learning Rate": 1.9404008117394726e-05, "Finetune/Loss": 0.9936245083808899, "Finetune/Loss (Raw)": 0.7218053936958313, "Finetune/Step": 712, "Finetune/Step Time": 4.673075156286359} +{"Finetune/Learning Rate": 1.9401887800910525e-05, "Finetune/Loss": 0.9928700923919678, "Finetune/Loss (Raw)": 0.9861665368080139, "Finetune/Step": 713, "Finetune/Step Time": 4.673550980165601} +{"Finetune/Learning Rate": 1.939976383573722e-05, "Finetune/Loss": 0.9974703192710876, "Finetune/Loss (Raw)": 0.9354202747344971, "Finetune/Step": 714, "Finetune/Step Time": 4.636850604787469} +{"Finetune/Learning Rate": 1.9397636222699088e-05, "Finetune/Loss": 0.9969366788864136, "Finetune/Loss (Raw)": 1.1222866773605347, "Finetune/Step": 715, "Finetune/Step Time": 4.6358896773308516} +{"Finetune/Learning Rate": 1.939550496262181e-05, "Finetune/Loss": 0.995928168296814, "Finetune/Loss (Raw)": 0.8687351942062378, "Finetune/Step": 716, "Finetune/Step Time": 4.61032291688025} +{"Finetune/Learning Rate": 1.939337005633249e-05, "Finetune/Loss": 0.9963544607162476, "Finetune/Loss (Raw)": 1.1415457725524902, "Finetune/Step": 717, "Finetune/Step Time": 4.6223588436841965} +{"Finetune/Learning Rate": 1.939123150465964e-05, "Finetune/Loss": 0.9960700869560242, "Finetune/Loss (Raw)": 1.055803894996643, "Finetune/Step": 718, "Finetune/Step Time": 4.6265987902879715} +{"Finetune/Learning Rate": 1.938908930843319e-05, "Finetune/Loss": 0.996782660484314, "Finetune/Loss (Raw)": 1.0408536195755005, "Finetune/Step": 719, "Finetune/Step Time": 4.627080895006657} +{"Finetune/Learning Rate": 1.9386943468484487e-05, "Finetune/Loss": 0.9986568689346313, "Finetune/Loss (Raw)": 1.2120813131332397, "Finetune/Step": 720, "Finetune/Step Time": 4.634755278006196} +{"Finetune/Learning Rate": 1.938479398564629e-05, "Finetune/Loss": 0.9981191158294678, "Finetune/Loss (Raw)": 0.9320497512817383, "Finetune/Step": 721, "Finetune/Step Time": 4.638856207951903} +{"Finetune/Learning Rate": 1.938264086075277e-05, "Finetune/Loss": 0.9979867935180664, "Finetune/Loss (Raw)": 0.995366096496582, "Finetune/Step": 722, "Finetune/Step Time": 4.665338695049286} +{"Finetune/Learning Rate": 1.938048409463951e-05, "Finetune/Loss": 0.9988104701042175, "Finetune/Loss (Raw)": 1.1193156242370605, "Finetune/Step": 723, "Finetune/Step Time": 4.662480382248759} +{"Finetune/Learning Rate": 1.9378323688143515e-05, "Finetune/Loss": 0.9999375939369202, "Finetune/Loss (Raw)": 1.0825952291488647, "Finetune/Step": 724, "Finetune/Step Time": 4.677998719736934} +{"Finetune/Learning Rate": 1.937615964210319e-05, "Finetune/Loss": 1.0019803047180176, "Finetune/Loss (Raw)": 1.1408867835998535, "Finetune/Step": 725, "Finetune/Step Time": 4.67655111849308} +{"Finetune/Learning Rate": 1.9373991957358362e-05, "Finetune/Loss": 1.0001428127288818, "Finetune/Loss (Raw)": 0.7691336274147034, "Finetune/Step": 726, "Finetune/Step Time": 4.664458321407437} +{"Finetune/Learning Rate": 1.9371820634750266e-05, "Finetune/Loss": 1.0002301931381226, "Finetune/Loss (Raw)": 1.0324598550796509, "Finetune/Step": 727, "Finetune/Step Time": 4.671341467648745} +{"Finetune/Learning Rate": 1.9369645675121554e-05, "Finetune/Loss": 1.0009715557098389, "Finetune/Loss (Raw)": 1.1981648206710815, "Finetune/Step": 728, "Finetune/Step Time": 4.670918971300125} +{"Finetune/Learning Rate": 1.936746707931628e-05, "Finetune/Loss": 0.9984598159790039, "Finetune/Loss (Raw)": 0.654758870601654, "Finetune/Step": 729, "Finetune/Step Time": 4.660407930612564} +{"Finetune/Learning Rate": 1.9365284848179917e-05, "Finetune/Loss": 0.9979597926139832, "Finetune/Loss (Raw)": 1.0086567401885986, "Finetune/Step": 730, "Finetune/Step Time": 4.664107980206609} +{"Finetune/Learning Rate": 1.9363098982559346e-05, "Finetune/Loss": 0.9991970062255859, "Finetune/Loss (Raw)": 1.093902349472046, "Finetune/Step": 731, "Finetune/Step Time": 4.659866629168391} +{"Finetune/Learning Rate": 1.9360909483302862e-05, "Finetune/Loss": 0.9979391098022461, "Finetune/Loss (Raw)": 0.9078949689865112, "Finetune/Step": 732, "Finetune/Step Time": 4.691926060244441} +{"Finetune/Learning Rate": 1.9358716351260164e-05, "Finetune/Loss": 0.998065173625946, "Finetune/Loss (Raw)": 0.9620604515075684, "Finetune/Step": 733, "Finetune/Step Time": 4.689868627116084} +{"Finetune/Learning Rate": 1.935651958728236e-05, "Finetune/Loss": 0.9976493120193481, "Finetune/Loss (Raw)": 0.8745489120483398, "Finetune/Step": 734, "Finetune/Step Time": 4.677565228193998} +{"Finetune/Learning Rate": 1.9354319192221975e-05, "Finetune/Loss": 0.9983202815055847, "Finetune/Loss (Raw)": 0.907257080078125, "Finetune/Step": 735, "Finetune/Step Time": 4.654651202261448} +{"Finetune/Learning Rate": 1.9352115166932943e-05, "Finetune/Loss": 0.9970548152923584, "Finetune/Loss (Raw)": 0.9330247044563293, "Finetune/Step": 736, "Finetune/Step Time": 4.690641256049275} +{"Finetune/Learning Rate": 1.9349907512270597e-05, "Finetune/Loss": 0.9986472725868225, "Finetune/Loss (Raw)": 1.121514916419983, "Finetune/Step": 737, "Finetune/Step Time": 4.686586041003466} +{"Finetune/Learning Rate": 1.9347696229091685e-05, "Finetune/Loss": 0.9990463852882385, "Finetune/Loss (Raw)": 1.0215356349945068, "Finetune/Step": 738, "Finetune/Step Time": 4.682874081656337} +{"Finetune/Learning Rate": 1.934548131825437e-05, "Finetune/Loss": 0.9945502877235413, "Finetune/Loss (Raw)": 0.4615418016910553, "Finetune/Step": 739, "Finetune/Step Time": 4.725339058786631} +{"Finetune/Learning Rate": 1.9343262780618206e-05, "Finetune/Loss": 0.9940337538719177, "Finetune/Loss (Raw)": 1.144360065460205, "Finetune/Step": 740, "Finetune/Step Time": 4.727582458406687} +{"Finetune/Learning Rate": 1.934104061704417e-05, "Finetune/Loss": 0.9932906627655029, "Finetune/Loss (Raw)": 0.8090225458145142, "Finetune/Step": 741, "Finetune/Step Time": 4.74558549746871} +{"Finetune/Learning Rate": 1.9338814828394638e-05, "Finetune/Loss": 0.9932425618171692, "Finetune/Loss (Raw)": 0.9818156957626343, "Finetune/Step": 742, "Finetune/Step Time": 4.7510375287383795} +{"Finetune/Learning Rate": 1.9336585415533398e-05, "Finetune/Loss": 0.9935423731803894, "Finetune/Loss (Raw)": 1.073805809020996, "Finetune/Step": 743, "Finetune/Step Time": 4.744231591001153} +{"Finetune/Learning Rate": 1.9334352379325638e-05, "Finetune/Loss": 0.9935169219970703, "Finetune/Loss (Raw)": 0.9986687898635864, "Finetune/Step": 744, "Finetune/Step Time": 4.745902042835951} +{"Finetune/Learning Rate": 1.9332115720637957e-05, "Finetune/Loss": 0.9926879405975342, "Finetune/Loss (Raw)": 0.8950099945068359, "Finetune/Step": 745, "Finetune/Step Time": 4.763835649937391} +{"Finetune/Learning Rate": 1.9329875440338363e-05, "Finetune/Loss": 0.9931080341339111, "Finetune/Loss (Raw)": 1.101274847984314, "Finetune/Step": 746, "Finetune/Step Time": 4.754173381254077} +{"Finetune/Learning Rate": 1.9327631539296258e-05, "Finetune/Loss": 0.9946366548538208, "Finetune/Loss (Raw)": 1.1210821866989136, "Finetune/Step": 747, "Finetune/Step Time": 4.733423136174679} +{"Finetune/Learning Rate": 1.932538401838246e-05, "Finetune/Loss": 0.995128870010376, "Finetune/Loss (Raw)": 1.098863124847412, "Finetune/Step": 748, "Finetune/Step Time": 4.735217260196805} +{"Finetune/Learning Rate": 1.9323132878469185e-05, "Finetune/Loss": 0.9950444102287292, "Finetune/Loss (Raw)": 1.0113452672958374, "Finetune/Step": 749, "Finetune/Step Time": 4.724518155679107} +{"Finetune/Learning Rate": 1.9320878120430063e-05, "Finetune/Loss": 0.9955012798309326, "Finetune/Loss (Raw)": 1.0280636548995972, "Finetune/Step": 750, "Finetune/Step Time": 4.730316385626793} +{"Finetune/Learning Rate": 1.9318619745140115e-05, "Finetune/Loss": 0.9941600561141968, "Finetune/Loss (Raw)": 0.8977521657943726, "Finetune/Step": 751, "Finetune/Step Time": 4.733039002865553} +{"Finetune/Learning Rate": 1.9316357753475776e-05, "Finetune/Loss": 0.9947580695152283, "Finetune/Loss (Raw)": 1.0740382671356201, "Finetune/Step": 752, "Finetune/Step Time": 4.733911396935582} +{"Finetune/Learning Rate": 1.9314092146314872e-05, "Finetune/Loss": 0.9896234273910522, "Finetune/Loss (Raw)": 0.3993740677833557, "Finetune/Step": 753, "Finetune/Step Time": 4.769362537190318} +{"Finetune/Learning Rate": 1.9311822924536656e-05, "Finetune/Loss": 0.9881530404090881, "Finetune/Loss (Raw)": 0.878351628780365, "Finetune/Step": 754, "Finetune/Step Time": 4.7695413790643215} +{"Finetune/Learning Rate": 1.930955008902176e-05, "Finetune/Loss": 0.9878119826316833, "Finetune/Loss (Raw)": 0.9332676529884338, "Finetune/Step": 755, "Finetune/Step Time": 4.791226740926504} +{"Finetune/Learning Rate": 1.9307273640652228e-05, "Finetune/Loss": 0.987504243850708, "Finetune/Loss (Raw)": 0.9516286253929138, "Finetune/Step": 756, "Finetune/Step Time": 4.788109922781587} +{"Finetune/Learning Rate": 1.9304993580311505e-05, "Finetune/Loss": 0.987172544002533, "Finetune/Loss (Raw)": 0.9875473976135254, "Finetune/Step": 757, "Finetune/Step Time": 4.79947586171329} +{"Finetune/Learning Rate": 1.9302709908884437e-05, "Finetune/Loss": 0.9873166680335999, "Finetune/Loss (Raw)": 1.1566417217254639, "Finetune/Step": 758, "Finetune/Step Time": 4.802076591178775} +{"Finetune/Learning Rate": 1.9300422627257275e-05, "Finetune/Loss": 0.9885240793228149, "Finetune/Loss (Raw)": 1.067878246307373, "Finetune/Step": 759, "Finetune/Step Time": 4.7958068791776896} +{"Finetune/Learning Rate": 1.9298131736317666e-05, "Finetune/Loss": 0.9874248504638672, "Finetune/Loss (Raw)": 1.0309501886367798, "Finetune/Step": 760, "Finetune/Step Time": 4.794210582971573} +{"Finetune/Learning Rate": 1.929583723695466e-05, "Finetune/Loss": 0.9848136901855469, "Finetune/Loss (Raw)": 0.935867965221405, "Finetune/Step": 761, "Finetune/Step Time": 4.802911512553692} +{"Finetune/Learning Rate": 1.9293539130058708e-05, "Finetune/Loss": 0.9803532361984253, "Finetune/Loss (Raw)": 0.3825409412384033, "Finetune/Step": 762, "Finetune/Step Time": 4.843174781650305} +{"Finetune/Learning Rate": 1.9291237416521662e-05, "Finetune/Loss": 0.9794754981994629, "Finetune/Loss (Raw)": 0.8952641487121582, "Finetune/Step": 763, "Finetune/Step Time": 4.8467479553073645} +{"Finetune/Learning Rate": 1.9288932097236772e-05, "Finetune/Loss": 0.9789330959320068, "Finetune/Loss (Raw)": 1.010223388671875, "Finetune/Step": 764, "Finetune/Step Time": 4.823917001485825} +{"Finetune/Learning Rate": 1.9286623173098683e-05, "Finetune/Loss": 0.9779626131057739, "Finetune/Loss (Raw)": 0.9781339764595032, "Finetune/Step": 765, "Finetune/Step Time": 4.827959764748812} +{"Finetune/Learning Rate": 1.928431064500345e-05, "Finetune/Loss": 0.9779289960861206, "Finetune/Loss (Raw)": 1.0992200374603271, "Finetune/Step": 766, "Finetune/Step Time": 4.840377043932676} +{"Finetune/Learning Rate": 1.9281994513848515e-05, "Finetune/Loss": 0.9771507978439331, "Finetune/Loss (Raw)": 0.9224541187286377, "Finetune/Step": 767, "Finetune/Step Time": 4.849346160888672} +{"Finetune/Learning Rate": 1.9279674780532726e-05, "Finetune/Loss": 0.9765504002571106, "Finetune/Loss (Raw)": 0.9312680959701538, "Finetune/Step": 768, "Finetune/Step Time": 4.869715234264731} +{"Finetune/Learning Rate": 1.9277351445956323e-05, "Finetune/Loss": 0.9744544625282288, "Finetune/Loss (Raw)": 0.8313361406326294, "Finetune/Step": 769, "Finetune/Step Time": 4.871899537742138} +{"Finetune/Learning Rate": 1.927502451102095e-05, "Finetune/Loss": 0.9798392653465271, "Finetune/Loss (Raw)": 1.032728910446167, "Finetune/Step": 770, "Finetune/Step Time": 4.848065273836255} +{"Finetune/Learning Rate": 1.9272693976629642e-05, "Finetune/Loss": 0.9804361462593079, "Finetune/Loss (Raw)": 1.0779207944869995, "Finetune/Step": 771, "Finetune/Step Time": 4.852982183918357} +{"Finetune/Learning Rate": 1.9270359843686837e-05, "Finetune/Loss": 0.9801152348518372, "Finetune/Loss (Raw)": 1.0046201944351196, "Finetune/Step": 772, "Finetune/Step Time": 4.85266499966383} +{"Finetune/Learning Rate": 1.9268022113098366e-05, "Finetune/Loss": 0.9799600839614868, "Finetune/Loss (Raw)": 1.0645532608032227, "Finetune/Step": 773, "Finetune/Step Time": 4.851836943998933} +{"Finetune/Learning Rate": 1.9265680785771455e-05, "Finetune/Loss": 0.9788122773170471, "Finetune/Loss (Raw)": 0.8783182501792908, "Finetune/Step": 774, "Finetune/Step Time": 4.863132953643799} +{"Finetune/Learning Rate": 1.926333586261473e-05, "Finetune/Loss": 0.979466438293457, "Finetune/Loss (Raw)": 1.0346773862838745, "Finetune/Step": 775, "Finetune/Step Time": 4.8652932569384575} +{"Finetune/Learning Rate": 1.9260987344538205e-05, "Finetune/Loss": 0.9776211977005005, "Finetune/Loss (Raw)": 0.7989290952682495, "Finetune/Step": 776, "Finetune/Step Time": 4.86062067002058} +{"Finetune/Learning Rate": 1.9258635232453296e-05, "Finetune/Loss": 0.9784554839134216, "Finetune/Loss (Raw)": 1.1397801637649536, "Finetune/Step": 777, "Finetune/Step Time": 4.877793483436108} +{"Finetune/Learning Rate": 1.9256279527272815e-05, "Finetune/Loss": 0.9792881608009338, "Finetune/Loss (Raw)": 1.0951719284057617, "Finetune/Step": 778, "Finetune/Step Time": 4.862359523773193} +{"Finetune/Learning Rate": 1.9253920229910966e-05, "Finetune/Loss": 0.9785509705543518, "Finetune/Loss (Raw)": 0.9390358328819275, "Finetune/Step": 779, "Finetune/Step Time": 4.85919969342649} +{"Finetune/Learning Rate": 1.925155734128334e-05, "Finetune/Loss": 0.9733487963676453, "Finetune/Loss (Raw)": 0.33787479996681213, "Finetune/Step": 780, "Finetune/Step Time": 4.890503080561757} +{"Finetune/Learning Rate": 1.9249190862306935e-05, "Finetune/Loss": 0.9735076427459717, "Finetune/Loss (Raw)": 0.9402947425842285, "Finetune/Step": 781, "Finetune/Step Time": 4.891086608171463} +{"Finetune/Learning Rate": 1.9246820793900127e-05, "Finetune/Loss": 0.9698460102081299, "Finetune/Loss (Raw)": 0.7406473755836487, "Finetune/Step": 782, "Finetune/Step Time": 4.921301124617457} +{"Finetune/Learning Rate": 1.92444471369827e-05, "Finetune/Loss": 0.9693635106086731, "Finetune/Loss (Raw)": 0.9301873445510864, "Finetune/Step": 783, "Finetune/Step Time": 4.96950557641685} +{"Finetune/Learning Rate": 1.924206989247582e-05, "Finetune/Loss": 0.9676463603973389, "Finetune/Loss (Raw)": 0.7513118982315063, "Finetune/Step": 784, "Finetune/Step Time": 4.986696710810065} +{"Finetune/Learning Rate": 1.9239689061302053e-05, "Finetune/Loss": 0.9685633182525635, "Finetune/Loss (Raw)": 1.0786139965057373, "Finetune/Step": 785, "Finetune/Step Time": 4.981724733486772} +{"Finetune/Learning Rate": 1.923730464438535e-05, "Finetune/Loss": 0.9697480201721191, "Finetune/Loss (Raw)": 1.1388506889343262, "Finetune/Step": 786, "Finetune/Step Time": 4.979515057057142} +{"Finetune/Learning Rate": 1.923491664265106e-05, "Finetune/Loss": 0.9721741080284119, "Finetune/Loss (Raw)": 1.1187083721160889, "Finetune/Step": 787, "Finetune/Step Time": 4.983719544485211} +{"Finetune/Learning Rate": 1.9232525057025915e-05, "Finetune/Loss": 0.9721751809120178, "Finetune/Loss (Raw)": 1.1321067810058594, "Finetune/Step": 788, "Finetune/Step Time": 4.980300763621926} +{"Finetune/Learning Rate": 1.9230129888438046e-05, "Finetune/Loss": 0.9721153974533081, "Finetune/Loss (Raw)": 1.0063199996948242, "Finetune/Step": 789, "Finetune/Step Time": 4.978748451918364} +{"Finetune/Learning Rate": 1.922773113781697e-05, "Finetune/Loss": 0.9698323011398315, "Finetune/Loss (Raw)": 0.766996443271637, "Finetune/Step": 790, "Finetune/Step Time": 5.008985158056021} +{"Finetune/Learning Rate": 1.9225328806093596e-05, "Finetune/Loss": 0.9684332609176636, "Finetune/Loss (Raw)": 0.9355500340461731, "Finetune/Step": 791, "Finetune/Step Time": 5.010479839518666} +{"Finetune/Learning Rate": 1.922292289420022e-05, "Finetune/Loss": 0.9670560359954834, "Finetune/Loss (Raw)": 0.8486037850379944, "Finetune/Step": 792, "Finetune/Step Time": 5.000807322561741} +{"Finetune/Learning Rate": 1.9220513403070534e-05, "Finetune/Loss": 0.9664620757102966, "Finetune/Loss (Raw)": 1.0869872570037842, "Finetune/Step": 793, "Finetune/Step Time": 5.003140615299344} +{"Finetune/Learning Rate": 1.9218100333639608e-05, "Finetune/Loss": 0.9646444320678711, "Finetune/Loss (Raw)": 0.9408161044120789, "Finetune/Step": 794, "Finetune/Step Time": 5.005827585235238} +{"Finetune/Learning Rate": 1.9215683686843912e-05, "Finetune/Loss": 0.963553786277771, "Finetune/Loss (Raw)": 0.9860066771507263, "Finetune/Step": 795, "Finetune/Step Time": 5.0086606331169605} +{"Finetune/Learning Rate": 1.92132634636213e-05, "Finetune/Loss": 0.9642390608787537, "Finetune/Loss (Raw)": 0.9525144696235657, "Finetune/Step": 796, "Finetune/Step Time": 5.004619067534804} +{"Finetune/Learning Rate": 1.9210839664911013e-05, "Finetune/Loss": 0.9637758135795593, "Finetune/Loss (Raw)": 0.9783152937889099, "Finetune/Step": 797, "Finetune/Step Time": 5.010949108749628} +{"Finetune/Learning Rate": 1.9208412291653674e-05, "Finetune/Loss": 0.9645453095436096, "Finetune/Loss (Raw)": 1.017115592956543, "Finetune/Step": 798, "Finetune/Step Time": 5.012858130037785} +{"Finetune/Learning Rate": 1.9205981344791307e-05, "Finetune/Loss": 0.963889479637146, "Finetune/Loss (Raw)": 0.9085894227027893, "Finetune/Step": 799, "Finetune/Step Time": 5.019620284438133} +{"Finetune/Learning Rate": 1.9203546825267313e-05, "Finetune/Loss": 0.963844895362854, "Finetune/Loss (Raw)": 1.0576266050338745, "Finetune/Step": 800, "Finetune/Step Time": 5.02491357550025} +{"Finetune/Learning Rate": 1.9201108734026477e-05, "Finetune/Loss": 0.9634963274002075, "Finetune/Loss (Raw)": 1.0770533084869385, "Finetune/Step": 801, "Finetune/Step Time": 5.0500067211687565} +{"Finetune/Learning Rate": 1.9198667072014978e-05, "Finetune/Loss": 0.9624898433685303, "Finetune/Loss (Raw)": 0.9933575987815857, "Finetune/Step": 802, "Finetune/Step Time": 5.0534142181277275} +{"Finetune/Learning Rate": 1.9196221840180378e-05, "Finetune/Loss": 0.9603883028030396, "Finetune/Loss (Raw)": 0.7994536757469177, "Finetune/Step": 803, "Finetune/Step Time": 5.059453951194882} +{"Finetune/Learning Rate": 1.9193773039471622e-05, "Finetune/Loss": 0.9600027799606323, "Finetune/Loss (Raw)": 0.9878866076469421, "Finetune/Step": 804, "Finetune/Step Time": 5.050429465249181} +{"Finetune/Learning Rate": 1.9191320670839047e-05, "Finetune/Loss": 0.9609054327011108, "Finetune/Loss (Raw)": 1.0411767959594727, "Finetune/Step": 805, "Finetune/Step Time": 5.020073173567653} +{"Finetune/Learning Rate": 1.918886473523436e-05, "Finetune/Loss": 0.9586168527603149, "Finetune/Loss (Raw)": 0.8428459167480469, "Finetune/Step": 806, "Finetune/Step Time": 5.034182608127594} +{"Finetune/Learning Rate": 1.918640523361067e-05, "Finetune/Loss": 0.9573583602905273, "Finetune/Loss (Raw)": 0.9853242635726929, "Finetune/Step": 807, "Finetune/Step Time": 5.0261589251458645} +{"Finetune/Learning Rate": 1.9183942166922455e-05, "Finetune/Loss": 0.957939624786377, "Finetune/Loss (Raw)": 1.0419601202011108, "Finetune/Step": 808, "Finetune/Step Time": 5.045414816588163} +{"Finetune/Learning Rate": 1.9181475536125588e-05, "Finetune/Loss": 0.9570140242576599, "Finetune/Loss (Raw)": 0.9568663239479065, "Finetune/Step": 809, "Finetune/Step Time": 5.049289269372821} +{"Finetune/Learning Rate": 1.917900534217732e-05, "Finetune/Loss": 0.9571096897125244, "Finetune/Loss (Raw)": 1.0971629619598389, "Finetune/Step": 810, "Finetune/Step Time": 5.0525323655456305} +{"Finetune/Learning Rate": 1.9176531586036282e-05, "Finetune/Loss": 0.9550135135650635, "Finetune/Loss (Raw)": 0.8461378812789917, "Finetune/Step": 811, "Finetune/Step Time": 5.049661828204989} +{"Finetune/Learning Rate": 1.917405426866249e-05, "Finetune/Loss": 0.9570581912994385, "Finetune/Loss (Raw)": 1.2228827476501465, "Finetune/Step": 812, "Finetune/Step Time": 5.047021145001054} +{"Finetune/Learning Rate": 1.9171573391017348e-05, "Finetune/Loss": 0.95881187915802, "Finetune/Loss (Raw)": 0.9895992279052734, "Finetune/Step": 813, "Finetune/Step Time": 5.034061662852764} +{"Finetune/Learning Rate": 1.9169088954063634e-05, "Finetune/Loss": 0.959028422832489, "Finetune/Loss (Raw)": 0.9656187295913696, "Finetune/Step": 814, "Finetune/Step Time": 5.023467339575291} +{"Finetune/Learning Rate": 1.9166600958765508e-05, "Finetune/Loss": 0.9590767025947571, "Finetune/Loss (Raw)": 0.9809621572494507, "Finetune/Step": 815, "Finetune/Step Time": 5.027487905696034} +{"Finetune/Learning Rate": 1.916410940608851e-05, "Finetune/Loss": 0.9594690799713135, "Finetune/Loss (Raw)": 1.0693998336791992, "Finetune/Step": 816, "Finetune/Step Time": 5.027779879048467} +{"Finetune/Learning Rate": 1.9161614296999565e-05, "Finetune/Loss": 0.9621565937995911, "Finetune/Loss (Raw)": 1.079487681388855, "Finetune/Step": 817, "Finetune/Step Time": 5.016803311184049} +{"Finetune/Learning Rate": 1.9159115632466978e-05, "Finetune/Loss": 0.9581474661827087, "Finetune/Loss (Raw)": 0.330108106136322, "Finetune/Step": 818, "Finetune/Step Time": 5.002364490181208} +{"Finetune/Learning Rate": 1.9156613413460432e-05, "Finetune/Loss": 0.9563164710998535, "Finetune/Loss (Raw)": 0.8273177146911621, "Finetune/Step": 819, "Finetune/Step Time": 5.0081622917205095} +{"Finetune/Learning Rate": 1.9154107640950986e-05, "Finetune/Loss": 0.954767107963562, "Finetune/Loss (Raw)": 0.7390469312667847, "Finetune/Step": 820, "Finetune/Step Time": 4.984186382964253} +{"Finetune/Learning Rate": 1.9151598315911088e-05, "Finetune/Loss": 0.9548971652984619, "Finetune/Loss (Raw)": 1.00102961063385, "Finetune/Step": 821, "Finetune/Step Time": 4.9848541636019945} +{"Finetune/Learning Rate": 1.9149085439314548e-05, "Finetune/Loss": 0.9590119123458862, "Finetune/Loss (Raw)": 0.9290549159049988, "Finetune/Step": 822, "Finetune/Step Time": 4.9513407573103905} +{"Finetune/Learning Rate": 1.9146569012136572e-05, "Finetune/Loss": 0.960808277130127, "Finetune/Loss (Raw)": 1.098164677619934, "Finetune/Step": 823, "Finetune/Step Time": 4.931988809257746} +{"Finetune/Learning Rate": 1.914404903535373e-05, "Finetune/Loss": 0.9606059789657593, "Finetune/Loss (Raw)": 1.0867195129394531, "Finetune/Step": 824, "Finetune/Step Time": 4.930698661133647} +{"Finetune/Learning Rate": 1.9141525509943987e-05, "Finetune/Loss": 0.9604354500770569, "Finetune/Loss (Raw)": 0.9191098809242249, "Finetune/Step": 825, "Finetune/Step Time": 4.919582337141037} +{"Finetune/Learning Rate": 1.9138998436886664e-05, "Finetune/Loss": 0.9577930569648743, "Finetune/Loss (Raw)": 0.8550704121589661, "Finetune/Step": 826, "Finetune/Step Time": 4.92587548494339} +{"Finetune/Learning Rate": 1.913646781716247e-05, "Finetune/Loss": 0.9561989307403564, "Finetune/Loss (Raw)": 1.002748966217041, "Finetune/Step": 827, "Finetune/Step Time": 4.912642342969775} +{"Finetune/Learning Rate": 1.9133933651753493e-05, "Finetune/Loss": 0.9513639211654663, "Finetune/Loss (Raw)": 0.38204556703567505, "Finetune/Step": 828, "Finetune/Step Time": 4.929748687893152} +{"Finetune/Learning Rate": 1.913139594164319e-05, "Finetune/Loss": 0.9573053121566772, "Finetune/Loss (Raw)": 1.0739336013793945, "Finetune/Step": 829, "Finetune/Step Time": 4.905422134324908} +{"Finetune/Learning Rate": 1.9128854687816394e-05, "Finetune/Loss": 0.9574834108352661, "Finetune/Loss (Raw)": 0.9953141808509827, "Finetune/Step": 830, "Finetune/Step Time": 4.902272330597043} +{"Finetune/Learning Rate": 1.9126309891259324e-05, "Finetune/Loss": 0.9587414264678955, "Finetune/Loss (Raw)": 1.0406272411346436, "Finetune/Step": 831, "Finetune/Step Time": 4.913282783702016} +{"Finetune/Learning Rate": 1.9123761552959562e-05, "Finetune/Loss": 0.9595261216163635, "Finetune/Loss (Raw)": 1.093171238899231, "Finetune/Step": 832, "Finetune/Step Time": 4.889320367947221} +{"Finetune/Learning Rate": 1.9121209673906064e-05, "Finetune/Loss": 0.9548900723457336, "Finetune/Loss (Raw)": 0.42653244733810425, "Finetune/Step": 833, "Finetune/Step Time": 4.927896831184626} +{"Finetune/Learning Rate": 1.911865425508917e-05, "Finetune/Loss": 0.956519603729248, "Finetune/Loss (Raw)": 0.9962027668952942, "Finetune/Step": 834, "Finetune/Step Time": 4.936119606718421} +{"Finetune/Learning Rate": 1.9116095297500588e-05, "Finetune/Loss": 0.9571606516838074, "Finetune/Loss (Raw)": 0.9895809888839722, "Finetune/Step": 835, "Finetune/Step Time": 4.946935381740332} +{"Finetune/Learning Rate": 1.9113532802133394e-05, "Finetune/Loss": 0.9559630751609802, "Finetune/Loss (Raw)": 1.0133717060089111, "Finetune/Step": 836, "Finetune/Step Time": 4.940805900841951} +{"Finetune/Learning Rate": 1.911096676998205e-05, "Finetune/Loss": 0.95566326379776, "Finetune/Loss (Raw)": 0.9327467083930969, "Finetune/Step": 837, "Finetune/Step Time": 4.944466618821025} +{"Finetune/Learning Rate": 1.9108397202042376e-05, "Finetune/Loss": 0.9570432901382446, "Finetune/Loss (Raw)": 1.06098210811615, "Finetune/Step": 838, "Finetune/Step Time": 4.9483935087919235} +{"Finetune/Learning Rate": 1.910582409931158e-05, "Finetune/Loss": 0.9557845592498779, "Finetune/Loss (Raw)": 0.9221790432929993, "Finetune/Step": 839, "Finetune/Step Time": 4.940148249268532} +{"Finetune/Learning Rate": 1.910324746278822e-05, "Finetune/Loss": 0.9593183994293213, "Finetune/Loss (Raw)": 1.1741304397583008, "Finetune/Step": 840, "Finetune/Step Time": 4.928720971569419} +{"Finetune/Learning Rate": 1.9100667293472252e-05, "Finetune/Loss": 0.9586340188980103, "Finetune/Loss (Raw)": 0.8985705971717834, "Finetune/Step": 841, "Finetune/Step Time": 4.9326941426843405} +{"Finetune/Learning Rate": 1.9098083592364976e-05, "Finetune/Loss": 0.959713876247406, "Finetune/Loss (Raw)": 1.0736407041549683, "Finetune/Step": 842, "Finetune/Step Time": 4.931379459798336} +{"Finetune/Learning Rate": 1.9095496360469087e-05, "Finetune/Loss": 0.9591455459594727, "Finetune/Loss (Raw)": 1.0495362281799316, "Finetune/Step": 843, "Finetune/Step Time": 4.945424864068627} +{"Finetune/Learning Rate": 1.9092905598788633e-05, "Finetune/Loss": 0.9601540565490723, "Finetune/Loss (Raw)": 0.997835099697113, "Finetune/Step": 844, "Finetune/Step Time": 4.946979584172368} +{"Finetune/Learning Rate": 1.9090311308329045e-05, "Finetune/Loss": 0.9596651792526245, "Finetune/Loss (Raw)": 1.078955888748169, "Finetune/Step": 845, "Finetune/Step Time": 4.9331672843545675} +{"Finetune/Learning Rate": 1.9087713490097108e-05, "Finetune/Loss": 0.9595913290977478, "Finetune/Loss (Raw)": 1.0463602542877197, "Finetune/Step": 846, "Finetune/Step Time": 4.931649932637811} +{"Finetune/Learning Rate": 1.9085112145100987e-05, "Finetune/Loss": 0.955102801322937, "Finetune/Loss (Raw)": 0.4663218557834625, "Finetune/Step": 847, "Finetune/Step Time": 4.963690413162112} +{"Finetune/Learning Rate": 1.9082507274350216e-05, "Finetune/Loss": 0.9522560834884644, "Finetune/Loss (Raw)": 0.8476975560188293, "Finetune/Step": 848, "Finetune/Step Time": 4.961597345769405} +{"Finetune/Learning Rate": 1.9079898878855695e-05, "Finetune/Loss": 0.9528542160987854, "Finetune/Loss (Raw)": 1.0086103677749634, "Finetune/Step": 849, "Finetune/Step Time": 4.939030045643449} +{"Finetune/Learning Rate": 1.9077286959629684e-05, "Finetune/Loss": 0.952060341835022, "Finetune/Loss (Raw)": 0.893748939037323, "Finetune/Step": 850, "Finetune/Step Time": 4.928533028811216} +{"Finetune/Learning Rate": 1.907467151768583e-05, "Finetune/Loss": 0.9513292908668518, "Finetune/Loss (Raw)": 1.0257415771484375, "Finetune/Step": 851, "Finetune/Step Time": 4.93525829911232} +{"Finetune/Learning Rate": 1.9072052554039123e-05, "Finetune/Loss": 0.9497607946395874, "Finetune/Loss (Raw)": 0.8818349242210388, "Finetune/Step": 852, "Finetune/Step Time": 4.919738199561834} +{"Finetune/Learning Rate": 1.9069430069705936e-05, "Finetune/Loss": 0.9482877254486084, "Finetune/Loss (Raw)": 0.9523271322250366, "Finetune/Step": 853, "Finetune/Step Time": 4.916335197165608} +{"Finetune/Learning Rate": 1.9066804065704006e-05, "Finetune/Loss": 0.9496570229530334, "Finetune/Loss (Raw)": 0.9444075226783752, "Finetune/Step": 854, "Finetune/Step Time": 4.919170495122671} +{"Finetune/Learning Rate": 1.9064174543052435e-05, "Finetune/Loss": 0.9478858113288879, "Finetune/Loss (Raw)": 0.8057479858398438, "Finetune/Step": 855, "Finetune/Step Time": 4.955397492274642} +{"Finetune/Learning Rate": 1.9061541502771682e-05, "Finetune/Loss": 0.9462956190109253, "Finetune/Loss (Raw)": 0.9946203827857971, "Finetune/Step": 856, "Finetune/Step Time": 4.952764766290784} +{"Finetune/Learning Rate": 1.905890494588359e-05, "Finetune/Loss": 0.9485747814178467, "Finetune/Loss (Raw)": 0.946485161781311, "Finetune/Step": 857, "Finetune/Step Time": 4.942902456969023} +{"Finetune/Learning Rate": 1.9056264873411343e-05, "Finetune/Loss": 0.9499717950820923, "Finetune/Loss (Raw)": 1.1874773502349854, "Finetune/Step": 858, "Finetune/Step Time": 4.94051163084805} +{"Finetune/Learning Rate": 1.905362128637951e-05, "Finetune/Loss": 0.9500318765640259, "Finetune/Loss (Raw)": 1.1015852689743042, "Finetune/Step": 859, "Finetune/Step Time": 4.9556313920766115} +{"Finetune/Learning Rate": 1.905097418581401e-05, "Finetune/Loss": 0.9503809213638306, "Finetune/Loss (Raw)": 0.9525805711746216, "Finetune/Step": 860, "Finetune/Step Time": 4.923433488234878} +{"Finetune/Learning Rate": 1.904832357274214e-05, "Finetune/Loss": 0.948551595211029, "Finetune/Loss (Raw)": 0.7279016971588135, "Finetune/Step": 861, "Finetune/Step Time": 4.913141816854477} +{"Finetune/Learning Rate": 1.9045669448192537e-05, "Finetune/Loss": 0.9496902227401733, "Finetune/Loss (Raw)": 1.0203007459640503, "Finetune/Step": 862, "Finetune/Step Time": 4.915469281375408} +{"Finetune/Learning Rate": 1.904301181319523e-05, "Finetune/Loss": 0.9491708874702454, "Finetune/Loss (Raw)": 0.8407788872718811, "Finetune/Step": 863, "Finetune/Step Time": 4.921562826260924} +{"Finetune/Learning Rate": 1.9040350668781584e-05, "Finetune/Loss": 0.9500323534011841, "Finetune/Loss (Raw)": 1.0432947874069214, "Finetune/Step": 864, "Finetune/Step Time": 4.888727502897382} +{"Finetune/Learning Rate": 1.9037686015984343e-05, "Finetune/Loss": 0.9487820863723755, "Finetune/Loss (Raw)": 0.9614787101745605, "Finetune/Step": 865, "Finetune/Step Time": 4.910689763724804} +{"Finetune/Learning Rate": 1.9035017855837604e-05, "Finetune/Loss": 0.9495573043823242, "Finetune/Loss (Raw)": 1.1207689046859741, "Finetune/Step": 866, "Finetune/Step Time": 4.912264561280608} +{"Finetune/Learning Rate": 1.9032346189376828e-05, "Finetune/Loss": 0.9542495012283325, "Finetune/Loss (Raw)": 1.062139868736267, "Finetune/Step": 867, "Finetune/Step Time": 4.873551892116666} +{"Finetune/Learning Rate": 1.902967101763884e-05, "Finetune/Loss": 0.9527093172073364, "Finetune/Loss (Raw)": 0.9472152590751648, "Finetune/Step": 868, "Finetune/Step Time": 4.86689187027514} +{"Finetune/Learning Rate": 1.902699234166182e-05, "Finetune/Loss": 0.9548866748809814, "Finetune/Loss (Raw)": 1.08772873878479, "Finetune/Step": 869, "Finetune/Step Time": 4.857904888689518} +{"Finetune/Learning Rate": 1.9024310162485308e-05, "Finetune/Loss": 0.9542691707611084, "Finetune/Loss (Raw)": 0.9027702808380127, "Finetune/Step": 870, "Finetune/Step Time": 4.861341351643205} +{"Finetune/Learning Rate": 1.9021624481150208e-05, "Finetune/Loss": 0.9525064826011658, "Finetune/Loss (Raw)": 0.8481839895248413, "Finetune/Step": 871, "Finetune/Step Time": 4.86486897058785} +{"Finetune/Learning Rate": 1.9018935298698777e-05, "Finetune/Loss": 0.9524916410446167, "Finetune/Loss (Raw)": 0.9967665672302246, "Finetune/Step": 872, "Finetune/Step Time": 4.8641002885997295} +{"Finetune/Learning Rate": 1.9016242616174637e-05, "Finetune/Loss": 0.9536646604537964, "Finetune/Loss (Raw)": 1.0451524257659912, "Finetune/Step": 873, "Finetune/Step Time": 4.847946163266897} +{"Finetune/Learning Rate": 1.9013546434622766e-05, "Finetune/Loss": 0.9519757032394409, "Finetune/Loss (Raw)": 0.8850964307785034, "Finetune/Step": 874, "Finetune/Step Time": 4.853189853951335} +{"Finetune/Learning Rate": 1.90108467550895e-05, "Finetune/Loss": 0.9506422281265259, "Finetune/Loss (Raw)": 0.9503952264785767, "Finetune/Step": 875, "Finetune/Step Time": 4.86482048407197} +{"Finetune/Learning Rate": 1.9008143578622527e-05, "Finetune/Loss": 0.9493623971939087, "Finetune/Loss (Raw)": 0.935041069984436, "Finetune/Step": 876, "Finetune/Step Time": 4.866058135405183} +{"Finetune/Learning Rate": 1.9005436906270902e-05, "Finetune/Loss": 0.9495264887809753, "Finetune/Loss (Raw)": 1.0323501825332642, "Finetune/Step": 877, "Finetune/Step Time": 4.864363696426153} +{"Finetune/Learning Rate": 1.9002726739085027e-05, "Finetune/Loss": 0.9492945671081543, "Finetune/Loss (Raw)": 0.9983702898025513, "Finetune/Step": 878, "Finetune/Step Time": 4.857017681002617} +{"Finetune/Learning Rate": 1.900001307811667e-05, "Finetune/Loss": 0.9492263197898865, "Finetune/Loss (Raw)": 0.8890281319618225, "Finetune/Step": 879, "Finetune/Step Time": 4.8776986338198185} +{"Finetune/Learning Rate": 1.899729592441895e-05, "Finetune/Loss": 0.9487622380256653, "Finetune/Loss (Raw)": 1.0146334171295166, "Finetune/Step": 880, "Finetune/Step Time": 4.881894191727042} +{"Finetune/Learning Rate": 1.899457527904634e-05, "Finetune/Loss": 0.95262610912323, "Finetune/Loss (Raw)": 0.8939545750617981, "Finetune/Step": 881, "Finetune/Step Time": 4.8580726981163025} +{"Finetune/Learning Rate": 1.8991851143054662e-05, "Finetune/Loss": 0.9540952444076538, "Finetune/Loss (Raw)": 1.0663959980010986, "Finetune/Step": 882, "Finetune/Step Time": 4.8576729111373425} +{"Finetune/Learning Rate": 1.8989123517501112e-05, "Finetune/Loss": 0.9550725221633911, "Finetune/Loss (Raw)": 1.0583603382110596, "Finetune/Step": 883, "Finetune/Step Time": 4.828548185527325} +{"Finetune/Learning Rate": 1.8986392403444222e-05, "Finetune/Loss": 0.9560123085975647, "Finetune/Loss (Raw)": 1.0719202756881714, "Finetune/Step": 884, "Finetune/Step Time": 4.829211972653866} +{"Finetune/Learning Rate": 1.8983657801943884e-05, "Finetune/Loss": 0.9500976800918579, "Finetune/Loss (Raw)": 0.23047928512096405, "Finetune/Step": 885, "Finetune/Step Time": 4.858756793662906} +{"Finetune/Learning Rate": 1.8980919714061343e-05, "Finetune/Loss": 0.9490035772323608, "Finetune/Loss (Raw)": 1.0165907144546509, "Finetune/Step": 886, "Finetune/Step Time": 4.862395789474249} +{"Finetune/Learning Rate": 1.89781781408592e-05, "Finetune/Loss": 0.9438279867172241, "Finetune/Loss (Raw)": 0.40540340542793274, "Finetune/Step": 887, "Finetune/Step Time": 4.899814736098051} +{"Finetune/Learning Rate": 1.8975433083401403e-05, "Finetune/Loss": 0.9442540407180786, "Finetune/Loss (Raw)": 1.0854876041412354, "Finetune/Step": 888, "Finetune/Step Time": 4.905528599396348} +{"Finetune/Learning Rate": 1.897268454275326e-05, "Finetune/Loss": 0.9451367855072021, "Finetune/Loss (Raw)": 1.0488606691360474, "Finetune/Step": 889, "Finetune/Step Time": 4.905677329748869} +{"Finetune/Learning Rate": 1.896993251998142e-05, "Finetune/Loss": 0.9497449398040771, "Finetune/Loss (Raw)": 0.9723784327507019, "Finetune/Step": 890, "Finetune/Step Time": 4.861134761944413} +{"Finetune/Learning Rate": 1.8967177016153895e-05, "Finetune/Loss": 0.9505103826522827, "Finetune/Loss (Raw)": 0.9932382106781006, "Finetune/Step": 891, "Finetune/Step Time": 4.864549562335014} +{"Finetune/Learning Rate": 1.8964418032340037e-05, "Finetune/Loss": 0.9495946168899536, "Finetune/Loss (Raw)": 0.8930070400238037, "Finetune/Step": 892, "Finetune/Step Time": 4.869177550077438} +{"Finetune/Learning Rate": 1.8961655569610557e-05, "Finetune/Loss": 0.9507946968078613, "Finetune/Loss (Raw)": 1.1317452192306519, "Finetune/Step": 893, "Finetune/Step Time": 4.865691864863038} +{"Finetune/Learning Rate": 1.8958889629037514e-05, "Finetune/Loss": 0.9498065710067749, "Finetune/Loss (Raw)": 0.9727373123168945, "Finetune/Step": 894, "Finetune/Step Time": 4.855623075738549} +{"Finetune/Learning Rate": 1.895612021169431e-05, "Finetune/Loss": 0.9502537250518799, "Finetune/Loss (Raw)": 0.9796947240829468, "Finetune/Step": 895, "Finetune/Step Time": 4.862602053210139} +{"Finetune/Learning Rate": 1.8953347318655704e-05, "Finetune/Loss": 0.9512993097305298, "Finetune/Loss (Raw)": 1.065104603767395, "Finetune/Step": 896, "Finetune/Step Time": 4.843927493318915} +{"Finetune/Learning Rate": 1.8950570950997808e-05, "Finetune/Loss": 0.9514622092247009, "Finetune/Loss (Raw)": 0.852184534072876, "Finetune/Step": 897, "Finetune/Step Time": 4.852461786940694} +{"Finetune/Learning Rate": 1.894779110979807e-05, "Finetune/Loss": 0.9462214708328247, "Finetune/Loss (Raw)": 0.36191216111183167, "Finetune/Step": 898, "Finetune/Step Time": 4.881470950320363} +{"Finetune/Learning Rate": 1.894500779613529e-05, "Finetune/Loss": 0.9442240595817566, "Finetune/Loss (Raw)": 0.8222500681877136, "Finetune/Step": 899, "Finetune/Step Time": 4.9540267661213875} +{"Finetune/Learning Rate": 1.8942221011089623e-05, "Finetune/Loss": 0.9450790286064148, "Finetune/Loss (Raw)": 1.1140636205673218, "Finetune/Step": 900, "Finetune/Step Time": 4.952723477035761} +{"Finetune/Learning Rate": 1.8939430755742564e-05, "Finetune/Loss": 0.945408821105957, "Finetune/Loss (Raw)": 1.1067695617675781, "Finetune/Step": 901, "Finetune/Step Time": 4.958285184577107} +{"Finetune/Learning Rate": 1.8936637031176957e-05, "Finetune/Loss": 0.9468489289283752, "Finetune/Loss (Raw)": 1.062642216682434, "Finetune/Step": 902, "Finetune/Step Time": 4.940800737589598} +{"Finetune/Learning Rate": 1.893383983847699e-05, "Finetune/Loss": 0.9472190141677856, "Finetune/Loss (Raw)": 1.0820540189743042, "Finetune/Step": 903, "Finetune/Step Time": 4.950367724522948} +{"Finetune/Learning Rate": 1.89310391787282e-05, "Finetune/Loss": 0.948508620262146, "Finetune/Loss (Raw)": 0.9639918804168701, "Finetune/Step": 904, "Finetune/Step Time": 4.946334313601255} +{"Finetune/Learning Rate": 1.892823505301747e-05, "Finetune/Loss": 0.9478874802589417, "Finetune/Loss (Raw)": 1.0602788925170898, "Finetune/Step": 905, "Finetune/Step Time": 4.925720449537039} +{"Finetune/Learning Rate": 1.8925427462433027e-05, "Finetune/Loss": 0.9467052221298218, "Finetune/Loss (Raw)": 0.9438445568084717, "Finetune/Step": 906, "Finetune/Step Time": 4.923375338315964} +{"Finetune/Learning Rate": 1.8922616408064436e-05, "Finetune/Loss": 0.9475637674331665, "Finetune/Loss (Raw)": 1.0489314794540405, "Finetune/Step": 907, "Finetune/Step Time": 4.910694405436516} +{"Finetune/Learning Rate": 1.8919801891002615e-05, "Finetune/Loss": 0.9525364637374878, "Finetune/Loss (Raw)": 0.9743773937225342, "Finetune/Step": 908, "Finetune/Step Time": 4.873618649318814} +{"Finetune/Learning Rate": 1.891698391233983e-05, "Finetune/Loss": 0.9530136585235596, "Finetune/Loss (Raw)": 1.0013772249221802, "Finetune/Step": 909, "Finetune/Step Time": 4.877581773325801} +{"Finetune/Learning Rate": 1.8914162473169676e-05, "Finetune/Loss": 0.9547890424728394, "Finetune/Loss (Raw)": 0.9678859114646912, "Finetune/Step": 910, "Finetune/Step Time": 4.8490879610180855} +{"Finetune/Learning Rate": 1.8911337574587098e-05, "Finetune/Loss": 0.9548487067222595, "Finetune/Loss (Raw)": 0.9378302693367004, "Finetune/Step": 911, "Finetune/Step Time": 4.8015909269452095} +{"Finetune/Learning Rate": 1.8908509217688387e-05, "Finetune/Loss": 0.9577800035476685, "Finetune/Loss (Raw)": 1.126513957977295, "Finetune/Step": 912, "Finetune/Step Time": 4.7927365321666} +{"Finetune/Learning Rate": 1.8905677403571177e-05, "Finetune/Loss": 0.9549965858459473, "Finetune/Loss (Raw)": 0.7223448157310486, "Finetune/Step": 913, "Finetune/Step Time": 4.787964375689626} +{"Finetune/Learning Rate": 1.890284213333443e-05, "Finetune/Loss": 0.9548863172531128, "Finetune/Loss (Raw)": 1.1247309446334839, "Finetune/Step": 914, "Finetune/Step Time": 4.7975531332194805} +{"Finetune/Learning Rate": 1.8900003408078468e-05, "Finetune/Loss": 0.9540616273880005, "Finetune/Loss (Raw)": 1.0131523609161377, "Finetune/Step": 915, "Finetune/Step Time": 4.770293049514294} +{"Finetune/Learning Rate": 1.889716122890494e-05, "Finetune/Loss": 0.9533126950263977, "Finetune/Loss (Raw)": 1.0362436771392822, "Finetune/Step": 916, "Finetune/Step Time": 4.776926742866635} +{"Finetune/Learning Rate": 1.889431559691684e-05, "Finetune/Loss": 0.9547169208526611, "Finetune/Loss (Raw)": 1.186055064201355, "Finetune/Step": 917, "Finetune/Step Time": 4.815626008436084} +{"Finetune/Learning Rate": 1.8891466513218507e-05, "Finetune/Loss": 0.9563666582107544, "Finetune/Loss (Raw)": 0.9781625270843506, "Finetune/Step": 918, "Finetune/Step Time": 4.792499413713813} +{"Finetune/Learning Rate": 1.888861397891561e-05, "Finetune/Loss": 0.955035924911499, "Finetune/Loss (Raw)": 0.7652260065078735, "Finetune/Step": 919, "Finetune/Step Time": 4.807108864188194} +{"Finetune/Learning Rate": 1.888575799511516e-05, "Finetune/Loss": 0.9550033807754517, "Finetune/Loss (Raw)": 0.8444307446479797, "Finetune/Step": 920, "Finetune/Step Time": 4.813682546839118} +{"Finetune/Learning Rate": 1.888289856292552e-05, "Finetune/Loss": 0.9542241096496582, "Finetune/Loss (Raw)": 0.9872412085533142, "Finetune/Step": 921, "Finetune/Step Time": 4.803862733766437} +{"Finetune/Learning Rate": 1.8880035683456365e-05, "Finetune/Loss": 0.954084038734436, "Finetune/Loss (Raw)": 0.9228895306587219, "Finetune/Step": 922, "Finetune/Step Time": 4.802581213414669} +{"Finetune/Learning Rate": 1.887716935781873e-05, "Finetune/Loss": 0.9545096158981323, "Finetune/Loss (Raw)": 1.040488600730896, "Finetune/Step": 923, "Finetune/Step Time": 4.7947792410850525} +{"Finetune/Learning Rate": 1.887429958712498e-05, "Finetune/Loss": 0.9544147849082947, "Finetune/Loss (Raw)": 0.9403735399246216, "Finetune/Step": 924, "Finetune/Step Time": 4.803646383807063} +{"Finetune/Learning Rate": 1.8871426372488818e-05, "Finetune/Loss": 0.9531222581863403, "Finetune/Loss (Raw)": 0.8128724098205566, "Finetune/Step": 925, "Finetune/Step Time": 4.823142832145095} +{"Finetune/Learning Rate": 1.886854971502528e-05, "Finetune/Loss": 0.9525879621505737, "Finetune/Loss (Raw)": 0.9487209916114807, "Finetune/Step": 926, "Finetune/Step Time": 4.827185668051243} +{"Finetune/Learning Rate": 1.886566961585074e-05, "Finetune/Loss": 0.9536369442939758, "Finetune/Loss (Raw)": 1.0428563356399536, "Finetune/Step": 927, "Finetune/Step Time": 4.820268418639898} +{"Finetune/Learning Rate": 1.886278607608291e-05, "Finetune/Loss": 0.9532610177993774, "Finetune/Loss (Raw)": 1.0095142126083374, "Finetune/Step": 928, "Finetune/Step Time": 4.812485821545124} +{"Finetune/Learning Rate": 1.8859899096840836e-05, "Finetune/Loss": 0.9529210329055786, "Finetune/Loss (Raw)": 1.0335290431976318, "Finetune/Step": 929, "Finetune/Step Time": 4.787844453006983} +{"Finetune/Learning Rate": 1.8857008679244897e-05, "Finetune/Loss": 0.9522515535354614, "Finetune/Loss (Raw)": 0.9076620936393738, "Finetune/Step": 930, "Finetune/Step Time": 4.7927151825279} +{"Finetune/Learning Rate": 1.885411482441681e-05, "Finetune/Loss": 0.9540435075759888, "Finetune/Loss (Raw)": 1.0288325548171997, "Finetune/Step": 931, "Finetune/Step Time": 4.7742217797786} +{"Finetune/Learning Rate": 1.885121753347962e-05, "Finetune/Loss": 0.9546306133270264, "Finetune/Loss (Raw)": 1.0630278587341309, "Finetune/Step": 932, "Finetune/Step Time": 4.780436521396041} +{"Finetune/Learning Rate": 1.884831680755771e-05, "Finetune/Loss": 0.9546248316764832, "Finetune/Loss (Raw)": 1.0404419898986816, "Finetune/Step": 933, "Finetune/Step Time": 4.790317812934518} +{"Finetune/Learning Rate": 1.8845412647776795e-05, "Finetune/Loss": 0.9559800028800964, "Finetune/Loss (Raw)": 1.0163072347640991, "Finetune/Step": 934, "Finetune/Step Time": 4.776869451627135} +{"Finetune/Learning Rate": 1.8842505055263928e-05, "Finetune/Loss": 0.9550014734268188, "Finetune/Loss (Raw)": 0.8600743412971497, "Finetune/Step": 935, "Finetune/Step Time": 4.777806427329779} +{"Finetune/Learning Rate": 1.883959403114748e-05, "Finetune/Loss": 0.9542479515075684, "Finetune/Loss (Raw)": 0.9455106854438782, "Finetune/Step": 936, "Finetune/Step Time": 4.770015671849251} +{"Finetune/Learning Rate": 1.8836679576557165e-05, "Finetune/Loss": 0.9556149244308472, "Finetune/Loss (Raw)": 1.1318368911743164, "Finetune/Step": 937, "Finetune/Step Time": 4.76488739438355} +{"Finetune/Learning Rate": 1.8833761692624034e-05, "Finetune/Loss": 0.9559168815612793, "Finetune/Loss (Raw)": 1.1358152627944946, "Finetune/Step": 938, "Finetune/Step Time": 4.768742905929685} +{"Finetune/Learning Rate": 1.883084038048045e-05, "Finetune/Loss": 0.9565726518630981, "Finetune/Loss (Raw)": 0.9300703406333923, "Finetune/Step": 939, "Finetune/Step Time": 4.769745115190744} +{"Finetune/Learning Rate": 1.8827915641260127e-05, "Finetune/Loss": 0.9550290703773499, "Finetune/Loss (Raw)": 1.0253093242645264, "Finetune/Step": 940, "Finetune/Step Time": 4.772230017930269} +{"Finetune/Learning Rate": 1.882498747609809e-05, "Finetune/Loss": 0.9549206495285034, "Finetune/Loss (Raw)": 0.9757189154624939, "Finetune/Step": 941, "Finetune/Step Time": 4.770967373624444} +{"Finetune/Learning Rate": 1.8822055886130708e-05, "Finetune/Loss": 0.9547372460365295, "Finetune/Loss (Raw)": 0.9421464204788208, "Finetune/Step": 942, "Finetune/Step Time": 4.776936946436763} +{"Finetune/Learning Rate": 1.8819120872495675e-05, "Finetune/Loss": 0.9545099139213562, "Finetune/Loss (Raw)": 0.9518624544143677, "Finetune/Step": 943, "Finetune/Step Time": 4.775947717949748} +{"Finetune/Learning Rate": 1.8816182436332006e-05, "Finetune/Loss": 0.9540104866027832, "Finetune/Loss (Raw)": 1.005471110343933, "Finetune/Step": 944, "Finetune/Step Time": 4.7710070591419935} +{"Finetune/Learning Rate": 1.881324057878006e-05, "Finetune/Loss": 0.9523822665214539, "Finetune/Loss (Raw)": 0.8710758090019226, "Finetune/Step": 945, "Finetune/Step Time": 4.780921073630452} +{"Finetune/Learning Rate": 1.8810295300981504e-05, "Finetune/Loss": 0.956913948059082, "Finetune/Loss (Raw)": 0.9101665616035461, "Finetune/Step": 946, "Finetune/Step Time": 4.760516673326492} +{"Finetune/Learning Rate": 1.8807346604079352e-05, "Finetune/Loss": 0.9580310583114624, "Finetune/Loss (Raw)": 0.9703010320663452, "Finetune/Step": 947, "Finetune/Step Time": 4.773871963843703} +{"Finetune/Learning Rate": 1.880439448921793e-05, "Finetune/Loss": 0.9552727937698364, "Finetune/Loss (Raw)": 0.38598722219467163, "Finetune/Step": 948, "Finetune/Step Time": 4.80126572959125} +{"Finetune/Learning Rate": 1.88014389575429e-05, "Finetune/Loss": 0.9563754200935364, "Finetune/Loss (Raw)": 1.1421747207641602, "Finetune/Step": 949, "Finetune/Step Time": 4.786592097952962} +{"Finetune/Learning Rate": 1.879848001020124e-05, "Finetune/Loss": 0.9562243819236755, "Finetune/Loss (Raw)": 0.909724235534668, "Finetune/Step": 950, "Finetune/Step Time": 4.784413397312164} +{"Finetune/Learning Rate": 1.8795517648341267e-05, "Finetune/Loss": 0.9568765163421631, "Finetune/Loss (Raw)": 1.1816374063491821, "Finetune/Step": 951, "Finetune/Step Time": 4.783136645331979} +{"Finetune/Learning Rate": 1.8792551873112614e-05, "Finetune/Loss": 0.9550403356552124, "Finetune/Loss (Raw)": 0.8516847491264343, "Finetune/Step": 952, "Finetune/Step Time": 4.7883022129535675} +{"Finetune/Learning Rate": 1.8789582685666237e-05, "Finetune/Loss": 0.955556333065033, "Finetune/Loss (Raw)": 0.985153317451477, "Finetune/Step": 953, "Finetune/Step Time": 4.775571325793862} +{"Finetune/Learning Rate": 1.878661008715442e-05, "Finetune/Loss": 0.9560493230819702, "Finetune/Loss (Raw)": 0.9181750416755676, "Finetune/Step": 954, "Finetune/Step Time": 4.777917103841901} +{"Finetune/Learning Rate": 1.8783634078730774e-05, "Finetune/Loss": 0.9559909701347351, "Finetune/Loss (Raw)": 0.9952802658081055, "Finetune/Step": 955, "Finetune/Step Time": 4.777080472558737} +{"Finetune/Learning Rate": 1.878065466155023e-05, "Finetune/Loss": 0.9610607624053955, "Finetune/Loss (Raw)": 1.0309796333312988, "Finetune/Step": 956, "Finetune/Step Time": 4.742072271183133} +{"Finetune/Learning Rate": 1.877767183676904e-05, "Finetune/Loss": 0.9597377777099609, "Finetune/Loss (Raw)": 0.9045903086662292, "Finetune/Step": 957, "Finetune/Step Time": 4.738291436806321} +{"Finetune/Learning Rate": 1.8774685605544776e-05, "Finetune/Loss": 0.9595975875854492, "Finetune/Loss (Raw)": 0.9773692488670349, "Finetune/Step": 958, "Finetune/Step Time": 4.741631610319018} +{"Finetune/Learning Rate": 1.8771695969036342e-05, "Finetune/Loss": 0.9587693214416504, "Finetune/Loss (Raw)": 0.9346087574958801, "Finetune/Step": 959, "Finetune/Step Time": 4.717684527859092} +{"Finetune/Learning Rate": 1.8768702928403954e-05, "Finetune/Loss": 0.9577102065086365, "Finetune/Loss (Raw)": 0.9576051235198975, "Finetune/Step": 960, "Finetune/Step Time": 4.725653616711497} +{"Finetune/Learning Rate": 1.8765706484809155e-05, "Finetune/Loss": 0.9612707495689392, "Finetune/Loss (Raw)": 0.8822808265686035, "Finetune/Step": 961, "Finetune/Step Time": 4.688799437135458} +{"Finetune/Learning Rate": 1.8762706639414806e-05, "Finetune/Loss": 0.9621466398239136, "Finetune/Loss (Raw)": 1.1083184480667114, "Finetune/Step": 962, "Finetune/Step Time": 4.685620421543717} +{"Finetune/Learning Rate": 1.8759703393385087e-05, "Finetune/Loss": 0.9614949226379395, "Finetune/Loss (Raw)": 0.9061572551727295, "Finetune/Step": 963, "Finetune/Step Time": 4.678762344643474} +{"Finetune/Learning Rate": 1.87566967478855e-05, "Finetune/Loss": 0.9608693718910217, "Finetune/Loss (Raw)": 0.9333006143569946, "Finetune/Step": 964, "Finetune/Step Time": 4.672551829367876} +{"Finetune/Learning Rate": 1.8753686704082868e-05, "Finetune/Loss": 0.959890604019165, "Finetune/Loss (Raw)": 0.8074694871902466, "Finetune/Step": 965, "Finetune/Step Time": 4.672851227223873} +{"Finetune/Learning Rate": 1.8750673263145323e-05, "Finetune/Loss": 0.9590032696723938, "Finetune/Loss (Raw)": 0.9474040269851685, "Finetune/Step": 966, "Finetune/Step Time": 4.674184255301952} +{"Finetune/Learning Rate": 1.874765642624233e-05, "Finetune/Loss": 0.9595772624015808, "Finetune/Loss (Raw)": 0.9956454038619995, "Finetune/Step": 967, "Finetune/Step Time": 4.679029319435358} +{"Finetune/Learning Rate": 1.874463619454466e-05, "Finetune/Loss": 0.9586870670318604, "Finetune/Loss (Raw)": 1.0601886510849, "Finetune/Step": 968, "Finetune/Step Time": 4.682828977704048} +{"Finetune/Learning Rate": 1.874161256922441e-05, "Finetune/Loss": 0.9589362144470215, "Finetune/Loss (Raw)": 0.9304569363594055, "Finetune/Step": 969, "Finetune/Step Time": 4.689834721386433} +{"Finetune/Learning Rate": 1.8738585551454982e-05, "Finetune/Loss": 0.9560767412185669, "Finetune/Loss (Raw)": 0.7076278924942017, "Finetune/Step": 970, "Finetune/Step Time": 4.697184823453426} +{"Finetune/Learning Rate": 1.873555514241111e-05, "Finetune/Loss": 0.9557522535324097, "Finetune/Loss (Raw)": 1.008002519607544, "Finetune/Step": 971, "Finetune/Step Time": 4.7101060803979635} +{"Finetune/Learning Rate": 1.8732521343268838e-05, "Finetune/Loss": 0.9510179758071899, "Finetune/Loss (Raw)": 0.39184606075286865, "Finetune/Step": 972, "Finetune/Step Time": 4.7462024968117476} +{"Finetune/Learning Rate": 1.8729484155205517e-05, "Finetune/Loss": 0.9495259523391724, "Finetune/Loss (Raw)": 0.8879876732826233, "Finetune/Step": 973, "Finetune/Step Time": 4.751738298684359} +{"Finetune/Learning Rate": 1.8726443579399828e-05, "Finetune/Loss": 0.9489631652832031, "Finetune/Loss (Raw)": 0.9743141531944275, "Finetune/Step": 974, "Finetune/Step Time": 4.767513521015644} +{"Finetune/Learning Rate": 1.8723399617031754e-05, "Finetune/Loss": 0.9540089964866638, "Finetune/Loss (Raw)": 1.1121915578842163, "Finetune/Step": 975, "Finetune/Step Time": 4.747346254065633} +{"Finetune/Learning Rate": 1.8720352269282592e-05, "Finetune/Loss": 0.955081582069397, "Finetune/Loss (Raw)": 0.9849941730499268, "Finetune/Step": 976, "Finetune/Step Time": 4.749707009643316} +{"Finetune/Learning Rate": 1.8717301537334975e-05, "Finetune/Loss": 0.9555745124816895, "Finetune/Loss (Raw)": 1.071696162223816, "Finetune/Step": 977, "Finetune/Step Time": 4.755271816626191} +{"Finetune/Learning Rate": 1.8714247422372816e-05, "Finetune/Loss": 0.9560083150863647, "Finetune/Loss (Raw)": 0.9492771625518799, "Finetune/Step": 978, "Finetune/Step Time": 4.734862394630909} +{"Finetune/Learning Rate": 1.8711189925581366e-05, "Finetune/Loss": 0.9549176096916199, "Finetune/Loss (Raw)": 0.8861349821090698, "Finetune/Step": 979, "Finetune/Step Time": 4.7355542704463005} +{"Finetune/Learning Rate": 1.8708129048147176e-05, "Finetune/Loss": 0.9571372270584106, "Finetune/Loss (Raw)": 1.1659421920776367, "Finetune/Step": 980, "Finetune/Step Time": 4.734161999076605} +{"Finetune/Learning Rate": 1.8705064791258118e-05, "Finetune/Loss": 0.9574809074401855, "Finetune/Loss (Raw)": 0.9963154792785645, "Finetune/Step": 981, "Finetune/Step Time": 4.735075356438756} +{"Finetune/Learning Rate": 1.870199715610336e-05, "Finetune/Loss": 0.9581866264343262, "Finetune/Loss (Raw)": 1.0347442626953125, "Finetune/Step": 982, "Finetune/Step Time": 4.722622592002153} +{"Finetune/Learning Rate": 1.8698926143873404e-05, "Finetune/Loss": 0.959888219833374, "Finetune/Loss (Raw)": 1.0235552787780762, "Finetune/Step": 983, "Finetune/Step Time": 4.686315290629864} +{"Finetune/Learning Rate": 1.8695851755760042e-05, "Finetune/Loss": 0.9598129391670227, "Finetune/Loss (Raw)": 0.9849849343299866, "Finetune/Step": 984, "Finetune/Step Time": 4.697097798809409} +{"Finetune/Learning Rate": 1.8692773992956383e-05, "Finetune/Loss": 0.9598493576049805, "Finetune/Loss (Raw)": 0.951138973236084, "Finetune/Step": 985, "Finetune/Step Time": 4.695645904168487} +{"Finetune/Learning Rate": 1.8689692856656854e-05, "Finetune/Loss": 0.9574862718582153, "Finetune/Loss (Raw)": 0.8850035071372986, "Finetune/Step": 986, "Finetune/Step Time": 4.698904860764742} +{"Finetune/Learning Rate": 1.8686608348057178e-05, "Finetune/Loss": 0.9567371010780334, "Finetune/Loss (Raw)": 1.0056935548782349, "Finetune/Step": 987, "Finetune/Step Time": 4.695991491898894} +{"Finetune/Learning Rate": 1.868352046835439e-05, "Finetune/Loss": 0.9560531377792358, "Finetune/Loss (Raw)": 0.8650308847427368, "Finetune/Step": 988, "Finetune/Step Time": 4.698722146451473} +{"Finetune/Learning Rate": 1.8680429218746842e-05, "Finetune/Loss": 0.9584822654724121, "Finetune/Loss (Raw)": 1.038833737373352, "Finetune/Step": 989, "Finetune/Step Time": 4.702171739190817} +{"Finetune/Learning Rate": 1.867733460043419e-05, "Finetune/Loss": 0.9563683867454529, "Finetune/Loss (Raw)": 0.7497223019599915, "Finetune/Step": 990, "Finetune/Step Time": 4.7080196756869555} +{"Finetune/Learning Rate": 1.8674236614617386e-05, "Finetune/Loss": 0.9579576253890991, "Finetune/Loss (Raw)": 1.0442081689834595, "Finetune/Step": 991, "Finetune/Step Time": 4.698758395388722} +{"Finetune/Learning Rate": 1.86711352624987e-05, "Finetune/Loss": 0.9574934840202332, "Finetune/Loss (Raw)": 0.9838821887969971, "Finetune/Step": 992, "Finetune/Step Time": 4.696419067680836} +{"Finetune/Learning Rate": 1.866803054528171e-05, "Finetune/Loss": 0.9577295184135437, "Finetune/Loss (Raw)": 0.9916886687278748, "Finetune/Step": 993, "Finetune/Step Time": 4.706739041954279} +{"Finetune/Learning Rate": 1.8664922464171295e-05, "Finetune/Loss": 0.9550227522850037, "Finetune/Loss (Raw)": 0.7743018269538879, "Finetune/Step": 994, "Finetune/Step Time": 4.715177888050675} +{"Finetune/Learning Rate": 1.866181102037364e-05, "Finetune/Loss": 0.9540989398956299, "Finetune/Loss (Raw)": 0.9438950419425964, "Finetune/Step": 995, "Finetune/Step Time": 4.7134172059595585} +{"Finetune/Learning Rate": 1.8658696215096235e-05, "Finetune/Loss": 0.9541953802108765, "Finetune/Loss (Raw)": 0.9595589637756348, "Finetune/Step": 996, "Finetune/Step Time": 4.7174368891865015} +{"Finetune/Learning Rate": 1.8655578049547878e-05, "Finetune/Loss": 0.9530155658721924, "Finetune/Loss (Raw)": 0.9367139935493469, "Finetune/Step": 997, "Finetune/Step Time": 4.696884928271174} +{"Finetune/Learning Rate": 1.8652456524938665e-05, "Finetune/Loss": 0.9534519910812378, "Finetune/Loss (Raw)": 0.9586317539215088, "Finetune/Step": 998, "Finetune/Step Time": 4.709798410534859} +{"Finetune/Learning Rate": 1.864933164248e-05, "Finetune/Loss": 0.9555096626281738, "Finetune/Loss (Raw)": 1.1115694046020508, "Finetune/Step": 999, "Finetune/Step Time": 4.703397490084171} +{"Finetune/Learning Rate": 1.8646203403384583e-05, "Finetune/Loss": 0.9566614627838135, "Finetune/Loss (Raw)": 1.1441948413848877, "Finetune/Step": 1000, "Finetune/Step Time": 4.697406290099025} +{"Finetune/Learning Rate": 1.8643071808866434e-05, "Finetune/Loss": 0.951039731502533, "Finetune/Loss (Raw)": 0.32556548714637756, "Finetune/Step": 1001, "Finetune/Step Time": 4.73807155713439} +{"Finetune/Learning Rate": 1.8639936860140856e-05, "Finetune/Loss": 0.951797604560852, "Finetune/Loss (Raw)": 0.9821091890335083, "Finetune/Step": 1002, "Finetune/Step Time": 4.7389242723584175} +{"Finetune/Learning Rate": 1.8636798558424464e-05, "Finetune/Loss": 0.9526489973068237, "Finetune/Loss (Raw)": 1.059373378753662, "Finetune/Step": 1003, "Finetune/Step Time": 4.741897895932198} +{"Finetune/Learning Rate": 1.8633656904935168e-05, "Finetune/Loss": 0.9536831378936768, "Finetune/Loss (Raw)": 1.0674026012420654, "Finetune/Step": 1004, "Finetune/Step Time": 4.741424156352878} +{"Finetune/Learning Rate": 1.863051190089219e-05, "Finetune/Loss": 0.9536027908325195, "Finetune/Loss (Raw)": 1.022068738937378, "Finetune/Step": 1005, "Finetune/Step Time": 4.733932664617896} +{"Finetune/Learning Rate": 1.8627363547516032e-05, "Finetune/Loss": 0.9526512026786804, "Finetune/Loss (Raw)": 0.8765695691108704, "Finetune/Step": 1006, "Finetune/Step Time": 4.749685326591134} +{"Finetune/Learning Rate": 1.8624211846028522e-05, "Finetune/Loss": 0.9530125856399536, "Finetune/Loss (Raw)": 0.935286819934845, "Finetune/Step": 1007, "Finetune/Step Time": 4.73081081174314} +{"Finetune/Learning Rate": 1.862105679765277e-05, "Finetune/Loss": 0.9527812004089355, "Finetune/Loss (Raw)": 0.9850154519081116, "Finetune/Step": 1008, "Finetune/Step Time": 4.729796625673771} +{"Finetune/Learning Rate": 1.8617898403613183e-05, "Finetune/Loss": 0.9491504430770874, "Finetune/Loss (Raw)": 0.42921316623687744, "Finetune/Step": 1009, "Finetune/Step Time": 4.753389669582248} +{"Finetune/Learning Rate": 1.8614736665135476e-05, "Finetune/Loss": 0.9479007124900818, "Finetune/Loss (Raw)": 0.9064368009567261, "Finetune/Step": 1010, "Finetune/Step Time": 4.755808301270008} +{"Finetune/Learning Rate": 1.861157158344666e-05, "Finetune/Loss": 0.9468288421630859, "Finetune/Loss (Raw)": 0.921158492565155, "Finetune/Step": 1011, "Finetune/Step Time": 4.755414815619588} +{"Finetune/Learning Rate": 1.8608403159775037e-05, "Finetune/Loss": 0.9463650584220886, "Finetune/Loss (Raw)": 1.0125524997711182, "Finetune/Step": 1012, "Finetune/Step Time": 4.76046852581203} +{"Finetune/Learning Rate": 1.8605231395350213e-05, "Finetune/Loss": 0.9517455101013184, "Finetune/Loss (Raw)": 0.9191776514053345, "Finetune/Step": 1013, "Finetune/Step Time": 4.717985976487398} +{"Finetune/Learning Rate": 1.860205629140309e-05, "Finetune/Loss": 0.9519912600517273, "Finetune/Loss (Raw)": 1.0480523109436035, "Finetune/Step": 1014, "Finetune/Step Time": 4.712625151500106} +{"Finetune/Learning Rate": 1.8598877849165857e-05, "Finetune/Loss": 0.9512848854064941, "Finetune/Loss (Raw)": 0.31498926877975464, "Finetune/Step": 1015, "Finetune/Step Time": 4.715177813544869} +{"Finetune/Learning Rate": 1.8595696069872013e-05, "Finetune/Loss": 0.9506927728652954, "Finetune/Loss (Raw)": 1.0096893310546875, "Finetune/Step": 1016, "Finetune/Step Time": 4.7082817405462265} +{"Finetune/Learning Rate": 1.859251095475634e-05, "Finetune/Loss": 0.9509641528129578, "Finetune/Loss (Raw)": 1.0835996866226196, "Finetune/Step": 1017, "Finetune/Step Time": 4.699780190363526} +{"Finetune/Learning Rate": 1.858932250505492e-05, "Finetune/Loss": 0.9511574506759644, "Finetune/Loss (Raw)": 0.9971267580986023, "Finetune/Step": 1018, "Finetune/Step Time": 4.744444206357002} +{"Finetune/Learning Rate": 1.8586130722005128e-05, "Finetune/Loss": 0.9506034851074219, "Finetune/Loss (Raw)": 0.9223265647888184, "Finetune/Step": 1019, "Finetune/Step Time": 4.742198770865798} +{"Finetune/Learning Rate": 1.8582935606845636e-05, "Finetune/Loss": 0.9513591527938843, "Finetune/Loss (Raw)": 0.9897328615188599, "Finetune/Step": 1020, "Finetune/Step Time": 4.737554537132382} +{"Finetune/Learning Rate": 1.8579737160816397e-05, "Finetune/Loss": 0.9493298530578613, "Finetune/Loss (Raw)": 0.8719973564147949, "Finetune/Step": 1021, "Finetune/Step Time": 4.749196941033006} +{"Finetune/Learning Rate": 1.8576535385158675e-05, "Finetune/Loss": 0.9497328996658325, "Finetune/Loss (Raw)": 1.024329662322998, "Finetune/Step": 1022, "Finetune/Step Time": 4.750200284644961} +{"Finetune/Learning Rate": 1.8573330281115008e-05, "Finetune/Loss": 0.948676347732544, "Finetune/Loss (Raw)": 0.8444498181343079, "Finetune/Step": 1023, "Finetune/Step Time": 4.753403766080737} +{"Finetune/Learning Rate": 1.8570121849929245e-05, "Finetune/Loss": 0.9476097822189331, "Finetune/Loss (Raw)": 0.9285804033279419, "Finetune/Step": 1024, "Finetune/Step Time": 4.7549822479486465} +{"Finetune/Learning Rate": 1.8566910092846508e-05, "Finetune/Loss": 0.9478857517242432, "Finetune/Loss (Raw)": 0.8875163197517395, "Finetune/Step": 1025, "Finetune/Step Time": 4.746121075004339} +{"Finetune/Learning Rate": 1.8563695011113217e-05, "Finetune/Loss": 0.953066349029541, "Finetune/Loss (Raw)": 1.0250285863876343, "Finetune/Step": 1026, "Finetune/Step Time": 4.717391423881054} +{"Finetune/Learning Rate": 1.8560476605977085e-05, "Finetune/Loss": 0.9495618939399719, "Finetune/Loss (Raw)": 0.37367522716522217, "Finetune/Step": 1027, "Finetune/Step Time": 4.684158174321055} +{"Finetune/Learning Rate": 1.8557254878687114e-05, "Finetune/Loss": 0.9490835666656494, "Finetune/Loss (Raw)": 1.0528326034545898, "Finetune/Step": 1028, "Finetune/Step Time": 4.682738246396184} +{"Finetune/Learning Rate": 1.8554029830493587e-05, "Finetune/Loss": 0.9472770690917969, "Finetune/Loss (Raw)": 0.8755403757095337, "Finetune/Step": 1029, "Finetune/Step Time": 4.690316958352923} +{"Finetune/Learning Rate": 1.855080146264809e-05, "Finetune/Loss": 0.9470279216766357, "Finetune/Loss (Raw)": 1.0307600498199463, "Finetune/Step": 1030, "Finetune/Step Time": 4.699825854972005} +{"Finetune/Learning Rate": 1.854756977640348e-05, "Finetune/Loss": 0.9463001489639282, "Finetune/Loss (Raw)": 0.9888884425163269, "Finetune/Step": 1031, "Finetune/Step Time": 4.674942338839173} +{"Finetune/Learning Rate": 1.8544334773013924e-05, "Finetune/Loss": 0.9415176510810852, "Finetune/Loss (Raw)": 0.35183680057525635, "Finetune/Step": 1032, "Finetune/Step Time": 4.709694301709533} +{"Finetune/Learning Rate": 1.8541096453734852e-05, "Finetune/Loss": 0.9411424398422241, "Finetune/Loss (Raw)": 1.0122582912445068, "Finetune/Step": 1033, "Finetune/Step Time": 4.736605249345303} +{"Finetune/Learning Rate": 1.8537854819822998e-05, "Finetune/Loss": 0.9403492212295532, "Finetune/Loss (Raw)": 0.842308759689331, "Finetune/Step": 1034, "Finetune/Step Time": 4.743929037824273} +{"Finetune/Learning Rate": 1.853460987253638e-05, "Finetune/Loss": 0.9400227069854736, "Finetune/Loss (Raw)": 1.0071349143981934, "Finetune/Step": 1035, "Finetune/Step Time": 4.756337815895677} +{"Finetune/Learning Rate": 1.8531361613134286e-05, "Finetune/Loss": 0.9399752616882324, "Finetune/Loss (Raw)": 0.9683065414428711, "Finetune/Step": 1036, "Finetune/Step Time": 4.761214690282941} +{"Finetune/Learning Rate": 1.8528110042877316e-05, "Finetune/Loss": 0.9390294551849365, "Finetune/Loss (Raw)": 0.8803115487098694, "Finetune/Step": 1037, "Finetune/Step Time": 4.764319106936455} +{"Finetune/Learning Rate": 1.8524855163027337e-05, "Finetune/Loss": 0.9406057000160217, "Finetune/Loss (Raw)": 1.1696466207504272, "Finetune/Step": 1038, "Finetune/Step Time": 4.7575857397168875} +{"Finetune/Learning Rate": 1.85215969748475e-05, "Finetune/Loss": 0.9411858320236206, "Finetune/Loss (Raw)": 1.0120819807052612, "Finetune/Step": 1039, "Finetune/Step Time": 4.750715898349881} +{"Finetune/Learning Rate": 1.851833547960225e-05, "Finetune/Loss": 0.9397094249725342, "Finetune/Loss (Raw)": 0.9375425577163696, "Finetune/Step": 1040, "Finetune/Step Time": 4.746697157621384} +{"Finetune/Learning Rate": 1.8515070678557304e-05, "Finetune/Loss": 0.9425210356712341, "Finetune/Loss (Raw)": 1.0822304487228394, "Finetune/Step": 1041, "Finetune/Step Time": 4.728238582611084} +{"Finetune/Learning Rate": 1.8511802572979672e-05, "Finetune/Loss": 0.9422119855880737, "Finetune/Loss (Raw)": 1.0851725339889526, "Finetune/Step": 1042, "Finetune/Step Time": 4.723625490441918} +{"Finetune/Learning Rate": 1.8508531164137644e-05, "Finetune/Loss": 0.937302827835083, "Finetune/Loss (Raw)": 0.3847753405570984, "Finetune/Step": 1043, "Finetune/Step Time": 4.770327743142843} +{"Finetune/Learning Rate": 1.8505256453300778e-05, "Finetune/Loss": 0.9359167218208313, "Finetune/Loss (Raw)": 0.8588232398033142, "Finetune/Step": 1044, "Finetune/Step Time": 4.77695300988853} +{"Finetune/Learning Rate": 1.850197844173994e-05, "Finetune/Loss": 0.934906005859375, "Finetune/Loss (Raw)": 1.0566829442977905, "Finetune/Step": 1045, "Finetune/Step Time": 4.761302364990115} +{"Finetune/Learning Rate": 1.8498697130727255e-05, "Finetune/Loss": 0.9346784353256226, "Finetune/Loss (Raw)": 0.9490396976470947, "Finetune/Step": 1046, "Finetune/Step Time": 4.764015629887581} +{"Finetune/Learning Rate": 1.8495412521536135e-05, "Finetune/Loss": 0.9365523457527161, "Finetune/Loss (Raw)": 1.005078673362732, "Finetune/Step": 1047, "Finetune/Step Time": 4.750286806374788} +{"Finetune/Learning Rate": 1.8492124615441276e-05, "Finetune/Loss": 0.9391065239906311, "Finetune/Loss (Raw)": 1.1713660955429077, "Finetune/Step": 1048, "Finetune/Step Time": 4.741791691631079} +{"Finetune/Learning Rate": 1.8488833413718646e-05, "Finetune/Loss": 0.938249945640564, "Finetune/Loss (Raw)": 0.8775998950004578, "Finetune/Step": 1049, "Finetune/Step Time": 4.739054879173636} +{"Finetune/Learning Rate": 1.84855389176455e-05, "Finetune/Loss": 0.9374063014984131, "Finetune/Loss (Raw)": 0.8149043917655945, "Finetune/Step": 1050, "Finetune/Step Time": 4.751354917883873} +{"Finetune/Learning Rate": 1.8482241128500366e-05, "Finetune/Loss": 0.9366869330406189, "Finetune/Loss (Raw)": 0.9484074711799622, "Finetune/Step": 1051, "Finetune/Step Time": 4.754878133535385} +{"Finetune/Learning Rate": 1.847894004756305e-05, "Finetune/Loss": 0.9383687376976013, "Finetune/Loss (Raw)": 1.1556451320648193, "Finetune/Step": 1052, "Finetune/Step Time": 4.752685071900487} +{"Finetune/Learning Rate": 1.8475635676114643e-05, "Finetune/Loss": 0.9397166967391968, "Finetune/Loss (Raw)": 0.985407292842865, "Finetune/Step": 1053, "Finetune/Step Time": 4.740070542320609} +{"Finetune/Learning Rate": 1.8472328015437503e-05, "Finetune/Loss": 0.9410640001296997, "Finetune/Loss (Raw)": 1.1211845874786377, "Finetune/Step": 1054, "Finetune/Step Time": 4.735795360058546} +{"Finetune/Learning Rate": 1.8469017066815265e-05, "Finetune/Loss": 0.9399636387825012, "Finetune/Loss (Raw)": 0.9020048379898071, "Finetune/Step": 1055, "Finetune/Step Time": 4.736424628645182} +{"Finetune/Learning Rate": 1.8465702831532848e-05, "Finetune/Loss": 0.9405829906463623, "Finetune/Loss (Raw)": 1.088786244392395, "Finetune/Step": 1056, "Finetune/Step Time": 4.7376505974680185} +{"Finetune/Learning Rate": 1.8462385310876444e-05, "Finetune/Loss": 0.9396723508834839, "Finetune/Loss (Raw)": 0.9169769287109375, "Finetune/Step": 1057, "Finetune/Step Time": 4.7324449475854635} +{"Finetune/Learning Rate": 1.845906450613351e-05, "Finetune/Loss": 0.9398304224014282, "Finetune/Loss (Raw)": 0.9278964400291443, "Finetune/Step": 1058, "Finetune/Step Time": 4.725109016522765} +{"Finetune/Learning Rate": 1.8455740418592793e-05, "Finetune/Loss": 0.939209520816803, "Finetune/Loss (Raw)": 0.9493510127067566, "Finetune/Step": 1059, "Finetune/Step Time": 4.743548080325127} +{"Finetune/Learning Rate": 1.8452413049544304e-05, "Finetune/Loss": 0.9385566711425781, "Finetune/Loss (Raw)": 0.9794600605964661, "Finetune/Step": 1060, "Finetune/Step Time": 4.741630459204316} +{"Finetune/Learning Rate": 1.844908240027933e-05, "Finetune/Loss": 0.9370150566101074, "Finetune/Loss (Raw)": 0.8431169986724854, "Finetune/Step": 1061, "Finetune/Step Time": 4.722324952483177} +{"Finetune/Learning Rate": 1.8445748472090432e-05, "Finetune/Loss": 0.9368745684623718, "Finetune/Loss (Raw)": 0.9983274340629578, "Finetune/Step": 1062, "Finetune/Step Time": 4.717343652620912} +{"Finetune/Learning Rate": 1.844241126627144e-05, "Finetune/Loss": 0.9378466010093689, "Finetune/Loss (Raw)": 0.9844958186149597, "Finetune/Step": 1063, "Finetune/Step Time": 4.725346799939871} +{"Finetune/Learning Rate": 1.843907078411746e-05, "Finetune/Loss": 0.9397452473640442, "Finetune/Loss (Raw)": 1.188534140586853, "Finetune/Step": 1064, "Finetune/Step Time": 4.712787130847573} +{"Finetune/Learning Rate": 1.8435727026924864e-05, "Finetune/Loss": 0.9397969841957092, "Finetune/Loss (Raw)": 1.1384633779525757, "Finetune/Step": 1065, "Finetune/Step Time": 4.714053021743894} +{"Finetune/Learning Rate": 1.8432379995991307e-05, "Finetune/Loss": 0.9380001425743103, "Finetune/Loss (Raw)": 0.9058170318603516, "Finetune/Step": 1066, "Finetune/Step Time": 4.707879079505801} +{"Finetune/Learning Rate": 1.8429029692615702e-05, "Finetune/Loss": 0.9394299387931824, "Finetune/Loss (Raw)": 1.1130852699279785, "Finetune/Step": 1067, "Finetune/Step Time": 4.715155305340886} +{"Finetune/Learning Rate": 1.8425676118098236e-05, "Finetune/Loss": 0.9385261535644531, "Finetune/Loss (Raw)": 0.9096269607543945, "Finetune/Step": 1068, "Finetune/Step Time": 4.719912629574537} +{"Finetune/Learning Rate": 1.8422319273740364e-05, "Finetune/Loss": 0.9384744167327881, "Finetune/Loss (Raw)": 0.9690893888473511, "Finetune/Step": 1069, "Finetune/Step Time": 4.746431946754456} +{"Finetune/Learning Rate": 1.8418959160844813e-05, "Finetune/Loss": 0.9392201900482178, "Finetune/Loss (Raw)": 1.0376137495040894, "Finetune/Step": 1070, "Finetune/Step Time": 4.7357702404260635} +{"Finetune/Learning Rate": 1.8415595780715583e-05, "Finetune/Loss": 0.9399535059928894, "Finetune/Loss (Raw)": 1.045722246170044, "Finetune/Step": 1071, "Finetune/Step Time": 4.733545633032918} +{"Finetune/Learning Rate": 1.8412229134657926e-05, "Finetune/Loss": 0.9402562379837036, "Finetune/Loss (Raw)": 1.0442241430282593, "Finetune/Step": 1072, "Finetune/Step Time": 4.743051944300532} +{"Finetune/Learning Rate": 1.8408859223978378e-05, "Finetune/Loss": 0.9409446120262146, "Finetune/Loss (Raw)": 0.9591852426528931, "Finetune/Step": 1073, "Finetune/Step Time": 4.725968707352877} +{"Finetune/Learning Rate": 1.840548604998474e-05, "Finetune/Loss": 0.9420515298843384, "Finetune/Loss (Raw)": 1.0518544912338257, "Finetune/Step": 1074, "Finetune/Step Time": 4.711315946653485} +{"Finetune/Learning Rate": 1.840210961398606e-05, "Finetune/Loss": 0.9427245855331421, "Finetune/Loss (Raw)": 1.0564497709274292, "Finetune/Step": 1075, "Finetune/Step Time": 4.7061553075909615} +{"Finetune/Learning Rate": 1.8398729917292683e-05, "Finetune/Loss": 0.9481461048126221, "Finetune/Loss (Raw)": 1.0799418687820435, "Finetune/Step": 1076, "Finetune/Step Time": 4.669179566204548} +{"Finetune/Learning Rate": 1.8395346961216193e-05, "Finetune/Loss": 0.9477843046188354, "Finetune/Loss (Raw)": 1.0958625078201294, "Finetune/Step": 1077, "Finetune/Step Time": 4.672739282250404} +{"Finetune/Learning Rate": 1.8391960747069453e-05, "Finetune/Loss": 0.9490183591842651, "Finetune/Loss (Raw)": 1.0676863193511963, "Finetune/Step": 1078, "Finetune/Step Time": 4.675263687968254} +{"Finetune/Learning Rate": 1.8388571276166588e-05, "Finetune/Loss": 0.9465655088424683, "Finetune/Loss (Raw)": 0.8676733374595642, "Finetune/Step": 1079, "Finetune/Step Time": 4.673774238675833} +{"Finetune/Learning Rate": 1.8385178549822982e-05, "Finetune/Loss": 0.9475319385528564, "Finetune/Loss (Raw)": 0.9753892421722412, "Finetune/Step": 1080, "Finetune/Step Time": 4.6734849736094475} +{"Finetune/Learning Rate": 1.8381782569355288e-05, "Finetune/Loss": 0.9482895731925964, "Finetune/Loss (Raw)": 1.0821248292922974, "Finetune/Step": 1081, "Finetune/Step Time": 4.677213963121176} +{"Finetune/Learning Rate": 1.837838333608142e-05, "Finetune/Loss": 0.9477909803390503, "Finetune/Loss (Raw)": 0.8543528914451599, "Finetune/Step": 1082, "Finetune/Step Time": 4.692556355148554} +{"Finetune/Learning Rate": 1.8374980851320555e-05, "Finetune/Loss": 0.9480053186416626, "Finetune/Loss (Raw)": 1.0227150917053223, "Finetune/Step": 1083, "Finetune/Step Time": 4.691166687756777} +{"Finetune/Learning Rate": 1.8371575116393125e-05, "Finetune/Loss": 0.948331356048584, "Finetune/Loss (Raw)": 1.072713017463684, "Finetune/Step": 1084, "Finetune/Step Time": 4.711022570729256} +{"Finetune/Learning Rate": 1.8368166132620838e-05, "Finetune/Loss": 0.9492401480674744, "Finetune/Loss (Raw)": 1.020915150642395, "Finetune/Step": 1085, "Finetune/Step Time": 4.706843374297023} +{"Finetune/Learning Rate": 1.8364753901326647e-05, "Finetune/Loss": 0.9497054815292358, "Finetune/Loss (Raw)": 1.0369296073913574, "Finetune/Step": 1086, "Finetune/Step Time": 4.6934482883661985} +{"Finetune/Learning Rate": 1.8361338423834776e-05, "Finetune/Loss": 0.9503979682922363, "Finetune/Loss (Raw)": 1.023253321647644, "Finetune/Step": 1087, "Finetune/Step Time": 4.695849092677236} +{"Finetune/Learning Rate": 1.8357919701470702e-05, "Finetune/Loss": 0.949553370475769, "Finetune/Loss (Raw)": 0.8494937419891357, "Finetune/Step": 1088, "Finetune/Step Time": 4.6978628896176815} +{"Finetune/Learning Rate": 1.835449773556117e-05, "Finetune/Loss": 0.9505150318145752, "Finetune/Loss (Raw)": 1.0053691864013672, "Finetune/Step": 1089, "Finetune/Step Time": 4.699444964528084} +{"Finetune/Learning Rate": 1.8351072527434173e-05, "Finetune/Loss": 0.950520932674408, "Finetune/Loss (Raw)": 1.1090770959854126, "Finetune/Step": 1090, "Finetune/Step Time": 4.7096228916198015} +{"Finetune/Learning Rate": 1.8347644078418968e-05, "Finetune/Loss": 0.9507834911346436, "Finetune/Loss (Raw)": 0.9397696256637573, "Finetune/Step": 1091, "Finetune/Step Time": 4.708478910848498} +{"Finetune/Learning Rate": 1.834421238984607e-05, "Finetune/Loss": 0.9516241550445557, "Finetune/Loss (Raw)": 1.0409038066864014, "Finetune/Step": 1092, "Finetune/Step Time": 4.7075639851391315} +{"Finetune/Learning Rate": 1.8340777463047253e-05, "Finetune/Loss": 0.9522342681884766, "Finetune/Loss (Raw)": 0.8855671882629395, "Finetune/Step": 1093, "Finetune/Step Time": 4.704917849972844} +{"Finetune/Learning Rate": 1.833733929935554e-05, "Finetune/Loss": 0.9525587558746338, "Finetune/Loss (Raw)": 0.9889287352561951, "Finetune/Step": 1094, "Finetune/Step Time": 4.699332697317004} +{"Finetune/Learning Rate": 1.833389790010522e-05, "Finetune/Loss": 0.9529315233230591, "Finetune/Loss (Raw)": 1.04336416721344, "Finetune/Step": 1095, "Finetune/Step Time": 4.688579294830561} +{"Finetune/Learning Rate": 1.8330453266631828e-05, "Finetune/Loss": 0.952958345413208, "Finetune/Loss (Raw)": 1.0636286735534668, "Finetune/Step": 1096, "Finetune/Step Time": 4.678901979699731} +{"Finetune/Learning Rate": 1.8327005400272165e-05, "Finetune/Loss": 0.9534449577331543, "Finetune/Loss (Raw)": 0.9927380084991455, "Finetune/Step": 1097, "Finetune/Step Time": 4.680068861693144} +{"Finetune/Learning Rate": 1.8323554302364273e-05, "Finetune/Loss": 0.9556403756141663, "Finetune/Loss (Raw)": 0.9886411428451538, "Finetune/Step": 1098, "Finetune/Step Time": 4.687169020995498} +{"Finetune/Learning Rate": 1.8320099974247462e-05, "Finetune/Loss": 0.9555630087852478, "Finetune/Loss (Raw)": 0.9980983734130859, "Finetune/Step": 1099, "Finetune/Step Time": 4.660434892401099} +{"Finetune/Learning Rate": 1.831664241726229e-05, "Finetune/Loss": 0.960045576095581, "Finetune/Loss (Raw)": 0.9656156301498413, "Finetune/Step": 1100, "Finetune/Step Time": 4.627860218286514} +{"Finetune/Learning Rate": 1.831318163275056e-05, "Finetune/Loss": 0.9615870714187622, "Finetune/Loss (Raw)": 1.0853034257888794, "Finetune/Step": 1101, "Finetune/Step Time": 4.627806494012475} +{"Finetune/Learning Rate": 1.8309717622055343e-05, "Finetune/Loss": 0.9602490663528442, "Finetune/Loss (Raw)": 0.8030473589897156, "Finetune/Step": 1102, "Finetune/Step Time": 4.608171904459596} +{"Finetune/Learning Rate": 1.830625038652095e-05, "Finetune/Loss": 0.9603906869888306, "Finetune/Loss (Raw)": 1.1303123235702515, "Finetune/Step": 1103, "Finetune/Step Time": 4.594635531306267} +{"Finetune/Learning Rate": 1.8302779927492945e-05, "Finetune/Loss": 0.9609935879707336, "Finetune/Loss (Raw)": 1.0621709823608398, "Finetune/Step": 1104, "Finetune/Step Time": 4.59780508838594} +{"Finetune/Learning Rate": 1.829930624631815e-05, "Finetune/Loss": 0.9613560438156128, "Finetune/Loss (Raw)": 1.1180928945541382, "Finetune/Step": 1105, "Finetune/Step Time": 4.6029123198240995} +{"Finetune/Learning Rate": 1.829582934434463e-05, "Finetune/Loss": 0.9617250561714172, "Finetune/Loss (Raw)": 0.996507465839386, "Finetune/Step": 1106, "Finetune/Step Time": 4.615278992801905} +{"Finetune/Learning Rate": 1.829234922292171e-05, "Finetune/Loss": 0.962814450263977, "Finetune/Loss (Raw)": 1.0255743265151978, "Finetune/Step": 1107, "Finetune/Step Time": 4.615403868257999} +{"Finetune/Learning Rate": 1.8288865883399946e-05, "Finetune/Loss": 0.9622610807418823, "Finetune/Loss (Raw)": 1.0951173305511475, "Finetune/Step": 1108, "Finetune/Step Time": 4.615764414891601} +{"Finetune/Learning Rate": 1.8285379327131157e-05, "Finetune/Loss": 0.9608550667762756, "Finetune/Loss (Raw)": 0.8163439631462097, "Finetune/Step": 1109, "Finetune/Step Time": 4.62748290412128} +{"Finetune/Learning Rate": 1.828188955546841e-05, "Finetune/Loss": 0.9605118036270142, "Finetune/Loss (Raw)": 0.99080890417099, "Finetune/Step": 1110, "Finetune/Step Time": 4.626463528722525} +{"Finetune/Learning Rate": 1.827839656976602e-05, "Finetune/Loss": 0.9613687992095947, "Finetune/Loss (Raw)": 1.1332544088363647, "Finetune/Step": 1111, "Finetune/Step Time": 4.628013329580426} +{"Finetune/Learning Rate": 1.8274900371379543e-05, "Finetune/Loss": 0.9609452486038208, "Finetune/Loss (Raw)": 0.9307680130004883, "Finetune/Step": 1112, "Finetune/Step Time": 4.616308938711882} +{"Finetune/Learning Rate": 1.8271400961665784e-05, "Finetune/Loss": 0.9609311819076538, "Finetune/Loss (Raw)": 0.9493380784988403, "Finetune/Step": 1113, "Finetune/Step Time": 4.612597031518817} +{"Finetune/Learning Rate": 1.8267898341982796e-05, "Finetune/Loss": 0.9620972871780396, "Finetune/Loss (Raw)": 1.0342662334442139, "Finetune/Step": 1114, "Finetune/Step Time": 4.636237097904086} +{"Finetune/Learning Rate": 1.8264392513689882e-05, "Finetune/Loss": 0.962200403213501, "Finetune/Loss (Raw)": 1.0188919305801392, "Finetune/Step": 1115, "Finetune/Step Time": 4.633129166439176} +{"Finetune/Learning Rate": 1.8260883478147578e-05, "Finetune/Loss": 0.9639829397201538, "Finetune/Loss (Raw)": 1.0931882858276367, "Finetune/Step": 1116, "Finetune/Step Time": 4.62790366820991} +{"Finetune/Learning Rate": 1.8257371236717682e-05, "Finetune/Loss": 0.9627945423126221, "Finetune/Loss (Raw)": 0.8867197036743164, "Finetune/Step": 1117, "Finetune/Step Time": 4.6273108292371035} +{"Finetune/Learning Rate": 1.8253855790763215e-05, "Finetune/Loss": 0.9654454588890076, "Finetune/Loss (Raw)": 1.0890438556671143, "Finetune/Step": 1118, "Finetune/Step Time": 4.614018719643354} +{"Finetune/Learning Rate": 1.8250337141648464e-05, "Finetune/Loss": 0.9630343914031982, "Finetune/Loss (Raw)": 0.7355932593345642, "Finetune/Step": 1119, "Finetune/Step Time": 4.62374615482986} +{"Finetune/Learning Rate": 1.8246815290738937e-05, "Finetune/Loss": 0.962997317314148, "Finetune/Loss (Raw)": 0.979132354259491, "Finetune/Step": 1120, "Finetune/Step Time": 4.626490822061896} +{"Finetune/Learning Rate": 1.8243290239401402e-05, "Finetune/Loss": 0.9642781019210815, "Finetune/Loss (Raw)": 1.1556321382522583, "Finetune/Step": 1121, "Finetune/Step Time": 4.595393994823098} +{"Finetune/Learning Rate": 1.8239761989003866e-05, "Finetune/Loss": 0.966424822807312, "Finetune/Loss (Raw)": 1.049086093902588, "Finetune/Step": 1122, "Finetune/Step Time": 4.601719358935952} +{"Finetune/Learning Rate": 1.8236230540915567e-05, "Finetune/Loss": 0.9663608074188232, "Finetune/Loss (Raw)": 0.9356932640075684, "Finetune/Step": 1123, "Finetune/Step Time": 4.619836110621691} +{"Finetune/Learning Rate": 1.8232695896506995e-05, "Finetune/Loss": 0.968475878238678, "Finetune/Loss (Raw)": 1.2302895784378052, "Finetune/Step": 1124, "Finetune/Step Time": 4.6158876940608025} +{"Finetune/Learning Rate": 1.8229158057149875e-05, "Finetune/Loss": 0.9685149192810059, "Finetune/Loss (Raw)": 0.9417154788970947, "Finetune/Step": 1125, "Finetune/Step Time": 4.612469645217061} +{"Finetune/Learning Rate": 1.8225617024217174e-05, "Finetune/Loss": 0.9687221646308899, "Finetune/Loss (Raw)": 0.9851576685905457, "Finetune/Step": 1126, "Finetune/Step Time": 4.594454042613506} +{"Finetune/Learning Rate": 1.8222072799083096e-05, "Finetune/Loss": 0.9679967164993286, "Finetune/Loss (Raw)": 1.018701434135437, "Finetune/Step": 1127, "Finetune/Step Time": 4.593891642987728} +{"Finetune/Learning Rate": 1.821852538312309e-05, "Finetune/Loss": 0.9666780829429626, "Finetune/Loss (Raw)": 0.9754177927970886, "Finetune/Step": 1128, "Finetune/Step Time": 4.606044260784984} +{"Finetune/Learning Rate": 1.8214974777713834e-05, "Finetune/Loss": 0.9728391766548157, "Finetune/Loss (Raw)": 1.114185094833374, "Finetune/Step": 1129, "Finetune/Step Time": 4.562474753707647} +{"Finetune/Learning Rate": 1.821142098423325e-05, "Finetune/Loss": 0.9713914394378662, "Finetune/Loss (Raw)": 0.7967953085899353, "Finetune/Step": 1130, "Finetune/Step Time": 4.570592880249023} +{"Finetune/Learning Rate": 1.8207864004060497e-05, "Finetune/Loss": 0.9715973138809204, "Finetune/Loss (Raw)": 1.0857272148132324, "Finetune/Step": 1131, "Finetune/Step Time": 4.5533904284238815} +{"Finetune/Learning Rate": 1.820430383857597e-05, "Finetune/Loss": 0.9712921380996704, "Finetune/Loss (Raw)": 1.0283455848693848, "Finetune/Step": 1132, "Finetune/Step Time": 4.555149395018816} +{"Finetune/Learning Rate": 1.8200740489161296e-05, "Finetune/Loss": 0.9709951281547546, "Finetune/Loss (Raw)": 0.9840471744537354, "Finetune/Step": 1133, "Finetune/Step Time": 4.550408935174346} +{"Finetune/Learning Rate": 1.819717395719935e-05, "Finetune/Loss": 0.9716464281082153, "Finetune/Loss (Raw)": 0.9599332213401794, "Finetune/Step": 1134, "Finetune/Step Time": 4.53216059692204} +{"Finetune/Learning Rate": 1.819360424407422e-05, "Finetune/Loss": 0.9729655385017395, "Finetune/Loss (Raw)": 1.1041396856307983, "Finetune/Step": 1135, "Finetune/Step Time": 4.527386633679271} +{"Finetune/Learning Rate": 1.8190031351171256e-05, "Finetune/Loss": 0.9728615283966064, "Finetune/Loss (Raw)": 0.9717008471488953, "Finetune/Step": 1136, "Finetune/Step Time": 4.525065645575523} +{"Finetune/Learning Rate": 1.818645527987702e-05, "Finetune/Loss": 0.9777013659477234, "Finetune/Loss (Raw)": 1.0487104654312134, "Finetune/Step": 1137, "Finetune/Step Time": 4.488511029630899} +{"Finetune/Learning Rate": 1.8182876031579312e-05, "Finetune/Loss": 0.9786432981491089, "Finetune/Loss (Raw)": 1.027005672454834, "Finetune/Step": 1138, "Finetune/Step Time": 4.497680734843016} +{"Finetune/Learning Rate": 1.8179293607667177e-05, "Finetune/Loss": 0.9790042638778687, "Finetune/Loss (Raw)": 0.96735680103302, "Finetune/Step": 1139, "Finetune/Step Time": 4.510351886972785} +{"Finetune/Learning Rate": 1.817570800953088e-05, "Finetune/Loss": 0.9790667295455933, "Finetune/Loss (Raw)": 1.0205527544021606, "Finetune/Step": 1140, "Finetune/Step Time": 4.50330519862473} +{"Finetune/Learning Rate": 1.8172119238561924e-05, "Finetune/Loss": 0.9793801307678223, "Finetune/Loss (Raw)": 0.9592914581298828, "Finetune/Step": 1141, "Finetune/Step Time": 4.508837606757879} +{"Finetune/Learning Rate": 1.8168527296153037e-05, "Finetune/Loss": 0.9735939502716064, "Finetune/Loss (Raw)": 0.30741798877716064, "Finetune/Step": 1142, "Finetune/Step Time": 4.54080031439662} +{"Finetune/Learning Rate": 1.8164932183698184e-05, "Finetune/Loss": 0.9784374237060547, "Finetune/Loss (Raw)": 0.9349618554115295, "Finetune/Step": 1143, "Finetune/Step Time": 4.515231743454933} +{"Finetune/Learning Rate": 1.8161333902592554e-05, "Finetune/Loss": 0.9784808158874512, "Finetune/Loss (Raw)": 1.0152373313903809, "Finetune/Step": 1144, "Finetune/Step Time": 4.519348964095116} +{"Finetune/Learning Rate": 1.8157732454232577e-05, "Finetune/Loss": 0.9775278568267822, "Finetune/Loss (Raw)": 0.9616224765777588, "Finetune/Step": 1145, "Finetune/Step Time": 4.519689613953233} +{"Finetune/Learning Rate": 1.81541278400159e-05, "Finetune/Loss": 0.9779808521270752, "Finetune/Loss (Raw)": 1.0551095008850098, "Finetune/Step": 1146, "Finetune/Step Time": 4.478682244196534} +{"Finetune/Learning Rate": 1.81505200613414e-05, "Finetune/Loss": 0.9771542549133301, "Finetune/Loss (Raw)": 0.8165242671966553, "Finetune/Step": 1147, "Finetune/Step Time": 4.4816611763089895} +{"Finetune/Learning Rate": 1.8146909119609196e-05, "Finetune/Loss": 0.9721603989601135, "Finetune/Loss (Raw)": 0.3505234122276306, "Finetune/Step": 1148, "Finetune/Step Time": 4.527925588190556} +{"Finetune/Learning Rate": 1.8143295016220612e-05, "Finetune/Loss": 0.9740196466445923, "Finetune/Loss (Raw)": 1.1099759340286255, "Finetune/Step": 1149, "Finetune/Step Time": 4.533620312809944} +{"Finetune/Learning Rate": 1.813967775257822e-05, "Finetune/Loss": 0.9736530780792236, "Finetune/Loss (Raw)": 0.9774046540260315, "Finetune/Step": 1150, "Finetune/Step Time": 4.523981772363186} +{"Finetune/Learning Rate": 1.8136057330085805e-05, "Finetune/Loss": 0.9739681482315063, "Finetune/Loss (Raw)": 0.88479083776474, "Finetune/Step": 1151, "Finetune/Step Time": 4.5084235686808825} +{"Finetune/Learning Rate": 1.8132433750148383e-05, "Finetune/Loss": 0.9745653867721558, "Finetune/Loss (Raw)": 1.0050204992294312, "Finetune/Step": 1152, "Finetune/Step Time": 4.526070909574628} +{"Finetune/Learning Rate": 1.8128807014172197e-05, "Finetune/Loss": 0.976271390914917, "Finetune/Loss (Raw)": 1.1058849096298218, "Finetune/Step": 1153, "Finetune/Step Time": 4.528754778206348} +{"Finetune/Learning Rate": 1.812517712356471e-05, "Finetune/Loss": 0.970879316329956, "Finetune/Loss (Raw)": 0.33485034108161926, "Finetune/Step": 1154, "Finetune/Step Time": 4.553210634738207} +{"Finetune/Learning Rate": 1.812154407973461e-05, "Finetune/Loss": 0.9706547260284424, "Finetune/Loss (Raw)": 0.34491685032844543, "Finetune/Step": 1155, "Finetune/Step Time": 4.549891980364919} +{"Finetune/Learning Rate": 1.8117907884091815e-05, "Finetune/Loss": 0.9714688062667847, "Finetune/Loss (Raw)": 1.1570426225662231, "Finetune/Step": 1156, "Finetune/Step Time": 4.546001434326172} +{"Finetune/Learning Rate": 1.8114268538047457e-05, "Finetune/Loss": 0.9727702736854553, "Finetune/Loss (Raw)": 1.0421239137649536, "Finetune/Step": 1157, "Finetune/Step Time": 4.543247498571873} +{"Finetune/Learning Rate": 1.81106260430139e-05, "Finetune/Loss": 0.971890926361084, "Finetune/Loss (Raw)": 0.918197751045227, "Finetune/Step": 1158, "Finetune/Step Time": 4.5284023974090815} +{"Finetune/Learning Rate": 1.810698040040472e-05, "Finetune/Loss": 0.970880389213562, "Finetune/Loss (Raw)": 0.859542965888977, "Finetune/Step": 1159, "Finetune/Step Time": 4.539271546527743} +{"Finetune/Learning Rate": 1.810333161163472e-05, "Finetune/Loss": 0.9756696224212646, "Finetune/Loss (Raw)": 0.9648619294166565, "Finetune/Step": 1160, "Finetune/Step Time": 4.523178370669484} +{"Finetune/Learning Rate": 1.809967967811993e-05, "Finetune/Loss": 0.9755325317382812, "Finetune/Loss (Raw)": 0.9947121143341064, "Finetune/Step": 1161, "Finetune/Step Time": 4.5157016310840845} +{"Finetune/Learning Rate": 1.8096024601277586e-05, "Finetune/Loss": 0.9768515229225159, "Finetune/Loss (Raw)": 1.011137843132019, "Finetune/Step": 1162, "Finetune/Step Time": 4.512861145660281} +{"Finetune/Learning Rate": 1.8092366382526157e-05, "Finetune/Loss": 0.9752578735351562, "Finetune/Loss (Raw)": 0.8031481504440308, "Finetune/Step": 1163, "Finetune/Step Time": 4.526577722281218} +{"Finetune/Learning Rate": 1.8088705023285324e-05, "Finetune/Loss": 0.9771049618721008, "Finetune/Loss (Raw)": 1.2047327756881714, "Finetune/Step": 1164, "Finetune/Step Time": 4.524342896416783} +{"Finetune/Learning Rate": 1.8085040524975994e-05, "Finetune/Loss": 0.9731041193008423, "Finetune/Loss (Raw)": 0.36820292472839355, "Finetune/Step": 1165, "Finetune/Step Time": 4.550862891599536} +{"Finetune/Learning Rate": 1.8081372889020282e-05, "Finetune/Loss": 0.9708505868911743, "Finetune/Loss (Raw)": 0.8811967968940735, "Finetune/Step": 1166, "Finetune/Step Time": 4.559803988784552} +{"Finetune/Learning Rate": 1.8077702116841524e-05, "Finetune/Loss": 0.9694743156433105, "Finetune/Loss (Raw)": 0.8359177708625793, "Finetune/Step": 1167, "Finetune/Step Time": 4.574841968715191} +{"Finetune/Learning Rate": 1.807402820986428e-05, "Finetune/Loss": 0.9696410298347473, "Finetune/Loss (Raw)": 0.9588833451271057, "Finetune/Step": 1168, "Finetune/Step Time": 4.573497138917446} +{"Finetune/Learning Rate": 1.8070351169514324e-05, "Finetune/Loss": 0.9690492153167725, "Finetune/Loss (Raw)": 1.0064791440963745, "Finetune/Step": 1169, "Finetune/Step Time": 4.575463419780135} +{"Finetune/Learning Rate": 1.8066670997218633e-05, "Finetune/Loss": 0.964268684387207, "Finetune/Loss (Raw)": 0.4732670783996582, "Finetune/Step": 1170, "Finetune/Step Time": 4.6142581179738045} +{"Finetune/Learning Rate": 1.806298769440542e-05, "Finetune/Loss": 0.9700219631195068, "Finetune/Loss (Raw)": 1.1211864948272705, "Finetune/Step": 1171, "Finetune/Step Time": 4.566584385931492} +{"Finetune/Learning Rate": 1.8059301262504097e-05, "Finetune/Loss": 0.971491813659668, "Finetune/Loss (Raw)": 1.0469701290130615, "Finetune/Step": 1172, "Finetune/Step Time": 4.554651170969009} +{"Finetune/Learning Rate": 1.80556117029453e-05, "Finetune/Loss": 0.9709979295730591, "Finetune/Loss (Raw)": 0.9934599995613098, "Finetune/Step": 1173, "Finetune/Step Time": 4.53357139788568} +{"Finetune/Learning Rate": 1.8051919017160875e-05, "Finetune/Loss": 0.9664479494094849, "Finetune/Loss (Raw)": 0.3666463792324066, "Finetune/Step": 1174, "Finetune/Step Time": 4.569611731916666} +{"Finetune/Learning Rate": 1.804822320658388e-05, "Finetune/Loss": 0.9661051630973816, "Finetune/Loss (Raw)": 0.9611989855766296, "Finetune/Step": 1175, "Finetune/Step Time": 4.583691721782088} +{"Finetune/Learning Rate": 1.8044524272648588e-05, "Finetune/Loss": 0.9642536640167236, "Finetune/Loss (Raw)": 0.9343798160552979, "Finetune/Step": 1176, "Finetune/Step Time": 4.578789696097374} +{"Finetune/Learning Rate": 1.8040822216790475e-05, "Finetune/Loss": 0.9596825838088989, "Finetune/Loss (Raw)": 0.29250040650367737, "Finetune/Step": 1177, "Finetune/Step Time": 4.618689810857177} +{"Finetune/Learning Rate": 1.803711704044625e-05, "Finetune/Loss": 0.9558706283569336, "Finetune/Loss (Raw)": 0.32697346806526184, "Finetune/Step": 1178, "Finetune/Step Time": 4.641715465113521} +{"Finetune/Learning Rate": 1.803340874505381e-05, "Finetune/Loss": 0.9575421810150146, "Finetune/Loss (Raw)": 1.1623677015304565, "Finetune/Step": 1179, "Finetune/Step Time": 4.634801249951124} +{"Finetune/Learning Rate": 1.8029697332052277e-05, "Finetune/Loss": 0.955788254737854, "Finetune/Loss (Raw)": 0.931138277053833, "Finetune/Step": 1180, "Finetune/Step Time": 4.651234356686473} +{"Finetune/Learning Rate": 1.8025982802881976e-05, "Finetune/Loss": 0.9565839767456055, "Finetune/Loss (Raw)": 1.0872567892074585, "Finetune/Step": 1181, "Finetune/Step Time": 4.6386679392308} +{"Finetune/Learning Rate": 1.802226515898444e-05, "Finetune/Loss": 0.9533566236495972, "Finetune/Loss (Raw)": 0.7080914378166199, "Finetune/Step": 1182, "Finetune/Step Time": 4.64003761485219} +{"Finetune/Learning Rate": 1.8018544401802422e-05, "Finetune/Loss": 0.9544100761413574, "Finetune/Loss (Raw)": 1.036837100982666, "Finetune/Step": 1183, "Finetune/Step Time": 4.6416968032717705} +{"Finetune/Learning Rate": 1.801482053277987e-05, "Finetune/Loss": 0.952377438545227, "Finetune/Loss (Raw)": 0.8286185264587402, "Finetune/Step": 1184, "Finetune/Step Time": 4.671326393261552} +{"Finetune/Learning Rate": 1.8011093553361944e-05, "Finetune/Loss": 0.9531486630439758, "Finetune/Loss (Raw)": 1.0156869888305664, "Finetune/Step": 1185, "Finetune/Step Time": 4.6741602048277855} +{"Finetune/Learning Rate": 1.8007363464995015e-05, "Finetune/Loss": 0.9546656608581543, "Finetune/Loss (Raw)": 1.1220707893371582, "Finetune/Step": 1186, "Finetune/Step Time": 4.67423197068274} +{"Finetune/Learning Rate": 1.800363026912665e-05, "Finetune/Loss": 0.9500037431716919, "Finetune/Loss (Raw)": 0.3526340425014496, "Finetune/Step": 1187, "Finetune/Step Time": 4.697696769610047} +{"Finetune/Learning Rate": 1.7999893967205643e-05, "Finetune/Loss": 0.9506466388702393, "Finetune/Loss (Raw)": 1.0617495775222778, "Finetune/Step": 1188, "Finetune/Step Time": 4.697190495207906} +{"Finetune/Learning Rate": 1.7996154560681963e-05, "Finetune/Loss": 0.9523828029632568, "Finetune/Loss (Raw)": 1.0653482675552368, "Finetune/Step": 1189, "Finetune/Step Time": 4.702113205567002} +{"Finetune/Learning Rate": 1.7992412051006817e-05, "Finetune/Loss": 0.9522591233253479, "Finetune/Loss (Raw)": 0.9824911952018738, "Finetune/Step": 1190, "Finetune/Step Time": 4.7005585096776485} +{"Finetune/Learning Rate": 1.7988666439632593e-05, "Finetune/Loss": 0.9527975916862488, "Finetune/Loss (Raw)": 1.0534169673919678, "Finetune/Step": 1191, "Finetune/Step Time": 4.694144017994404} +{"Finetune/Learning Rate": 1.7984917728012886e-05, "Finetune/Loss": 0.9515477418899536, "Finetune/Loss (Raw)": 1.0285567045211792, "Finetune/Step": 1192, "Finetune/Step Time": 4.701650559902191} +{"Finetune/Learning Rate": 1.7981165917602502e-05, "Finetune/Loss": 0.9506532549858093, "Finetune/Loss (Raw)": 1.0239653587341309, "Finetune/Step": 1193, "Finetune/Step Time": 4.700634056702256} +{"Finetune/Learning Rate": 1.7977411009857446e-05, "Finetune/Loss": 0.9519360065460205, "Finetune/Loss (Raw)": 1.0700074434280396, "Finetune/Step": 1194, "Finetune/Step Time": 4.714451540261507} +{"Finetune/Learning Rate": 1.797365300623492e-05, "Finetune/Loss": 0.9510259628295898, "Finetune/Loss (Raw)": 0.9966063499450684, "Finetune/Step": 1195, "Finetune/Step Time": 4.7037999257445335} +{"Finetune/Learning Rate": 1.7969891908193338e-05, "Finetune/Loss": 0.951641857624054, "Finetune/Loss (Raw)": 0.9884570837020874, "Finetune/Step": 1196, "Finetune/Step Time": 4.700241677463055} +{"Finetune/Learning Rate": 1.7966127717192305e-05, "Finetune/Loss": 0.9527620077133179, "Finetune/Loss (Raw)": 1.1124708652496338, "Finetune/Step": 1197, "Finetune/Step Time": 4.666994474828243} +{"Finetune/Learning Rate": 1.796236043469263e-05, "Finetune/Loss": 0.9532454013824463, "Finetune/Loss (Raw)": 1.0994858741760254, "Finetune/Step": 1198, "Finetune/Step Time": 4.672434547916055} +{"Finetune/Learning Rate": 1.7958590062156318e-05, "Finetune/Loss": 0.9532265067100525, "Finetune/Loss (Raw)": 1.043306589126587, "Finetune/Step": 1199, "Finetune/Step Time": 4.668652260676026} +{"Finetune/Learning Rate": 1.7954816601046587e-05, "Finetune/Loss": 0.9533171057701111, "Finetune/Loss (Raw)": 1.055817723274231, "Finetune/Step": 1200, "Finetune/Step Time": 4.665203412994742} +{"Finetune/Learning Rate": 1.7951040052827836e-05, "Finetune/Loss": 0.9540204405784607, "Finetune/Loss (Raw)": 1.0492113828659058, "Finetune/Step": 1201, "Finetune/Step Time": 4.678870363160968} +{"Finetune/Learning Rate": 1.794726041896567e-05, "Finetune/Loss": 0.9548848867416382, "Finetune/Loss (Raw)": 1.162501573562622, "Finetune/Step": 1202, "Finetune/Step Time": 4.682630740106106} +{"Finetune/Learning Rate": 1.7943477700926895e-05, "Finetune/Loss": 0.9554179906845093, "Finetune/Loss (Raw)": 1.1246881484985352, "Finetune/Step": 1203, "Finetune/Step Time": 4.673105226829648} +{"Finetune/Learning Rate": 1.7939691900179507e-05, "Finetune/Loss": 0.9554517269134521, "Finetune/Loss (Raw)": 1.0842710733413696, "Finetune/Step": 1204, "Finetune/Step Time": 4.681474028155208} +{"Finetune/Learning Rate": 1.79359030181927e-05, "Finetune/Loss": 0.9544392228126526, "Finetune/Loss (Raw)": 0.966258704662323, "Finetune/Step": 1205, "Finetune/Step Time": 4.699234971776605} +{"Finetune/Learning Rate": 1.7932111056436868e-05, "Finetune/Loss": 0.9550865888595581, "Finetune/Loss (Raw)": 1.1505500078201294, "Finetune/Step": 1206, "Finetune/Step Time": 4.6913510244339705} +{"Finetune/Learning Rate": 1.79283160163836e-05, "Finetune/Loss": 0.9558081030845642, "Finetune/Loss (Raw)": 0.9600231051445007, "Finetune/Step": 1207, "Finetune/Step Time": 4.698348013684154} +{"Finetune/Learning Rate": 1.792451789950567e-05, "Finetune/Loss": 0.9561471939086914, "Finetune/Loss (Raw)": 1.0187894105911255, "Finetune/Step": 1208, "Finetune/Step Time": 4.698655169457197} +{"Finetune/Learning Rate": 1.7920716707277054e-05, "Finetune/Loss": 0.9559463858604431, "Finetune/Loss (Raw)": 1.056424617767334, "Finetune/Step": 1209, "Finetune/Step Time": 4.696911917999387} +{"Finetune/Learning Rate": 1.7916912441172924e-05, "Finetune/Loss": 0.957053005695343, "Finetune/Loss (Raw)": 0.9959987998008728, "Finetune/Step": 1210, "Finetune/Step Time": 4.674392998218536} +{"Finetune/Learning Rate": 1.7913105102669643e-05, "Finetune/Loss": 0.9564824104309082, "Finetune/Loss (Raw)": 0.9496856331825256, "Finetune/Step": 1211, "Finetune/Step Time": 4.6776497438549995} +{"Finetune/Learning Rate": 1.790929469324476e-05, "Finetune/Loss": 0.9569015502929688, "Finetune/Loss (Raw)": 1.1263586282730103, "Finetune/Step": 1212, "Finetune/Step Time": 4.660557042807341} +{"Finetune/Learning Rate": 1.7905481214377027e-05, "Finetune/Loss": 0.9563919305801392, "Finetune/Loss (Raw)": 0.9556840062141418, "Finetune/Step": 1213, "Finetune/Step Time": 4.654178809374571} +{"Finetune/Learning Rate": 1.790166466754637e-05, "Finetune/Loss": 0.954990804195404, "Finetune/Loss (Raw)": 0.8575801849365234, "Finetune/Step": 1214, "Finetune/Step Time": 4.668323112651706} +{"Finetune/Learning Rate": 1.7897845054233927e-05, "Finetune/Loss": 0.9547512531280518, "Finetune/Loss (Raw)": 0.9926022291183472, "Finetune/Step": 1215, "Finetune/Step Time": 4.669002171605825} +{"Finetune/Learning Rate": 1.7894022375922006e-05, "Finetune/Loss": 0.9508764743804932, "Finetune/Loss (Raw)": 0.3535187542438507, "Finetune/Step": 1216, "Finetune/Step Time": 4.702124070376158} +{"Finetune/Learning Rate": 1.7890196634094122e-05, "Finetune/Loss": 0.9517194032669067, "Finetune/Loss (Raw)": 1.113257884979248, "Finetune/Step": 1217, "Finetune/Step Time": 4.7001917734742165} +{"Finetune/Learning Rate": 1.7886367830234968e-05, "Finetune/Loss": 0.951290488243103, "Finetune/Loss (Raw)": 1.0541836023330688, "Finetune/Step": 1218, "Finetune/Step Time": 4.684943770989776} +{"Finetune/Learning Rate": 1.788253596583043e-05, "Finetune/Loss": 0.9463613629341125, "Finetune/Loss (Raw)": 0.3088354766368866, "Finetune/Step": 1219, "Finetune/Step Time": 4.7235729824751616} +{"Finetune/Learning Rate": 1.787870104236757e-05, "Finetune/Loss": 0.9459823369979858, "Finetune/Loss (Raw)": 0.9923855662345886, "Finetune/Step": 1220, "Finetune/Step Time": 4.728486588224769} +{"Finetune/Learning Rate": 1.7874863061334658e-05, "Finetune/Loss": 0.9471328854560852, "Finetune/Loss (Raw)": 1.0328474044799805, "Finetune/Step": 1221, "Finetune/Step Time": 4.739364067092538} +{"Finetune/Learning Rate": 1.7871022024221133e-05, "Finetune/Loss": 0.9473096132278442, "Finetune/Loss (Raw)": 1.0115413665771484, "Finetune/Step": 1222, "Finetune/Step Time": 4.749593140557408} +{"Finetune/Learning Rate": 1.7867177932517634e-05, "Finetune/Loss": 0.9471880197525024, "Finetune/Loss (Raw)": 1.0278072357177734, "Finetune/Step": 1223, "Finetune/Step Time": 4.744508355855942} +{"Finetune/Learning Rate": 1.786333078771597e-05, "Finetune/Loss": 0.9467446804046631, "Finetune/Loss (Raw)": 1.006874680519104, "Finetune/Step": 1224, "Finetune/Step Time": 4.745961396023631} +{"Finetune/Learning Rate": 1.785948059130915e-05, "Finetune/Loss": 0.9458229541778564, "Finetune/Loss (Raw)": 0.8747624754905701, "Finetune/Step": 1225, "Finetune/Step Time": 4.742086358368397} +{"Finetune/Learning Rate": 1.785562734479135e-05, "Finetune/Loss": 0.945132851600647, "Finetune/Loss (Raw)": 0.9003117680549622, "Finetune/Step": 1226, "Finetune/Step Time": 4.732750112190843} +{"Finetune/Learning Rate": 1.785177104965795e-05, "Finetune/Loss": 0.9441253542900085, "Finetune/Loss (Raw)": 0.869133710861206, "Finetune/Step": 1227, "Finetune/Step Time": 4.7409530729055405} +{"Finetune/Learning Rate": 1.7847911707405494e-05, "Finetune/Loss": 0.9447838068008423, "Finetune/Loss (Raw)": 1.0498878955841064, "Finetune/Step": 1228, "Finetune/Step Time": 4.747391227632761} +{"Finetune/Learning Rate": 1.7844049319531725e-05, "Finetune/Loss": 0.9447399377822876, "Finetune/Loss (Raw)": 1.0796945095062256, "Finetune/Step": 1229, "Finetune/Step Time": 4.7434835489839315} +{"Finetune/Learning Rate": 1.7840183887535558e-05, "Finetune/Loss": 0.9416322708129883, "Finetune/Loss (Raw)": 0.4052635133266449, "Finetune/Step": 1230, "Finetune/Step Time": 4.781061900779605} +{"Finetune/Learning Rate": 1.7836315412917088e-05, "Finetune/Loss": 0.9415371417999268, "Finetune/Loss (Raw)": 1.1181381940841675, "Finetune/Step": 1231, "Finetune/Step Time": 4.774955587461591} +{"Finetune/Learning Rate": 1.78324438971776e-05, "Finetune/Loss": 0.941156804561615, "Finetune/Loss (Raw)": 1.013487458229065, "Finetune/Step": 1232, "Finetune/Step Time": 4.763587644323707} +{"Finetune/Learning Rate": 1.782856934181955e-05, "Finetune/Loss": 0.9405223727226257, "Finetune/Loss (Raw)": 1.03688645362854, "Finetune/Step": 1233, "Finetune/Step Time": 4.748849414288998} +{"Finetune/Learning Rate": 1.782469174834658e-05, "Finetune/Loss": 0.939041256904602, "Finetune/Loss (Raw)": 0.806925892829895, "Finetune/Step": 1234, "Finetune/Step Time": 4.758108923211694} +{"Finetune/Learning Rate": 1.7820811118263503e-05, "Finetune/Loss": 0.9384871125221252, "Finetune/Loss (Raw)": 0.9546427726745605, "Finetune/Step": 1235, "Finetune/Step Time": 4.751094734296203} +{"Finetune/Learning Rate": 1.7816927453076324e-05, "Finetune/Loss": 0.9386287927627563, "Finetune/Loss (Raw)": 1.1132431030273438, "Finetune/Step": 1236, "Finetune/Step Time": 4.747082998976111} +{"Finetune/Learning Rate": 1.781304075429221e-05, "Finetune/Loss": 0.9402287006378174, "Finetune/Loss (Raw)": 1.0211416482925415, "Finetune/Step": 1237, "Finetune/Step Time": 4.749397672712803} +{"Finetune/Learning Rate": 1.7809151023419517e-05, "Finetune/Loss": 0.941716194152832, "Finetune/Loss (Raw)": 1.18120276927948, "Finetune/Step": 1238, "Finetune/Step Time": 4.757460650056601} +{"Finetune/Learning Rate": 1.7805258261967774e-05, "Finetune/Loss": 0.940502941608429, "Finetune/Loss (Raw)": 0.9779627919197083, "Finetune/Step": 1239, "Finetune/Step Time": 4.75938930734992} +{"Finetune/Learning Rate": 1.780136247144768e-05, "Finetune/Loss": 0.9408514499664307, "Finetune/Loss (Raw)": 0.9753797054290771, "Finetune/Step": 1240, "Finetune/Step Time": 4.763296812772751} +{"Finetune/Learning Rate": 1.7797463653371125e-05, "Finetune/Loss": 0.9404466152191162, "Finetune/Loss (Raw)": 0.8975187540054321, "Finetune/Step": 1241, "Finetune/Step Time": 4.767118312418461} +{"Finetune/Learning Rate": 1.7793561809251153e-05, "Finetune/Loss": 0.9399338960647583, "Finetune/Loss (Raw)": 0.9686325788497925, "Finetune/Step": 1242, "Finetune/Step Time": 4.7400876600295305} +{"Finetune/Learning Rate": 1.7789656940602003e-05, "Finetune/Loss": 0.9402210712432861, "Finetune/Loss (Raw)": 1.0556544065475464, "Finetune/Step": 1243, "Finetune/Step Time": 4.7342415396124125} +{"Finetune/Learning Rate": 1.7785749048939076e-05, "Finetune/Loss": 0.9393014311790466, "Finetune/Loss (Raw)": 0.9754669666290283, "Finetune/Step": 1244, "Finetune/Step Time": 4.737730698660016} +{"Finetune/Learning Rate": 1.7781838135778945e-05, "Finetune/Loss": 0.9399682283401489, "Finetune/Loss (Raw)": 0.9720783829689026, "Finetune/Step": 1245, "Finetune/Step Time": 4.728074185550213} +{"Finetune/Learning Rate": 1.777792420263936e-05, "Finetune/Loss": 0.9386990666389465, "Finetune/Loss (Raw)": 0.9265897274017334, "Finetune/Step": 1246, "Finetune/Step Time": 4.750470697879791} +{"Finetune/Learning Rate": 1.7774007251039246e-05, "Finetune/Loss": 0.9413541555404663, "Finetune/Loss (Raw)": 1.075435996055603, "Finetune/Step": 1247, "Finetune/Step Time": 4.736203949898481} +{"Finetune/Learning Rate": 1.777008728249869e-05, "Finetune/Loss": 0.9401021003723145, "Finetune/Loss (Raw)": 0.8188782930374146, "Finetune/Step": 1248, "Finetune/Step Time": 4.741138994693756} +{"Finetune/Learning Rate": 1.776616429853896e-05, "Finetune/Loss": 0.9382696151733398, "Finetune/Loss (Raw)": 0.9210750460624695, "Finetune/Step": 1249, "Finetune/Step Time": 4.7420765701681376} +{"Finetune/Learning Rate": 1.7762238300682484e-05, "Finetune/Loss": 0.9381715059280396, "Finetune/Loss (Raw)": 1.0365277528762817, "Finetune/Step": 1250, "Finetune/Step Time": 4.722306478768587} +{"Finetune/Learning Rate": 1.775830929045287e-05, "Finetune/Loss": 0.9389042854309082, "Finetune/Loss (Raw)": 1.0294855833053589, "Finetune/Step": 1251, "Finetune/Step Time": 4.708433164283633} +{"Finetune/Learning Rate": 1.7754377269374893e-05, "Finetune/Loss": 0.9363425970077515, "Finetune/Loss (Raw)": 0.9023939967155457, "Finetune/Step": 1252, "Finetune/Step Time": 4.704320738092065} +{"Finetune/Learning Rate": 1.7750442238974483e-05, "Finetune/Loss": 0.9362595677375793, "Finetune/Loss (Raw)": 0.9310876727104187, "Finetune/Step": 1253, "Finetune/Step Time": 4.722918612882495} +{"Finetune/Learning Rate": 1.7746504200778756e-05, "Finetune/Loss": 0.9362616539001465, "Finetune/Loss (Raw)": 0.9854285717010498, "Finetune/Step": 1254, "Finetune/Step Time": 4.726430321112275} +{"Finetune/Learning Rate": 1.7742563156315987e-05, "Finetune/Loss": 0.9369282722473145, "Finetune/Loss (Raw)": 1.1040213108062744, "Finetune/Step": 1255, "Finetune/Step Time": 4.728663666173816} +{"Finetune/Learning Rate": 1.773861910711562e-05, "Finetune/Loss": 0.9372310638427734, "Finetune/Loss (Raw)": 1.0141806602478027, "Finetune/Step": 1256, "Finetune/Step Time": 4.724330436438322} +{"Finetune/Learning Rate": 1.773467205470826e-05, "Finetune/Loss": 0.9340840578079224, "Finetune/Loss (Raw)": 0.7113677859306335, "Finetune/Step": 1257, "Finetune/Step Time": 4.750726664438844} +{"Finetune/Learning Rate": 1.7730722000625683e-05, "Finetune/Loss": 0.9353484511375427, "Finetune/Loss (Raw)": 0.9586373567581177, "Finetune/Step": 1258, "Finetune/Step Time": 4.739229800179601} +{"Finetune/Learning Rate": 1.7726768946400825e-05, "Finetune/Loss": 0.9338951110839844, "Finetune/Loss (Raw)": 0.8996986746788025, "Finetune/Step": 1259, "Finetune/Step Time": 4.7500037513673306} +{"Finetune/Learning Rate": 1.772281289356779e-05, "Finetune/Loss": 0.9350628852844238, "Finetune/Loss (Raw)": 1.1778234243392944, "Finetune/Step": 1260, "Finetune/Step Time": 4.752169182524085} +{"Finetune/Learning Rate": 1.771885384366185e-05, "Finetune/Loss": 0.9339679479598999, "Finetune/Loss (Raw)": 0.8438986539840698, "Finetune/Step": 1261, "Finetune/Step Time": 4.751714583486319} +{"Finetune/Learning Rate": 1.7714891798219432e-05, "Finetune/Loss": 0.9342473149299622, "Finetune/Loss (Raw)": 0.9956880211830139, "Finetune/Step": 1262, "Finetune/Step Time": 4.747315429151058} +{"Finetune/Learning Rate": 1.7710926758778123e-05, "Finetune/Loss": 0.9338825345039368, "Finetune/Loss (Raw)": 1.0574463605880737, "Finetune/Step": 1263, "Finetune/Step Time": 4.744010193273425} +{"Finetune/Learning Rate": 1.7706958726876685e-05, "Finetune/Loss": 0.9340804815292358, "Finetune/Loss (Raw)": 0.9970358610153198, "Finetune/Step": 1264, "Finetune/Step Time": 4.741911532357335} +{"Finetune/Learning Rate": 1.770298770405503e-05, "Finetune/Loss": 0.9288641214370728, "Finetune/Loss (Raw)": 0.3810223340988159, "Finetune/Step": 1265, "Finetune/Step Time": 4.779100054875016} +{"Finetune/Learning Rate": 1.7699013691854238e-05, "Finetune/Loss": 0.9283385872840881, "Finetune/Loss (Raw)": 0.9597345590591431, "Finetune/Step": 1266, "Finetune/Step Time": 4.784494288265705} +{"Finetune/Learning Rate": 1.769503669181654e-05, "Finetune/Loss": 0.9289789199829102, "Finetune/Loss (Raw)": 1.0493195056915283, "Finetune/Step": 1267, "Finetune/Step Time": 4.770695932209492} +{"Finetune/Learning Rate": 1.7691056705485333e-05, "Finetune/Loss": 0.9294180274009705, "Finetune/Loss (Raw)": 1.0767635107040405, "Finetune/Step": 1268, "Finetune/Step Time": 4.774248592555523} +{"Finetune/Learning Rate": 1.7687073734405177e-05, "Finetune/Loss": 0.929401695728302, "Finetune/Loss (Raw)": 0.9571983814239502, "Finetune/Step": 1269, "Finetune/Step Time": 4.765995983034372} +{"Finetune/Learning Rate": 1.768308778012178e-05, "Finetune/Loss": 0.9358950853347778, "Finetune/Loss (Raw)": 1.1385632753372192, "Finetune/Step": 1270, "Finetune/Step Time": 4.73750233463943} +{"Finetune/Learning Rate": 1.767909884418202e-05, "Finetune/Loss": 0.9371987581253052, "Finetune/Loss (Raw)": 1.1018363237380981, "Finetune/Step": 1271, "Finetune/Step Time": 4.7223990969359875} +{"Finetune/Learning Rate": 1.7675106928133915e-05, "Finetune/Loss": 0.9356689453125, "Finetune/Loss (Raw)": 0.8194190263748169, "Finetune/Step": 1272, "Finetune/Step Time": 4.730662824586034} +{"Finetune/Learning Rate": 1.7671112033526655e-05, "Finetune/Loss": 0.935490608215332, "Finetune/Loss (Raw)": 0.9387953281402588, "Finetune/Step": 1273, "Finetune/Step Time": 4.74043876118958} +{"Finetune/Learning Rate": 1.7667114161910586e-05, "Finetune/Loss": 0.9340556859970093, "Finetune/Loss (Raw)": 0.8714433312416077, "Finetune/Step": 1274, "Finetune/Step Time": 4.7315999530255795} +{"Finetune/Learning Rate": 1.7663113314837192e-05, "Finetune/Loss": 0.9368239641189575, "Finetune/Loss (Raw)": 1.170863389968872, "Finetune/Step": 1275, "Finetune/Step Time": 4.722844395786524} +{"Finetune/Learning Rate": 1.7659109493859134e-05, "Finetune/Loss": 0.94153892993927, "Finetune/Loss (Raw)": 0.9540366530418396, "Finetune/Step": 1276, "Finetune/Step Time": 4.67819950170815} +{"Finetune/Learning Rate": 1.7655102700530212e-05, "Finetune/Loss": 0.9404622316360474, "Finetune/Loss (Raw)": 0.9721651077270508, "Finetune/Step": 1277, "Finetune/Step Time": 4.6762600261718035} +{"Finetune/Learning Rate": 1.7651092936405385e-05, "Finetune/Loss": 0.9386772513389587, "Finetune/Loss (Raw)": 0.7489207983016968, "Finetune/Step": 1278, "Finetune/Step Time": 4.682647416368127} +{"Finetune/Learning Rate": 1.7647080203040765e-05, "Finetune/Loss": 0.9395711421966553, "Finetune/Loss (Raw)": 0.9992160797119141, "Finetune/Step": 1279, "Finetune/Step Time": 4.68439644575119} +{"Finetune/Learning Rate": 1.7643064501993614e-05, "Finetune/Loss": 0.9407994747161865, "Finetune/Loss (Raw)": 1.1622428894042969, "Finetune/Step": 1280, "Finetune/Step Time": 4.663282670080662} +{"Finetune/Learning Rate": 1.763904583482235e-05, "Finetune/Loss": 0.9396204948425293, "Finetune/Loss (Raw)": 0.954979419708252, "Finetune/Step": 1281, "Finetune/Step Time": 4.647917030379176} +{"Finetune/Learning Rate": 1.7635024203086532e-05, "Finetune/Loss": 0.9447439312934875, "Finetune/Loss (Raw)": 0.9906485676765442, "Finetune/Step": 1282, "Finetune/Step Time": 4.611312096938491} +{"Finetune/Learning Rate": 1.7630999608346888e-05, "Finetune/Loss": 0.9489333629608154, "Finetune/Loss (Raw)": 0.881162703037262, "Finetune/Step": 1283, "Finetune/Step Time": 4.586540572345257} +{"Finetune/Learning Rate": 1.7626972052165275e-05, "Finetune/Loss": 0.9475170373916626, "Finetune/Loss (Raw)": 0.9757534265518188, "Finetune/Step": 1284, "Finetune/Step Time": 4.589628996327519} +{"Finetune/Learning Rate": 1.7622941536104715e-05, "Finetune/Loss": 0.9476640224456787, "Finetune/Loss (Raw)": 1.0609440803527832, "Finetune/Step": 1285, "Finetune/Step Time": 4.589913621544838} +{"Finetune/Learning Rate": 1.7618908061729373e-05, "Finetune/Loss": 0.9480116963386536, "Finetune/Loss (Raw)": 0.9626904726028442, "Finetune/Step": 1286, "Finetune/Step Time": 4.605435561388731} +{"Finetune/Learning Rate": 1.7614871630604554e-05, "Finetune/Loss": 0.9489594101905823, "Finetune/Loss (Raw)": 0.980855405330658, "Finetune/Step": 1287, "Finetune/Step Time": 4.599465610459447} +{"Finetune/Learning Rate": 1.761083224429672e-05, "Finetune/Loss": 0.9485106468200684, "Finetune/Loss (Raw)": 0.9074205160140991, "Finetune/Step": 1288, "Finetune/Step Time": 4.58476042188704} +{"Finetune/Learning Rate": 1.760678990437349e-05, "Finetune/Loss": 0.9478728175163269, "Finetune/Loss (Raw)": 0.9130643606185913, "Finetune/Step": 1289, "Finetune/Step Time": 4.581431342288852} +{"Finetune/Learning Rate": 1.76027446124036e-05, "Finetune/Loss": 0.9486830234527588, "Finetune/Loss (Raw)": 1.1148470640182495, "Finetune/Step": 1290, "Finetune/Step Time": 4.575828740373254} +{"Finetune/Learning Rate": 1.759869636995696e-05, "Finetune/Loss": 0.9501792788505554, "Finetune/Loss (Raw)": 0.9946675896644592, "Finetune/Step": 1291, "Finetune/Step Time": 4.567917061969638} +{"Finetune/Learning Rate": 1.7594645178604614e-05, "Finetune/Loss": 0.9483450651168823, "Finetune/Loss (Raw)": 0.9699513912200928, "Finetune/Step": 1292, "Finetune/Step Time": 4.562306756153703} +{"Finetune/Learning Rate": 1.7590591039918744e-05, "Finetune/Loss": 0.9529136419296265, "Finetune/Loss (Raw)": 0.952987015247345, "Finetune/Step": 1293, "Finetune/Step Time": 4.524527044966817} +{"Finetune/Learning Rate": 1.7586533955472686e-05, "Finetune/Loss": 0.9537746906280518, "Finetune/Loss (Raw)": 0.9914105534553528, "Finetune/Step": 1294, "Finetune/Step Time": 4.52732166275382} +{"Finetune/Learning Rate": 1.7582473926840916e-05, "Finetune/Loss": 0.9544922113418579, "Finetune/Loss (Raw)": 0.9277634024620056, "Finetune/Step": 1295, "Finetune/Step Time": 4.525658652186394} +{"Finetune/Learning Rate": 1.7578410955599046e-05, "Finetune/Loss": 0.954585075378418, "Finetune/Loss (Raw)": 0.9707611203193665, "Finetune/Step": 1296, "Finetune/Step Time": 4.529598290100694} +{"Finetune/Learning Rate": 1.7574345043323843e-05, "Finetune/Loss": 0.9545465707778931, "Finetune/Loss (Raw)": 1.0015517473220825, "Finetune/Step": 1297, "Finetune/Step Time": 4.544567482545972} +{"Finetune/Learning Rate": 1.7570276191593202e-05, "Finetune/Loss": 0.9575383067131042, "Finetune/Loss (Raw)": 0.8562109470367432, "Finetune/Step": 1298, "Finetune/Step Time": 4.497474377974868} +{"Finetune/Learning Rate": 1.7566204401986172e-05, "Finetune/Loss": 0.9546334147453308, "Finetune/Loss (Raw)": 0.7493593692779541, "Finetune/Step": 1299, "Finetune/Step Time": 4.50127706117928} +{"Finetune/Learning Rate": 1.7562129676082933e-05, "Finetune/Loss": 0.9534668922424316, "Finetune/Loss (Raw)": 0.8976643085479736, "Finetune/Step": 1300, "Finetune/Step Time": 4.499561550095677} +{"Finetune/Learning Rate": 1.75580520154648e-05, "Finetune/Loss": 0.953710675239563, "Finetune/Loss (Raw)": 1.0246528387069702, "Finetune/Step": 1301, "Finetune/Step Time": 4.517428237944841} +{"Finetune/Learning Rate": 1.755397142171424e-05, "Finetune/Loss": 0.9582269787788391, "Finetune/Loss (Raw)": 0.9447401762008667, "Finetune/Step": 1302, "Finetune/Step Time": 4.4940315913408995} +{"Finetune/Learning Rate": 1.7549887896414853e-05, "Finetune/Loss": 0.9584417343139648, "Finetune/Loss (Raw)": 0.9886903166770935, "Finetune/Step": 1303, "Finetune/Step Time": 4.482919789850712} +{"Finetune/Learning Rate": 1.7545801441151368e-05, "Finetune/Loss": 0.9589886665344238, "Finetune/Loss (Raw)": 1.0043851137161255, "Finetune/Step": 1304, "Finetune/Step Time": 4.488783897832036} +{"Finetune/Learning Rate": 1.7541712057509666e-05, "Finetune/Loss": 0.9642922878265381, "Finetune/Loss (Raw)": 0.9713556170463562, "Finetune/Step": 1305, "Finetune/Step Time": 4.454236375167966} +{"Finetune/Learning Rate": 1.7537619747076752e-05, "Finetune/Loss": 0.9695514440536499, "Finetune/Loss (Raw)": 1.0001511573791504, "Finetune/Step": 1306, "Finetune/Step Time": 4.433339303359389} +{"Finetune/Learning Rate": 1.7533524511440777e-05, "Finetune/Loss": 0.9689528942108154, "Finetune/Loss (Raw)": 1.0857497453689575, "Finetune/Step": 1307, "Finetune/Step Time": 4.426975667476654} +{"Finetune/Learning Rate": 1.7529426352191014e-05, "Finetune/Loss": 0.9700822830200195, "Finetune/Loss (Raw)": 1.0757060050964355, "Finetune/Step": 1308, "Finetune/Step Time": 4.409704431891441} +{"Finetune/Learning Rate": 1.752532527091789e-05, "Finetune/Loss": 0.9690678119659424, "Finetune/Loss (Raw)": 0.9574017524719238, "Finetune/Step": 1309, "Finetune/Step Time": 4.428327266126871} +{"Finetune/Learning Rate": 1.7521221269212944e-05, "Finetune/Loss": 0.9706621170043945, "Finetune/Loss (Raw)": 0.9121572375297546, "Finetune/Step": 1310, "Finetune/Step Time": 4.426460480317473} +{"Finetune/Learning Rate": 1.751711434866886e-05, "Finetune/Loss": 0.970439076423645, "Finetune/Loss (Raw)": 1.0082924365997314, "Finetune/Step": 1311, "Finetune/Step Time": 4.428512517362833} +{"Finetune/Learning Rate": 1.751300451087946e-05, "Finetune/Loss": 0.9700382947921753, "Finetune/Loss (Raw)": 0.7773175239562988, "Finetune/Step": 1312, "Finetune/Step Time": 4.42289225757122} +{"Finetune/Learning Rate": 1.7508891757439687e-05, "Finetune/Loss": 0.9700756072998047, "Finetune/Loss (Raw)": 1.0204569101333618, "Finetune/Step": 1313, "Finetune/Step Time": 4.429703371599317} +{"Finetune/Learning Rate": 1.7504776089945626e-05, "Finetune/Loss": 0.9691246747970581, "Finetune/Loss (Raw)": 1.0003606081008911, "Finetune/Step": 1314, "Finetune/Step Time": 4.433515353128314} +{"Finetune/Learning Rate": 1.7500657509994478e-05, "Finetune/Loss": 0.9741367101669312, "Finetune/Loss (Raw)": 0.9941679239273071, "Finetune/Step": 1315, "Finetune/Step Time": 4.397237569093704} +{"Finetune/Learning Rate": 1.7496536019184588e-05, "Finetune/Loss": 0.9749503135681152, "Finetune/Loss (Raw)": 1.1658962965011597, "Finetune/Step": 1316, "Finetune/Step Time": 4.402054304257035} +{"Finetune/Learning Rate": 1.749241161911543e-05, "Finetune/Loss": 0.9749166965484619, "Finetune/Loss (Raw)": 1.061037540435791, "Finetune/Step": 1317, "Finetune/Step Time": 4.394645366817713} +{"Finetune/Learning Rate": 1.7488284311387597e-05, "Finetune/Loss": 0.9747217893600464, "Finetune/Loss (Raw)": 0.9575549364089966, "Finetune/Step": 1318, "Finetune/Step Time": 4.3947647251188755} +{"Finetune/Learning Rate": 1.7484154097602822e-05, "Finetune/Loss": 0.9743050336837769, "Finetune/Loss (Raw)": 1.0000593662261963, "Finetune/Step": 1319, "Finetune/Step Time": 4.392208553850651} +{"Finetune/Learning Rate": 1.7480020979363956e-05, "Finetune/Loss": 0.9738985300064087, "Finetune/Loss (Raw)": 0.9765340685844421, "Finetune/Step": 1320, "Finetune/Step Time": 4.393517833203077} +{"Finetune/Learning Rate": 1.747588495827498e-05, "Finetune/Loss": 0.9738086462020874, "Finetune/Loss (Raw)": 1.012452244758606, "Finetune/Step": 1321, "Finetune/Step Time": 4.3965040519833565} +{"Finetune/Learning Rate": 1.7471746035941007e-05, "Finetune/Loss": 0.9731222987174988, "Finetune/Loss (Raw)": 0.982162356376648, "Finetune/Step": 1322, "Finetune/Step Time": 4.385479034855962} +{"Finetune/Learning Rate": 1.7467604213968274e-05, "Finetune/Loss": 0.9735637903213501, "Finetune/Loss (Raw)": 1.0531108379364014, "Finetune/Step": 1323, "Finetune/Step Time": 4.386232869699597} +{"Finetune/Learning Rate": 1.746345949396414e-05, "Finetune/Loss": 0.9738638401031494, "Finetune/Loss (Raw)": 1.0268704891204834, "Finetune/Step": 1324, "Finetune/Step Time": 4.38812674023211} +{"Finetune/Learning Rate": 1.7459311877537085e-05, "Finetune/Loss": 0.9734561443328857, "Finetune/Loss (Raw)": 1.0602823495864868, "Finetune/Step": 1325, "Finetune/Step Time": 4.405900159850717} +{"Finetune/Learning Rate": 1.745516136629672e-05, "Finetune/Loss": 0.9729681611061096, "Finetune/Loss (Raw)": 1.0370248556137085, "Finetune/Step": 1326, "Finetune/Step Time": 4.398731065914035} +{"Finetune/Learning Rate": 1.7451007961853778e-05, "Finetune/Loss": 0.9719857573509216, "Finetune/Loss (Raw)": 0.917553722858429, "Finetune/Step": 1327, "Finetune/Step Time": 4.436335373669863} +{"Finetune/Learning Rate": 1.7446851665820118e-05, "Finetune/Loss": 0.9727175235748291, "Finetune/Loss (Raw)": 1.149491548538208, "Finetune/Step": 1328, "Finetune/Step Time": 4.4318412356078625} +{"Finetune/Learning Rate": 1.744269247980871e-05, "Finetune/Loss": 0.9737603068351746, "Finetune/Loss (Raw)": 1.1826823949813843, "Finetune/Step": 1329, "Finetune/Step Time": 4.423688178882003} +{"Finetune/Learning Rate": 1.7438530405433663e-05, "Finetune/Loss": 0.9707541465759277, "Finetune/Loss (Raw)": 0.7777190208435059, "Finetune/Step": 1330, "Finetune/Step Time": 4.417200671508908} +{"Finetune/Learning Rate": 1.7434365444310185e-05, "Finetune/Loss": 0.9701997637748718, "Finetune/Loss (Raw)": 1.0537189245224, "Finetune/Step": 1331, "Finetune/Step Time": 4.41077920421958} +{"Finetune/Learning Rate": 1.7430197598054622e-05, "Finetune/Loss": 0.9685746431350708, "Finetune/Loss (Raw)": 0.8762593269348145, "Finetune/Step": 1332, "Finetune/Step Time": 4.4136339370161295} +{"Finetune/Learning Rate": 1.7426026868284438e-05, "Finetune/Loss": 0.9685711860656738, "Finetune/Loss (Raw)": 0.9658077955245972, "Finetune/Step": 1333, "Finetune/Step Time": 4.400275036692619} +{"Finetune/Learning Rate": 1.7421853256618204e-05, "Finetune/Loss": 0.9674851894378662, "Finetune/Loss (Raw)": 1.011550784111023, "Finetune/Step": 1334, "Finetune/Step Time": 4.404203148558736} +{"Finetune/Learning Rate": 1.741767676467562e-05, "Finetune/Loss": 0.9680812954902649, "Finetune/Loss (Raw)": 1.0363208055496216, "Finetune/Step": 1335, "Finetune/Step Time": 4.396388791501522} +{"Finetune/Learning Rate": 1.7413497394077502e-05, "Finetune/Loss": 0.9681745767593384, "Finetune/Loss (Raw)": 1.0307371616363525, "Finetune/Step": 1336, "Finetune/Step Time": 4.39179041236639} +{"Finetune/Learning Rate": 1.7409315146445783e-05, "Finetune/Loss": 0.9687668681144714, "Finetune/Loss (Raw)": 1.1322331428527832, "Finetune/Step": 1337, "Finetune/Step Time": 4.394590884447098} +{"Finetune/Learning Rate": 1.7405130023403513e-05, "Finetune/Loss": 0.96345055103302, "Finetune/Loss (Raw)": 0.31551507115364075, "Finetune/Step": 1338, "Finetune/Step Time": 4.43432111479342} +{"Finetune/Learning Rate": 1.740094202657485e-05, "Finetune/Loss": 0.9631749987602234, "Finetune/Loss (Raw)": 0.9144054651260376, "Finetune/Step": 1339, "Finetune/Step Time": 4.440004924312234} +{"Finetune/Learning Rate": 1.7396751157585078e-05, "Finetune/Loss": 0.9621639251708984, "Finetune/Loss (Raw)": 0.9969491958618164, "Finetune/Step": 1340, "Finetune/Step Time": 4.445979531854391} +{"Finetune/Learning Rate": 1.739255741806059e-05, "Finetune/Loss": 0.9621320962905884, "Finetune/Loss (Raw)": 0.9516119360923767, "Finetune/Step": 1341, "Finetune/Step Time": 4.447371957823634} +{"Finetune/Learning Rate": 1.73883608096289e-05, "Finetune/Loss": 0.9640988111495972, "Finetune/Loss (Raw)": 1.1093136072158813, "Finetune/Step": 1342, "Finetune/Step Time": 4.444433301687241} +{"Finetune/Learning Rate": 1.738416133391862e-05, "Finetune/Loss": 0.9634677171707153, "Finetune/Loss (Raw)": 0.9118210673332214, "Finetune/Step": 1343, "Finetune/Step Time": 4.466460967436433} +{"Finetune/Learning Rate": 1.7379958992559494e-05, "Finetune/Loss": 0.968137264251709, "Finetune/Loss (Raw)": 0.9512292146682739, "Finetune/Step": 1344, "Finetune/Step Time": 4.430022045969963} +{"Finetune/Learning Rate": 1.7375753787182364e-05, "Finetune/Loss": 0.967806339263916, "Finetune/Loss (Raw)": 1.0708974599838257, "Finetune/Step": 1345, "Finetune/Step Time": 4.429870251566172} +{"Finetune/Learning Rate": 1.7371545719419188e-05, "Finetune/Loss": 0.9670243263244629, "Finetune/Loss (Raw)": 0.9540804624557495, "Finetune/Step": 1346, "Finetune/Step Time": 4.431777440011501} +{"Finetune/Learning Rate": 1.7367334790903034e-05, "Finetune/Loss": 0.9737648367881775, "Finetune/Loss (Raw)": 1.171619176864624, "Finetune/Step": 1347, "Finetune/Step Time": 4.406136969104409} +{"Finetune/Learning Rate": 1.736312100326808e-05, "Finetune/Loss": 0.9694035649299622, "Finetune/Loss (Raw)": 0.43414559960365295, "Finetune/Step": 1348, "Finetune/Step Time": 4.4464032500982285} +{"Finetune/Learning Rate": 1.7358904358149625e-05, "Finetune/Loss": 0.969865083694458, "Finetune/Loss (Raw)": 1.0919255018234253, "Finetune/Step": 1349, "Finetune/Step Time": 4.420868428424001} +{"Finetune/Learning Rate": 1.7354684857184056e-05, "Finetune/Loss": 0.9701778888702393, "Finetune/Loss (Raw)": 1.0515823364257812, "Finetune/Step": 1350, "Finetune/Step Time": 4.424793912097812} +{"Finetune/Learning Rate": 1.735046250200888e-05, "Finetune/Loss": 0.9702789783477783, "Finetune/Loss (Raw)": 1.0407435894012451, "Finetune/Step": 1351, "Finetune/Step Time": 4.4342526365071535} +{"Finetune/Learning Rate": 1.7346237294262717e-05, "Finetune/Loss": 0.9657406806945801, "Finetune/Loss (Raw)": 0.42597419023513794, "Finetune/Step": 1352, "Finetune/Step Time": 4.471577487885952} +{"Finetune/Learning Rate": 1.734200923558528e-05, "Finetune/Loss": 0.9663777351379395, "Finetune/Loss (Raw)": 0.9563075304031372, "Finetune/Step": 1353, "Finetune/Step Time": 4.460831144824624} +{"Finetune/Learning Rate": 1.7337778327617397e-05, "Finetune/Loss": 0.9672118425369263, "Finetune/Loss (Raw)": 1.0070736408233643, "Finetune/Step": 1354, "Finetune/Step Time": 4.458045952022076} +{"Finetune/Learning Rate": 1.733354457200101e-05, "Finetune/Loss": 0.9690626859664917, "Finetune/Loss (Raw)": 1.1060431003570557, "Finetune/Step": 1355, "Finetune/Step Time": 4.451126102358103} +{"Finetune/Learning Rate": 1.7329307970379145e-05, "Finetune/Loss": 0.9692120552062988, "Finetune/Loss (Raw)": 1.069007158279419, "Finetune/Step": 1356, "Finetune/Step Time": 4.444894386455417} +{"Finetune/Learning Rate": 1.732506852439595e-05, "Finetune/Loss": 0.9691121578216553, "Finetune/Loss (Raw)": 1.0669070482254028, "Finetune/Step": 1357, "Finetune/Step Time": 4.4680310562253} +{"Finetune/Learning Rate": 1.7320826235696672e-05, "Finetune/Loss": 0.9736392498016357, "Finetune/Loss (Raw)": 0.9847233295440674, "Finetune/Step": 1358, "Finetune/Step Time": 4.429752744734287} +{"Finetune/Learning Rate": 1.7316581105927658e-05, "Finetune/Loss": 0.9716262817382812, "Finetune/Loss (Raw)": 0.8604823350906372, "Finetune/Step": 1359, "Finetune/Step Time": 4.443917006254196} +{"Finetune/Learning Rate": 1.731233313673636e-05, "Finetune/Loss": 0.9715427160263062, "Finetune/Loss (Raw)": 1.0027947425842285, "Finetune/Step": 1360, "Finetune/Step Time": 4.452160770073533} +{"Finetune/Learning Rate": 1.7308082329771335e-05, "Finetune/Loss": 0.9719607830047607, "Finetune/Loss (Raw)": 1.0903964042663574, "Finetune/Step": 1361, "Finetune/Step Time": 4.452793698757887} +{"Finetune/Learning Rate": 1.7303828686682233e-05, "Finetune/Loss": 0.9732146263122559, "Finetune/Loss (Raw)": 0.9674164056777954, "Finetune/Step": 1362, "Finetune/Step Time": 4.435006335377693} +{"Finetune/Learning Rate": 1.7299572209119813e-05, "Finetune/Loss": 0.9735448956489563, "Finetune/Loss (Raw)": 0.9969207048416138, "Finetune/Step": 1363, "Finetune/Step Time": 4.435990737751126} +{"Finetune/Learning Rate": 1.7295312898735933e-05, "Finetune/Loss": 0.972320020198822, "Finetune/Loss (Raw)": 0.9564602375030518, "Finetune/Step": 1364, "Finetune/Step Time": 4.470369962975383} +{"Finetune/Learning Rate": 1.7291050757183546e-05, "Finetune/Loss": 0.9714118242263794, "Finetune/Loss (Raw)": 0.9048852920532227, "Finetune/Step": 1365, "Finetune/Step Time": 4.46140955388546} +{"Finetune/Learning Rate": 1.7286785786116707e-05, "Finetune/Loss": 0.9696766138076782, "Finetune/Loss (Raw)": 0.9591005444526672, "Finetune/Step": 1366, "Finetune/Step Time": 4.450642993673682} +{"Finetune/Learning Rate": 1.7282517987190567e-05, "Finetune/Loss": 0.9692703485488892, "Finetune/Loss (Raw)": 0.9259553551673889, "Finetune/Step": 1367, "Finetune/Step Time": 4.469195062294602} +{"Finetune/Learning Rate": 1.7278247362061372e-05, "Finetune/Loss": 0.9697210788726807, "Finetune/Loss (Raw)": 1.0330753326416016, "Finetune/Step": 1368, "Finetune/Step Time": 4.463130738586187} +{"Finetune/Learning Rate": 1.7273973912386473e-05, "Finetune/Loss": 0.9709287881851196, "Finetune/Loss (Raw)": 1.0521045923233032, "Finetune/Step": 1369, "Finetune/Step Time": 4.47288903221488} +{"Finetune/Learning Rate": 1.726969763982431e-05, "Finetune/Loss": 0.970028817653656, "Finetune/Loss (Raw)": 0.853439211845398, "Finetune/Step": 1370, "Finetune/Step Time": 4.475799972191453} +{"Finetune/Learning Rate": 1.7265418546034424e-05, "Finetune/Loss": 0.9694069623947144, "Finetune/Loss (Raw)": 0.9760621786117554, "Finetune/Step": 1371, "Finetune/Step Time": 4.476458039134741} +{"Finetune/Learning Rate": 1.7261136632677445e-05, "Finetune/Loss": 0.9647963047027588, "Finetune/Loss (Raw)": 0.38529887795448303, "Finetune/Step": 1372, "Finetune/Step Time": 4.5110800843685865} +{"Finetune/Learning Rate": 1.7256851901415102e-05, "Finetune/Loss": 0.966018557548523, "Finetune/Loss (Raw)": 1.1285256147384644, "Finetune/Step": 1373, "Finetune/Step Time": 4.511227823793888} +{"Finetune/Learning Rate": 1.725256435391021e-05, "Finetune/Loss": 0.9661288857460022, "Finetune/Loss (Raw)": 0.9407128095626831, "Finetune/Step": 1374, "Finetune/Step Time": 4.5026146322488785} +{"Finetune/Learning Rate": 1.7248273991826692e-05, "Finetune/Loss": 0.965944230556488, "Finetune/Loss (Raw)": 1.0517990589141846, "Finetune/Step": 1375, "Finetune/Step Time": 4.50025693513453} +{"Finetune/Learning Rate": 1.7243980816829543e-05, "Finetune/Loss": 0.9620518684387207, "Finetune/Loss (Raw)": 0.3206510543823242, "Finetune/Step": 1376, "Finetune/Step Time": 4.530084518715739} +{"Finetune/Learning Rate": 1.723968483058487e-05, "Finetune/Loss": 0.9632179737091064, "Finetune/Loss (Raw)": 1.0703448057174683, "Finetune/Step": 1377, "Finetune/Step Time": 4.53843536041677} +{"Finetune/Learning Rate": 1.7235386034759853e-05, "Finetune/Loss": 0.9641490578651428, "Finetune/Loss (Raw)": 1.1557049751281738, "Finetune/Step": 1378, "Finetune/Step Time": 4.556778535246849} +{"Finetune/Learning Rate": 1.723108443102278e-05, "Finetune/Loss": 0.9618375897407532, "Finetune/Loss (Raw)": 0.7336209416389465, "Finetune/Step": 1379, "Finetune/Step Time": 4.578470300883055} +{"Finetune/Learning Rate": 1.722678002104301e-05, "Finetune/Loss": 0.9621406197547913, "Finetune/Loss (Raw)": 0.9411792755126953, "Finetune/Step": 1380, "Finetune/Step Time": 4.578101051971316} +{"Finetune/Learning Rate": 1.722247280649101e-05, "Finetune/Loss": 0.9631240367889404, "Finetune/Loss (Raw)": 1.0569586753845215, "Finetune/Step": 1381, "Finetune/Step Time": 4.563499636948109} +{"Finetune/Learning Rate": 1.7218162789038312e-05, "Finetune/Loss": 0.9631903171539307, "Finetune/Loss (Raw)": 0.9939159750938416, "Finetune/Step": 1382, "Finetune/Step Time": 4.563028270378709} +{"Finetune/Learning Rate": 1.7213849970357565e-05, "Finetune/Loss": 0.9617329835891724, "Finetune/Loss (Raw)": 0.9174873232841492, "Finetune/Step": 1383, "Finetune/Step Time": 4.567105941474438} +{"Finetune/Learning Rate": 1.7209534352122484e-05, "Finetune/Loss": 0.9613858461380005, "Finetune/Loss (Raw)": 0.9697403311729431, "Finetune/Step": 1384, "Finetune/Step Time": 4.569763172417879} +{"Finetune/Learning Rate": 1.720521593600787e-05, "Finetune/Loss": 0.9638369083404541, "Finetune/Loss (Raw)": 1.025101661682129, "Finetune/Step": 1385, "Finetune/Step Time": 4.546046115458012} +{"Finetune/Learning Rate": 1.7200894723689626e-05, "Finetune/Loss": 0.9636189937591553, "Finetune/Loss (Raw)": 0.9307506084442139, "Finetune/Step": 1386, "Finetune/Step Time": 4.5466556288301945} +{"Finetune/Learning Rate": 1.7196570716844724e-05, "Finetune/Loss": 0.9633796215057373, "Finetune/Loss (Raw)": 0.8690634369850159, "Finetune/Step": 1387, "Finetune/Step Time": 4.5409818179905415} +{"Finetune/Learning Rate": 1.7192243917151228e-05, "Finetune/Loss": 0.9622457027435303, "Finetune/Loss (Raw)": 1.0326766967773438, "Finetune/Step": 1388, "Finetune/Step Time": 4.536250397562981} +{"Finetune/Learning Rate": 1.7187914326288286e-05, "Finetune/Loss": 0.9635302424430847, "Finetune/Loss (Raw)": 1.0083208084106445, "Finetune/Step": 1389, "Finetune/Step Time": 4.53721509501338} +{"Finetune/Learning Rate": 1.718358194593612e-05, "Finetune/Loss": 0.9648997187614441, "Finetune/Loss (Raw)": 1.1709827184677124, "Finetune/Step": 1390, "Finetune/Step Time": 4.545465549454093} +{"Finetune/Learning Rate": 1.7179246777776053e-05, "Finetune/Loss": 0.9635319113731384, "Finetune/Loss (Raw)": 0.882361888885498, "Finetune/Step": 1391, "Finetune/Step Time": 4.561184884980321} +{"Finetune/Learning Rate": 1.7174908823490475e-05, "Finetune/Loss": 0.9628453254699707, "Finetune/Loss (Raw)": 0.9091518521308899, "Finetune/Step": 1392, "Finetune/Step Time": 4.594618471339345} +{"Finetune/Learning Rate": 1.717056808476286e-05, "Finetune/Loss": 0.967572033405304, "Finetune/Loss (Raw)": 0.9860449433326721, "Finetune/Step": 1393, "Finetune/Step Time": 4.594975762069225} +{"Finetune/Learning Rate": 1.716622456327776e-05, "Finetune/Loss": 0.9682328701019287, "Finetune/Loss (Raw)": 1.0443183183670044, "Finetune/Step": 1394, "Finetune/Step Time": 4.581563971936703} +{"Finetune/Learning Rate": 1.7161878260720826e-05, "Finetune/Loss": 0.967577338218689, "Finetune/Loss (Raw)": 0.9654129147529602, "Finetune/Step": 1395, "Finetune/Step Time": 4.58390723913908} +{"Finetune/Learning Rate": 1.7157529178778757e-05, "Finetune/Loss": 0.9650452733039856, "Finetune/Loss (Raw)": 0.752659797668457, "Finetune/Step": 1396, "Finetune/Step Time": 4.591801276430488} +{"Finetune/Learning Rate": 1.7153177319139352e-05, "Finetune/Loss": 0.9658111333847046, "Finetune/Loss (Raw)": 1.0552301406860352, "Finetune/Step": 1397, "Finetune/Step Time": 4.615279709920287} +{"Finetune/Learning Rate": 1.7148822683491483e-05, "Finetune/Loss": 0.9647595882415771, "Finetune/Loss (Raw)": 1.003964900970459, "Finetune/Step": 1398, "Finetune/Step Time": 4.60495688021183} +{"Finetune/Learning Rate": 1.7144465273525105e-05, "Finetune/Loss": 0.9626282453536987, "Finetune/Loss (Raw)": 0.8290324807167053, "Finetune/Step": 1399, "Finetune/Step Time": 4.611260890960693} +{"Finetune/Learning Rate": 1.7140105090931232e-05, "Finetune/Loss": 0.9644448757171631, "Finetune/Loss (Raw)": 1.0519315004348755, "Finetune/Step": 1400, "Finetune/Step Time": 4.60616772249341} +{"Finetune/Learning Rate": 1.7135742137401976e-05, "Finetune/Loss": 0.9643236398696899, "Finetune/Loss (Raw)": 0.9232825040817261, "Finetune/Step": 1401, "Finetune/Step Time": 4.604764223098755} +{"Finetune/Learning Rate": 1.7131376414630506e-05, "Finetune/Loss": 0.9606198072433472, "Finetune/Loss (Raw)": 0.3973486125469208, "Finetune/Step": 1402, "Finetune/Step Time": 4.643985761329532} +{"Finetune/Learning Rate": 1.712700792431108e-05, "Finetune/Loss": 0.9588168859481812, "Finetune/Loss (Raw)": 0.940093994140625, "Finetune/Step": 1403, "Finetune/Step Time": 4.644963791593909} +{"Finetune/Learning Rate": 1.7122636668139022e-05, "Finetune/Loss": 0.9590226411819458, "Finetune/Loss (Raw)": 0.9803785681724548, "Finetune/Step": 1404, "Finetune/Step Time": 4.641420243307948} +{"Finetune/Learning Rate": 1.711826264781073e-05, "Finetune/Loss": 0.9580857753753662, "Finetune/Loss (Raw)": 0.8522432446479797, "Finetune/Step": 1405, "Finetune/Step Time": 4.639427047222853} +{"Finetune/Learning Rate": 1.711388586502368e-05, "Finetune/Loss": 0.9598011374473572, "Finetune/Loss (Raw)": 0.9684898853302002, "Finetune/Step": 1406, "Finetune/Step Time": 4.632629655301571} +{"Finetune/Learning Rate": 1.710950632147641e-05, "Finetune/Loss": 0.9549310803413391, "Finetune/Loss (Raw)": 0.37584441900253296, "Finetune/Step": 1407, "Finetune/Step Time": 4.66456763073802} +{"Finetune/Learning Rate": 1.710512401886854e-05, "Finetune/Loss": 0.9526944160461426, "Finetune/Loss (Raw)": 0.8759467005729675, "Finetune/Step": 1408, "Finetune/Step Time": 4.6675893403589725} +{"Finetune/Learning Rate": 1.7100738958900752e-05, "Finetune/Loss": 0.9530448913574219, "Finetune/Loss (Raw)": 0.9998447299003601, "Finetune/Step": 1409, "Finetune/Step Time": 4.664999578148127} +{"Finetune/Learning Rate": 1.7096351143274804e-05, "Finetune/Loss": 0.9523825645446777, "Finetune/Loss (Raw)": 0.9058703780174255, "Finetune/Step": 1410, "Finetune/Step Time": 4.694170853123069} +{"Finetune/Learning Rate": 1.7091960573693523e-05, "Finetune/Loss": 0.9537835121154785, "Finetune/Loss (Raw)": 1.060490608215332, "Finetune/Step": 1411, "Finetune/Step Time": 4.677593544125557} +{"Finetune/Learning Rate": 1.70875672518608e-05, "Finetune/Loss": 0.9520226716995239, "Finetune/Loss (Raw)": 0.7503568530082703, "Finetune/Step": 1412, "Finetune/Step Time": 4.677348926663399} +{"Finetune/Learning Rate": 1.7083171179481603e-05, "Finetune/Loss": 0.9517779350280762, "Finetune/Loss (Raw)": 1.0296200513839722, "Finetune/Step": 1413, "Finetune/Step Time": 4.680935760959983} +{"Finetune/Learning Rate": 1.7078772358261957e-05, "Finetune/Loss": 0.9529026746749878, "Finetune/Loss (Raw)": 1.1066569089889526, "Finetune/Step": 1414, "Finetune/Step Time": 4.670364273712039} +{"Finetune/Learning Rate": 1.707437078990896e-05, "Finetune/Loss": 0.9529481530189514, "Finetune/Loss (Raw)": 0.986680805683136, "Finetune/Step": 1415, "Finetune/Step Time": 4.669489432126284} +{"Finetune/Learning Rate": 1.7069966476130773e-05, "Finetune/Loss": 0.9543789625167847, "Finetune/Loss (Raw)": 1.090554118156433, "Finetune/Step": 1416, "Finetune/Step Time": 4.668068241328001} +{"Finetune/Learning Rate": 1.706555941863663e-05, "Finetune/Loss": 0.9544806480407715, "Finetune/Loss (Raw)": 0.9260864853858948, "Finetune/Step": 1417, "Finetune/Step Time": 4.6552269365638494} +{"Finetune/Learning Rate": 1.7061149619136818e-05, "Finetune/Loss": 0.9525547623634338, "Finetune/Loss (Raw)": 0.8683311939239502, "Finetune/Step": 1418, "Finetune/Step Time": 4.662717562168837} +{"Finetune/Learning Rate": 1.7056737079342694e-05, "Finetune/Loss": 0.9516843557357788, "Finetune/Loss (Raw)": 0.8832592368125916, "Finetune/Step": 1419, "Finetune/Step Time": 4.65104154497385} +{"Finetune/Learning Rate": 1.7052321800966684e-05, "Finetune/Loss": 0.9519644975662231, "Finetune/Loss (Raw)": 1.0058107376098633, "Finetune/Step": 1420, "Finetune/Step Time": 4.6628419272601604} +{"Finetune/Learning Rate": 1.7047903785722268e-05, "Finetune/Loss": 0.9517099857330322, "Finetune/Loss (Raw)": 0.9204003810882568, "Finetune/Step": 1421, "Finetune/Step Time": 4.6631696708500385} +{"Finetune/Learning Rate": 1.704348303532399e-05, "Finetune/Loss": 0.9513351321220398, "Finetune/Loss (Raw)": 0.9434385299682617, "Finetune/Step": 1422, "Finetune/Step Time": 4.659571930766106} +{"Finetune/Learning Rate": 1.7039059551487457e-05, "Finetune/Loss": 0.9522672891616821, "Finetune/Loss (Raw)": 1.0470787286758423, "Finetune/Step": 1423, "Finetune/Step Time": 4.649861304089427} +{"Finetune/Learning Rate": 1.703463333592934e-05, "Finetune/Loss": 0.9513893723487854, "Finetune/Loss (Raw)": 0.8583892583847046, "Finetune/Step": 1424, "Finetune/Step Time": 4.644382514059544} +{"Finetune/Learning Rate": 1.7030204390367368e-05, "Finetune/Loss": 0.951417863368988, "Finetune/Loss (Raw)": 1.0051977634429932, "Finetune/Step": 1425, "Finetune/Step Time": 4.6381547171622515} +{"Finetune/Learning Rate": 1.7025772716520324e-05, "Finetune/Loss": 0.9516537189483643, "Finetune/Loss (Raw)": 0.8863925933837891, "Finetune/Step": 1426, "Finetune/Step Time": 4.651229355484247} +{"Finetune/Learning Rate": 1.702133831610805e-05, "Finetune/Loss": 0.9532089829444885, "Finetune/Loss (Raw)": 0.9484397768974304, "Finetune/Step": 1427, "Finetune/Step Time": 4.656375641003251} +{"Finetune/Learning Rate": 1.701690119085146e-05, "Finetune/Loss": 0.9535079002380371, "Finetune/Loss (Raw)": 0.935920000076294, "Finetune/Step": 1428, "Finetune/Step Time": 4.662406241521239} +{"Finetune/Learning Rate": 1.701246134247251e-05, "Finetune/Loss": 0.9523527026176453, "Finetune/Loss (Raw)": 0.8767884373664856, "Finetune/Step": 1429, "Finetune/Step Time": 4.653034640476108} +{"Finetune/Learning Rate": 1.700801877269422e-05, "Finetune/Loss": 0.9524511098861694, "Finetune/Loss (Raw)": 0.9573427438735962, "Finetune/Step": 1430, "Finetune/Step Time": 4.644434377551079} +{"Finetune/Learning Rate": 1.7003573483240666e-05, "Finetune/Loss": 0.9517502188682556, "Finetune/Loss (Raw)": 0.898975670337677, "Finetune/Step": 1431, "Finetune/Step Time": 4.655227327719331} +{"Finetune/Learning Rate": 1.6999125475836973e-05, "Finetune/Loss": 0.9514158964157104, "Finetune/Loss (Raw)": 0.9615945219993591, "Finetune/Step": 1432, "Finetune/Step Time": 4.658433837816119} +{"Finetune/Learning Rate": 1.6994674752209334e-05, "Finetune/Loss": 0.9516054391860962, "Finetune/Loss (Raw)": 0.9956107139587402, "Finetune/Step": 1433, "Finetune/Step Time": 4.660139463841915} +{"Finetune/Learning Rate": 1.6990221314084976e-05, "Finetune/Loss": 0.9467438459396362, "Finetune/Loss (Raw)": 0.37786370515823364, "Finetune/Step": 1434, "Finetune/Step Time": 4.681782485917211} +{"Finetune/Learning Rate": 1.6985765163192204e-05, "Finetune/Loss": 0.9462350606918335, "Finetune/Loss (Raw)": 1.020637035369873, "Finetune/Step": 1435, "Finetune/Step Time": 4.689192159101367} +{"Finetune/Learning Rate": 1.698130630126036e-05, "Finetune/Loss": 0.944170355796814, "Finetune/Loss (Raw)": 0.8114149570465088, "Finetune/Step": 1436, "Finetune/Step Time": 4.699910765513778} +{"Finetune/Learning Rate": 1.697684473001983e-05, "Finetune/Loss": 0.9449355602264404, "Finetune/Loss (Raw)": 1.0553468465805054, "Finetune/Step": 1437, "Finetune/Step Time": 4.6825416553765535} +{"Finetune/Learning Rate": 1.6972380451202083e-05, "Finetune/Loss": 0.9455230236053467, "Finetune/Loss (Raw)": 0.987360417842865, "Finetune/Step": 1438, "Finetune/Step Time": 4.677900919690728} +{"Finetune/Learning Rate": 1.69679134665396e-05, "Finetune/Loss": 0.9457725286483765, "Finetune/Loss (Raw)": 1.0402286052703857, "Finetune/Step": 1439, "Finetune/Step Time": 4.674501191824675} +{"Finetune/Learning Rate": 1.696344377776594e-05, "Finetune/Loss": 0.9452903866767883, "Finetune/Loss (Raw)": 0.7156046628952026, "Finetune/Step": 1440, "Finetune/Step Time": 4.679751042276621} +{"Finetune/Learning Rate": 1.69589713866157e-05, "Finetune/Loss": 0.9446969032287598, "Finetune/Loss (Raw)": 0.9444913268089294, "Finetune/Step": 1441, "Finetune/Step Time": 4.675791595131159} +{"Finetune/Learning Rate": 1.6954496294824526e-05, "Finetune/Loss": 0.9446312189102173, "Finetune/Loss (Raw)": 0.9919427037239075, "Finetune/Step": 1442, "Finetune/Step Time": 4.675002049654722} +{"Finetune/Learning Rate": 1.6950018504129112e-05, "Finetune/Loss": 0.9440314769744873, "Finetune/Loss (Raw)": 0.9174020886421204, "Finetune/Step": 1443, "Finetune/Step Time": 4.678437082096934} +{"Finetune/Learning Rate": 1.6945538016267206e-05, "Finetune/Loss": 0.9420876502990723, "Finetune/Loss (Raw)": 0.9170955419540405, "Finetune/Step": 1444, "Finetune/Step Time": 4.675722256302834} +{"Finetune/Learning Rate": 1.6941054832977598e-05, "Finetune/Loss": 0.9410128593444824, "Finetune/Loss (Raw)": 0.9234568476676941, "Finetune/Step": 1445, "Finetune/Step Time": 4.678415359929204} +{"Finetune/Learning Rate": 1.6936568956000123e-05, "Finetune/Loss": 0.9402171969413757, "Finetune/Loss (Raw)": 0.8557172417640686, "Finetune/Step": 1446, "Finetune/Step Time": 4.693454558029771} +{"Finetune/Learning Rate": 1.693208038707566e-05, "Finetune/Loss": 0.9399942755699158, "Finetune/Loss (Raw)": 0.971523106098175, "Finetune/Step": 1447, "Finetune/Step Time": 4.698050471022725} +{"Finetune/Learning Rate": 1.6927589127946136e-05, "Finetune/Loss": 0.9406948089599609, "Finetune/Loss (Raw)": 1.0662022829055786, "Finetune/Step": 1448, "Finetune/Step Time": 4.689402837306261} +{"Finetune/Learning Rate": 1.692309518035452e-05, "Finetune/Loss": 0.9394806623458862, "Finetune/Loss (Raw)": 0.857039749622345, "Finetune/Step": 1449, "Finetune/Step Time": 4.6920173820108175} +{"Finetune/Learning Rate": 1.691859854604483e-05, "Finetune/Loss": 0.9386733770370483, "Finetune/Loss (Raw)": 0.8788288235664368, "Finetune/Step": 1450, "Finetune/Step Time": 4.7336620185524225} +{"Finetune/Learning Rate": 1.6914099226762116e-05, "Finetune/Loss": 0.9376113414764404, "Finetune/Loss (Raw)": 0.9171777963638306, "Finetune/Step": 1451, "Finetune/Step Time": 4.736720148473978} +{"Finetune/Learning Rate": 1.6909597224252478e-05, "Finetune/Loss": 0.937947154045105, "Finetune/Loss (Raw)": 1.0698463916778564, "Finetune/Step": 1452, "Finetune/Step Time": 4.728972563520074} +{"Finetune/Learning Rate": 1.6905092540263055e-05, "Finetune/Loss": 0.9367920160293579, "Finetune/Loss (Raw)": 0.9124258756637573, "Finetune/Step": 1453, "Finetune/Step Time": 4.723840607330203} +{"Finetune/Learning Rate": 1.690058517654203e-05, "Finetune/Loss": 0.9380799531936646, "Finetune/Loss (Raw)": 1.2018769979476929, "Finetune/Step": 1454, "Finetune/Step Time": 4.726559866219759} +{"Finetune/Learning Rate": 1.689607513483862e-05, "Finetune/Loss": 0.9388422966003418, "Finetune/Loss (Raw)": 1.015134334564209, "Finetune/Step": 1455, "Finetune/Step Time": 4.693689584732056} +{"Finetune/Learning Rate": 1.6891562416903083e-05, "Finetune/Loss": 0.9371979236602783, "Finetune/Loss (Raw)": 0.9390214681625366, "Finetune/Step": 1456, "Finetune/Step Time": 4.695654513314366} +{"Finetune/Learning Rate": 1.688704702448672e-05, "Finetune/Loss": 0.9309031963348389, "Finetune/Loss (Raw)": 0.37695184350013733, "Finetune/Step": 1457, "Finetune/Step Time": 4.731640694662929} +{"Finetune/Learning Rate": 1.6882528959341865e-05, "Finetune/Loss": 0.9322495460510254, "Finetune/Loss (Raw)": 0.9500571489334106, "Finetune/Step": 1458, "Finetune/Step Time": 4.728790504857898} +{"Finetune/Learning Rate": 1.6878008223221895e-05, "Finetune/Loss": 0.9325940608978271, "Finetune/Loss (Raw)": 1.0978089570999146, "Finetune/Step": 1459, "Finetune/Step Time": 4.737548511475325} +{"Finetune/Learning Rate": 1.687348481788121e-05, "Finetune/Loss": 0.934343695640564, "Finetune/Loss (Raw)": 1.1002140045166016, "Finetune/Step": 1460, "Finetune/Step Time": 4.727190785109997} +{"Finetune/Learning Rate": 1.6868958745075262e-05, "Finetune/Loss": 0.9355644583702087, "Finetune/Loss (Raw)": 1.122067928314209, "Finetune/Step": 1461, "Finetune/Step Time": 4.72055190615356} +{"Finetune/Learning Rate": 1.6864430006560527e-05, "Finetune/Loss": 0.9358226656913757, "Finetune/Loss (Raw)": 1.0445988178253174, "Finetune/Step": 1462, "Finetune/Step Time": 4.722668882459402} +{"Finetune/Learning Rate": 1.685989860409453e-05, "Finetune/Loss": 0.9359676837921143, "Finetune/Loss (Raw)": 1.0548828840255737, "Finetune/Step": 1463, "Finetune/Step Time": 4.723537232726812} +{"Finetune/Learning Rate": 1.6855364539435805e-05, "Finetune/Loss": 0.9348664283752441, "Finetune/Loss (Raw)": 0.8897833228111267, "Finetune/Step": 1464, "Finetune/Step Time": 4.7286283522844315} +{"Finetune/Learning Rate": 1.685082781434395e-05, "Finetune/Loss": 0.9336903095245361, "Finetune/Loss (Raw)": 0.9816797971725464, "Finetune/Step": 1465, "Finetune/Step Time": 4.721514604985714} +{"Finetune/Learning Rate": 1.6846288430579565e-05, "Finetune/Loss": 0.9388260841369629, "Finetune/Loss (Raw)": 0.9729053974151611, "Finetune/Step": 1466, "Finetune/Step Time": 4.688910225406289} +{"Finetune/Learning Rate": 1.6841746389904306e-05, "Finetune/Loss": 0.9391368627548218, "Finetune/Loss (Raw)": 0.9541783928871155, "Finetune/Step": 1467, "Finetune/Step Time": 4.706318957731128} +{"Finetune/Learning Rate": 1.683720169408085e-05, "Finetune/Loss": 0.9393908977508545, "Finetune/Loss (Raw)": 1.0294647216796875, "Finetune/Step": 1468, "Finetune/Step Time": 4.698963150382042} +{"Finetune/Learning Rate": 1.6832654344872893e-05, "Finetune/Loss": 0.9350127577781677, "Finetune/Loss (Raw)": 0.3912111818790436, "Finetune/Step": 1469, "Finetune/Step Time": 4.740428190678358} +{"Finetune/Learning Rate": 1.6828104344045186e-05, "Finetune/Loss": 0.9340928792953491, "Finetune/Loss (Raw)": 0.9915720224380493, "Finetune/Step": 1470, "Finetune/Step Time": 4.728277483955026} +{"Finetune/Learning Rate": 1.682355169336349e-05, "Finetune/Loss": 0.9347604513168335, "Finetune/Loss (Raw)": 0.9972606301307678, "Finetune/Step": 1471, "Finetune/Step Time": 4.702740728855133} +{"Finetune/Learning Rate": 1.6818996394594603e-05, "Finetune/Loss": 0.9341100454330444, "Finetune/Loss (Raw)": 0.8679796457290649, "Finetune/Step": 1472, "Finetune/Step Time": 4.6957911886274815} +{"Finetune/Learning Rate": 1.681443844950634e-05, "Finetune/Loss": 0.9313002228736877, "Finetune/Loss (Raw)": 0.7112417817115784, "Finetune/Step": 1473, "Finetune/Step Time": 4.702147997915745} +{"Finetune/Learning Rate": 1.6809877859867557e-05, "Finetune/Loss": 0.9326875805854797, "Finetune/Loss (Raw)": 1.1316683292388916, "Finetune/Step": 1474, "Finetune/Step Time": 4.696879161521792} +{"Finetune/Learning Rate": 1.680531462744813e-05, "Finetune/Loss": 0.9294955730438232, "Finetune/Loss (Raw)": 0.7630329728126526, "Finetune/Step": 1475, "Finetune/Step Time": 4.689774915575981} +{"Finetune/Learning Rate": 1.680074875401896e-05, "Finetune/Loss": 0.9339765906333923, "Finetune/Loss (Raw)": 1.0077176094055176, "Finetune/Step": 1476, "Finetune/Step Time": 4.649093030020595} +{"Finetune/Learning Rate": 1.679618024135197e-05, "Finetune/Loss": 0.9279030561447144, "Finetune/Loss (Raw)": 0.3145209550857544, "Finetune/Step": 1477, "Finetune/Step Time": 4.692318845540285} +{"Finetune/Learning Rate": 1.679160909122011e-05, "Finetune/Loss": 0.9280765056610107, "Finetune/Loss (Raw)": 1.073779821395874, "Finetune/Step": 1478, "Finetune/Step Time": 4.679002711549401} +{"Finetune/Learning Rate": 1.6787035305397363e-05, "Finetune/Loss": 0.9270293712615967, "Finetune/Loss (Raw)": 0.906708836555481, "Finetune/Step": 1479, "Finetune/Step Time": 4.687842361629009} +{"Finetune/Learning Rate": 1.6782458885658716e-05, "Finetune/Loss": 0.9313161373138428, "Finetune/Loss (Raw)": 0.9746803045272827, "Finetune/Step": 1480, "Finetune/Step Time": 4.654829487204552} +{"Finetune/Learning Rate": 1.6777879833780192e-05, "Finetune/Loss": 0.931613028049469, "Finetune/Loss (Raw)": 0.9943077564239502, "Finetune/Step": 1481, "Finetune/Step Time": 4.66615610383451} +{"Finetune/Learning Rate": 1.677329815153883e-05, "Finetune/Loss": 0.9312648773193359, "Finetune/Loss (Raw)": 0.9625157713890076, "Finetune/Step": 1482, "Finetune/Step Time": 4.6634634137153625} +{"Finetune/Learning Rate": 1.6768713840712692e-05, "Finetune/Loss": 0.9298821687698364, "Finetune/Loss (Raw)": 0.9290554523468018, "Finetune/Step": 1483, "Finetune/Step Time": 4.6935875080525875} +{"Finetune/Learning Rate": 1.6764126903080865e-05, "Finetune/Loss": 0.9296417236328125, "Finetune/Loss (Raw)": 1.0382311344146729, "Finetune/Step": 1484, "Finetune/Step Time": 4.689579186961055} +{"Finetune/Learning Rate": 1.675953734042344e-05, "Finetune/Loss": 0.928451657295227, "Finetune/Loss (Raw)": 0.9145759344100952, "Finetune/Step": 1485, "Finetune/Step Time": 4.667150128632784} +{"Finetune/Learning Rate": 1.6754945154521548e-05, "Finetune/Loss": 0.9231829643249512, "Finetune/Loss (Raw)": 0.3103235363960266, "Finetune/Step": 1486, "Finetune/Step Time": 4.704800287261605} +{"Finetune/Learning Rate": 1.6750350347157315e-05, "Finetune/Loss": 0.9250831604003906, "Finetune/Loss (Raw)": 1.1037163734436035, "Finetune/Step": 1487, "Finetune/Step Time": 4.705519394949079} +{"Finetune/Learning Rate": 1.6745752920113907e-05, "Finetune/Loss": 0.9266149997711182, "Finetune/Loss (Raw)": 1.198872447013855, "Finetune/Step": 1488, "Finetune/Step Time": 4.697934255003929} +{"Finetune/Learning Rate": 1.6741152875175485e-05, "Finetune/Loss": 0.9244534373283386, "Finetune/Loss (Raw)": 0.8137097358703613, "Finetune/Step": 1489, "Finetune/Step Time": 4.712284145876765} +{"Finetune/Learning Rate": 1.6736550214127247e-05, "Finetune/Loss": 0.924318253993988, "Finetune/Loss (Raw)": 0.9501153230667114, "Finetune/Step": 1490, "Finetune/Step Time": 4.733482036739588} +{"Finetune/Learning Rate": 1.673194493875539e-05, "Finetune/Loss": 0.9230741262435913, "Finetune/Loss (Raw)": 0.8376703262329102, "Finetune/Step": 1491, "Finetune/Step Time": 4.752118740230799} +{"Finetune/Learning Rate": 1.672733705084713e-05, "Finetune/Loss": 0.9240609407424927, "Finetune/Loss (Raw)": 1.082772135734558, "Finetune/Step": 1492, "Finetune/Step Time": 4.718363856896758} +{"Finetune/Learning Rate": 1.67227265521907e-05, "Finetune/Loss": 0.9196231365203857, "Finetune/Loss (Raw)": 0.3368527889251709, "Finetune/Step": 1493, "Finetune/Step Time": 4.752050848677754} +{"Finetune/Learning Rate": 1.671811344457535e-05, "Finetune/Loss": 0.919489860534668, "Finetune/Loss (Raw)": 0.9420332908630371, "Finetune/Step": 1494, "Finetune/Step Time": 4.756149107590318} +{"Finetune/Learning Rate": 1.671349772979133e-05, "Finetune/Loss": 0.9204027652740479, "Finetune/Loss (Raw)": 1.0428088903427124, "Finetune/Step": 1495, "Finetune/Step Time": 4.747490679845214} +{"Finetune/Learning Rate": 1.6708879409629908e-05, "Finetune/Loss": 0.9200564622879028, "Finetune/Loss (Raw)": 0.9887524843215942, "Finetune/Step": 1496, "Finetune/Step Time": 4.748055653646588} +{"Finetune/Learning Rate": 1.670425848588337e-05, "Finetune/Loss": 0.9184480905532837, "Finetune/Loss (Raw)": 0.8462269306182861, "Finetune/Step": 1497, "Finetune/Step Time": 4.742171430960298} +{"Finetune/Learning Rate": 1.6699634960345005e-05, "Finetune/Loss": 0.9201822280883789, "Finetune/Loss (Raw)": 1.0754088163375854, "Finetune/Step": 1498, "Finetune/Step Time": 4.744638528674841} +{"Finetune/Learning Rate": 1.6695008834809107e-05, "Finetune/Loss": 0.9159103631973267, "Finetune/Loss (Raw)": 0.4292663633823395, "Finetune/Step": 1499, "Finetune/Step Time": 4.776206523180008} +{"Finetune/Learning Rate": 1.669038011107099e-05, "Finetune/Loss": 0.9156742691993713, "Finetune/Loss (Raw)": 0.35508424043655396, "Finetune/Step": 1500, "Finetune/Step Time": 4.775961447507143} +{"Finetune/Learning Rate": 1.6685748790926975e-05, "Finetune/Loss": 0.9150280952453613, "Finetune/Loss (Raw)": 1.0458186864852905, "Finetune/Step": 1501, "Finetune/Step Time": 4.808607617393136} +{"Finetune/Learning Rate": 1.6681114876174377e-05, "Finetune/Loss": 0.9163576364517212, "Finetune/Loss (Raw)": 1.1108882427215576, "Finetune/Step": 1502, "Finetune/Step Time": 4.8007149156183} +{"Finetune/Learning Rate": 1.6676478368611536e-05, "Finetune/Loss": 0.9149417877197266, "Finetune/Loss (Raw)": 0.870570182800293, "Finetune/Step": 1503, "Finetune/Step Time": 4.803438533097506} +{"Finetune/Learning Rate": 1.667183927003779e-05, "Finetune/Loss": 0.9203057289123535, "Finetune/Loss (Raw)": 1.0072351694107056, "Finetune/Step": 1504, "Finetune/Step Time": 4.776988323777914} +{"Finetune/Learning Rate": 1.6667197582253474e-05, "Finetune/Loss": 0.9196926355361938, "Finetune/Loss (Raw)": 0.9918728470802307, "Finetune/Step": 1505, "Finetune/Step Time": 4.763174777850509} +{"Finetune/Learning Rate": 1.6662553307059947e-05, "Finetune/Loss": 0.917586624622345, "Finetune/Loss (Raw)": 0.8861262798309326, "Finetune/Step": 1506, "Finetune/Step Time": 4.7526686023920774} +{"Finetune/Learning Rate": 1.6657906446259554e-05, "Finetune/Loss": 0.9201446771621704, "Finetune/Loss (Raw)": 1.0610496997833252, "Finetune/Step": 1507, "Finetune/Step Time": 4.72577996365726} +{"Finetune/Learning Rate": 1.6653257001655652e-05, "Finetune/Loss": 0.9207506775856018, "Finetune/Loss (Raw)": 1.0187580585479736, "Finetune/Step": 1508, "Finetune/Step Time": 4.729705534875393} +{"Finetune/Learning Rate": 1.664860497505261e-05, "Finetune/Loss": 0.9204385280609131, "Finetune/Loss (Raw)": 1.017006754875183, "Finetune/Step": 1509, "Finetune/Step Time": 4.723426437005401} +{"Finetune/Learning Rate": 1.664395036825577e-05, "Finetune/Loss": 0.9211899042129517, "Finetune/Loss (Raw)": 1.090084433555603, "Finetune/Step": 1510, "Finetune/Step Time": 4.722338762134314} +{"Finetune/Learning Rate": 1.6639293183071514e-05, "Finetune/Loss": 0.9227515459060669, "Finetune/Loss (Raw)": 1.1173769235610962, "Finetune/Step": 1511, "Finetune/Step Time": 4.721584873273969} +{"Finetune/Learning Rate": 1.6634633421307187e-05, "Finetune/Loss": 0.9223470687866211, "Finetune/Loss (Raw)": 0.9179665446281433, "Finetune/Step": 1512, "Finetune/Step Time": 4.7252830024808645} +{"Finetune/Learning Rate": 1.6629971084771165e-05, "Finetune/Loss": 0.9233797788619995, "Finetune/Loss (Raw)": 1.157287836074829, "Finetune/Step": 1513, "Finetune/Step Time": 4.727246845141053} +{"Finetune/Learning Rate": 1.66253061752728e-05, "Finetune/Loss": 0.9237586855888367, "Finetune/Loss (Raw)": 0.9792516827583313, "Finetune/Step": 1514, "Finetune/Step Time": 4.736875446513295} +{"Finetune/Learning Rate": 1.662063869462246e-05, "Finetune/Loss": 0.9244675636291504, "Finetune/Loss (Raw)": 0.9598039984703064, "Finetune/Step": 1515, "Finetune/Step Time": 4.735176494345069} +{"Finetune/Learning Rate": 1.6615968644631498e-05, "Finetune/Loss": 0.9239014387130737, "Finetune/Loss (Raw)": 0.9602110385894775, "Finetune/Step": 1516, "Finetune/Step Time": 4.737889086827636} +{"Finetune/Learning Rate": 1.661129602711227e-05, "Finetune/Loss": 0.9228622913360596, "Finetune/Loss (Raw)": 0.8753154277801514, "Finetune/Step": 1517, "Finetune/Step Time": 4.746020920574665} +{"Finetune/Learning Rate": 1.6606620843878128e-05, "Finetune/Loss": 0.9221857786178589, "Finetune/Loss (Raw)": 1.084381103515625, "Finetune/Step": 1518, "Finetune/Step Time": 4.739729896187782} +{"Finetune/Learning Rate": 1.6601943096743426e-05, "Finetune/Loss": 0.9218825101852417, "Finetune/Loss (Raw)": 0.8435443043708801, "Finetune/Step": 1519, "Finetune/Step Time": 4.72493689507246} +{"Finetune/Learning Rate": 1.6597262787523493e-05, "Finetune/Loss": 0.91785728931427, "Finetune/Loss (Raw)": 0.39391905069351196, "Finetune/Step": 1520, "Finetune/Step Time": 4.726565508171916} +{"Finetune/Learning Rate": 1.6592579918034678e-05, "Finetune/Loss": 0.9183797240257263, "Finetune/Loss (Raw)": 1.0529232025146484, "Finetune/Step": 1521, "Finetune/Step Time": 4.689041793346405} +{"Finetune/Learning Rate": 1.65878944900943e-05, "Finetune/Loss": 0.917453944683075, "Finetune/Loss (Raw)": 0.9258120656013489, "Finetune/Step": 1522, "Finetune/Step Time": 4.700801840052009} +{"Finetune/Learning Rate": 1.6583206505520695e-05, "Finetune/Loss": 0.9163953065872192, "Finetune/Loss (Raw)": 0.8299094438552856, "Finetune/Step": 1523, "Finetune/Step Time": 4.714954994618893} +{"Finetune/Learning Rate": 1.6578515966133167e-05, "Finetune/Loss": 0.9176080822944641, "Finetune/Loss (Raw)": 0.9078955054283142, "Finetune/Step": 1524, "Finetune/Step Time": 4.7328334543854} +{"Finetune/Learning Rate": 1.6573822873752026e-05, "Finetune/Loss": 0.916232705116272, "Finetune/Loss (Raw)": 0.8791853189468384, "Finetune/Step": 1525, "Finetune/Step Time": 4.721189949661493} +{"Finetune/Learning Rate": 1.6569127230198575e-05, "Finetune/Loss": 0.9169065952301025, "Finetune/Loss (Raw)": 1.0902196168899536, "Finetune/Step": 1526, "Finetune/Step Time": 4.72026096843183} +{"Finetune/Learning Rate": 1.6564429037295097e-05, "Finetune/Loss": 0.917822003364563, "Finetune/Loss (Raw)": 0.9461986422538757, "Finetune/Step": 1527, "Finetune/Step Time": 4.7147705517709255} +{"Finetune/Learning Rate": 1.6559728296864868e-05, "Finetune/Loss": 0.9128687381744385, "Finetune/Loss (Raw)": 0.41792038083076477, "Finetune/Step": 1528, "Finetune/Step Time": 4.746682887896895} +{"Finetune/Learning Rate": 1.6555025010732156e-05, "Finetune/Loss": 0.9139021635055542, "Finetune/Loss (Raw)": 1.0555590391159058, "Finetune/Step": 1529, "Finetune/Step Time": 4.73733096010983} +{"Finetune/Learning Rate": 1.6550319180722213e-05, "Finetune/Loss": 0.9167748689651489, "Finetune/Loss (Raw)": 0.7650554776191711, "Finetune/Step": 1530, "Finetune/Step Time": 4.718668011948466} +{"Finetune/Learning Rate": 1.6545610808661278e-05, "Finetune/Loss": 0.9172961711883545, "Finetune/Loss (Raw)": 1.0068304538726807, "Finetune/Step": 1531, "Finetune/Step Time": 4.722137842327356} +{"Finetune/Learning Rate": 1.654089989637658e-05, "Finetune/Loss": 0.9171596765518188, "Finetune/Loss (Raw)": 0.9628977179527283, "Finetune/Step": 1532, "Finetune/Step Time": 4.73158179409802} +{"Finetune/Learning Rate": 1.6536186445696337e-05, "Finetune/Loss": 0.9175058603286743, "Finetune/Loss (Raw)": 0.8965569138526917, "Finetune/Step": 1533, "Finetune/Step Time": 4.725242307409644} +{"Finetune/Learning Rate": 1.653147045844974e-05, "Finetune/Loss": 0.9175332188606262, "Finetune/Loss (Raw)": 0.9719917178153992, "Finetune/Step": 1534, "Finetune/Step Time": 4.732831325381994} +{"Finetune/Learning Rate": 1.6526751936466974e-05, "Finetune/Loss": 0.9215925931930542, "Finetune/Loss (Raw)": 0.8954466581344604, "Finetune/Step": 1535, "Finetune/Step Time": 4.697253789752722} +{"Finetune/Learning Rate": 1.6522030881579207e-05, "Finetune/Loss": 0.9217916131019592, "Finetune/Loss (Raw)": 0.9014163613319397, "Finetune/Step": 1536, "Finetune/Step Time": 4.702784122899175} +{"Finetune/Learning Rate": 1.6517307295618582e-05, "Finetune/Loss": 0.9210429191589355, "Finetune/Loss (Raw)": 0.9040173888206482, "Finetune/Step": 1537, "Finetune/Step Time": 4.703254710882902} +{"Finetune/Learning Rate": 1.6512581180418245e-05, "Finetune/Loss": 0.9173869490623474, "Finetune/Loss (Raw)": 0.43790337443351746, "Finetune/Step": 1538, "Finetune/Step Time": 4.709479695186019} +{"Finetune/Learning Rate": 1.650785253781229e-05, "Finetune/Loss": 0.9168356657028198, "Finetune/Loss (Raw)": 0.9899347424507141, "Finetune/Step": 1539, "Finetune/Step Time": 4.710521053522825} +{"Finetune/Learning Rate": 1.6503121369635822e-05, "Finetune/Loss": 0.9186564683914185, "Finetune/Loss (Raw)": 0.9834066033363342, "Finetune/Step": 1540, "Finetune/Step Time": 4.710173266008496} +{"Finetune/Learning Rate": 1.6498387677724917e-05, "Finetune/Loss": 0.9190788269042969, "Finetune/Loss (Raw)": 1.0836882591247559, "Finetune/Step": 1541, "Finetune/Step Time": 4.705274846404791} +{"Finetune/Learning Rate": 1.6493651463916622e-05, "Finetune/Loss": 0.9175916910171509, "Finetune/Loss (Raw)": 0.9163061380386353, "Finetune/Step": 1542, "Finetune/Step Time": 4.713012024760246} +{"Finetune/Learning Rate": 1.6488912730048973e-05, "Finetune/Loss": 0.9165376424789429, "Finetune/Loss (Raw)": 0.8517531156539917, "Finetune/Step": 1543, "Finetune/Step Time": 4.708780221641064} +{"Finetune/Learning Rate": 1.6484171477960978e-05, "Finetune/Loss": 0.9157131910324097, "Finetune/Loss (Raw)": 0.9850329756736755, "Finetune/Step": 1544, "Finetune/Step Time": 4.701862515881658} +{"Finetune/Learning Rate": 1.6479427709492622e-05, "Finetune/Loss": 0.915625274181366, "Finetune/Loss (Raw)": 0.914831817150116, "Finetune/Step": 1545, "Finetune/Step Time": 4.704287812113762} +{"Finetune/Learning Rate": 1.6474681426484878e-05, "Finetune/Loss": 0.9172447919845581, "Finetune/Loss (Raw)": 1.0756298303604126, "Finetune/Step": 1546, "Finetune/Step Time": 4.7084042727947235} +{"Finetune/Learning Rate": 1.6469932630779677e-05, "Finetune/Loss": 0.9179832935333252, "Finetune/Loss (Raw)": 0.9777840375900269, "Finetune/Step": 1547, "Finetune/Step Time": 4.708389153704047} +{"Finetune/Learning Rate": 1.6465181324219937e-05, "Finetune/Loss": 0.9182742238044739, "Finetune/Loss (Raw)": 1.0430465936660767, "Finetune/Step": 1548, "Finetune/Step Time": 4.717998567968607} +{"Finetune/Learning Rate": 1.6460427508649546e-05, "Finetune/Loss": 0.9142160415649414, "Finetune/Loss (Raw)": 0.4009639322757721, "Finetune/Step": 1549, "Finetune/Step Time": 4.755517616868019} +{"Finetune/Learning Rate": 1.6455671185913367e-05, "Finetune/Loss": 0.9147412776947021, "Finetune/Loss (Raw)": 1.010664701461792, "Finetune/Step": 1550, "Finetune/Step Time": 4.758954409509897} +{"Finetune/Learning Rate": 1.645091235785724e-05, "Finetune/Loss": 0.9133199453353882, "Finetune/Loss (Raw)": 0.8651429414749146, "Finetune/Step": 1551, "Finetune/Step Time": 4.753284716978669} +{"Finetune/Learning Rate": 1.644615102632797e-05, "Finetune/Loss": 0.9145329594612122, "Finetune/Loss (Raw)": 1.0136593580245972, "Finetune/Step": 1552, "Finetune/Step Time": 4.754105033352971} +{"Finetune/Learning Rate": 1.6441387193173337e-05, "Finetune/Loss": 0.913715124130249, "Finetune/Loss (Raw)": 0.9005128145217896, "Finetune/Step": 1553, "Finetune/Step Time": 4.770420324057341} +{"Finetune/Learning Rate": 1.6436620860242093e-05, "Finetune/Loss": 0.9135510921478271, "Finetune/Loss (Raw)": 0.8653987646102905, "Finetune/Step": 1554, "Finetune/Step Time": 4.761018877848983} +{"Finetune/Learning Rate": 1.6431852029383955e-05, "Finetune/Loss": 0.9144183397293091, "Finetune/Loss (Raw)": 1.0594428777694702, "Finetune/Step": 1555, "Finetune/Step Time": 4.763469347730279} +{"Finetune/Learning Rate": 1.6427080702449616e-05, "Finetune/Loss": 0.9140125513076782, "Finetune/Loss (Raw)": 0.8839831948280334, "Finetune/Step": 1556, "Finetune/Step Time": 4.769752951338887} +{"Finetune/Learning Rate": 1.6422306881290734e-05, "Finetune/Loss": 0.9138367176055908, "Finetune/Loss (Raw)": 0.8542851805686951, "Finetune/Step": 1557, "Finetune/Step Time": 4.773709047585726} +{"Finetune/Learning Rate": 1.641753056775994e-05, "Finetune/Loss": 0.9136574268341064, "Finetune/Loss (Raw)": 0.9343874454498291, "Finetune/Step": 1558, "Finetune/Step Time": 4.763545570895076} +{"Finetune/Learning Rate": 1.641275176371082e-05, "Finetune/Loss": 0.9157372713088989, "Finetune/Loss (Raw)": 1.1651986837387085, "Finetune/Step": 1559, "Finetune/Step Time": 4.749014239758253} +{"Finetune/Learning Rate": 1.6407970470997943e-05, "Finetune/Loss": 0.9163366556167603, "Finetune/Loss (Raw)": 1.038313627243042, "Finetune/Step": 1560, "Finetune/Step Time": 4.754231156781316} +{"Finetune/Learning Rate": 1.6403186691476828e-05, "Finetune/Loss": 0.9164920449256897, "Finetune/Loss (Raw)": 1.015501618385315, "Finetune/Step": 1561, "Finetune/Step Time": 4.751141095533967} +{"Finetune/Learning Rate": 1.639840042700397e-05, "Finetune/Loss": 0.9213893413543701, "Finetune/Loss (Raw)": 1.0047156810760498, "Finetune/Step": 1562, "Finetune/Step Time": 4.722746253013611} +{"Finetune/Learning Rate": 1.6393611679436828e-05, "Finetune/Loss": 0.9207791090011597, "Finetune/Loss (Raw)": 0.9425315260887146, "Finetune/Step": 1563, "Finetune/Step Time": 4.740679187700152} +{"Finetune/Learning Rate": 1.6388820450633813e-05, "Finetune/Loss": 0.9215984344482422, "Finetune/Loss (Raw)": 0.9162803292274475, "Finetune/Step": 1564, "Finetune/Step Time": 4.7259494829922915} +{"Finetune/Learning Rate": 1.6384026742454315e-05, "Finetune/Loss": 0.9200283288955688, "Finetune/Loss (Raw)": 0.854381263256073, "Finetune/Step": 1565, "Finetune/Step Time": 4.725468879565597} +{"Finetune/Learning Rate": 1.6379230556758676e-05, "Finetune/Loss": 0.9198340177536011, "Finetune/Loss (Raw)": 0.962486982345581, "Finetune/Step": 1566, "Finetune/Step Time": 4.732336973771453} +{"Finetune/Learning Rate": 1.63744318954082e-05, "Finetune/Loss": 0.9194589853286743, "Finetune/Loss (Raw)": 0.9922230243682861, "Finetune/Step": 1567, "Finetune/Step Time": 4.738928455859423} +{"Finetune/Learning Rate": 1.636963076026516e-05, "Finetune/Loss": 0.9214000701904297, "Finetune/Loss (Raw)": 0.9640621542930603, "Finetune/Step": 1568, "Finetune/Step Time": 4.710541313514113} +{"Finetune/Learning Rate": 1.6364827153192773e-05, "Finetune/Loss": 0.92059326171875, "Finetune/Loss (Raw)": 0.8412236571311951, "Finetune/Step": 1569, "Finetune/Step Time": 4.723246522247791} +{"Finetune/Learning Rate": 1.6360021076055234e-05, "Finetune/Loss": 0.9209926128387451, "Finetune/Loss (Raw)": 1.0430610179901123, "Finetune/Step": 1570, "Finetune/Step Time": 4.720254253596067} +{"Finetune/Learning Rate": 1.6355212530717683e-05, "Finetune/Loss": 0.9213650226593018, "Finetune/Loss (Raw)": 0.9650639295578003, "Finetune/Step": 1571, "Finetune/Step Time": 4.716089136898518} +{"Finetune/Learning Rate": 1.6350401519046226e-05, "Finetune/Loss": 0.9221706390380859, "Finetune/Loss (Raw)": 1.0202147960662842, "Finetune/Step": 1572, "Finetune/Step Time": 4.709888098761439} +{"Finetune/Learning Rate": 1.634558804290792e-05, "Finetune/Loss": 0.9226634502410889, "Finetune/Loss (Raw)": 0.9865432381629944, "Finetune/Step": 1573, "Finetune/Step Time": 4.725846793502569} +{"Finetune/Learning Rate": 1.634077210417078e-05, "Finetune/Loss": 0.9224985837936401, "Finetune/Loss (Raw)": 0.8346083760261536, "Finetune/Step": 1574, "Finetune/Step Time": 4.709344007074833} +{"Finetune/Learning Rate": 1.633595370470378e-05, "Finetune/Loss": 0.9242546558380127, "Finetune/Loss (Raw)": 1.1963084936141968, "Finetune/Step": 1575, "Finetune/Step Time": 4.719176867976785} +{"Finetune/Learning Rate": 1.6331132846376843e-05, "Finetune/Loss": 0.9234966039657593, "Finetune/Loss (Raw)": 0.9691638946533203, "Finetune/Step": 1576, "Finetune/Step Time": 4.72219174169004} +{"Finetune/Learning Rate": 1.632630953106086e-05, "Finetune/Loss": 0.9242852330207825, "Finetune/Loss (Raw)": 0.9579839706420898, "Finetune/Step": 1577, "Finetune/Step Time": 4.71330600976944} +{"Finetune/Learning Rate": 1.6321483760627658e-05, "Finetune/Loss": 0.9252288937568665, "Finetune/Loss (Raw)": 0.9996151328086853, "Finetune/Step": 1578, "Finetune/Step Time": 4.665738126263022} +{"Finetune/Learning Rate": 1.6316655536950017e-05, "Finetune/Loss": 0.9256277680397034, "Finetune/Loss (Raw)": 0.968235194683075, "Finetune/Step": 1579, "Finetune/Step Time": 4.672082729637623} +{"Finetune/Learning Rate": 1.6311824861901693e-05, "Finetune/Loss": 0.9233758449554443, "Finetune/Loss (Raw)": 0.781596302986145, "Finetune/Step": 1580, "Finetune/Step Time": 4.707065526396036} +{"Finetune/Learning Rate": 1.6306991737357366e-05, "Finetune/Loss": 0.9229512214660645, "Finetune/Loss (Raw)": 0.8580747246742249, "Finetune/Step": 1581, "Finetune/Step Time": 4.702017547562718} +{"Finetune/Learning Rate": 1.630215616519268e-05, "Finetune/Loss": 0.9211103916168213, "Finetune/Loss (Raw)": 0.9662591814994812, "Finetune/Step": 1582, "Finetune/Step Time": 4.703196091577411} +{"Finetune/Learning Rate": 1.6297318147284223e-05, "Finetune/Loss": 0.92106693983078, "Finetune/Loss (Raw)": 1.009576439857483, "Finetune/Step": 1583, "Finetune/Step Time": 4.712765704840422} +{"Finetune/Learning Rate": 1.6292477685509538e-05, "Finetune/Loss": 0.9213159084320068, "Finetune/Loss (Raw)": 0.9708796143531799, "Finetune/Step": 1584, "Finetune/Step Time": 4.714223243296146} +{"Finetune/Learning Rate": 1.6287634781747113e-05, "Finetune/Loss": 0.9266475439071655, "Finetune/Loss (Raw)": 1.0594078302383423, "Finetune/Step": 1585, "Finetune/Step Time": 4.677190788090229} +{"Finetune/Learning Rate": 1.628278943787639e-05, "Finetune/Loss": 0.9279484748840332, "Finetune/Loss (Raw)": 1.1165727376937866, "Finetune/Step": 1586, "Finetune/Step Time": 4.730812514200807} +{"Finetune/Learning Rate": 1.6277941655777744e-05, "Finetune/Loss": 0.9276050329208374, "Finetune/Loss (Raw)": 1.0538510084152222, "Finetune/Step": 1587, "Finetune/Step Time": 4.723206914961338} +{"Finetune/Learning Rate": 1.6273091437332506e-05, "Finetune/Loss": 0.9264596700668335, "Finetune/Loss (Raw)": 0.9536074995994568, "Finetune/Step": 1588, "Finetune/Step Time": 4.733416680246592} +{"Finetune/Learning Rate": 1.6268238784422954e-05, "Finetune/Loss": 0.9249893426895142, "Finetune/Loss (Raw)": 0.9338601231575012, "Finetune/Step": 1589, "Finetune/Step Time": 4.749368324875832} +{"Finetune/Learning Rate": 1.6263383698932307e-05, "Finetune/Loss": 0.9248526692390442, "Finetune/Loss (Raw)": 1.0271084308624268, "Finetune/Step": 1590, "Finetune/Step Time": 4.755195425823331} +{"Finetune/Learning Rate": 1.6258526182744727e-05, "Finetune/Loss": 0.9230906963348389, "Finetune/Loss (Raw)": 0.8293465375900269, "Finetune/Step": 1591, "Finetune/Step Time": 4.770741745829582} +{"Finetune/Learning Rate": 1.6253666237745326e-05, "Finetune/Loss": 0.9229930639266968, "Finetune/Loss (Raw)": 0.8772849440574646, "Finetune/Step": 1592, "Finetune/Step Time": 4.772268610075116} +{"Finetune/Learning Rate": 1.6248803865820152e-05, "Finetune/Loss": 0.9225925207138062, "Finetune/Loss (Raw)": 0.9304161667823792, "Finetune/Step": 1593, "Finetune/Step Time": 4.775138631463051} +{"Finetune/Learning Rate": 1.624393906885619e-05, "Finetune/Loss": 0.9223030209541321, "Finetune/Loss (Raw)": 0.9358469247817993, "Finetune/Step": 1594, "Finetune/Step Time": 4.780417285859585} +{"Finetune/Learning Rate": 1.6239071848741385e-05, "Finetune/Loss": 0.9215797185897827, "Finetune/Loss (Raw)": 0.8615925312042236, "Finetune/Step": 1595, "Finetune/Step Time": 4.7584158182144165} +{"Finetune/Learning Rate": 1.6234202207364596e-05, "Finetune/Loss": 0.9222311973571777, "Finetune/Loss (Raw)": 1.1128549575805664, "Finetune/Step": 1596, "Finetune/Step Time": 4.763268018141389} +{"Finetune/Learning Rate": 1.6229330146615646e-05, "Finetune/Loss": 0.925796627998352, "Finetune/Loss (Raw)": 0.8475852608680725, "Finetune/Step": 1597, "Finetune/Step Time": 4.727114386856556} +{"Finetune/Learning Rate": 1.6224455668385283e-05, "Finetune/Loss": 0.9258071184158325, "Finetune/Loss (Raw)": 0.9929134249687195, "Finetune/Step": 1598, "Finetune/Step Time": 4.740315232425928} +{"Finetune/Learning Rate": 1.62195787745652e-05, "Finetune/Loss": 0.9209687113761902, "Finetune/Loss (Raw)": 0.37795284390449524, "Finetune/Step": 1599, "Finetune/Step Time": 4.779354875907302} +{"Finetune/Learning Rate": 1.6214699467048017e-05, "Finetune/Loss": 0.9219846129417419, "Finetune/Loss (Raw)": 0.9980126619338989, "Finetune/Step": 1600, "Finetune/Step Time": 4.782923458144069} +{"Finetune/Learning Rate": 1.6209817747727304e-05, "Finetune/Loss": 0.9194662570953369, "Finetune/Loss (Raw)": 0.3888937830924988, "Finetune/Step": 1601, "Finetune/Step Time": 4.820902770385146} +{"Finetune/Learning Rate": 1.620493361849756e-05, "Finetune/Loss": 0.9191018342971802, "Finetune/Loss (Raw)": 1.0850147008895874, "Finetune/Step": 1602, "Finetune/Step Time": 4.830490319058299} +{"Finetune/Learning Rate": 1.6200047081254223e-05, "Finetune/Loss": 0.9211423397064209, "Finetune/Loss (Raw)": 1.0242201089859009, "Finetune/Step": 1603, "Finetune/Step Time": 4.825985949486494} +{"Finetune/Learning Rate": 1.6195158137893656e-05, "Finetune/Loss": 0.920267641544342, "Finetune/Loss (Raw)": 0.895762026309967, "Finetune/Step": 1604, "Finetune/Step Time": 4.825869986787438} +{"Finetune/Learning Rate": 1.6190266790313165e-05, "Finetune/Loss": 0.9242868423461914, "Finetune/Loss (Raw)": 0.8289772868156433, "Finetune/Step": 1605, "Finetune/Step Time": 4.796962006017566} +{"Finetune/Learning Rate": 1.618537304041099e-05, "Finetune/Loss": 0.9219478368759155, "Finetune/Loss (Raw)": 0.774386465549469, "Finetune/Step": 1606, "Finetune/Step Time": 4.832994006574154} +{"Finetune/Learning Rate": 1.6180476890086297e-05, "Finetune/Loss": 0.9219244718551636, "Finetune/Loss (Raw)": 0.9037219882011414, "Finetune/Step": 1607, "Finetune/Step Time": 4.819598162546754} +{"Finetune/Learning Rate": 1.6175578341239188e-05, "Finetune/Loss": 0.9173468351364136, "Finetune/Loss (Raw)": 0.38873764872550964, "Finetune/Step": 1608, "Finetune/Step Time": 4.852155325934291} +{"Finetune/Learning Rate": 1.617067739577069e-05, "Finetune/Loss": 0.9170827865600586, "Finetune/Loss (Raw)": 0.9605114459991455, "Finetune/Step": 1609, "Finetune/Step Time": 4.840532787144184} +{"Finetune/Learning Rate": 1.616577405558277e-05, "Finetune/Loss": 0.9169092178344727, "Finetune/Loss (Raw)": 0.9403039216995239, "Finetune/Step": 1610, "Finetune/Step Time": 4.8366794269531965} +{"Finetune/Learning Rate": 1.616086832257831e-05, "Finetune/Loss": 0.9164004325866699, "Finetune/Loss (Raw)": 0.8639218807220459, "Finetune/Step": 1611, "Finetune/Step Time": 4.823201131075621} +{"Finetune/Learning Rate": 1.615596019866114e-05, "Finetune/Loss": 0.9154777526855469, "Finetune/Loss (Raw)": 0.9201360940933228, "Finetune/Step": 1612, "Finetune/Step Time": 4.825463257730007} +{"Finetune/Learning Rate": 1.6151049685736e-05, "Finetune/Loss": 0.9150145053863525, "Finetune/Loss (Raw)": 0.855273962020874, "Finetune/Step": 1613, "Finetune/Step Time": 4.837562987580895} +{"Finetune/Learning Rate": 1.6146136785708564e-05, "Finetune/Loss": 0.9203577041625977, "Finetune/Loss (Raw)": 0.9942499995231628, "Finetune/Step": 1614, "Finetune/Step Time": 4.799255646765232} +{"Finetune/Learning Rate": 1.6141221500485437e-05, "Finetune/Loss": 0.9200379848480225, "Finetune/Loss (Raw)": 1.0627943277359009, "Finetune/Step": 1615, "Finetune/Step Time": 4.787293745204806} +{"Finetune/Learning Rate": 1.6136303831974146e-05, "Finetune/Loss": 0.9182496666908264, "Finetune/Loss (Raw)": 0.9699700474739075, "Finetune/Step": 1616, "Finetune/Step Time": 4.786626713350415} +{"Finetune/Learning Rate": 1.6131383782083135e-05, "Finetune/Loss": 0.9189041256904602, "Finetune/Loss (Raw)": 0.8974812030792236, "Finetune/Step": 1617, "Finetune/Step Time": 4.7885549776256084} +{"Finetune/Learning Rate": 1.6126461352721784e-05, "Finetune/Loss": 0.9185447692871094, "Finetune/Loss (Raw)": 0.9041163325309753, "Finetune/Step": 1618, "Finetune/Step Time": 4.773505941033363} +{"Finetune/Learning Rate": 1.612153654580039e-05, "Finetune/Loss": 0.9185123443603516, "Finetune/Loss (Raw)": 0.8335248231887817, "Finetune/Step": 1619, "Finetune/Step Time": 4.762812778353691} +{"Finetune/Learning Rate": 1.6116609363230176e-05, "Finetune/Loss": 0.9190412759780884, "Finetune/Loss (Raw)": 1.1504707336425781, "Finetune/Step": 1620, "Finetune/Step Time": 4.771146237850189} +{"Finetune/Learning Rate": 1.6111679806923287e-05, "Finetune/Loss": 0.9230931997299194, "Finetune/Loss (Raw)": 0.8554984331130981, "Finetune/Step": 1621, "Finetune/Step Time": 4.738525668159127} +{"Finetune/Learning Rate": 1.6106747878792784e-05, "Finetune/Loss": 0.9240604639053345, "Finetune/Loss (Raw)": 1.0658423900604248, "Finetune/Step": 1622, "Finetune/Step Time": 4.736237671226263} +{"Finetune/Learning Rate": 1.610181358075265e-05, "Finetune/Loss": 0.9248278141021729, "Finetune/Loss (Raw)": 1.1410348415374756, "Finetune/Step": 1623, "Finetune/Step Time": 4.724523423239589} +{"Finetune/Learning Rate": 1.60968769147178e-05, "Finetune/Loss": 0.924294650554657, "Finetune/Loss (Raw)": 0.9205044507980347, "Finetune/Step": 1624, "Finetune/Step Time": 4.732140427455306} +{"Finetune/Learning Rate": 1.6091937882604047e-05, "Finetune/Loss": 0.9254130125045776, "Finetune/Loss (Raw)": 0.9893703460693359, "Finetune/Step": 1625, "Finetune/Step Time": 4.737789191305637} +{"Finetune/Learning Rate": 1.608699648632814e-05, "Finetune/Loss": 0.9248604774475098, "Finetune/Loss (Raw)": 1.0046918392181396, "Finetune/Step": 1626, "Finetune/Step Time": 4.728612704202533} +{"Finetune/Learning Rate": 1.6082052727807732e-05, "Finetune/Loss": 0.9296283721923828, "Finetune/Loss (Raw)": 1.0395582914352417, "Finetune/Step": 1627, "Finetune/Step Time": 4.687504816800356} +{"Finetune/Learning Rate": 1.60771066089614e-05, "Finetune/Loss": 0.9345697164535522, "Finetune/Loss (Raw)": 0.9875791072845459, "Finetune/Step": 1628, "Finetune/Step Time": 4.649074275046587} +{"Finetune/Learning Rate": 1.6072158131708643e-05, "Finetune/Loss": 0.9338669180870056, "Finetune/Loss (Raw)": 0.9558537006378174, "Finetune/Step": 1629, "Finetune/Step Time": 4.619818057864904} +{"Finetune/Learning Rate": 1.6067207297969864e-05, "Finetune/Loss": 0.9332996010780334, "Finetune/Loss (Raw)": 1.0382755994796753, "Finetune/Step": 1630, "Finetune/Step Time": 4.6174785029143095} +{"Finetune/Learning Rate": 1.6062254109666383e-05, "Finetune/Loss": 0.933911144733429, "Finetune/Loss (Raw)": 0.9488429427146912, "Finetune/Step": 1631, "Finetune/Step Time": 4.620037836953998} +{"Finetune/Learning Rate": 1.6057298568720436e-05, "Finetune/Loss": 0.9333711862564087, "Finetune/Loss (Raw)": 0.9381226897239685, "Finetune/Step": 1632, "Finetune/Step Time": 4.611164467409253} +{"Finetune/Learning Rate": 1.6052340677055175e-05, "Finetune/Loss": 0.9325215816497803, "Finetune/Loss (Raw)": 0.8831256031990051, "Finetune/Step": 1633, "Finetune/Step Time": 4.611585820093751} +{"Finetune/Learning Rate": 1.6047380436594663e-05, "Finetune/Loss": 0.9334686994552612, "Finetune/Loss (Raw)": 1.007359266281128, "Finetune/Step": 1634, "Finetune/Step Time": 4.603986049070954} +{"Finetune/Learning Rate": 1.6042417849263863e-05, "Finetune/Loss": 0.9328457117080688, "Finetune/Loss (Raw)": 0.9813058972358704, "Finetune/Step": 1635, "Finetune/Step Time": 4.625897197052836} +{"Finetune/Learning Rate": 1.6037452916988665e-05, "Finetune/Loss": 0.9325491189956665, "Finetune/Loss (Raw)": 0.9807857871055603, "Finetune/Step": 1636, "Finetune/Step Time": 4.622679831460118} +{"Finetune/Learning Rate": 1.6032485641695862e-05, "Finetune/Loss": 0.9321646690368652, "Finetune/Loss (Raw)": 0.967795729637146, "Finetune/Step": 1637, "Finetune/Step Time": 4.623967435210943} +{"Finetune/Learning Rate": 1.602751602531316e-05, "Finetune/Loss": 0.9307122230529785, "Finetune/Loss (Raw)": 0.9041715860366821, "Finetune/Step": 1638, "Finetune/Step Time": 4.625469138845801} +{"Finetune/Learning Rate": 1.602254406976916e-05, "Finetune/Loss": 0.9285762906074524, "Finetune/Loss (Raw)": 0.8439880609512329, "Finetune/Step": 1639, "Finetune/Step Time": 4.622648656368256} +{"Finetune/Learning Rate": 1.6017569776993388e-05, "Finetune/Loss": 0.9294164776802063, "Finetune/Loss (Raw)": 1.0255043506622314, "Finetune/Step": 1640, "Finetune/Step Time": 4.611201541498303} +{"Finetune/Learning Rate": 1.601259314891627e-05, "Finetune/Loss": 0.92840975522995, "Finetune/Loss (Raw)": 1.0284302234649658, "Finetune/Step": 1641, "Finetune/Step Time": 4.608895981684327} +{"Finetune/Learning Rate": 1.6007614187469137e-05, "Finetune/Loss": 0.9284800291061401, "Finetune/Loss (Raw)": 0.9882505536079407, "Finetune/Step": 1642, "Finetune/Step Time": 4.5982883628457785} +{"Finetune/Learning Rate": 1.6002632894584227e-05, "Finetune/Loss": 0.9294293522834778, "Finetune/Loss (Raw)": 1.0813109874725342, "Finetune/Step": 1643, "Finetune/Step Time": 4.599797561764717} +{"Finetune/Learning Rate": 1.599764927219468e-05, "Finetune/Loss": 0.9295929074287415, "Finetune/Loss (Raw)": 0.9811491966247559, "Finetune/Step": 1644, "Finetune/Step Time": 4.630785416811705} +{"Finetune/Learning Rate": 1.5992663322234548e-05, "Finetune/Loss": 0.929972231388092, "Finetune/Loss (Raw)": 0.923865795135498, "Finetune/Step": 1645, "Finetune/Step Time": 4.6251452546566725} +{"Finetune/Learning Rate": 1.5987675046638777e-05, "Finetune/Loss": 0.9295586943626404, "Finetune/Loss (Raw)": 1.0314513444900513, "Finetune/Step": 1646, "Finetune/Step Time": 4.628766518086195} +{"Finetune/Learning Rate": 1.5982684447343214e-05, "Finetune/Loss": 0.9299910664558411, "Finetune/Loss (Raw)": 0.8988861441612244, "Finetune/Step": 1647, "Finetune/Step Time": 4.6418643817305565} +{"Finetune/Learning Rate": 1.5977691526284623e-05, "Finetune/Loss": 0.9351151585578918, "Finetune/Loss (Raw)": 1.049800157546997, "Finetune/Step": 1648, "Finetune/Step Time": 4.608432721346617} +{"Finetune/Learning Rate": 1.5972696285400656e-05, "Finetune/Loss": 0.9342407584190369, "Finetune/Loss (Raw)": 0.9410027265548706, "Finetune/Step": 1649, "Finetune/Step Time": 4.610244246199727} +{"Finetune/Learning Rate": 1.596769872662987e-05, "Finetune/Loss": 0.9345306754112244, "Finetune/Loss (Raw)": 0.9629235863685608, "Finetune/Step": 1650, "Finetune/Step Time": 4.5984825268387794} +{"Finetune/Learning Rate": 1.5962698851911715e-05, "Finetune/Loss": 0.9349449872970581, "Finetune/Loss (Raw)": 0.8829387426376343, "Finetune/Step": 1651, "Finetune/Step Time": 4.600231662392616} +{"Finetune/Learning Rate": 1.5957696663186547e-05, "Finetune/Loss": 0.9348019361495972, "Finetune/Loss (Raw)": 0.8895831108093262, "Finetune/Step": 1652, "Finetune/Step Time": 4.583002822473645} +{"Finetune/Learning Rate": 1.595269216239562e-05, "Finetune/Loss": 0.9355316162109375, "Finetune/Loss (Raw)": 0.9725837707519531, "Finetune/Step": 1653, "Finetune/Step Time": 4.574696283787489} +{"Finetune/Learning Rate": 1.594768535148108e-05, "Finetune/Loss": 0.9349552989006042, "Finetune/Loss (Raw)": 1.016452431678772, "Finetune/Step": 1654, "Finetune/Step Time": 4.588123258203268} +{"Finetune/Learning Rate": 1.5942676232385973e-05, "Finetune/Loss": 0.9305751323699951, "Finetune/Loss (Raw)": 0.38554173707962036, "Finetune/Step": 1655, "Finetune/Step Time": 4.624779416248202} +{"Finetune/Learning Rate": 1.5937664807054242e-05, "Finetune/Loss": 0.9348918199539185, "Finetune/Loss (Raw)": 0.9704506397247314, "Finetune/Step": 1656, "Finetune/Step Time": 4.589774312451482} +{"Finetune/Learning Rate": 1.593265107743072e-05, "Finetune/Loss": 0.9335527420043945, "Finetune/Loss (Raw)": 0.8841648697853088, "Finetune/Step": 1657, "Finetune/Step Time": 4.588331094011664} +{"Finetune/Learning Rate": 1.5927635045461145e-05, "Finetune/Loss": 0.9351852536201477, "Finetune/Loss (Raw)": 0.9740135669708252, "Finetune/Step": 1658, "Finetune/Step Time": 4.572734666988254} +{"Finetune/Learning Rate": 1.5922616713092135e-05, "Finetune/Loss": 0.9358269572257996, "Finetune/Loss (Raw)": 1.0889630317687988, "Finetune/Step": 1659, "Finetune/Step Time": 4.573179529979825} +{"Finetune/Learning Rate": 1.5917596082271206e-05, "Finetune/Loss": 0.9358870983123779, "Finetune/Loss (Raw)": 0.9705997109413147, "Finetune/Step": 1660, "Finetune/Step Time": 4.569919366389513} +{"Finetune/Learning Rate": 1.5912573154946768e-05, "Finetune/Loss": 0.9361323118209839, "Finetune/Loss (Raw)": 0.9279394149780273, "Finetune/Step": 1661, "Finetune/Step Time": 4.565203377977014} +{"Finetune/Learning Rate": 1.5907547933068125e-05, "Finetune/Loss": 0.9368859529495239, "Finetune/Loss (Raw)": 1.0684610605239868, "Finetune/Step": 1662, "Finetune/Step Time": 4.564627250656486} +{"Finetune/Learning Rate": 1.590252041858546e-05, "Finetune/Loss": 0.9354571104049683, "Finetune/Loss (Raw)": 0.712558925151825, "Finetune/Step": 1663, "Finetune/Step Time": 4.5743805803358555} +{"Finetune/Learning Rate": 1.5897490613449864e-05, "Finetune/Loss": 0.9346296787261963, "Finetune/Loss (Raw)": 0.7954996228218079, "Finetune/Step": 1664, "Finetune/Step Time": 4.567311329767108} +{"Finetune/Learning Rate": 1.5892458519613296e-05, "Finetune/Loss": 0.9358994364738464, "Finetune/Loss (Raw)": 1.066550374031067, "Finetune/Step": 1665, "Finetune/Step Time": 4.5671056397259235} +{"Finetune/Learning Rate": 1.588742413902862e-05, "Finetune/Loss": 0.9395381212234497, "Finetune/Loss (Raw)": 0.9036538600921631, "Finetune/Step": 1666, "Finetune/Step Time": 4.524776961654425} +{"Finetune/Learning Rate": 1.5882387473649572e-05, "Finetune/Loss": 0.9395787119865417, "Finetune/Loss (Raw)": 0.9951339960098267, "Finetune/Step": 1667, "Finetune/Step Time": 4.522551652044058} +{"Finetune/Learning Rate": 1.5877348525430793e-05, "Finetune/Loss": 0.9402441382408142, "Finetune/Loss (Raw)": 1.068578839302063, "Finetune/Step": 1668, "Finetune/Step Time": 4.5275388322770596} +{"Finetune/Learning Rate": 1.5872307296327795e-05, "Finetune/Loss": 0.9392496943473816, "Finetune/Loss (Raw)": 0.9564012885093689, "Finetune/Step": 1669, "Finetune/Step Time": 4.524258736521006} +{"Finetune/Learning Rate": 1.5867263788296984e-05, "Finetune/Loss": 0.9398964643478394, "Finetune/Loss (Raw)": 0.999093770980835, "Finetune/Step": 1670, "Finetune/Step Time": 4.510811522603035} +{"Finetune/Learning Rate": 1.5862218003295647e-05, "Finetune/Loss": 0.9402914047241211, "Finetune/Loss (Raw)": 0.9023036956787109, "Finetune/Step": 1671, "Finetune/Step Time": 4.518665909767151} +{"Finetune/Learning Rate": 1.5857169943281948e-05, "Finetune/Loss": 0.9407552480697632, "Finetune/Loss (Raw)": 1.0443966388702393, "Finetune/Step": 1672, "Finetune/Step Time": 4.535453567281365} +{"Finetune/Learning Rate": 1.585211961021495e-05, "Finetune/Loss": 0.9401978254318237, "Finetune/Loss (Raw)": 0.8434882164001465, "Finetune/Step": 1673, "Finetune/Step Time": 4.54077641479671} +{"Finetune/Learning Rate": 1.5847067006054584e-05, "Finetune/Loss": 0.9380095601081848, "Finetune/Loss (Raw)": 0.7955321073532104, "Finetune/Step": 1674, "Finetune/Step Time": 4.542061721906066} +{"Finetune/Learning Rate": 1.584201213276167e-05, "Finetune/Loss": 0.9376425743103027, "Finetune/Loss (Raw)": 0.9308122396469116, "Finetune/Step": 1675, "Finetune/Step Time": 4.551438337191939} +{"Finetune/Learning Rate": 1.5836954992297897e-05, "Finetune/Loss": 0.9379087686538696, "Finetune/Loss (Raw)": 1.077111840248108, "Finetune/Step": 1676, "Finetune/Step Time": 4.5344985499978065} +{"Finetune/Learning Rate": 1.583189558662585e-05, "Finetune/Loss": 0.9426635503768921, "Finetune/Loss (Raw)": 1.0095778703689575, "Finetune/Step": 1677, "Finetune/Step Time": 4.496716333553195} +{"Finetune/Learning Rate": 1.5826833917708983e-05, "Finetune/Loss": 0.94144606590271, "Finetune/Loss (Raw)": 0.8548333644866943, "Finetune/Step": 1678, "Finetune/Step Time": 4.489018822088838} +{"Finetune/Learning Rate": 1.5821769987511634e-05, "Finetune/Loss": 0.9377468228340149, "Finetune/Loss (Raw)": 0.39164361357688904, "Finetune/Step": 1679, "Finetune/Step Time": 4.528825281187892} +{"Finetune/Learning Rate": 1.5816703797999015e-05, "Finetune/Loss": 0.937252938747406, "Finetune/Loss (Raw)": 0.950431764125824, "Finetune/Step": 1680, "Finetune/Step Time": 4.540989026427269} +{"Finetune/Learning Rate": 1.5811635351137212e-05, "Finetune/Loss": 0.9378401041030884, "Finetune/Loss (Raw)": 0.9756783246994019, "Finetune/Step": 1681, "Finetune/Step Time": 4.517538581043482} +{"Finetune/Learning Rate": 1.580656464889319e-05, "Finetune/Loss": 0.9398002624511719, "Finetune/Loss (Raw)": 1.1162910461425781, "Finetune/Step": 1682, "Finetune/Step Time": 4.525324013084173} +{"Finetune/Learning Rate": 1.5801491693234793e-05, "Finetune/Loss": 0.938389003276825, "Finetune/Loss (Raw)": 0.8788098692893982, "Finetune/Step": 1683, "Finetune/Step Time": 4.528024645522237} +{"Finetune/Learning Rate": 1.5796416486130735e-05, "Finetune/Loss": 0.9385923147201538, "Finetune/Loss (Raw)": 0.9100027084350586, "Finetune/Step": 1684, "Finetune/Step Time": 4.524741457775235} +{"Finetune/Learning Rate": 1.5791339029550604e-05, "Finetune/Loss": 0.9388208389282227, "Finetune/Loss (Raw)": 0.883535623550415, "Finetune/Step": 1685, "Finetune/Step Time": 4.5232681427150965} +{"Finetune/Learning Rate": 1.578625932546486e-05, "Finetune/Loss": 0.9379059672355652, "Finetune/Loss (Raw)": 0.817287266254425, "Finetune/Step": 1686, "Finetune/Step Time": 4.539760380983353} +{"Finetune/Learning Rate": 1.578117737584484e-05, "Finetune/Loss": 0.9368083477020264, "Finetune/Loss (Raw)": 1.0247005224227905, "Finetune/Step": 1687, "Finetune/Step Time": 4.548058787360787} +{"Finetune/Learning Rate": 1.577609318266275e-05, "Finetune/Loss": 0.9368035793304443, "Finetune/Loss (Raw)": 1.0376960039138794, "Finetune/Step": 1688, "Finetune/Step Time": 4.5497965179383755} +{"Finetune/Learning Rate": 1.5771006747891663e-05, "Finetune/Loss": 0.9312961101531982, "Finetune/Loss (Raw)": 0.3105570673942566, "Finetune/Step": 1689, "Finetune/Step Time": 4.5846948903054} +{"Finetune/Learning Rate": 1.576591807350553e-05, "Finetune/Loss": 0.9318051338195801, "Finetune/Loss (Raw)": 1.069862723350525, "Finetune/Step": 1690, "Finetune/Step Time": 4.57538016140461} +{"Finetune/Learning Rate": 1.5760827161479158e-05, "Finetune/Loss": 0.9318717122077942, "Finetune/Loss (Raw)": 0.9510640501976013, "Finetune/Step": 1691, "Finetune/Step Time": 4.557200333103538} +{"Finetune/Learning Rate": 1.5755734013788235e-05, "Finetune/Loss": 0.9316977858543396, "Finetune/Loss (Raw)": 0.8940105438232422, "Finetune/Step": 1692, "Finetune/Step Time": 4.579570561647415} +{"Finetune/Learning Rate": 1.5750638632409315e-05, "Finetune/Loss": 0.9316715002059937, "Finetune/Loss (Raw)": 0.8510218262672424, "Finetune/Step": 1693, "Finetune/Step Time": 4.598289243876934} +{"Finetune/Learning Rate": 1.574554101931981e-05, "Finetune/Loss": 0.9299108982086182, "Finetune/Loss (Raw)": 0.7371281981468201, "Finetune/Step": 1694, "Finetune/Step Time": 4.6007756516337395} +{"Finetune/Learning Rate": 1.5740441176498015e-05, "Finetune/Loss": 0.9292274713516235, "Finetune/Loss (Raw)": 0.9047530293464661, "Finetune/Step": 1695, "Finetune/Step Time": 4.606525348499417} +{"Finetune/Learning Rate": 1.5735339105923068e-05, "Finetune/Loss": 0.9292845129966736, "Finetune/Loss (Raw)": 0.9713523983955383, "Finetune/Step": 1696, "Finetune/Step Time": 4.604659976437688} +{"Finetune/Learning Rate": 1.5730234809574986e-05, "Finetune/Loss": 0.9305337071418762, "Finetune/Loss (Raw)": 1.001123070716858, "Finetune/Step": 1697, "Finetune/Step Time": 4.590656576678157} +{"Finetune/Learning Rate": 1.5725128289434647e-05, "Finetune/Loss": 0.9297486543655396, "Finetune/Loss (Raw)": 0.9425753951072693, "Finetune/Step": 1698, "Finetune/Step Time": 4.593305576592684} +{"Finetune/Learning Rate": 1.5720019547483795e-05, "Finetune/Loss": 0.9303040504455566, "Finetune/Loss (Raw)": 1.036158561706543, "Finetune/Step": 1699, "Finetune/Step Time": 4.591680355370045} +{"Finetune/Learning Rate": 1.571490858570503e-05, "Finetune/Loss": 0.9303134679794312, "Finetune/Loss (Raw)": 1.021411657333374, "Finetune/Step": 1700, "Finetune/Step Time": 4.601532885804772} +{"Finetune/Learning Rate": 1.5709795406081818e-05, "Finetune/Loss": 0.9309306144714355, "Finetune/Loss (Raw)": 1.0655378103256226, "Finetune/Step": 1701, "Finetune/Step Time": 4.586137857288122} +{"Finetune/Learning Rate": 1.570468001059848e-05, "Finetune/Loss": 0.9321970343589783, "Finetune/Loss (Raw)": 0.9967122673988342, "Finetune/Step": 1702, "Finetune/Step Time": 4.58582554012537} +{"Finetune/Learning Rate": 1.5699562401240206e-05, "Finetune/Loss": 0.9296629428863525, "Finetune/Loss (Raw)": 0.8719415068626404, "Finetune/Step": 1703, "Finetune/Step Time": 4.592843785881996} +{"Finetune/Learning Rate": 1.5694442579993042e-05, "Finetune/Loss": 0.9247949719429016, "Finetune/Loss (Raw)": 0.34606289863586426, "Finetune/Step": 1704, "Finetune/Step Time": 4.626278651878238} +{"Finetune/Learning Rate": 1.5689320548843882e-05, "Finetune/Loss": 0.9246925711631775, "Finetune/Loss (Raw)": 0.9448822140693665, "Finetune/Step": 1705, "Finetune/Step Time": 4.62878043204546} +{"Finetune/Learning Rate": 1.5684196309780495e-05, "Finetune/Loss": 0.9250377416610718, "Finetune/Loss (Raw)": 1.0437872409820557, "Finetune/Step": 1706, "Finetune/Step Time": 4.630724098533392} +{"Finetune/Learning Rate": 1.5679069864791493e-05, "Finetune/Loss": 0.9249389171600342, "Finetune/Loss (Raw)": 0.9555919766426086, "Finetune/Step": 1707, "Finetune/Step Time": 4.6274726670235395} +{"Finetune/Learning Rate": 1.567394121586635e-05, "Finetune/Loss": 0.9263631105422974, "Finetune/Loss (Raw)": 0.9638934135437012, "Finetune/Step": 1708, "Finetune/Step Time": 4.607664035633206} +{"Finetune/Learning Rate": 1.56688103649954e-05, "Finetune/Loss": 0.9282065629959106, "Finetune/Loss (Raw)": 1.0940332412719727, "Finetune/Step": 1709, "Finetune/Step Time": 4.611725728958845} +{"Finetune/Learning Rate": 1.566367731416982e-05, "Finetune/Loss": 0.9284758567810059, "Finetune/Loss (Raw)": 1.000733494758606, "Finetune/Step": 1710, "Finetune/Step Time": 4.620938170701265} +{"Finetune/Learning Rate": 1.5658542065381644e-05, "Finetune/Loss": 0.9284777641296387, "Finetune/Loss (Raw)": 1.009825348854065, "Finetune/Step": 1711, "Finetune/Step Time": 4.617807248607278} +{"Finetune/Learning Rate": 1.565340462062377e-05, "Finetune/Loss": 0.9297792911529541, "Finetune/Loss (Raw)": 1.1374675035476685, "Finetune/Step": 1712, "Finetune/Step Time": 4.618299404159188} +{"Finetune/Learning Rate": 1.5648264981889936e-05, "Finetune/Loss": 0.9293390512466431, "Finetune/Loss (Raw)": 1.0030533075332642, "Finetune/Step": 1713, "Finetune/Step Time": 4.617870161309838} +{"Finetune/Learning Rate": 1.5643123151174736e-05, "Finetune/Loss": 0.9284132719039917, "Finetune/Loss (Raw)": 0.9980762004852295, "Finetune/Step": 1714, "Finetune/Step Time": 4.572223665192723} +{"Finetune/Learning Rate": 1.5637979130473612e-05, "Finetune/Loss": 0.9266656637191772, "Finetune/Loss (Raw)": 0.8301681280136108, "Finetune/Step": 1715, "Finetune/Step Time": 4.5816277749836445} +{"Finetune/Learning Rate": 1.563283292178286e-05, "Finetune/Loss": 0.9264369606971741, "Finetune/Loss (Raw)": 0.9243229031562805, "Finetune/Step": 1716, "Finetune/Step Time": 4.569857833907008} +{"Finetune/Learning Rate": 1.5627684527099624e-05, "Finetune/Loss": 0.9274903535842896, "Finetune/Loss (Raw)": 1.0686912536621094, "Finetune/Step": 1717, "Finetune/Step Time": 4.569858519360423} +{"Finetune/Learning Rate": 1.5622533948421895e-05, "Finetune/Loss": 0.9271489381790161, "Finetune/Loss (Raw)": 0.9834108352661133, "Finetune/Step": 1718, "Finetune/Step Time": 4.560026181861758} +{"Finetune/Learning Rate": 1.5617381187748507e-05, "Finetune/Loss": 0.9281620383262634, "Finetune/Loss (Raw)": 0.9590252637863159, "Finetune/Step": 1719, "Finetune/Step Time": 4.546361889690161} +{"Finetune/Learning Rate": 1.5612226247079155e-05, "Finetune/Loss": 0.9294329285621643, "Finetune/Loss (Raw)": 1.0399597883224487, "Finetune/Step": 1720, "Finetune/Step Time": 4.545075004920363} +{"Finetune/Learning Rate": 1.5607069128414366e-05, "Finetune/Loss": 0.9299468994140625, "Finetune/Loss (Raw)": 0.996196985244751, "Finetune/Step": 1721, "Finetune/Step Time": 4.5466140024363995} +{"Finetune/Learning Rate": 1.5601909833755514e-05, "Finetune/Loss": 0.930397629737854, "Finetune/Loss (Raw)": 0.9935449957847595, "Finetune/Step": 1722, "Finetune/Step Time": 4.537338683381677} +{"Finetune/Learning Rate": 1.559674836510482e-05, "Finetune/Loss": 0.9317703247070312, "Finetune/Loss (Raw)": 1.0373055934906006, "Finetune/Step": 1723, "Finetune/Step Time": 4.553643299266696} +{"Finetune/Learning Rate": 1.5591584724465363e-05, "Finetune/Loss": 0.9300591945648193, "Finetune/Loss (Raw)": 0.8938250541687012, "Finetune/Step": 1724, "Finetune/Step Time": 4.546166148036718} +{"Finetune/Learning Rate": 1.5586418913841034e-05, "Finetune/Loss": 0.9311427474021912, "Finetune/Loss (Raw)": 0.9862746000289917, "Finetune/Step": 1725, "Finetune/Step Time": 4.536082271486521} +{"Finetune/Learning Rate": 1.558125093523659e-05, "Finetune/Loss": 0.9303900003433228, "Finetune/Loss (Raw)": 0.8965591788291931, "Finetune/Step": 1726, "Finetune/Step Time": 4.528348693624139} +{"Finetune/Learning Rate": 1.5576080790657626e-05, "Finetune/Loss": 0.9352138638496399, "Finetune/Loss (Raw)": 0.9954105615615845, "Finetune/Step": 1727, "Finetune/Step Time": 4.506545007228851} +{"Finetune/Learning Rate": 1.5570908482110566e-05, "Finetune/Loss": 0.9357805252075195, "Finetune/Loss (Raw)": 1.070550560951233, "Finetune/Step": 1728, "Finetune/Step Time": 4.50333109498024} +{"Finetune/Learning Rate": 1.5565734011602693e-05, "Finetune/Loss": 0.9407770037651062, "Finetune/Loss (Raw)": 1.0284396409988403, "Finetune/Step": 1729, "Finetune/Step Time": 4.461127696558833} +{"Finetune/Learning Rate": 1.556055738114211e-05, "Finetune/Loss": 0.9387882947921753, "Finetune/Loss (Raw)": 0.8304623961448669, "Finetune/Step": 1730, "Finetune/Step Time": 4.453755782917142} +{"Finetune/Learning Rate": 1.555537859273777e-05, "Finetune/Loss": 0.9394721984863281, "Finetune/Loss (Raw)": 1.1117618083953857, "Finetune/Step": 1731, "Finetune/Step Time": 4.450609538704157} +{"Finetune/Learning Rate": 1.5550197648399453e-05, "Finetune/Loss": 0.9410848617553711, "Finetune/Loss (Raw)": 1.1021815538406372, "Finetune/Step": 1732, "Finetune/Step Time": 4.46555070579052} +{"Finetune/Learning Rate": 1.554501455013779e-05, "Finetune/Loss": 0.9422717094421387, "Finetune/Loss (Raw)": 0.9808861017227173, "Finetune/Step": 1733, "Finetune/Step Time": 4.456766501069069} +{"Finetune/Learning Rate": 1.5539829299964236e-05, "Finetune/Loss": 0.9441116452217102, "Finetune/Loss (Raw)": 1.0099029541015625, "Finetune/Step": 1734, "Finetune/Step Time": 4.435524934902787} +{"Finetune/Learning Rate": 1.5534641899891084e-05, "Finetune/Loss": 0.944266676902771, "Finetune/Loss (Raw)": 0.9235670566558838, "Finetune/Step": 1735, "Finetune/Step Time": 4.4356371853500605} +{"Finetune/Learning Rate": 1.5529452351931473e-05, "Finetune/Loss": 0.9488106966018677, "Finetune/Loss (Raw)": 0.9703712463378906, "Finetune/Step": 1736, "Finetune/Step Time": 4.397772870957851} +{"Finetune/Learning Rate": 1.5524260658099352e-05, "Finetune/Loss": 0.9483629465103149, "Finetune/Loss (Raw)": 0.9031959772109985, "Finetune/Step": 1737, "Finetune/Step Time": 4.397198086604476} +{"Finetune/Learning Rate": 1.551906682040952e-05, "Finetune/Loss": 0.9438072443008423, "Finetune/Loss (Raw)": 0.35718223452568054, "Finetune/Step": 1738, "Finetune/Step Time": 4.436954401433468} +{"Finetune/Learning Rate": 1.5513870840877608e-05, "Finetune/Loss": 0.9443973302841187, "Finetune/Loss (Raw)": 0.9394555687904358, "Finetune/Step": 1739, "Finetune/Step Time": 4.415142605081201} +{"Finetune/Learning Rate": 1.5508672721520076e-05, "Finetune/Loss": 0.944100022315979, "Finetune/Loss (Raw)": 0.8820705413818359, "Finetune/Step": 1740, "Finetune/Step Time": 4.43156629987061} +{"Finetune/Learning Rate": 1.5503472464354207e-05, "Finetune/Loss": 0.9461435675621033, "Finetune/Loss (Raw)": 1.116848349571228, "Finetune/Step": 1741, "Finetune/Step Time": 4.4219623152166605} +{"Finetune/Learning Rate": 1.5498270071398116e-05, "Finetune/Loss": 0.9450879096984863, "Finetune/Loss (Raw)": 0.8591248393058777, "Finetune/Step": 1742, "Finetune/Step Time": 4.430255621671677} +{"Finetune/Learning Rate": 1.549306554467076e-05, "Finetune/Loss": 0.944309651851654, "Finetune/Loss (Raw)": 0.963178813457489, "Finetune/Step": 1743, "Finetune/Step Time": 4.451364109292626} +{"Finetune/Learning Rate": 1.5487858886191903e-05, "Finetune/Loss": 0.9439136981964111, "Finetune/Loss (Raw)": 0.9192855358123779, "Finetune/Step": 1744, "Finetune/Step Time": 4.452250182628632} +{"Finetune/Learning Rate": 1.548265009798216e-05, "Finetune/Loss": 0.9445873498916626, "Finetune/Loss (Raw)": 0.9837116003036499, "Finetune/Step": 1745, "Finetune/Step Time": 4.448032535612583} +{"Finetune/Learning Rate": 1.547743918206295e-05, "Finetune/Loss": 0.9456135034561157, "Finetune/Loss (Raw)": 1.0354728698730469, "Finetune/Step": 1746, "Finetune/Step Time": 4.447831999510527} +{"Finetune/Learning Rate": 1.5472226140456537e-05, "Finetune/Loss": 0.9425125122070312, "Finetune/Loss (Raw)": 0.43659842014312744, "Finetune/Step": 1747, "Finetune/Step Time": 4.48414952494204} +{"Finetune/Learning Rate": 1.546701097518599e-05, "Finetune/Loss": 0.9401817321777344, "Finetune/Loss (Raw)": 0.8521163463592529, "Finetune/Step": 1748, "Finetune/Step Time": 4.489014049991965} +{"Finetune/Learning Rate": 1.546179368827522e-05, "Finetune/Loss": 0.9410892724990845, "Finetune/Loss (Raw)": 0.9716705679893494, "Finetune/Step": 1749, "Finetune/Step Time": 4.487342674285173} +{"Finetune/Learning Rate": 1.545657428174895e-05, "Finetune/Loss": 0.9413078427314758, "Finetune/Loss (Raw)": 1.0938233137130737, "Finetune/Step": 1750, "Finetune/Step Time": 4.506529830396175} +{"Finetune/Learning Rate": 1.5451352757632734e-05, "Finetune/Loss": 0.9404408931732178, "Finetune/Loss (Raw)": 1.030063509941101, "Finetune/Step": 1751, "Finetune/Step Time": 4.510544590651989} +{"Finetune/Learning Rate": 1.544612911795294e-05, "Finetune/Loss": 0.9424880743026733, "Finetune/Loss (Raw)": 1.1825473308563232, "Finetune/Step": 1752, "Finetune/Step Time": 4.504082167521119} +{"Finetune/Learning Rate": 1.5440903364736763e-05, "Finetune/Loss": 0.9421074390411377, "Finetune/Loss (Raw)": 0.9406455755233765, "Finetune/Step": 1753, "Finetune/Step Time": 4.496325872838497} +{"Finetune/Learning Rate": 1.5435675500012212e-05, "Finetune/Loss": 0.94202721118927, "Finetune/Loss (Raw)": 0.9944189786911011, "Finetune/Step": 1754, "Finetune/Step Time": 4.5037830248475075} +{"Finetune/Learning Rate": 1.5430445525808123e-05, "Finetune/Loss": 0.941542387008667, "Finetune/Loss (Raw)": 0.9775046706199646, "Finetune/Step": 1755, "Finetune/Step Time": 4.510752389207482} +{"Finetune/Learning Rate": 1.5425213444154152e-05, "Finetune/Loss": 0.9423707723617554, "Finetune/Loss (Raw)": 1.0936028957366943, "Finetune/Step": 1756, "Finetune/Step Time": 4.519731041043997} +{"Finetune/Learning Rate": 1.5419979257080755e-05, "Finetune/Loss": 0.9377216100692749, "Finetune/Loss (Raw)": 0.3607640564441681, "Finetune/Step": 1757, "Finetune/Step Time": 4.562541930004954} +{"Finetune/Learning Rate": 1.541474296661923e-05, "Finetune/Loss": 0.9362401962280273, "Finetune/Loss (Raw)": 0.8486579060554504, "Finetune/Step": 1758, "Finetune/Step Time": 4.566845012828708} +{"Finetune/Learning Rate": 1.5409504574801667e-05, "Finetune/Loss": 0.9368985891342163, "Finetune/Loss (Raw)": 1.033116340637207, "Finetune/Step": 1759, "Finetune/Step Time": 4.572390912100673} +{"Finetune/Learning Rate": 1.5404264083660993e-05, "Finetune/Loss": 0.9377288818359375, "Finetune/Loss (Raw)": 1.0443998575210571, "Finetune/Step": 1760, "Finetune/Step Time": 4.570079766213894} +{"Finetune/Learning Rate": 1.539902149523094e-05, "Finetune/Loss": 0.9375277757644653, "Finetune/Loss (Raw)": 0.8573853969573975, "Finetune/Step": 1761, "Finetune/Step Time": 4.574432635679841} +{"Finetune/Learning Rate": 1.5393776811546047e-05, "Finetune/Loss": 0.9362553358078003, "Finetune/Loss (Raw)": 0.8444957733154297, "Finetune/Step": 1762, "Finetune/Step Time": 4.583765182644129} +{"Finetune/Learning Rate": 1.538853003464168e-05, "Finetune/Loss": 0.9371325969696045, "Finetune/Loss (Raw)": 1.0935944318771362, "Finetune/Step": 1763, "Finetune/Step Time": 4.578533813357353} +{"Finetune/Learning Rate": 1.538328116655401e-05, "Finetune/Loss": 0.937714695930481, "Finetune/Loss (Raw)": 1.0552821159362793, "Finetune/Step": 1764, "Finetune/Step Time": 4.598765766248107} +{"Finetune/Learning Rate": 1.5378030209320018e-05, "Finetune/Loss": 0.9372228384017944, "Finetune/Loss (Raw)": 0.9048450589179993, "Finetune/Step": 1765, "Finetune/Step Time": 4.611609065905213} +{"Finetune/Learning Rate": 1.53727771649775e-05, "Finetune/Loss": 0.9380149245262146, "Finetune/Loss (Raw)": 1.0055571794509888, "Finetune/Step": 1766, "Finetune/Step Time": 4.613251123577356} +{"Finetune/Learning Rate": 1.5367522035565055e-05, "Finetune/Loss": 0.9378656148910522, "Finetune/Loss (Raw)": 0.8248779773712158, "Finetune/Step": 1767, "Finetune/Step Time": 4.6139915734529495} +{"Finetune/Learning Rate": 1.5362264823122107e-05, "Finetune/Loss": 0.9368460774421692, "Finetune/Loss (Raw)": 0.8950024843215942, "Finetune/Step": 1768, "Finetune/Step Time": 4.614832367748022} +{"Finetune/Learning Rate": 1.5357005529688868e-05, "Finetune/Loss": 0.9351314306259155, "Finetune/Loss (Raw)": 0.8089545965194702, "Finetune/Step": 1769, "Finetune/Step Time": 4.6297038570046425} +{"Finetune/Learning Rate": 1.5351744157306366e-05, "Finetune/Loss": 0.935378909111023, "Finetune/Loss (Raw)": 1.019930124282837, "Finetune/Step": 1770, "Finetune/Step Time": 4.630592914298177} +{"Finetune/Learning Rate": 1.5346480708016445e-05, "Finetune/Loss": 0.9339973330497742, "Finetune/Loss (Raw)": 0.9044675230979919, "Finetune/Step": 1771, "Finetune/Step Time": 4.628974117338657} +{"Finetune/Learning Rate": 1.5341215183861738e-05, "Finetune/Loss": 0.9348562955856323, "Finetune/Loss (Raw)": 1.0910910367965698, "Finetune/Step": 1772, "Finetune/Step Time": 4.5977266151458025} +{"Finetune/Learning Rate": 1.53359475868857e-05, "Finetune/Loss": 0.9350959062576294, "Finetune/Loss (Raw)": 0.9545410871505737, "Finetune/Step": 1773, "Finetune/Step Time": 4.606921911239624} +{"Finetune/Learning Rate": 1.5330677919132577e-05, "Finetune/Loss": 0.9345154762268066, "Finetune/Loss (Raw)": 0.9571555852890015, "Finetune/Step": 1774, "Finetune/Step Time": 4.619874231517315} +{"Finetune/Learning Rate": 1.5325406182647426e-05, "Finetune/Loss": 0.9356033802032471, "Finetune/Loss (Raw)": 1.0381373167037964, "Finetune/Step": 1775, "Finetune/Step Time": 4.606709498912096} +{"Finetune/Learning Rate": 1.5320132379476107e-05, "Finetune/Loss": 0.9348287582397461, "Finetune/Loss (Raw)": 0.9506462216377258, "Finetune/Step": 1776, "Finetune/Step Time": 4.6054925713688135} +{"Finetune/Learning Rate": 1.531485651166528e-05, "Finetune/Loss": 0.9353724718093872, "Finetune/Loss (Raw)": 1.0105984210968018, "Finetune/Step": 1777, "Finetune/Step Time": 4.606734532862902} +{"Finetune/Learning Rate": 1.5309578581262404e-05, "Finetune/Loss": 0.9352620840072632, "Finetune/Loss (Raw)": 0.9487902522087097, "Finetune/Step": 1778, "Finetune/Step Time": 4.609006829559803} +{"Finetune/Learning Rate": 1.530429859031574e-05, "Finetune/Loss": 0.9369584321975708, "Finetune/Loss (Raw)": 1.1000795364379883, "Finetune/Step": 1779, "Finetune/Step Time": 4.593678329139948} +{"Finetune/Learning Rate": 1.5299016540874346e-05, "Finetune/Loss": 0.937281608581543, "Finetune/Loss (Raw)": 0.930944561958313, "Finetune/Step": 1780, "Finetune/Step Time": 4.585504541173577} +{"Finetune/Learning Rate": 1.5293732434988095e-05, "Finetune/Loss": 0.9360913038253784, "Finetune/Loss (Raw)": 0.8202233910560608, "Finetune/Step": 1781, "Finetune/Step Time": 4.587223749607801} +{"Finetune/Learning Rate": 1.5288446274707635e-05, "Finetune/Loss": 0.935455322265625, "Finetune/Loss (Raw)": 0.9350489377975464, "Finetune/Step": 1782, "Finetune/Step Time": 4.5745147205889225} +{"Finetune/Learning Rate": 1.5283158062084423e-05, "Finetune/Loss": 0.9391975402832031, "Finetune/Loss (Raw)": 0.8645467758178711, "Finetune/Step": 1783, "Finetune/Step Time": 4.537865549325943} +{"Finetune/Learning Rate": 1.5277867799170708e-05, "Finetune/Loss": 0.9399616122245789, "Finetune/Loss (Raw)": 1.0682506561279297, "Finetune/Step": 1784, "Finetune/Step Time": 4.5422243513166904} +{"Finetune/Learning Rate": 1.527257548801955e-05, "Finetune/Loss": 0.9390149712562561, "Finetune/Loss (Raw)": 0.7629987001419067, "Finetune/Step": 1785, "Finetune/Step Time": 4.55655569024384} +{"Finetune/Learning Rate": 1.526728113068478e-05, "Finetune/Loss": 0.9385485649108887, "Finetune/Loss (Raw)": 0.914302408695221, "Finetune/Step": 1786, "Finetune/Step Time": 4.554296502843499} +{"Finetune/Learning Rate": 1.526198472922104e-05, "Finetune/Loss": 0.9362925291061401, "Finetune/Loss (Raw)": 0.8001945614814758, "Finetune/Step": 1787, "Finetune/Step Time": 4.559718070551753} +{"Finetune/Learning Rate": 1.5256686285683753e-05, "Finetune/Loss": 0.9355214834213257, "Finetune/Loss (Raw)": 0.8719109296798706, "Finetune/Step": 1788, "Finetune/Step Time": 4.555329559370875} +{"Finetune/Learning Rate": 1.5251385802129149e-05, "Finetune/Loss": 0.9306016564369202, "Finetune/Loss (Raw)": 0.2982006072998047, "Finetune/Step": 1789, "Finetune/Step Time": 4.593178812414408} +{"Finetune/Learning Rate": 1.5246083280614241e-05, "Finetune/Loss": 0.9303510189056396, "Finetune/Loss (Raw)": 1.0363743305206299, "Finetune/Step": 1790, "Finetune/Step Time": 4.596146775409579} +{"Finetune/Learning Rate": 1.5240778723196833e-05, "Finetune/Loss": 0.9332391619682312, "Finetune/Loss (Raw)": 1.0822452306747437, "Finetune/Step": 1791, "Finetune/Step Time": 4.58878162689507} +{"Finetune/Learning Rate": 1.523547213193552e-05, "Finetune/Loss": 0.9348889589309692, "Finetune/Loss (Raw)": 1.0066708326339722, "Finetune/Step": 1792, "Finetune/Step Time": 4.5892618000507355} +{"Finetune/Learning Rate": 1.5230163508889686e-05, "Finetune/Loss": 0.9329755902290344, "Finetune/Loss (Raw)": 0.8216461539268494, "Finetune/Step": 1793, "Finetune/Step Time": 4.603911612182856} +{"Finetune/Learning Rate": 1.5224852856119503e-05, "Finetune/Loss": 0.9340904951095581, "Finetune/Loss (Raw)": 1.0463576316833496, "Finetune/Step": 1794, "Finetune/Step Time": 4.610285382717848} +{"Finetune/Learning Rate": 1.5219540175685938e-05, "Finetune/Loss": 0.9292330145835876, "Finetune/Loss (Raw)": 0.3733786642551422, "Finetune/Step": 1795, "Finetune/Step Time": 4.6488057896494865} +{"Finetune/Learning Rate": 1.5214225469650727e-05, "Finetune/Loss": 0.928006112575531, "Finetune/Loss (Raw)": 0.9115378260612488, "Finetune/Step": 1796, "Finetune/Step Time": 4.641392005607486} +{"Finetune/Learning Rate": 1.5208908740076416e-05, "Finetune/Loss": 0.9284243583679199, "Finetune/Loss (Raw)": 1.0099400281906128, "Finetune/Step": 1797, "Finetune/Step Time": 4.640726130455732} +{"Finetune/Learning Rate": 1.5203589989026318e-05, "Finetune/Loss": 0.9278845191001892, "Finetune/Loss (Raw)": 0.9299871921539307, "Finetune/Step": 1798, "Finetune/Step Time": 4.6442707646638155} +{"Finetune/Learning Rate": 1.5198269218564535e-05, "Finetune/Loss": 0.9280651807785034, "Finetune/Loss (Raw)": 0.9254262447357178, "Finetune/Step": 1799, "Finetune/Step Time": 4.635600835084915} +{"Finetune/Learning Rate": 1.5192946430755955e-05, "Finetune/Loss": 0.9270246624946594, "Finetune/Loss (Raw)": 0.9112081527709961, "Finetune/Step": 1800, "Finetune/Step Time": 4.634817898273468} +{"Finetune/Learning Rate": 1.518762162766625e-05, "Finetune/Loss": 0.9287753701210022, "Finetune/Loss (Raw)": 1.0675886869430542, "Finetune/Step": 1801, "Finetune/Step Time": 4.638996073976159} +{"Finetune/Learning Rate": 1.5182294811361871e-05, "Finetune/Loss": 0.9300500154495239, "Finetune/Loss (Raw)": 0.9586741328239441, "Finetune/Step": 1802, "Finetune/Step Time": 4.641052121296525} +{"Finetune/Learning Rate": 1.5176965983910053e-05, "Finetune/Loss": 0.9300441145896912, "Finetune/Loss (Raw)": 0.9300618171691895, "Finetune/Step": 1803, "Finetune/Step Time": 4.648084023967385} +{"Finetune/Learning Rate": 1.5171635147378804e-05, "Finetune/Loss": 0.9293279051780701, "Finetune/Loss (Raw)": 0.9854395985603333, "Finetune/Step": 1804, "Finetune/Step Time": 4.651035441085696} +{"Finetune/Learning Rate": 1.5166302303836928e-05, "Finetune/Loss": 0.9265583157539368, "Finetune/Loss (Raw)": 0.6550700068473816, "Finetune/Step": 1805, "Finetune/Step Time": 4.659591894596815} +{"Finetune/Learning Rate": 1.5160967455353985e-05, "Finetune/Loss": 0.92664635181427, "Finetune/Loss (Raw)": 0.8660971522331238, "Finetune/Step": 1806, "Finetune/Step Time": 4.671661235392094} +{"Finetune/Learning Rate": 1.5155630604000338e-05, "Finetune/Loss": 0.9309613704681396, "Finetune/Loss (Raw)": 0.943967342376709, "Finetune/Step": 1807, "Finetune/Step Time": 4.632891060784459} +{"Finetune/Learning Rate": 1.5150291751847106e-05, "Finetune/Loss": 0.9310554265975952, "Finetune/Loss (Raw)": 0.9624711275100708, "Finetune/Step": 1808, "Finetune/Step Time": 4.616943039000034} +{"Finetune/Learning Rate": 1.5144950900966198e-05, "Finetune/Loss": 0.9309174418449402, "Finetune/Loss (Raw)": 0.958023726940155, "Finetune/Step": 1809, "Finetune/Step Time": 4.616740707308054} +{"Finetune/Learning Rate": 1.5139608053430288e-05, "Finetune/Loss": 0.929726779460907, "Finetune/Loss (Raw)": 0.9638813734054565, "Finetune/Step": 1810, "Finetune/Step Time": 4.619812348857522} +{"Finetune/Learning Rate": 1.5134263211312837e-05, "Finetune/Loss": 0.9302401542663574, "Finetune/Loss (Raw)": 0.9445257782936096, "Finetune/Step": 1811, "Finetune/Step Time": 4.612725239247084} +{"Finetune/Learning Rate": 1.5128916376688071e-05, "Finetune/Loss": 0.9303511381149292, "Finetune/Loss (Raw)": 0.9242095947265625, "Finetune/Step": 1812, "Finetune/Step Time": 4.614605510607362} +{"Finetune/Learning Rate": 1.5123567551630993e-05, "Finetune/Loss": 0.9307446479797363, "Finetune/Loss (Raw)": 0.933900773525238, "Finetune/Step": 1813, "Finetune/Step Time": 4.6040513552725315} +{"Finetune/Learning Rate": 1.511821673821738e-05, "Finetune/Loss": 0.932113528251648, "Finetune/Loss (Raw)": 0.9925019145011902, "Finetune/Step": 1814, "Finetune/Step Time": 4.587030880153179} +{"Finetune/Learning Rate": 1.5112863938523774e-05, "Finetune/Loss": 0.9310229420661926, "Finetune/Loss (Raw)": 0.8851099014282227, "Finetune/Step": 1815, "Finetune/Step Time": 4.585038755089045} +{"Finetune/Learning Rate": 1.5107509154627491e-05, "Finetune/Loss": 0.9311367273330688, "Finetune/Loss (Raw)": 1.0522587299346924, "Finetune/Step": 1816, "Finetune/Step Time": 4.580456757918} +{"Finetune/Learning Rate": 1.5102152388606625e-05, "Finetune/Loss": 0.935899019241333, "Finetune/Loss (Raw)": 0.9201322197914124, "Finetune/Step": 1817, "Finetune/Step Time": 4.55069262906909} +{"Finetune/Learning Rate": 1.5096793642540027e-05, "Finetune/Loss": 0.9346828460693359, "Finetune/Loss (Raw)": 0.9141910076141357, "Finetune/Step": 1818, "Finetune/Step Time": 4.562730986624956} +{"Finetune/Learning Rate": 1.5091432918507322e-05, "Finetune/Loss": 0.9348958134651184, "Finetune/Loss (Raw)": 0.9783251285552979, "Finetune/Step": 1819, "Finetune/Step Time": 4.560432996600866} +{"Finetune/Learning Rate": 1.5086070218588904e-05, "Finetune/Loss": 0.9346105456352234, "Finetune/Loss (Raw)": 0.8574953675270081, "Finetune/Step": 1820, "Finetune/Step Time": 4.538356138393283} +{"Finetune/Learning Rate": 1.5080705544865932e-05, "Finetune/Loss": 0.9355248808860779, "Finetune/Loss (Raw)": 0.9680534601211548, "Finetune/Step": 1821, "Finetune/Step Time": 4.523835623636842} +{"Finetune/Learning Rate": 1.5075338899420326e-05, "Finetune/Loss": 0.9391045570373535, "Finetune/Loss (Raw)": 1.1953275203704834, "Finetune/Step": 1822, "Finetune/Step Time": 4.515852082520723} +{"Finetune/Learning Rate": 1.5069970284334785e-05, "Finetune/Loss": 0.9395809173583984, "Finetune/Loss (Raw)": 0.9657337665557861, "Finetune/Step": 1823, "Finetune/Step Time": 4.507360462099314} +{"Finetune/Learning Rate": 1.506459970169276e-05, "Finetune/Loss": 0.9386845231056213, "Finetune/Loss (Raw)": 0.8566061854362488, "Finetune/Step": 1824, "Finetune/Step Time": 4.51179133169353} +{"Finetune/Learning Rate": 1.5059227153578466e-05, "Finetune/Loss": 0.9380590915679932, "Finetune/Loss (Raw)": 0.9210712909698486, "Finetune/Step": 1825, "Finetune/Step Time": 4.512935781851411} +{"Finetune/Learning Rate": 1.505385264207689e-05, "Finetune/Loss": 0.9386804103851318, "Finetune/Loss (Raw)": 1.0221045017242432, "Finetune/Step": 1826, "Finetune/Step Time": 4.513663355261087} +{"Finetune/Learning Rate": 1.5048476169273767e-05, "Finetune/Loss": 0.9380051493644714, "Finetune/Loss (Raw)": 0.9497305154800415, "Finetune/Step": 1827, "Finetune/Step Time": 4.545885084196925} +{"Finetune/Learning Rate": 1.5043097737255607e-05, "Finetune/Loss": 0.9384914040565491, "Finetune/Loss (Raw)": 1.0836468935012817, "Finetune/Step": 1828, "Finetune/Step Time": 4.543604588136077} +{"Finetune/Learning Rate": 1.5037717348109677e-05, "Finetune/Loss": 0.9389952421188354, "Finetune/Loss (Raw)": 1.1300230026245117, "Finetune/Step": 1829, "Finetune/Step Time": 4.547501929104328} +{"Finetune/Learning Rate": 1.5032335003923994e-05, "Finetune/Loss": 0.9394921064376831, "Finetune/Loss (Raw)": 1.0603188276290894, "Finetune/Step": 1830, "Finetune/Step Time": 4.559470742940903} +{"Finetune/Learning Rate": 1.5026950706787346e-05, "Finetune/Loss": 0.9398806095123291, "Finetune/Loss (Raw)": 0.9216729402542114, "Finetune/Step": 1831, "Finetune/Step Time": 4.544541792944074} +{"Finetune/Learning Rate": 1.502156445878927e-05, "Finetune/Loss": 0.9453293085098267, "Finetune/Loss (Raw)": 1.0434916019439697, "Finetune/Step": 1832, "Finetune/Step Time": 4.511106578633189} +{"Finetune/Learning Rate": 1.5016176262020069e-05, "Finetune/Loss": 0.9446099400520325, "Finetune/Loss (Raw)": 0.85280442237854, "Finetune/Step": 1833, "Finetune/Step Time": 4.530353279784322} +{"Finetune/Learning Rate": 1.5010786118570791e-05, "Finetune/Loss": 0.94496750831604, "Finetune/Loss (Raw)": 1.089558720588684, "Finetune/Step": 1834, "Finetune/Step Time": 4.529981719329953} +{"Finetune/Learning Rate": 1.5005394030533246e-05, "Finetune/Loss": 0.9450518488883972, "Finetune/Loss (Raw)": 0.9663905501365662, "Finetune/Step": 1835, "Finetune/Step Time": 4.5386701468378305} +{"Finetune/Learning Rate": 1.5000000000000002e-05, "Finetune/Loss": 0.9452847242355347, "Finetune/Loss (Raw)": 0.9936972856521606, "Finetune/Step": 1836, "Finetune/Step Time": 4.533716784790158} +{"Finetune/Learning Rate": 1.4994604029064372e-05, "Finetune/Loss": 0.9441695809364319, "Finetune/Loss (Raw)": 0.9512950778007507, "Finetune/Step": 1837, "Finetune/Step Time": 4.529153324663639} +{"Finetune/Learning Rate": 1.4989206119820432e-05, "Finetune/Loss": 0.9441729784011841, "Finetune/Loss (Raw)": 1.0011675357818604, "Finetune/Step": 1838, "Finetune/Step Time": 4.521826129406691} +{"Finetune/Learning Rate": 1.4983806274363005e-05, "Finetune/Loss": 0.9440330266952515, "Finetune/Loss (Raw)": 0.9919118881225586, "Finetune/Step": 1839, "Finetune/Step Time": 4.523764558136463} +{"Finetune/Learning Rate": 1.497840449478766e-05, "Finetune/Loss": 0.943203330039978, "Finetune/Loss (Raw)": 1.0312644243240356, "Finetune/Step": 1840, "Finetune/Step Time": 4.533842908218503} +{"Finetune/Learning Rate": 1.4973000783190728e-05, "Finetune/Loss": 0.9419081807136536, "Finetune/Loss (Raw)": 0.837275505065918, "Finetune/Step": 1841, "Finetune/Step Time": 4.53556926175952} +{"Finetune/Learning Rate": 1.4967595141669278e-05, "Finetune/Loss": 0.9424829483032227, "Finetune/Loss (Raw)": 1.0716474056243896, "Finetune/Step": 1842, "Finetune/Step Time": 4.540386959910393} +{"Finetune/Learning Rate": 1.4962187572321143e-05, "Finetune/Loss": 0.9437035918235779, "Finetune/Loss (Raw)": 0.9864131808280945, "Finetune/Step": 1843, "Finetune/Step Time": 4.545820215716958} +{"Finetune/Learning Rate": 1.4956778077244884e-05, "Finetune/Loss": 0.9439185261726379, "Finetune/Loss (Raw)": 0.9518287181854248, "Finetune/Step": 1844, "Finetune/Step Time": 4.556935973465443} +{"Finetune/Learning Rate": 1.4951366658539832e-05, "Finetune/Loss": 0.9429163932800293, "Finetune/Loss (Raw)": 0.9404141902923584, "Finetune/Step": 1845, "Finetune/Step Time": 4.5503743048757315} +{"Finetune/Learning Rate": 1.4945953318306044e-05, "Finetune/Loss": 0.943638801574707, "Finetune/Loss (Raw)": 1.0758821964263916, "Finetune/Step": 1846, "Finetune/Step Time": 4.55191046372056} +{"Finetune/Learning Rate": 1.4940538058644332e-05, "Finetune/Loss": 0.9441260099411011, "Finetune/Loss (Raw)": 1.021390438079834, "Finetune/Step": 1847, "Finetune/Step Time": 4.560386734083295} +{"Finetune/Learning Rate": 1.4935120881656263e-05, "Finetune/Loss": 0.9435429573059082, "Finetune/Loss (Raw)": 0.9653282165527344, "Finetune/Step": 1848, "Finetune/Step Time": 4.552382355555892} +{"Finetune/Learning Rate": 1.4929701789444122e-05, "Finetune/Loss": 0.9439408183097839, "Finetune/Loss (Raw)": 1.0471247434616089, "Finetune/Step": 1849, "Finetune/Step Time": 4.545609934255481} +{"Finetune/Learning Rate": 1.4924280784110965e-05, "Finetune/Loss": 0.9443259239196777, "Finetune/Loss (Raw)": 1.0428338050842285, "Finetune/Step": 1850, "Finetune/Step Time": 4.551451185718179} +{"Finetune/Learning Rate": 1.4918857867760571e-05, "Finetune/Loss": 0.9428632855415344, "Finetune/Loss (Raw)": 0.8500944972038269, "Finetune/Step": 1851, "Finetune/Step Time": 4.5545914974063635} +{"Finetune/Learning Rate": 1.4913433042497473e-05, "Finetune/Loss": 0.9437308311462402, "Finetune/Loss (Raw)": 1.004872441291809, "Finetune/Step": 1852, "Finetune/Step Time": 4.555995786562562} +{"Finetune/Learning Rate": 1.4908006310426942e-05, "Finetune/Loss": 0.9437886476516724, "Finetune/Loss (Raw)": 0.9936736226081848, "Finetune/Step": 1853, "Finetune/Step Time": 4.557087734341621} +{"Finetune/Learning Rate": 1.4902577673654977e-05, "Finetune/Loss": 0.9445956349372864, "Finetune/Loss (Raw)": 0.9998493194580078, "Finetune/Step": 1854, "Finetune/Step Time": 4.55087068118155} +{"Finetune/Learning Rate": 1.4897147134288334e-05, "Finetune/Loss": 0.9449343085289001, "Finetune/Loss (Raw)": 1.0387667417526245, "Finetune/Step": 1855, "Finetune/Step Time": 4.535691566765308} +{"Finetune/Learning Rate": 1.4891714694434495e-05, "Finetune/Loss": 0.945061206817627, "Finetune/Loss (Raw)": 1.0867925882339478, "Finetune/Step": 1856, "Finetune/Step Time": 4.5355457831174135} +{"Finetune/Learning Rate": 1.4886280356201688e-05, "Finetune/Loss": 0.94418865442276, "Finetune/Loss (Raw)": 0.9167510867118835, "Finetune/Step": 1857, "Finetune/Step Time": 4.572620641440153} +{"Finetune/Learning Rate": 1.4880844121698872e-05, "Finetune/Loss": 0.9456093311309814, "Finetune/Loss (Raw)": 1.0123052597045898, "Finetune/Step": 1858, "Finetune/Step Time": 4.580260617658496} +{"Finetune/Learning Rate": 1.487540599303574e-05, "Finetune/Loss": 0.9452787637710571, "Finetune/Loss (Raw)": 1.0694490671157837, "Finetune/Step": 1859, "Finetune/Step Time": 4.574768332764506} +{"Finetune/Learning Rate": 1.4869965972322728e-05, "Finetune/Loss": 0.9438098073005676, "Finetune/Loss (Raw)": 0.9141596555709839, "Finetune/Step": 1860, "Finetune/Step Time": 4.563215831294656} +{"Finetune/Learning Rate": 1.4864524061670999e-05, "Finetune/Loss": 0.9428354501724243, "Finetune/Loss (Raw)": 0.8561615347862244, "Finetune/Step": 1861, "Finetune/Step Time": 4.563662486150861} +{"Finetune/Learning Rate": 1.4859080263192456e-05, "Finetune/Loss": 0.9417136907577515, "Finetune/Loss (Raw)": 0.8663201332092285, "Finetune/Step": 1862, "Finetune/Step Time": 4.548190193250775} +{"Finetune/Learning Rate": 1.4853634578999728e-05, "Finetune/Loss": 0.9410080909729004, "Finetune/Loss (Raw)": 0.8332551717758179, "Finetune/Step": 1863, "Finetune/Step Time": 4.547470059245825} +{"Finetune/Learning Rate": 1.4848187011206177e-05, "Finetune/Loss": 0.9400556683540344, "Finetune/Loss (Raw)": 0.848463237285614, "Finetune/Step": 1864, "Finetune/Step Time": 4.552149498835206} +{"Finetune/Learning Rate": 1.4842737561925901e-05, "Finetune/Loss": 0.9411281943321228, "Finetune/Loss (Raw)": 1.0404748916625977, "Finetune/Step": 1865, "Finetune/Step Time": 4.560225253924727} +{"Finetune/Learning Rate": 1.4837286233273722e-05, "Finetune/Loss": 0.9464432001113892, "Finetune/Loss (Raw)": 1.037500023841858, "Finetune/Step": 1866, "Finetune/Step Time": 4.5308042243123055} +{"Finetune/Learning Rate": 1.48318330273652e-05, "Finetune/Loss": 0.9476355314254761, "Finetune/Loss (Raw)": 1.0920714139938354, "Finetune/Step": 1867, "Finetune/Step Time": 4.537158351391554} +{"Finetune/Learning Rate": 1.4826377946316611e-05, "Finetune/Loss": 0.9483211636543274, "Finetune/Loss (Raw)": 0.9698426723480225, "Finetune/Step": 1868, "Finetune/Step Time": 4.538835296407342} +{"Finetune/Learning Rate": 1.4820920992244964e-05, "Finetune/Loss": 0.9478977918624878, "Finetune/Loss (Raw)": 1.0626498460769653, "Finetune/Step": 1869, "Finetune/Step Time": 4.536015797406435} +{"Finetune/Learning Rate": 1.481546216726801e-05, "Finetune/Loss": 0.9482990503311157, "Finetune/Loss (Raw)": 0.9104784727096558, "Finetune/Step": 1870, "Finetune/Step Time": 4.528338374570012} +{"Finetune/Learning Rate": 1.4810001473504194e-05, "Finetune/Loss": 0.9483078718185425, "Finetune/Loss (Raw)": 0.9643147587776184, "Finetune/Step": 1871, "Finetune/Step Time": 4.53662890009582} +{"Finetune/Learning Rate": 1.4804538913072719e-05, "Finetune/Loss": 0.947748601436615, "Finetune/Loss (Raw)": 0.8477009534835815, "Finetune/Step": 1872, "Finetune/Step Time": 4.5415413454174995} +{"Finetune/Learning Rate": 1.479907448809349e-05, "Finetune/Loss": 0.9429471492767334, "Finetune/Loss (Raw)": 0.36912259459495544, "Finetune/Step": 1873, "Finetune/Step Time": 4.568255800753832} +{"Finetune/Learning Rate": 1.4793608200687146e-05, "Finetune/Loss": 0.943128228187561, "Finetune/Loss (Raw)": 1.0586484670639038, "Finetune/Step": 1874, "Finetune/Step Time": 4.564371047541499} +{"Finetune/Learning Rate": 1.478814005297505e-05, "Finetune/Loss": 0.9468637704849243, "Finetune/Loss (Raw)": 0.9147540926933289, "Finetune/Step": 1875, "Finetune/Step Time": 4.532523484900594} +{"Finetune/Learning Rate": 1.4782670047079275e-05, "Finetune/Loss": 0.9477311968803406, "Finetune/Loss (Raw)": 0.9631497859954834, "Finetune/Step": 1876, "Finetune/Step Time": 4.524517767131329} +{"Finetune/Learning Rate": 1.477719818512263e-05, "Finetune/Loss": 0.9476534128189087, "Finetune/Loss (Raw)": 0.9617146253585815, "Finetune/Step": 1877, "Finetune/Step Time": 4.529166588559747} +{"Finetune/Learning Rate": 1.4771724469228638e-05, "Finetune/Loss": 0.9453380107879639, "Finetune/Loss (Raw)": 0.7974467277526855, "Finetune/Step": 1878, "Finetune/Step Time": 4.521888416260481} +{"Finetune/Learning Rate": 1.4766248901521539e-05, "Finetune/Loss": 0.9456877708435059, "Finetune/Loss (Raw)": 1.0748316049575806, "Finetune/Step": 1879, "Finetune/Step Time": 4.513127775862813} +{"Finetune/Learning Rate": 1.4760771484126296e-05, "Finetune/Loss": 0.9427673816680908, "Finetune/Loss (Raw)": 0.8087457418441772, "Finetune/Step": 1880, "Finetune/Step Time": 4.53401841968298} +{"Finetune/Learning Rate": 1.4755292219168581e-05, "Finetune/Loss": 0.9419499635696411, "Finetune/Loss (Raw)": 0.83601313829422, "Finetune/Step": 1881, "Finetune/Step Time": 4.541713250800967} +{"Finetune/Learning Rate": 1.4749811108774798e-05, "Finetune/Loss": 0.9419437646865845, "Finetune/Loss (Raw)": 0.9936160445213318, "Finetune/Step": 1882, "Finetune/Step Time": 4.538013419136405} +{"Finetune/Learning Rate": 1.4744328155072058e-05, "Finetune/Loss": 0.9412033557891846, "Finetune/Loss (Raw)": 0.8827442526817322, "Finetune/Step": 1883, "Finetune/Step Time": 4.562262898311019} +{"Finetune/Learning Rate": 1.4738843360188184e-05, "Finetune/Loss": 0.9354614615440369, "Finetune/Loss (Raw)": 0.35863929986953735, "Finetune/Step": 1884, "Finetune/Step Time": 4.597470512613654} +{"Finetune/Learning Rate": 1.4733356726251722e-05, "Finetune/Loss": 0.9403077960014343, "Finetune/Loss (Raw)": 0.9810880422592163, "Finetune/Step": 1885, "Finetune/Step Time": 4.562274392694235} +{"Finetune/Learning Rate": 1.4727868255391924e-05, "Finetune/Loss": 0.9421653747558594, "Finetune/Loss (Raw)": 1.0864311456680298, "Finetune/Step": 1886, "Finetune/Step Time": 4.566656863316894} +{"Finetune/Learning Rate": 1.472237794973876e-05, "Finetune/Loss": 0.940540075302124, "Finetune/Loss (Raw)": 0.8250740170478821, "Finetune/Step": 1887, "Finetune/Step Time": 4.56187834776938} +{"Finetune/Learning Rate": 1.471688581142291e-05, "Finetune/Loss": 0.9384199976921082, "Finetune/Loss (Raw)": 0.7730311751365662, "Finetune/Step": 1888, "Finetune/Step Time": 4.567341269925237} +{"Finetune/Learning Rate": 1.4711391842575765e-05, "Finetune/Loss": 0.9390379190444946, "Finetune/Loss (Raw)": 0.9364776611328125, "Finetune/Step": 1889, "Finetune/Step Time": 4.573221197351813} +{"Finetune/Learning Rate": 1.4705896045329432e-05, "Finetune/Loss": 0.9396748542785645, "Finetune/Loss (Raw)": 0.9260271787643433, "Finetune/Step": 1890, "Finetune/Step Time": 4.566896377131343} +{"Finetune/Learning Rate": 1.4700398421816714e-05, "Finetune/Loss": 0.9390881657600403, "Finetune/Loss (Raw)": 1.018494963645935, "Finetune/Step": 1891, "Finetune/Step Time": 4.5510059259831905} +{"Finetune/Learning Rate": 1.469489897417114e-05, "Finetune/Loss": 0.9396196007728577, "Finetune/Loss (Raw)": 1.1233134269714355, "Finetune/Step": 1892, "Finetune/Step Time": 4.531378900632262} +{"Finetune/Learning Rate": 1.468939770452693e-05, "Finetune/Loss": 0.9399106502532959, "Finetune/Loss (Raw)": 0.9420979022979736, "Finetune/Step": 1893, "Finetune/Step Time": 4.533081956207752} +{"Finetune/Learning Rate": 1.4683894615019028e-05, "Finetune/Loss": 0.9385579824447632, "Finetune/Loss (Raw)": 0.8324111700057983, "Finetune/Step": 1894, "Finetune/Step Time": 4.524752993136644} +{"Finetune/Learning Rate": 1.4678389707783072e-05, "Finetune/Loss": 0.9392777681350708, "Finetune/Loss (Raw)": 0.9170135855674744, "Finetune/Step": 1895, "Finetune/Step Time": 4.524348685517907} +{"Finetune/Learning Rate": 1.4672882984955404e-05, "Finetune/Loss": 0.9398326873779297, "Finetune/Loss (Raw)": 0.966034471988678, "Finetune/Step": 1896, "Finetune/Step Time": 4.526045145466924} +{"Finetune/Learning Rate": 1.4667374448673081e-05, "Finetune/Loss": 0.9421147108078003, "Finetune/Loss (Raw)": 1.1010503768920898, "Finetune/Step": 1897, "Finetune/Step Time": 4.512771926820278} +{"Finetune/Learning Rate": 1.4661864101073857e-05, "Finetune/Loss": 0.9419606924057007, "Finetune/Loss (Raw)": 1.0002096891403198, "Finetune/Step": 1898, "Finetune/Step Time": 4.517804911360145} +{"Finetune/Learning Rate": 1.465635194429619e-05, "Finetune/Loss": 0.941399335861206, "Finetune/Loss (Raw)": 0.8326206803321838, "Finetune/Step": 1899, "Finetune/Step Time": 4.536678498610854} +{"Finetune/Learning Rate": 1.4650837980479246e-05, "Finetune/Loss": 0.9398723840713501, "Finetune/Loss (Raw)": 0.8956395387649536, "Finetune/Step": 1900, "Finetune/Step Time": 4.5374315325170755} +{"Finetune/Learning Rate": 1.4645322211762876e-05, "Finetune/Loss": 0.9394862055778503, "Finetune/Loss (Raw)": 0.9051079154014587, "Finetune/Step": 1901, "Finetune/Step Time": 4.533826105296612} +{"Finetune/Learning Rate": 1.4639804640287652e-05, "Finetune/Loss": 0.939775288105011, "Finetune/Loss (Raw)": 0.9941611289978027, "Finetune/Step": 1902, "Finetune/Step Time": 4.524725710973144} +{"Finetune/Learning Rate": 1.4634285268194828e-05, "Finetune/Loss": 0.9383116960525513, "Finetune/Loss (Raw)": 0.8507975339889526, "Finetune/Step": 1903, "Finetune/Step Time": 4.550563083961606} +{"Finetune/Learning Rate": 1.462876409762637e-05, "Finetune/Loss": 0.9382573366165161, "Finetune/Loss (Raw)": 0.9436866641044617, "Finetune/Step": 1904, "Finetune/Step Time": 4.555511679500341} +{"Finetune/Learning Rate": 1.4623241130724937e-05, "Finetune/Loss": 0.9328882694244385, "Finetune/Loss (Raw)": 0.3233567476272583, "Finetune/Step": 1905, "Finetune/Step Time": 4.5892306212335825} +{"Finetune/Learning Rate": 1.4617716369633876e-05, "Finetune/Loss": 0.9328069686889648, "Finetune/Loss (Raw)": 0.9383853673934937, "Finetune/Step": 1906, "Finetune/Step Time": 4.601250242441893} +{"Finetune/Learning Rate": 1.461218981649725e-05, "Finetune/Loss": 0.931678056716919, "Finetune/Loss (Raw)": 0.9555845260620117, "Finetune/Step": 1907, "Finetune/Step Time": 4.616537597030401} +{"Finetune/Learning Rate": 1.4606661473459801e-05, "Finetune/Loss": 0.9314218759536743, "Finetune/Loss (Raw)": 0.8981567025184631, "Finetune/Step": 1908, "Finetune/Step Time": 4.614293925464153} +{"Finetune/Learning Rate": 1.4601131342666973e-05, "Finetune/Loss": 0.9307832717895508, "Finetune/Loss (Raw)": 0.7384713888168335, "Finetune/Step": 1909, "Finetune/Step Time": 4.614787546917796} +{"Finetune/Learning Rate": 1.45955994262649e-05, "Finetune/Loss": 0.9307886362075806, "Finetune/Loss (Raw)": 0.9357326030731201, "Finetune/Step": 1910, "Finetune/Step Time": 4.616902261972427} +{"Finetune/Learning Rate": 1.459006572640041e-05, "Finetune/Loss": 0.9317535758018494, "Finetune/Loss (Raw)": 0.9880656003952026, "Finetune/Step": 1911, "Finetune/Step Time": 4.622331975027919} +{"Finetune/Learning Rate": 1.4584530245221025e-05, "Finetune/Loss": 0.9310219287872314, "Finetune/Loss (Raw)": 0.9745960831642151, "Finetune/Step": 1912, "Finetune/Step Time": 4.615799941122532} +{"Finetune/Learning Rate": 1.4578992984874956e-05, "Finetune/Loss": 0.932684600353241, "Finetune/Loss (Raw)": 0.9758211374282837, "Finetune/Step": 1913, "Finetune/Step Time": 4.600188255310059} +{"Finetune/Learning Rate": 1.457345394751111e-05, "Finetune/Loss": 0.9340624809265137, "Finetune/Loss (Raw)": 1.0906776189804077, "Finetune/Step": 1914, "Finetune/Step Time": 4.597984218969941} +{"Finetune/Learning Rate": 1.4567913135279075e-05, "Finetune/Loss": 0.9360162019729614, "Finetune/Loss (Raw)": 1.0502687692642212, "Finetune/Step": 1915, "Finetune/Step Time": 4.620041897520423} +{"Finetune/Learning Rate": 1.456237055032913e-05, "Finetune/Loss": 0.9364258050918579, "Finetune/Loss (Raw)": 0.9243346452713013, "Finetune/Step": 1916, "Finetune/Step Time": 4.619703345000744} +{"Finetune/Learning Rate": 1.455682619481225e-05, "Finetune/Loss": 0.9407212734222412, "Finetune/Loss (Raw)": 0.8480279445648193, "Finetune/Step": 1917, "Finetune/Step Time": 4.5815197210758924} +{"Finetune/Learning Rate": 1.4551280070880089e-05, "Finetune/Loss": 0.9396313428878784, "Finetune/Loss (Raw)": 0.8968662023544312, "Finetune/Step": 1918, "Finetune/Step Time": 4.569310121238232} +{"Finetune/Learning Rate": 1.4545732180684989e-05, "Finetune/Loss": 0.9386047124862671, "Finetune/Loss (Raw)": 0.950835108757019, "Finetune/Step": 1919, "Finetune/Step Time": 4.565243059769273} +{"Finetune/Learning Rate": 1.4540182526379976e-05, "Finetune/Loss": 0.9374512434005737, "Finetune/Loss (Raw)": 0.8590224385261536, "Finetune/Step": 1920, "Finetune/Step Time": 4.574980305507779} +{"Finetune/Learning Rate": 1.4534631110118763e-05, "Finetune/Loss": 0.9390701055526733, "Finetune/Loss (Raw)": 1.028855562210083, "Finetune/Step": 1921, "Finetune/Step Time": 4.558949518948793} +{"Finetune/Learning Rate": 1.4529077934055754e-05, "Finetune/Loss": 0.9332778453826904, "Finetune/Loss (Raw)": 0.30495205521583557, "Finetune/Step": 1922, "Finetune/Step Time": 4.595022862777114} +{"Finetune/Learning Rate": 1.4523523000346012e-05, "Finetune/Loss": 0.9379377365112305, "Finetune/Loss (Raw)": 0.9698399305343628, "Finetune/Step": 1923, "Finetune/Step Time": 4.558041961863637} +{"Finetune/Learning Rate": 1.4517966311145313e-05, "Finetune/Loss": 0.9389896392822266, "Finetune/Loss (Raw)": 1.0461828708648682, "Finetune/Step": 1924, "Finetune/Step Time": 4.55672044120729} +{"Finetune/Learning Rate": 1.4512407868610094e-05, "Finetune/Loss": 0.940172553062439, "Finetune/Loss (Raw)": 1.1613560914993286, "Finetune/Step": 1925, "Finetune/Step Time": 4.554353807121515} +{"Finetune/Learning Rate": 1.450684767489748e-05, "Finetune/Loss": 0.9355579614639282, "Finetune/Loss (Raw)": 0.3393196165561676, "Finetune/Step": 1926, "Finetune/Step Time": 4.587334776297212} +{"Finetune/Learning Rate": 1.4501285732165271e-05, "Finetune/Loss": 0.9314596652984619, "Finetune/Loss (Raw)": 0.40084779262542725, "Finetune/Step": 1927, "Finetune/Step Time": 4.62683979049325} +{"Finetune/Learning Rate": 1.4495722042571946e-05, "Finetune/Loss": 0.9324442744255066, "Finetune/Loss (Raw)": 1.037231206893921, "Finetune/Step": 1928, "Finetune/Step Time": 4.609724348410964} +{"Finetune/Learning Rate": 1.4490156608276674e-05, "Finetune/Loss": 0.9305568337440491, "Finetune/Loss (Raw)": 0.8259996771812439, "Finetune/Step": 1929, "Finetune/Step Time": 4.601168781518936} +{"Finetune/Learning Rate": 1.4484589431439283e-05, "Finetune/Loss": 0.9301853179931641, "Finetune/Loss (Raw)": 0.9111229181289673, "Finetune/Step": 1930, "Finetune/Step Time": 4.586720813065767} +{"Finetune/Learning Rate": 1.4479020514220285e-05, "Finetune/Loss": 0.9304642677307129, "Finetune/Loss (Raw)": 0.9657713770866394, "Finetune/Step": 1931, "Finetune/Step Time": 4.564761757850647} +{"Finetune/Learning Rate": 1.4473449858780874e-05, "Finetune/Loss": 0.9293667674064636, "Finetune/Loss (Raw)": 0.8449543714523315, "Finetune/Step": 1932, "Finetune/Step Time": 4.579071065410972} +{"Finetune/Learning Rate": 1.4467877467282906e-05, "Finetune/Loss": 0.9318569898605347, "Finetune/Loss (Raw)": 0.9738200902938843, "Finetune/Step": 1933, "Finetune/Step Time": 4.572069019079208} +{"Finetune/Learning Rate": 1.4462303341888925e-05, "Finetune/Loss": 0.9329644441604614, "Finetune/Loss (Raw)": 1.0078524351119995, "Finetune/Step": 1934, "Finetune/Step Time": 4.55837893858552} +{"Finetune/Learning Rate": 1.4456727484762137e-05, "Finetune/Loss": 0.9337457418441772, "Finetune/Loss (Raw)": 1.0439648628234863, "Finetune/Step": 1935, "Finetune/Step Time": 4.564668353646994} +{"Finetune/Learning Rate": 1.4451149898066418e-05, "Finetune/Loss": 0.9327342510223389, "Finetune/Loss (Raw)": 0.8330009579658508, "Finetune/Step": 1936, "Finetune/Step Time": 4.5784653797745705} +{"Finetune/Learning Rate": 1.4445570583966329e-05, "Finetune/Loss": 0.9328586459159851, "Finetune/Loss (Raw)": 0.9739508628845215, "Finetune/Step": 1937, "Finetune/Step Time": 4.577444804832339} +{"Finetune/Learning Rate": 1.4439989544627085e-05, "Finetune/Loss": 0.9284838438034058, "Finetune/Loss (Raw)": 0.4039052724838257, "Finetune/Step": 1938, "Finetune/Step Time": 4.603589596226811} +{"Finetune/Learning Rate": 1.4434406782214584e-05, "Finetune/Loss": 0.9285017251968384, "Finetune/Loss (Raw)": 0.94681715965271, "Finetune/Step": 1939, "Finetune/Step Time": 4.599027153104544} +{"Finetune/Learning Rate": 1.4428822298895386e-05, "Finetune/Loss": 0.9259631037712097, "Finetune/Loss (Raw)": 0.5992664098739624, "Finetune/Step": 1940, "Finetune/Step Time": 4.599887797608972} +{"Finetune/Learning Rate": 1.442323609683672e-05, "Finetune/Loss": 0.9257463216781616, "Finetune/Loss (Raw)": 0.9061498641967773, "Finetune/Step": 1941, "Finetune/Step Time": 4.604229023680091} +{"Finetune/Learning Rate": 1.4417648178206481e-05, "Finetune/Loss": 0.926903247833252, "Finetune/Loss (Raw)": 1.140587329864502, "Finetune/Step": 1942, "Finetune/Step Time": 4.611684799194336} +{"Finetune/Learning Rate": 1.4412058545173228e-05, "Finetune/Loss": 0.928119957447052, "Finetune/Loss (Raw)": 1.040848970413208, "Finetune/Step": 1943, "Finetune/Step Time": 4.605642458423972} +{"Finetune/Learning Rate": 1.4406467199906194e-05, "Finetune/Loss": 0.9274355173110962, "Finetune/Loss (Raw)": 0.964651882648468, "Finetune/Step": 1944, "Finetune/Step Time": 4.605029912665486} +{"Finetune/Learning Rate": 1.440087414457527e-05, "Finetune/Loss": 0.9232591986656189, "Finetune/Loss (Raw)": 0.38556450605392456, "Finetune/Step": 1945, "Finetune/Step Time": 4.636934086680412} +{"Finetune/Learning Rate": 1.4395279381351008e-05, "Finetune/Loss": 0.9235037565231323, "Finetune/Loss (Raw)": 0.9454978108406067, "Finetune/Step": 1946, "Finetune/Step Time": 4.643706142902374} +{"Finetune/Learning Rate": 1.438968291240463e-05, "Finetune/Loss": 0.9246947765350342, "Finetune/Loss (Raw)": 1.1307653188705444, "Finetune/Step": 1947, "Finetune/Step Time": 4.641408029943705} +{"Finetune/Learning Rate": 1.4384084739908016e-05, "Finetune/Loss": 0.9265066385269165, "Finetune/Loss (Raw)": 1.0894274711608887, "Finetune/Step": 1948, "Finetune/Step Time": 4.641019502654672} +{"Finetune/Learning Rate": 1.4378484866033705e-05, "Finetune/Loss": 0.9273021817207336, "Finetune/Loss (Raw)": 1.0698708295822144, "Finetune/Step": 1949, "Finetune/Step Time": 4.639293264597654} +{"Finetune/Learning Rate": 1.43728832929549e-05, "Finetune/Loss": 0.926423192024231, "Finetune/Loss (Raw)": 1.0828170776367188, "Finetune/Step": 1950, "Finetune/Step Time": 4.635583478957415} +{"Finetune/Learning Rate": 1.4367280022845462e-05, "Finetune/Loss": 0.9272587895393372, "Finetune/Loss (Raw)": 1.0726943016052246, "Finetune/Step": 1951, "Finetune/Step Time": 4.639956144616008} +{"Finetune/Learning Rate": 1.4361675057879913e-05, "Finetune/Loss": 0.9290089011192322, "Finetune/Loss (Raw)": 1.0806183815002441, "Finetune/Step": 1952, "Finetune/Step Time": 4.634228937327862} +{"Finetune/Learning Rate": 1.4356068400233429e-05, "Finetune/Loss": 0.9275000095367432, "Finetune/Loss (Raw)": 0.7279404401779175, "Finetune/Step": 1953, "Finetune/Step Time": 4.6459658201783895} +{"Finetune/Learning Rate": 1.435046005208184e-05, "Finetune/Loss": 0.9261764287948608, "Finetune/Loss (Raw)": 0.8526788949966431, "Finetune/Step": 1954, "Finetune/Step Time": 4.659505957737565} +{"Finetune/Learning Rate": 1.4344850015601644e-05, "Finetune/Loss": 0.9253901243209839, "Finetune/Loss (Raw)": 0.8490820527076721, "Finetune/Step": 1955, "Finetune/Step Time": 4.644226875156164} +{"Finetune/Learning Rate": 1.4339238292969981e-05, "Finetune/Loss": 0.9236321449279785, "Finetune/Loss (Raw)": 0.8586358428001404, "Finetune/Step": 1956, "Finetune/Step Time": 4.638317430391908} +{"Finetune/Learning Rate": 1.4333624886364655e-05, "Finetune/Loss": 0.9232335686683655, "Finetune/Loss (Raw)": 1.0789990425109863, "Finetune/Step": 1957, "Finetune/Step Time": 4.631664905697107} +{"Finetune/Learning Rate": 1.4328009797964115e-05, "Finetune/Loss": 0.9233312606811523, "Finetune/Loss (Raw)": 1.0728192329406738, "Finetune/Step": 1958, "Finetune/Step Time": 4.628178674727678} +{"Finetune/Learning Rate": 1.432239302994747e-05, "Finetune/Loss": 0.9227354526519775, "Finetune/Loss (Raw)": 0.8454086780548096, "Finetune/Step": 1959, "Finetune/Step Time": 4.630505319684744} +{"Finetune/Learning Rate": 1.431677458449448e-05, "Finetune/Loss": 0.923403799533844, "Finetune/Loss (Raw)": 1.129043459892273, "Finetune/Step": 1960, "Finetune/Step Time": 4.635638173669577} +{"Finetune/Learning Rate": 1.431115446378555e-05, "Finetune/Loss": 0.9232117533683777, "Finetune/Loss (Raw)": 0.8282240629196167, "Finetune/Step": 1961, "Finetune/Step Time": 4.623247139155865} +{"Finetune/Learning Rate": 1.4305532670001743e-05, "Finetune/Loss": 0.9179120063781738, "Finetune/Loss (Raw)": 0.4111931025981903, "Finetune/Step": 1962, "Finetune/Step Time": 4.663420790806413} +{"Finetune/Learning Rate": 1.4299909205324762e-05, "Finetune/Loss": 0.9134760499000549, "Finetune/Loss (Raw)": 0.39858299493789673, "Finetune/Step": 1963, "Finetune/Step Time": 4.687726188451052} +{"Finetune/Learning Rate": 1.4294284071936968e-05, "Finetune/Loss": 0.913678765296936, "Finetune/Loss (Raw)": 1.0196529626846313, "Finetune/Step": 1964, "Finetune/Step Time": 4.694480713456869} +{"Finetune/Learning Rate": 1.4288657272021367e-05, "Finetune/Loss": 0.912759006023407, "Finetune/Loss (Raw)": 0.8335580229759216, "Finetune/Step": 1965, "Finetune/Step Time": 4.693562172353268} +{"Finetune/Learning Rate": 1.4283028807761603e-05, "Finetune/Loss": 0.9118071794509888, "Finetune/Loss (Raw)": 0.879336416721344, "Finetune/Step": 1966, "Finetune/Step Time": 4.688389286398888} +{"Finetune/Learning Rate": 1.4277398681341982e-05, "Finetune/Loss": 0.9106333255767822, "Finetune/Loss (Raw)": 0.8416571617126465, "Finetune/Step": 1967, "Finetune/Step Time": 4.676310615614057} +{"Finetune/Learning Rate": 1.4271766894947441e-05, "Finetune/Loss": 0.9105490446090698, "Finetune/Loss (Raw)": 1.0204739570617676, "Finetune/Step": 1968, "Finetune/Step Time": 4.662427186965942} +{"Finetune/Learning Rate": 1.4266133450763564e-05, "Finetune/Loss": 0.9074978232383728, "Finetune/Loss (Raw)": 0.4467237889766693, "Finetune/Step": 1969, "Finetune/Step Time": 4.6979331243783236} +{"Finetune/Learning Rate": 1.426049835097659e-05, "Finetune/Loss": 0.906315267086029, "Finetune/Loss (Raw)": 0.9202803373336792, "Finetune/Step": 1970, "Finetune/Step Time": 4.686532638967037} +{"Finetune/Learning Rate": 1.425486159777338e-05, "Finetune/Loss": 0.9017802476882935, "Finetune/Loss (Raw)": 0.4059317111968994, "Finetune/Step": 1971, "Finetune/Step Time": 4.707940179854631} +{"Finetune/Learning Rate": 1.4249223193341456e-05, "Finetune/Loss": 0.9014849662780762, "Finetune/Loss (Raw)": 0.9140255451202393, "Finetune/Step": 1972, "Finetune/Step Time": 4.696055233478546} +{"Finetune/Learning Rate": 1.4243583139868969e-05, "Finetune/Loss": 0.9009524583816528, "Finetune/Loss (Raw)": 0.8722590804100037, "Finetune/Step": 1973, "Finetune/Step Time": 4.723348313942552} +{"Finetune/Learning Rate": 1.423794143954471e-05, "Finetune/Loss": 0.9010002613067627, "Finetune/Loss (Raw)": 1.0819964408874512, "Finetune/Step": 1974, "Finetune/Step Time": 4.722349653020501} +{"Finetune/Learning Rate": 1.4232298094558125e-05, "Finetune/Loss": 0.9002394676208496, "Finetune/Loss (Raw)": 0.9240103960037231, "Finetune/Step": 1975, "Finetune/Step Time": 4.712249666452408} +{"Finetune/Learning Rate": 1.4226653107099273e-05, "Finetune/Loss": 0.9001672267913818, "Finetune/Loss (Raw)": 0.9560801386833191, "Finetune/Step": 1976, "Finetune/Step Time": 4.721273168921471} +{"Finetune/Learning Rate": 1.422100647935887e-05, "Finetune/Loss": 0.8997277617454529, "Finetune/Loss (Raw)": 0.9908771514892578, "Finetune/Step": 1977, "Finetune/Step Time": 4.729727378115058} +{"Finetune/Learning Rate": 1.421535821352826e-05, "Finetune/Loss": 0.8998125791549683, "Finetune/Loss (Raw)": 1.053693175315857, "Finetune/Step": 1978, "Finetune/Step Time": 4.723127270117402} +{"Finetune/Learning Rate": 1.4209708311799424e-05, "Finetune/Loss": 0.8961503505706787, "Finetune/Loss (Raw)": 0.3813228905200958, "Finetune/Step": 1979, "Finetune/Step Time": 4.737187931314111} +{"Finetune/Learning Rate": 1.4204056776364985e-05, "Finetune/Loss": 0.8948875069618225, "Finetune/Loss (Raw)": 0.8432307839393616, "Finetune/Step": 1980, "Finetune/Step Time": 4.750409198924899} +{"Finetune/Learning Rate": 1.4198403609418184e-05, "Finetune/Loss": 0.8949921727180481, "Finetune/Loss (Raw)": 1.0070695877075195, "Finetune/Step": 1981, "Finetune/Step Time": 4.750660127028823} +{"Finetune/Learning Rate": 1.419274881315291e-05, "Finetune/Loss": 0.8947503566741943, "Finetune/Loss (Raw)": 0.9688957929611206, "Finetune/Step": 1982, "Finetune/Step Time": 4.758640099316835} +{"Finetune/Learning Rate": 1.418709238976368e-05, "Finetune/Loss": 0.8937028050422668, "Finetune/Loss (Raw)": 0.9046808481216431, "Finetune/Step": 1983, "Finetune/Step Time": 4.7574455719441175} +{"Finetune/Learning Rate": 1.4181434341445634e-05, "Finetune/Loss": 0.8938297629356384, "Finetune/Loss (Raw)": 1.1030480861663818, "Finetune/Step": 1984, "Finetune/Step Time": 4.7574719693511724} +{"Finetune/Learning Rate": 1.4175774670394561e-05, "Finetune/Loss": 0.8945814371109009, "Finetune/Loss (Raw)": 1.0129611492156982, "Finetune/Step": 1985, "Finetune/Step Time": 4.717588018625975} +{"Finetune/Learning Rate": 1.4170113378806862e-05, "Finetune/Loss": 0.8958990573883057, "Finetune/Loss (Raw)": 1.1809638738632202, "Finetune/Step": 1986, "Finetune/Step Time": 4.707626739516854} +{"Finetune/Learning Rate": 1.4164450468879578e-05, "Finetune/Loss": 0.8949872255325317, "Finetune/Loss (Raw)": 0.9527406096458435, "Finetune/Step": 1987, "Finetune/Step Time": 4.734204113483429} +{"Finetune/Learning Rate": 1.4158785942810367e-05, "Finetune/Loss": 0.8947327733039856, "Finetune/Loss (Raw)": 0.8815820217132568, "Finetune/Step": 1988, "Finetune/Step Time": 4.7321465369313955} +{"Finetune/Learning Rate": 1.4153119802797526e-05, "Finetune/Loss": 0.8956636190414429, "Finetune/Loss (Raw)": 0.9753046035766602, "Finetune/Step": 1989, "Finetune/Step Time": 4.7340754475444555} +{"Finetune/Learning Rate": 1.4147452051039974e-05, "Finetune/Loss": 0.8967190384864807, "Finetune/Loss (Raw)": 1.0014163255691528, "Finetune/Step": 1990, "Finetune/Step Time": 4.734604915603995} +{"Finetune/Learning Rate": 1.4141782689737252e-05, "Finetune/Loss": 0.8966535329818726, "Finetune/Loss (Raw)": 0.824872612953186, "Finetune/Step": 1991, "Finetune/Step Time": 4.730500923469663} +{"Finetune/Learning Rate": 1.413611172108953e-05, "Finetune/Loss": 0.897653341293335, "Finetune/Loss (Raw)": 0.9764420986175537, "Finetune/Step": 1992, "Finetune/Step Time": 4.725709026679397} +{"Finetune/Learning Rate": 1.41304391472976e-05, "Finetune/Loss": 0.8963389992713928, "Finetune/Loss (Raw)": 0.8722352981567383, "Finetune/Step": 1993, "Finetune/Step Time": 4.724672131240368} +{"Finetune/Learning Rate": 1.4124764970562874e-05, "Finetune/Loss": 0.8941811323165894, "Finetune/Loss (Raw)": 0.7612954378128052, "Finetune/Step": 1994, "Finetune/Step Time": 4.7206455655395985} +{"Finetune/Learning Rate": 1.4119089193087396e-05, "Finetune/Loss": 0.8919337391853333, "Finetune/Loss (Raw)": 0.8044049739837646, "Finetune/Step": 1995, "Finetune/Step Time": 4.724264228716493} +{"Finetune/Learning Rate": 1.4113411817073818e-05, "Finetune/Loss": 0.8909457921981812, "Finetune/Loss (Raw)": 0.8433777689933777, "Finetune/Step": 1996, "Finetune/Step Time": 4.711416369304061} +{"Finetune/Learning Rate": 1.4107732844725424e-05, "Finetune/Loss": 0.8910598158836365, "Finetune/Loss (Raw)": 1.0772510766983032, "Finetune/Step": 1997, "Finetune/Step Time": 4.7136589307338} +{"Finetune/Learning Rate": 1.4102052278246107e-05, "Finetune/Loss": 0.8910962343215942, "Finetune/Loss (Raw)": 0.9151366353034973, "Finetune/Step": 1998, "Finetune/Step Time": 4.714395670220256} +{"Finetune/Learning Rate": 1.4096370119840389e-05, "Finetune/Loss": 0.8868011832237244, "Finetune/Loss (Raw)": 0.41455021500587463, "Finetune/Step": 1999, "Finetune/Step Time": 4.7253748662769794} +{"Finetune/Learning Rate": 1.4090686371713403e-05, "Finetune/Loss": 0.8871241807937622, "Finetune/Loss (Raw)": 0.8890517354011536, "Finetune/Step": 2000, "Finetune/Step Time": 4.748552307486534} +{"Finetune/Learning Rate": 1.4085001036070896e-05, "Finetune/Loss": 0.891718864440918, "Finetune/Loss (Raw)": 0.9572310447692871, "Finetune/Step": 2001, "Finetune/Step Time": 4.715354852378368} +{"Finetune/Learning Rate": 1.4079314115119242e-05, "Finetune/Loss": 0.8907550573348999, "Finetune/Loss (Raw)": 0.9352915287017822, "Finetune/Step": 2002, "Finetune/Step Time": 4.712997917085886} +{"Finetune/Learning Rate": 1.4073625611065422e-05, "Finetune/Loss": 0.8904728889465332, "Finetune/Loss (Raw)": 0.8786296844482422, "Finetune/Step": 2003, "Finetune/Step Time": 4.712403662502766} +{"Finetune/Learning Rate": 1.4067935526117033e-05, "Finetune/Loss": 0.889983057975769, "Finetune/Loss (Raw)": 0.9004587531089783, "Finetune/Step": 2004, "Finetune/Step Time": 4.711342491209507} +{"Finetune/Learning Rate": 1.4062243862482286e-05, "Finetune/Loss": 0.8896437883377075, "Finetune/Loss (Raw)": 0.9182835221290588, "Finetune/Step": 2005, "Finetune/Step Time": 4.705103760585189} +{"Finetune/Learning Rate": 1.4056550622370004e-05, "Finetune/Loss": 0.8907455205917358, "Finetune/Loss (Raw)": 0.9384639263153076, "Finetune/Step": 2006, "Finetune/Step Time": 4.6942242458462715} +{"Finetune/Learning Rate": 1.4050855807989626e-05, "Finetune/Loss": 0.8912513256072998, "Finetune/Loss (Raw)": 1.139574646949768, "Finetune/Step": 2007, "Finetune/Step Time": 4.6945836916565895} +{"Finetune/Learning Rate": 1.4045159421551191e-05, "Finetune/Loss": 0.8925976753234863, "Finetune/Loss (Raw)": 0.9810871481895447, "Finetune/Step": 2008, "Finetune/Step Time": 4.684392446652055} +{"Finetune/Learning Rate": 1.4039461465265362e-05, "Finetune/Loss": 0.8930011987686157, "Finetune/Loss (Raw)": 0.8876568078994751, "Finetune/Step": 2009, "Finetune/Step Time": 4.673258664086461} +{"Finetune/Learning Rate": 1.4033761941343403e-05, "Finetune/Loss": 0.8933687210083008, "Finetune/Loss (Raw)": 1.0406615734100342, "Finetune/Step": 2010, "Finetune/Step Time": 4.678105004131794} +{"Finetune/Learning Rate": 1.4028060851997183e-05, "Finetune/Loss": 0.8950496912002563, "Finetune/Loss (Raw)": 1.097902536392212, "Finetune/Step": 2011, "Finetune/Step Time": 4.655104450881481} +{"Finetune/Learning Rate": 1.4022358199439193e-05, "Finetune/Loss": 0.8995648622512817, "Finetune/Loss (Raw)": 0.9365932941436768, "Finetune/Step": 2012, "Finetune/Step Time": 4.611147882416844} +{"Finetune/Learning Rate": 1.4016653985882512e-05, "Finetune/Loss": 0.8998769521713257, "Finetune/Loss (Raw)": 1.0210323333740234, "Finetune/Step": 2013, "Finetune/Step Time": 4.621889105066657} +{"Finetune/Learning Rate": 1.4010948213540838e-05, "Finetune/Loss": 0.8975496292114258, "Finetune/Loss (Raw)": 0.7885239124298096, "Finetune/Step": 2014, "Finetune/Step Time": 4.6306090746074915} +{"Finetune/Learning Rate": 1.4005240884628473e-05, "Finetune/Loss": 0.89418625831604, "Finetune/Loss (Raw)": 0.3945716619491577, "Finetune/Step": 2015, "Finetune/Step Time": 4.670246405526996} +{"Finetune/Learning Rate": 1.3999532001360309e-05, "Finetune/Loss": 0.891302227973938, "Finetune/Loss (Raw)": 0.4038723409175873, "Finetune/Step": 2016, "Finetune/Step Time": 4.703059466555715} +{"Finetune/Learning Rate": 1.3993821565951868e-05, "Finetune/Loss": 0.8906502723693848, "Finetune/Loss (Raw)": 0.8530257940292358, "Finetune/Step": 2017, "Finetune/Step Time": 4.708785770460963} +{"Finetune/Learning Rate": 1.3988109580619244e-05, "Finetune/Loss": 0.8909355401992798, "Finetune/Loss (Raw)": 0.962544858455658, "Finetune/Step": 2018, "Finetune/Step Time": 4.715557875111699} +{"Finetune/Learning Rate": 1.3982396047579156e-05, "Finetune/Loss": 0.8897706270217896, "Finetune/Loss (Raw)": 0.8693884015083313, "Finetune/Step": 2019, "Finetune/Step Time": 4.733503518626094} +{"Finetune/Learning Rate": 1.3976680969048912e-05, "Finetune/Loss": 0.8886043429374695, "Finetune/Loss (Raw)": 0.9740263819694519, "Finetune/Step": 2020, "Finetune/Step Time": 4.737268576398492} +{"Finetune/Learning Rate": 1.3970964347246419e-05, "Finetune/Loss": 0.8887721300125122, "Finetune/Loss (Raw)": 0.9635710120201111, "Finetune/Step": 2021, "Finetune/Step Time": 4.733738228678703} +{"Finetune/Learning Rate": 1.3965246184390193e-05, "Finetune/Loss": 0.8894721865653992, "Finetune/Loss (Raw)": 0.9220148921012878, "Finetune/Step": 2022, "Finetune/Step Time": 4.7402629517018795} +{"Finetune/Learning Rate": 1.3959526482699335e-05, "Finetune/Loss": 0.8883882164955139, "Finetune/Loss (Raw)": 0.7782694101333618, "Finetune/Step": 2023, "Finetune/Step Time": 4.7651981599628925} +{"Finetune/Learning Rate": 1.3953805244393554e-05, "Finetune/Loss": 0.8881951570510864, "Finetune/Loss (Raw)": 0.9413294196128845, "Finetune/Step": 2024, "Finetune/Step Time": 4.764319257810712} +{"Finetune/Learning Rate": 1.394808247169315e-05, "Finetune/Loss": 0.8865344524383545, "Finetune/Loss (Raw)": 0.8884788751602173, "Finetune/Step": 2025, "Finetune/Step Time": 4.776503473520279} +{"Finetune/Learning Rate": 1.3942358166819016e-05, "Finetune/Loss": 0.8862276673316956, "Finetune/Loss (Raw)": 0.9609380960464478, "Finetune/Step": 2026, "Finetune/Step Time": 4.763103689998388} +{"Finetune/Learning Rate": 1.3936632331992648e-05, "Finetune/Loss": 0.8864284753799438, "Finetune/Loss (Raw)": 0.8583215475082397, "Finetune/Step": 2027, "Finetune/Step Time": 4.743103815242648} +{"Finetune/Learning Rate": 1.3930904969436125e-05, "Finetune/Loss": 0.8869541883468628, "Finetune/Loss (Raw)": 0.9629313945770264, "Finetune/Step": 2028, "Finetune/Step Time": 4.7572041768580675} +{"Finetune/Learning Rate": 1.3925176081372129e-05, "Finetune/Loss": 0.8881050944328308, "Finetune/Loss (Raw)": 1.0524275302886963, "Finetune/Step": 2029, "Finetune/Step Time": 4.750259339809418} +{"Finetune/Learning Rate": 1.3919445670023933e-05, "Finetune/Loss": 0.8830579519271851, "Finetune/Loss (Raw)": 0.34812602400779724, "Finetune/Step": 2030, "Finetune/Step Time": 4.7809839230030775} +{"Finetune/Learning Rate": 1.3913713737615387e-05, "Finetune/Loss": 0.8834407329559326, "Finetune/Loss (Raw)": 0.8997872471809387, "Finetune/Step": 2031, "Finetune/Step Time": 4.753962920978665} +{"Finetune/Learning Rate": 1.3907980286370952e-05, "Finetune/Loss": 0.884225606918335, "Finetune/Loss (Raw)": 1.0441546440124512, "Finetune/Step": 2032, "Finetune/Step Time": 4.749386033043265} +{"Finetune/Learning Rate": 1.3902245318515661e-05, "Finetune/Loss": 0.8896751403808594, "Finetune/Loss (Raw)": 1.0208929777145386, "Finetune/Step": 2033, "Finetune/Step Time": 4.7152453400194645} +{"Finetune/Learning Rate": 1.389650883627515e-05, "Finetune/Loss": 0.8906750679016113, "Finetune/Loss (Raw)": 1.0663812160491943, "Finetune/Step": 2034, "Finetune/Step Time": 4.702939914539456} +{"Finetune/Learning Rate": 1.3890770841875632e-05, "Finetune/Loss": 0.8896679878234863, "Finetune/Loss (Raw)": 0.8266804218292236, "Finetune/Step": 2035, "Finetune/Step Time": 4.718387927860022} +{"Finetune/Learning Rate": 1.388503133754391e-05, "Finetune/Loss": 0.8903509378433228, "Finetune/Loss (Raw)": 0.9855697751045227, "Finetune/Step": 2036, "Finetune/Step Time": 4.731815550476313} +{"Finetune/Learning Rate": 1.3879290325507378e-05, "Finetune/Loss": 0.891956627368927, "Finetune/Loss (Raw)": 0.9440008997917175, "Finetune/Step": 2037, "Finetune/Step Time": 4.746372986584902} +{"Finetune/Learning Rate": 1.3873547807994e-05, "Finetune/Loss": 0.8910074234008789, "Finetune/Loss (Raw)": 0.8142350316047668, "Finetune/Step": 2038, "Finetune/Step Time": 4.748450059443712} +{"Finetune/Learning Rate": 1.386780378723235e-05, "Finetune/Loss": 0.8915276527404785, "Finetune/Loss (Raw)": 1.0546540021896362, "Finetune/Step": 2039, "Finetune/Step Time": 4.7429409716278315} +{"Finetune/Learning Rate": 1.3862058265451561e-05, "Finetune/Loss": 0.8918560743331909, "Finetune/Loss (Raw)": 1.0166397094726562, "Finetune/Step": 2040, "Finetune/Step Time": 4.741797531023622} +{"Finetune/Learning Rate": 1.385631124488136e-05, "Finetune/Loss": 0.8919751644134521, "Finetune/Loss (Raw)": 0.9910660982131958, "Finetune/Step": 2041, "Finetune/Step Time": 4.78552383929491} +{"Finetune/Learning Rate": 1.385056272775205e-05, "Finetune/Loss": 0.8898833990097046, "Finetune/Loss (Raw)": 0.8229238390922546, "Finetune/Step": 2042, "Finetune/Step Time": 4.7868078127503395} +{"Finetune/Learning Rate": 1.3844812716294523e-05, "Finetune/Loss": 0.8890942931175232, "Finetune/Loss (Raw)": 0.9492692947387695, "Finetune/Step": 2043, "Finetune/Step Time": 4.756797935813665} +{"Finetune/Learning Rate": 1.3839061212740244e-05, "Finetune/Loss": 0.8892397880554199, "Finetune/Loss (Raw)": 0.942959189414978, "Finetune/Step": 2044, "Finetune/Step Time": 4.756813749670982} +{"Finetune/Learning Rate": 1.383330821932126e-05, "Finetune/Loss": 0.8907085657119751, "Finetune/Loss (Raw)": 1.0360239744186401, "Finetune/Step": 2045, "Finetune/Step Time": 4.768894108012319} +{"Finetune/Learning Rate": 1.3827553738270196e-05, "Finetune/Loss": 0.8919575214385986, "Finetune/Loss (Raw)": 1.0567306280136108, "Finetune/Step": 2046, "Finetune/Step Time": 4.768837206065655} +{"Finetune/Learning Rate": 1.382179777182025e-05, "Finetune/Loss": 0.8920627236366272, "Finetune/Loss (Raw)": 0.9643018245697021, "Finetune/Step": 2047, "Finetune/Step Time": 4.772184636443853} +{"Finetune/Learning Rate": 1.3816040322205206e-05, "Finetune/Loss": 0.8916850090026855, "Finetune/Loss (Raw)": 0.8106730580329895, "Finetune/Step": 2048, "Finetune/Step Time": 4.761147569864988} +{"Finetune/Learning Rate": 1.3810281391659414e-05, "Finetune/Loss": 0.8891787528991699, "Finetune/Loss (Raw)": 0.7080637216567993, "Finetune/Step": 2049, "Finetune/Step Time": 4.7757537718862295} +{"Finetune/Learning Rate": 1.3804520982417802e-05, "Finetune/Loss": 0.8933219909667969, "Finetune/Loss (Raw)": 0.8352832198143005, "Finetune/Step": 2050, "Finetune/Step Time": 4.751173593103886} +{"Finetune/Learning Rate": 1.3798759096715877e-05, "Finetune/Loss": 0.8941612243652344, "Finetune/Loss (Raw)": 1.077262282371521, "Finetune/Step": 2051, "Finetune/Step Time": 4.7550117168575525} +{"Finetune/Learning Rate": 1.379299573678971e-05, "Finetune/Loss": 0.8937015533447266, "Finetune/Loss (Raw)": 0.9873496294021606, "Finetune/Step": 2052, "Finetune/Step Time": 4.766853721812367} +{"Finetune/Learning Rate": 1.3787230904875947e-05, "Finetune/Loss": 0.8919973373413086, "Finetune/Loss (Raw)": 0.9432141780853271, "Finetune/Step": 2053, "Finetune/Step Time": 4.769219493493438} +{"Finetune/Learning Rate": 1.3781464603211811e-05, "Finetune/Loss": 0.8975421190261841, "Finetune/Loss (Raw)": 1.0490539073944092, "Finetune/Step": 2054, "Finetune/Step Time": 4.733741827309132} +{"Finetune/Learning Rate": 1.3775696834035088e-05, "Finetune/Loss": 0.9023462533950806, "Finetune/Loss (Raw)": 1.0157747268676758, "Finetune/Step": 2055, "Finetune/Step Time": 4.700589306652546} +{"Finetune/Learning Rate": 1.3769927599584141e-05, "Finetune/Loss": 0.9021563529968262, "Finetune/Loss (Raw)": 1.0129188299179077, "Finetune/Step": 2056, "Finetune/Step Time": 4.700929516926408} +{"Finetune/Learning Rate": 1.376415690209789e-05, "Finetune/Loss": 0.9026333689689636, "Finetune/Loss (Raw)": 0.8870642781257629, "Finetune/Step": 2057, "Finetune/Step Time": 4.709884976968169} +{"Finetune/Learning Rate": 1.3758384743815838e-05, "Finetune/Loss": 0.9025992751121521, "Finetune/Loss (Raw)": 0.9067530035972595, "Finetune/Step": 2058, "Finetune/Step Time": 4.721175465732813} +{"Finetune/Learning Rate": 1.3752611126978045e-05, "Finetune/Loss": 0.901614248752594, "Finetune/Loss (Raw)": 0.8396937847137451, "Finetune/Step": 2059, "Finetune/Step Time": 4.7225673124194145} +{"Finetune/Learning Rate": 1.3746836053825133e-05, "Finetune/Loss": 0.9024035334587097, "Finetune/Loss (Raw)": 0.9459797143936157, "Finetune/Step": 2060, "Finetune/Step Time": 4.702624198049307} +{"Finetune/Learning Rate": 1.3741059526598299e-05, "Finetune/Loss": 0.9029330015182495, "Finetune/Loss (Raw)": 1.041591763496399, "Finetune/Step": 2061, "Finetune/Step Time": 4.70156005397439} +{"Finetune/Learning Rate": 1.3735281547539304e-05, "Finetune/Loss": 0.9018142223358154, "Finetune/Loss (Raw)": 0.8646513819694519, "Finetune/Step": 2062, "Finetune/Step Time": 4.70137196034193} +{"Finetune/Learning Rate": 1.3729502118890463e-05, "Finetune/Loss": 0.901591420173645, "Finetune/Loss (Raw)": 1.0154443979263306, "Finetune/Step": 2063, "Finetune/Step Time": 4.693485848605633} +{"Finetune/Learning Rate": 1.3723721242894663e-05, "Finetune/Loss": 0.9024658203125, "Finetune/Loss (Raw)": 0.9449272751808167, "Finetune/Step": 2064, "Finetune/Step Time": 4.7020658776164055} +{"Finetune/Learning Rate": 1.3717938921795347e-05, "Finetune/Loss": 0.9025421142578125, "Finetune/Loss (Raw)": 0.9837164878845215, "Finetune/Step": 2065, "Finetune/Step Time": 4.709624614566565} +{"Finetune/Learning Rate": 1.3712155157836522e-05, "Finetune/Loss": 0.9020444750785828, "Finetune/Loss (Raw)": 0.3401998281478882, "Finetune/Step": 2066, "Finetune/Step Time": 4.709806747734547} +{"Finetune/Learning Rate": 1.3706369953262754e-05, "Finetune/Loss": 0.9020892381668091, "Finetune/Loss (Raw)": 0.952555239200592, "Finetune/Step": 2067, "Finetune/Step Time": 4.711252871900797} +{"Finetune/Learning Rate": 1.3700583310319171e-05, "Finetune/Loss": 0.9046792984008789, "Finetune/Loss (Raw)": 0.9307851791381836, "Finetune/Step": 2068, "Finetune/Step Time": 4.700224258005619} +{"Finetune/Learning Rate": 1.3694795231251453e-05, "Finetune/Loss": 0.9047704935073853, "Finetune/Loss (Raw)": 0.9178246855735779, "Finetune/Step": 2069, "Finetune/Step Time": 4.6967331394553185} +{"Finetune/Learning Rate": 1.368900571830584e-05, "Finetune/Loss": 0.9013500213623047, "Finetune/Loss (Raw)": 0.7027624845504761, "Finetune/Step": 2070, "Finetune/Step Time": 4.702948773279786} +{"Finetune/Learning Rate": 1.3683214773729123e-05, "Finetune/Loss": 0.9008066654205322, "Finetune/Loss (Raw)": 0.9713120460510254, "Finetune/Step": 2071, "Finetune/Step Time": 4.709149429574609} +{"Finetune/Learning Rate": 1.3677422399768667e-05, "Finetune/Loss": 0.9019765853881836, "Finetune/Loss (Raw)": 1.114394187927246, "Finetune/Step": 2072, "Finetune/Step Time": 4.706714479252696} +{"Finetune/Learning Rate": 1.3671628598672373e-05, "Finetune/Loss": 0.9061459898948669, "Finetune/Loss (Raw)": 0.9192553758621216, "Finetune/Step": 2073, "Finetune/Step Time": 4.678931234404445} +{"Finetune/Learning Rate": 1.3665833372688702e-05, "Finetune/Loss": 0.9067971706390381, "Finetune/Loss (Raw)": 1.028841257095337, "Finetune/Step": 2074, "Finetune/Step Time": 4.664533538743854} +{"Finetune/Learning Rate": 1.366003672406667e-05, "Finetune/Loss": 0.9049615859985352, "Finetune/Loss (Raw)": 0.8958158493041992, "Finetune/Step": 2075, "Finetune/Step Time": 4.664186159148812} +{"Finetune/Learning Rate": 1.3654238655055836e-05, "Finetune/Loss": 0.902599573135376, "Finetune/Loss (Raw)": 0.7870861291885376, "Finetune/Step": 2076, "Finetune/Step Time": 4.681861056014895} +{"Finetune/Learning Rate": 1.364843916790633e-05, "Finetune/Loss": 0.9010846614837646, "Finetune/Loss (Raw)": 0.8759646415710449, "Finetune/Step": 2077, "Finetune/Step Time": 4.6817650478333235} +{"Finetune/Learning Rate": 1.3642638264868808e-05, "Finetune/Loss": 0.899577260017395, "Finetune/Loss (Raw)": 0.8898685574531555, "Finetune/Step": 2078, "Finetune/Step Time": 4.6816045716404915} +{"Finetune/Learning Rate": 1.3636835948194494e-05, "Finetune/Loss": 0.8984376192092896, "Finetune/Loss (Raw)": 0.9268254637718201, "Finetune/Step": 2079, "Finetune/Step Time": 4.682120595127344} +{"Finetune/Learning Rate": 1.3631032220135152e-05, "Finetune/Loss": 0.8960813283920288, "Finetune/Loss (Raw)": 0.7790104746818542, "Finetune/Step": 2080, "Finetune/Step Time": 4.689982917159796} +{"Finetune/Learning Rate": 1.3625227082943093e-05, "Finetune/Loss": 0.8976464867591858, "Finetune/Loss (Raw)": 0.9282809495925903, "Finetune/Step": 2081, "Finetune/Step Time": 4.689916506409645} +{"Finetune/Learning Rate": 1.361942053887118e-05, "Finetune/Loss": 0.8976536989212036, "Finetune/Loss (Raw)": 0.8535941243171692, "Finetune/Step": 2082, "Finetune/Step Time": 4.677080783993006} +{"Finetune/Learning Rate": 1.361361259017282e-05, "Finetune/Loss": 0.8986664414405823, "Finetune/Loss (Raw)": 0.9787171483039856, "Finetune/Step": 2083, "Finetune/Step Time": 4.659931244328618} +{"Finetune/Learning Rate": 1.3607803239101965e-05, "Finetune/Loss": 0.9006417989730835, "Finetune/Loss (Raw)": 1.1114791631698608, "Finetune/Step": 2084, "Finetune/Step Time": 4.655756209045649} +{"Finetune/Learning Rate": 1.360199248791311e-05, "Finetune/Loss": 0.8947106003761292, "Finetune/Loss (Raw)": 0.3198103606700897, "Finetune/Step": 2085, "Finetune/Step Time": 4.698324615135789} +{"Finetune/Learning Rate": 1.359618033886129e-05, "Finetune/Loss": 0.8940709829330444, "Finetune/Loss (Raw)": 0.9909404516220093, "Finetune/Step": 2086, "Finetune/Step Time": 4.697674302384257} +{"Finetune/Learning Rate": 1.359036679420209e-05, "Finetune/Loss": 0.8957949280738831, "Finetune/Loss (Raw)": 1.0660841464996338, "Finetune/Step": 2087, "Finetune/Step Time": 4.6876706797629595} +{"Finetune/Learning Rate": 1.3584551856191635e-05, "Finetune/Loss": 0.8952593207359314, "Finetune/Loss (Raw)": 1.0604819059371948, "Finetune/Step": 2088, "Finetune/Step Time": 4.677845258265734} +{"Finetune/Learning Rate": 1.3578735527086585e-05, "Finetune/Loss": 0.896496057510376, "Finetune/Loss (Raw)": 0.9865283966064453, "Finetune/Step": 2089, "Finetune/Step Time": 4.672596482560039} +{"Finetune/Learning Rate": 1.357291780914415e-05, "Finetune/Loss": 0.8954249620437622, "Finetune/Loss (Raw)": 0.27408555150032043, "Finetune/Step": 2090, "Finetune/Step Time": 4.673802156001329} +{"Finetune/Learning Rate": 1.3567098704622064e-05, "Finetune/Loss": 0.8989207744598389, "Finetune/Loss (Raw)": 0.8460549712181091, "Finetune/Step": 2091, "Finetune/Step Time": 4.6395668014883995} +{"Finetune/Learning Rate": 1.3561278215778618e-05, "Finetune/Loss": 0.8988742232322693, "Finetune/Loss (Raw)": 1.0136890411376953, "Finetune/Step": 2092, "Finetune/Step Time": 4.627131201326847} +{"Finetune/Learning Rate": 1.355545634487262e-05, "Finetune/Loss": 0.8999892473220825, "Finetune/Loss (Raw)": 0.9762786626815796, "Finetune/Step": 2093, "Finetune/Step Time": 4.625043040141463} +{"Finetune/Learning Rate": 1.354963309416343e-05, "Finetune/Loss": 0.9014662504196167, "Finetune/Loss (Raw)": 1.0683934688568115, "Finetune/Step": 2094, "Finetune/Step Time": 4.625895645469427} +{"Finetune/Learning Rate": 1.3543808465910938e-05, "Finetune/Loss": 0.9025121331214905, "Finetune/Loss (Raw)": 0.9755377769470215, "Finetune/Step": 2095, "Finetune/Step Time": 4.630382515490055} +{"Finetune/Learning Rate": 1.3537982462375566e-05, "Finetune/Loss": 0.901903510093689, "Finetune/Loss (Raw)": 0.9425657987594604, "Finetune/Step": 2096, "Finetune/Step Time": 4.632662486284971} +{"Finetune/Learning Rate": 1.3532155085818276e-05, "Finetune/Loss": 0.9066884517669678, "Finetune/Loss (Raw)": 1.059201955795288, "Finetune/Step": 2097, "Finetune/Step Time": 4.594127995893359} +{"Finetune/Learning Rate": 1.3526326338500553e-05, "Finetune/Loss": 0.9066051244735718, "Finetune/Loss (Raw)": 0.9096071124076843, "Finetune/Step": 2098, "Finetune/Step Time": 4.603069128468633} +{"Finetune/Learning Rate": 1.3520496222684424e-05, "Finetune/Loss": 0.906282901763916, "Finetune/Loss (Raw)": 0.36469385027885437, "Finetune/Step": 2099, "Finetune/Step Time": 4.605527648702264} +{"Finetune/Learning Rate": 1.3514664740632446e-05, "Finetune/Loss": 0.9053399562835693, "Finetune/Loss (Raw)": 0.7933229804039001, "Finetune/Step": 2100, "Finetune/Step Time": 4.611770249903202} +{"Finetune/Learning Rate": 1.3508831894607696e-05, "Finetune/Loss": 0.9064969420433044, "Finetune/Loss (Raw)": 1.0203570127487183, "Finetune/Step": 2101, "Finetune/Step Time": 4.57606346718967} +{"Finetune/Learning Rate": 1.3502997686873797e-05, "Finetune/Loss": 0.905098557472229, "Finetune/Loss (Raw)": 0.9030020833015442, "Finetune/Step": 2102, "Finetune/Step Time": 4.585423622280359} +{"Finetune/Learning Rate": 1.3497162119694883e-05, "Finetune/Loss": 0.9007806181907654, "Finetune/Loss (Raw)": 0.3713175058364868, "Finetune/Step": 2103, "Finetune/Step Time": 4.6248833406716585} +{"Finetune/Learning Rate": 1.3491325195335627e-05, "Finetune/Loss": 0.9004395008087158, "Finetune/Loss (Raw)": 0.9124086499214172, "Finetune/Step": 2104, "Finetune/Step Time": 4.626136653125286} +{"Finetune/Learning Rate": 1.3485486916061228e-05, "Finetune/Loss": 0.8999636173248291, "Finetune/Loss (Raw)": 0.9299718141555786, "Finetune/Step": 2105, "Finetune/Step Time": 4.636402549222112} +{"Finetune/Learning Rate": 1.3479647284137398e-05, "Finetune/Loss": 0.8990387320518494, "Finetune/Loss (Raw)": 0.935298502445221, "Finetune/Step": 2106, "Finetune/Step Time": 4.634921029210091} +{"Finetune/Learning Rate": 1.3473806301830398e-05, "Finetune/Loss": 0.9034005999565125, "Finetune/Loss (Raw)": 0.9396461844444275, "Finetune/Step": 2107, "Finetune/Step Time": 4.596605118364096} +{"Finetune/Learning Rate": 1.346796397140699e-05, "Finetune/Loss": 0.9038213491439819, "Finetune/Loss (Raw)": 0.8970803618431091, "Finetune/Step": 2108, "Finetune/Step Time": 4.584064930677414} +{"Finetune/Learning Rate": 1.3462120295134476e-05, "Finetune/Loss": 0.9033926725387573, "Finetune/Loss (Raw)": 0.9522029161453247, "Finetune/Step": 2109, "Finetune/Step Time": 4.587662283331156} +{"Finetune/Learning Rate": 1.3456275275280661e-05, "Finetune/Loss": 0.9034258127212524, "Finetune/Loss (Raw)": 0.9731361865997314, "Finetune/Step": 2110, "Finetune/Step Time": 4.595466872677207} +{"Finetune/Learning Rate": 1.3450428914113892e-05, "Finetune/Loss": 0.9036082029342651, "Finetune/Loss (Raw)": 0.9280390739440918, "Finetune/Step": 2111, "Finetune/Step Time": 4.6098750196397305} +{"Finetune/Learning Rate": 1.3444581213903026e-05, "Finetune/Loss": 0.8985756635665894, "Finetune/Loss (Raw)": 0.4588741064071655, "Finetune/Step": 2112, "Finetune/Step Time": 4.647329056635499} +{"Finetune/Learning Rate": 1.3438732176917437e-05, "Finetune/Loss": 0.8980616331100464, "Finetune/Loss (Raw)": 0.947170078754425, "Finetune/Step": 2113, "Finetune/Step Time": 4.6450458616018295} +{"Finetune/Learning Rate": 1.3432881805427032e-05, "Finetune/Loss": 0.8976009488105774, "Finetune/Loss (Raw)": 1.121989369392395, "Finetune/Step": 2114, "Finetune/Step Time": 4.663461115211248} +{"Finetune/Learning Rate": 1.3427030101702215e-05, "Finetune/Loss": 0.8980312943458557, "Finetune/Loss (Raw)": 1.0078318119049072, "Finetune/Step": 2115, "Finetune/Step Time": 4.639379108324647} +{"Finetune/Learning Rate": 1.3421177068013925e-05, "Finetune/Loss": 0.894444465637207, "Finetune/Loss (Raw)": 0.42246678471565247, "Finetune/Step": 2116, "Finetune/Step Time": 4.6738367564976215} +{"Finetune/Learning Rate": 1.3415322706633606e-05, "Finetune/Loss": 0.8927895426750183, "Finetune/Loss (Raw)": 0.763475775718689, "Finetune/Step": 2117, "Finetune/Step Time": 4.689845761284232} +{"Finetune/Learning Rate": 1.3409467019833223e-05, "Finetune/Loss": 0.8918454647064209, "Finetune/Loss (Raw)": 0.8805690407752991, "Finetune/Step": 2118, "Finetune/Step Time": 4.685187598690391} +{"Finetune/Learning Rate": 1.3403610009885258e-05, "Finetune/Loss": 0.8922786712646484, "Finetune/Loss (Raw)": 0.8803324103355408, "Finetune/Step": 2119, "Finetune/Step Time": 4.698732972145081} +{"Finetune/Learning Rate": 1.3397751679062692e-05, "Finetune/Loss": 0.8872833251953125, "Finetune/Loss (Raw)": 0.33702072501182556, "Finetune/Step": 2120, "Finetune/Step Time": 4.7368802689015865} +{"Finetune/Learning Rate": 1.3391892029639037e-05, "Finetune/Loss": 0.8878339529037476, "Finetune/Loss (Raw)": 0.9427258372306824, "Finetune/Step": 2121, "Finetune/Step Time": 4.728655379265547} +{"Finetune/Learning Rate": 1.3386031063888309e-05, "Finetune/Loss": 0.8897675275802612, "Finetune/Loss (Raw)": 1.0087867975234985, "Finetune/Step": 2122, "Finetune/Step Time": 4.722676573321223} +{"Finetune/Learning Rate": 1.3380168784085028e-05, "Finetune/Loss": 0.8914105892181396, "Finetune/Loss (Raw)": 1.0147243738174438, "Finetune/Step": 2123, "Finetune/Step Time": 4.713948506861925} +{"Finetune/Learning Rate": 1.337430519250424e-05, "Finetune/Loss": 0.8925808072090149, "Finetune/Loss (Raw)": 0.9931616187095642, "Finetune/Step": 2124, "Finetune/Step Time": 4.710440557450056} +{"Finetune/Learning Rate": 1.3368440291421475e-05, "Finetune/Loss": 0.8920842409133911, "Finetune/Loss (Raw)": 1.0136899948120117, "Finetune/Step": 2125, "Finetune/Step Time": 4.72405163757503} +{"Finetune/Learning Rate": 1.3362574083112804e-05, "Finetune/Loss": 0.8929011821746826, "Finetune/Loss (Raw)": 1.0197075605392456, "Finetune/Step": 2126, "Finetune/Step Time": 4.72086045704782} +{"Finetune/Learning Rate": 1.3356706569854778e-05, "Finetune/Loss": 0.8969942331314087, "Finetune/Loss (Raw)": 0.9384657144546509, "Finetune/Step": 2127, "Finetune/Step Time": 4.680547183379531} +{"Finetune/Learning Rate": 1.3350837753924466e-05, "Finetune/Loss": 0.8928279876708984, "Finetune/Loss (Raw)": 0.3557702898979187, "Finetune/Step": 2128, "Finetune/Step Time": 4.699016977101564} +{"Finetune/Learning Rate": 1.3344967637599445e-05, "Finetune/Loss": 0.8875391483306885, "Finetune/Loss (Raw)": 0.28025588393211365, "Finetune/Step": 2129, "Finetune/Step Time": 4.7323587369173765} +{"Finetune/Learning Rate": 1.3339096223157782e-05, "Finetune/Loss": 0.8879015445709229, "Finetune/Loss (Raw)": 0.9816720485687256, "Finetune/Step": 2130, "Finetune/Step Time": 4.73083994910121} +{"Finetune/Learning Rate": 1.3333223512878068e-05, "Finetune/Loss": 0.8888100981712341, "Finetune/Loss (Raw)": 0.9949332475662231, "Finetune/Step": 2131, "Finetune/Step Time": 4.7205372247844934} +{"Finetune/Learning Rate": 1.3327349509039387e-05, "Finetune/Loss": 0.8886668682098389, "Finetune/Loss (Raw)": 0.8821255564689636, "Finetune/Step": 2132, "Finetune/Step Time": 4.724986776709557} +{"Finetune/Learning Rate": 1.3321474213921317e-05, "Finetune/Loss": 0.8905720114707947, "Finetune/Loss (Raw)": 1.1621410846710205, "Finetune/Step": 2133, "Finetune/Step Time": 4.722175344824791} +{"Finetune/Learning Rate": 1.3315597629803959e-05, "Finetune/Loss": 0.890283465385437, "Finetune/Loss (Raw)": 0.901533305644989, "Finetune/Step": 2134, "Finetune/Step Time": 4.732975257560611} +{"Finetune/Learning Rate": 1.3309719758967885e-05, "Finetune/Loss": 0.8882317543029785, "Finetune/Loss (Raw)": 0.8769540786743164, "Finetune/Step": 2135, "Finetune/Step Time": 4.72857510112226} +{"Finetune/Learning Rate": 1.3303840603694192e-05, "Finetune/Loss": 0.888210654258728, "Finetune/Loss (Raw)": 0.9783862233161926, "Finetune/Step": 2136, "Finetune/Step Time": 4.721161883324385} +{"Finetune/Learning Rate": 1.3297960166264462e-05, "Finetune/Loss": 0.884611189365387, "Finetune/Loss (Raw)": 0.4269290864467621, "Finetune/Step": 2137, "Finetune/Step Time": 4.766022743657231} +{"Finetune/Learning Rate": 1.3292078448960778e-05, "Finetune/Loss": 0.8836878538131714, "Finetune/Loss (Raw)": 0.9224616289138794, "Finetune/Step": 2138, "Finetune/Step Time": 4.76435031928122} +{"Finetune/Learning Rate": 1.3286195454065726e-05, "Finetune/Loss": 0.8820139169692993, "Finetune/Loss (Raw)": 0.8836515545845032, "Finetune/Step": 2139, "Finetune/Step Time": 4.778935780748725} +{"Finetune/Learning Rate": 1.3280311183862371e-05, "Finetune/Loss": 0.8823015689849854, "Finetune/Loss (Raw)": 0.973408043384552, "Finetune/Step": 2140, "Finetune/Step Time": 4.781792936846614} +{"Finetune/Learning Rate": 1.3274425640634287e-05, "Finetune/Loss": 0.8828730583190918, "Finetune/Loss (Raw)": 1.0941866636276245, "Finetune/Step": 2141, "Finetune/Step Time": 4.772867506369948} +{"Finetune/Learning Rate": 1.3268538826665544e-05, "Finetune/Loss": 0.8834618330001831, "Finetune/Loss (Raw)": 0.8638808727264404, "Finetune/Step": 2142, "Finetune/Step Time": 4.761695142835379} +{"Finetune/Learning Rate": 1.326265074424069e-05, "Finetune/Loss": 0.8877052068710327, "Finetune/Loss (Raw)": 0.9377210140228271, "Finetune/Step": 2143, "Finetune/Step Time": 4.735764009878039} +{"Finetune/Learning Rate": 1.3256761395644787e-05, "Finetune/Loss": 0.8916807174682617, "Finetune/Loss (Raw)": 0.9127466082572937, "Finetune/Step": 2144, "Finetune/Step Time": 4.714455271139741} +{"Finetune/Learning Rate": 1.3250870783163363e-05, "Finetune/Loss": 0.8905762434005737, "Finetune/Loss (Raw)": 0.7116544842720032, "Finetune/Step": 2145, "Finetune/Step Time": 4.714357594028115} +{"Finetune/Learning Rate": 1.3244978909082457e-05, "Finetune/Loss": 0.8903234004974365, "Finetune/Loss (Raw)": 0.9301663041114807, "Finetune/Step": 2146, "Finetune/Step Time": 4.724093152210116} +{"Finetune/Learning Rate": 1.3239085775688591e-05, "Finetune/Loss": 0.8909823894500732, "Finetune/Loss (Raw)": 0.9537418484687805, "Finetune/Step": 2147, "Finetune/Step Time": 4.71475332044065} +{"Finetune/Learning Rate": 1.3233191385268771e-05, "Finetune/Loss": 0.8917832374572754, "Finetune/Loss (Raw)": 1.0765376091003418, "Finetune/Step": 2148, "Finetune/Step Time": 4.710683714598417} +{"Finetune/Learning Rate": 1.3227295740110502e-05, "Finetune/Loss": 0.8920656442642212, "Finetune/Loss (Raw)": 0.9997181296348572, "Finetune/Step": 2149, "Finetune/Step Time": 4.699446650221944} +{"Finetune/Learning Rate": 1.3221398842501758e-05, "Finetune/Loss": 0.8920372724533081, "Finetune/Loss (Raw)": 0.918383777141571, "Finetune/Step": 2150, "Finetune/Step Time": 4.697424551472068} +{"Finetune/Learning Rate": 1.321550069473102e-05, "Finetune/Loss": 0.8949805498123169, "Finetune/Loss (Raw)": 1.1550133228302002, "Finetune/Step": 2151, "Finetune/Step Time": 4.68143199197948} +{"Finetune/Learning Rate": 1.3209601299087238e-05, "Finetune/Loss": 0.8940937519073486, "Finetune/Loss (Raw)": 0.8278162479400635, "Finetune/Step": 2152, "Finetune/Step Time": 4.688011929392815} +{"Finetune/Learning Rate": 1.3203700657859855e-05, "Finetune/Loss": 0.8939062356948853, "Finetune/Loss (Raw)": 0.8644834160804749, "Finetune/Step": 2153, "Finetune/Step Time": 4.677412420511246} +{"Finetune/Learning Rate": 1.3197798773338793e-05, "Finetune/Loss": 0.8933213949203491, "Finetune/Loss (Raw)": 0.8860729336738586, "Finetune/Step": 2154, "Finetune/Step Time": 4.697666490450501} +{"Finetune/Learning Rate": 1.3191895647814457e-05, "Finetune/Loss": 0.8938331604003906, "Finetune/Loss (Raw)": 0.923823356628418, "Finetune/Step": 2155, "Finetune/Step Time": 4.704417992383242} +{"Finetune/Learning Rate": 1.318599128357774e-05, "Finetune/Loss": 0.892371416091919, "Finetune/Loss (Raw)": 0.7758340835571289, "Finetune/Step": 2156, "Finetune/Step Time": 4.699791006743908} +{"Finetune/Learning Rate": 1.3180085682920005e-05, "Finetune/Loss": 0.891150951385498, "Finetune/Loss (Raw)": 0.8962087035179138, "Finetune/Step": 2157, "Finetune/Step Time": 4.709366619586945} +{"Finetune/Learning Rate": 1.31741788481331e-05, "Finetune/Loss": 0.8960471153259277, "Finetune/Loss (Raw)": 0.9748350381851196, "Finetune/Step": 2158, "Finetune/Step Time": 4.671490902081132} +{"Finetune/Learning Rate": 1.316827078150936e-05, "Finetune/Loss": 0.8968151807785034, "Finetune/Loss (Raw)": 0.9980913996696472, "Finetune/Step": 2159, "Finetune/Step Time": 4.688149839639664} +{"Finetune/Learning Rate": 1.316236148534158e-05, "Finetune/Loss": 0.8960897922515869, "Finetune/Loss (Raw)": 0.951309859752655, "Finetune/Step": 2160, "Finetune/Step Time": 4.683549927547574} +{"Finetune/Learning Rate": 1.3156450961923051e-05, "Finetune/Loss": 0.8948994874954224, "Finetune/Loss (Raw)": 0.8685407042503357, "Finetune/Step": 2161, "Finetune/Step Time": 4.677816251292825} +{"Finetune/Learning Rate": 1.3150539213547528e-05, "Finetune/Loss": 0.8937524557113647, "Finetune/Loss (Raw)": 0.9195521473884583, "Finetune/Step": 2162, "Finetune/Step Time": 4.682408096268773} +{"Finetune/Learning Rate": 1.3144626242509243e-05, "Finetune/Loss": 0.8942540287971497, "Finetune/Loss (Raw)": 0.8908825516700745, "Finetune/Step": 2163, "Finetune/Step Time": 4.661211796104908} +{"Finetune/Learning Rate": 1.3138712051102908e-05, "Finetune/Loss": 0.894808292388916, "Finetune/Loss (Raw)": 1.0565156936645508, "Finetune/Step": 2164, "Finetune/Step Time": 4.647815968841314} +{"Finetune/Learning Rate": 1.3132796641623703e-05, "Finetune/Loss": 0.8948045969009399, "Finetune/Loss (Raw)": 0.9435304999351501, "Finetune/Step": 2165, "Finetune/Step Time": 4.657576123252511} +{"Finetune/Learning Rate": 1.3126880016367287e-05, "Finetune/Loss": 0.8959997892379761, "Finetune/Loss (Raw)": 0.9672133922576904, "Finetune/Step": 2166, "Finetune/Step Time": 4.6579674780368805} +{"Finetune/Learning Rate": 1.3120962177629783e-05, "Finetune/Loss": 0.8961538672447205, "Finetune/Loss (Raw)": 1.0743765830993652, "Finetune/Step": 2167, "Finetune/Step Time": 4.657535603269935} +{"Finetune/Learning Rate": 1.311504312770779e-05, "Finetune/Loss": 0.8954335451126099, "Finetune/Loss (Raw)": 0.9244384765625, "Finetune/Step": 2168, "Finetune/Step Time": 4.65653271228075} +{"Finetune/Learning Rate": 1.3109122868898374e-05, "Finetune/Loss": 0.8952677249908447, "Finetune/Loss (Raw)": 0.9698525071144104, "Finetune/Step": 2169, "Finetune/Step Time": 4.613499723374844} +{"Finetune/Learning Rate": 1.310320140349907e-05, "Finetune/Loss": 0.8954389095306396, "Finetune/Loss (Raw)": 0.8448349833488464, "Finetune/Step": 2170, "Finetune/Step Time": 4.6207634303718805} +{"Finetune/Learning Rate": 1.3097278733807892e-05, "Finetune/Loss": 0.8952110409736633, "Finetune/Loss (Raw)": 0.9200951457023621, "Finetune/Step": 2171, "Finetune/Step Time": 4.630480004474521} +{"Finetune/Learning Rate": 1.3091354862123304e-05, "Finetune/Loss": 0.8949558734893799, "Finetune/Loss (Raw)": 0.9103039503097534, "Finetune/Step": 2172, "Finetune/Step Time": 4.630328483879566} +{"Finetune/Learning Rate": 1.3085429790744248e-05, "Finetune/Loss": 0.8942822217941284, "Finetune/Loss (Raw)": 0.94979327917099, "Finetune/Step": 2173, "Finetune/Step Time": 4.634133696556091} +{"Finetune/Learning Rate": 1.3079503521970127e-05, "Finetune/Loss": 0.8921841382980347, "Finetune/Loss (Raw)": 0.7881685495376587, "Finetune/Step": 2174, "Finetune/Step Time": 4.647010890766978} +{"Finetune/Learning Rate": 1.3073576058100815e-05, "Finetune/Loss": 0.8920794129371643, "Finetune/Loss (Raw)": 0.9508993625640869, "Finetune/Step": 2175, "Finetune/Step Time": 4.66291631013155} +{"Finetune/Learning Rate": 1.3067647401436636e-05, "Finetune/Loss": 0.8935225605964661, "Finetune/Loss (Raw)": 0.9954007267951965, "Finetune/Step": 2176, "Finetune/Step Time": 4.662568787112832} +{"Finetune/Learning Rate": 1.3061717554278393e-05, "Finetune/Loss": 0.8964095115661621, "Finetune/Loss (Raw)": 1.0775974988937378, "Finetune/Step": 2177, "Finetune/Step Time": 4.651626918464899} +{"Finetune/Learning Rate": 1.3055786518927342e-05, "Finetune/Loss": 0.8962059020996094, "Finetune/Loss (Raw)": 0.8092154860496521, "Finetune/Step": 2178, "Finetune/Step Time": 4.639422003179789} +{"Finetune/Learning Rate": 1.3049854297685206e-05, "Finetune/Loss": 0.8953778147697449, "Finetune/Loss (Raw)": 0.971264123916626, "Finetune/Step": 2179, "Finetune/Step Time": 4.649358009919524} +{"Finetune/Learning Rate": 1.3043920892854155e-05, "Finetune/Loss": 0.8952803611755371, "Finetune/Loss (Raw)": 0.9748821258544922, "Finetune/Step": 2180, "Finetune/Step Time": 4.6430672984570265} +{"Finetune/Learning Rate": 1.3037986306736839e-05, "Finetune/Loss": 0.8955678343772888, "Finetune/Loss (Raw)": 0.9800029993057251, "Finetune/Step": 2181, "Finetune/Step Time": 4.640958065167069} +{"Finetune/Learning Rate": 1.3032050541636343e-05, "Finetune/Loss": 0.8951998949050903, "Finetune/Loss (Raw)": 1.0019612312316895, "Finetune/Step": 2182, "Finetune/Step Time": 4.6446786522865295} +{"Finetune/Learning Rate": 1.3026113599856229e-05, "Finetune/Loss": 0.8934518098831177, "Finetune/Loss (Raw)": 0.7920231223106384, "Finetune/Step": 2183, "Finetune/Step Time": 4.661314742639661} +{"Finetune/Learning Rate": 1.302017548370051e-05, "Finetune/Loss": 0.8919890522956848, "Finetune/Loss (Raw)": 0.8256850838661194, "Finetune/Step": 2184, "Finetune/Step Time": 4.675938511267304} +{"Finetune/Learning Rate": 1.3014236195473649e-05, "Finetune/Loss": 0.8923554420471191, "Finetune/Loss (Raw)": 0.9339661598205566, "Finetune/Step": 2185, "Finetune/Step Time": 4.66204921528697} +{"Finetune/Learning Rate": 1.3008295737480567e-05, "Finetune/Loss": 0.8923795223236084, "Finetune/Loss (Raw)": 0.9098278284072876, "Finetune/Step": 2186, "Finetune/Step Time": 4.647142171859741} +{"Finetune/Learning Rate": 1.300235411202664e-05, "Finetune/Loss": 0.8917232751846313, "Finetune/Loss (Raw)": 0.7556925415992737, "Finetune/Step": 2187, "Finetune/Step Time": 4.680366672575474} +{"Finetune/Learning Rate": 1.2996411321417699e-05, "Finetune/Loss": 0.8915723562240601, "Finetune/Loss (Raw)": 0.9266660213470459, "Finetune/Step": 2188, "Finetune/Step Time": 4.71351283043623} +{"Finetune/Learning Rate": 1.2990467367960027e-05, "Finetune/Loss": 0.8901056051254272, "Finetune/Loss (Raw)": 0.8538452982902527, "Finetune/Step": 2189, "Finetune/Step Time": 4.7211337853223085} +{"Finetune/Learning Rate": 1.298452225396035e-05, "Finetune/Loss": 0.8900307416915894, "Finetune/Loss (Raw)": 0.8550663590431213, "Finetune/Step": 2190, "Finetune/Step Time": 4.73076525516808} +{"Finetune/Learning Rate": 1.2978575981725858e-05, "Finetune/Loss": 0.8896648287773132, "Finetune/Loss (Raw)": 0.9686118364334106, "Finetune/Step": 2191, "Finetune/Step Time": 4.745771365240216} +{"Finetune/Learning Rate": 1.2972628553564178e-05, "Finetune/Loss": 0.8886420726776123, "Finetune/Loss (Raw)": 0.8140200972557068, "Finetune/Step": 2192, "Finetune/Step Time": 4.726906273514032} +{"Finetune/Learning Rate": 1.2966679971783387e-05, "Finetune/Loss": 0.8887501358985901, "Finetune/Loss (Raw)": 0.9975473284721375, "Finetune/Step": 2193, "Finetune/Step Time": 4.7217614110559225} +{"Finetune/Learning Rate": 1.2960730238692023e-05, "Finetune/Loss": 0.8938120007514954, "Finetune/Loss (Raw)": 0.9881150126457214, "Finetune/Step": 2194, "Finetune/Step Time": 4.687488157302141} +{"Finetune/Learning Rate": 1.2954779356599056e-05, "Finetune/Loss": 0.8942315578460693, "Finetune/Loss (Raw)": 1.0062533617019653, "Finetune/Step": 2195, "Finetune/Step Time": 4.685963269323111} +{"Finetune/Learning Rate": 1.2948827327813908e-05, "Finetune/Loss": 0.8936847448348999, "Finetune/Loss (Raw)": 0.8607961535453796, "Finetune/Step": 2196, "Finetune/Step Time": 4.6951488964259624} +{"Finetune/Learning Rate": 1.2942874154646447e-05, "Finetune/Loss": 0.8943589329719543, "Finetune/Loss (Raw)": 1.0041229724884033, "Finetune/Step": 2197, "Finetune/Step Time": 4.691258139908314} +{"Finetune/Learning Rate": 1.2936919839406978e-05, "Finetune/Loss": 0.8957533240318298, "Finetune/Loss (Raw)": 0.8812429904937744, "Finetune/Step": 2198, "Finetune/Step Time": 4.6851868107914925} +{"Finetune/Learning Rate": 1.2930964384406259e-05, "Finetune/Loss": 0.8948701620101929, "Finetune/Loss (Raw)": 0.8582668900489807, "Finetune/Step": 2199, "Finetune/Step Time": 4.6767630986869335} +{"Finetune/Learning Rate": 1.2925007791955487e-05, "Finetune/Loss": 0.8931382894515991, "Finetune/Loss (Raw)": 0.8927171230316162, "Finetune/Step": 2200, "Finetune/Step Time": 4.688538474962115} +{"Finetune/Learning Rate": 1.2919050064366294e-05, "Finetune/Loss": 0.8887493014335632, "Finetune/Loss (Raw)": 0.357466459274292, "Finetune/Step": 2201, "Finetune/Step Time": 4.714163916185498} +{"Finetune/Learning Rate": 1.2913091203950764e-05, "Finetune/Loss": 0.8879051208496094, "Finetune/Loss (Raw)": 0.9207836389541626, "Finetune/Step": 2202, "Finetune/Step Time": 4.711356611922383} +{"Finetune/Learning Rate": 1.2907131213021404e-05, "Finetune/Loss": 0.8877940773963928, "Finetune/Loss (Raw)": 0.8815971612930298, "Finetune/Step": 2203, "Finetune/Step Time": 4.712366471067071} +{"Finetune/Learning Rate": 1.2901170093891182e-05, "Finetune/Loss": 0.8883149027824402, "Finetune/Loss (Raw)": 0.8537535071372986, "Finetune/Step": 2204, "Finetune/Step Time": 4.710066758096218} +{"Finetune/Learning Rate": 1.2895207848873488e-05, "Finetune/Loss": 0.8848214149475098, "Finetune/Loss (Raw)": 0.4288105368614197, "Finetune/Step": 2205, "Finetune/Step Time": 4.751533528789878} +{"Finetune/Learning Rate": 1.288924448028215e-05, "Finetune/Loss": 0.8832207918167114, "Finetune/Loss (Raw)": 0.6849751472473145, "Finetune/Step": 2206, "Finetune/Step Time": 4.806315686553717} +{"Finetune/Learning Rate": 1.288327999043144e-05, "Finetune/Loss": 0.8835369944572449, "Finetune/Loss (Raw)": 0.9673025608062744, "Finetune/Step": 2207, "Finetune/Step Time": 4.801203107461333} +{"Finetune/Learning Rate": 1.2877314381636052e-05, "Finetune/Loss": 0.8842902183532715, "Finetune/Loss (Raw)": 0.8754274249076843, "Finetune/Step": 2208, "Finetune/Step Time": 4.804912988096476} +{"Finetune/Learning Rate": 1.287134765621113e-05, "Finetune/Loss": 0.8840688467025757, "Finetune/Loss (Raw)": 0.8999487161636353, "Finetune/Step": 2209, "Finetune/Step Time": 4.794428955763578} +{"Finetune/Learning Rate": 1.2865379816472244e-05, "Finetune/Loss": 0.8804748058319092, "Finetune/Loss (Raw)": 0.3935437500476837, "Finetune/Step": 2210, "Finetune/Step Time": 4.825487880036235} +{"Finetune/Learning Rate": 1.2859410864735389e-05, "Finetune/Loss": 0.880811333656311, "Finetune/Loss (Raw)": 1.0218037366867065, "Finetune/Step": 2211, "Finetune/Step Time": 4.849693983793259} +{"Finetune/Learning Rate": 1.2853440803317003e-05, "Finetune/Loss": 0.8777718544006348, "Finetune/Loss (Raw)": 0.7224233746528625, "Finetune/Step": 2212, "Finetune/Step Time": 4.872609714046121} +{"Finetune/Learning Rate": 1.2847469634533952e-05, "Finetune/Loss": 0.8817296624183655, "Finetune/Loss (Raw)": 0.8264071345329285, "Finetune/Step": 2213, "Finetune/Step Time": 4.835950097069144} +{"Finetune/Learning Rate": 1.2841497360703528e-05, "Finetune/Loss": 0.8796217441558838, "Finetune/Loss (Raw)": 0.7211247682571411, "Finetune/Step": 2214, "Finetune/Step Time": 4.836553975939751} +{"Finetune/Learning Rate": 1.283552398414345e-05, "Finetune/Loss": 0.8789771199226379, "Finetune/Loss (Raw)": 0.9835733771324158, "Finetune/Step": 2215, "Finetune/Step Time": 4.836349926888943} +{"Finetune/Learning Rate": 1.2829549507171877e-05, "Finetune/Loss": 0.8781548738479614, "Finetune/Loss (Raw)": 0.9552350640296936, "Finetune/Step": 2216, "Finetune/Step Time": 4.838814305141568} +{"Finetune/Learning Rate": 1.2823573932107381e-05, "Finetune/Loss": 0.8784348368644714, "Finetune/Loss (Raw)": 1.0223675966262817, "Finetune/Step": 2217, "Finetune/Step Time": 4.840050315484405} +{"Finetune/Learning Rate": 1.2817597261268965e-05, "Finetune/Loss": 0.8821694254875183, "Finetune/Loss (Raw)": 0.7521047592163086, "Finetune/Step": 2218, "Finetune/Step Time": 4.806628393009305} +{"Finetune/Learning Rate": 1.2811619496976066e-05, "Finetune/Loss": 0.8823822736740112, "Finetune/Loss (Raw)": 0.8732972741127014, "Finetune/Step": 2219, "Finetune/Step Time": 4.841624913737178} +{"Finetune/Learning Rate": 1.2805640641548529e-05, "Finetune/Loss": 0.8822305202484131, "Finetune/Loss (Raw)": 0.9942737817764282, "Finetune/Step": 2220, "Finetune/Step Time": 4.8413779232650995} +{"Finetune/Learning Rate": 1.2799660697306636e-05, "Finetune/Loss": 0.8812053799629211, "Finetune/Loss (Raw)": 0.8450590968132019, "Finetune/Step": 2221, "Finetune/Step Time": 4.858070259913802} +{"Finetune/Learning Rate": 1.2793679666571084e-05, "Finetune/Loss": 0.8804969787597656, "Finetune/Loss (Raw)": 0.9777233004570007, "Finetune/Step": 2222, "Finetune/Step Time": 4.867061680182815} +{"Finetune/Learning Rate": 1.2787697551662994e-05, "Finetune/Loss": 0.8802776336669922, "Finetune/Loss (Raw)": 0.9474537968635559, "Finetune/Step": 2223, "Finetune/Step Time": 4.871121857315302} +{"Finetune/Learning Rate": 1.2781714354903913e-05, "Finetune/Loss": 0.8800802230834961, "Finetune/Loss (Raw)": 0.9172917008399963, "Finetune/Step": 2224, "Finetune/Step Time": 4.872006326913834} +{"Finetune/Learning Rate": 1.2775730078615798e-05, "Finetune/Loss": 0.8788920640945435, "Finetune/Loss (Raw)": 0.9071306586265564, "Finetune/Step": 2225, "Finetune/Step Time": 4.888979585841298} +{"Finetune/Learning Rate": 1.276974472512103e-05, "Finetune/Loss": 0.8791189193725586, "Finetune/Loss (Raw)": 0.9386342763900757, "Finetune/Step": 2226, "Finetune/Step Time": 4.88864865899086} +{"Finetune/Learning Rate": 1.276375829674241e-05, "Finetune/Loss": 0.8839826583862305, "Finetune/Loss (Raw)": 0.9872555136680603, "Finetune/Step": 2227, "Finetune/Step Time": 4.854993149638176} +{"Finetune/Learning Rate": 1.275777079580315e-05, "Finetune/Loss": 0.8848296999931335, "Finetune/Loss (Raw)": 0.9017469882965088, "Finetune/Step": 2228, "Finetune/Step Time": 4.852980861440301} +{"Finetune/Learning Rate": 1.2751782224626891e-05, "Finetune/Loss": 0.8840748071670532, "Finetune/Loss (Raw)": 0.9237225651741028, "Finetune/Step": 2229, "Finetune/Step Time": 4.852252207696438} +{"Finetune/Learning Rate": 1.274579258553767e-05, "Finetune/Loss": 0.8838028907775879, "Finetune/Loss (Raw)": 0.8682098984718323, "Finetune/Step": 2230, "Finetune/Step Time": 4.846482142806053} +{"Finetune/Learning Rate": 1.2739801880859954e-05, "Finetune/Loss": 0.8886970281600952, "Finetune/Loss (Raw)": 0.9977540373802185, "Finetune/Step": 2231, "Finetune/Step Time": 4.808240689337254} +{"Finetune/Learning Rate": 1.2733810112918618e-05, "Finetune/Loss": 0.8896676301956177, "Finetune/Loss (Raw)": 1.0366538763046265, "Finetune/Step": 2232, "Finetune/Step Time": 4.799389503896236} +{"Finetune/Learning Rate": 1.2727817284038953e-05, "Finetune/Loss": 0.8887163400650024, "Finetune/Loss (Raw)": 0.8082068562507629, "Finetune/Step": 2233, "Finetune/Step Time": 4.797246349975467} +{"Finetune/Learning Rate": 1.2721823396546655e-05, "Finetune/Loss": 0.8890875577926636, "Finetune/Loss (Raw)": 0.9828125238418579, "Finetune/Step": 2234, "Finetune/Step Time": 4.800725340843201} +{"Finetune/Learning Rate": 1.2715828452767833e-05, "Finetune/Loss": 0.8898698091506958, "Finetune/Loss (Raw)": 1.0397734642028809, "Finetune/Step": 2235, "Finetune/Step Time": 4.812152441591024} +{"Finetune/Learning Rate": 1.2709832455029017e-05, "Finetune/Loss": 0.8907015323638916, "Finetune/Loss (Raw)": 1.0035420656204224, "Finetune/Step": 2236, "Finetune/Step Time": 4.813079101964831} +{"Finetune/Learning Rate": 1.2703835405657124e-05, "Finetune/Loss": 0.890372633934021, "Finetune/Loss (Raw)": 0.9101014733314514, "Finetune/Step": 2237, "Finetune/Step Time": 4.819743640720844} +{"Finetune/Learning Rate": 1.2697837306979499e-05, "Finetune/Loss": 0.8888740539550781, "Finetune/Loss (Raw)": 0.7813178300857544, "Finetune/Step": 2238, "Finetune/Step Time": 4.824014892801642} +{"Finetune/Learning Rate": 1.2691838161323888e-05, "Finetune/Loss": 0.8908345699310303, "Finetune/Loss (Raw)": 1.1789907217025757, "Finetune/Step": 2239, "Finetune/Step Time": 4.810193996876478} +{"Finetune/Learning Rate": 1.2685837971018438e-05, "Finetune/Loss": 0.8955987691879272, "Finetune/Loss (Raw)": 1.0686848163604736, "Finetune/Step": 2240, "Finetune/Step Time": 4.775165159255266} +{"Finetune/Learning Rate": 1.2679836738391714e-05, "Finetune/Loss": 0.8960216045379639, "Finetune/Loss (Raw)": 1.0012940168380737, "Finetune/Step": 2241, "Finetune/Step Time": 4.783908884972334} +{"Finetune/Learning Rate": 1.2673834465772662e-05, "Finetune/Loss": 0.8942832350730896, "Finetune/Loss (Raw)": 0.8994836211204529, "Finetune/Step": 2242, "Finetune/Step Time": 4.766603380441666} +{"Finetune/Learning Rate": 1.2667831155490662e-05, "Finetune/Loss": 0.8934675455093384, "Finetune/Loss (Raw)": 0.9034164547920227, "Finetune/Step": 2243, "Finetune/Step Time": 4.769425503909588} +{"Finetune/Learning Rate": 1.2661826809875474e-05, "Finetune/Loss": 0.8993294835090637, "Finetune/Loss (Raw)": 1.1728017330169678, "Finetune/Step": 2244, "Finetune/Step Time": 4.729469133540988} +{"Finetune/Learning Rate": 1.265582143125727e-05, "Finetune/Loss": 0.9009532928466797, "Finetune/Loss (Raw)": 0.9713197350502014, "Finetune/Step": 2245, "Finetune/Step Time": 4.70962973497808} +{"Finetune/Learning Rate": 1.264981502196662e-05, "Finetune/Loss": 0.9026232957839966, "Finetune/Loss (Raw)": 1.0943268537521362, "Finetune/Step": 2246, "Finetune/Step Time": 4.712650280445814} +{"Finetune/Learning Rate": 1.2643807584334491e-05, "Finetune/Loss": 0.9024678468704224, "Finetune/Loss (Raw)": 0.8604393601417542, "Finetune/Step": 2247, "Finetune/Step Time": 4.711442783474922} +{"Finetune/Learning Rate": 1.2637799120692253e-05, "Finetune/Loss": 0.9059057831764221, "Finetune/Loss (Raw)": 0.7770773768424988, "Finetune/Step": 2248, "Finetune/Step Time": 4.669607391580939} +{"Finetune/Learning Rate": 1.2631789633371678e-05, "Finetune/Loss": 0.9053753614425659, "Finetune/Loss (Raw)": 0.8748310208320618, "Finetune/Step": 2249, "Finetune/Step Time": 4.669650783762336} +{"Finetune/Learning Rate": 1.2625779124704925e-05, "Finetune/Loss": 0.9049947261810303, "Finetune/Loss (Raw)": 0.9600650668144226, "Finetune/Step": 2250, "Finetune/Step Time": 4.690667245537043} +{"Finetune/Learning Rate": 1.2619767597024561e-05, "Finetune/Loss": 0.9045897722244263, "Finetune/Loss (Raw)": 0.9628866314888, "Finetune/Step": 2251, "Finetune/Step Time": 4.691973354667425} +{"Finetune/Learning Rate": 1.2613755052663536e-05, "Finetune/Loss": 0.8993532657623291, "Finetune/Loss (Raw)": 0.3228881061077118, "Finetune/Step": 2252, "Finetune/Step Time": 4.725118892267346} +{"Finetune/Learning Rate": 1.2607741493955208e-05, "Finetune/Loss": 0.8995083570480347, "Finetune/Loss (Raw)": 1.0335497856140137, "Finetune/Step": 2253, "Finetune/Step Time": 4.706347579136491} +{"Finetune/Learning Rate": 1.2601726923233315e-05, "Finetune/Loss": 0.8997491002082825, "Finetune/Loss (Raw)": 1.0505173206329346, "Finetune/Step": 2254, "Finetune/Step Time": 4.712779011577368} +{"Finetune/Learning Rate": 1.2595711342831998e-05, "Finetune/Loss": 0.8999810218811035, "Finetune/Loss (Raw)": 0.9681506156921387, "Finetune/Step": 2255, "Finetune/Step Time": 4.710723822936416} +{"Finetune/Learning Rate": 1.2589694755085789e-05, "Finetune/Loss": 0.904109001159668, "Finetune/Loss (Raw)": 0.8841577768325806, "Finetune/Step": 2256, "Finetune/Step Time": 4.667096244171262} +{"Finetune/Learning Rate": 1.25836771623296e-05, "Finetune/Loss": 0.9095671772956848, "Finetune/Loss (Raw)": 0.9789012670516968, "Finetune/Step": 2257, "Finetune/Step Time": 4.632001860067248} +{"Finetune/Learning Rate": 1.257765856689875e-05, "Finetune/Loss": 0.9078587293624878, "Finetune/Loss (Raw)": 0.7629890441894531, "Finetune/Step": 2258, "Finetune/Step Time": 4.632084861397743} +{"Finetune/Learning Rate": 1.2571638971128933e-05, "Finetune/Loss": 0.9067502617835999, "Finetune/Loss (Raw)": 0.8530474305152893, "Finetune/Step": 2259, "Finetune/Step Time": 4.629404291510582} +{"Finetune/Learning Rate": 1.2565618377356237e-05, "Finetune/Loss": 0.9066766500473022, "Finetune/Loss (Raw)": 0.8727083802223206, "Finetune/Step": 2260, "Finetune/Step Time": 4.634496299549937} +{"Finetune/Learning Rate": 1.2559596787917144e-05, "Finetune/Loss": 0.9051585793495178, "Finetune/Loss (Raw)": 0.9678241014480591, "Finetune/Step": 2261, "Finetune/Step Time": 4.6366912461817265} +{"Finetune/Learning Rate": 1.2553574205148502e-05, "Finetune/Loss": 0.9063377380371094, "Finetune/Loss (Raw)": 1.0524632930755615, "Finetune/Step": 2262, "Finetune/Step Time": 4.630777949467301} +{"Finetune/Learning Rate": 1.2547550631387571e-05, "Finetune/Loss": 0.9061455130577087, "Finetune/Loss (Raw)": 0.8523532152175903, "Finetune/Step": 2263, "Finetune/Step Time": 4.635246284306049} +{"Finetune/Learning Rate": 1.2541526068971975e-05, "Finetune/Loss": 0.9060568809509277, "Finetune/Loss (Raw)": 0.9670392274856567, "Finetune/Step": 2264, "Finetune/Step Time": 4.632772883400321} +{"Finetune/Learning Rate": 1.2535500520239727e-05, "Finetune/Loss": 0.9098300337791443, "Finetune/Loss (Raw)": 0.9098945260047913, "Finetune/Step": 2265, "Finetune/Step Time": 4.591062603518367} +{"Finetune/Learning Rate": 1.2529473987529235e-05, "Finetune/Loss": 0.9086421728134155, "Finetune/Loss (Raw)": 0.7704169154167175, "Finetune/Step": 2266, "Finetune/Step Time": 4.598102327436209} +{"Finetune/Learning Rate": 1.2523446473179267e-05, "Finetune/Loss": 0.9086719155311584, "Finetune/Loss (Raw)": 0.8874573111534119, "Finetune/Step": 2267, "Finetune/Step Time": 4.57785315439105} +{"Finetune/Learning Rate": 1.2517417979528991e-05, "Finetune/Loss": 0.9076664447784424, "Finetune/Loss (Raw)": 0.8447009325027466, "Finetune/Step": 2268, "Finetune/Step Time": 4.575214581564069} +{"Finetune/Learning Rate": 1.2511388508917946e-05, "Finetune/Loss": 0.9065536260604858, "Finetune/Loss (Raw)": 0.9517505168914795, "Finetune/Step": 2269, "Finetune/Step Time": 4.5628368742764} +{"Finetune/Learning Rate": 1.250535806368605e-05, "Finetune/Loss": 0.9059945344924927, "Finetune/Loss (Raw)": 0.7923222780227661, "Finetune/Step": 2270, "Finetune/Step Time": 4.5769975408911705} +{"Finetune/Learning Rate": 1.2499326646173606e-05, "Finetune/Loss": 0.9062443375587463, "Finetune/Loss (Raw)": 0.9696925282478333, "Finetune/Step": 2271, "Finetune/Step Time": 4.556409489363432} +{"Finetune/Learning Rate": 1.2493294258721282e-05, "Finetune/Loss": 0.9068269729614258, "Finetune/Loss (Raw)": 0.9873183369636536, "Finetune/Step": 2272, "Finetune/Step Time": 4.551147470250726} +{"Finetune/Learning Rate": 1.2487260903670136e-05, "Finetune/Loss": 0.9085675477981567, "Finetune/Loss (Raw)": 0.9344497919082642, "Finetune/Step": 2273, "Finetune/Step Time": 4.541956963017583} +{"Finetune/Learning Rate": 1.2481226583361592e-05, "Finetune/Loss": 0.9079114198684692, "Finetune/Loss (Raw)": 0.846183717250824, "Finetune/Step": 2274, "Finetune/Step Time": 4.556625302881002} +{"Finetune/Learning Rate": 1.2475191300137452e-05, "Finetune/Loss": 0.9085801243782043, "Finetune/Loss (Raw)": 1.039334774017334, "Finetune/Step": 2275, "Finetune/Step Time": 4.54828411526978} +{"Finetune/Learning Rate": 1.2469155056339894e-05, "Finetune/Loss": 0.9075182676315308, "Finetune/Loss (Raw)": 0.9406249523162842, "Finetune/Step": 2276, "Finetune/Step Time": 4.550600217655301} +{"Finetune/Learning Rate": 1.2463117854311463e-05, "Finetune/Loss": 0.9056359529495239, "Finetune/Loss (Raw)": 0.7587809562683105, "Finetune/Step": 2277, "Finetune/Step Time": 4.55586526542902} +{"Finetune/Learning Rate": 1.2457079696395082e-05, "Finetune/Loss": 0.9061533212661743, "Finetune/Loss (Raw)": 0.9846094250679016, "Finetune/Step": 2278, "Finetune/Step Time": 4.5537307895720005} +{"Finetune/Learning Rate": 1.2451040584934042e-05, "Finetune/Loss": 0.9052779674530029, "Finetune/Loss (Raw)": 1.0429607629776, "Finetune/Step": 2279, "Finetune/Step Time": 4.542978476732969} +{"Finetune/Learning Rate": 1.2445000522272005e-05, "Finetune/Loss": 0.9058189988136292, "Finetune/Loss (Raw)": 0.8970693349838257, "Finetune/Step": 2280, "Finetune/Step Time": 4.53755259513855} +{"Finetune/Learning Rate": 1.2438959510752997e-05, "Finetune/Loss": 0.9062173962593079, "Finetune/Loss (Raw)": 0.9154810905456543, "Finetune/Step": 2281, "Finetune/Step Time": 4.53864181227982} +{"Finetune/Learning Rate": 1.243291755272142e-05, "Finetune/Loss": 0.9064481258392334, "Finetune/Loss (Raw)": 0.915611207485199, "Finetune/Step": 2282, "Finetune/Step Time": 4.527304865419865} +{"Finetune/Learning Rate": 1.2426874650522039e-05, "Finetune/Loss": 0.907402515411377, "Finetune/Loss (Raw)": 1.0459775924682617, "Finetune/Step": 2283, "Finetune/Step Time": 4.5143440421670675} +{"Finetune/Learning Rate": 1.2420830806499988e-05, "Finetune/Loss": 0.9083526134490967, "Finetune/Loss (Raw)": 0.8974435925483704, "Finetune/Step": 2284, "Finetune/Step Time": 4.528605228289962} +{"Finetune/Learning Rate": 1.2414786023000765e-05, "Finetune/Loss": 0.9089361429214478, "Finetune/Loss (Raw)": 0.9709028005599976, "Finetune/Step": 2285, "Finetune/Step Time": 4.5280318055301905} +{"Finetune/Learning Rate": 1.240874030237023e-05, "Finetune/Loss": 0.9088885188102722, "Finetune/Loss (Raw)": 0.9687430262565613, "Finetune/Step": 2286, "Finetune/Step Time": 4.53853858076036} +{"Finetune/Learning Rate": 1.2402693646954607e-05, "Finetune/Loss": 0.9071621894836426, "Finetune/Loss (Raw)": 0.7771199941635132, "Finetune/Step": 2287, "Finetune/Step Time": 4.523312693461776} +{"Finetune/Learning Rate": 1.2396646059100492e-05, "Finetune/Loss": 0.9072149991989136, "Finetune/Loss (Raw)": 0.9580605626106262, "Finetune/Step": 2288, "Finetune/Step Time": 4.527386276051402} +{"Finetune/Learning Rate": 1.2390597541154832e-05, "Finetune/Loss": 0.9062992334365845, "Finetune/Loss (Raw)": 0.7513306140899658, "Finetune/Step": 2289, "Finetune/Step Time": 4.540840540081263} +{"Finetune/Learning Rate": 1.238454809546494e-05, "Finetune/Loss": 0.9070076942443848, "Finetune/Loss (Raw)": 1.0102375745773315, "Finetune/Step": 2290, "Finetune/Step Time": 4.5390092842280865} +{"Finetune/Learning Rate": 1.2378497724378484e-05, "Finetune/Loss": 0.9066295623779297, "Finetune/Loss (Raw)": 0.8424824476242065, "Finetune/Step": 2291, "Finetune/Step Time": 4.5510344840586185} +{"Finetune/Learning Rate": 1.2372446430243497e-05, "Finetune/Loss": 0.9051145315170288, "Finetune/Loss (Raw)": 0.8625866770744324, "Finetune/Step": 2292, "Finetune/Step Time": 4.558984663337469} +{"Finetune/Learning Rate": 1.2366394215408369e-05, "Finetune/Loss": 0.9036087393760681, "Finetune/Loss (Raw)": 0.7507902979850769, "Finetune/Step": 2293, "Finetune/Step Time": 4.543452996760607} +{"Finetune/Learning Rate": 1.2360341082221847e-05, "Finetune/Loss": 0.9034318923950195, "Finetune/Loss (Raw)": 0.9445793628692627, "Finetune/Step": 2294, "Finetune/Step Time": 4.541189398616552} +{"Finetune/Learning Rate": 1.2354287033033032e-05, "Finetune/Loss": 0.9023362398147583, "Finetune/Loss (Raw)": 0.934130847454071, "Finetune/Step": 2295, "Finetune/Step Time": 4.5592330899089575} +{"Finetune/Learning Rate": 1.2348232070191383e-05, "Finetune/Loss": 0.9010650515556335, "Finetune/Loss (Raw)": 0.7617274522781372, "Finetune/Step": 2296, "Finetune/Step Time": 4.5659946501255035} +{"Finetune/Learning Rate": 1.2342176196046715e-05, "Finetune/Loss": 0.9007861614227295, "Finetune/Loss (Raw)": 0.9341543316841125, "Finetune/Step": 2297, "Finetune/Step Time": 4.568441819399595} +{"Finetune/Learning Rate": 1.2336119412949193e-05, "Finetune/Loss": 0.9010606408119202, "Finetune/Loss (Raw)": 0.8799698948860168, "Finetune/Step": 2298, "Finetune/Step Time": 4.563453435897827} +{"Finetune/Learning Rate": 1.2330061723249328e-05, "Finetune/Loss": 0.9015955924987793, "Finetune/Loss (Raw)": 0.988562285900116, "Finetune/Step": 2299, "Finetune/Step Time": 4.561131896451116} +{"Finetune/Learning Rate": 1.2324003129298006e-05, "Finetune/Loss": 0.9021592140197754, "Finetune/Loss (Raw)": 0.9824474453926086, "Finetune/Step": 2300, "Finetune/Step Time": 4.566753696650267} +{"Finetune/Learning Rate": 1.2317943633446441e-05, "Finetune/Loss": 0.9013419151306152, "Finetune/Loss (Raw)": 0.8451758027076721, "Finetune/Step": 2301, "Finetune/Step Time": 4.553109239786863} +{"Finetune/Learning Rate": 1.2311883238046204e-05, "Finetune/Loss": 0.9014955759048462, "Finetune/Loss (Raw)": 0.8078457713127136, "Finetune/Step": 2302, "Finetune/Step Time": 4.5457462053745985} +{"Finetune/Learning Rate": 1.2305821945449222e-05, "Finetune/Loss": 0.9012939929962158, "Finetune/Loss (Raw)": 0.9250954985618591, "Finetune/Step": 2303, "Finetune/Step Time": 4.528238648548722} +{"Finetune/Learning Rate": 1.2299759758007755e-05, "Finetune/Loss": 0.8991774320602417, "Finetune/Loss (Raw)": 0.7244783043861389, "Finetune/Step": 2304, "Finetune/Step Time": 4.536721458658576} +{"Finetune/Learning Rate": 1.2293696678074428e-05, "Finetune/Loss": 0.8970863223075867, "Finetune/Loss (Raw)": 0.8099391460418701, "Finetune/Step": 2305, "Finetune/Step Time": 4.532190343365073} +{"Finetune/Learning Rate": 1.22876327080022e-05, "Finetune/Loss": 0.8990715146064758, "Finetune/Loss (Raw)": 1.063322901725769, "Finetune/Step": 2306, "Finetune/Step Time": 4.534509940072894} +{"Finetune/Learning Rate": 1.2281567850144383e-05, "Finetune/Loss": 0.8991739749908447, "Finetune/Loss (Raw)": 0.9843832850456238, "Finetune/Step": 2307, "Finetune/Step Time": 4.51859749481082} +{"Finetune/Learning Rate": 1.2275502106854624e-05, "Finetune/Loss": 0.8964023590087891, "Finetune/Loss (Raw)": 0.6201061606407166, "Finetune/Step": 2308, "Finetune/Step Time": 4.521487759426236} +{"Finetune/Learning Rate": 1.2269435480486923e-05, "Finetune/Loss": 0.8961794376373291, "Finetune/Loss (Raw)": 0.9514752626419067, "Finetune/Step": 2309, "Finetune/Step Time": 4.521607330068946} +{"Finetune/Learning Rate": 1.226336797339562e-05, "Finetune/Loss": 0.8958958387374878, "Finetune/Loss (Raw)": 0.965658962726593, "Finetune/Step": 2310, "Finetune/Step Time": 4.516116891056299} +{"Finetune/Learning Rate": 1.2257299587935394e-05, "Finetune/Loss": 0.8978170156478882, "Finetune/Loss (Raw)": 1.037923812866211, "Finetune/Step": 2311, "Finetune/Step Time": 4.495807154104114} +{"Finetune/Learning Rate": 1.2251230326461268e-05, "Finetune/Loss": 0.8982561230659485, "Finetune/Loss (Raw)": 0.881894588470459, "Finetune/Step": 2312, "Finetune/Step Time": 4.505141153931618} +{"Finetune/Learning Rate": 1.2245160191328605e-05, "Finetune/Loss": 0.898395299911499, "Finetune/Loss (Raw)": 0.9517824053764343, "Finetune/Step": 2313, "Finetune/Step Time": 4.510616747662425} +{"Finetune/Learning Rate": 1.2239089184893102e-05, "Finetune/Loss": 0.8980761170387268, "Finetune/Loss (Raw)": 0.8689737319946289, "Finetune/Step": 2314, "Finetune/Step Time": 4.519229354336858} +{"Finetune/Learning Rate": 1.2233017309510804e-05, "Finetune/Loss": 0.8989423513412476, "Finetune/Loss (Raw)": 0.8665737509727478, "Finetune/Step": 2315, "Finetune/Step Time": 4.485478555783629} +{"Finetune/Learning Rate": 1.2226944567538084e-05, "Finetune/Loss": 0.8993874788284302, "Finetune/Loss (Raw)": 0.9836289286613464, "Finetune/Step": 2316, "Finetune/Step Time": 4.450546111911535} +{"Finetune/Learning Rate": 1.2220870961331654e-05, "Finetune/Loss": 0.8995245695114136, "Finetune/Loss (Raw)": 0.8713972568511963, "Finetune/Step": 2317, "Finetune/Step Time": 4.447040783241391} +{"Finetune/Learning Rate": 1.2214796493248564e-05, "Finetune/Loss": 0.8997994661331177, "Finetune/Loss (Raw)": 0.8902594447135925, "Finetune/Step": 2318, "Finetune/Step Time": 4.435046510770917} +{"Finetune/Learning Rate": 1.2208721165646193e-05, "Finetune/Loss": 0.9007689952850342, "Finetune/Loss (Raw)": 1.0927034616470337, "Finetune/Step": 2319, "Finetune/Step Time": 4.4261273220181465} +{"Finetune/Learning Rate": 1.2202644980882258e-05, "Finetune/Loss": 0.9017593860626221, "Finetune/Loss (Raw)": 0.9407971501350403, "Finetune/Step": 2320, "Finetune/Step Time": 4.425009120255709} +{"Finetune/Learning Rate": 1.2196567941314815e-05, "Finetune/Loss": 0.9012916088104248, "Finetune/Loss (Raw)": 0.9376732110977173, "Finetune/Step": 2321, "Finetune/Step Time": 4.432443859055638} +{"Finetune/Learning Rate": 1.2190490049302233e-05, "Finetune/Loss": 0.9013978242874146, "Finetune/Loss (Raw)": 1.0017110109329224, "Finetune/Step": 2322, "Finetune/Step Time": 4.4328353852033615} +{"Finetune/Learning Rate": 1.218441130720323e-05, "Finetune/Loss": 0.8998676538467407, "Finetune/Loss (Raw)": 0.8103861212730408, "Finetune/Step": 2323, "Finetune/Step Time": 4.447199922055006} +{"Finetune/Learning Rate": 1.2178331717376848e-05, "Finetune/Loss": 0.8998913168907166, "Finetune/Loss (Raw)": 0.8638238310813904, "Finetune/Step": 2324, "Finetune/Step Time": 4.4371540285646915} +{"Finetune/Learning Rate": 1.217225128218245e-05, "Finetune/Loss": 0.9001578092575073, "Finetune/Loss (Raw)": 1.0382424592971802, "Finetune/Step": 2325, "Finetune/Step Time": 4.443853667005897} +{"Finetune/Learning Rate": 1.2166170003979742e-05, "Finetune/Loss": 0.8966944813728333, "Finetune/Loss (Raw)": 0.43793073296546936, "Finetune/Step": 2326, "Finetune/Step Time": 4.475039537996054} +{"Finetune/Learning Rate": 1.2160087885128745e-05, "Finetune/Loss": 0.8971583843231201, "Finetune/Loss (Raw)": 0.9176393747329712, "Finetune/Step": 2327, "Finetune/Step Time": 4.490418469533324} +{"Finetune/Learning Rate": 1.2154004927989815e-05, "Finetune/Loss": 0.8969199657440186, "Finetune/Loss (Raw)": 0.8622083067893982, "Finetune/Step": 2328, "Finetune/Step Time": 4.47927363216877} +{"Finetune/Learning Rate": 1.2147921134923624e-05, "Finetune/Loss": 0.9018594026565552, "Finetune/Loss (Raw)": 0.9897170662879944, "Finetune/Step": 2329, "Finetune/Step Time": 4.443433919921517} +{"Finetune/Learning Rate": 1.2141836508291175e-05, "Finetune/Loss": 0.9010357856750488, "Finetune/Loss (Raw)": 0.8153582215309143, "Finetune/Step": 2330, "Finetune/Step Time": 4.450395476073027} +{"Finetune/Learning Rate": 1.2135751050453793e-05, "Finetune/Loss": 0.9015759229660034, "Finetune/Loss (Raw)": 0.9507362842559814, "Finetune/Step": 2331, "Finetune/Step Time": 4.458247708156705} +{"Finetune/Learning Rate": 1.2129664763773125e-05, "Finetune/Loss": 0.9037240147590637, "Finetune/Loss (Raw)": 1.1287050247192383, "Finetune/Step": 2332, "Finetune/Step Time": 4.448622273281217} +{"Finetune/Learning Rate": 1.212357765061114e-05, "Finetune/Loss": 0.9082035422325134, "Finetune/Loss (Raw)": 1.0021955966949463, "Finetune/Step": 2333, "Finetune/Step Time": 4.429213088005781} +{"Finetune/Learning Rate": 1.211748971333013e-05, "Finetune/Loss": 0.9102871417999268, "Finetune/Loss (Raw)": 0.9516742825508118, "Finetune/Step": 2334, "Finetune/Step Time": 4.399864515289664} +{"Finetune/Learning Rate": 1.21114009542927e-05, "Finetune/Loss": 0.9096074104309082, "Finetune/Loss (Raw)": 0.8802865147590637, "Finetune/Step": 2335, "Finetune/Step Time": 4.396582970395684} +{"Finetune/Learning Rate": 1.2105311375861786e-05, "Finetune/Loss": 0.9097537994384766, "Finetune/Loss (Raw)": 0.8941760659217834, "Finetune/Step": 2336, "Finetune/Step Time": 4.4064927734434605} +{"Finetune/Learning Rate": 1.2099220980400625e-05, "Finetune/Loss": 0.9103004932403564, "Finetune/Loss (Raw)": 0.9699265956878662, "Finetune/Step": 2337, "Finetune/Step Time": 4.406316561624408} +{"Finetune/Learning Rate": 1.2093129770272787e-05, "Finetune/Loss": 0.9149296879768372, "Finetune/Loss (Raw)": 0.9860789775848389, "Finetune/Step": 2338, "Finetune/Step Time": 4.369436388835311} +{"Finetune/Learning Rate": 1.208703774784215e-05, "Finetune/Loss": 0.913610577583313, "Finetune/Loss (Raw)": 0.8529616594314575, "Finetune/Step": 2339, "Finetune/Step Time": 4.365152910351753} +{"Finetune/Learning Rate": 1.2080944915472908e-05, "Finetune/Loss": 0.9150608777999878, "Finetune/Loss (Raw)": 0.9080622792243958, "Finetune/Step": 2340, "Finetune/Step Time": 4.3505670838057995} +{"Finetune/Learning Rate": 1.2074851275529575e-05, "Finetune/Loss": 0.9167789816856384, "Finetune/Loss (Raw)": 1.0463201999664307, "Finetune/Step": 2341, "Finetune/Step Time": 4.377986926585436} +{"Finetune/Learning Rate": 1.2068756830376966e-05, "Finetune/Loss": 0.913941502571106, "Finetune/Loss (Raw)": 0.35791948437690735, "Finetune/Step": 2342, "Finetune/Step Time": 4.411501241847873} +{"Finetune/Learning Rate": 1.2062661582380222e-05, "Finetune/Loss": 0.9140942096710205, "Finetune/Loss (Raw)": 1.0031180381774902, "Finetune/Step": 2343, "Finetune/Step Time": 4.429364409297705} +{"Finetune/Learning Rate": 1.2056565533904787e-05, "Finetune/Loss": 0.9137758016586304, "Finetune/Loss (Raw)": 0.9144883751869202, "Finetune/Step": 2344, "Finetune/Step Time": 4.45287187024951} +{"Finetune/Learning Rate": 1.2050468687316418e-05, "Finetune/Loss": 0.9148582220077515, "Finetune/Loss (Raw)": 1.16091787815094, "Finetune/Step": 2345, "Finetune/Step Time": 4.446572296321392} +{"Finetune/Learning Rate": 1.2044371044981188e-05, "Finetune/Loss": 0.9161162972450256, "Finetune/Loss (Raw)": 0.9131386876106262, "Finetune/Step": 2346, "Finetune/Step Time": 4.44232089817524} +{"Finetune/Learning Rate": 1.2038272609265464e-05, "Finetune/Loss": 0.9163632988929749, "Finetune/Loss (Raw)": 0.9049143195152283, "Finetune/Step": 2347, "Finetune/Step Time": 4.405661594122648} +{"Finetune/Learning Rate": 1.2032173382535936e-05, "Finetune/Loss": 0.9165194034576416, "Finetune/Loss (Raw)": 1.0142558813095093, "Finetune/Step": 2348, "Finetune/Step Time": 4.4077508840709925} +{"Finetune/Learning Rate": 1.2026073367159597e-05, "Finetune/Loss": 0.917224645614624, "Finetune/Loss (Raw)": 0.9353235960006714, "Finetune/Step": 2349, "Finetune/Step Time": 4.399146355688572} +{"Finetune/Learning Rate": 1.2019972565503733e-05, "Finetune/Loss": 0.9167487025260925, "Finetune/Loss (Raw)": 0.9168080687522888, "Finetune/Step": 2350, "Finetune/Step Time": 4.397863175719976} +{"Finetune/Learning Rate": 1.2013870979935957e-05, "Finetune/Loss": 0.9174864292144775, "Finetune/Loss (Raw)": 1.0418750047683716, "Finetune/Step": 2351, "Finetune/Step Time": 4.3948045782744884} +{"Finetune/Learning Rate": 1.200776861282417e-05, "Finetune/Loss": 0.918809175491333, "Finetune/Loss (Raw)": 1.0866159200668335, "Finetune/Step": 2352, "Finetune/Step Time": 4.403190674260259} +{"Finetune/Learning Rate": 1.2001665466536582e-05, "Finetune/Loss": 0.9188871383666992, "Finetune/Loss (Raw)": 0.9171014428138733, "Finetune/Step": 2353, "Finetune/Step Time": 4.397219397127628} +{"Finetune/Learning Rate": 1.199556154344171e-05, "Finetune/Loss": 0.9188061952590942, "Finetune/Loss (Raw)": 0.9282801151275635, "Finetune/Step": 2354, "Finetune/Step Time": 4.38951058126986} +{"Finetune/Learning Rate": 1.1989456845908356e-05, "Finetune/Loss": 0.917560338973999, "Finetune/Loss (Raw)": 0.8277881741523743, "Finetune/Step": 2355, "Finetune/Step Time": 4.386559868231416} +{"Finetune/Learning Rate": 1.1983351376305648e-05, "Finetune/Loss": 0.9168499112129211, "Finetune/Loss (Raw)": 0.8108100891113281, "Finetune/Step": 2356, "Finetune/Step Time": 4.387198029085994} +{"Finetune/Learning Rate": 1.197724513700299e-05, "Finetune/Loss": 0.9167643785476685, "Finetune/Loss (Raw)": 0.9127688407897949, "Finetune/Step": 2357, "Finetune/Step Time": 4.391333069652319} +{"Finetune/Learning Rate": 1.1971138130370097e-05, "Finetune/Loss": 0.918060302734375, "Finetune/Loss (Raw)": 1.034088373184204, "Finetune/Step": 2358, "Finetune/Step Time": 4.388240301981568} +{"Finetune/Learning Rate": 1.1965030358776975e-05, "Finetune/Loss": 0.9125707149505615, "Finetune/Loss (Raw)": 0.295085072517395, "Finetune/Step": 2359, "Finetune/Step Time": 4.4248806443065405} +{"Finetune/Learning Rate": 1.1958921824593936e-05, "Finetune/Loss": 0.911699116230011, "Finetune/Loss (Raw)": 0.9250917434692383, "Finetune/Step": 2360, "Finetune/Step Time": 4.424032524228096} +{"Finetune/Learning Rate": 1.1952812530191586e-05, "Finetune/Loss": 0.9127983450889587, "Finetune/Loss (Raw)": 0.9489085078239441, "Finetune/Step": 2361, "Finetune/Step Time": 4.418117597699165} +{"Finetune/Learning Rate": 1.1946702477940815e-05, "Finetune/Loss": 0.9125703573226929, "Finetune/Loss (Raw)": 0.953630805015564, "Finetune/Step": 2362, "Finetune/Step Time": 4.411348866298795} +{"Finetune/Learning Rate": 1.194059167021282e-05, "Finetune/Loss": 0.9101277589797974, "Finetune/Loss (Raw)": 0.7271248698234558, "Finetune/Step": 2363, "Finetune/Step Time": 4.422884564846754} +{"Finetune/Learning Rate": 1.1934480109379083e-05, "Finetune/Loss": 0.908973753452301, "Finetune/Loss (Raw)": 0.8558248281478882, "Finetune/Step": 2364, "Finetune/Step Time": 4.420030726119876} +{"Finetune/Learning Rate": 1.1928367797811384e-05, "Finetune/Loss": 0.9099640846252441, "Finetune/Loss (Raw)": 1.0368680953979492, "Finetune/Step": 2365, "Finetune/Step Time": 4.40970428660512} +{"Finetune/Learning Rate": 1.1922254737881792e-05, "Finetune/Loss": 0.9098649621009827, "Finetune/Loss (Raw)": 0.7686304450035095, "Finetune/Step": 2366, "Finetune/Step Time": 4.4061578232795} +{"Finetune/Learning Rate": 1.1916140931962661e-05, "Finetune/Loss": 0.9082906246185303, "Finetune/Loss (Raw)": 0.9774743914604187, "Finetune/Step": 2367, "Finetune/Step Time": 4.411396021023393} +{"Finetune/Learning Rate": 1.191002638242665e-05, "Finetune/Loss": 0.9071540832519531, "Finetune/Loss (Raw)": 0.923202395439148, "Finetune/Step": 2368, "Finetune/Step Time": 4.407022766768932} +{"Finetune/Learning Rate": 1.1903911091646684e-05, "Finetune/Loss": 0.9072592258453369, "Finetune/Loss (Raw)": 1.0147548913955688, "Finetune/Step": 2369, "Finetune/Step Time": 4.404081584885716} +{"Finetune/Learning Rate": 1.1897795061995998e-05, "Finetune/Loss": 0.9073889255523682, "Finetune/Loss (Raw)": 0.9160827398300171, "Finetune/Step": 2370, "Finetune/Step Time": 4.406154001131654} +{"Finetune/Learning Rate": 1.18916782958481e-05, "Finetune/Loss": 0.9083316922187805, "Finetune/Loss (Raw)": 1.024092674255371, "Finetune/Step": 2371, "Finetune/Step Time": 4.404985349625349} +{"Finetune/Learning Rate": 1.1885560795576782e-05, "Finetune/Loss": 0.906245768070221, "Finetune/Loss (Raw)": 0.9058061242103577, "Finetune/Step": 2372, "Finetune/Step Time": 4.407673954963684} +{"Finetune/Learning Rate": 1.1879442563556141e-05, "Finetune/Loss": 0.9066305160522461, "Finetune/Loss (Raw)": 1.0205610990524292, "Finetune/Step": 2373, "Finetune/Step Time": 4.430020188912749} +{"Finetune/Learning Rate": 1.1873323602160526e-05, "Finetune/Loss": 0.904511034488678, "Finetune/Loss (Raw)": 0.8230372667312622, "Finetune/Step": 2374, "Finetune/Step Time": 4.431435082107782} +{"Finetune/Learning Rate": 1.1867203913764599e-05, "Finetune/Loss": 0.9038686752319336, "Finetune/Loss (Raw)": 0.77821946144104, "Finetune/Step": 2375, "Finetune/Step Time": 4.43459040299058} +{"Finetune/Learning Rate": 1.1861083500743284e-05, "Finetune/Loss": 0.9052959084510803, "Finetune/Loss (Raw)": 0.9597607254981995, "Finetune/Step": 2376, "Finetune/Step Time": 4.439652977511287} +{"Finetune/Learning Rate": 1.1854962365471798e-05, "Finetune/Loss": 0.9068331122398376, "Finetune/Loss (Raw)": 1.0715950727462769, "Finetune/Step": 2377, "Finetune/Step Time": 4.439641244709492} +{"Finetune/Learning Rate": 1.1848840510325635e-05, "Finetune/Loss": 0.9072480797767639, "Finetune/Loss (Raw)": 1.0131776332855225, "Finetune/Step": 2378, "Finetune/Step Time": 4.417758207768202} +{"Finetune/Learning Rate": 1.184271793768056e-05, "Finetune/Loss": 0.9070842266082764, "Finetune/Loss (Raw)": 0.9419175982475281, "Finetune/Step": 2379, "Finetune/Step Time": 4.434973301365972} +{"Finetune/Learning Rate": 1.1836594649912636e-05, "Finetune/Loss": 0.9108273983001709, "Finetune/Loss (Raw)": 0.8020150065422058, "Finetune/Step": 2380, "Finetune/Step Time": 4.396160040050745} +{"Finetune/Learning Rate": 1.1830470649398183e-05, "Finetune/Loss": 0.9100182056427002, "Finetune/Loss (Raw)": 0.9299728274345398, "Finetune/Step": 2381, "Finetune/Step Time": 4.399469215422869} +{"Finetune/Learning Rate": 1.1824345938513806e-05, "Finetune/Loss": 0.9080120325088501, "Finetune/Loss (Raw)": 0.7937242984771729, "Finetune/Step": 2382, "Finetune/Step Time": 4.411028377711773} +{"Finetune/Learning Rate": 1.181822051963639e-05, "Finetune/Loss": 0.9058327078819275, "Finetune/Loss (Raw)": 0.6891952753067017, "Finetune/Step": 2383, "Finetune/Step Time": 4.424246931448579} +{"Finetune/Learning Rate": 1.1812094395143086e-05, "Finetune/Loss": 0.9075204133987427, "Finetune/Loss (Raw)": 1.100179672241211, "Finetune/Step": 2384, "Finetune/Step Time": 4.419933795928955} +{"Finetune/Learning Rate": 1.1805967567411325e-05, "Finetune/Loss": 0.9070220589637756, "Finetune/Loss (Raw)": 0.9151192307472229, "Finetune/Step": 2385, "Finetune/Step Time": 4.425072845071554} +{"Finetune/Learning Rate": 1.1799840038818813e-05, "Finetune/Loss": 0.9091938138008118, "Finetune/Loss (Raw)": 1.0409729480743408, "Finetune/Step": 2386, "Finetune/Step Time": 4.431865448132157} +{"Finetune/Learning Rate": 1.1793711811743515e-05, "Finetune/Loss": 0.9094090461730957, "Finetune/Loss (Raw)": 0.8805992603302002, "Finetune/Step": 2387, "Finetune/Step Time": 4.445863202214241} +{"Finetune/Learning Rate": 1.1787582888563689e-05, "Finetune/Loss": 0.9103235602378845, "Finetune/Loss (Raw)": 0.9897632598876953, "Finetune/Step": 2388, "Finetune/Step Time": 4.456220664083958} +{"Finetune/Learning Rate": 1.1781453271657839e-05, "Finetune/Loss": 0.9109164476394653, "Finetune/Loss (Raw)": 1.043716549873352, "Finetune/Step": 2389, "Finetune/Step Time": 4.454487580806017} +{"Finetune/Learning Rate": 1.1775322963404757e-05, "Finetune/Loss": 0.9107815027236938, "Finetune/Loss (Raw)": 1.0351881980895996, "Finetune/Step": 2390, "Finetune/Step Time": 4.4582216162234545} +{"Finetune/Learning Rate": 1.176919196618349e-05, "Finetune/Loss": 0.9119405746459961, "Finetune/Loss (Raw)": 1.0007113218307495, "Finetune/Step": 2391, "Finetune/Step Time": 4.457119885832071} +{"Finetune/Learning Rate": 1.1763060282373364e-05, "Finetune/Loss": 0.9117344617843628, "Finetune/Loss (Raw)": 0.9406536817550659, "Finetune/Step": 2392, "Finetune/Step Time": 4.457405982539058} +{"Finetune/Learning Rate": 1.1756927914353963e-05, "Finetune/Loss": 0.9126623272895813, "Finetune/Loss (Raw)": 1.028668999671936, "Finetune/Step": 2393, "Finetune/Step Time": 4.470523297786713} +{"Finetune/Learning Rate": 1.175079486450514e-05, "Finetune/Loss": 0.9097601175308228, "Finetune/Loss (Raw)": 0.398933082818985, "Finetune/Step": 2394, "Finetune/Step Time": 4.496850790455937} +{"Finetune/Learning Rate": 1.1744661135207009e-05, "Finetune/Loss": 0.9102332592010498, "Finetune/Loss (Raw)": 0.9480125904083252, "Finetune/Step": 2395, "Finetune/Step Time": 4.501909375190735} +{"Finetune/Learning Rate": 1.1738526728839956e-05, "Finetune/Loss": 0.9104965925216675, "Finetune/Loss (Raw)": 0.878414511680603, "Finetune/Step": 2396, "Finetune/Step Time": 4.499018419533968} +{"Finetune/Learning Rate": 1.1732391647784618e-05, "Finetune/Loss": 0.9104005694389343, "Finetune/Loss (Raw)": 0.9394578337669373, "Finetune/Step": 2397, "Finetune/Step Time": 4.5080086924135685} +{"Finetune/Learning Rate": 1.1726255894421908e-05, "Finetune/Loss": 0.9126817584037781, "Finetune/Loss (Raw)": 1.0843127965927124, "Finetune/Step": 2398, "Finetune/Step Time": 4.494006933644414} +{"Finetune/Learning Rate": 1.1720119471132985e-05, "Finetune/Loss": 0.9129038453102112, "Finetune/Loss (Raw)": 0.9981198906898499, "Finetune/Step": 2399, "Finetune/Step Time": 4.506276648491621} +{"Finetune/Learning Rate": 1.1713982380299278e-05, "Finetune/Loss": 0.9116101861000061, "Finetune/Loss (Raw)": 0.8217272162437439, "Finetune/Step": 2400, "Finetune/Step Time": 4.492554731667042} +{"Finetune/Learning Rate": 1.1707844624302471e-05, "Finetune/Loss": 0.9117599725723267, "Finetune/Loss (Raw)": 0.9536228775978088, "Finetune/Step": 2401, "Finetune/Step Time": 4.490784876048565} +{"Finetune/Learning Rate": 1.170170620552451e-05, "Finetune/Loss": 0.9116028547286987, "Finetune/Loss (Raw)": 0.8260769844055176, "Finetune/Step": 2402, "Finetune/Step Time": 4.475949002429843} +{"Finetune/Learning Rate": 1.1695567126347593e-05, "Finetune/Loss": 0.9107996821403503, "Finetune/Loss (Raw)": 0.9365285038948059, "Finetune/Step": 2403, "Finetune/Step Time": 4.484875848516822} +{"Finetune/Learning Rate": 1.1689427389154176e-05, "Finetune/Loss": 0.9109116792678833, "Finetune/Loss (Raw)": 0.9549631476402283, "Finetune/Step": 2404, "Finetune/Step Time": 4.484377855435014} +{"Finetune/Learning Rate": 1.1683286996326975e-05, "Finetune/Loss": 0.9123598337173462, "Finetune/Loss (Raw)": 0.9441465735435486, "Finetune/Step": 2405, "Finetune/Step Time": 4.480089224874973} +{"Finetune/Learning Rate": 1.1677145950248954e-05, "Finetune/Loss": 0.9125081896781921, "Finetune/Loss (Raw)": 1.0035942792892456, "Finetune/Step": 2406, "Finetune/Step Time": 4.481499668210745} +{"Finetune/Learning Rate": 1.167100425330333e-05, "Finetune/Loss": 0.9098151922225952, "Finetune/Loss (Raw)": 0.6982574462890625, "Finetune/Step": 2407, "Finetune/Step Time": 4.502984011545777} +{"Finetune/Learning Rate": 1.1664861907873583e-05, "Finetune/Loss": 0.9115763902664185, "Finetune/Loss (Raw)": 1.1224972009658813, "Finetune/Step": 2408, "Finetune/Step Time": 4.511732576414943} +{"Finetune/Learning Rate": 1.165871891634343e-05, "Finetune/Loss": 0.9114277362823486, "Finetune/Loss (Raw)": 0.8964594602584839, "Finetune/Step": 2409, "Finetune/Step Time": 4.517255108803511} +{"Finetune/Learning Rate": 1.165257528109685e-05, "Finetune/Loss": 0.9115355014801025, "Finetune/Loss (Raw)": 0.9293947219848633, "Finetune/Step": 2410, "Finetune/Step Time": 4.520740494132042} +{"Finetune/Learning Rate": 1.1646431004518068e-05, "Finetune/Loss": 0.9093645215034485, "Finetune/Loss (Raw)": 0.7681035995483398, "Finetune/Step": 2411, "Finetune/Step Time": 4.528321646153927} +{"Finetune/Learning Rate": 1.1640286088991558e-05, "Finetune/Loss": 0.909338116645813, "Finetune/Loss (Raw)": 0.8940690159797668, "Finetune/Step": 2412, "Finetune/Step Time": 4.506548693403602} +{"Finetune/Learning Rate": 1.1634140536902039e-05, "Finetune/Loss": 0.9081519842147827, "Finetune/Loss (Raw)": 0.8190782070159912, "Finetune/Step": 2413, "Finetune/Step Time": 4.496902687475085} +{"Finetune/Learning Rate": 1.1627994350634482e-05, "Finetune/Loss": 0.9080673456192017, "Finetune/Loss (Raw)": 0.9579054713249207, "Finetune/Step": 2414, "Finetune/Step Time": 4.482652544975281} +{"Finetune/Learning Rate": 1.1621847532574101e-05, "Finetune/Loss": 0.9089618921279907, "Finetune/Loss (Raw)": 0.8916193842887878, "Finetune/Step": 2415, "Finetune/Step Time": 4.482538966462016} +{"Finetune/Learning Rate": 1.1615700085106357e-05, "Finetune/Loss": 0.908965528011322, "Finetune/Loss (Raw)": 0.9585236310958862, "Finetune/Step": 2416, "Finetune/Step Time": 4.480255298316479} +{"Finetune/Learning Rate": 1.1609552010616953e-05, "Finetune/Loss": 0.9102583527565002, "Finetune/Loss (Raw)": 0.9168151021003723, "Finetune/Step": 2417, "Finetune/Step Time": 4.478305343538523} +{"Finetune/Learning Rate": 1.1603403311491841e-05, "Finetune/Loss": 0.9058345556259155, "Finetune/Loss (Raw)": 0.4439903795719147, "Finetune/Step": 2418, "Finetune/Step Time": 4.5096050295978785} +{"Finetune/Learning Rate": 1.1597253990117205e-05, "Finetune/Loss": 0.9063433408737183, "Finetune/Loss (Raw)": 0.907612681388855, "Finetune/Step": 2419, "Finetune/Step Time": 4.516976624727249} +{"Finetune/Learning Rate": 1.1591104048879479e-05, "Finetune/Loss": 0.9080297946929932, "Finetune/Loss (Raw)": 1.0784424543380737, "Finetune/Step": 2420, "Finetune/Step Time": 4.512472363188863} +{"Finetune/Learning Rate": 1.1584953490165334e-05, "Finetune/Loss": 0.9082335829734802, "Finetune/Loss (Raw)": 0.7768816947937012, "Finetune/Step": 2421, "Finetune/Step Time": 4.522130347788334} +{"Finetune/Learning Rate": 1.1578802316361685e-05, "Finetune/Loss": 0.9070398807525635, "Finetune/Loss (Raw)": 0.7917836904525757, "Finetune/Step": 2422, "Finetune/Step Time": 4.537621553987265} +{"Finetune/Learning Rate": 1.157265052985568e-05, "Finetune/Loss": 0.9055384397506714, "Finetune/Loss (Raw)": 0.7419469952583313, "Finetune/Step": 2423, "Finetune/Step Time": 4.530784761533141} +{"Finetune/Learning Rate": 1.156649813303471e-05, "Finetune/Loss": 0.9069344401359558, "Finetune/Loss (Raw)": 0.9404125213623047, "Finetune/Step": 2424, "Finetune/Step Time": 4.5242018438875675} +{"Finetune/Learning Rate": 1.1560345128286396e-05, "Finetune/Loss": 0.9062429666519165, "Finetune/Loss (Raw)": 0.845651388168335, "Finetune/Step": 2425, "Finetune/Step Time": 4.534576006233692} +{"Finetune/Learning Rate": 1.1554191517998598e-05, "Finetune/Loss": 0.9022887945175171, "Finetune/Loss (Raw)": 0.3738311231136322, "Finetune/Step": 2426, "Finetune/Step Time": 4.571345210075378} +{"Finetune/Learning Rate": 1.1548037304559419e-05, "Finetune/Loss": 0.9021692276000977, "Finetune/Loss (Raw)": 0.9732609987258911, "Finetune/Step": 2427, "Finetune/Step Time": 4.562214406207204} +{"Finetune/Learning Rate": 1.1541882490357183e-05, "Finetune/Loss": 0.9007717370986938, "Finetune/Loss (Raw)": 0.8035658001899719, "Finetune/Step": 2428, "Finetune/Step Time": 4.556899691000581} +{"Finetune/Learning Rate": 1.1535727077780456e-05, "Finetune/Loss": 0.9018118381500244, "Finetune/Loss (Raw)": 0.9783137440681458, "Finetune/Step": 2429, "Finetune/Step Time": 4.574949983507395} +{"Finetune/Learning Rate": 1.1529571069218033e-05, "Finetune/Loss": 0.9014341831207275, "Finetune/Loss (Raw)": 0.7595009803771973, "Finetune/Step": 2430, "Finetune/Step Time": 4.599527085199952} +{"Finetune/Learning Rate": 1.152341446705894e-05, "Finetune/Loss": 0.902154803276062, "Finetune/Loss (Raw)": 1.0173442363739014, "Finetune/Step": 2431, "Finetune/Step Time": 4.597428534179926} +{"Finetune/Learning Rate": 1.1517257273692432e-05, "Finetune/Loss": 0.9037089347839355, "Finetune/Loss (Raw)": 0.9234005212783813, "Finetune/Step": 2432, "Finetune/Step Time": 4.600165605545044} +{"Finetune/Learning Rate": 1.1511099491508003e-05, "Finetune/Loss": 0.9042214155197144, "Finetune/Loss (Raw)": 0.8755366206169128, "Finetune/Step": 2433, "Finetune/Step Time": 4.599643470719457} +{"Finetune/Learning Rate": 1.1504941122895361e-05, "Finetune/Loss": 0.9031648635864258, "Finetune/Loss (Raw)": 0.9280812740325928, "Finetune/Step": 2434, "Finetune/Step Time": 4.598509259521961} +{"Finetune/Learning Rate": 1.1498782170244449e-05, "Finetune/Loss": 0.9024099707603455, "Finetune/Loss (Raw)": 0.8877608776092529, "Finetune/Step": 2435, "Finetune/Step Time": 4.607768120244145} +{"Finetune/Learning Rate": 1.1492622635945438e-05, "Finetune/Loss": 0.9052343964576721, "Finetune/Loss (Raw)": 0.9816291928291321, "Finetune/Step": 2436, "Finetune/Step Time": 4.600650819018483} +{"Finetune/Learning Rate": 1.1486462522388724e-05, "Finetune/Loss": 0.9046825766563416, "Finetune/Loss (Raw)": 0.8808440566062927, "Finetune/Step": 2437, "Finetune/Step Time": 4.6060463059693575} +{"Finetune/Learning Rate": 1.1480301831964926e-05, "Finetune/Loss": 0.9050898551940918, "Finetune/Loss (Raw)": 1.0177925825119019, "Finetune/Step": 2438, "Finetune/Step Time": 4.610275654122233} +{"Finetune/Learning Rate": 1.1474140567064889e-05, "Finetune/Loss": 0.9047884345054626, "Finetune/Loss (Raw)": 0.9993427395820618, "Finetune/Step": 2439, "Finetune/Step Time": 4.610510574653745} +{"Finetune/Learning Rate": 1.1467978730079677e-05, "Finetune/Loss": 0.9057156443595886, "Finetune/Loss (Raw)": 1.0005714893341064, "Finetune/Step": 2440, "Finetune/Step Time": 4.590565092861652} +{"Finetune/Learning Rate": 1.1461816323400579e-05, "Finetune/Loss": 0.9058326482772827, "Finetune/Loss (Raw)": 0.9667580127716064, "Finetune/Step": 2441, "Finetune/Step Time": 4.5863668117672205} +{"Finetune/Learning Rate": 1.1455653349419105e-05, "Finetune/Loss": 0.9062817096710205, "Finetune/Loss (Raw)": 0.926457405090332, "Finetune/Step": 2442, "Finetune/Step Time": 4.582387514412403} +{"Finetune/Learning Rate": 1.1449489810526985e-05, "Finetune/Loss": 0.9071732759475708, "Finetune/Loss (Raw)": 0.980688750743866, "Finetune/Step": 2443, "Finetune/Step Time": 4.587812673300505} +{"Finetune/Learning Rate": 1.144332570911617e-05, "Finetune/Loss": 0.907102108001709, "Finetune/Loss (Raw)": 0.9745209813117981, "Finetune/Step": 2444, "Finetune/Step Time": 4.595047067850828} +{"Finetune/Learning Rate": 1.1437161047578826e-05, "Finetune/Loss": 0.9033068418502808, "Finetune/Loss (Raw)": 0.385602205991745, "Finetune/Step": 2445, "Finetune/Step Time": 4.62890512123704} +{"Finetune/Learning Rate": 1.1430995828307337e-05, "Finetune/Loss": 0.9036376476287842, "Finetune/Loss (Raw)": 0.932602047920227, "Finetune/Step": 2446, "Finetune/Step Time": 4.630634620785713} +{"Finetune/Learning Rate": 1.1424830053694303e-05, "Finetune/Loss": 0.9022191762924194, "Finetune/Loss (Raw)": 0.9111475348472595, "Finetune/Step": 2447, "Finetune/Step Time": 4.63530052639544} +{"Finetune/Learning Rate": 1.1418663726132543e-05, "Finetune/Loss": 0.9027073383331299, "Finetune/Loss (Raw)": 1.0032777786254883, "Finetune/Step": 2448, "Finetune/Step Time": 4.6367192491889} +{"Finetune/Learning Rate": 1.141249684801509e-05, "Finetune/Loss": 0.9017640352249146, "Finetune/Loss (Raw)": 0.8169254064559937, "Finetune/Step": 2449, "Finetune/Step Time": 4.6285109259188175} +{"Finetune/Learning Rate": 1.140632942173519e-05, "Finetune/Loss": 0.8977557420730591, "Finetune/Loss (Raw)": 0.4886517822742462, "Finetune/Step": 2450, "Finetune/Step Time": 4.663020936772227} +{"Finetune/Learning Rate": 1.1400161449686293e-05, "Finetune/Loss": 0.8939336538314819, "Finetune/Loss (Raw)": 0.321161150932312, "Finetune/Step": 2451, "Finetune/Step Time": 4.688334539532661} +{"Finetune/Learning Rate": 1.1393992934262073e-05, "Finetune/Loss": 0.8947299122810364, "Finetune/Loss (Raw)": 0.9657447934150696, "Finetune/Step": 2452, "Finetune/Step Time": 4.689930649474263} +{"Finetune/Learning Rate": 1.1387823877856413e-05, "Finetune/Loss": 0.893926203250885, "Finetune/Loss (Raw)": 0.9353659152984619, "Finetune/Step": 2453, "Finetune/Step Time": 4.707562817260623} +{"Finetune/Learning Rate": 1.1381654282863399e-05, "Finetune/Loss": 0.8977676630020142, "Finetune/Loss (Raw)": 0.9296462535858154, "Finetune/Step": 2454, "Finetune/Step Time": 4.679840033873916} +{"Finetune/Learning Rate": 1.1375484151677334e-05, "Finetune/Loss": 0.8984434604644775, "Finetune/Loss (Raw)": 1.0041297674179077, "Finetune/Step": 2455, "Finetune/Step Time": 4.666820937767625} +{"Finetune/Learning Rate": 1.1369313486692723e-05, "Finetune/Loss": 0.8992189168930054, "Finetune/Loss (Raw)": 0.9614733457565308, "Finetune/Step": 2456, "Finetune/Step Time": 4.656686130911112} +{"Finetune/Learning Rate": 1.136314229030428e-05, "Finetune/Loss": 0.8989790678024292, "Finetune/Loss (Raw)": 0.9590184688568115, "Finetune/Step": 2457, "Finetune/Step Time": 4.654087176546454} +{"Finetune/Learning Rate": 1.1356970564906928e-05, "Finetune/Loss": 0.9001734256744385, "Finetune/Loss (Raw)": 0.9682270884513855, "Finetune/Step": 2458, "Finetune/Step Time": 4.659989891573787} +{"Finetune/Learning Rate": 1.1350798312895795e-05, "Finetune/Loss": 0.899651050567627, "Finetune/Loss (Raw)": 0.8838804364204407, "Finetune/Step": 2459, "Finetune/Step Time": 4.6538384929299355} +{"Finetune/Learning Rate": 1.1344625536666208e-05, "Finetune/Loss": 0.8981992602348328, "Finetune/Loss (Raw)": 0.94287109375, "Finetune/Step": 2460, "Finetune/Step Time": 4.6479428466409445} +{"Finetune/Learning Rate": 1.13384522386137e-05, "Finetune/Loss": 0.8987658023834229, "Finetune/Loss (Raw)": 1.074710726737976, "Finetune/Step": 2461, "Finetune/Step Time": 4.639051949605346} +{"Finetune/Learning Rate": 1.1332278421134007e-05, "Finetune/Loss": 0.8988877534866333, "Finetune/Loss (Raw)": 0.9672863483428955, "Finetune/Step": 2462, "Finetune/Step Time": 4.614802440628409} +{"Finetune/Learning Rate": 1.1326104086623075e-05, "Finetune/Loss": 0.89939284324646, "Finetune/Loss (Raw)": 0.9449419975280762, "Finetune/Step": 2463, "Finetune/Step Time": 4.615140948444605} +{"Finetune/Learning Rate": 1.1319929237477033e-05, "Finetune/Loss": 0.9004364609718323, "Finetune/Loss (Raw)": 1.027758240699768, "Finetune/Step": 2464, "Finetune/Step Time": 4.600710766389966} +{"Finetune/Learning Rate": 1.1313753876092223e-05, "Finetune/Loss": 0.8999179005622864, "Finetune/Loss (Raw)": 0.9035497903823853, "Finetune/Step": 2465, "Finetune/Step Time": 4.6031789649277925} +{"Finetune/Learning Rate": 1.1307578004865188e-05, "Finetune/Loss": 0.898589015007019, "Finetune/Loss (Raw)": 0.8159813284873962, "Finetune/Step": 2466, "Finetune/Step Time": 4.6043082401156425} +{"Finetune/Learning Rate": 1.130140162619265e-05, "Finetune/Loss": 0.8990983963012695, "Finetune/Loss (Raw)": 0.9181631207466125, "Finetune/Step": 2467, "Finetune/Step Time": 4.589823953807354} +{"Finetune/Learning Rate": 1.1295224742471555e-05, "Finetune/Loss": 0.9006985425949097, "Finetune/Loss (Raw)": 1.1128813028335571, "Finetune/Step": 2468, "Finetune/Step Time": 4.600253250449896} +{"Finetune/Learning Rate": 1.128904735609902e-05, "Finetune/Loss": 0.9004219770431519, "Finetune/Loss (Raw)": 1.0109182596206665, "Finetune/Step": 2469, "Finetune/Step Time": 4.571120619773865} +{"Finetune/Learning Rate": 1.1282869469472375e-05, "Finetune/Loss": 0.9042283296585083, "Finetune/Loss (Raw)": 0.845130205154419, "Finetune/Step": 2470, "Finetune/Step Time": 4.5409855879843235} +{"Finetune/Learning Rate": 1.1276691084989135e-05, "Finetune/Loss": 0.9041290283203125, "Finetune/Loss (Raw)": 0.9904053211212158, "Finetune/Step": 2471, "Finetune/Step Time": 4.530111238360405} +{"Finetune/Learning Rate": 1.1270512205047008e-05, "Finetune/Loss": 0.904381513595581, "Finetune/Loss (Raw)": 0.9468168020248413, "Finetune/Step": 2472, "Finetune/Step Time": 4.504109621047974} +{"Finetune/Learning Rate": 1.1264332832043902e-05, "Finetune/Loss": 0.9027091264724731, "Finetune/Loss (Raw)": 0.9468477368354797, "Finetune/Step": 2473, "Finetune/Step Time": 4.510359948500991} +{"Finetune/Learning Rate": 1.1258152968377907e-05, "Finetune/Loss": 0.9028996229171753, "Finetune/Loss (Raw)": 0.9375198483467102, "Finetune/Step": 2474, "Finetune/Step Time": 4.514405246824026} +{"Finetune/Learning Rate": 1.125197261644731e-05, "Finetune/Loss": 0.9028115272521973, "Finetune/Loss (Raw)": 0.8936414122581482, "Finetune/Step": 2475, "Finetune/Step Time": 4.513998402282596} +{"Finetune/Learning Rate": 1.1245791778650578e-05, "Finetune/Loss": 0.9022390842437744, "Finetune/Loss (Raw)": 0.9409819841384888, "Finetune/Step": 2476, "Finetune/Step Time": 4.511803166940808} +{"Finetune/Learning Rate": 1.1239610457386381e-05, "Finetune/Loss": 0.9022682309150696, "Finetune/Loss (Raw)": 0.9390528202056885, "Finetune/Step": 2477, "Finetune/Step Time": 4.508705478161573} +{"Finetune/Learning Rate": 1.1233428655053572e-05, "Finetune/Loss": 0.9022848606109619, "Finetune/Loss (Raw)": 0.9189437627792358, "Finetune/Step": 2478, "Finetune/Step Time": 4.502310823649168} +{"Finetune/Learning Rate": 1.1227246374051175e-05, "Finetune/Loss": 0.9007102251052856, "Finetune/Loss (Raw)": 0.840319037437439, "Finetune/Step": 2479, "Finetune/Step Time": 4.50302323885262} +{"Finetune/Learning Rate": 1.1221063616778425e-05, "Finetune/Loss": 0.8988567590713501, "Finetune/Loss (Raw)": 0.8493726849555969, "Finetune/Step": 2480, "Finetune/Step Time": 4.501595539972186} +{"Finetune/Learning Rate": 1.1214880385634723e-05, "Finetune/Loss": 0.8946398496627808, "Finetune/Loss (Raw)": 0.37733322381973267, "Finetune/Step": 2481, "Finetune/Step Time": 4.537268543615937} +{"Finetune/Learning Rate": 1.120869668301966e-05, "Finetune/Loss": 0.8959650993347168, "Finetune/Loss (Raw)": 1.0979127883911133, "Finetune/Step": 2482, "Finetune/Step Time": 4.539911670610309} +{"Finetune/Learning Rate": 1.1202512511333017e-05, "Finetune/Loss": 0.8966451287269592, "Finetune/Loss (Raw)": 0.9148308038711548, "Finetune/Step": 2483, "Finetune/Step Time": 4.551260018721223} +{"Finetune/Learning Rate": 1.119632787297474e-05, "Finetune/Loss": 0.8972132205963135, "Finetune/Loss (Raw)": 0.8835259079933167, "Finetune/Step": 2484, "Finetune/Step Time": 4.55647274851799} +{"Finetune/Learning Rate": 1.1190142770344976e-05, "Finetune/Loss": 0.8974186182022095, "Finetune/Loss (Raw)": 0.9390607476234436, "Finetune/Step": 2485, "Finetune/Step Time": 4.55688969604671} +{"Finetune/Learning Rate": 1.1183957205844037e-05, "Finetune/Loss": 0.8957850933074951, "Finetune/Loss (Raw)": 0.8249996900558472, "Finetune/Step": 2486, "Finetune/Step Time": 4.5674595143646} +{"Finetune/Learning Rate": 1.1177771181872416e-05, "Finetune/Loss": 0.8959413766860962, "Finetune/Loss (Raw)": 0.3150911033153534, "Finetune/Step": 2487, "Finetune/Step Time": 4.565891092643142} +{"Finetune/Learning Rate": 1.1171584700830803e-05, "Finetune/Loss": 0.8968179821968079, "Finetune/Loss (Raw)": 1.0372984409332275, "Finetune/Step": 2488, "Finetune/Step Time": 4.5645100604742765} +{"Finetune/Learning Rate": 1.1165397765120035e-05, "Finetune/Loss": 0.8969722986221313, "Finetune/Loss (Raw)": 0.9686523675918579, "Finetune/Step": 2489, "Finetune/Step Time": 4.582776861265302} +{"Finetune/Learning Rate": 1.115921037714115e-05, "Finetune/Loss": 0.8920249938964844, "Finetune/Loss (Raw)": 0.3203828036785126, "Finetune/Step": 2490, "Finetune/Step Time": 4.62393631413579} +{"Finetune/Learning Rate": 1.1153022539295345e-05, "Finetune/Loss": 0.893385648727417, "Finetune/Loss (Raw)": 0.9012883901596069, "Finetune/Step": 2491, "Finetune/Step Time": 4.611244166269898} +{"Finetune/Learning Rate": 1.1146834253984008e-05, "Finetune/Loss": 0.8929626941680908, "Finetune/Loss (Raw)": 0.8016793727874756, "Finetune/Step": 2492, "Finetune/Step Time": 4.612854989245534} +{"Finetune/Learning Rate": 1.1140645523608684e-05, "Finetune/Loss": 0.8925707936286926, "Finetune/Loss (Raw)": 0.9867105484008789, "Finetune/Step": 2493, "Finetune/Step Time": 4.620866063982248} +{"Finetune/Learning Rate": 1.1134456350571099e-05, "Finetune/Loss": 0.894572913646698, "Finetune/Loss (Raw)": 1.0249027013778687, "Finetune/Step": 2494, "Finetune/Step Time": 4.615443505346775} +{"Finetune/Learning Rate": 1.112826673727316e-05, "Finetune/Loss": 0.8942962288856506, "Finetune/Loss (Raw)": 0.9420576691627502, "Finetune/Step": 2495, "Finetune/Step Time": 4.608545899391174} +{"Finetune/Learning Rate": 1.112207668611692e-05, "Finetune/Loss": 0.8943088054656982, "Finetune/Loss (Raw)": 0.9248108863830566, "Finetune/Step": 2496, "Finetune/Step Time": 4.6300945635885} +{"Finetune/Learning Rate": 1.1115886199504631e-05, "Finetune/Loss": 0.8927209377288818, "Finetune/Loss (Raw)": 0.8115007877349854, "Finetune/Step": 2497, "Finetune/Step Time": 4.628183672204614} +{"Finetune/Learning Rate": 1.110969527983869e-05, "Finetune/Loss": 0.8934804201126099, "Finetune/Loss (Raw)": 1.0133033990859985, "Finetune/Step": 2498, "Finetune/Step Time": 4.628448665142059} +{"Finetune/Learning Rate": 1.1103503929521677e-05, "Finetune/Loss": 0.8907092809677124, "Finetune/Loss (Raw)": 0.6693859100341797, "Finetune/Step": 2499, "Finetune/Step Time": 4.640894526615739} +{"Finetune/Learning Rate": 1.1097312150956336e-05, "Finetune/Loss": 0.8910622000694275, "Finetune/Loss (Raw)": 0.9509795308113098, "Finetune/Step": 2500, "Finetune/Step Time": 4.640997361391783} +{"Finetune/Learning Rate": 1.1091119946545568e-05, "Finetune/Loss": 0.8904322385787964, "Finetune/Loss (Raw)": 0.9399287104606628, "Finetune/Step": 2501, "Finetune/Step Time": 4.621183793991804} +{"Finetune/Learning Rate": 1.1084927318692453e-05, "Finetune/Loss": 0.8914190530776978, "Finetune/Loss (Raw)": 0.9493566155433655, "Finetune/Step": 2502, "Finetune/Step Time": 4.6215846091508865} +{"Finetune/Learning Rate": 1.1078734269800227e-05, "Finetune/Loss": 0.8923043012619019, "Finetune/Loss (Raw)": 0.8915257453918457, "Finetune/Step": 2503, "Finetune/Step Time": 4.610215075314045} +{"Finetune/Learning Rate": 1.1072540802272291e-05, "Finetune/Loss": 0.8918380737304688, "Finetune/Loss (Raw)": 0.9000813961029053, "Finetune/Step": 2504, "Finetune/Step Time": 4.614979172125459} +{"Finetune/Learning Rate": 1.1066346918512211e-05, "Finetune/Loss": 0.8905192613601685, "Finetune/Loss (Raw)": 0.9027853608131409, "Finetune/Step": 2505, "Finetune/Step Time": 4.614507148042321} +{"Finetune/Learning Rate": 1.1060152620923707e-05, "Finetune/Loss": 0.8893663883209229, "Finetune/Loss (Raw)": 0.8656076788902283, "Finetune/Step": 2506, "Finetune/Step Time": 4.618512695655227} +{"Finetune/Learning Rate": 1.105395791191067e-05, "Finetune/Loss": 0.8899372816085815, "Finetune/Loss (Raw)": 1.0149903297424316, "Finetune/Step": 2507, "Finetune/Step Time": 4.606018556281924} +{"Finetune/Learning Rate": 1.1047762793877146e-05, "Finetune/Loss": 0.8902875185012817, "Finetune/Loss (Raw)": 0.8468536138534546, "Finetune/Step": 2508, "Finetune/Step Time": 4.613817950710654} +{"Finetune/Learning Rate": 1.1041567269227336e-05, "Finetune/Loss": 0.889591634273529, "Finetune/Loss (Raw)": 0.8408983945846558, "Finetune/Step": 2509, "Finetune/Step Time": 4.6164416298270226} +{"Finetune/Learning Rate": 1.1035371340365606e-05, "Finetune/Loss": 0.8915739059448242, "Finetune/Loss (Raw)": 1.047452449798584, "Finetune/Step": 2510, "Finetune/Step Time": 4.599766248837113} +{"Finetune/Learning Rate": 1.1029175009696468e-05, "Finetune/Loss": 0.8938666582107544, "Finetune/Loss (Raw)": 0.9826671481132507, "Finetune/Step": 2511, "Finetune/Step Time": 4.596293235197663} +{"Finetune/Learning Rate": 1.1022978279624606e-05, "Finetune/Loss": 0.8916277289390564, "Finetune/Loss (Raw)": 0.813596785068512, "Finetune/Step": 2512, "Finetune/Step Time": 4.605585081502795} +{"Finetune/Learning Rate": 1.1016781152554846e-05, "Finetune/Loss": 0.8911696076393127, "Finetune/Loss (Raw)": 0.8564801812171936, "Finetune/Step": 2513, "Finetune/Step Time": 4.6047933250665665} +{"Finetune/Learning Rate": 1.1010583630892166e-05, "Finetune/Loss": 0.8910179734230042, "Finetune/Loss (Raw)": 1.0215638875961304, "Finetune/Step": 2514, "Finetune/Step Time": 4.611153790727258} +{"Finetune/Learning Rate": 1.1004385717041716e-05, "Finetune/Loss": 0.890164852142334, "Finetune/Loss (Raw)": 0.7714018821716309, "Finetune/Step": 2515, "Finetune/Step Time": 4.61636727117002} +{"Finetune/Learning Rate": 1.0998187413408774e-05, "Finetune/Loss": 0.8897214531898499, "Finetune/Loss (Raw)": 0.9330092072486877, "Finetune/Step": 2516, "Finetune/Step Time": 4.594124998897314} +{"Finetune/Learning Rate": 1.0991988722398786e-05, "Finetune/Loss": 0.8888971209526062, "Finetune/Loss (Raw)": 0.9381973147392273, "Finetune/Step": 2517, "Finetune/Step Time": 4.597684009000659} +{"Finetune/Learning Rate": 1.0985789646417341e-05, "Finetune/Loss": 0.8875992298126221, "Finetune/Loss (Raw)": 0.8690658211708069, "Finetune/Step": 2518, "Finetune/Step Time": 4.603466374799609} +{"Finetune/Learning Rate": 1.097959018787018e-05, "Finetune/Loss": 0.8867014646530151, "Finetune/Loss (Raw)": 0.8857936859130859, "Finetune/Step": 2519, "Finetune/Step Time": 4.608193738386035} +{"Finetune/Learning Rate": 1.0973390349163195e-05, "Finetune/Loss": 0.8871461153030396, "Finetune/Loss (Raw)": 0.9975759387016296, "Finetune/Step": 2520, "Finetune/Step Time": 4.607975943014026} +{"Finetune/Learning Rate": 1.0967190132702414e-05, "Finetune/Loss": 0.8822717070579529, "Finetune/Loss (Raw)": 0.40473589301109314, "Finetune/Step": 2521, "Finetune/Step Time": 4.6294872891157866} +{"Finetune/Learning Rate": 1.096098954089403e-05, "Finetune/Loss": 0.8865916728973389, "Finetune/Loss (Raw)": 0.9518805742263794, "Finetune/Step": 2522, "Finetune/Step Time": 4.609220456331968} +{"Finetune/Learning Rate": 1.0954788576144365e-05, "Finetune/Loss": 0.8847938776016235, "Finetune/Loss (Raw)": 0.7179054021835327, "Finetune/Step": 2523, "Finetune/Step Time": 4.6103877648711205} +{"Finetune/Learning Rate": 1.0948587240859893e-05, "Finetune/Loss": 0.8848345279693604, "Finetune/Loss (Raw)": 0.8836172223091125, "Finetune/Step": 2524, "Finetune/Step Time": 4.609464081004262} +{"Finetune/Learning Rate": 1.0942385537447236e-05, "Finetune/Loss": 0.8841702938079834, "Finetune/Loss (Raw)": 0.8544308543205261, "Finetune/Step": 2525, "Finetune/Step Time": 4.610995465889573} +{"Finetune/Learning Rate": 1.0936183468313147e-05, "Finetune/Loss": 0.8831142783164978, "Finetune/Loss (Raw)": 0.9491485357284546, "Finetune/Step": 2526, "Finetune/Step Time": 4.613477086648345} +{"Finetune/Learning Rate": 1.0929981035864532e-05, "Finetune/Loss": 0.8830999732017517, "Finetune/Loss (Raw)": 0.9962842464447021, "Finetune/Step": 2527, "Finetune/Step Time": 4.618178112432361} +{"Finetune/Learning Rate": 1.0923778242508437e-05, "Finetune/Loss": 0.8844969868659973, "Finetune/Loss (Raw)": 1.0005484819412231, "Finetune/Step": 2528, "Finetune/Step Time": 4.624228363856673} +{"Finetune/Learning Rate": 1.0917575090652038e-05, "Finetune/Loss": 0.8791627883911133, "Finetune/Loss (Raw)": 0.2708369493484497, "Finetune/Step": 2529, "Finetune/Step Time": 4.660295249894261} +{"Finetune/Learning Rate": 1.091137158270266e-05, "Finetune/Loss": 0.8799791932106018, "Finetune/Loss (Raw)": 0.9305837154388428, "Finetune/Step": 2530, "Finetune/Step Time": 4.646942323073745} +{"Finetune/Learning Rate": 1.0905167721067762e-05, "Finetune/Loss": 0.8792167901992798, "Finetune/Loss (Raw)": 0.838939368724823, "Finetune/Step": 2531, "Finetune/Step Time": 4.645881025120616} +{"Finetune/Learning Rate": 1.0898963508154947e-05, "Finetune/Loss": 0.8795099854469299, "Finetune/Loss (Raw)": 0.9924932718276978, "Finetune/Step": 2532, "Finetune/Step Time": 4.662579454481602} +{"Finetune/Learning Rate": 1.0892758946371943e-05, "Finetune/Loss": 0.8791784048080444, "Finetune/Loss (Raw)": 0.9017062783241272, "Finetune/Step": 2533, "Finetune/Step Time": 4.680666401982307} +{"Finetune/Learning Rate": 1.0886554038126624e-05, "Finetune/Loss": 0.8742902874946594, "Finetune/Loss (Raw)": 0.37791359424591064, "Finetune/Step": 2534, "Finetune/Step Time": 4.717229217290878} +{"Finetune/Learning Rate": 1.0880348785826991e-05, "Finetune/Loss": 0.8768210411071777, "Finetune/Loss (Raw)": 1.0221900939941406, "Finetune/Step": 2535, "Finetune/Step Time": 4.701376898214221} +{"Finetune/Learning Rate": 1.087414319188118e-05, "Finetune/Loss": 0.8737900853157043, "Finetune/Loss (Raw)": 0.7345404028892517, "Finetune/Step": 2536, "Finetune/Step Time": 4.714442366734147} +{"Finetune/Learning Rate": 1.0867937258697463e-05, "Finetune/Loss": 0.8738595843315125, "Finetune/Loss (Raw)": 0.9053524136543274, "Finetune/Step": 2537, "Finetune/Step Time": 4.71618084050715} +{"Finetune/Learning Rate": 1.086173098868424e-05, "Finetune/Loss": 0.8744131326675415, "Finetune/Loss (Raw)": 1.0002416372299194, "Finetune/Step": 2538, "Finetune/Step Time": 4.725101528689265} +{"Finetune/Learning Rate": 1.0855524384250048e-05, "Finetune/Loss": 0.8753390908241272, "Finetune/Loss (Raw)": 0.886635422706604, "Finetune/Step": 2539, "Finetune/Step Time": 4.723383953794837} +{"Finetune/Learning Rate": 1.0849317447803542e-05, "Finetune/Loss": 0.8762447834014893, "Finetune/Loss (Raw)": 1.0100001096725464, "Finetune/Step": 2540, "Finetune/Step Time": 4.723513362929225} +{"Finetune/Learning Rate": 1.0843110181753513e-05, "Finetune/Loss": 0.876431405544281, "Finetune/Loss (Raw)": 0.8429540395736694, "Finetune/Step": 2541, "Finetune/Step Time": 4.74033004231751} +{"Finetune/Learning Rate": 1.0836902588508888e-05, "Finetune/Loss": 0.8761403560638428, "Finetune/Loss (Raw)": 0.9206604957580566, "Finetune/Step": 2542, "Finetune/Step Time": 4.744796747341752} +{"Finetune/Learning Rate": 1.0830694670478706e-05, "Finetune/Loss": 0.876475989818573, "Finetune/Loss (Raw)": 0.9345782399177551, "Finetune/Step": 2543, "Finetune/Step Time": 4.739611353725195} +{"Finetune/Learning Rate": 1.082448643007214e-05, "Finetune/Loss": 0.8747351765632629, "Finetune/Loss (Raw)": 0.7356985807418823, "Finetune/Step": 2544, "Finetune/Step Time": 4.749657826498151} +{"Finetune/Learning Rate": 1.0818277869698488e-05, "Finetune/Loss": 0.8744173645973206, "Finetune/Loss (Raw)": 0.8761306405067444, "Finetune/Step": 2545, "Finetune/Step Time": 4.748310351744294} +{"Finetune/Learning Rate": 1.0812068991767171e-05, "Finetune/Loss": 0.8731290102005005, "Finetune/Loss (Raw)": 0.27909108996391296, "Finetune/Step": 2546, "Finetune/Step Time": 4.7483047023415565} +{"Finetune/Learning Rate": 1.0805859798687727e-05, "Finetune/Loss": 0.8728792071342468, "Finetune/Loss (Raw)": 0.8756334185600281, "Finetune/Step": 2547, "Finetune/Step Time": 4.719793163239956} +{"Finetune/Learning Rate": 1.0799650292869835e-05, "Finetune/Loss": 0.872573733329773, "Finetune/Loss (Raw)": 1.0393462181091309, "Finetune/Step": 2548, "Finetune/Step Time": 4.716444322839379} +{"Finetune/Learning Rate": 1.0793440476723274e-05, "Finetune/Loss": 0.869047224521637, "Finetune/Loss (Raw)": 0.3254818618297577, "Finetune/Step": 2549, "Finetune/Step Time": 4.739685285836458} +{"Finetune/Learning Rate": 1.0787230352657954e-05, "Finetune/Loss": 0.870164692401886, "Finetune/Loss (Raw)": 0.93482506275177, "Finetune/Step": 2550, "Finetune/Step Time": 4.72374071739614} +{"Finetune/Learning Rate": 1.0781019923083905e-05, "Finetune/Loss": 0.8718457221984863, "Finetune/Loss (Raw)": 0.9571157693862915, "Finetune/Step": 2551, "Finetune/Step Time": 4.7196211740374565} +{"Finetune/Learning Rate": 1.077480919041127e-05, "Finetune/Loss": 0.8730541467666626, "Finetune/Loss (Raw)": 1.0950853824615479, "Finetune/Step": 2552, "Finetune/Step Time": 4.720357179641724} +{"Finetune/Learning Rate": 1.0768598157050313e-05, "Finetune/Loss": 0.8732025623321533, "Finetune/Loss (Raw)": 0.8646523952484131, "Finetune/Step": 2553, "Finetune/Step Time": 4.721840528771281} +{"Finetune/Learning Rate": 1.0762386825411422e-05, "Finetune/Loss": 0.8777555227279663, "Finetune/Loss (Raw)": 0.9566171765327454, "Finetune/Step": 2554, "Finetune/Step Time": 4.712826631963253} +{"Finetune/Learning Rate": 1.0756175197905086e-05, "Finetune/Loss": 0.8727308511734009, "Finetune/Loss (Raw)": 0.3300953805446625, "Finetune/Step": 2555, "Finetune/Step Time": 4.749100169166923} +{"Finetune/Learning Rate": 1.0749963276941919e-05, "Finetune/Loss": 0.8735299706459045, "Finetune/Loss (Raw)": 0.9058562517166138, "Finetune/Step": 2556, "Finetune/Step Time": 4.7492290660738945} +{"Finetune/Learning Rate": 1.0743751064932648e-05, "Finetune/Loss": 0.8718616366386414, "Finetune/Loss (Raw)": 0.7647616863250732, "Finetune/Step": 2557, "Finetune/Step Time": 4.731987630948424} +{"Finetune/Learning Rate": 1.0737538564288104e-05, "Finetune/Loss": 0.8727458715438843, "Finetune/Loss (Raw)": 0.8726872801780701, "Finetune/Step": 2558, "Finetune/Step Time": 4.701675755903125} +{"Finetune/Learning Rate": 1.0731325777419246e-05, "Finetune/Loss": 0.8718605041503906, "Finetune/Loss (Raw)": 0.9040126204490662, "Finetune/Step": 2559, "Finetune/Step Time": 4.700521914288402} +{"Finetune/Learning Rate": 1.0725112706737133e-05, "Finetune/Loss": 0.8716081380844116, "Finetune/Loss (Raw)": 0.8910978436470032, "Finetune/Step": 2560, "Finetune/Step Time": 4.7089441902935505} +{"Finetune/Learning Rate": 1.0718899354652931e-05, "Finetune/Loss": 0.8717802166938782, "Finetune/Loss (Raw)": 0.8975622057914734, "Finetune/Step": 2561, "Finetune/Step Time": 4.712362648919225} +{"Finetune/Learning Rate": 1.071268572357793e-05, "Finetune/Loss": 0.8724905252456665, "Finetune/Loss (Raw)": 1.0190035104751587, "Finetune/Step": 2562, "Finetune/Step Time": 4.711694283410907} +{"Finetune/Learning Rate": 1.0706471815923511e-05, "Finetune/Loss": 0.8706644773483276, "Finetune/Loss (Raw)": 0.6540284156799316, "Finetune/Step": 2563, "Finetune/Step Time": 4.709577102214098} +{"Finetune/Learning Rate": 1.070025763410117e-05, "Finetune/Loss": 0.8702492713928223, "Finetune/Loss (Raw)": 0.9284864068031311, "Finetune/Step": 2564, "Finetune/Step Time": 4.712155034765601} +{"Finetune/Learning Rate": 1.0694043180522515e-05, "Finetune/Loss": 0.871010422706604, "Finetune/Loss (Raw)": 0.978263795375824, "Finetune/Step": 2565, "Finetune/Step Time": 4.709667630493641} +{"Finetune/Learning Rate": 1.0687828457599254e-05, "Finetune/Loss": 0.8703200817108154, "Finetune/Loss (Raw)": 0.9294338226318359, "Finetune/Step": 2566, "Finetune/Step Time": 4.712614722549915} +{"Finetune/Learning Rate": 1.0681613467743195e-05, "Finetune/Loss": 0.8688204884529114, "Finetune/Loss (Raw)": 0.8073956370353699, "Finetune/Step": 2567, "Finetune/Step Time": 4.7173977587372065} +{"Finetune/Learning Rate": 1.0675398213366257e-05, "Finetune/Loss": 0.8672695159912109, "Finetune/Loss (Raw)": 0.8020376563072205, "Finetune/Step": 2568, "Finetune/Step Time": 4.720798531547189} +{"Finetune/Learning Rate": 1.0669182696880459e-05, "Finetune/Loss": 0.8685492277145386, "Finetune/Loss (Raw)": 1.1305714845657349, "Finetune/Step": 2569, "Finetune/Step Time": 4.71817546710372} +{"Finetune/Learning Rate": 1.0662966920697918e-05, "Finetune/Loss": 0.8685083389282227, "Finetune/Loss (Raw)": 0.9212256669998169, "Finetune/Step": 2570, "Finetune/Step Time": 4.722458548843861} +{"Finetune/Learning Rate": 1.0656750887230862e-05, "Finetune/Loss": 0.869141161441803, "Finetune/Loss (Raw)": 1.0616856813430786, "Finetune/Step": 2571, "Finetune/Step Time": 4.720813881605864} +{"Finetune/Learning Rate": 1.0650534598891608e-05, "Finetune/Loss": 0.8684180974960327, "Finetune/Loss (Raw)": 0.8819675445556641, "Finetune/Step": 2572, "Finetune/Step Time": 4.717755740508437} +{"Finetune/Learning Rate": 1.0644318058092577e-05, "Finetune/Loss": 0.8724606037139893, "Finetune/Loss (Raw)": 0.9030469655990601, "Finetune/Step": 2573, "Finetune/Step Time": 4.687221363186836} +{"Finetune/Learning Rate": 1.0638101267246283e-05, "Finetune/Loss": 0.872161865234375, "Finetune/Loss (Raw)": 0.8943619728088379, "Finetune/Step": 2574, "Finetune/Step Time": 4.709883272647858} +{"Finetune/Learning Rate": 1.063188422876535e-05, "Finetune/Loss": 0.8715240955352783, "Finetune/Loss (Raw)": 0.8295062184333801, "Finetune/Step": 2575, "Finetune/Step Time": 4.701800929382443} +{"Finetune/Learning Rate": 1.0625666945062484e-05, "Finetune/Loss": 0.8710947632789612, "Finetune/Loss (Raw)": 0.9483252763748169, "Finetune/Step": 2576, "Finetune/Step Time": 4.702848309651017} +{"Finetune/Learning Rate": 1.061944941855049e-05, "Finetune/Loss": 0.8723669052124023, "Finetune/Loss (Raw)": 0.979762852191925, "Finetune/Step": 2577, "Finetune/Step Time": 4.707551803439856} +{"Finetune/Learning Rate": 1.061323165164227e-05, "Finetune/Loss": 0.8765981197357178, "Finetune/Loss (Raw)": 1.0302454233169556, "Finetune/Step": 2578, "Finetune/Step Time": 4.670197254046798} +{"Finetune/Learning Rate": 1.0607013646750818e-05, "Finetune/Loss": 0.8820472955703735, "Finetune/Loss (Raw)": 1.018660068511963, "Finetune/Step": 2579, "Finetune/Step Time": 4.631881991401315} +{"Finetune/Learning Rate": 1.0600795406289221e-05, "Finetune/Loss": 0.8800396919250488, "Finetune/Loss (Raw)": 0.7087668180465698, "Finetune/Step": 2580, "Finetune/Step Time": 4.642407169565558} +{"Finetune/Learning Rate": 1.059457693267066e-05, "Finetune/Loss": 0.8809386491775513, "Finetune/Loss (Raw)": 1.050439476966858, "Finetune/Step": 2581, "Finetune/Step Time": 4.619283908978105} +{"Finetune/Learning Rate": 1.0588358228308395e-05, "Finetune/Loss": 0.8805690407752991, "Finetune/Loss (Raw)": 0.8823311924934387, "Finetune/Step": 2582, "Finetune/Step Time": 4.609105139970779} +{"Finetune/Learning Rate": 1.0582139295615793e-05, "Finetune/Loss": 0.8791435956954956, "Finetune/Loss (Raw)": 0.8216742873191833, "Finetune/Step": 2583, "Finetune/Step Time": 4.618024319410324} +{"Finetune/Learning Rate": 1.0575920137006288e-05, "Finetune/Loss": 0.8798097372055054, "Finetune/Loss (Raw)": 1.0467356443405151, "Finetune/Step": 2584, "Finetune/Step Time": 4.61861296184361} +{"Finetune/Learning Rate": 1.056970075489343e-05, "Finetune/Loss": 0.8786787986755371, "Finetune/Loss (Raw)": 0.8142672181129456, "Finetune/Step": 2585, "Finetune/Step Time": 4.62049113959074} +{"Finetune/Learning Rate": 1.0563481151690827e-05, "Finetune/Loss": 0.8787332773208618, "Finetune/Loss (Raw)": 0.9751999974250793, "Finetune/Step": 2586, "Finetune/Step Time": 4.606763832271099} +{"Finetune/Learning Rate": 1.055726132981219e-05, "Finetune/Loss": 0.8792466521263123, "Finetune/Loss (Raw)": 0.9495880007743835, "Finetune/Step": 2587, "Finetune/Step Time": 4.607120705768466} +{"Finetune/Learning Rate": 1.0551041291671311e-05, "Finetune/Loss": 0.8785871863365173, "Finetune/Loss (Raw)": 0.85845947265625, "Finetune/Step": 2588, "Finetune/Step Time": 4.615728102624416} +{"Finetune/Learning Rate": 1.0544821039682066e-05, "Finetune/Loss": 0.8779991865158081, "Finetune/Loss (Raw)": 0.9994529485702515, "Finetune/Step": 2589, "Finetune/Step Time": 4.620092457160354} +{"Finetune/Learning Rate": 1.0538600576258413e-05, "Finetune/Loss": 0.8777240514755249, "Finetune/Loss (Raw)": 0.9320623278617859, "Finetune/Step": 2590, "Finetune/Step Time": 4.622983993962407} +{"Finetune/Learning Rate": 1.0532379903814387e-05, "Finetune/Loss": 0.8767437934875488, "Finetune/Loss (Raw)": 0.8194653987884521, "Finetune/Step": 2591, "Finetune/Step Time": 4.661737056449056} +{"Finetune/Learning Rate": 1.0526159024764118e-05, "Finetune/Loss": 0.8758461475372314, "Finetune/Loss (Raw)": 0.9128574132919312, "Finetune/Step": 2592, "Finetune/Step Time": 4.6700820326805115} +{"Finetune/Learning Rate": 1.0519937941521805e-05, "Finetune/Loss": 0.8753900527954102, "Finetune/Loss (Raw)": 0.8451830744743347, "Finetune/Step": 2593, "Finetune/Step Time": 4.690169487148523} +{"Finetune/Learning Rate": 1.0513716656501723e-05, "Finetune/Loss": 0.8755496740341187, "Finetune/Loss (Raw)": 0.8364121913909912, "Finetune/Step": 2594, "Finetune/Step Time": 4.701370995491743} +{"Finetune/Learning Rate": 1.0507495172118242e-05, "Finetune/Loss": 0.8753466606140137, "Finetune/Loss (Raw)": 0.892169177532196, "Finetune/Step": 2595, "Finetune/Step Time": 4.700455151498318} +{"Finetune/Learning Rate": 1.0501273490785789e-05, "Finetune/Loss": 0.8734675049781799, "Finetune/Loss (Raw)": 0.8723505735397339, "Finetune/Step": 2596, "Finetune/Step Time": 4.687094988301396} +{"Finetune/Learning Rate": 1.049505161491888e-05, "Finetune/Loss": 0.8731001615524292, "Finetune/Loss (Raw)": 0.9638931155204773, "Finetune/Step": 2597, "Finetune/Step Time": 4.690418317914009} +{"Finetune/Learning Rate": 1.0488829546932106e-05, "Finetune/Loss": 0.8722800016403198, "Finetune/Loss (Raw)": 0.7401545643806458, "Finetune/Step": 2598, "Finetune/Step Time": 4.696988712996244} +{"Finetune/Learning Rate": 1.0482607289240128e-05, "Finetune/Loss": 0.8720327615737915, "Finetune/Loss (Raw)": 0.9587628245353699, "Finetune/Step": 2599, "Finetune/Step Time": 4.700883936136961} +{"Finetune/Learning Rate": 1.0476384844257686e-05, "Finetune/Loss": 0.8715720176696777, "Finetune/Loss (Raw)": 0.887832760810852, "Finetune/Step": 2600, "Finetune/Step Time": 4.705338461324573} +{"Finetune/Learning Rate": 1.0470162214399582e-05, "Finetune/Loss": 0.8713430166244507, "Finetune/Loss (Raw)": 0.917545735836029, "Finetune/Step": 2601, "Finetune/Step Time": 4.71755138784647} +{"Finetune/Learning Rate": 1.0463939402080706e-05, "Finetune/Loss": 0.8720384240150452, "Finetune/Loss (Raw)": 1.0265274047851562, "Finetune/Step": 2602, "Finetune/Step Time": 4.715316584333777} +{"Finetune/Learning Rate": 1.0457716409716002e-05, "Finetune/Loss": 0.8727864027023315, "Finetune/Loss (Raw)": 0.9893853068351746, "Finetune/Step": 2603, "Finetune/Step Time": 4.714617636054754} +{"Finetune/Learning Rate": 1.0451493239720496e-05, "Finetune/Loss": 0.8723086714744568, "Finetune/Loss (Raw)": 0.8798304796218872, "Finetune/Step": 2604, "Finetune/Step Time": 4.731683110818267} +{"Finetune/Learning Rate": 1.0445269894509283e-05, "Finetune/Loss": 0.8711390495300293, "Finetune/Loss (Raw)": 0.7893356680870056, "Finetune/Step": 2605, "Finetune/Step Time": 4.730158930644393} +{"Finetune/Learning Rate": 1.043904637649751e-05, "Finetune/Loss": 0.8705427646636963, "Finetune/Loss (Raw)": 0.842621386051178, "Finetune/Step": 2606, "Finetune/Step Time": 4.759498730301857} +{"Finetune/Learning Rate": 1.0432822688100414e-05, "Finetune/Loss": 0.8714730739593506, "Finetune/Loss (Raw)": 0.9593984484672546, "Finetune/Step": 2607, "Finetune/Step Time": 4.779812356457114} +{"Finetune/Learning Rate": 1.0426598831733283e-05, "Finetune/Loss": 0.8723194599151611, "Finetune/Loss (Raw)": 0.9577149152755737, "Finetune/Step": 2608, "Finetune/Step Time": 4.7724054250866175} +{"Finetune/Learning Rate": 1.0420374809811472e-05, "Finetune/Loss": 0.8766636252403259, "Finetune/Loss (Raw)": 0.9333865642547607, "Finetune/Step": 2609, "Finetune/Step Time": 4.730814877897501} +{"Finetune/Learning Rate": 1.0414150624750408e-05, "Finetune/Loss": 0.8749424815177917, "Finetune/Loss (Raw)": 0.8776038289070129, "Finetune/Step": 2610, "Finetune/Step Time": 4.741920495405793} +{"Finetune/Learning Rate": 1.0407926278965568e-05, "Finetune/Loss": 0.8733792304992676, "Finetune/Loss (Raw)": 0.7147395610809326, "Finetune/Step": 2611, "Finetune/Step Time": 4.742268394678831} +{"Finetune/Learning Rate": 1.0401701774872508e-05, "Finetune/Loss": 0.8734169006347656, "Finetune/Loss (Raw)": 0.8883389234542847, "Finetune/Step": 2612, "Finetune/Step Time": 4.7304052375257015} +{"Finetune/Learning Rate": 1.0395477114886827e-05, "Finetune/Loss": 0.8740537166595459, "Finetune/Loss (Raw)": 1.0205708742141724, "Finetune/Step": 2613, "Finetune/Step Time": 4.7291696183383465} +{"Finetune/Learning Rate": 1.0389252301424197e-05, "Finetune/Loss": 0.8742192387580872, "Finetune/Loss (Raw)": 0.8461924195289612, "Finetune/Step": 2614, "Finetune/Step Time": 4.72157920524478} +{"Finetune/Learning Rate": 1.0383027336900356e-05, "Finetune/Loss": 0.8793210387229919, "Finetune/Loss (Raw)": 0.9681243300437927, "Finetune/Step": 2615, "Finetune/Step Time": 4.693985616788268} +{"Finetune/Learning Rate": 1.0376802223731077e-05, "Finetune/Loss": 0.8786770105361938, "Finetune/Loss (Raw)": 0.9548643827438354, "Finetune/Step": 2616, "Finetune/Step Time": 4.697182962670922} +{"Finetune/Learning Rate": 1.0370576964332217e-05, "Finetune/Loss": 0.8742066621780396, "Finetune/Loss (Raw)": 0.396442711353302, "Finetune/Step": 2617, "Finetune/Step Time": 4.711180724203587} +{"Finetune/Learning Rate": 1.0364351561119667e-05, "Finetune/Loss": 0.8779196739196777, "Finetune/Loss (Raw)": 0.7956507205963135, "Finetune/Step": 2618, "Finetune/Step Time": 4.685355149209499} +{"Finetune/Learning Rate": 1.0358126016509392e-05, "Finetune/Loss": 0.8784162402153015, "Finetune/Loss (Raw)": 0.9648485779762268, "Finetune/Step": 2619, "Finetune/Step Time": 4.679100267589092} +{"Finetune/Learning Rate": 1.0351900332917401e-05, "Finetune/Loss": 0.878691554069519, "Finetune/Loss (Raw)": 0.836915135383606, "Finetune/Step": 2620, "Finetune/Step Time": 4.674956385046244} +{"Finetune/Learning Rate": 1.034567451275976e-05, "Finetune/Loss": 0.8779200315475464, "Finetune/Loss (Raw)": 0.8879583477973938, "Finetune/Step": 2621, "Finetune/Step Time": 4.671654129400849} +{"Finetune/Learning Rate": 1.0339448558452594e-05, "Finetune/Loss": 0.8772940635681152, "Finetune/Loss (Raw)": 0.9447794556617737, "Finetune/Step": 2622, "Finetune/Step Time": 4.6664019376039505} +{"Finetune/Learning Rate": 1.0333222472412064e-05, "Finetune/Loss": 0.8770154714584351, "Finetune/Loss (Raw)": 0.9063935279846191, "Finetune/Step": 2623, "Finetune/Step Time": 4.686798175796866} +{"Finetune/Learning Rate": 1.03269962570544e-05, "Finetune/Loss": 0.876907229423523, "Finetune/Loss (Raw)": 0.9109529256820679, "Finetune/Step": 2624, "Finetune/Step Time": 4.675252532586455} +{"Finetune/Learning Rate": 1.0320769914795874e-05, "Finetune/Loss": 0.8765987157821655, "Finetune/Loss (Raw)": 0.7720127701759338, "Finetune/Step": 2625, "Finetune/Step Time": 4.689362671226263} +{"Finetune/Learning Rate": 1.0314543448052806e-05, "Finetune/Loss": 0.8711130619049072, "Finetune/Loss (Raw)": 0.31114161014556885, "Finetune/Step": 2626, "Finetune/Step Time": 4.727503197267652} +{"Finetune/Learning Rate": 1.0308316859241567e-05, "Finetune/Loss": 0.8727928400039673, "Finetune/Loss (Raw)": 0.8843994140625, "Finetune/Step": 2627, "Finetune/Step Time": 4.7102180905640125} +{"Finetune/Learning Rate": 1.0302090150778572e-05, "Finetune/Loss": 0.8731015920639038, "Finetune/Loss (Raw)": 0.9904978275299072, "Finetune/Step": 2628, "Finetune/Step Time": 4.709376055747271} +{"Finetune/Learning Rate": 1.029586332508029e-05, "Finetune/Loss": 0.8708709478378296, "Finetune/Loss (Raw)": 0.6544065475463867, "Finetune/Step": 2629, "Finetune/Step Time": 4.725099332630634} +{"Finetune/Learning Rate": 1.0289636384563226e-05, "Finetune/Loss": 0.8697882890701294, "Finetune/Loss (Raw)": 0.8107778429985046, "Finetune/Step": 2630, "Finetune/Step Time": 4.725119927898049} +{"Finetune/Learning Rate": 1.0283409331643936e-05, "Finetune/Loss": 0.8709324598312378, "Finetune/Loss (Raw)": 1.0379811525344849, "Finetune/Step": 2631, "Finetune/Step Time": 4.7222754415124655} +{"Finetune/Learning Rate": 1.0277182168739019e-05, "Finetune/Loss": 0.8702049255371094, "Finetune/Loss (Raw)": 0.806956946849823, "Finetune/Step": 2632, "Finetune/Step Time": 4.7175616435706615} +{"Finetune/Learning Rate": 1.0270954898265113e-05, "Finetune/Loss": 0.8706158399581909, "Finetune/Loss (Raw)": 0.955385684967041, "Finetune/Step": 2633, "Finetune/Step Time": 4.7195106111466885} +{"Finetune/Learning Rate": 1.0264727522638899e-05, "Finetune/Loss": 0.8713565468788147, "Finetune/Loss (Raw)": 0.9604155421257019, "Finetune/Step": 2634, "Finetune/Step Time": 4.720308471471071} +{"Finetune/Learning Rate": 1.0258500044277106e-05, "Finetune/Loss": 0.870672345161438, "Finetune/Loss (Raw)": 0.927420973777771, "Finetune/Step": 2635, "Finetune/Step Time": 4.714442878961563} +{"Finetune/Learning Rate": 1.025227246559649e-05, "Finetune/Loss": 0.8719533681869507, "Finetune/Loss (Raw)": 1.010813593864441, "Finetune/Step": 2636, "Finetune/Step Time": 4.713728668168187} +{"Finetune/Learning Rate": 1.0246044789013855e-05, "Finetune/Loss": 0.8718855381011963, "Finetune/Loss (Raw)": 0.8322189450263977, "Finetune/Step": 2637, "Finetune/Step Time": 4.7188626658171415} +{"Finetune/Learning Rate": 1.023981701694604e-05, "Finetune/Loss": 0.8713846206665039, "Finetune/Loss (Raw)": 0.9833299517631531, "Finetune/Step": 2638, "Finetune/Step Time": 4.739179676398635} +{"Finetune/Learning Rate": 1.0233589151809927e-05, "Finetune/Loss": 0.8701858520507812, "Finetune/Loss (Raw)": 0.8292248845100403, "Finetune/Step": 2639, "Finetune/Step Time": 4.729833906516433} +{"Finetune/Learning Rate": 1.0227361196022421e-05, "Finetune/Loss": 0.8665536642074585, "Finetune/Loss (Raw)": 0.3486783802509308, "Finetune/Step": 2640, "Finetune/Step Time": 4.758483098819852} +{"Finetune/Learning Rate": 1.0221133152000474e-05, "Finetune/Loss": 0.8669772744178772, "Finetune/Loss (Raw)": 0.9107071161270142, "Finetune/Step": 2641, "Finetune/Step Time": 4.7498580403625965} +{"Finetune/Learning Rate": 1.021490502216107e-05, "Finetune/Loss": 0.8660343885421753, "Finetune/Loss (Raw)": 0.9008768796920776, "Finetune/Step": 2642, "Finetune/Step Time": 4.7392285615205765} +{"Finetune/Learning Rate": 1.020867680892122e-05, "Finetune/Loss": 0.865756094455719, "Finetune/Loss (Raw)": 0.7357836365699768, "Finetune/Step": 2643, "Finetune/Step Time": 4.736409084871411} +{"Finetune/Learning Rate": 1.0202448514697975e-05, "Finetune/Loss": 0.8645349740982056, "Finetune/Loss (Raw)": 0.7766954898834229, "Finetune/Step": 2644, "Finetune/Step Time": 4.7697737365961075} +{"Finetune/Learning Rate": 1.0196220141908417e-05, "Finetune/Loss": 0.8646039962768555, "Finetune/Loss (Raw)": 0.9470350742340088, "Finetune/Step": 2645, "Finetune/Step Time": 4.772869562730193} +{"Finetune/Learning Rate": 1.018999169296965e-05, "Finetune/Loss": 0.8652166128158569, "Finetune/Loss (Raw)": 0.9474773406982422, "Finetune/Step": 2646, "Finetune/Step Time": 4.759065195918083} +{"Finetune/Learning Rate": 1.0183763170298818e-05, "Finetune/Loss": 0.8647252917289734, "Finetune/Loss (Raw)": 0.8229088187217712, "Finetune/Step": 2647, "Finetune/Step Time": 4.764897460117936} +{"Finetune/Learning Rate": 1.0177534576313084e-05, "Finetune/Loss": 0.8644857406616211, "Finetune/Loss (Raw)": 0.9669102430343628, "Finetune/Step": 2648, "Finetune/Step Time": 4.7756585497409105} +{"Finetune/Learning Rate": 1.0171305913429648e-05, "Finetune/Loss": 0.8680292367935181, "Finetune/Loss (Raw)": 0.8583061695098877, "Finetune/Step": 2649, "Finetune/Step Time": 4.751274766400456} +{"Finetune/Learning Rate": 1.0165077184065732e-05, "Finetune/Loss": 0.8669146299362183, "Finetune/Loss (Raw)": 0.8092113137245178, "Finetune/Step": 2650, "Finetune/Step Time": 4.7343238312751055} +{"Finetune/Learning Rate": 1.0158848390638588e-05, "Finetune/Loss": 0.8679678440093994, "Finetune/Loss (Raw)": 0.8527135848999023, "Finetune/Step": 2651, "Finetune/Step Time": 4.729888446629047} +{"Finetune/Learning Rate": 1.015261953556548e-05, "Finetune/Loss": 0.8690555095672607, "Finetune/Loss (Raw)": 1.0228415727615356, "Finetune/Step": 2652, "Finetune/Step Time": 4.736458430066705} +{"Finetune/Learning Rate": 1.0146390621263709e-05, "Finetune/Loss": 0.8693616986274719, "Finetune/Loss (Raw)": 0.893622100353241, "Finetune/Step": 2653, "Finetune/Step Time": 4.748107887804508} +{"Finetune/Learning Rate": 1.0140161650150598e-05, "Finetune/Loss": 0.8646243810653687, "Finetune/Loss (Raw)": 0.34276828169822693, "Finetune/Step": 2654, "Finetune/Step Time": 4.778255665674806} +{"Finetune/Learning Rate": 1.0133932624643487e-05, "Finetune/Loss": 0.8641101717948914, "Finetune/Loss (Raw)": 0.9304670095443726, "Finetune/Step": 2655, "Finetune/Step Time": 4.776852622628212} +{"Finetune/Learning Rate": 1.012770354715974e-05, "Finetune/Loss": 0.8633424639701843, "Finetune/Loss (Raw)": 0.902280867099762, "Finetune/Step": 2656, "Finetune/Step Time": 4.77366565912962} +{"Finetune/Learning Rate": 1.012147442011674e-05, "Finetune/Loss": 0.8668744564056396, "Finetune/Loss (Raw)": 0.7229355573654175, "Finetune/Step": 2657, "Finetune/Step Time": 4.75091758556664} +{"Finetune/Learning Rate": 1.0115245245931883e-05, "Finetune/Loss": 0.865184485912323, "Finetune/Loss (Raw)": 0.714265763759613, "Finetune/Step": 2658, "Finetune/Step Time": 4.769716648384929} +{"Finetune/Learning Rate": 1.0109016027022602e-05, "Finetune/Loss": 0.8664931654930115, "Finetune/Loss (Raw)": 1.0064562559127808, "Finetune/Step": 2659, "Finetune/Step Time": 4.760524535551667} +{"Finetune/Learning Rate": 1.0102786765806329e-05, "Finetune/Loss": 0.8660560846328735, "Finetune/Loss (Raw)": 0.9365431070327759, "Finetune/Step": 2660, "Finetune/Step Time": 4.74435905739665} +{"Finetune/Learning Rate": 1.0096557464700518e-05, "Finetune/Loss": 0.8676440715789795, "Finetune/Loss (Raw)": 1.1049671173095703, "Finetune/Step": 2661, "Finetune/Step Time": 4.732023509219289} +{"Finetune/Learning Rate": 1.0090328126122642e-05, "Finetune/Loss": 0.8726043105125427, "Finetune/Loss (Raw)": 1.012824535369873, "Finetune/Step": 2662, "Finetune/Step Time": 4.691587282344699} +{"Finetune/Learning Rate": 1.0084098752490179e-05, "Finetune/Loss": 0.8723200559616089, "Finetune/Loss (Raw)": 0.9858022332191467, "Finetune/Step": 2663, "Finetune/Step Time": 4.7035616263747215} +{"Finetune/Learning Rate": 1.0077869346220635e-05, "Finetune/Loss": 0.869996190071106, "Finetune/Loss (Raw)": 0.43709126114845276, "Finetune/Step": 2664, "Finetune/Step Time": 4.723980726674199} +{"Finetune/Learning Rate": 1.0071639909731519e-05, "Finetune/Loss": 0.8690349459648132, "Finetune/Loss (Raw)": 0.7823134660720825, "Finetune/Step": 2665, "Finetune/Step Time": 4.717004604637623} +{"Finetune/Learning Rate": 1.0065410445440352e-05, "Finetune/Loss": 0.8688241243362427, "Finetune/Loss (Raw)": 0.9732510447502136, "Finetune/Step": 2666, "Finetune/Step Time": 4.713700095191598} +{"Finetune/Learning Rate": 1.0059180955764666e-05, "Finetune/Loss": 0.8681246042251587, "Finetune/Loss (Raw)": 0.7970967888832092, "Finetune/Step": 2667, "Finetune/Step Time": 4.714578693732619} +{"Finetune/Learning Rate": 1.005295144312201e-05, "Finetune/Loss": 0.8669540286064148, "Finetune/Loss (Raw)": 0.8601678609848022, "Finetune/Step": 2668, "Finetune/Step Time": 4.716412749141455} +{"Finetune/Learning Rate": 1.0046721909929928e-05, "Finetune/Loss": 0.8682337403297424, "Finetune/Loss (Raw)": 1.0067611932754517, "Finetune/Step": 2669, "Finetune/Step Time": 4.699937034398317} +{"Finetune/Learning Rate": 1.0040492358605989e-05, "Finetune/Loss": 0.8682801723480225, "Finetune/Loss (Raw)": 0.9266030788421631, "Finetune/Step": 2670, "Finetune/Step Time": 4.694737551733851} +{"Finetune/Learning Rate": 1.0034262791567755e-05, "Finetune/Loss": 0.8670896291732788, "Finetune/Loss (Raw)": 0.7821888327598572, "Finetune/Step": 2671, "Finetune/Step Time": 4.700942594558001} +{"Finetune/Learning Rate": 1.0028033211232802e-05, "Finetune/Loss": 0.8667004108428955, "Finetune/Loss (Raw)": 0.685879647731781, "Finetune/Step": 2672, "Finetune/Step Time": 4.705934962257743} +{"Finetune/Learning Rate": 1.0021803620018706e-05, "Finetune/Loss": 0.8666252493858337, "Finetune/Loss (Raw)": 0.8665019869804382, "Finetune/Step": 2673, "Finetune/Step Time": 4.700720127671957} +{"Finetune/Learning Rate": 1.0015574020343048e-05, "Finetune/Loss": 0.8710348606109619, "Finetune/Loss (Raw)": 0.8435283899307251, "Finetune/Step": 2674, "Finetune/Step Time": 4.674148743972182} +{"Finetune/Learning Rate": 1.0009344414623418e-05, "Finetune/Loss": 0.8709409236907959, "Finetune/Loss (Raw)": 0.8636091351509094, "Finetune/Step": 2675, "Finetune/Step Time": 4.678903294727206} +{"Finetune/Learning Rate": 1.0003114805277405e-05, "Finetune/Loss": 0.8693574666976929, "Finetune/Loss (Raw)": 0.8366593718528748, "Finetune/Step": 2676, "Finetune/Step Time": 4.682415530085564} +{"Finetune/Learning Rate": 9.9968851947226e-06, "Finetune/Loss": 0.8745677471160889, "Finetune/Loss (Raw)": 0.9923951029777527, "Finetune/Step": 2677, "Finetune/Step Time": 4.637311277911067} +{"Finetune/Learning Rate": 9.990655585376583e-06, "Finetune/Loss": 0.8741587400436401, "Finetune/Loss (Raw)": 0.8824744820594788, "Finetune/Step": 2678, "Finetune/Step Time": 4.636704161763191} +{"Finetune/Learning Rate": 9.984425979656955e-06, "Finetune/Loss": 0.8747082948684692, "Finetune/Loss (Raw)": 1.0274560451507568, "Finetune/Step": 2679, "Finetune/Step Time": 4.634283421561122} +{"Finetune/Learning Rate": 9.978196379981297e-06, "Finetune/Loss": 0.8733965754508972, "Finetune/Loss (Raw)": 0.9271915555000305, "Finetune/Step": 2680, "Finetune/Step Time": 4.6419426910579205} +{"Finetune/Learning Rate": 9.971966788767198e-06, "Finetune/Loss": 0.8693150281906128, "Finetune/Loss (Raw)": 0.3422161936759949, "Finetune/Step": 2681, "Finetune/Step Time": 4.666678819805384} +{"Finetune/Learning Rate": 9.96573720843225e-06, "Finetune/Loss": 0.8688422441482544, "Finetune/Loss (Raw)": 0.896102249622345, "Finetune/Step": 2682, "Finetune/Step Time": 4.648147767409682} +{"Finetune/Learning Rate": 9.959507641394015e-06, "Finetune/Loss": 0.8733844757080078, "Finetune/Loss (Raw)": 0.9114904999732971, "Finetune/Step": 2683, "Finetune/Step Time": 4.609126856550574} +{"Finetune/Learning Rate": 9.953278090070075e-06, "Finetune/Loss": 0.8732398748397827, "Finetune/Loss (Raw)": 0.8873507976531982, "Finetune/Step": 2684, "Finetune/Step Time": 4.6090972777456045} +{"Finetune/Learning Rate": 9.947048556877992e-06, "Finetune/Loss": 0.8743391036987305, "Finetune/Loss (Raw)": 0.9054701328277588, "Finetune/Step": 2685, "Finetune/Step Time": 4.613178923726082} +{"Finetune/Learning Rate": 9.940819044235335e-06, "Finetune/Loss": 0.8737589716911316, "Finetune/Loss (Raw)": 0.7984294891357422, "Finetune/Step": 2686, "Finetune/Step Time": 4.615956390276551} +{"Finetune/Learning Rate": 9.934589554559653e-06, "Finetune/Loss": 0.873702883720398, "Finetune/Loss (Raw)": 0.8968346118927002, "Finetune/Step": 2687, "Finetune/Step Time": 4.6174279153347015} +{"Finetune/Learning Rate": 9.928360090268484e-06, "Finetune/Loss": 0.8739960193634033, "Finetune/Loss (Raw)": 0.928613007068634, "Finetune/Step": 2688, "Finetune/Step Time": 4.598226845264435} +{"Finetune/Learning Rate": 9.922130653779368e-06, "Finetune/Loss": 0.8750542402267456, "Finetune/Loss (Raw)": 1.0330151319503784, "Finetune/Step": 2689, "Finetune/Step Time": 4.595889750868082} +{"Finetune/Learning Rate": 9.915901247509821e-06, "Finetune/Loss": 0.8699835538864136, "Finetune/Loss (Raw)": 0.36996448040008545, "Finetune/Step": 2690, "Finetune/Step Time": 4.634091982617974} +{"Finetune/Learning Rate": 9.909671873877363e-06, "Finetune/Loss": 0.8731400966644287, "Finetune/Loss (Raw)": 1.0580558776855469, "Finetune/Step": 2691, "Finetune/Step Time": 4.626764280721545} +{"Finetune/Learning Rate": 9.903442535299487e-06, "Finetune/Loss": 0.8725784420967102, "Finetune/Loss (Raw)": 0.856596827507019, "Finetune/Step": 2692, "Finetune/Step Time": 4.626310866326094} +{"Finetune/Learning Rate": 9.897213234193673e-06, "Finetune/Loss": 0.8724339008331299, "Finetune/Loss (Raw)": 0.9597650766372681, "Finetune/Step": 2693, "Finetune/Step Time": 4.623786402866244} +{"Finetune/Learning Rate": 9.890983972977401e-06, "Finetune/Loss": 0.8722429871559143, "Finetune/Loss (Raw)": 0.904991626739502, "Finetune/Step": 2694, "Finetune/Step Time": 4.625027609989047} +{"Finetune/Learning Rate": 9.884754754068116e-06, "Finetune/Loss": 0.87257981300354, "Finetune/Loss (Raw)": 0.8505114316940308, "Finetune/Step": 2695, "Finetune/Step Time": 4.624330535531044} +{"Finetune/Learning Rate": 9.878525579883264e-06, "Finetune/Loss": 0.8739163875579834, "Finetune/Loss (Raw)": 0.9731215238571167, "Finetune/Step": 2696, "Finetune/Step Time": 4.636257002130151} +{"Finetune/Learning Rate": 9.872296452840266e-06, "Finetune/Loss": 0.8719079494476318, "Finetune/Loss (Raw)": 0.8734949231147766, "Finetune/Step": 2697, "Finetune/Step Time": 4.63907509855926} +{"Finetune/Learning Rate": 9.866067375356516e-06, "Finetune/Loss": 0.8711220026016235, "Finetune/Loss (Raw)": 0.8206255435943604, "Finetune/Step": 2698, "Finetune/Step Time": 4.634021380916238} +{"Finetune/Learning Rate": 9.859838349849404e-06, "Finetune/Loss": 0.8706320524215698, "Finetune/Loss (Raw)": 0.9989703893661499, "Finetune/Step": 2699, "Finetune/Step Time": 4.648270856589079} +{"Finetune/Learning Rate": 9.853609378736293e-06, "Finetune/Loss": 0.87051922082901, "Finetune/Loss (Raw)": 0.8675166964530945, "Finetune/Step": 2700, "Finetune/Step Time": 4.647726874798536} +{"Finetune/Learning Rate": 9.847380464434525e-06, "Finetune/Loss": 0.8702894449234009, "Finetune/Loss (Raw)": 0.8736376166343689, "Finetune/Step": 2701, "Finetune/Step Time": 4.670325096696615} +{"Finetune/Learning Rate": 9.841151609361419e-06, "Finetune/Loss": 0.8705418109893799, "Finetune/Loss (Raw)": 0.9266692399978638, "Finetune/Step": 2702, "Finetune/Step Time": 4.658660672605038} +{"Finetune/Learning Rate": 9.83492281593427e-06, "Finetune/Loss": 0.8705501556396484, "Finetune/Loss (Raw)": 0.8305729627609253, "Finetune/Step": 2703, "Finetune/Step Time": 4.675969975069165} +{"Finetune/Learning Rate": 9.828694086570353e-06, "Finetune/Loss": 0.8694337606430054, "Finetune/Loss (Raw)": 0.8054347634315491, "Finetune/Step": 2704, "Finetune/Step Time": 4.68841553106904} +{"Finetune/Learning Rate": 9.822465423686917e-06, "Finetune/Loss": 0.8686200380325317, "Finetune/Loss (Raw)": 0.8755984306335449, "Finetune/Step": 2705, "Finetune/Step Time": 4.688375463709235} +{"Finetune/Learning Rate": 9.816236829701187e-06, "Finetune/Loss": 0.8682566285133362, "Finetune/Loss (Raw)": 0.983728289604187, "Finetune/Step": 2706, "Finetune/Step Time": 4.695298565551639} +{"Finetune/Learning Rate": 9.810008307030356e-06, "Finetune/Loss": 0.8681291937828064, "Finetune/Loss (Raw)": 1.0023465156555176, "Finetune/Step": 2707, "Finetune/Step Time": 4.691489588469267} +{"Finetune/Learning Rate": 9.803779858091586e-06, "Finetune/Loss": 0.870499312877655, "Finetune/Loss (Raw)": 1.0121430158615112, "Finetune/Step": 2708, "Finetune/Step Time": 4.688435014337301} +{"Finetune/Learning Rate": 9.797551485302027e-06, "Finetune/Loss": 0.8690669536590576, "Finetune/Loss (Raw)": 0.8670923709869385, "Finetune/Step": 2709, "Finetune/Step Time": 4.698206784203649} +{"Finetune/Learning Rate": 9.791323191078781e-06, "Finetune/Loss": 0.8651410341262817, "Finetune/Loss (Raw)": 0.37981921434402466, "Finetune/Step": 2710, "Finetune/Step Time": 4.737151093780994} +{"Finetune/Learning Rate": 9.785094977838935e-06, "Finetune/Loss": 0.8662072420120239, "Finetune/Loss (Raw)": 0.9581429362297058, "Finetune/Step": 2711, "Finetune/Step Time": 4.727662848308682} +{"Finetune/Learning Rate": 9.77886684799953e-06, "Finetune/Loss": 0.8653147220611572, "Finetune/Loss (Raw)": 0.9325065016746521, "Finetune/Step": 2712, "Finetune/Step Time": 4.746013512834907} +{"Finetune/Learning Rate": 9.77263880397758e-06, "Finetune/Loss": 0.8658517599105835, "Finetune/Loss (Raw)": 0.8830018043518066, "Finetune/Step": 2713, "Finetune/Step Time": 4.753098949790001} +{"Finetune/Learning Rate": 9.766410848190076e-06, "Finetune/Loss": 0.8656710386276245, "Finetune/Loss (Raw)": 0.9520660638809204, "Finetune/Step": 2714, "Finetune/Step Time": 4.752961816266179} +{"Finetune/Learning Rate": 9.76018298305396e-06, "Finetune/Loss": 0.8646878600120544, "Finetune/Loss (Raw)": 0.8237407803535461, "Finetune/Step": 2715, "Finetune/Step Time": 4.745663542300463} +{"Finetune/Learning Rate": 9.75395521098615e-06, "Finetune/Loss": 0.8646948933601379, "Finetune/Loss (Raw)": 0.859356701374054, "Finetune/Step": 2716, "Finetune/Step Time": 4.7380997240543365} +{"Finetune/Learning Rate": 9.747727534403515e-06, "Finetune/Loss": 0.8639242649078369, "Finetune/Loss (Raw)": 0.900806725025177, "Finetune/Step": 2717, "Finetune/Step Time": 4.719814758747816} +{"Finetune/Learning Rate": 9.741499955722897e-06, "Finetune/Loss": 0.8634248971939087, "Finetune/Loss (Raw)": 0.8681578636169434, "Finetune/Step": 2718, "Finetune/Step Time": 4.719705099239945} +{"Finetune/Learning Rate": 9.735272477361103e-06, "Finetune/Loss": 0.8643659949302673, "Finetune/Loss (Raw)": 0.9399150013923645, "Finetune/Step": 2719, "Finetune/Step Time": 4.6822267062962055} +{"Finetune/Learning Rate": 9.729045101734889e-06, "Finetune/Loss": 0.8646290302276611, "Finetune/Loss (Raw)": 0.9465385675430298, "Finetune/Step": 2720, "Finetune/Step Time": 4.672105407342315} +{"Finetune/Learning Rate": 9.722817831260985e-06, "Finetune/Loss": 0.8647827506065369, "Finetune/Loss (Raw)": 0.8648547530174255, "Finetune/Step": 2721, "Finetune/Step Time": 4.662954572588205} +{"Finetune/Learning Rate": 9.716590668356067e-06, "Finetune/Loss": 0.865204930305481, "Finetune/Loss (Raw)": 0.8904421925544739, "Finetune/Step": 2722, "Finetune/Step Time": 4.653597479686141} +{"Finetune/Learning Rate": 9.710363615436777e-06, "Finetune/Loss": 0.8648960590362549, "Finetune/Loss (Raw)": 0.852634847164154, "Finetune/Step": 2723, "Finetune/Step Time": 4.648474110290408} +{"Finetune/Learning Rate": 9.704136674919712e-06, "Finetune/Loss": 0.8652945756912231, "Finetune/Loss (Raw)": 0.9233648180961609, "Finetune/Step": 2724, "Finetune/Step Time": 4.651751473546028} +{"Finetune/Learning Rate": 9.697909849221428e-06, "Finetune/Loss": 0.8649210929870605, "Finetune/Loss (Raw)": 0.9160866737365723, "Finetune/Step": 2725, "Finetune/Step Time": 4.645367251709104} +{"Finetune/Learning Rate": 9.691683140758439e-06, "Finetune/Loss": 0.8662918210029602, "Finetune/Loss (Raw)": 0.9156078696250916, "Finetune/Step": 2726, "Finetune/Step Time": 4.642280243337154} +{"Finetune/Learning Rate": 9.685456551947199e-06, "Finetune/Loss": 0.865707516670227, "Finetune/Loss (Raw)": 0.8839688301086426, "Finetune/Step": 2727, "Finetune/Step Time": 4.63997633010149} +{"Finetune/Learning Rate": 9.67923008520413e-06, "Finetune/Loss": 0.8649462461471558, "Finetune/Loss (Raw)": 0.7903890609741211, "Finetune/Step": 2728, "Finetune/Step Time": 4.6499023251235485} +{"Finetune/Learning Rate": 9.673003742945602e-06, "Finetune/Loss": 0.8610690832138062, "Finetune/Loss (Raw)": 0.4212709069252014, "Finetune/Step": 2729, "Finetune/Step Time": 4.672388035804033} +{"Finetune/Learning Rate": 9.666777527587936e-06, "Finetune/Loss": 0.8591874241828918, "Finetune/Loss (Raw)": 0.7856761813163757, "Finetune/Step": 2730, "Finetune/Step Time": 4.674320254474878} +{"Finetune/Learning Rate": 9.660551441547411e-06, "Finetune/Loss": 0.8585168123245239, "Finetune/Loss (Raw)": 0.903542697429657, "Finetune/Step": 2731, "Finetune/Step Time": 4.676904806867242} +{"Finetune/Learning Rate": 9.654325487240243e-06, "Finetune/Loss": 0.8589814305305481, "Finetune/Loss (Raw)": 0.9393061995506287, "Finetune/Step": 2732, "Finetune/Step Time": 4.666469482704997} +{"Finetune/Learning Rate": 9.6480996670826e-06, "Finetune/Loss": 0.8598109483718872, "Finetune/Loss (Raw)": 0.8955127000808716, "Finetune/Step": 2733, "Finetune/Step Time": 4.66678617335856} +{"Finetune/Learning Rate": 9.64187398349061e-06, "Finetune/Loss": 0.8606833219528198, "Finetune/Loss (Raw)": 0.9542835354804993, "Finetune/Step": 2734, "Finetune/Step Time": 4.648576445877552} +{"Finetune/Learning Rate": 9.635648438880334e-06, "Finetune/Loss": 0.8607314825057983, "Finetune/Loss (Raw)": 0.9655637741088867, "Finetune/Step": 2735, "Finetune/Step Time": 4.625486049801111} +{"Finetune/Learning Rate": 9.629423035667788e-06, "Finetune/Loss": 0.8568216562271118, "Finetune/Loss (Raw)": 0.4572644829750061, "Finetune/Step": 2736, "Finetune/Step Time": 4.661556785926223} +{"Finetune/Learning Rate": 9.623197776268926e-06, "Finetune/Loss": 0.8543306589126587, "Finetune/Loss (Raw)": 0.6145345568656921, "Finetune/Step": 2737, "Finetune/Step Time": 4.666420863941312} +{"Finetune/Learning Rate": 9.616972663099648e-06, "Finetune/Loss": 0.8546644449234009, "Finetune/Loss (Raw)": 0.92032790184021, "Finetune/Step": 2738, "Finetune/Step Time": 4.655891379341483} +{"Finetune/Learning Rate": 9.610747698575801e-06, "Finetune/Loss": 0.8564273118972778, "Finetune/Loss (Raw)": 0.940392792224884, "Finetune/Step": 2739, "Finetune/Step Time": 4.642304539680481} +{"Finetune/Learning Rate": 9.60452288511318e-06, "Finetune/Loss": 0.8560562133789062, "Finetune/Loss (Raw)": 0.8408346176147461, "Finetune/Step": 2740, "Finetune/Step Time": 4.651373650878668} +{"Finetune/Learning Rate": 9.598298225127499e-06, "Finetune/Loss": 0.8558993339538574, "Finetune/Loss (Raw)": 1.000490427017212, "Finetune/Step": 2741, "Finetune/Step Time": 4.651718344539404} +{"Finetune/Learning Rate": 9.592073721034435e-06, "Finetune/Loss": 0.8564310669898987, "Finetune/Loss (Raw)": 0.914253294467926, "Finetune/Step": 2742, "Finetune/Step Time": 4.656746281310916} +{"Finetune/Learning Rate": 9.585849375249595e-06, "Finetune/Loss": 0.8564149141311646, "Finetune/Loss (Raw)": 0.9660585522651672, "Finetune/Step": 2743, "Finetune/Step Time": 4.648872662335634} +{"Finetune/Learning Rate": 9.57962519018853e-06, "Finetune/Loss": 0.8562498688697815, "Finetune/Loss (Raw)": 0.9337343573570251, "Finetune/Step": 2744, "Finetune/Step Time": 4.645426243543625} +{"Finetune/Learning Rate": 9.573401168266722e-06, "Finetune/Loss": 0.8608455657958984, "Finetune/Loss (Raw)": 0.9846920371055603, "Finetune/Step": 2745, "Finetune/Step Time": 4.607999365776777} +{"Finetune/Learning Rate": 9.567177311899589e-06, "Finetune/Loss": 0.8617379665374756, "Finetune/Loss (Raw)": 0.9098740816116333, "Finetune/Step": 2746, "Finetune/Step Time": 4.609542045742273} +{"Finetune/Learning Rate": 9.560953623502493e-06, "Finetune/Loss": 0.8611030578613281, "Finetune/Loss (Raw)": 0.8835924863815308, "Finetune/Step": 2747, "Finetune/Step Time": 4.626377584412694} +{"Finetune/Learning Rate": 9.554730105490722e-06, "Finetune/Loss": 0.8625401854515076, "Finetune/Loss (Raw)": 1.0208642482757568, "Finetune/Step": 2748, "Finetune/Step Time": 4.634199922904372} +{"Finetune/Learning Rate": 9.548506760279506e-06, "Finetune/Loss": 0.8629838228225708, "Finetune/Loss (Raw)": 0.9447476267814636, "Finetune/Step": 2749, "Finetune/Step Time": 4.6421787813305855} +{"Finetune/Learning Rate": 9.542283590284003e-06, "Finetune/Loss": 0.862341046333313, "Finetune/Loss (Raw)": 0.8624959588050842, "Finetune/Step": 2750, "Finetune/Step Time": 4.640529593452811} +{"Finetune/Learning Rate": 9.5360605979193e-06, "Finetune/Loss": 0.8625848293304443, "Finetune/Loss (Raw)": 0.9375964403152466, "Finetune/Step": 2751, "Finetune/Step Time": 4.629629587754607} +{"Finetune/Learning Rate": 9.52983778560042e-06, "Finetune/Loss": 0.8629180788993835, "Finetune/Loss (Raw)": 0.953612744808197, "Finetune/Step": 2752, "Finetune/Step Time": 4.62932075932622} +{"Finetune/Learning Rate": 9.523615155742317e-06, "Finetune/Loss": 0.864589273929596, "Finetune/Loss (Raw)": 0.9859253168106079, "Finetune/Step": 2753, "Finetune/Step Time": 4.615220993757248} +{"Finetune/Learning Rate": 9.517392710759872e-06, "Finetune/Loss": 0.8695074319839478, "Finetune/Loss (Raw)": 0.9406617283821106, "Finetune/Step": 2754, "Finetune/Step Time": 4.574576457962394} +{"Finetune/Learning Rate": 9.511170453067899e-06, "Finetune/Loss": 0.8653883934020996, "Finetune/Loss (Raw)": 0.3571610450744629, "Finetune/Step": 2755, "Finetune/Step Time": 4.6158016212284565} +{"Finetune/Learning Rate": 9.504948385081121e-06, "Finetune/Loss": 0.8652389049530029, "Finetune/Loss (Raw)": 0.9713670611381531, "Finetune/Step": 2756, "Finetune/Step Time": 4.6327151991426945} +{"Finetune/Learning Rate": 9.498726509214216e-06, "Finetune/Loss": 0.8662282228469849, "Finetune/Loss (Raw)": 0.7810384035110474, "Finetune/Step": 2757, "Finetune/Step Time": 4.649436181411147} +{"Finetune/Learning Rate": 9.492504827881761e-06, "Finetune/Loss": 0.8673290610313416, "Finetune/Loss (Raw)": 0.9516909718513489, "Finetune/Step": 2758, "Finetune/Step Time": 4.649429669603705} +{"Finetune/Learning Rate": 9.486283343498277e-06, "Finetune/Loss": 0.8657361268997192, "Finetune/Loss (Raw)": 0.8340796232223511, "Finetune/Step": 2759, "Finetune/Step Time": 4.650404796004295} +{"Finetune/Learning Rate": 9.480062058478201e-06, "Finetune/Loss": 0.8652222752571106, "Finetune/Loss (Raw)": 0.7411852478981018, "Finetune/Step": 2760, "Finetune/Step Time": 4.644556390121579} +{"Finetune/Learning Rate": 9.473840975235883e-06, "Finetune/Loss": 0.8610213398933411, "Finetune/Loss (Raw)": 0.41766759753227234, "Finetune/Step": 2761, "Finetune/Step Time": 4.682969296351075} +{"Finetune/Learning Rate": 9.467620096185616e-06, "Finetune/Loss": 0.860037088394165, "Finetune/Loss (Raw)": 0.8344317674636841, "Finetune/Step": 2762, "Finetune/Step Time": 4.692299742251635} +{"Finetune/Learning Rate": 9.46139942374159e-06, "Finetune/Loss": 0.859795331954956, "Finetune/Loss (Raw)": 0.8964746594429016, "Finetune/Step": 2763, "Finetune/Step Time": 4.695250798016787} +{"Finetune/Learning Rate": 9.455178960317936e-06, "Finetune/Loss": 0.8587627410888672, "Finetune/Loss (Raw)": 0.8786420822143555, "Finetune/Step": 2764, "Finetune/Step Time": 4.696106020361185} +{"Finetune/Learning Rate": 9.448958708328694e-06, "Finetune/Loss": 0.8550270795822144, "Finetune/Loss (Raw)": 0.35405486822128296, "Finetune/Step": 2765, "Finetune/Step Time": 4.727953638881445} +{"Finetune/Learning Rate": 9.442738670187814e-06, "Finetune/Loss": 0.8541064262390137, "Finetune/Loss (Raw)": 0.865483283996582, "Finetune/Step": 2766, "Finetune/Step Time": 4.713631369173527} +{"Finetune/Learning Rate": 9.436518848309178e-06, "Finetune/Loss": 0.8554432392120361, "Finetune/Loss (Raw)": 1.0003418922424316, "Finetune/Step": 2767, "Finetune/Step Time": 4.727102247998118} +{"Finetune/Learning Rate": 9.430299245106574e-06, "Finetune/Loss": 0.8602055311203003, "Finetune/Loss (Raw)": 0.9582482576370239, "Finetune/Step": 2768, "Finetune/Step Time": 4.6911162957549095} +{"Finetune/Learning Rate": 9.424079862993712e-06, "Finetune/Loss": 0.860396146774292, "Finetune/Loss (Raw)": 0.9351104497909546, "Finetune/Step": 2769, "Finetune/Step Time": 4.699537597596645} +{"Finetune/Learning Rate": 9.417860704384214e-06, "Finetune/Loss": 0.8586780428886414, "Finetune/Loss (Raw)": 0.6809554696083069, "Finetune/Step": 2770, "Finetune/Step Time": 4.7158819530159235} +{"Finetune/Learning Rate": 9.411641771691608e-06, "Finetune/Loss": 0.860001266002655, "Finetune/Loss (Raw)": 0.9051575064659119, "Finetune/Step": 2771, "Finetune/Step Time": 4.708869220688939} +{"Finetune/Learning Rate": 9.405423067329344e-06, "Finetune/Loss": 0.8611206412315369, "Finetune/Loss (Raw)": 0.9199745059013367, "Finetune/Step": 2772, "Finetune/Step Time": 4.677766906097531} +{"Finetune/Learning Rate": 9.399204593710777e-06, "Finetune/Loss": 0.8611878156661987, "Finetune/Loss (Raw)": 0.9556432366371155, "Finetune/Step": 2773, "Finetune/Step Time": 4.670824598520994} +{"Finetune/Learning Rate": 9.392986353249183e-06, "Finetune/Loss": 0.8563750982284546, "Finetune/Loss (Raw)": 0.33144256472587585, "Finetune/Step": 2774, "Finetune/Step Time": 4.708476956933737} +{"Finetune/Learning Rate": 9.386768348357735e-06, "Finetune/Loss": 0.8572012186050415, "Finetune/Loss (Raw)": 0.9286531805992126, "Finetune/Step": 2775, "Finetune/Step Time": 4.6984196696430445} +{"Finetune/Learning Rate": 9.380550581449512e-06, "Finetune/Loss": 0.8559464812278748, "Finetune/Loss (Raw)": 0.8063023686408997, "Finetune/Step": 2776, "Finetune/Step Time": 4.690999390557408} +{"Finetune/Learning Rate": 9.37433305493752e-06, "Finetune/Loss": 0.8559472560882568, "Finetune/Loss (Raw)": 0.8584029674530029, "Finetune/Step": 2777, "Finetune/Step Time": 4.6777317598462105} +{"Finetune/Learning Rate": 9.368115771234651e-06, "Finetune/Loss": 0.856640636920929, "Finetune/Loss (Raw)": 0.8979617953300476, "Finetune/Step": 2778, "Finetune/Step Time": 4.678133826702833} +{"Finetune/Learning Rate": 9.361898732753715e-06, "Finetune/Loss": 0.8565540909767151, "Finetune/Loss (Raw)": 0.8416388630867004, "Finetune/Step": 2779, "Finetune/Step Time": 4.6838754415512085} +{"Finetune/Learning Rate": 9.355681941907428e-06, "Finetune/Loss": 0.8555268049240112, "Finetune/Loss (Raw)": 0.8913443684577942, "Finetune/Step": 2780, "Finetune/Step Time": 4.682499239221215} +{"Finetune/Learning Rate": 9.349465401108393e-06, "Finetune/Loss": 0.8563464283943176, "Finetune/Loss (Raw)": 0.9985431432723999, "Finetune/Step": 2781, "Finetune/Step Time": 4.667240625247359} +{"Finetune/Learning Rate": 9.34324911276914e-06, "Finetune/Loss": 0.8604348301887512, "Finetune/Loss (Raw)": 0.8660799860954285, "Finetune/Step": 2782, "Finetune/Step Time": 4.63190639950335} +{"Finetune/Learning Rate": 9.337033079302082e-06, "Finetune/Loss": 0.8613607287406921, "Finetune/Loss (Raw)": 1.048980474472046, "Finetune/Step": 2783, "Finetune/Step Time": 4.635847872123122} +{"Finetune/Learning Rate": 9.330817303119545e-06, "Finetune/Loss": 0.8599178791046143, "Finetune/Loss (Raw)": 0.7175964117050171, "Finetune/Step": 2784, "Finetune/Step Time": 4.648296523839235} +{"Finetune/Learning Rate": 9.324601786633747e-06, "Finetune/Loss": 0.8611255884170532, "Finetune/Loss (Raw)": 0.8775241374969482, "Finetune/Step": 2785, "Finetune/Step Time": 4.6371134147048} +{"Finetune/Learning Rate": 9.318386532256807e-06, "Finetune/Loss": 0.8631496429443359, "Finetune/Loss (Raw)": 0.9733408093452454, "Finetune/Step": 2786, "Finetune/Step Time": 4.614229328930378} +{"Finetune/Learning Rate": 9.312171542400749e-06, "Finetune/Loss": 0.8617646098136902, "Finetune/Loss (Raw)": 0.8291789889335632, "Finetune/Step": 2787, "Finetune/Step Time": 4.614574860781431} +{"Finetune/Learning Rate": 9.305956819477485e-06, "Finetune/Loss": 0.8617594242095947, "Finetune/Loss (Raw)": 0.9358739852905273, "Finetune/Step": 2788, "Finetune/Step Time": 4.624775815755129} +{"Finetune/Learning Rate": 9.299742365898831e-06, "Finetune/Loss": 0.859896183013916, "Finetune/Loss (Raw)": 0.8664706349372864, "Finetune/Step": 2789, "Finetune/Step Time": 4.638178940862417} +{"Finetune/Learning Rate": 9.293528184076495e-06, "Finetune/Loss": 0.8591480255126953, "Finetune/Loss (Raw)": 0.9170604944229126, "Finetune/Step": 2790, "Finetune/Step Time": 4.641293194144964} +{"Finetune/Learning Rate": 9.287314276422074e-06, "Finetune/Loss": 0.8573060631752014, "Finetune/Loss (Raw)": 0.750031590461731, "Finetune/Step": 2791, "Finetune/Step Time": 4.6510704811662436} +{"Finetune/Learning Rate": 9.28110064534707e-06, "Finetune/Loss": 0.8615478873252869, "Finetune/Loss (Raw)": 0.9800466895103455, "Finetune/Step": 2792, "Finetune/Step Time": 4.621517036110163} +{"Finetune/Learning Rate": 9.274887293262869e-06, "Finetune/Loss": 0.863198459148407, "Finetune/Loss (Raw)": 0.9935829043388367, "Finetune/Step": 2793, "Finetune/Step Time": 4.619344737380743} +{"Finetune/Learning Rate": 9.268674222580757e-06, "Finetune/Loss": 0.8620444536209106, "Finetune/Loss (Raw)": 0.8255353569984436, "Finetune/Step": 2794, "Finetune/Step Time": 4.622214967384934} +{"Finetune/Learning Rate": 9.262461435711898e-06, "Finetune/Loss": 0.8629336953163147, "Finetune/Loss (Raw)": 0.9109272360801697, "Finetune/Step": 2795, "Finetune/Step Time": 4.627054888755083} +{"Finetune/Learning Rate": 9.256248935067356e-06, "Finetune/Loss": 0.8629094362258911, "Finetune/Loss (Raw)": 0.8570650219917297, "Finetune/Step": 2796, "Finetune/Step Time": 4.648661520332098} +{"Finetune/Learning Rate": 9.250036723058083e-06, "Finetune/Loss": 0.8611050248146057, "Finetune/Loss (Raw)": 0.7757948040962219, "Finetune/Step": 2797, "Finetune/Step Time": 4.665611032396555} +{"Finetune/Learning Rate": 9.243824802094914e-06, "Finetune/Loss": 0.8568224906921387, "Finetune/Loss (Raw)": 0.37843504548072815, "Finetune/Step": 2798, "Finetune/Step Time": 4.707734540104866} +{"Finetune/Learning Rate": 9.237613174588583e-06, "Finetune/Loss": 0.856529712677002, "Finetune/Loss (Raw)": 0.7447112202644348, "Finetune/Step": 2799, "Finetune/Step Time": 4.71115498803556} +{"Finetune/Learning Rate": 9.231401842949688e-06, "Finetune/Loss": 0.8569456338882446, "Finetune/Loss (Raw)": 0.739128053188324, "Finetune/Step": 2800, "Finetune/Step Time": 4.7058098670095205} +{"Finetune/Learning Rate": 9.225190809588731e-06, "Finetune/Loss": 0.8578407168388367, "Finetune/Loss (Raw)": 0.9810613989830017, "Finetune/Step": 2801, "Finetune/Step Time": 4.709487102925777} +{"Finetune/Learning Rate": 9.218980076916099e-06, "Finetune/Loss": 0.8589487075805664, "Finetune/Loss (Raw)": 0.9853579998016357, "Finetune/Step": 2802, "Finetune/Step Time": 4.708619775250554} +{"Finetune/Learning Rate": 9.212769647342046e-06, "Finetune/Loss": 0.8594191074371338, "Finetune/Loss (Raw)": 0.9238245487213135, "Finetune/Step": 2803, "Finetune/Step Time": 4.707451859489083} +{"Finetune/Learning Rate": 9.206559523276731e-06, "Finetune/Loss": 0.8600870370864868, "Finetune/Loss (Raw)": 0.922150194644928, "Finetune/Step": 2804, "Finetune/Step Time": 4.701972162351012} +{"Finetune/Learning Rate": 9.200349707130168e-06, "Finetune/Loss": 0.8603558540344238, "Finetune/Loss (Raw)": 1.0268007516860962, "Finetune/Step": 2805, "Finetune/Step Time": 4.700307397171855} +{"Finetune/Learning Rate": 9.194140201312274e-06, "Finetune/Loss": 0.860045313835144, "Finetune/Loss (Raw)": 0.8427306413650513, "Finetune/Step": 2806, "Finetune/Step Time": 4.706683021038771} +{"Finetune/Learning Rate": 9.187931008232832e-06, "Finetune/Loss": 0.858622133731842, "Finetune/Loss (Raw)": 0.8452858328819275, "Finetune/Step": 2807, "Finetune/Step Time": 4.705120267346501} +{"Finetune/Learning Rate": 9.181722130301512e-06, "Finetune/Loss": 0.8573610186576843, "Finetune/Loss (Raw)": 0.7657652497291565, "Finetune/Step": 2808, "Finetune/Step Time": 4.710565367713571} +{"Finetune/Learning Rate": 9.175513569927865e-06, "Finetune/Loss": 0.8574312329292297, "Finetune/Loss (Raw)": 0.3512020409107208, "Finetune/Step": 2809, "Finetune/Step Time": 4.711136231198907} +{"Finetune/Learning Rate": 9.169305329521296e-06, "Finetune/Loss": 0.8564199209213257, "Finetune/Loss (Raw)": 0.766654908657074, "Finetune/Step": 2810, "Finetune/Step Time": 4.703799434006214} +{"Finetune/Learning Rate": 9.163097411491115e-06, "Finetune/Loss": 0.8565154075622559, "Finetune/Loss (Raw)": 0.9237225651741028, "Finetune/Step": 2811, "Finetune/Step Time": 4.7128109112381935} +{"Finetune/Learning Rate": 9.156889818246485e-06, "Finetune/Loss": 0.8564373254776001, "Finetune/Loss (Raw)": 0.877353310585022, "Finetune/Step": 2812, "Finetune/Step Time": 4.7119655553251505} +{"Finetune/Learning Rate": 9.150682552196461e-06, "Finetune/Loss": 0.8571759462356567, "Finetune/Loss (Raw)": 1.0000046491622925, "Finetune/Step": 2813, "Finetune/Step Time": 4.704796958714724} +{"Finetune/Learning Rate": 9.144475615749957e-06, "Finetune/Loss": 0.8536840677261353, "Finetune/Loss (Raw)": 0.35147684812545776, "Finetune/Step": 2814, "Finetune/Step Time": 4.742001786828041} +{"Finetune/Learning Rate": 9.138269011315761e-06, "Finetune/Loss": 0.8546695113182068, "Finetune/Loss (Raw)": 1.0229709148406982, "Finetune/Step": 2815, "Finetune/Step Time": 4.744321284815669} +{"Finetune/Learning Rate": 9.13206274130254e-06, "Finetune/Loss": 0.8547364473342896, "Finetune/Loss (Raw)": 0.9371844530105591, "Finetune/Step": 2816, "Finetune/Step Time": 4.7428826577961445} +{"Finetune/Learning Rate": 9.125856808118822e-06, "Finetune/Loss": 0.8534376621246338, "Finetune/Loss (Raw)": 0.8667656779289246, "Finetune/Step": 2817, "Finetune/Step Time": 4.749682817608118} +{"Finetune/Learning Rate": 9.119651214173012e-06, "Finetune/Loss": 0.8571088314056396, "Finetune/Loss (Raw)": 0.8398697972297668, "Finetune/Step": 2818, "Finetune/Step Time": 4.722437785938382} +{"Finetune/Learning Rate": 9.11344596187338e-06, "Finetune/Loss": 0.8569597005844116, "Finetune/Loss (Raw)": 1.0389649868011475, "Finetune/Step": 2819, "Finetune/Step Time": 4.731184370815754} +{"Finetune/Learning Rate": 9.107241053628058e-06, "Finetune/Loss": 0.8581764101982117, "Finetune/Loss (Raw)": 1.0123406648635864, "Finetune/Step": 2820, "Finetune/Step Time": 4.7310521472245455} +{"Finetune/Learning Rate": 9.101036491845056e-06, "Finetune/Loss": 0.8563817739486694, "Finetune/Loss (Raw)": 0.730049729347229, "Finetune/Step": 2821, "Finetune/Step Time": 4.744147971272469} +{"Finetune/Learning Rate": 9.094832278932238e-06, "Finetune/Loss": 0.8564419746398926, "Finetune/Loss (Raw)": 0.9126980900764465, "Finetune/Step": 2822, "Finetune/Step Time": 4.739973688498139} +{"Finetune/Learning Rate": 9.088628417297346e-06, "Finetune/Loss": 0.8529504537582397, "Finetune/Loss (Raw)": 0.4036012589931488, "Finetune/Step": 2823, "Finetune/Step Time": 4.771952971816063} +{"Finetune/Learning Rate": 9.082424909347969e-06, "Finetune/Loss": 0.8517229557037354, "Finetune/Loss (Raw)": 0.8160008192062378, "Finetune/Step": 2824, "Finetune/Step Time": 4.757959706708789} +{"Finetune/Learning Rate": 9.076221757491568e-06, "Finetune/Loss": 0.8511740565299988, "Finetune/Loss (Raw)": 0.8032326102256775, "Finetune/Step": 2825, "Finetune/Step Time": 4.756863370537758} +{"Finetune/Learning Rate": 9.070018964135471e-06, "Finetune/Loss": 0.8508593440055847, "Finetune/Loss (Raw)": 0.7803404927253723, "Finetune/Step": 2826, "Finetune/Step Time": 4.759814804419875} +{"Finetune/Learning Rate": 9.063816531686855e-06, "Finetune/Loss": 0.8504493832588196, "Finetune/Loss (Raw)": 0.946502149105072, "Finetune/Step": 2827, "Finetune/Step Time": 4.742316519841552} +{"Finetune/Learning Rate": 9.057614462552769e-06, "Finetune/Loss": 0.8518781661987305, "Finetune/Loss (Raw)": 1.0503941774368286, "Finetune/Step": 2828, "Finetune/Step Time": 4.74198572896421} +{"Finetune/Learning Rate": 9.05141275914011e-06, "Finetune/Loss": 0.8523311018943787, "Finetune/Loss (Raw)": 0.9316148161888123, "Finetune/Step": 2829, "Finetune/Step Time": 4.724569998681545} +{"Finetune/Learning Rate": 9.045211423855637e-06, "Finetune/Loss": 0.8526703715324402, "Finetune/Loss (Raw)": 0.9700964093208313, "Finetune/Step": 2830, "Finetune/Step Time": 4.716619720682502} +{"Finetune/Learning Rate": 9.039010459105974e-06, "Finetune/Loss": 0.8544399738311768, "Finetune/Loss (Raw)": 1.0570839643478394, "Finetune/Step": 2831, "Finetune/Step Time": 4.700246024876833} +{"Finetune/Learning Rate": 9.032809867297586e-06, "Finetune/Loss": 0.8559709191322327, "Finetune/Loss (Raw)": 1.0013959407806396, "Finetune/Step": 2832, "Finetune/Step Time": 4.695571998134255} +{"Finetune/Learning Rate": 9.02660965083681e-06, "Finetune/Loss": 0.8560890555381775, "Finetune/Loss (Raw)": 0.8907213807106018, "Finetune/Step": 2833, "Finetune/Step Time": 4.702261738479137} +{"Finetune/Learning Rate": 9.020409812129824e-06, "Finetune/Loss": 0.8511753082275391, "Finetune/Loss (Raw)": 0.35475847125053406, "Finetune/Step": 2834, "Finetune/Step Time": 4.733496701344848} +{"Finetune/Learning Rate": 9.014210353582662e-06, "Finetune/Loss": 0.8506261110305786, "Finetune/Loss (Raw)": 0.9320617318153381, "Finetune/Step": 2835, "Finetune/Step Time": 4.736506471410394} +{"Finetune/Learning Rate": 9.008011277601217e-06, "Finetune/Loss": 0.8485806584358215, "Finetune/Loss (Raw)": 0.7503232955932617, "Finetune/Step": 2836, "Finetune/Step Time": 4.728773707523942} +{"Finetune/Learning Rate": 9.001812586591228e-06, "Finetune/Loss": 0.8492714166641235, "Finetune/Loss (Raw)": 0.9555072784423828, "Finetune/Step": 2837, "Finetune/Step Time": 4.7170186918228865} +{"Finetune/Learning Rate": 8.995614282958289e-06, "Finetune/Loss": 0.8528280258178711, "Finetune/Loss (Raw)": 0.8350655436515808, "Finetune/Step": 2838, "Finetune/Step Time": 4.674487333744764} +{"Finetune/Learning Rate": 8.989416369107837e-06, "Finetune/Loss": 0.8519011735916138, "Finetune/Loss (Raw)": 0.8395073413848877, "Finetune/Step": 2839, "Finetune/Step Time": 4.679580280557275} +{"Finetune/Learning Rate": 8.983218847445157e-06, "Finetune/Loss": 0.8519738912582397, "Finetune/Loss (Raw)": 0.941817581653595, "Finetune/Step": 2840, "Finetune/Step Time": 4.6633617747575045} +{"Finetune/Learning Rate": 8.977021720375395e-06, "Finetune/Loss": 0.8534824252128601, "Finetune/Loss (Raw)": 1.0760884284973145, "Finetune/Step": 2841, "Finetune/Step Time": 4.657697157934308} +{"Finetune/Learning Rate": 8.970824990303532e-06, "Finetune/Loss": 0.8531216382980347, "Finetune/Loss (Raw)": 0.9058876037597656, "Finetune/Step": 2842, "Finetune/Step Time": 4.659614738076925} +{"Finetune/Learning Rate": 8.9646286596344e-06, "Finetune/Loss": 0.8525630235671997, "Finetune/Loss (Raw)": 0.7522408962249756, "Finetune/Step": 2843, "Finetune/Step Time": 4.664780514314771} +{"Finetune/Learning Rate": 8.958432730772667e-06, "Finetune/Loss": 0.852961540222168, "Finetune/Loss (Raw)": 0.9103613495826721, "Finetune/Step": 2844, "Finetune/Step Time": 4.665071101859212} +{"Finetune/Learning Rate": 8.952237206122855e-06, "Finetune/Loss": 0.8534530401229858, "Finetune/Loss (Raw)": 0.9637245535850525, "Finetune/Step": 2845, "Finetune/Step Time": 4.675880741328001} +{"Finetune/Learning Rate": 8.946042088089331e-06, "Finetune/Loss": 0.8533488512039185, "Finetune/Loss (Raw)": 0.8548116683959961, "Finetune/Step": 2846, "Finetune/Step Time": 4.674140417948365} +{"Finetune/Learning Rate": 8.939847379076293e-06, "Finetune/Loss": 0.8513883948326111, "Finetune/Loss (Raw)": 0.6889817714691162, "Finetune/Step": 2847, "Finetune/Step Time": 4.685948619619012} +{"Finetune/Learning Rate": 8.933653081487794e-06, "Finetune/Loss": 0.851599931716919, "Finetune/Loss (Raw)": 0.9736177325248718, "Finetune/Step": 2848, "Finetune/Step Time": 4.692695425823331} +{"Finetune/Learning Rate": 8.927459197727712e-06, "Finetune/Loss": 0.8515959978103638, "Finetune/Loss (Raw)": 0.8643521666526794, "Finetune/Step": 2849, "Finetune/Step Time": 4.688420979306102} +{"Finetune/Learning Rate": 8.921265730199775e-06, "Finetune/Loss": 0.8514752388000488, "Finetune/Loss (Raw)": 0.8749786019325256, "Finetune/Step": 2850, "Finetune/Step Time": 4.683365613222122} +{"Finetune/Learning Rate": 8.91507268130755e-06, "Finetune/Loss": 0.8521212339401245, "Finetune/Loss (Raw)": 0.9353322386741638, "Finetune/Step": 2851, "Finetune/Step Time": 4.6799474731087685} +{"Finetune/Learning Rate": 8.908880053454432e-06, "Finetune/Loss": 0.8526453971862793, "Finetune/Loss (Raw)": 0.9904541969299316, "Finetune/Step": 2852, "Finetune/Step Time": 4.680140282958746} +{"Finetune/Learning Rate": 8.90268784904367e-06, "Finetune/Loss": 0.8523828387260437, "Finetune/Loss (Raw)": 0.8824777007102966, "Finetune/Step": 2853, "Finetune/Step Time": 4.680379990488291} +{"Finetune/Learning Rate": 8.896496070478326e-06, "Finetune/Loss": 0.8526313304901123, "Finetune/Loss (Raw)": 0.9474136233329773, "Finetune/Step": 2854, "Finetune/Step Time": 4.673535333946347} +{"Finetune/Learning Rate": 8.890304720161311e-06, "Finetune/Loss": 0.8531571626663208, "Finetune/Loss (Raw)": 0.9512772560119629, "Finetune/Step": 2855, "Finetune/Step Time": 4.678269352763891} +{"Finetune/Learning Rate": 8.884113800495372e-06, "Finetune/Loss": 0.8544793128967285, "Finetune/Loss (Raw)": 0.9596251249313354, "Finetune/Step": 2856, "Finetune/Step Time": 4.6804707236588} +{"Finetune/Learning Rate": 8.87792331388308e-06, "Finetune/Loss": 0.8588910102844238, "Finetune/Loss (Raw)": 0.9859596490859985, "Finetune/Step": 2857, "Finetune/Step Time": 4.650736033916473} +{"Finetune/Learning Rate": 8.871733262726846e-06, "Finetune/Loss": 0.8591325283050537, "Finetune/Loss (Raw)": 0.8165989518165588, "Finetune/Step": 2858, "Finetune/Step Time": 4.649636739864945} +{"Finetune/Learning Rate": 8.865543649428903e-06, "Finetune/Loss": 0.8591257333755493, "Finetune/Loss (Raw)": 0.9026634693145752, "Finetune/Step": 2859, "Finetune/Step Time": 4.651221502572298} +{"Finetune/Learning Rate": 8.859354476391319e-06, "Finetune/Loss": 0.8579592704772949, "Finetune/Loss (Raw)": 0.7900063991546631, "Finetune/Step": 2860, "Finetune/Step Time": 4.656289152801037} +{"Finetune/Learning Rate": 8.853165746015997e-06, "Finetune/Loss": 0.8590461015701294, "Finetune/Loss (Raw)": 1.0346266031265259, "Finetune/Step": 2861, "Finetune/Step Time": 4.66188558191061} +{"Finetune/Learning Rate": 8.846977460704656e-06, "Finetune/Loss": 0.8580111265182495, "Finetune/Loss (Raw)": 0.8218058347702026, "Finetune/Step": 2862, "Finetune/Step Time": 4.672462414950132} +{"Finetune/Learning Rate": 8.840789622858856e-06, "Finetune/Loss": 0.8577660322189331, "Finetune/Loss (Raw)": 0.9341965317726135, "Finetune/Step": 2863, "Finetune/Step Time": 4.6753650307655334} +{"Finetune/Learning Rate": 8.834602234879968e-06, "Finetune/Loss": 0.8608635663986206, "Finetune/Loss (Raw)": 0.8537439107894897, "Finetune/Step": 2864, "Finetune/Step Time": 4.636146828532219} +{"Finetune/Learning Rate": 8.8284152991692e-06, "Finetune/Loss": 0.8638716340065002, "Finetune/Loss (Raw)": 0.9995707273483276, "Finetune/Step": 2865, "Finetune/Step Time": 4.628251256421208} +{"Finetune/Learning Rate": 8.822228818127584e-06, "Finetune/Loss": 0.8597752451896667, "Finetune/Loss (Raw)": 0.39598971605300903, "Finetune/Step": 2866, "Finetune/Step Time": 4.662435673177242} +{"Finetune/Learning Rate": 8.81604279415597e-06, "Finetune/Loss": 0.8595496416091919, "Finetune/Loss (Raw)": 0.9115140438079834, "Finetune/Step": 2867, "Finetune/Step Time": 4.671042367815971} +{"Finetune/Learning Rate": 8.809857229655029e-06, "Finetune/Loss": 0.8610391020774841, "Finetune/Loss (Raw)": 1.0314860343933105, "Finetune/Step": 2868, "Finetune/Step Time": 4.665758000686765} +{"Finetune/Learning Rate": 8.803672127025264e-06, "Finetune/Loss": 0.8588923215866089, "Finetune/Loss (Raw)": 0.7256982922554016, "Finetune/Step": 2869, "Finetune/Step Time": 4.697336740791798} +{"Finetune/Learning Rate": 8.797487488666986e-06, "Finetune/Loss": 0.8546907305717468, "Finetune/Loss (Raw)": 0.37645411491394043, "Finetune/Step": 2870, "Finetune/Step Time": 4.728666728362441} +{"Finetune/Learning Rate": 8.791303316980342e-06, "Finetune/Loss": 0.8551105856895447, "Finetune/Loss (Raw)": 1.019795536994934, "Finetune/Step": 2871, "Finetune/Step Time": 4.726985938847065} +{"Finetune/Learning Rate": 8.785119614365282e-06, "Finetune/Loss": 0.8554889559745789, "Finetune/Loss (Raw)": 0.9821664094924927, "Finetune/Step": 2872, "Finetune/Step Time": 4.727243687957525} +{"Finetune/Learning Rate": 8.77893638322158e-06, "Finetune/Loss": 0.8547763824462891, "Finetune/Loss (Raw)": 0.893488883972168, "Finetune/Step": 2873, "Finetune/Step Time": 4.728613708168268} +{"Finetune/Learning Rate": 8.772753625948828e-06, "Finetune/Loss": 0.8504887223243713, "Finetune/Loss (Raw)": 0.3610496520996094, "Finetune/Step": 2874, "Finetune/Step Time": 4.752744352445006} +{"Finetune/Learning Rate": 8.766571344946433e-06, "Finetune/Loss": 0.8514969348907471, "Finetune/Loss (Raw)": 1.0126436948776245, "Finetune/Step": 2875, "Finetune/Step Time": 4.7378558944910765} +{"Finetune/Learning Rate": 8.760389542613619e-06, "Finetune/Loss": 0.8504408001899719, "Finetune/Loss (Raw)": 0.8856769800186157, "Finetune/Step": 2876, "Finetune/Step Time": 4.7409527618438005} +{"Finetune/Learning Rate": 8.754208221349427e-06, "Finetune/Loss": 0.8496305346488953, "Finetune/Loss (Raw)": 0.8410332798957825, "Finetune/Step": 2877, "Finetune/Step Time": 4.7322036642581224} +{"Finetune/Learning Rate": 8.748027383552696e-06, "Finetune/Loss": 0.8502376079559326, "Finetune/Loss (Raw)": 0.9402069449424744, "Finetune/Step": 2878, "Finetune/Step Time": 4.73725675418973} +{"Finetune/Learning Rate": 8.741847031622096e-06, "Finetune/Loss": 0.8490506410598755, "Finetune/Loss (Raw)": 0.785658597946167, "Finetune/Step": 2879, "Finetune/Step Time": 4.738613102585077} +{"Finetune/Learning Rate": 8.7356671679561e-06, "Finetune/Loss": 0.8492828607559204, "Finetune/Loss (Raw)": 0.9833419322967529, "Finetune/Step": 2880, "Finetune/Step Time": 4.730148803442717} +{"Finetune/Learning Rate": 8.729487794952992e-06, "Finetune/Loss": 0.8492682576179504, "Finetune/Loss (Raw)": 0.9840533137321472, "Finetune/Step": 2881, "Finetune/Step Time": 4.735296750441194} +{"Finetune/Learning Rate": 8.72330891501087e-06, "Finetune/Loss": 0.8496971130371094, "Finetune/Loss (Raw)": 0.995559573173523, "Finetune/Step": 2882, "Finetune/Step Time": 4.7367167472839355} +{"Finetune/Learning Rate": 8.717130530527626e-06, "Finetune/Loss": 0.8547466993331909, "Finetune/Loss (Raw)": 1.0035067796707153, "Finetune/Step": 2883, "Finetune/Step Time": 4.701505733653903} +{"Finetune/Learning Rate": 8.710952643900983e-06, "Finetune/Loss": 0.8539034128189087, "Finetune/Loss (Raw)": 0.8634242415428162, "Finetune/Step": 2884, "Finetune/Step Time": 4.712781209498644} +{"Finetune/Learning Rate": 8.704775257528446e-06, "Finetune/Loss": 0.8551410436630249, "Finetune/Loss (Raw)": 0.9394587278366089, "Finetune/Step": 2885, "Finetune/Step Time": 4.6856905203312635} +{"Finetune/Learning Rate": 8.69859837380735e-06, "Finetune/Loss": 0.853134036064148, "Finetune/Loss (Raw)": 0.6947869658470154, "Finetune/Step": 2886, "Finetune/Step Time": 4.709076784551144} +{"Finetune/Learning Rate": 8.692421995134819e-06, "Finetune/Loss": 0.8527635335922241, "Finetune/Loss (Raw)": 0.7866533994674683, "Finetune/Step": 2887, "Finetune/Step Time": 4.71392940543592} +{"Finetune/Learning Rate": 8.686246123907778e-06, "Finetune/Loss": 0.8552682399749756, "Finetune/Loss (Raw)": 1.061788558959961, "Finetune/Step": 2888, "Finetune/Step Time": 4.721684757620096} +{"Finetune/Learning Rate": 8.680070762522969e-06, "Finetune/Loss": 0.8545819520950317, "Finetune/Loss (Raw)": 0.3298303782939911, "Finetune/Step": 2889, "Finetune/Step Time": 4.72054629586637} +{"Finetune/Learning Rate": 8.673895913376928e-06, "Finetune/Loss": 0.8544209003448486, "Finetune/Loss (Raw)": 0.8138073682785034, "Finetune/Step": 2890, "Finetune/Step Time": 4.709535108879209} +{"Finetune/Learning Rate": 8.667721578865993e-06, "Finetune/Loss": 0.8544516563415527, "Finetune/Loss (Raw)": 0.9004214406013489, "Finetune/Step": 2891, "Finetune/Step Time": 4.708465855568647} +{"Finetune/Learning Rate": 8.661547761386307e-06, "Finetune/Loss": 0.8561867475509644, "Finetune/Loss (Raw)": 1.1007258892059326, "Finetune/Step": 2892, "Finetune/Step Time": 4.717490656301379} +{"Finetune/Learning Rate": 8.655374463333797e-06, "Finetune/Loss": 0.8603121042251587, "Finetune/Loss (Raw)": 0.8821017146110535, "Finetune/Step": 2893, "Finetune/Step Time": 4.676958177238703} +{"Finetune/Learning Rate": 8.649201687104209e-06, "Finetune/Loss": 0.861494779586792, "Finetune/Loss (Raw)": 1.0168678760528564, "Finetune/Step": 2894, "Finetune/Step Time": 4.675956733524799} +{"Finetune/Learning Rate": 8.643029435093072e-06, "Finetune/Loss": 0.860633373260498, "Finetune/Loss (Raw)": 0.8900827169418335, "Finetune/Step": 2895, "Finetune/Step Time": 4.662665413692594} +{"Finetune/Learning Rate": 8.63685770969572e-06, "Finetune/Loss": 0.8605287671089172, "Finetune/Loss (Raw)": 0.9448534250259399, "Finetune/Step": 2896, "Finetune/Step Time": 4.664711190387607} +{"Finetune/Learning Rate": 8.630686513307282e-06, "Finetune/Loss": 0.8608503341674805, "Finetune/Loss (Raw)": 0.9762794971466064, "Finetune/Step": 2897, "Finetune/Step Time": 4.659169198945165} +{"Finetune/Learning Rate": 8.62451584832267e-06, "Finetune/Loss": 0.8627023696899414, "Finetune/Loss (Raw)": 0.9180136322975159, "Finetune/Step": 2898, "Finetune/Step Time": 4.642244225367904} +{"Finetune/Learning Rate": 8.618345717136604e-06, "Finetune/Loss": 0.8624236583709717, "Finetune/Loss (Raw)": 0.8694893717765808, "Finetune/Step": 2899, "Finetune/Step Time": 4.642750134691596} +{"Finetune/Learning Rate": 8.612176122143589e-06, "Finetune/Loss": 0.8614143133163452, "Finetune/Loss (Raw)": 0.7907671928405762, "Finetune/Step": 2900, "Finetune/Step Time": 4.646142682060599} +{"Finetune/Learning Rate": 8.606007065737928e-06, "Finetune/Loss": 0.8601484894752502, "Finetune/Loss (Raw)": 0.7936244010925293, "Finetune/Step": 2901, "Finetune/Step Time": 4.655657893046737} +{"Finetune/Learning Rate": 8.599838550313714e-06, "Finetune/Loss": 0.8648287057876587, "Finetune/Loss (Raw)": 0.9305019378662109, "Finetune/Step": 2902, "Finetune/Step Time": 4.6407118160277605} +{"Finetune/Learning Rate": 8.593670578264814e-06, "Finetune/Loss": 0.8650770783424377, "Finetune/Loss (Raw)": 0.9604482650756836, "Finetune/Step": 2903, "Finetune/Step Time": 4.643672419711947} +{"Finetune/Learning Rate": 8.587503151984911e-06, "Finetune/Loss": 0.8655251860618591, "Finetune/Loss (Raw)": 0.8636569380760193, "Finetune/Step": 2904, "Finetune/Step Time": 4.641835814341903} +{"Finetune/Learning Rate": 8.581336273867455e-06, "Finetune/Loss": 0.8658605217933655, "Finetune/Loss (Raw)": 0.9013323783874512, "Finetune/Step": 2905, "Finetune/Step Time": 4.643485639244318} +{"Finetune/Learning Rate": 8.575169946305699e-06, "Finetune/Loss": 0.8661630153656006, "Finetune/Loss (Raw)": 0.9366791248321533, "Finetune/Step": 2906, "Finetune/Step Time": 4.64469419233501} +{"Finetune/Learning Rate": 8.569004171692667e-06, "Finetune/Loss": 0.866301953792572, "Finetune/Loss (Raw)": 0.8594242334365845, "Finetune/Step": 2907, "Finetune/Step Time": 4.636999988928437} +{"Finetune/Learning Rate": 8.562838952421177e-06, "Finetune/Loss": 0.8667142391204834, "Finetune/Loss (Raw)": 0.9441170692443848, "Finetune/Step": 2908, "Finetune/Step Time": 4.640028910711408} +{"Finetune/Learning Rate": 8.556674290883833e-06, "Finetune/Loss": 0.8673787117004395, "Finetune/Loss (Raw)": 1.0835952758789062, "Finetune/Step": 2909, "Finetune/Step Time": 4.634434854611754} +{"Finetune/Learning Rate": 8.550510189473016e-06, "Finetune/Loss": 0.8683157563209534, "Finetune/Loss (Raw)": 0.9860185384750366, "Finetune/Step": 2910, "Finetune/Step Time": 4.635653292760253} +{"Finetune/Learning Rate": 8.544346650580899e-06, "Finetune/Loss": 0.8683413863182068, "Finetune/Loss (Raw)": 1.0522613525390625, "Finetune/Step": 2911, "Finetune/Step Time": 4.6214446891099215} +{"Finetune/Learning Rate": 8.538183676599426e-06, "Finetune/Loss": 0.870112955570221, "Finetune/Loss (Raw)": 0.9443550109863281, "Finetune/Step": 2912, "Finetune/Step Time": 4.613476054742932} +{"Finetune/Learning Rate": 8.532021269920326e-06, "Finetune/Loss": 0.8706032037734985, "Finetune/Loss (Raw)": 0.940281867980957, "Finetune/Step": 2913, "Finetune/Step Time": 4.606081910431385} +{"Finetune/Learning Rate": 8.525859432935114e-06, "Finetune/Loss": 0.8684820532798767, "Finetune/Loss (Raw)": 0.701831042766571, "Finetune/Step": 2914, "Finetune/Step Time": 4.632595784962177} +{"Finetune/Learning Rate": 8.519698168035074e-06, "Finetune/Loss": 0.8678080439567566, "Finetune/Loss (Raw)": 0.7429060935974121, "Finetune/Step": 2915, "Finetune/Step Time": 4.644540540874004} +{"Finetune/Learning Rate": 8.513537477611278e-06, "Finetune/Loss": 0.8677660226821899, "Finetune/Loss (Raw)": 0.9304929971694946, "Finetune/Step": 2916, "Finetune/Step Time": 4.631339689716697} +{"Finetune/Learning Rate": 8.507377364054566e-06, "Finetune/Loss": 0.8686498403549194, "Finetune/Loss (Raw)": 0.979608416557312, "Finetune/Step": 2917, "Finetune/Step Time": 4.615486359223723} +{"Finetune/Learning Rate": 8.501217829755554e-06, "Finetune/Loss": 0.8681426644325256, "Finetune/Loss (Raw)": 0.8521389365196228, "Finetune/Step": 2918, "Finetune/Step Time": 4.621020890772343} +{"Finetune/Learning Rate": 8.495058877104642e-06, "Finetune/Loss": 0.8693859577178955, "Finetune/Loss (Raw)": 0.9091721177101135, "Finetune/Step": 2919, "Finetune/Step Time": 4.594236008822918} +{"Finetune/Learning Rate": 8.488900508491999e-06, "Finetune/Loss": 0.8644352555274963, "Finetune/Loss (Raw)": 0.3463522493839264, "Finetune/Step": 2920, "Finetune/Step Time": 4.618881141766906} +{"Finetune/Learning Rate": 8.48274272630757e-06, "Finetune/Loss": 0.8636816143989563, "Finetune/Loss (Raw)": 0.8971194624900818, "Finetune/Step": 2921, "Finetune/Step Time": 4.6420542020350695} +{"Finetune/Learning Rate": 8.476585532941064e-06, "Finetune/Loss": 0.8639583587646484, "Finetune/Loss (Raw)": 0.8609582781791687, "Finetune/Step": 2922, "Finetune/Step Time": 4.641031434759498} +{"Finetune/Learning Rate": 8.470428930781969e-06, "Finetune/Loss": 0.8637900352478027, "Finetune/Loss (Raw)": 0.8893771767616272, "Finetune/Step": 2923, "Finetune/Step Time": 4.6628663539886475} +{"Finetune/Learning Rate": 8.464272922219547e-06, "Finetune/Loss": 0.8649500012397766, "Finetune/Loss (Raw)": 1.0055444240570068, "Finetune/Step": 2924, "Finetune/Step Time": 4.635591270402074} +{"Finetune/Learning Rate": 8.458117509642817e-06, "Finetune/Loss": 0.8668307065963745, "Finetune/Loss (Raw)": 1.0165283679962158, "Finetune/Step": 2925, "Finetune/Step Time": 4.620358092710376} +{"Finetune/Learning Rate": 8.451962695440586e-06, "Finetune/Loss": 0.8708113431930542, "Finetune/Loss (Raw)": 0.8879517912864685, "Finetune/Step": 2926, "Finetune/Step Time": 4.61528504639864} +{"Finetune/Learning Rate": 8.445808482001405e-06, "Finetune/Loss": 0.8704614043235779, "Finetune/Loss (Raw)": 0.6999218463897705, "Finetune/Step": 2927, "Finetune/Step Time": 4.624700279906392} +{"Finetune/Learning Rate": 8.439654871713608e-06, "Finetune/Loss": 0.8717525005340576, "Finetune/Loss (Raw)": 0.9043865203857422, "Finetune/Step": 2928, "Finetune/Step Time": 4.633764024823904} +{"Finetune/Learning Rate": 8.433501866965294e-06, "Finetune/Loss": 0.8722875714302063, "Finetune/Loss (Raw)": 1.0495514869689941, "Finetune/Step": 2929, "Finetune/Step Time": 4.623833704739809} +{"Finetune/Learning Rate": 8.42734947014432e-06, "Finetune/Loss": 0.8721861839294434, "Finetune/Loss (Raw)": 0.972381055355072, "Finetune/Step": 2930, "Finetune/Step Time": 4.62152012437582} +{"Finetune/Learning Rate": 8.42119768363832e-06, "Finetune/Loss": 0.8723245859146118, "Finetune/Loss (Raw)": 0.9415375590324402, "Finetune/Step": 2931, "Finetune/Step Time": 4.624372327700257} +{"Finetune/Learning Rate": 8.415046509834668e-06, "Finetune/Loss": 0.8724501132965088, "Finetune/Loss (Raw)": 0.9382213354110718, "Finetune/Step": 2932, "Finetune/Step Time": 4.628023013472557} +{"Finetune/Learning Rate": 8.408895951120526e-06, "Finetune/Loss": 0.8715610504150391, "Finetune/Loss (Raw)": 0.9129977226257324, "Finetune/Step": 2933, "Finetune/Step Time": 4.639540247619152} +{"Finetune/Learning Rate": 8.402746009882798e-06, "Finetune/Loss": 0.8722023963928223, "Finetune/Loss (Raw)": 0.9248281717300415, "Finetune/Step": 2934, "Finetune/Step Time": 4.633262129500508} +{"Finetune/Learning Rate": 8.39659668850816e-06, "Finetune/Loss": 0.8727002739906311, "Finetune/Loss (Raw)": 0.9090099930763245, "Finetune/Step": 2935, "Finetune/Step Time": 4.638182520866394} +{"Finetune/Learning Rate": 8.39044798938305e-06, "Finetune/Loss": 0.8746919631958008, "Finetune/Loss (Raw)": 1.0206983089447021, "Finetune/Step": 2936, "Finetune/Step Time": 4.644346533343196} +{"Finetune/Learning Rate": 8.384299914893645e-06, "Finetune/Loss": 0.8802342414855957, "Finetune/Loss (Raw)": 1.0606167316436768, "Finetune/Step": 2937, "Finetune/Step Time": 4.60731428489089} +{"Finetune/Learning Rate": 8.378152467425902e-06, "Finetune/Loss": 0.8816013336181641, "Finetune/Loss (Raw)": 0.9416395425796509, "Finetune/Step": 2938, "Finetune/Step Time": 4.6038234885782} +{"Finetune/Learning Rate": 8.37200564936552e-06, "Finetune/Loss": 0.881434440612793, "Finetune/Loss (Raw)": 0.9023557901382446, "Finetune/Step": 2939, "Finetune/Step Time": 4.597239874303341} +{"Finetune/Learning Rate": 8.365859463097963e-06, "Finetune/Loss": 0.880212128162384, "Finetune/Loss (Raw)": 0.7209042906761169, "Finetune/Step": 2940, "Finetune/Step Time": 4.599054662510753} +{"Finetune/Learning Rate": 8.359713911008447e-06, "Finetune/Loss": 0.8776731491088867, "Finetune/Loss (Raw)": 0.6750146746635437, "Finetune/Step": 2941, "Finetune/Step Time": 4.617172414436936} +{"Finetune/Learning Rate": 8.353568995481934e-06, "Finetune/Loss": 0.8829790353775024, "Finetune/Loss (Raw)": 1.0306313037872314, "Finetune/Step": 2942, "Finetune/Step Time": 4.583149453625083} +{"Finetune/Learning Rate": 8.347424718903152e-06, "Finetune/Loss": 0.882373034954071, "Finetune/Loss (Raw)": 0.9454051852226257, "Finetune/Step": 2943, "Finetune/Step Time": 4.59354680031538} +{"Finetune/Learning Rate": 8.341281083656571e-06, "Finetune/Loss": 0.8822897672653198, "Finetune/Loss (Raw)": 0.9265260696411133, "Finetune/Step": 2944, "Finetune/Step Time": 4.600812871009111} +{"Finetune/Learning Rate": 8.33513809212642e-06, "Finetune/Loss": 0.8820044994354248, "Finetune/Loss (Raw)": 0.830244779586792, "Finetune/Step": 2945, "Finetune/Step Time": 4.614585237577558} +{"Finetune/Learning Rate": 8.328995746696674e-06, "Finetune/Loss": 0.8826369643211365, "Finetune/Loss (Raw)": 0.9208334684371948, "Finetune/Step": 2946, "Finetune/Step Time": 4.600236451253295} +{"Finetune/Learning Rate": 8.32285404975105e-06, "Finetune/Loss": 0.881332516670227, "Finetune/Loss (Raw)": 0.8719881176948547, "Finetune/Step": 2947, "Finetune/Step Time": 4.611138425767422} +{"Finetune/Learning Rate": 8.316713003673028e-06, "Finetune/Loss": 0.8810237646102905, "Finetune/Loss (Raw)": 0.9728195667266846, "Finetune/Step": 2948, "Finetune/Step Time": 4.625985264778137} +{"Finetune/Learning Rate": 8.310572610845824e-06, "Finetune/Loss": 0.882764458656311, "Finetune/Loss (Raw)": 0.9528615474700928, "Finetune/Step": 2949, "Finetune/Step Time": 4.615932727232575} +{"Finetune/Learning Rate": 8.304432873652412e-06, "Finetune/Loss": 0.8827366828918457, "Finetune/Loss (Raw)": 0.9091444611549377, "Finetune/Step": 2950, "Finetune/Step Time": 4.615663081407547} +{"Finetune/Learning Rate": 8.298293794475495e-06, "Finetune/Loss": 0.8819576501846313, "Finetune/Loss (Raw)": 0.3038829267024994, "Finetune/Step": 2951, "Finetune/Step Time": 4.6161106657236814} +{"Finetune/Learning Rate": 8.29215537569753e-06, "Finetune/Loss": 0.8830223083496094, "Finetune/Loss (Raw)": 0.9522851705551147, "Finetune/Step": 2952, "Finetune/Step Time": 4.613855008035898} +{"Finetune/Learning Rate": 8.286017619700725e-06, "Finetune/Loss": 0.8844565153121948, "Finetune/Loss (Raw)": 0.9868077635765076, "Finetune/Step": 2953, "Finetune/Step Time": 4.612702300772071} +{"Finetune/Learning Rate": 8.279880528867017e-06, "Finetune/Loss": 0.8845551609992981, "Finetune/Loss (Raw)": 0.7929643392562866, "Finetune/Step": 2954, "Finetune/Step Time": 4.617411037907004} +{"Finetune/Learning Rate": 8.273744105578097e-06, "Finetune/Loss": 0.885425329208374, "Finetune/Loss (Raw)": 1.057881474494934, "Finetune/Step": 2955, "Finetune/Step Time": 4.624349152669311} +{"Finetune/Learning Rate": 8.267608352215385e-06, "Finetune/Loss": 0.8845735788345337, "Finetune/Loss (Raw)": 0.9413694739341736, "Finetune/Step": 2956, "Finetune/Step Time": 4.621413433924317} +{"Finetune/Learning Rate": 8.261473271160046e-06, "Finetune/Loss": 0.8843334317207336, "Finetune/Loss (Raw)": 0.9008752107620239, "Finetune/Step": 2957, "Finetune/Step Time": 4.609849901869893} +{"Finetune/Learning Rate": 8.255338864792993e-06, "Finetune/Loss": 0.8838851451873779, "Finetune/Loss (Raw)": 0.9127158522605896, "Finetune/Step": 2958, "Finetune/Step Time": 4.6129167303442955} +{"Finetune/Learning Rate": 8.249205135494862e-06, "Finetune/Loss": 0.8781167268753052, "Finetune/Loss (Raw)": 0.3187335431575775, "Finetune/Step": 2959, "Finetune/Step Time": 4.647688791155815} +{"Finetune/Learning Rate": 8.24307208564604e-06, "Finetune/Loss": 0.8774142265319824, "Finetune/Loss (Raw)": 0.9114683270454407, "Finetune/Step": 2960, "Finetune/Step Time": 4.644800234586} +{"Finetune/Learning Rate": 8.236939717626639e-06, "Finetune/Loss": 0.8770493268966675, "Finetune/Loss (Raw)": 0.844014585018158, "Finetune/Step": 2961, "Finetune/Step Time": 4.631296021863818} +{"Finetune/Learning Rate": 8.23080803381651e-06, "Finetune/Loss": 0.882676362991333, "Finetune/Loss (Raw)": 1.0750246047973633, "Finetune/Step": 2962, "Finetune/Step Time": 4.59947950579226} +{"Finetune/Learning Rate": 8.224677036595245e-06, "Finetune/Loss": 0.8830357193946838, "Finetune/Loss (Raw)": 0.9780527353286743, "Finetune/Step": 2963, "Finetune/Step Time": 4.609979402273893} +{"Finetune/Learning Rate": 8.218546728342161e-06, "Finetune/Loss": 0.8849347829818726, "Finetune/Loss (Raw)": 0.9933983683586121, "Finetune/Step": 2964, "Finetune/Step Time": 4.612611185759306} +{"Finetune/Learning Rate": 8.212417111436315e-06, "Finetune/Loss": 0.8850260972976685, "Finetune/Loss (Raw)": 0.9672021865844727, "Finetune/Step": 2965, "Finetune/Step Time": 4.608871651813388} +{"Finetune/Learning Rate": 8.206288188256486e-06, "Finetune/Loss": 0.8849344253540039, "Finetune/Loss (Raw)": 0.8233338594436646, "Finetune/Step": 2966, "Finetune/Step Time": 4.616829462349415} +{"Finetune/Learning Rate": 8.20015996118119e-06, "Finetune/Loss": 0.8845788836479187, "Finetune/Loss (Raw)": 0.7939991354942322, "Finetune/Step": 2967, "Finetune/Step Time": 4.628767250105739} +{"Finetune/Learning Rate": 8.194032432588677e-06, "Finetune/Loss": 0.885262668132782, "Finetune/Loss (Raw)": 1.0293430089950562, "Finetune/Step": 2968, "Finetune/Step Time": 4.6377660389989614} +{"Finetune/Learning Rate": 8.187905604856916e-06, "Finetune/Loss": 0.8836380243301392, "Finetune/Loss (Raw)": 0.8681371212005615, "Finetune/Step": 2969, "Finetune/Step Time": 4.634558359161019} +{"Finetune/Learning Rate": 8.181779480363614e-06, "Finetune/Loss": 0.8837016820907593, "Finetune/Loss (Raw)": 0.9140329360961914, "Finetune/Step": 2970, "Finetune/Step Time": 4.64680808968842} +{"Finetune/Learning Rate": 8.175654061486197e-06, "Finetune/Loss": 0.8848220109939575, "Finetune/Loss (Raw)": 0.8956353664398193, "Finetune/Step": 2971, "Finetune/Step Time": 4.654078438878059} +{"Finetune/Learning Rate": 8.16952935060182e-06, "Finetune/Loss": 0.8849550485610962, "Finetune/Loss (Raw)": 0.9273961782455444, "Finetune/Step": 2972, "Finetune/Step Time": 4.649934709072113} +{"Finetune/Learning Rate": 8.163405350087366e-06, "Finetune/Loss": 0.8835822343826294, "Finetune/Loss (Raw)": 0.7879987359046936, "Finetune/Step": 2973, "Finetune/Step Time": 4.653363833203912} +{"Finetune/Learning Rate": 8.157282062319437e-06, "Finetune/Loss": 0.8842090368270874, "Finetune/Loss (Raw)": 0.93503737449646, "Finetune/Step": 2974, "Finetune/Step Time": 4.654441295191646} +{"Finetune/Learning Rate": 8.15115948967437e-06, "Finetune/Loss": 0.881869912147522, "Finetune/Loss (Raw)": 0.38957828283309937, "Finetune/Step": 2975, "Finetune/Step Time": 4.679303949698806} +{"Finetune/Learning Rate": 8.145037634528205e-06, "Finetune/Loss": 0.8807267546653748, "Finetune/Loss (Raw)": 0.827299177646637, "Finetune/Step": 2976, "Finetune/Step Time": 4.6736370753496885} +{"Finetune/Learning Rate": 8.138916499256718e-06, "Finetune/Loss": 0.8811987042427063, "Finetune/Loss (Raw)": 0.9247614145278931, "Finetune/Step": 2977, "Finetune/Step Time": 4.671358494088054} +{"Finetune/Learning Rate": 8.132796086235405e-06, "Finetune/Loss": 0.8821255564689636, "Finetune/Loss (Raw)": 0.9936137795448303, "Finetune/Step": 2978, "Finetune/Step Time": 4.6721866223961115} +{"Finetune/Learning Rate": 8.126676397839475e-06, "Finetune/Loss": 0.8818274140357971, "Finetune/Loss (Raw)": 0.897167444229126, "Finetune/Step": 2979, "Finetune/Step Time": 4.682959068566561} +{"Finetune/Learning Rate": 8.120557436443866e-06, "Finetune/Loss": 0.8806957006454468, "Finetune/Loss (Raw)": 0.8455916047096252, "Finetune/Step": 2980, "Finetune/Step Time": 4.686295252293348} +{"Finetune/Learning Rate": 8.11443920442322e-06, "Finetune/Loss": 0.8799399733543396, "Finetune/Loss (Raw)": 0.7857535481452942, "Finetune/Step": 2981, "Finetune/Step Time": 4.712868450209498} +{"Finetune/Learning Rate": 8.108321704151903e-06, "Finetune/Loss": 0.8789759874343872, "Finetune/Loss (Raw)": 0.8240211606025696, "Finetune/Step": 2982, "Finetune/Step Time": 4.705869399011135} +{"Finetune/Learning Rate": 8.102204938004004e-06, "Finetune/Loss": 0.8794318437576294, "Finetune/Loss (Raw)": 1.0096265077590942, "Finetune/Step": 2983, "Finetune/Step Time": 4.696379376575351} +{"Finetune/Learning Rate": 8.096088908353316e-06, "Finetune/Loss": 0.879718542098999, "Finetune/Loss (Raw)": 0.9963182806968689, "Finetune/Step": 2984, "Finetune/Step Time": 4.696330765262246} +{"Finetune/Learning Rate": 8.089973617573355e-06, "Finetune/Loss": 0.8788760304450989, "Finetune/Loss (Raw)": 0.8781238794326782, "Finetune/Step": 2985, "Finetune/Step Time": 4.6905247792601585} +{"Finetune/Learning Rate": 8.08385906803734e-06, "Finetune/Loss": 0.875069260597229, "Finetune/Loss (Raw)": 0.3293377757072449, "Finetune/Step": 2986, "Finetune/Step Time": 4.724452879279852} +{"Finetune/Learning Rate": 8.07774526211821e-06, "Finetune/Loss": 0.8745294809341431, "Finetune/Loss (Raw)": 0.8335645794868469, "Finetune/Step": 2987, "Finetune/Step Time": 4.737370910122991} +{"Finetune/Learning Rate": 8.071632202188618e-06, "Finetune/Loss": 0.875796377658844, "Finetune/Loss (Raw)": 0.9521695375442505, "Finetune/Step": 2988, "Finetune/Step Time": 4.734100943431258} +{"Finetune/Learning Rate": 8.065519890620922e-06, "Finetune/Loss": 0.8741159439086914, "Finetune/Loss (Raw)": 0.8195286393165588, "Finetune/Step": 2989, "Finetune/Step Time": 4.738055154681206} +{"Finetune/Learning Rate": 8.059408329787184e-06, "Finetune/Loss": 0.8756381273269653, "Finetune/Loss (Raw)": 1.0166523456573486, "Finetune/Step": 2990, "Finetune/Step Time": 4.719967583194375} +{"Finetune/Learning Rate": 8.053297522059189e-06, "Finetune/Loss": 0.8758141398429871, "Finetune/Loss (Raw)": 0.9567205309867859, "Finetune/Step": 2991, "Finetune/Step Time": 4.719101106747985} +{"Finetune/Learning Rate": 8.047187469808418e-06, "Finetune/Loss": 0.8753479719161987, "Finetune/Loss (Raw)": 0.7940768599510193, "Finetune/Step": 2992, "Finetune/Step Time": 4.730293188244104} +{"Finetune/Learning Rate": 8.041078175406064e-06, "Finetune/Loss": 0.8753663301467896, "Finetune/Loss (Raw)": 1.0019176006317139, "Finetune/Step": 2993, "Finetune/Step Time": 4.726230263710022} +{"Finetune/Learning Rate": 8.03496964122303e-06, "Finetune/Loss": 0.8803313970565796, "Finetune/Loss (Raw)": 1.0315210819244385, "Finetune/Step": 2994, "Finetune/Step Time": 4.684485606849194} +{"Finetune/Learning Rate": 8.02886186962991e-06, "Finetune/Loss": 0.8802983164787292, "Finetune/Loss (Raw)": 0.9072751998901367, "Finetune/Step": 2995, "Finetune/Step Time": 4.682334067299962} +{"Finetune/Learning Rate": 8.022754862997016e-06, "Finetune/Loss": 0.8791354894638062, "Finetune/Loss (Raw)": 0.8826420307159424, "Finetune/Step": 2996, "Finetune/Step Time": 4.697114726528525} +{"Finetune/Learning Rate": 8.016648623694354e-06, "Finetune/Loss": 0.8799857497215271, "Finetune/Loss (Raw)": 0.834535539150238, "Finetune/Step": 2997, "Finetune/Step Time": 4.677844364196062} +{"Finetune/Learning Rate": 8.010543154091644e-06, "Finetune/Loss": 0.8838428258895874, "Finetune/Loss (Raw)": 0.8701569437980652, "Finetune/Step": 2998, "Finetune/Step Time": 4.649012880399823} +{"Finetune/Learning Rate": 8.004438456558297e-06, "Finetune/Loss": 0.8842170238494873, "Finetune/Loss (Raw)": 1.0676939487457275, "Finetune/Step": 2999, "Finetune/Step Time": 4.65542221814394} +{"Finetune/Learning Rate": 7.99833453346342e-06, "Finetune/Loss": 0.8835541009902954, "Finetune/Loss (Raw)": 0.8973132371902466, "Finetune/Step": 3000, "Finetune/Step Time": 4.659717984497547} +{"Finetune/Learning Rate": 7.992231387175832e-06, "Finetune/Loss": 0.8833296298980713, "Finetune/Loss (Raw)": 0.864764928817749, "Finetune/Step": 3001, "Finetune/Step Time": 4.662986343726516} +{"Finetune/Learning Rate": 7.986129020064044e-06, "Finetune/Loss": 0.8878133296966553, "Finetune/Loss (Raw)": 0.9349597692489624, "Finetune/Step": 3002, "Finetune/Step Time": 4.628243703395128} +{"Finetune/Learning Rate": 7.980027434496267e-06, "Finetune/Loss": 0.886817216873169, "Finetune/Loss (Raw)": 0.8851401209831238, "Finetune/Step": 3003, "Finetune/Step Time": 4.629393830895424} +{"Finetune/Learning Rate": 7.97392663284041e-06, "Finetune/Loss": 0.887204110622406, "Finetune/Loss (Raw)": 0.9351940751075745, "Finetune/Step": 3004, "Finetune/Step Time": 4.622353216633201} +{"Finetune/Learning Rate": 7.967826617464065e-06, "Finetune/Loss": 0.8862650990486145, "Finetune/Loss (Raw)": 0.7208454608917236, "Finetune/Step": 3005, "Finetune/Step Time": 4.640963152050972} +{"Finetune/Learning Rate": 7.961727390734538e-06, "Finetune/Loss": 0.886275589466095, "Finetune/Loss (Raw)": 0.9415492415428162, "Finetune/Step": 3006, "Finetune/Step Time": 4.633532129228115} +{"Finetune/Learning Rate": 7.955628955018813e-06, "Finetune/Loss": 0.8861731290817261, "Finetune/Loss (Raw)": 0.7725369930267334, "Finetune/Step": 3007, "Finetune/Step Time": 4.62460513971746} +{"Finetune/Learning Rate": 7.949531312683582e-06, "Finetune/Loss": 0.885860800743103, "Finetune/Loss (Raw)": 0.9433697462081909, "Finetune/Step": 3008, "Finetune/Step Time": 4.625879587605596} +{"Finetune/Learning Rate": 7.943434466095216e-06, "Finetune/Loss": 0.8859145641326904, "Finetune/Loss (Raw)": 0.990943968296051, "Finetune/Step": 3009, "Finetune/Step Time": 4.618454473093152} +{"Finetune/Learning Rate": 7.937338417619781e-06, "Finetune/Loss": 0.8849603533744812, "Finetune/Loss (Raw)": 0.8734181523323059, "Finetune/Step": 3010, "Finetune/Step Time": 4.619216620922089} +{"Finetune/Learning Rate": 7.931243169623037e-06, "Finetune/Loss": 0.8851056098937988, "Finetune/Loss (Raw)": 1.0220915079116821, "Finetune/Step": 3011, "Finetune/Step Time": 4.6216224785894156} +{"Finetune/Learning Rate": 7.925148724470428e-06, "Finetune/Loss": 0.8857290744781494, "Finetune/Loss (Raw)": 0.943221390247345, "Finetune/Step": 3012, "Finetune/Step Time": 4.594169933348894} +{"Finetune/Learning Rate": 7.919055084527094e-06, "Finetune/Loss": 0.8853817582130432, "Finetune/Loss (Raw)": 0.8950119018554688, "Finetune/Step": 3013, "Finetune/Step Time": 4.585300516337156} +{"Finetune/Learning Rate": 7.912962252157854e-06, "Finetune/Loss": 0.8866063356399536, "Finetune/Loss (Raw)": 0.8515320420265198, "Finetune/Step": 3014, "Finetune/Step Time": 4.568618565797806} +{"Finetune/Learning Rate": 7.906870229727216e-06, "Finetune/Loss": 0.8864943385124207, "Finetune/Loss (Raw)": 0.7723167538642883, "Finetune/Step": 3015, "Finetune/Step Time": 4.57760544680059} +{"Finetune/Learning Rate": 7.900779019599378e-06, "Finetune/Loss": 0.8852521181106567, "Finetune/Loss (Raw)": 0.9027814865112305, "Finetune/Step": 3016, "Finetune/Step Time": 4.581116329878569} +{"Finetune/Learning Rate": 7.894688624138217e-06, "Finetune/Loss": 0.8887343406677246, "Finetune/Loss (Raw)": 0.7755631804466248, "Finetune/Step": 3017, "Finetune/Step Time": 4.549152521416545} +{"Finetune/Learning Rate": 7.8885990457073e-06, "Finetune/Loss": 0.8890082836151123, "Finetune/Loss (Raw)": 0.8488643169403076, "Finetune/Step": 3018, "Finetune/Step Time": 4.544189736247063} +{"Finetune/Learning Rate": 7.882510286669875e-06, "Finetune/Loss": 0.8892936110496521, "Finetune/Loss (Raw)": 0.9369467496871948, "Finetune/Step": 3019, "Finetune/Step Time": 4.545955363661051} +{"Finetune/Learning Rate": 7.876422349388861e-06, "Finetune/Loss": 0.887547492980957, "Finetune/Loss (Raw)": 0.8772212862968445, "Finetune/Step": 3020, "Finetune/Step Time": 4.527609396725893} +{"Finetune/Learning Rate": 7.870335236226878e-06, "Finetune/Loss": 0.8875967264175415, "Finetune/Loss (Raw)": 0.8884050846099854, "Finetune/Step": 3021, "Finetune/Step Time": 4.536728359758854} +{"Finetune/Learning Rate": 7.864248949546209e-06, "Finetune/Loss": 0.8872579336166382, "Finetune/Loss (Raw)": 0.9734948873519897, "Finetune/Step": 3022, "Finetune/Step Time": 4.536671483889222} +{"Finetune/Learning Rate": 7.858163491708827e-06, "Finetune/Loss": 0.8860690593719482, "Finetune/Loss (Raw)": 0.7379143834114075, "Finetune/Step": 3023, "Finetune/Step Time": 4.539461212232709} +{"Finetune/Learning Rate": 7.85207886507638e-06, "Finetune/Loss": 0.8859002590179443, "Finetune/Loss (Raw)": 0.9232404232025146, "Finetune/Step": 3024, "Finetune/Step Time": 4.544439444318414} +{"Finetune/Learning Rate": 7.845995072010188e-06, "Finetune/Loss": 0.8855479955673218, "Finetune/Loss (Raw)": 0.9311990141868591, "Finetune/Step": 3025, "Finetune/Step Time": 4.543784733861685} +{"Finetune/Learning Rate": 7.839912114871256e-06, "Finetune/Loss": 0.8856927752494812, "Finetune/Loss (Raw)": 0.9365445375442505, "Finetune/Step": 3026, "Finetune/Step Time": 4.548156777396798} +{"Finetune/Learning Rate": 7.83382999602026e-06, "Finetune/Loss": 0.8865994215011597, "Finetune/Loss (Raw)": 0.9855398535728455, "Finetune/Step": 3027, "Finetune/Step Time": 4.546648679301143} +{"Finetune/Learning Rate": 7.827748717817552e-06, "Finetune/Loss": 0.8874568939208984, "Finetune/Loss (Raw)": 0.9005224108695984, "Finetune/Step": 3028, "Finetune/Step Time": 4.549723660573363} +{"Finetune/Learning Rate": 7.821668282623159e-06, "Finetune/Loss": 0.8894551992416382, "Finetune/Loss (Raw)": 1.0494062900543213, "Finetune/Step": 3029, "Finetune/Step Time": 4.540590377524495} +{"Finetune/Learning Rate": 7.815588692796771e-06, "Finetune/Loss": 0.8891695737838745, "Finetune/Loss (Raw)": 0.8939460515975952, "Finetune/Step": 3030, "Finetune/Step Time": 4.532032987102866} +{"Finetune/Learning Rate": 7.809509950697769e-06, "Finetune/Loss": 0.888804018497467, "Finetune/Loss (Raw)": 0.9136595129966736, "Finetune/Step": 3031, "Finetune/Step Time": 4.529005207121372} +{"Finetune/Learning Rate": 7.803432058685187e-06, "Finetune/Loss": 0.8897314071655273, "Finetune/Loss (Raw)": 0.9823507070541382, "Finetune/Step": 3032, "Finetune/Step Time": 4.533509746193886} +{"Finetune/Learning Rate": 7.797355019117743e-06, "Finetune/Loss": 0.8897930979728699, "Finetune/Loss (Raw)": 0.9092320799827576, "Finetune/Step": 3033, "Finetune/Step Time": 4.530328210443258} +{"Finetune/Learning Rate": 7.79127883435381e-06, "Finetune/Loss": 0.8906341791152954, "Finetune/Loss (Raw)": 1.0443389415740967, "Finetune/Step": 3034, "Finetune/Step Time": 4.526124596595764} +{"Finetune/Learning Rate": 7.78520350675144e-06, "Finetune/Loss": 0.8909323215484619, "Finetune/Loss (Raw)": 0.8975850343704224, "Finetune/Step": 3035, "Finetune/Step Time": 4.527908410876989} +{"Finetune/Learning Rate": 7.779129038668347e-06, "Finetune/Loss": 0.8904402256011963, "Finetune/Loss (Raw)": 0.8811361789703369, "Finetune/Step": 3036, "Finetune/Step Time": 4.541141156107187} +{"Finetune/Learning Rate": 7.773055432461916e-06, "Finetune/Loss": 0.8899456858634949, "Finetune/Loss (Raw)": 1.0202910900115967, "Finetune/Step": 3037, "Finetune/Step Time": 4.543395381420851} +{"Finetune/Learning Rate": 7.766982690489198e-06, "Finetune/Loss": 0.8899874687194824, "Finetune/Loss (Raw)": 0.9913648366928101, "Finetune/Step": 3038, "Finetune/Step Time": 4.541963120922446} +{"Finetune/Learning Rate": 7.7609108151069e-06, "Finetune/Loss": 0.8882973194122314, "Finetune/Loss (Raw)": 0.8359231352806091, "Finetune/Step": 3039, "Finetune/Step Time": 4.541891930624843} +{"Finetune/Learning Rate": 7.754839808671397e-06, "Finetune/Loss": 0.8882073163986206, "Finetune/Loss (Raw)": 0.9328397512435913, "Finetune/Step": 3040, "Finetune/Step Time": 4.534601887688041} +{"Finetune/Learning Rate": 7.748769673538734e-06, "Finetune/Loss": 0.8869187235832214, "Finetune/Loss (Raw)": 0.7753363847732544, "Finetune/Step": 3041, "Finetune/Step Time": 4.539552390575409} +{"Finetune/Learning Rate": 7.742700412064608e-06, "Finetune/Loss": 0.8886803984642029, "Finetune/Loss (Raw)": 0.9273233413696289, "Finetune/Step": 3042, "Finetune/Step Time": 4.515195610001683} +{"Finetune/Learning Rate": 7.736632026604386e-06, "Finetune/Loss": 0.8852977752685547, "Finetune/Loss (Raw)": 0.30992719531059265, "Finetune/Step": 3043, "Finetune/Step Time": 4.542720962315798} +{"Finetune/Learning Rate": 7.730564519513082e-06, "Finetune/Loss": 0.8836996555328369, "Finetune/Loss (Raw)": 0.7259371876716614, "Finetune/Step": 3044, "Finetune/Step Time": 4.561709847301245} +{"Finetune/Learning Rate": 7.724497893145379e-06, "Finetune/Loss": 0.882375955581665, "Finetune/Loss (Raw)": 0.8101817965507507, "Finetune/Step": 3045, "Finetune/Step Time": 4.567825846374035} +{"Finetune/Learning Rate": 7.718432149855622e-06, "Finetune/Loss": 0.8838609457015991, "Finetune/Loss (Raw)": 1.0422130823135376, "Finetune/Step": 3046, "Finetune/Step Time": 4.561503864824772} +{"Finetune/Learning Rate": 7.7123672919978e-06, "Finetune/Loss": 0.8834744095802307, "Finetune/Loss (Raw)": 0.859698474407196, "Finetune/Step": 3047, "Finetune/Step Time": 4.567252075299621} +{"Finetune/Learning Rate": 7.706303321925577e-06, "Finetune/Loss": 0.8888998031616211, "Finetune/Loss (Raw)": 1.0407975912094116, "Finetune/Step": 3048, "Finetune/Step Time": 4.528912544250488} +{"Finetune/Learning Rate": 7.700240241992247e-06, "Finetune/Loss": 0.888836145401001, "Finetune/Loss (Raw)": 0.8889670968055725, "Finetune/Step": 3049, "Finetune/Step Time": 4.5120987724512815} +{"Finetune/Learning Rate": 7.694178054550782e-06, "Finetune/Loss": 0.8899198770523071, "Finetune/Loss (Raw)": 0.9996828436851501, "Finetune/Step": 3050, "Finetune/Step Time": 4.497345799580216} +{"Finetune/Learning Rate": 7.688116761953797e-06, "Finetune/Loss": 0.8910711407661438, "Finetune/Loss (Raw)": 1.0367372035980225, "Finetune/Step": 3051, "Finetune/Step Time": 4.475919606164098} +{"Finetune/Learning Rate": 7.682056366553559e-06, "Finetune/Loss": 0.8901035785675049, "Finetune/Loss (Raw)": 0.8817039132118225, "Finetune/Step": 3052, "Finetune/Step Time": 4.485678534954786} +{"Finetune/Learning Rate": 7.675996870701999e-06, "Finetune/Loss": 0.8897996544837952, "Finetune/Loss (Raw)": 0.9776200652122498, "Finetune/Step": 3053, "Finetune/Step Time": 4.488925697281957} +{"Finetune/Learning Rate": 7.669938276750673e-06, "Finetune/Loss": 0.8907742500305176, "Finetune/Loss (Raw)": 1.012704610824585, "Finetune/Step": 3054, "Finetune/Step Time": 4.458139006048441} +{"Finetune/Learning Rate": 7.663880587050812e-06, "Finetune/Loss": 0.8926084041595459, "Finetune/Loss (Raw)": 0.9346863031387329, "Finetune/Step": 3055, "Finetune/Step Time": 4.44937351718545} +{"Finetune/Learning Rate": 7.657823803953288e-06, "Finetune/Loss": 0.8932271599769592, "Finetune/Loss (Raw)": 0.9835907816886902, "Finetune/Step": 3056, "Finetune/Step Time": 4.428062418475747} +{"Finetune/Learning Rate": 7.651767929808615e-06, "Finetune/Loss": 0.8912646770477295, "Finetune/Loss (Raw)": 0.7983513474464417, "Finetune/Step": 3057, "Finetune/Step Time": 4.4409983810037374} +{"Finetune/Learning Rate": 7.645712966966973e-06, "Finetune/Loss": 0.8913578987121582, "Finetune/Loss (Raw)": 0.9843118786811829, "Finetune/Step": 3058, "Finetune/Step Time": 4.437990756705403} +{"Finetune/Learning Rate": 7.639658917778156e-06, "Finetune/Loss": 0.8914270401000977, "Finetune/Loss (Raw)": 0.9503913521766663, "Finetune/Step": 3059, "Finetune/Step Time": 4.434841761365533} +{"Finetune/Learning Rate": 7.633605784591633e-06, "Finetune/Loss": 0.8917464017868042, "Finetune/Loss (Raw)": 0.9791007041931152, "Finetune/Step": 3060, "Finetune/Step Time": 4.439297271892428} +{"Finetune/Learning Rate": 7.627553569756505e-06, "Finetune/Loss": 0.8920783400535583, "Finetune/Loss (Raw)": 0.9554858803749084, "Finetune/Step": 3061, "Finetune/Step Time": 4.431120598688722} +{"Finetune/Learning Rate": 7.621502275621518e-06, "Finetune/Loss": 0.8915409445762634, "Finetune/Loss (Raw)": 0.8560402393341064, "Finetune/Step": 3062, "Finetune/Step Time": 4.464498145505786} +{"Finetune/Learning Rate": 7.615451904535066e-06, "Finetune/Loss": 0.8912160396575928, "Finetune/Loss (Raw)": 0.8674250841140747, "Finetune/Step": 3063, "Finetune/Step Time": 4.456204455345869} +{"Finetune/Learning Rate": 7.609402458845171e-06, "Finetune/Loss": 0.8903615474700928, "Finetune/Loss (Raw)": 0.9113154411315918, "Finetune/Step": 3064, "Finetune/Step Time": 4.437679305672646} +{"Finetune/Learning Rate": 7.60335394089951e-06, "Finetune/Loss": 0.8883564472198486, "Finetune/Loss (Raw)": 0.8039631843566895, "Finetune/Step": 3065, "Finetune/Step Time": 4.454855253919959} +{"Finetune/Learning Rate": 7.597306353045393e-06, "Finetune/Loss": 0.8864403963088989, "Finetune/Loss (Raw)": 0.6963930130004883, "Finetune/Step": 3066, "Finetune/Step Time": 4.487933209165931} +{"Finetune/Learning Rate": 7.5912596976297735e-06, "Finetune/Loss": 0.8863266110420227, "Finetune/Loss (Raw)": 0.887788712978363, "Finetune/Step": 3067, "Finetune/Step Time": 4.488690130412579} +{"Finetune/Learning Rate": 7.58521397699924e-06, "Finetune/Loss": 0.8876280784606934, "Finetune/Loss (Raw)": 0.8875007629394531, "Finetune/Step": 3068, "Finetune/Step Time": 4.500722588971257} +{"Finetune/Learning Rate": 7.579169193500014e-06, "Finetune/Loss": 0.8885279297828674, "Finetune/Loss (Raw)": 0.7901882529258728, "Finetune/Step": 3069, "Finetune/Step Time": 4.491922080516815} +{"Finetune/Learning Rate": 7.573125349477964e-06, "Finetune/Loss": 0.8877202272415161, "Finetune/Loss (Raw)": 0.9272416234016418, "Finetune/Step": 3070, "Finetune/Step Time": 4.489595485851169} +{"Finetune/Learning Rate": 7.567082447278582e-06, "Finetune/Loss": 0.8882619142532349, "Finetune/Loss (Raw)": 1.014743447303772, "Finetune/Step": 3071, "Finetune/Step Time": 4.476202595978975} +{"Finetune/Learning Rate": 7.561040489247008e-06, "Finetune/Loss": 0.887776255607605, "Finetune/Loss (Raw)": 0.8643594980239868, "Finetune/Step": 3072, "Finetune/Step Time": 4.477878352627158} +{"Finetune/Learning Rate": 7.554999477728001e-06, "Finetune/Loss": 0.8882778882980347, "Finetune/Loss (Raw)": 0.8944558501243591, "Finetune/Step": 3073, "Finetune/Step Time": 4.469392366707325} +{"Finetune/Learning Rate": 7.548959415065961e-06, "Finetune/Loss": 0.88780277967453, "Finetune/Loss (Raw)": 0.8600204586982727, "Finetune/Step": 3074, "Finetune/Step Time": 4.471734402701259} +{"Finetune/Learning Rate": 7.54292030360492e-06, "Finetune/Loss": 0.8832919597625732, "Finetune/Loss (Raw)": 0.2945963740348816, "Finetune/Step": 3075, "Finetune/Step Time": 4.493722254410386} +{"Finetune/Learning Rate": 7.5368821456885375e-06, "Finetune/Loss": 0.8816449642181396, "Finetune/Loss (Raw)": 0.7620095014572144, "Finetune/Step": 3076, "Finetune/Step Time": 4.4807465225458145} +{"Finetune/Learning Rate": 7.530844943660111e-06, "Finetune/Loss": 0.8815402984619141, "Finetune/Loss (Raw)": 0.9394680857658386, "Finetune/Step": 3077, "Finetune/Step Time": 4.480655964463949} +{"Finetune/Learning Rate": 7.524808699862552e-06, "Finetune/Loss": 0.8808128237724304, "Finetune/Loss (Raw)": 0.8160229921340942, "Finetune/Step": 3078, "Finetune/Step Time": 4.477091237902641} +{"Finetune/Learning Rate": 7.518773416638412e-06, "Finetune/Loss": 0.8844634294509888, "Finetune/Loss (Raw)": 0.7711546421051025, "Finetune/Step": 3079, "Finetune/Step Time": 4.461709637194872} +{"Finetune/Learning Rate": 7.512739096329868e-06, "Finetune/Loss": 0.8829915523529053, "Finetune/Loss (Raw)": 0.763889729976654, "Finetune/Step": 3080, "Finetune/Step Time": 4.466151934117079} +{"Finetune/Learning Rate": 7.50670574127872e-06, "Finetune/Loss": 0.8816382884979248, "Finetune/Loss (Raw)": 0.8135963678359985, "Finetune/Step": 3081, "Finetune/Step Time": 4.469557464122772} +{"Finetune/Learning Rate": 7.500673353826399e-06, "Finetune/Loss": 0.8822691440582275, "Finetune/Loss (Raw)": 0.8737123608589172, "Finetune/Step": 3082, "Finetune/Step Time": 4.4643550626933575} +{"Finetune/Learning Rate": 7.4946419363139535e-06, "Finetune/Loss": 0.8813556432723999, "Finetune/Loss (Raw)": 0.940950334072113, "Finetune/Step": 3083, "Finetune/Step Time": 4.4750757198780775} +{"Finetune/Learning Rate": 7.488611491082056e-06, "Finetune/Loss": 0.8803867697715759, "Finetune/Loss (Raw)": 0.8173587918281555, "Finetune/Step": 3084, "Finetune/Step Time": 4.486277105286717} +{"Finetune/Learning Rate": 7.482582020471012e-06, "Finetune/Loss": 0.8817532062530518, "Finetune/Loss (Raw)": 1.075778603553772, "Finetune/Step": 3085, "Finetune/Step Time": 4.483452150598168} +{"Finetune/Learning Rate": 7.476553526820733e-06, "Finetune/Loss": 0.8822293281555176, "Finetune/Loss (Raw)": 0.9736507534980774, "Finetune/Step": 3086, "Finetune/Step Time": 4.48296014405787} +{"Finetune/Learning Rate": 7.4705260124707705e-06, "Finetune/Loss": 0.8867693543434143, "Finetune/Loss (Raw)": 0.8998555541038513, "Finetune/Step": 3087, "Finetune/Step Time": 4.450058491900563} +{"Finetune/Learning Rate": 7.464499479760275e-06, "Finetune/Loss": 0.885236918926239, "Finetune/Loss (Raw)": 0.7153233885765076, "Finetune/Step": 3088, "Finetune/Step Time": 4.44889946654439} +{"Finetune/Learning Rate": 7.458473931028029e-06, "Finetune/Loss": 0.8860246539115906, "Finetune/Loss (Raw)": 0.9448497295379639, "Finetune/Step": 3089, "Finetune/Step Time": 4.45291687361896} +{"Finetune/Learning Rate": 7.452449368612433e-06, "Finetune/Loss": 0.8840981721878052, "Finetune/Loss (Raw)": 0.8284345865249634, "Finetune/Step": 3090, "Finetune/Step Time": 4.446537360548973} +{"Finetune/Learning Rate": 7.446425794851498e-06, "Finetune/Loss": 0.8829450607299805, "Finetune/Loss (Raw)": 0.8304494023323059, "Finetune/Step": 3091, "Finetune/Step Time": 4.45028081163764} +{"Finetune/Learning Rate": 7.440403212082863e-06, "Finetune/Loss": 0.8775413036346436, "Finetune/Loss (Raw)": 0.30171316862106323, "Finetune/Step": 3092, "Finetune/Step Time": 4.48569774068892} +{"Finetune/Learning Rate": 7.434381622643766e-06, "Finetune/Loss": 0.876940906047821, "Finetune/Loss (Raw)": 0.8903548717498779, "Finetune/Step": 3093, "Finetune/Step Time": 4.4927275981754065} +{"Finetune/Learning Rate": 7.42836102887107e-06, "Finetune/Loss": 0.877619743347168, "Finetune/Loss (Raw)": 0.9102194309234619, "Finetune/Step": 3094, "Finetune/Step Time": 4.487178863957524} +{"Finetune/Learning Rate": 7.422341433101254e-06, "Finetune/Loss": 0.878373384475708, "Finetune/Loss (Raw)": 0.8904649615287781, "Finetune/Step": 3095, "Finetune/Step Time": 4.468389833346009} +{"Finetune/Learning Rate": 7.4163228376704e-06, "Finetune/Loss": 0.8774102926254272, "Finetune/Loss (Raw)": 0.9060754776000977, "Finetune/Step": 3096, "Finetune/Step Time": 4.462798733264208} +{"Finetune/Learning Rate": 7.410305244914216e-06, "Finetune/Loss": 0.8730162382125854, "Finetune/Loss (Raw)": 0.30569955706596375, "Finetune/Step": 3097, "Finetune/Step Time": 4.502490069717169} +{"Finetune/Learning Rate": 7.404288657168006e-06, "Finetune/Loss": 0.8715638518333435, "Finetune/Loss (Raw)": 0.7281270623207092, "Finetune/Step": 3098, "Finetune/Step Time": 4.490426652133465} +{"Finetune/Learning Rate": 7.398273076766687e-06, "Finetune/Loss": 0.8718871474266052, "Finetune/Loss (Raw)": 0.93701171875, "Finetune/Step": 3099, "Finetune/Step Time": 4.493928089737892} +{"Finetune/Learning Rate": 7.392258506044796e-06, "Finetune/Loss": 0.873115062713623, "Finetune/Loss (Raw)": 1.084577202796936, "Finetune/Step": 3100, "Finetune/Step Time": 4.496143335476518} +{"Finetune/Learning Rate": 7.386244947336463e-06, "Finetune/Loss": 0.8720475435256958, "Finetune/Loss (Raw)": 0.6513563990592957, "Finetune/Step": 3101, "Finetune/Step Time": 4.4983931221067905} +{"Finetune/Learning Rate": 7.380232402975443e-06, "Finetune/Loss": 0.8723804354667664, "Finetune/Loss (Raw)": 0.9776411652565002, "Finetune/Step": 3102, "Finetune/Step Time": 4.505847645923495} +{"Finetune/Learning Rate": 7.3742208752950776e-06, "Finetune/Loss": 0.8773577809333801, "Finetune/Loss (Raw)": 1.0266746282577515, "Finetune/Step": 3103, "Finetune/Step Time": 4.4803089033812284} +{"Finetune/Learning Rate": 7.3682103666283245e-06, "Finetune/Loss": 0.8781588673591614, "Finetune/Loss (Raw)": 0.9298453330993652, "Finetune/Step": 3104, "Finetune/Step Time": 4.474951162934303} +{"Finetune/Learning Rate": 7.362200879307748e-06, "Finetune/Loss": 0.8780279159545898, "Finetune/Loss (Raw)": 0.9080000519752502, "Finetune/Step": 3105, "Finetune/Step Time": 4.465408286079764} +{"Finetune/Learning Rate": 7.3561924156655105e-06, "Finetune/Loss": 0.876971423625946, "Finetune/Loss (Raw)": 0.8583797812461853, "Finetune/Step": 3106, "Finetune/Step Time": 4.465745326131582} +{"Finetune/Learning Rate": 7.350184978033386e-06, "Finetune/Loss": 0.8778764009475708, "Finetune/Loss (Raw)": 1.0130048990249634, "Finetune/Step": 3107, "Finetune/Step Time": 4.458770763128996} +{"Finetune/Learning Rate": 7.344178568742734e-06, "Finetune/Loss": 0.8776459693908691, "Finetune/Loss (Raw)": 0.8160921931266785, "Finetune/Step": 3108, "Finetune/Step Time": 4.449881309643388} +{"Finetune/Learning Rate": 7.338173190124526e-06, "Finetune/Loss": 0.8785272836685181, "Finetune/Loss (Raw)": 0.8985648155212402, "Finetune/Step": 3109, "Finetune/Step Time": 4.425770554691553} +{"Finetune/Learning Rate": 7.3321688445093395e-06, "Finetune/Loss": 0.8787020444869995, "Finetune/Loss (Raw)": 0.8463898301124573, "Finetune/Step": 3110, "Finetune/Step Time": 4.422983584925532} +{"Finetune/Learning Rate": 7.326165534227341e-06, "Finetune/Loss": 0.8776891827583313, "Finetune/Loss (Raw)": 0.8799799680709839, "Finetune/Step": 3111, "Finetune/Step Time": 4.431978711858392} +{"Finetune/Learning Rate": 7.320163261608293e-06, "Finetune/Loss": 0.8774791955947876, "Finetune/Loss (Raw)": 0.9694387316703796, "Finetune/Step": 3112, "Finetune/Step Time": 4.416612951084971} +{"Finetune/Learning Rate": 7.314162028981564e-06, "Finetune/Loss": 0.8774908781051636, "Finetune/Loss (Raw)": 0.8796243071556091, "Finetune/Step": 3113, "Finetune/Step Time": 4.430075826123357} +{"Finetune/Learning Rate": 7.308161838676113e-06, "Finetune/Loss": 0.8822906017303467, "Finetune/Loss (Raw)": 0.9436999559402466, "Finetune/Step": 3114, "Finetune/Step Time": 4.398207886144519} +{"Finetune/Learning Rate": 7.302162693020503e-06, "Finetune/Loss": 0.8824789524078369, "Finetune/Loss (Raw)": 0.8576666712760925, "Finetune/Step": 3115, "Finetune/Step Time": 4.386333230882883} +{"Finetune/Learning Rate": 7.2961645943428804e-06, "Finetune/Loss": 0.8807333707809448, "Finetune/Loss (Raw)": 0.7287395000457764, "Finetune/Step": 3116, "Finetune/Step Time": 4.413728177547455} +{"Finetune/Learning Rate": 7.290167544970989e-06, "Finetune/Loss": 0.8813596963882446, "Finetune/Loss (Raw)": 0.8996971249580383, "Finetune/Step": 3117, "Finetune/Step Time": 4.40358048863709} +{"Finetune/Learning Rate": 7.284171547232168e-06, "Finetune/Loss": 0.8771405220031738, "Finetune/Loss (Raw)": 0.476594477891922, "Finetune/Step": 3118, "Finetune/Step Time": 4.437769930809736} +{"Finetune/Learning Rate": 7.278176603453347e-06, "Finetune/Loss": 0.8764623403549194, "Finetune/Loss (Raw)": 0.8699166774749756, "Finetune/Step": 3119, "Finetune/Step Time": 4.442962244153023} +{"Finetune/Learning Rate": 7.272182715961049e-06, "Finetune/Loss": 0.8773723244667053, "Finetune/Loss (Raw)": 0.910559892654419, "Finetune/Step": 3120, "Finetune/Step Time": 4.4370098281651735} +{"Finetune/Learning Rate": 7.266189887081385e-06, "Finetune/Loss": 0.8753558993339539, "Finetune/Loss (Raw)": 0.7438146471977234, "Finetune/Step": 3121, "Finetune/Step Time": 4.448279624804854} +{"Finetune/Learning Rate": 7.2601981191400484e-06, "Finetune/Loss": 0.8743083477020264, "Finetune/Loss (Raw)": 0.8974369168281555, "Finetune/Step": 3122, "Finetune/Step Time": 4.461376942694187} +{"Finetune/Learning Rate": 7.254207414462333e-06, "Finetune/Loss": 0.8737750053405762, "Finetune/Loss (Raw)": 0.8390045762062073, "Finetune/Step": 3123, "Finetune/Step Time": 4.455816175788641} +{"Finetune/Learning Rate": 7.248217775373112e-06, "Finetune/Loss": 0.8746370077133179, "Finetune/Loss (Raw)": 0.9929780960083008, "Finetune/Step": 3124, "Finetune/Step Time": 4.445379119366407} +{"Finetune/Learning Rate": 7.242229204196849e-06, "Finetune/Loss": 0.8754073977470398, "Finetune/Loss (Raw)": 0.9331467151641846, "Finetune/Step": 3125, "Finetune/Step Time": 4.4343141950666904} +{"Finetune/Learning Rate": 7.236241703257594e-06, "Finetune/Loss": 0.8761950731277466, "Finetune/Loss (Raw)": 0.9709786176681519, "Finetune/Step": 3126, "Finetune/Step Time": 4.424711547791958} +{"Finetune/Learning Rate": 7.230255274878972e-06, "Finetune/Loss": 0.875723123550415, "Finetune/Loss (Raw)": 1.0072803497314453, "Finetune/Step": 3127, "Finetune/Step Time": 4.420892497524619} +{"Finetune/Learning Rate": 7.224269921384206e-06, "Finetune/Loss": 0.8742200136184692, "Finetune/Loss (Raw)": 0.7049254179000854, "Finetune/Step": 3128, "Finetune/Step Time": 4.457168273627758} +{"Finetune/Learning Rate": 7.218285645096088e-06, "Finetune/Loss": 0.8741244077682495, "Finetune/Loss (Raw)": 0.8525141477584839, "Finetune/Step": 3129, "Finetune/Step Time": 4.453419588506222} +{"Finetune/Learning Rate": 7.2123024483370056e-06, "Finetune/Loss": 0.8732525706291199, "Finetune/Loss (Raw)": 0.8233672976493835, "Finetune/Step": 3130, "Finetune/Step Time": 4.468025114387274} +{"Finetune/Learning Rate": 7.20632033342892e-06, "Finetune/Loss": 0.8730483651161194, "Finetune/Loss (Raw)": 0.8590008616447449, "Finetune/Step": 3131, "Finetune/Step Time": 4.469636771827936} +{"Finetune/Learning Rate": 7.200339302693367e-06, "Finetune/Loss": 0.8689811825752258, "Finetune/Loss (Raw)": 0.41460126638412476, "Finetune/Step": 3132, "Finetune/Step Time": 4.507382970303297} +{"Finetune/Learning Rate": 7.194359358451475e-06, "Finetune/Loss": 0.8705371022224426, "Finetune/Loss (Raw)": 0.9199983477592468, "Finetune/Step": 3133, "Finetune/Step Time": 4.4878016244620085} +{"Finetune/Learning Rate": 7.188380503023936e-06, "Finetune/Loss": 0.8700056672096252, "Finetune/Loss (Raw)": 0.8735266327857971, "Finetune/Step": 3134, "Finetune/Step Time": 4.5028578862547874} +{"Finetune/Learning Rate": 7.182402738731036e-06, "Finetune/Loss": 0.8712900280952454, "Finetune/Loss (Raw)": 0.9369351267814636, "Finetune/Step": 3135, "Finetune/Step Time": 4.50549802929163} +{"Finetune/Learning Rate": 7.176426067892623e-06, "Finetune/Loss": 0.8714675307273865, "Finetune/Loss (Raw)": 0.966094970703125, "Finetune/Step": 3136, "Finetune/Step Time": 4.5070821326226} +{"Finetune/Learning Rate": 7.170450492828126e-06, "Finetune/Loss": 0.8702054023742676, "Finetune/Loss (Raw)": 0.8293867111206055, "Finetune/Step": 3137, "Finetune/Step Time": 4.523257344961166} +{"Finetune/Learning Rate": 7.164476015856552e-06, "Finetune/Loss": 0.8693815469741821, "Finetune/Loss (Raw)": 0.7679640650749207, "Finetune/Step": 3138, "Finetune/Step Time": 4.530745197087526} +{"Finetune/Learning Rate": 7.158502639296475e-06, "Finetune/Loss": 0.8682924509048462, "Finetune/Loss (Raw)": 0.8826888799667358, "Finetune/Step": 3139, "Finetune/Step Time": 4.5322318989783525} +{"Finetune/Learning Rate": 7.152530365466051e-06, "Finetune/Loss": 0.8677777051925659, "Finetune/Loss (Raw)": 0.8773289918899536, "Finetune/Step": 3140, "Finetune/Step Time": 4.543031135573983} +{"Finetune/Learning Rate": 7.146559196683001e-06, "Finetune/Loss": 0.8673101663589478, "Finetune/Loss (Raw)": 0.8351744413375854, "Finetune/Step": 3141, "Finetune/Step Time": 4.5448431186378} +{"Finetune/Learning Rate": 7.140589135264614e-06, "Finetune/Loss": 0.8672614097595215, "Finetune/Loss (Raw)": 0.8452910780906677, "Finetune/Step": 3142, "Finetune/Step Time": 4.549573794007301} +{"Finetune/Learning Rate": 7.13462018352776e-06, "Finetune/Loss": 0.8666418790817261, "Finetune/Loss (Raw)": 0.6930184960365295, "Finetune/Step": 3143, "Finetune/Step Time": 4.548968875780702} +{"Finetune/Learning Rate": 7.12865234378887e-06, "Finetune/Loss": 0.867356538772583, "Finetune/Loss (Raw)": 0.994260847568512, "Finetune/Step": 3144, "Finetune/Step Time": 4.540828613564372} +{"Finetune/Learning Rate": 7.122685618363948e-06, "Finetune/Loss": 0.8689866662025452, "Finetune/Loss (Raw)": 0.984208345413208, "Finetune/Step": 3145, "Finetune/Step Time": 4.535619894042611} +{"Finetune/Learning Rate": 7.116720009568565e-06, "Finetune/Loss": 0.8696955442428589, "Finetune/Loss (Raw)": 0.9395970702171326, "Finetune/Step": 3146, "Finetune/Step Time": 4.541650542989373} +{"Finetune/Learning Rate": 7.110755519717851e-06, "Finetune/Loss": 0.8700354099273682, "Finetune/Loss (Raw)": 0.9804561734199524, "Finetune/Step": 3147, "Finetune/Step Time": 4.548892671242356} +{"Finetune/Learning Rate": 7.104792151126515e-06, "Finetune/Loss": 0.8713542222976685, "Finetune/Loss (Raw)": 1.0460339784622192, "Finetune/Step": 3148, "Finetune/Step Time": 4.569311656057835} +{"Finetune/Learning Rate": 7.098829906108819e-06, "Finetune/Loss": 0.8677108287811279, "Finetune/Loss (Raw)": 0.42205119132995605, "Finetune/Step": 3149, "Finetune/Step Time": 4.599383583292365} +{"Finetune/Learning Rate": 7.092868786978596e-06, "Finetune/Loss": 0.8670399188995361, "Finetune/Loss (Raw)": 0.8876112103462219, "Finetune/Step": 3150, "Finetune/Step Time": 4.596313875168562} +{"Finetune/Learning Rate": 7.086908796049243e-06, "Finetune/Loss": 0.8682639598846436, "Finetune/Loss (Raw)": 0.8945993185043335, "Finetune/Step": 3151, "Finetune/Step Time": 4.597993498668075} +{"Finetune/Learning Rate": 7.08094993563371e-06, "Finetune/Loss": 0.8675239682197571, "Finetune/Loss (Raw)": 0.8285226225852966, "Finetune/Step": 3152, "Finetune/Step Time": 4.593014029785991} +{"Finetune/Learning Rate": 7.074992208044518e-06, "Finetune/Loss": 0.8674513101577759, "Finetune/Loss (Raw)": 0.9218892455101013, "Finetune/Step": 3153, "Finetune/Step Time": 4.593112554401159} +{"Finetune/Learning Rate": 7.069035615593743e-06, "Finetune/Loss": 0.8671072125434875, "Finetune/Loss (Raw)": 0.8925013542175293, "Finetune/Step": 3154, "Finetune/Step Time": 4.585093043744564} +{"Finetune/Learning Rate": 7.063080160593026e-06, "Finetune/Loss": 0.8669512271881104, "Finetune/Loss (Raw)": 0.965580940246582, "Finetune/Step": 3155, "Finetune/Step Time": 4.579166740179062} +{"Finetune/Learning Rate": 7.0571258453535585e-06, "Finetune/Loss": 0.8662635684013367, "Finetune/Loss (Raw)": 0.8124980926513672, "Finetune/Step": 3156, "Finetune/Step Time": 4.574045471847057} +{"Finetune/Learning Rate": 7.051172672186093e-06, "Finetune/Loss": 0.8660038709640503, "Finetune/Loss (Raw)": 1.0161727666854858, "Finetune/Step": 3157, "Finetune/Step Time": 4.579760886728764} +{"Finetune/Learning Rate": 7.045220643400946e-06, "Finetune/Loss": 0.8612520694732666, "Finetune/Loss (Raw)": 0.2857050895690918, "Finetune/Step": 3158, "Finetune/Step Time": 4.604577086865902} +{"Finetune/Learning Rate": 7.039269761307977e-06, "Finetune/Loss": 0.8608278036117554, "Finetune/Loss (Raw)": 0.8593589067459106, "Finetune/Step": 3159, "Finetune/Step Time": 4.608272913843393} +{"Finetune/Learning Rate": 7.0333200282166146e-06, "Finetune/Loss": 0.859963595867157, "Finetune/Loss (Raw)": 0.871728777885437, "Finetune/Step": 3160, "Finetune/Step Time": 4.5984416250139475} +{"Finetune/Learning Rate": 7.027371446435829e-06, "Finetune/Loss": 0.8599468469619751, "Finetune/Loss (Raw)": 0.9070864319801331, "Finetune/Step": 3161, "Finetune/Step Time": 4.617932872846723} +{"Finetune/Learning Rate": 7.0214240182741455e-06, "Finetune/Loss": 0.8549922704696655, "Finetune/Loss (Raw)": 0.41015344858169556, "Finetune/Step": 3162, "Finetune/Step Time": 4.656084347516298} +{"Finetune/Learning Rate": 7.015477746039652e-06, "Finetune/Loss": 0.8561410307884216, "Finetune/Loss (Raw)": 1.0446269512176514, "Finetune/Step": 3163, "Finetune/Step Time": 4.66711045615375} +{"Finetune/Learning Rate": 7.009532632039975e-06, "Finetune/Loss": 0.8567062616348267, "Finetune/Loss (Raw)": 0.9534891843795776, "Finetune/Step": 3164, "Finetune/Step Time": 4.647651636973023} +{"Finetune/Learning Rate": 7.003588678582303e-06, "Finetune/Loss": 0.8541951179504395, "Finetune/Loss (Raw)": 0.6988575458526611, "Finetune/Step": 3165, "Finetune/Step Time": 4.649800660088658} +{"Finetune/Learning Rate": 6.9976458879733635e-06, "Finetune/Loss": 0.8529068231582642, "Finetune/Loss (Raw)": 0.8264652490615845, "Finetune/Step": 3166, "Finetune/Step Time": 4.650467481464148} +{"Finetune/Learning Rate": 6.991704262519436e-06, "Finetune/Loss": 0.8535466194152832, "Finetune/Loss (Raw)": 0.9178164005279541, "Finetune/Step": 3167, "Finetune/Step Time": 4.648204578086734} +{"Finetune/Learning Rate": 6.985763804526354e-06, "Finetune/Loss": 0.8542360067367554, "Finetune/Loss (Raw)": 1.0210819244384766, "Finetune/Step": 3168, "Finetune/Step Time": 4.6539400443434715} +{"Finetune/Learning Rate": 6.979824516299491e-06, "Finetune/Loss": 0.8506449460983276, "Finetune/Loss (Raw)": 0.31568747758865356, "Finetune/Step": 3169, "Finetune/Step Time": 4.688798885792494} +{"Finetune/Learning Rate": 6.973886400143772e-06, "Finetune/Loss": 0.8495283722877502, "Finetune/Loss (Raw)": 0.7843942046165466, "Finetune/Step": 3170, "Finetune/Step Time": 4.689479045569897} +{"Finetune/Learning Rate": 6.9679494583636595e-06, "Finetune/Loss": 0.8535116314888, "Finetune/Loss (Raw)": 0.8197912573814392, "Finetune/Step": 3171, "Finetune/Step Time": 4.65392703935504} +{"Finetune/Learning Rate": 6.9620136932631655e-06, "Finetune/Loss": 0.8547903299331665, "Finetune/Loss (Raw)": 0.8896117806434631, "Finetune/Step": 3172, "Finetune/Step Time": 4.635604726150632} +{"Finetune/Learning Rate": 6.956079107145845e-06, "Finetune/Loss": 0.8546679615974426, "Finetune/Loss (Raw)": 0.7945153117179871, "Finetune/Step": 3173, "Finetune/Step Time": 4.646698663011193} +{"Finetune/Learning Rate": 6.950145702314795e-06, "Finetune/Loss": 0.8546568751335144, "Finetune/Loss (Raw)": 1.0407969951629639, "Finetune/Step": 3174, "Finetune/Step Time": 4.652113115414977} +{"Finetune/Learning Rate": 6.9442134810726615e-06, "Finetune/Loss": 0.8548846244812012, "Finetune/Loss (Raw)": 0.8888436555862427, "Finetune/Step": 3175, "Finetune/Step Time": 4.648656256496906} +{"Finetune/Learning Rate": 6.938282445721609e-06, "Finetune/Loss": 0.8548808097839355, "Finetune/Loss (Raw)": 1.0403075218200684, "Finetune/Step": 3176, "Finetune/Step Time": 4.65052112005651} +{"Finetune/Learning Rate": 6.932352598563365e-06, "Finetune/Loss": 0.8546494841575623, "Finetune/Loss (Raw)": 0.8593615293502808, "Finetune/Step": 3177, "Finetune/Step Time": 4.648483334109187} +{"Finetune/Learning Rate": 6.92642394189919e-06, "Finetune/Loss": 0.8535630702972412, "Finetune/Loss (Raw)": 0.8606176972389221, "Finetune/Step": 3178, "Finetune/Step Time": 4.654590431600809} +{"Finetune/Learning Rate": 6.920496478029873e-06, "Finetune/Loss": 0.8538714647293091, "Finetune/Loss (Raw)": 1.0762122869491577, "Finetune/Step": 3179, "Finetune/Step Time": 4.6606156174093485} +{"Finetune/Learning Rate": 6.914570209255758e-06, "Finetune/Loss": 0.8534029722213745, "Finetune/Loss (Raw)": 0.8217430114746094, "Finetune/Step": 3180, "Finetune/Step Time": 4.650770446285605} +{"Finetune/Learning Rate": 6.908645137876699e-06, "Finetune/Loss": 0.8532166481018066, "Finetune/Loss (Raw)": 0.9537642598152161, "Finetune/Step": 3181, "Finetune/Step Time": 4.646081319078803} +{"Finetune/Learning Rate": 6.902721266192112e-06, "Finetune/Loss": 0.8524528741836548, "Finetune/Loss (Raw)": 0.9149432182312012, "Finetune/Step": 3182, "Finetune/Step Time": 4.649190973490477} +{"Finetune/Learning Rate": 6.896798596500929e-06, "Finetune/Loss": 0.8518804907798767, "Finetune/Loss (Raw)": 0.8614248037338257, "Finetune/Step": 3183, "Finetune/Step Time": 4.649170149117708} +{"Finetune/Learning Rate": 6.8908771311016276e-06, "Finetune/Loss": 0.8513176441192627, "Finetune/Loss (Raw)": 0.9115465879440308, "Finetune/Step": 3184, "Finetune/Step Time": 4.659715391695499} +{"Finetune/Learning Rate": 6.884956872292216e-06, "Finetune/Loss": 0.8523187637329102, "Finetune/Loss (Raw)": 0.9264922142028809, "Finetune/Step": 3185, "Finetune/Step Time": 4.653742134571075} +{"Finetune/Learning Rate": 6.87903782237022e-06, "Finetune/Loss": 0.8512781262397766, "Finetune/Loss (Raw)": 0.8511121869087219, "Finetune/Step": 3186, "Finetune/Step Time": 4.652589853852987} +{"Finetune/Learning Rate": 6.8731199836327155e-06, "Finetune/Loss": 0.8512437343597412, "Finetune/Loss (Raw)": 0.9459890723228455, "Finetune/Step": 3187, "Finetune/Step Time": 4.656191416084766} +{"Finetune/Learning Rate": 6.867203358376295e-06, "Finetune/Loss": 0.8503068685531616, "Finetune/Loss (Raw)": 0.8591833114624023, "Finetune/Step": 3188, "Finetune/Step Time": 4.6699891500175} +{"Finetune/Learning Rate": 6.861287948897091e-06, "Finetune/Loss": 0.8494793176651001, "Finetune/Loss (Raw)": 0.8495504856109619, "Finetune/Step": 3189, "Finetune/Step Time": 4.672247558832169} +{"Finetune/Learning Rate": 6.855373757490761e-06, "Finetune/Loss": 0.8499125242233276, "Finetune/Loss (Raw)": 0.9114967584609985, "Finetune/Step": 3190, "Finetune/Step Time": 4.642444059252739} +{"Finetune/Learning Rate": 6.849460786452476e-06, "Finetune/Loss": 0.8506805300712585, "Finetune/Loss (Raw)": 0.9657304883003235, "Finetune/Step": 3191, "Finetune/Step Time": 4.6726033464074135} +{"Finetune/Learning Rate": 6.843549038076951e-06, "Finetune/Loss": 0.8501967191696167, "Finetune/Loss (Raw)": 0.8493885397911072, "Finetune/Step": 3192, "Finetune/Step Time": 4.678045354783535} +{"Finetune/Learning Rate": 6.83763851465842e-06, "Finetune/Loss": 0.8506315350532532, "Finetune/Loss (Raw)": 0.8596172332763672, "Finetune/Step": 3193, "Finetune/Step Time": 4.6623147912323475} +{"Finetune/Learning Rate": 6.831729218490645e-06, "Finetune/Loss": 0.8512527942657471, "Finetune/Loss (Raw)": 0.7759203910827637, "Finetune/Step": 3194, "Finetune/Step Time": 4.634819120168686} +{"Finetune/Learning Rate": 6.825821151866903e-06, "Finetune/Loss": 0.8519473075866699, "Finetune/Loss (Raw)": 0.9766781330108643, "Finetune/Step": 3195, "Finetune/Step Time": 4.6415996588766575} +{"Finetune/Learning Rate": 6.819914317079999e-06, "Finetune/Loss": 0.8522046804428101, "Finetune/Loss (Raw)": 0.920444905757904, "Finetune/Step": 3196, "Finetune/Step Time": 4.631092531606555} +{"Finetune/Learning Rate": 6.814008716422265e-06, "Finetune/Loss": 0.8537710905075073, "Finetune/Loss (Raw)": 0.9906922578811646, "Finetune/Step": 3197, "Finetune/Step Time": 4.622870244085789} +{"Finetune/Learning Rate": 6.808104352185544e-06, "Finetune/Loss": 0.8527005910873413, "Finetune/Loss (Raw)": 0.7902165055274963, "Finetune/Step": 3198, "Finetune/Step Time": 4.631640309467912} +{"Finetune/Learning Rate": 6.8022012266612116e-06, "Finetune/Loss": 0.8518937230110168, "Finetune/Loss (Raw)": 0.9114668965339661, "Finetune/Step": 3199, "Finetune/Step Time": 4.631051525473595} +{"Finetune/Learning Rate": 6.79629934214015e-06, "Finetune/Loss": 0.8535261750221252, "Finetune/Loss (Raw)": 1.0733139514923096, "Finetune/Step": 3200, "Finetune/Step Time": 4.62055230140686} +{"Finetune/Learning Rate": 6.790398700912764e-06, "Finetune/Loss": 0.852757453918457, "Finetune/Loss (Raw)": 0.7960576415061951, "Finetune/Step": 3201, "Finetune/Step Time": 4.616136379539967} +{"Finetune/Learning Rate": 6.784499305268983e-06, "Finetune/Loss": 0.8527076244354248, "Finetune/Loss (Raw)": 0.8536461591720581, "Finetune/Step": 3202, "Finetune/Step Time": 4.61251262947917} +{"Finetune/Learning Rate": 6.778601157498241e-06, "Finetune/Loss": 0.8570322394371033, "Finetune/Loss (Raw)": 0.8481436967849731, "Finetune/Step": 3203, "Finetune/Step Time": 4.5737405605614185} +{"Finetune/Learning Rate": 6.772704259889503e-06, "Finetune/Loss": 0.857621967792511, "Finetune/Loss (Raw)": 0.8374922871589661, "Finetune/Step": 3204, "Finetune/Step Time": 4.575168590992689} +{"Finetune/Learning Rate": 6.76680861473123e-06, "Finetune/Loss": 0.8567031025886536, "Finetune/Loss (Raw)": 0.8218506574630737, "Finetune/Step": 3205, "Finetune/Step Time": 4.571768049150705} +{"Finetune/Learning Rate": 6.7609142243114115e-06, "Finetune/Loss": 0.8564666509628296, "Finetune/Loss (Raw)": 0.7857659459114075, "Finetune/Step": 3206, "Finetune/Step Time": 4.573037035763264} +{"Finetune/Learning Rate": 6.755021090917545e-06, "Finetune/Loss": 0.857608437538147, "Finetune/Loss (Raw)": 0.9172974228858948, "Finetune/Step": 3207, "Finetune/Step Time": 4.577583022415638} +{"Finetune/Learning Rate": 6.7491292168366386e-06, "Finetune/Loss": 0.8591232299804688, "Finetune/Loss (Raw)": 0.9577879309654236, "Finetune/Step": 3208, "Finetune/Step Time": 4.571144657209516} +{"Finetune/Learning Rate": 6.743238604355219e-06, "Finetune/Loss": 0.8594532012939453, "Finetune/Loss (Raw)": 0.855830192565918, "Finetune/Step": 3209, "Finetune/Step Time": 4.57906080968678} +{"Finetune/Learning Rate": 6.737349255759312e-06, "Finetune/Loss": 0.8587996959686279, "Finetune/Loss (Raw)": 0.7900684475898743, "Finetune/Step": 3210, "Finetune/Step Time": 4.5778783634305} +{"Finetune/Learning Rate": 6.731461173334459e-06, "Finetune/Loss": 0.858759880065918, "Finetune/Loss (Raw)": 0.9358519315719604, "Finetune/Step": 3211, "Finetune/Step Time": 4.5665187649428844} +{"Finetune/Learning Rate": 6.725574359365714e-06, "Finetune/Loss": 0.8554818630218506, "Finetune/Loss (Raw)": 0.3977698087692261, "Finetune/Step": 3212, "Finetune/Step Time": 4.601005896925926} +{"Finetune/Learning Rate": 6.7196888161376305e-06, "Finetune/Loss": 0.8544995784759521, "Finetune/Loss (Raw)": 0.9500510096549988, "Finetune/Step": 3213, "Finetune/Step Time": 4.598939668387175} +{"Finetune/Learning Rate": 6.71380454593428e-06, "Finetune/Loss": 0.8541580438613892, "Finetune/Loss (Raw)": 0.9299262166023254, "Finetune/Step": 3214, "Finetune/Step Time": 4.6114579271525145} +{"Finetune/Learning Rate": 6.707921551039224e-06, "Finetune/Loss": 0.8533411026000977, "Finetune/Loss (Raw)": 0.7952885031700134, "Finetune/Step": 3215, "Finetune/Step Time": 4.619906475767493} +{"Finetune/Learning Rate": 6.70203983373554e-06, "Finetune/Loss": 0.8535022139549255, "Finetune/Loss (Raw)": 0.7359476089477539, "Finetune/Step": 3216, "Finetune/Step Time": 4.640254553407431} +{"Finetune/Learning Rate": 6.69615939630581e-06, "Finetune/Loss": 0.8532793521881104, "Finetune/Loss (Raw)": 0.9163236618041992, "Finetune/Step": 3217, "Finetune/Step Time": 4.631551554426551} +{"Finetune/Learning Rate": 6.690280241032117e-06, "Finetune/Loss": 0.8539862036705017, "Finetune/Loss (Raw)": 0.9189110398292542, "Finetune/Step": 3218, "Finetune/Step Time": 4.629283873364329} +{"Finetune/Learning Rate": 6.684402370196048e-06, "Finetune/Loss": 0.8535493016242981, "Finetune/Loss (Raw)": 0.7745267152786255, "Finetune/Step": 3219, "Finetune/Step Time": 4.613457731902599} +{"Finetune/Learning Rate": 6.678525786078684e-06, "Finetune/Loss": 0.8595343828201294, "Finetune/Loss (Raw)": 1.0677968263626099, "Finetune/Step": 3220, "Finetune/Step Time": 4.574255494400859} +{"Finetune/Learning Rate": 6.672650490960617e-06, "Finetune/Loss": 0.8590535521507263, "Finetune/Loss (Raw)": 0.8288148045539856, "Finetune/Step": 3221, "Finetune/Step Time": 4.578092595562339} +{"Finetune/Learning Rate": 6.666776487121934e-06, "Finetune/Loss": 0.8545514345169067, "Finetune/Loss (Raw)": 0.33394771814346313, "Finetune/Step": 3222, "Finetune/Step Time": 4.61468324624002} +{"Finetune/Learning Rate": 6.660903776842218e-06, "Finetune/Loss": 0.8542541265487671, "Finetune/Loss (Raw)": 0.8524160385131836, "Finetune/Step": 3223, "Finetune/Step Time": 4.620458446443081} +{"Finetune/Learning Rate": 6.655032362400562e-06, "Finetune/Loss": 0.8543820381164551, "Finetune/Loss (Raw)": 0.9224395155906677, "Finetune/Step": 3224, "Finetune/Step Time": 4.621560389176011} +{"Finetune/Learning Rate": 6.649162246075538e-06, "Finetune/Loss": 0.8588187098503113, "Finetune/Loss (Raw)": 0.873595654964447, "Finetune/Step": 3225, "Finetune/Step Time": 4.586770884692669} +{"Finetune/Learning Rate": 6.6432934301452235e-06, "Finetune/Loss": 0.8601312637329102, "Finetune/Loss (Raw)": 0.8961378335952759, "Finetune/Step": 3226, "Finetune/Step Time": 4.585469836369157} +{"Finetune/Learning Rate": 6.637425916887198e-06, "Finetune/Loss": 0.8593603372573853, "Finetune/Loss (Raw)": 0.8383285403251648, "Finetune/Step": 3227, "Finetune/Step Time": 4.585948929190636} +{"Finetune/Learning Rate": 6.631559708578524e-06, "Finetune/Loss": 0.8576998710632324, "Finetune/Loss (Raw)": 0.8720329403877258, "Finetune/Step": 3228, "Finetune/Step Time": 4.583467474207282} +{"Finetune/Learning Rate": 6.625694807495766e-06, "Finetune/Loss": 0.8602004051208496, "Finetune/Loss (Raw)": 0.9714304804801941, "Finetune/Step": 3229, "Finetune/Step Time": 4.5678123366087675} +{"Finetune/Learning Rate": 6.619831215914974e-06, "Finetune/Loss": 0.8591259717941284, "Finetune/Loss (Raw)": 0.840106189250946, "Finetune/Step": 3230, "Finetune/Step Time": 4.570329798385501} +{"Finetune/Learning Rate": 6.6139689361116945e-06, "Finetune/Loss": 0.8579990863800049, "Finetune/Loss (Raw)": 0.8824453353881836, "Finetune/Step": 3231, "Finetune/Step Time": 4.583062136545777} +{"Finetune/Learning Rate": 6.608107970360964e-06, "Finetune/Loss": 0.8586846590042114, "Finetune/Loss (Raw)": 1.0175930261611938, "Finetune/Step": 3232, "Finetune/Step Time": 4.584962420165539} +{"Finetune/Learning Rate": 6.602248320937308e-06, "Finetune/Loss": 0.8602712154388428, "Finetune/Loss (Raw)": 1.111084222793579, "Finetune/Step": 3233, "Finetune/Step Time": 4.587967442348599} +{"Finetune/Learning Rate": 6.596389990114748e-06, "Finetune/Loss": 0.8596376776695251, "Finetune/Loss (Raw)": 0.7772830724716187, "Finetune/Step": 3234, "Finetune/Step Time": 4.609602468088269} +{"Finetune/Learning Rate": 6.5905329801667796e-06, "Finetune/Loss": 0.8597671985626221, "Finetune/Loss (Raw)": 1.0295809507369995, "Finetune/Step": 3235, "Finetune/Step Time": 4.614231372252107} +{"Finetune/Learning Rate": 6.584677293366396e-06, "Finetune/Loss": 0.8592350482940674, "Finetune/Loss (Raw)": 0.7479845881462097, "Finetune/Step": 3236, "Finetune/Step Time": 4.621369909495115} +{"Finetune/Learning Rate": 6.578822931986078e-06, "Finetune/Loss": 0.8598198294639587, "Finetune/Loss (Raw)": 0.973413348197937, "Finetune/Step": 3237, "Finetune/Step Time": 4.619200302287936} +{"Finetune/Learning Rate": 6.57296989829779e-06, "Finetune/Loss": 0.8563281297683716, "Finetune/Loss (Raw)": 0.399448037147522, "Finetune/Step": 3238, "Finetune/Step Time": 4.658881204202771} +{"Finetune/Learning Rate": 6.567118194572972e-06, "Finetune/Loss": 0.857474148273468, "Finetune/Loss (Raw)": 1.0266741514205933, "Finetune/Step": 3239, "Finetune/Step Time": 4.654001595452428} +{"Finetune/Learning Rate": 6.5612678230825646e-06, "Finetune/Loss": 0.8583678007125854, "Finetune/Loss (Raw)": 1.083818793296814, "Finetune/Step": 3240, "Finetune/Step Time": 4.655331330373883} +{"Finetune/Learning Rate": 6.555418786096976e-06, "Finetune/Loss": 0.8590301275253296, "Finetune/Loss (Raw)": 0.9644039869308472, "Finetune/Step": 3241, "Finetune/Step Time": 4.644135899841785} +{"Finetune/Learning Rate": 6.549571085886111e-06, "Finetune/Loss": 0.859308123588562, "Finetune/Loss (Raw)": 0.9792855978012085, "Finetune/Step": 3242, "Finetune/Step Time": 4.64097517170012} +{"Finetune/Learning Rate": 6.5437247247193445e-06, "Finetune/Loss": 0.8600243330001831, "Finetune/Loss (Raw)": 0.9493420124053955, "Finetune/Step": 3243, "Finetune/Step Time": 4.633911389857531} +{"Finetune/Learning Rate": 6.537879704865531e-06, "Finetune/Loss": 0.8619075417518616, "Finetune/Loss (Raw)": 0.9697899222373962, "Finetune/Step": 3244, "Finetune/Step Time": 4.597848501056433} +{"Finetune/Learning Rate": 6.532036028593012e-06, "Finetune/Loss": 0.8626919984817505, "Finetune/Loss (Raw)": 1.0001181364059448, "Finetune/Step": 3245, "Finetune/Step Time": 4.6029105465859175} +{"Finetune/Learning Rate": 6.526193698169603e-06, "Finetune/Loss": 0.8646512031555176, "Finetune/Loss (Raw)": 0.7273645401000977, "Finetune/Step": 3246, "Finetune/Step Time": 4.573611229658127} +{"Finetune/Learning Rate": 6.520352715862601e-06, "Finetune/Loss": 0.8645577430725098, "Finetune/Loss (Raw)": 0.8579500317573547, "Finetune/Step": 3247, "Finetune/Step Time": 4.567985612899065} +{"Finetune/Learning Rate": 6.514513083938779e-06, "Finetune/Loss": 0.8652554750442505, "Finetune/Loss (Raw)": 0.999871551990509, "Finetune/Step": 3248, "Finetune/Step Time": 4.571738950908184} +{"Finetune/Learning Rate": 6.508674804664374e-06, "Finetune/Loss": 0.8672617673873901, "Finetune/Loss (Raw)": 1.0006295442581177, "Finetune/Step": 3249, "Finetune/Step Time": 4.561047326773405} +{"Finetune/Learning Rate": 6.50283788030512e-06, "Finetune/Loss": 0.8669174909591675, "Finetune/Loss (Raw)": 0.8533632755279541, "Finetune/Step": 3250, "Finetune/Step Time": 4.568983485922217} +{"Finetune/Learning Rate": 6.497002313126204e-06, "Finetune/Loss": 0.8670367002487183, "Finetune/Loss (Raw)": 0.8542647957801819, "Finetune/Step": 3251, "Finetune/Step Time": 4.596967479214072} +{"Finetune/Learning Rate": 6.491168105392303e-06, "Finetune/Loss": 0.867350697517395, "Finetune/Loss (Raw)": 1.0331681966781616, "Finetune/Step": 3252, "Finetune/Step Time": 4.5915312096476555} +{"Finetune/Learning Rate": 6.485335259367558e-06, "Finetune/Loss": 0.8678308725357056, "Finetune/Loss (Raw)": 0.9946095943450928, "Finetune/Step": 3253, "Finetune/Step Time": 4.594026623293757} +{"Finetune/Learning Rate": 6.4795037773155765e-06, "Finetune/Loss": 0.8676614761352539, "Finetune/Loss (Raw)": 0.9492886662483215, "Finetune/Step": 3254, "Finetune/Step Time": 4.604781907051802} +{"Finetune/Learning Rate": 6.473673661499451e-06, "Finetune/Loss": 0.8669856786727905, "Finetune/Loss (Raw)": 0.9207892417907715, "Finetune/Step": 3255, "Finetune/Step Time": 4.621291182935238} +{"Finetune/Learning Rate": 6.467844914181727e-06, "Finetune/Loss": 0.8692944049835205, "Finetune/Loss (Raw)": 1.0004364252090454, "Finetune/Step": 3256, "Finetune/Step Time": 4.587141778320074} +{"Finetune/Learning Rate": 6.462017537624437e-06, "Finetune/Loss": 0.868699312210083, "Finetune/Loss (Raw)": 0.7763406038284302, "Finetune/Step": 3257, "Finetune/Step Time": 4.597157757729292} +{"Finetune/Learning Rate": 6.4561915340890665e-06, "Finetune/Loss": 0.870661735534668, "Finetune/Loss (Raw)": 1.0745618343353271, "Finetune/Step": 3258, "Finetune/Step Time": 4.585850825533271} +{"Finetune/Learning Rate": 6.450366905836573e-06, "Finetune/Loss": 0.8706202507019043, "Finetune/Loss (Raw)": 0.8536888957023621, "Finetune/Step": 3259, "Finetune/Step Time": 4.594760486856103} +{"Finetune/Learning Rate": 6.444543655127383e-06, "Finetune/Loss": 0.8749023079872131, "Finetune/Loss (Raw)": 0.9627074599266052, "Finetune/Step": 3260, "Finetune/Step Time": 4.573740936815739} +{"Finetune/Learning Rate": 6.438721784221385e-06, "Finetune/Loss": 0.8740867972373962, "Finetune/Loss (Raw)": 0.8156136870384216, "Finetune/Step": 3261, "Finetune/Step Time": 4.574244376271963} +{"Finetune/Learning Rate": 6.432901295377936e-06, "Finetune/Loss": 0.87493896484375, "Finetune/Loss (Raw)": 0.9826014041900635, "Finetune/Step": 3262, "Finetune/Step Time": 4.560746729373932} +{"Finetune/Learning Rate": 6.427082190855855e-06, "Finetune/Loss": 0.875443696975708, "Finetune/Loss (Raw)": 1.0015441179275513, "Finetune/Step": 3263, "Finetune/Step Time": 4.558682767674327} +{"Finetune/Learning Rate": 6.4212644729134155e-06, "Finetune/Loss": 0.874602198600769, "Finetune/Loss (Raw)": 0.8583859205245972, "Finetune/Step": 3264, "Finetune/Step Time": 4.559451840817928} +{"Finetune/Learning Rate": 6.415448143808367e-06, "Finetune/Loss": 0.8765435814857483, "Finetune/Loss (Raw)": 1.0778779983520508, "Finetune/Step": 3265, "Finetune/Step Time": 4.542981997132301} +{"Finetune/Learning Rate": 6.409633205797911e-06, "Finetune/Loss": 0.877505898475647, "Finetune/Loss (Raw)": 0.8911392688751221, "Finetune/Step": 3266, "Finetune/Step Time": 4.537230383604765} +{"Finetune/Learning Rate": 6.403819661138712e-06, "Finetune/Loss": 0.8777052164077759, "Finetune/Loss (Raw)": 0.908199667930603, "Finetune/Step": 3267, "Finetune/Step Time": 4.54536147788167} +{"Finetune/Learning Rate": 6.398007512086897e-06, "Finetune/Loss": 0.8784105777740479, "Finetune/Loss (Raw)": 0.9676194190979004, "Finetune/Step": 3268, "Finetune/Step Time": 4.533696975558996} +{"Finetune/Learning Rate": 6.3921967608980375e-06, "Finetune/Loss": 0.8794434070587158, "Finetune/Loss (Raw)": 0.9673764705657959, "Finetune/Step": 3269, "Finetune/Step Time": 4.536382272839546} +{"Finetune/Learning Rate": 6.386387409827182e-06, "Finetune/Loss": 0.8794189095497131, "Finetune/Loss (Raw)": 0.8421496748924255, "Finetune/Step": 3270, "Finetune/Step Time": 4.531384689733386} +{"Finetune/Learning Rate": 6.38057946112882e-06, "Finetune/Loss": 0.8808097839355469, "Finetune/Loss (Raw)": 0.8710593581199646, "Finetune/Step": 3271, "Finetune/Step Time": 4.515749130398035} +{"Finetune/Learning Rate": 6.374772917056908e-06, "Finetune/Loss": 0.880496621131897, "Finetune/Loss (Raw)": 0.9541773200035095, "Finetune/Step": 3272, "Finetune/Step Time": 4.544583179056644} +{"Finetune/Learning Rate": 6.368967779864854e-06, "Finetune/Loss": 0.8790717720985413, "Finetune/Loss (Raw)": 0.80182284116745, "Finetune/Step": 3273, "Finetune/Step Time": 4.550127319991589} +{"Finetune/Learning Rate": 6.3631640518055085e-06, "Finetune/Loss": 0.8742650151252747, "Finetune/Loss (Raw)": 0.32433271408081055, "Finetune/Step": 3274, "Finetune/Step Time": 4.582748094573617} +{"Finetune/Learning Rate": 6.3573617351311945e-06, "Finetune/Loss": 0.872727632522583, "Finetune/Loss (Raw)": 0.7836671471595764, "Finetune/Step": 3275, "Finetune/Step Time": 4.580677980557084} +{"Finetune/Learning Rate": 6.351560832093673e-06, "Finetune/Loss": 0.8715580701828003, "Finetune/Loss (Raw)": 0.8963340520858765, "Finetune/Step": 3276, "Finetune/Step Time": 4.571049157530069} +{"Finetune/Learning Rate": 6.3457613449441654e-06, "Finetune/Loss": 0.8756489157676697, "Finetune/Loss (Raw)": 0.9456784129142761, "Finetune/Step": 3277, "Finetune/Step Time": 4.531779561191797} +{"Finetune/Learning Rate": 6.3399632759333364e-06, "Finetune/Loss": 0.8753964304924011, "Finetune/Loss (Raw)": 0.8552870750427246, "Finetune/Step": 3278, "Finetune/Step Time": 4.535148844122887} +{"Finetune/Learning Rate": 6.3341666273113e-06, "Finetune/Loss": 0.8759377598762512, "Finetune/Loss (Raw)": 0.9638956189155579, "Finetune/Step": 3279, "Finetune/Step Time": 4.5425501354038715} +{"Finetune/Learning Rate": 6.328371401327629e-06, "Finetune/Loss": 0.8772400617599487, "Finetune/Loss (Raw)": 0.9952165484428406, "Finetune/Step": 3280, "Finetune/Step Time": 4.541137954220176} +{"Finetune/Learning Rate": 6.322577600231332e-06, "Finetune/Loss": 0.8780661821365356, "Finetune/Loss (Raw)": 1.02763032913208, "Finetune/Step": 3281, "Finetune/Step Time": 4.548322984948754} +{"Finetune/Learning Rate": 6.316785226270878e-06, "Finetune/Loss": 0.8778157234191895, "Finetune/Loss (Raw)": 0.8604395389556885, "Finetune/Step": 3282, "Finetune/Step Time": 4.549645129591227} +{"Finetune/Learning Rate": 6.310994281694167e-06, "Finetune/Loss": 0.873171865940094, "Finetune/Loss (Raw)": 0.3711732029914856, "Finetune/Step": 3283, "Finetune/Step Time": 4.587393453344703} +{"Finetune/Learning Rate": 6.305204768748551e-06, "Finetune/Loss": 0.873611330986023, "Finetune/Loss (Raw)": 0.8687474727630615, "Finetune/Step": 3284, "Finetune/Step Time": 4.590575659647584} +{"Finetune/Learning Rate": 6.299416689680831e-06, "Finetune/Loss": 0.8732814788818359, "Finetune/Loss (Raw)": 0.9739511013031006, "Finetune/Step": 3285, "Finetune/Step Time": 4.58412024192512} +{"Finetune/Learning Rate": 6.293630046737245e-06, "Finetune/Loss": 0.8778011202812195, "Finetune/Loss (Raw)": 0.86421799659729, "Finetune/Step": 3286, "Finetune/Step Time": 4.551757454872131} +{"Finetune/Learning Rate": 6.28784484216348e-06, "Finetune/Loss": 0.8782260417938232, "Finetune/Loss (Raw)": 0.9137583374977112, "Finetune/Step": 3287, "Finetune/Step Time": 4.558665607124567} +{"Finetune/Learning Rate": 6.282061078204657e-06, "Finetune/Loss": 0.8790902495384216, "Finetune/Loss (Raw)": 0.9823417067527771, "Finetune/Step": 3288, "Finetune/Step Time": 4.564129687845707} +{"Finetune/Learning Rate": 6.276278757105341e-06, "Finetune/Loss": 0.8783360123634338, "Finetune/Loss (Raw)": 0.8105441331863403, "Finetune/Step": 3289, "Finetune/Step Time": 4.545471826568246} +{"Finetune/Learning Rate": 6.270497881109541e-06, "Finetune/Loss": 0.882466197013855, "Finetune/Loss (Raw)": 0.9388164281845093, "Finetune/Step": 3290, "Finetune/Step Time": 4.5084737204015255} +{"Finetune/Learning Rate": 6.264718452460699e-06, "Finetune/Loss": 0.8803697824478149, "Finetune/Loss (Raw)": 0.7762842774391174, "Finetune/Step": 3291, "Finetune/Step Time": 4.503653760999441} +{"Finetune/Learning Rate": 6.258940473401705e-06, "Finetune/Loss": 0.8795297145843506, "Finetune/Loss (Raw)": 0.8459560871124268, "Finetune/Step": 3292, "Finetune/Step Time": 4.504530485719442} +{"Finetune/Learning Rate": 6.253163946174872e-06, "Finetune/Loss": 0.8813188672065735, "Finetune/Loss (Raw)": 0.9278746843338013, "Finetune/Step": 3293, "Finetune/Step Time": 4.500181732699275} +{"Finetune/Learning Rate": 6.247388873021959e-06, "Finetune/Loss": 0.8814021944999695, "Finetune/Loss (Raw)": 0.8371305465698242, "Finetune/Step": 3294, "Finetune/Step Time": 4.526833163574338} +{"Finetune/Learning Rate": 6.2416152561841635e-06, "Finetune/Loss": 0.8807270526885986, "Finetune/Loss (Raw)": 0.8313956260681152, "Finetune/Step": 3295, "Finetune/Step Time": 4.529451144859195} +{"Finetune/Learning Rate": 6.2358430979021075e-06, "Finetune/Loss": 0.8799970746040344, "Finetune/Loss (Raw)": 0.9276469349861145, "Finetune/Step": 3296, "Finetune/Step Time": 4.530804393813014} +{"Finetune/Learning Rate": 6.230072400415863e-06, "Finetune/Loss": 0.8839681148529053, "Finetune/Loss (Raw)": 0.8239867687225342, "Finetune/Step": 3297, "Finetune/Step Time": 4.51301627419889} +{"Finetune/Learning Rate": 6.224303165964913e-06, "Finetune/Loss": 0.8855264186859131, "Finetune/Loss (Raw)": 0.9838463068008423, "Finetune/Step": 3298, "Finetune/Step Time": 4.51109267026186} +{"Finetune/Learning Rate": 6.21853539678819e-06, "Finetune/Loss": 0.8821412920951843, "Finetune/Loss (Raw)": 0.38650277256965637, "Finetune/Step": 3299, "Finetune/Step Time": 4.546775897964835} +{"Finetune/Learning Rate": 6.212769095124055e-06, "Finetune/Loss": 0.8824030756950378, "Finetune/Loss (Raw)": 0.923113226890564, "Finetune/Step": 3300, "Finetune/Step Time": 4.549411371350288} +{"Finetune/Learning Rate": 6.207004263210292e-06, "Finetune/Loss": 0.8820339441299438, "Finetune/Loss (Raw)": 0.7472731471061707, "Finetune/Step": 3301, "Finetune/Step Time": 4.538613621145487} +{"Finetune/Learning Rate": 6.2012409032841294e-06, "Finetune/Loss": 0.8814730644226074, "Finetune/Loss (Raw)": 0.9690042734146118, "Finetune/Step": 3302, "Finetune/Step Time": 4.536713810637593} +{"Finetune/Learning Rate": 6.1954790175822e-06, "Finetune/Loss": 0.8812990784645081, "Finetune/Loss (Raw)": 0.8665702939033508, "Finetune/Step": 3303, "Finetune/Step Time": 4.534536875784397} +{"Finetune/Learning Rate": 6.189718608340587e-06, "Finetune/Loss": 0.8797118663787842, "Finetune/Loss (Raw)": 0.8371523022651672, "Finetune/Step": 3304, "Finetune/Step Time": 4.544170709326863} +{"Finetune/Learning Rate": 6.183959677794795e-06, "Finetune/Loss": 0.8807453513145447, "Finetune/Loss (Raw)": 0.99164217710495, "Finetune/Step": 3305, "Finetune/Step Time": 4.539961179718375} +{"Finetune/Learning Rate": 6.1782022281797485e-06, "Finetune/Loss": 0.8806101679801941, "Finetune/Loss (Raw)": 0.8433104157447815, "Finetune/Step": 3306, "Finetune/Step Time": 4.531467808410525} +{"Finetune/Learning Rate": 6.17244626172981e-06, "Finetune/Loss": 0.8802053928375244, "Finetune/Loss (Raw)": 1.0244017839431763, "Finetune/Step": 3307, "Finetune/Step Time": 4.5211183074861765} +{"Finetune/Learning Rate": 6.166691780678743e-06, "Finetune/Loss": 0.8814667463302612, "Finetune/Loss (Raw)": 0.9831893444061279, "Finetune/Step": 3308, "Finetune/Step Time": 4.527285708114505} +{"Finetune/Learning Rate": 6.1609387872597595e-06, "Finetune/Loss": 0.8817248344421387, "Finetune/Loss (Raw)": 0.9868084788322449, "Finetune/Step": 3309, "Finetune/Step Time": 4.537616381421685} +{"Finetune/Learning Rate": 6.155187283705479e-06, "Finetune/Loss": 0.8809584975242615, "Finetune/Loss (Raw)": 0.8168541193008423, "Finetune/Step": 3310, "Finetune/Step Time": 4.534462595358491} +{"Finetune/Learning Rate": 6.14943727224795e-06, "Finetune/Loss": 0.880496621131897, "Finetune/Loss (Raw)": 0.8022971749305725, "Finetune/Step": 3311, "Finetune/Step Time": 4.536959649994969} +{"Finetune/Learning Rate": 6.1436887551186466e-06, "Finetune/Loss": 0.8799971342086792, "Finetune/Loss (Raw)": 0.8476197123527527, "Finetune/Step": 3312, "Finetune/Step Time": 4.53644016943872} +{"Finetune/Learning Rate": 6.137941734548443e-06, "Finetune/Loss": 0.8761284351348877, "Finetune/Loss (Raw)": 0.43130066990852356, "Finetune/Step": 3313, "Finetune/Step Time": 4.568343250080943} +{"Finetune/Learning Rate": 6.132196212767653e-06, "Finetune/Loss": 0.87574303150177, "Finetune/Loss (Raw)": 0.8017680048942566, "Finetune/Step": 3314, "Finetune/Step Time": 4.5701772421598434} +{"Finetune/Learning Rate": 6.126452192005999e-06, "Finetune/Loss": 0.8770676255226135, "Finetune/Loss (Raw)": 1.1155458688735962, "Finetune/Step": 3315, "Finetune/Step Time": 4.564585005864501} +{"Finetune/Learning Rate": 6.120709674492627e-06, "Finetune/Loss": 0.8768394589424133, "Finetune/Loss (Raw)": 0.8299779295921326, "Finetune/Step": 3316, "Finetune/Step Time": 4.561233328655362} +{"Finetune/Learning Rate": 6.114968662456093e-06, "Finetune/Loss": 0.8772228956222534, "Finetune/Loss (Raw)": 0.8986294269561768, "Finetune/Step": 3317, "Finetune/Step Time": 4.55675938539207} +{"Finetune/Learning Rate": 6.10922915812437e-06, "Finetune/Loss": 0.872732400894165, "Finetune/Loss (Raw)": 0.33671820163726807, "Finetune/Step": 3318, "Finetune/Step Time": 4.589877123013139} +{"Finetune/Learning Rate": 6.103491163724853e-06, "Finetune/Loss": 0.8728243112564087, "Finetune/Loss (Raw)": 0.9774883985519409, "Finetune/Step": 3319, "Finetune/Step Time": 4.566179709509015} +{"Finetune/Learning Rate": 6.09775468148434e-06, "Finetune/Loss": 0.8729415535926819, "Finetune/Loss (Raw)": 0.8643977046012878, "Finetune/Step": 3320, "Finetune/Step Time": 4.55825949460268} +{"Finetune/Learning Rate": 6.092019713629053e-06, "Finetune/Loss": 0.8726925849914551, "Finetune/Loss (Raw)": 0.8277559876441956, "Finetune/Step": 3321, "Finetune/Step Time": 4.558089729398489} +{"Finetune/Learning Rate": 6.086286262384617e-06, "Finetune/Loss": 0.8732618689537048, "Finetune/Loss (Raw)": 0.8487777709960938, "Finetune/Step": 3322, "Finetune/Step Time": 4.551029663532972} +{"Finetune/Learning Rate": 6.080554329976071e-06, "Finetune/Loss": 0.8735771775245667, "Finetune/Loss (Raw)": 1.0170422792434692, "Finetune/Step": 3323, "Finetune/Step Time": 4.5573827270418406} +{"Finetune/Learning Rate": 6.074823918627872e-06, "Finetune/Loss": 0.8729691505432129, "Finetune/Loss (Raw)": 0.842616081237793, "Finetune/Step": 3324, "Finetune/Step Time": 4.57045803964138} +{"Finetune/Learning Rate": 6.069095030563877e-06, "Finetune/Loss": 0.8715747594833374, "Finetune/Loss (Raw)": 0.8122042417526245, "Finetune/Step": 3325, "Finetune/Step Time": 4.57002535648644} +{"Finetune/Learning Rate": 6.063367668007356e-06, "Finetune/Loss": 0.8727672100067139, "Finetune/Loss (Raw)": 0.9428573250770569, "Finetune/Step": 3326, "Finetune/Step Time": 4.566615082323551} +{"Finetune/Learning Rate": 6.057641833180987e-06, "Finetune/Loss": 0.8714407682418823, "Finetune/Loss (Raw)": 0.7416821122169495, "Finetune/Step": 3327, "Finetune/Step Time": 4.579485941678286} +{"Finetune/Learning Rate": 6.0519175283068534e-06, "Finetune/Loss": 0.8710414171218872, "Finetune/Loss (Raw)": 1.0221980810165405, "Finetune/Step": 3328, "Finetune/Step Time": 4.583572732284665} +{"Finetune/Learning Rate": 6.046194755606448e-06, "Finetune/Loss": 0.8727782368659973, "Finetune/Loss (Raw)": 1.018369436264038, "Finetune/Step": 3329, "Finetune/Step Time": 4.586165279150009} +{"Finetune/Learning Rate": 6.040473517300665e-06, "Finetune/Loss": 0.8723640441894531, "Finetune/Loss (Raw)": 0.8006281852722168, "Finetune/Step": 3330, "Finetune/Step Time": 4.599642399698496} +{"Finetune/Learning Rate": 6.0347538156098104e-06, "Finetune/Loss": 0.87291419506073, "Finetune/Loss (Raw)": 0.918565571308136, "Finetune/Step": 3331, "Finetune/Step Time": 4.613336570560932} +{"Finetune/Learning Rate": 6.0290356527535836e-06, "Finetune/Loss": 0.8733371496200562, "Finetune/Loss (Raw)": 0.8916277885437012, "Finetune/Step": 3332, "Finetune/Step Time": 4.612565418705344} +{"Finetune/Learning Rate": 6.023319030951091e-06, "Finetune/Loss": 0.8742722868919373, "Finetune/Loss (Raw)": 0.9415456652641296, "Finetune/Step": 3333, "Finetune/Step Time": 4.612335052341223} +{"Finetune/Learning Rate": 6.017603952420846e-06, "Finetune/Loss": 0.8750662207603455, "Finetune/Loss (Raw)": 0.8873929381370544, "Finetune/Step": 3334, "Finetune/Step Time": 4.612802704796195} +{"Finetune/Learning Rate": 6.011890419380756e-06, "Finetune/Loss": 0.8744310140609741, "Finetune/Loss (Raw)": 0.8359968066215515, "Finetune/Step": 3335, "Finetune/Step Time": 4.5860102865844965} +{"Finetune/Learning Rate": 6.0061784340481366e-06, "Finetune/Loss": 0.8736850023269653, "Finetune/Loss (Raw)": 0.8622884750366211, "Finetune/Step": 3336, "Finetune/Step Time": 4.584962237626314} +{"Finetune/Learning Rate": 6.000467998639692e-06, "Finetune/Loss": 0.873150110244751, "Finetune/Loss (Raw)": 0.7873680591583252, "Finetune/Step": 3337, "Finetune/Step Time": 4.583285326138139} +{"Finetune/Learning Rate": 5.994759115371531e-06, "Finetune/Loss": 0.8739451766014099, "Finetune/Loss (Raw)": 0.8918331861495972, "Finetune/Step": 3338, "Finetune/Step Time": 4.586404649540782} +{"Finetune/Learning Rate": 5.989051786459163e-06, "Finetune/Loss": 0.8741156458854675, "Finetune/Loss (Raw)": 0.9576800465583801, "Finetune/Step": 3339, "Finetune/Step Time": 4.578697610646486} +{"Finetune/Learning Rate": 5.98334601411749e-06, "Finetune/Loss": 0.8780652284622192, "Finetune/Loss (Raw)": 0.9033103585243225, "Finetune/Step": 3340, "Finetune/Step Time": 4.538515228778124} +{"Finetune/Learning Rate": 5.977641800560811e-06, "Finetune/Loss": 0.8787407875061035, "Finetune/Loss (Raw)": 1.036521077156067, "Finetune/Step": 3341, "Finetune/Step Time": 4.550128867849708} +{"Finetune/Learning Rate": 5.971939148002819e-06, "Finetune/Loss": 0.8792614340782166, "Finetune/Loss (Raw)": 0.9965717196464539, "Finetune/Step": 3342, "Finetune/Step Time": 4.536381663754582} +{"Finetune/Learning Rate": 5.966238058656601e-06, "Finetune/Loss": 0.8803396224975586, "Finetune/Loss (Raw)": 0.9332963228225708, "Finetune/Step": 3343, "Finetune/Step Time": 4.524302996695042} +{"Finetune/Learning Rate": 5.960538534734641e-06, "Finetune/Loss": 0.8803118467330933, "Finetune/Loss (Raw)": 0.7323897480964661, "Finetune/Step": 3344, "Finetune/Step Time": 4.50498322956264} +{"Finetune/Learning Rate": 5.9548405784488095e-06, "Finetune/Loss": 0.8759010434150696, "Finetune/Loss (Raw)": 0.3517431318759918, "Finetune/Step": 3345, "Finetune/Step Time": 4.547362616285682} +{"Finetune/Learning Rate": 5.949144192010378e-06, "Finetune/Loss": 0.8767021894454956, "Finetune/Loss (Raw)": 1.021459698677063, "Finetune/Step": 3346, "Finetune/Step Time": 4.546348230913281} +{"Finetune/Learning Rate": 5.94344937763e-06, "Finetune/Loss": 0.8766641020774841, "Finetune/Loss (Raw)": 0.7696523666381836, "Finetune/Step": 3347, "Finetune/Step Time": 4.547517882660031} +{"Finetune/Learning Rate": 5.9377561375177164e-06, "Finetune/Loss": 0.8756994605064392, "Finetune/Loss (Raw)": 0.9443193674087524, "Finetune/Step": 3348, "Finetune/Step Time": 4.552279848605394} +{"Finetune/Learning Rate": 5.9320644738829706e-06, "Finetune/Loss": 0.876587450504303, "Finetune/Loss (Raw)": 0.9424777030944824, "Finetune/Step": 3349, "Finetune/Step Time": 4.548852955922484} +{"Finetune/Learning Rate": 5.926374388934579e-06, "Finetune/Loss": 0.8823038339614868, "Finetune/Loss (Raw)": 1.0656453371047974, "Finetune/Step": 3350, "Finetune/Step Time": 4.517681162804365} +{"Finetune/Learning Rate": 5.920685884880762e-06, "Finetune/Loss": 0.8822318315505981, "Finetune/Loss (Raw)": 0.8431954979896545, "Finetune/Step": 3351, "Finetune/Step Time": 4.524511381983757} +{"Finetune/Learning Rate": 5.914998963929109e-06, "Finetune/Loss": 0.8823774456977844, "Finetune/Loss (Raw)": 0.9410817623138428, "Finetune/Step": 3352, "Finetune/Step Time": 4.528404038399458} +{"Finetune/Learning Rate": 5.9093136282866014e-06, "Finetune/Loss": 0.883145272731781, "Finetune/Loss (Raw)": 0.9718773365020752, "Finetune/Step": 3353, "Finetune/Step Time": 4.544500095769763} +{"Finetune/Learning Rate": 5.903629880159614e-06, "Finetune/Loss": 0.8834282159805298, "Finetune/Loss (Raw)": 0.9323545694351196, "Finetune/Step": 3354, "Finetune/Step Time": 4.542666325345635} +{"Finetune/Learning Rate": 5.897947721753894e-06, "Finetune/Loss": 0.884462833404541, "Finetune/Loss (Raw)": 0.9707621335983276, "Finetune/Step": 3355, "Finetune/Step Time": 4.534433726221323} +{"Finetune/Learning Rate": 5.89226715527458e-06, "Finetune/Loss": 0.8849613666534424, "Finetune/Loss (Raw)": 0.935836911201477, "Finetune/Step": 3356, "Finetune/Step Time": 4.5368016716092825} +{"Finetune/Learning Rate": 5.886588182926185e-06, "Finetune/Loss": 0.8841283321380615, "Finetune/Loss (Raw)": 0.8648064136505127, "Finetune/Step": 3357, "Finetune/Step Time": 4.5372547674924135} +{"Finetune/Learning Rate": 5.880910806912607e-06, "Finetune/Loss": 0.8839098811149597, "Finetune/Loss (Raw)": 0.8121477365493774, "Finetune/Step": 3358, "Finetune/Step Time": 4.530994223430753} +{"Finetune/Learning Rate": 5.8752350294371274e-06, "Finetune/Loss": 0.885249674320221, "Finetune/Loss (Raw)": 1.0539406538009644, "Finetune/Step": 3359, "Finetune/Step Time": 4.5045680571347475} +{"Finetune/Learning Rate": 5.8695608527024075e-06, "Finetune/Loss": 0.8850835561752319, "Finetune/Loss (Raw)": 0.9963275194168091, "Finetune/Step": 3360, "Finetune/Step Time": 4.505647053942084} +{"Finetune/Learning Rate": 5.863888278910475e-06, "Finetune/Loss": 0.8837945461273193, "Finetune/Loss (Raw)": 0.9460950493812561, "Finetune/Step": 3361, "Finetune/Step Time": 4.519161529839039} +{"Finetune/Learning Rate": 5.8582173102627526e-06, "Finetune/Loss": 0.8841732740402222, "Finetune/Loss (Raw)": 0.825757622718811, "Finetune/Step": 3362, "Finetune/Step Time": 4.501884657889605} +{"Finetune/Learning Rate": 5.852547948960029e-06, "Finetune/Loss": 0.8834854364395142, "Finetune/Loss (Raw)": 0.9415398240089417, "Finetune/Step": 3363, "Finetune/Step Time": 4.501728113740683} +{"Finetune/Learning Rate": 5.8468801972024755e-06, "Finetune/Loss": 0.8852112293243408, "Finetune/Loss (Raw)": 0.9688849449157715, "Finetune/Step": 3364, "Finetune/Step Time": 4.493621939793229} +{"Finetune/Learning Rate": 5.841214057189638e-06, "Finetune/Loss": 0.8843874335289001, "Finetune/Loss (Raw)": 0.8679623007774353, "Finetune/Step": 3365, "Finetune/Step Time": 4.507536130025983} +{"Finetune/Learning Rate": 5.8355495311204266e-06, "Finetune/Loss": 0.8876590728759766, "Finetune/Loss (Raw)": 0.8182220458984375, "Finetune/Step": 3366, "Finetune/Step Time": 4.480892298743129} +{"Finetune/Learning Rate": 5.829886621193141e-06, "Finetune/Loss": 0.8860160708427429, "Finetune/Loss (Raw)": 0.8163709044456482, "Finetune/Step": 3367, "Finetune/Step Time": 4.478043671697378} +{"Finetune/Learning Rate": 5.824225329605442e-06, "Finetune/Loss": 0.8809686303138733, "Finetune/Loss (Raw)": 0.437744677066803, "Finetune/Step": 3368, "Finetune/Step Time": 4.514780147001147} +{"Finetune/Learning Rate": 5.818565658554364e-06, "Finetune/Loss": 0.8811781406402588, "Finetune/Loss (Raw)": 0.9912183284759521, "Finetune/Step": 3369, "Finetune/Step Time": 4.513503886759281} +{"Finetune/Learning Rate": 5.8129076102363266e-06, "Finetune/Loss": 0.8809956312179565, "Finetune/Loss (Raw)": 0.9559313058853149, "Finetune/Step": 3370, "Finetune/Step Time": 4.5107312630862} +{"Finetune/Learning Rate": 5.807251186847095e-06, "Finetune/Loss": 0.8789510726928711, "Finetune/Loss (Raw)": 0.6876282095909119, "Finetune/Step": 3371, "Finetune/Step Time": 4.532715508714318} +{"Finetune/Learning Rate": 5.801596390581817e-06, "Finetune/Loss": 0.8781711459159851, "Finetune/Loss (Raw)": 0.8699653148651123, "Finetune/Step": 3372, "Finetune/Step Time": 4.543972238898277} +{"Finetune/Learning Rate": 5.795943223635018e-06, "Finetune/Loss": 0.8778142929077148, "Finetune/Loss (Raw)": 0.9544350504875183, "Finetune/Step": 3373, "Finetune/Step Time": 4.549608904868364} +{"Finetune/Learning Rate": 5.7902916882005755e-06, "Finetune/Loss": 0.8791664838790894, "Finetune/Loss (Raw)": 0.9004501700401306, "Finetune/Step": 3374, "Finetune/Step Time": 4.5464368015527725} +{"Finetune/Learning Rate": 5.784641786471742e-06, "Finetune/Loss": 0.8812596797943115, "Finetune/Loss (Raw)": 1.1258771419525146, "Finetune/Step": 3375, "Finetune/Step Time": 4.5439095962792635} +{"Finetune/Learning Rate": 5.778993520641132e-06, "Finetune/Loss": 0.8813714385032654, "Finetune/Loss (Raw)": 1.0141775608062744, "Finetune/Step": 3376, "Finetune/Step Time": 4.535956714302301} +{"Finetune/Learning Rate": 5.7733468929007294e-06, "Finetune/Loss": 0.8798516988754272, "Finetune/Loss (Raw)": 0.80610191822052, "Finetune/Step": 3377, "Finetune/Step Time": 4.546633897349238} +{"Finetune/Learning Rate": 5.7677019054418805e-06, "Finetune/Loss": 0.8808923959732056, "Finetune/Loss (Raw)": 0.986575722694397, "Finetune/Step": 3378, "Finetune/Step Time": 4.542163040488958} +{"Finetune/Learning Rate": 5.762058560455287e-06, "Finetune/Loss": 0.8797827363014221, "Finetune/Loss (Raw)": 0.7122272253036499, "Finetune/Step": 3379, "Finetune/Step Time": 4.521761108189821} +{"Finetune/Learning Rate": 5.756416860131036e-06, "Finetune/Loss": 0.8778701424598694, "Finetune/Loss (Raw)": 0.7883544564247131, "Finetune/Step": 3380, "Finetune/Step Time": 4.519778179004788} +{"Finetune/Learning Rate": 5.750776806658549e-06, "Finetune/Loss": 0.8769996166229248, "Finetune/Loss (Raw)": 0.8831773996353149, "Finetune/Step": 3381, "Finetune/Step Time": 4.5195630472153425} +{"Finetune/Learning Rate": 5.745138402226621e-06, "Finetune/Loss": 0.8776810169219971, "Finetune/Loss (Raw)": 1.0365186929702759, "Finetune/Step": 3382, "Finetune/Step Time": 4.516234112903476} +{"Finetune/Learning Rate": 5.739501649023413e-06, "Finetune/Loss": 0.8771334886550903, "Finetune/Loss (Raw)": 0.850704550743103, "Finetune/Step": 3383, "Finetune/Step Time": 4.4986086543649435} +{"Finetune/Learning Rate": 5.7338665492364365e-06, "Finetune/Loss": 0.8766510486602783, "Finetune/Loss (Raw)": 0.9386804103851318, "Finetune/Step": 3384, "Finetune/Step Time": 4.499511677771807} +{"Finetune/Learning Rate": 5.728233105052563e-06, "Finetune/Loss": 0.8767132759094238, "Finetune/Loss (Raw)": 0.784305989742279, "Finetune/Step": 3385, "Finetune/Step Time": 4.494418036192656} +{"Finetune/Learning Rate": 5.722601318658021e-06, "Finetune/Loss": 0.8752330541610718, "Finetune/Loss (Raw)": 0.8850914239883423, "Finetune/Step": 3386, "Finetune/Step Time": 4.486537931486964} +{"Finetune/Learning Rate": 5.7169711922383984e-06, "Finetune/Loss": 0.8748137950897217, "Finetune/Loss (Raw)": 0.8000223636627197, "Finetune/Step": 3387, "Finetune/Step Time": 4.46950027346611} +{"Finetune/Learning Rate": 5.711342727978637e-06, "Finetune/Loss": 0.873677134513855, "Finetune/Loss (Raw)": 0.8172230124473572, "Finetune/Step": 3388, "Finetune/Step Time": 4.451952578499913} +{"Finetune/Learning Rate": 5.7057159280630316e-06, "Finetune/Loss": 0.8745723366737366, "Finetune/Loss (Raw)": 0.9301931262016296, "Finetune/Step": 3389, "Finetune/Step Time": 4.456830540671945} +{"Finetune/Learning Rate": 5.700090794675241e-06, "Finetune/Loss": 0.8743839859962463, "Finetune/Loss (Raw)": 0.9584911465644836, "Finetune/Step": 3390, "Finetune/Step Time": 4.461656356230378} +{"Finetune/Learning Rate": 5.694467329998264e-06, "Finetune/Loss": 0.8748645782470703, "Finetune/Loss (Raw)": 1.0630667209625244, "Finetune/Step": 3391, "Finetune/Step Time": 4.458214238286018} +{"Finetune/Learning Rate": 5.68884553621445e-06, "Finetune/Loss": 0.8748499155044556, "Finetune/Loss (Raw)": 0.8565034866333008, "Finetune/Step": 3392, "Finetune/Step Time": 4.457844618707895} +{"Finetune/Learning Rate": 5.683225415505521e-06, "Finetune/Loss": 0.8742333650588989, "Finetune/Loss (Raw)": 0.9989644289016724, "Finetune/Step": 3393, "Finetune/Step Time": 4.45449623093009} +{"Finetune/Learning Rate": 5.67760697005253e-06, "Finetune/Loss": 0.8743153810501099, "Finetune/Loss (Raw)": 0.9016333818435669, "Finetune/Step": 3394, "Finetune/Step Time": 4.474652333185077} +{"Finetune/Learning Rate": 5.671990202035886e-06, "Finetune/Loss": 0.874934196472168, "Finetune/Loss (Raw)": 0.9874091744422913, "Finetune/Step": 3395, "Finetune/Step Time": 4.458017868921161} +{"Finetune/Learning Rate": 5.666375113635347e-06, "Finetune/Loss": 0.8733147382736206, "Finetune/Loss (Raw)": 0.7603205442428589, "Finetune/Step": 3396, "Finetune/Step Time": 4.457884130999446} +{"Finetune/Learning Rate": 5.660761707030022e-06, "Finetune/Loss": 0.8727391958236694, "Finetune/Loss (Raw)": 0.8937203884124756, "Finetune/Step": 3397, "Finetune/Step Time": 4.455831775441766} +{"Finetune/Learning Rate": 5.655149984398359e-06, "Finetune/Loss": 0.873110830783844, "Finetune/Loss (Raw)": 0.8897122740745544, "Finetune/Step": 3398, "Finetune/Step Time": 4.449244856834412} +{"Finetune/Learning Rate": 5.649539947918159e-06, "Finetune/Loss": 0.8727487325668335, "Finetune/Loss (Raw)": 0.8247094750404358, "Finetune/Step": 3399, "Finetune/Step Time": 4.463609077036381} +{"Finetune/Learning Rate": 5.643931599766577e-06, "Finetune/Loss": 0.8718529939651489, "Finetune/Loss (Raw)": 0.8395273685455322, "Finetune/Step": 3400, "Finetune/Step Time": 4.437166329473257} +{"Finetune/Learning Rate": 5.638324942120091e-06, "Finetune/Loss": 0.8719229698181152, "Finetune/Loss (Raw)": 0.8107761144638062, "Finetune/Step": 3401, "Finetune/Step Time": 4.429046032950282} +{"Finetune/Learning Rate": 5.632719977154537e-06, "Finetune/Loss": 0.8763041496276855, "Finetune/Loss (Raw)": 0.8851252198219299, "Finetune/Step": 3402, "Finetune/Step Time": 4.396558521315455} +{"Finetune/Learning Rate": 5.6271167070451016e-06, "Finetune/Loss": 0.8766416907310486, "Finetune/Loss (Raw)": 0.8268734216690063, "Finetune/Step": 3403, "Finetune/Step Time": 4.392080029472709} +{"Finetune/Learning Rate": 5.621515133966301e-06, "Finetune/Loss": 0.876713752746582, "Finetune/Loss (Raw)": 0.9055619835853577, "Finetune/Step": 3404, "Finetune/Step Time": 4.38349974155426} +{"Finetune/Learning Rate": 5.615915260091987e-06, "Finetune/Loss": 0.8751533031463623, "Finetune/Loss (Raw)": 0.7459348440170288, "Finetune/Step": 3405, "Finetune/Step Time": 4.451615583151579} +{"Finetune/Learning Rate": 5.610317087595372e-06, "Finetune/Loss": 0.8757110834121704, "Finetune/Loss (Raw)": 0.9266843795776367, "Finetune/Step": 3406, "Finetune/Step Time": 4.448552655056119} +{"Finetune/Learning Rate": 5.604720618648993e-06, "Finetune/Loss": 0.8752754926681519, "Finetune/Loss (Raw)": 0.9081409573554993, "Finetune/Step": 3407, "Finetune/Step Time": 4.436132904142141} +{"Finetune/Learning Rate": 5.599125855424734e-06, "Finetune/Loss": 0.8748693466186523, "Finetune/Loss (Raw)": 0.9432325959205627, "Finetune/Step": 3408, "Finetune/Step Time": 4.436876008287072} +{"Finetune/Learning Rate": 5.593532800093808e-06, "Finetune/Loss": 0.8737360835075378, "Finetune/Loss (Raw)": 0.8825694918632507, "Finetune/Step": 3409, "Finetune/Step Time": 4.429533421993256} +{"Finetune/Learning Rate": 5.5879414548267754e-06, "Finetune/Loss": 0.8734613656997681, "Finetune/Loss (Raw)": 0.8252725601196289, "Finetune/Step": 3410, "Finetune/Step Time": 4.4385070856660604} +{"Finetune/Learning Rate": 5.582351821793525e-06, "Finetune/Loss": 0.878441572189331, "Finetune/Loss (Raw)": 1.0086383819580078, "Finetune/Step": 3411, "Finetune/Step Time": 4.4001075848937035} +{"Finetune/Learning Rate": 5.576763903163281e-06, "Finetune/Loss": 0.8790616393089294, "Finetune/Loss (Raw)": 0.9481183886528015, "Finetune/Step": 3412, "Finetune/Step Time": 4.395417051389813} +{"Finetune/Learning Rate": 5.571177701104614e-06, "Finetune/Loss": 0.8794549107551575, "Finetune/Loss (Raw)": 1.024293065071106, "Finetune/Step": 3413, "Finetune/Step Time": 4.40271944925189} +{"Finetune/Learning Rate": 5.56559321778542e-06, "Finetune/Loss": 0.8807939291000366, "Finetune/Loss (Raw)": 1.0356090068817139, "Finetune/Step": 3414, "Finetune/Step Time": 4.401145359501243} +{"Finetune/Learning Rate": 5.560010455372915e-06, "Finetune/Loss": 0.8794801235198975, "Finetune/Loss (Raw)": 0.7455976009368896, "Finetune/Step": 3415, "Finetune/Step Time": 4.3982252441346645} +{"Finetune/Learning Rate": 5.5544294160336735e-06, "Finetune/Loss": 0.8751192688941956, "Finetune/Loss (Raw)": 0.42415162920951843, "Finetune/Step": 3416, "Finetune/Step Time": 4.4330501314252615} +{"Finetune/Learning Rate": 5.5488501019335824e-06, "Finetune/Loss": 0.8758667707443237, "Finetune/Loss (Raw)": 0.9062234163284302, "Finetune/Step": 3417, "Finetune/Step Time": 4.439627084881067} +{"Finetune/Learning Rate": 5.5432725152378675e-06, "Finetune/Loss": 0.875525951385498, "Finetune/Loss (Raw)": 0.8951854705810547, "Finetune/Step": 3418, "Finetune/Step Time": 4.4508516527712345} +{"Finetune/Learning Rate": 5.537696658111077e-06, "Finetune/Loss": 0.876563310623169, "Finetune/Loss (Raw)": 0.909069836139679, "Finetune/Step": 3419, "Finetune/Step Time": 4.4434717036783695} +{"Finetune/Learning Rate": 5.532122532717095e-06, "Finetune/Loss": 0.8750367164611816, "Finetune/Loss (Raw)": 0.6505514979362488, "Finetune/Step": 3420, "Finetune/Step Time": 4.468732351437211} +{"Finetune/Learning Rate": 5.52655014121913e-06, "Finetune/Loss": 0.8749749660491943, "Finetune/Loss (Raw)": 0.9199740886688232, "Finetune/Step": 3421, "Finetune/Step Time": 4.4713953249156475} +{"Finetune/Learning Rate": 5.5209794857797135e-06, "Finetune/Loss": 0.8752561211585999, "Finetune/Loss (Raw)": 0.8731164932250977, "Finetune/Step": 3422, "Finetune/Step Time": 4.464487625285983} +{"Finetune/Learning Rate": 5.51541056856072e-06, "Finetune/Loss": 0.8751566410064697, "Finetune/Loss (Raw)": 0.8186604976654053, "Finetune/Step": 3423, "Finetune/Step Time": 4.463371094316244} +{"Finetune/Learning Rate": 5.509843391723332e-06, "Finetune/Loss": 0.8706679940223694, "Finetune/Loss (Raw)": 0.3531009554862976, "Finetune/Step": 3424, "Finetune/Step Time": 4.49491080455482} +{"Finetune/Learning Rate": 5.504277957428053e-06, "Finetune/Loss": 0.8711169958114624, "Finetune/Loss (Raw)": 0.8814603686332703, "Finetune/Step": 3425, "Finetune/Step Time": 4.482104774564505} +{"Finetune/Learning Rate": 5.498714267834731e-06, "Finetune/Loss": 0.8692227602005005, "Finetune/Loss (Raw)": 0.7413867712020874, "Finetune/Step": 3426, "Finetune/Step Time": 4.484105698764324} +{"Finetune/Learning Rate": 5.493152325102523e-06, "Finetune/Loss": 0.873841404914856, "Finetune/Loss (Raw)": 0.9776884317398071, "Finetune/Step": 3427, "Finetune/Step Time": 4.450477676466107} +{"Finetune/Learning Rate": 5.487592131389907e-06, "Finetune/Loss": 0.8736681938171387, "Finetune/Loss (Raw)": 0.9009372591972351, "Finetune/Step": 3428, "Finetune/Step Time": 4.458590444177389} +{"Finetune/Learning Rate": 5.482033688854689e-06, "Finetune/Loss": 0.8750714063644409, "Finetune/Loss (Raw)": 0.9268855452537537, "Finetune/Step": 3429, "Finetune/Step Time": 4.455146098509431} +{"Finetune/Learning Rate": 5.47647699965399e-06, "Finetune/Loss": 0.874087929725647, "Finetune/Loss (Raw)": 0.8431154489517212, "Finetune/Step": 3430, "Finetune/Step Time": 4.449584413319826} +{"Finetune/Learning Rate": 5.470922065944253e-06, "Finetune/Loss": 0.875022292137146, "Finetune/Loss (Raw)": 0.9861781597137451, "Finetune/Step": 3431, "Finetune/Step Time": 4.4539854396134615} +{"Finetune/Learning Rate": 5.465368889881235e-06, "Finetune/Loss": 0.8745355606079102, "Finetune/Loss (Raw)": 0.7748516201972961, "Finetune/Step": 3432, "Finetune/Step Time": 4.4563075341284275} +{"Finetune/Learning Rate": 5.459817473620024e-06, "Finetune/Loss": 0.8735326528549194, "Finetune/Loss (Raw)": 0.8632578253746033, "Finetune/Step": 3433, "Finetune/Step Time": 4.474117552861571} +{"Finetune/Learning Rate": 5.454267819315016e-06, "Finetune/Loss": 0.8736140727996826, "Finetune/Loss (Raw)": 0.8537325859069824, "Finetune/Step": 3434, "Finetune/Step Time": 4.498874615877867} +{"Finetune/Learning Rate": 5.448719929119916e-06, "Finetune/Loss": 0.8723958134651184, "Finetune/Loss (Raw)": 0.8684757351875305, "Finetune/Step": 3435, "Finetune/Step Time": 4.512177163735032} +{"Finetune/Learning Rate": 5.44317380518775e-06, "Finetune/Loss": 0.8708791732788086, "Finetune/Loss (Raw)": 0.7890574932098389, "Finetune/Step": 3436, "Finetune/Step Time": 4.505946047604084} +{"Finetune/Learning Rate": 5.43762944967087e-06, "Finetune/Loss": 0.870181143283844, "Finetune/Loss (Raw)": 0.8974559307098389, "Finetune/Step": 3437, "Finetune/Step Time": 4.494675483554602} +{"Finetune/Learning Rate": 5.432086864720929e-06, "Finetune/Loss": 0.8702847361564636, "Finetune/Loss (Raw)": 0.8301121592521667, "Finetune/Step": 3438, "Finetune/Step Time": 4.493222804740071} +{"Finetune/Learning Rate": 5.426546052488892e-06, "Finetune/Loss": 0.8716778755187988, "Finetune/Loss (Raw)": 0.9806190133094788, "Finetune/Step": 3439, "Finetune/Step Time": 4.498125579208136} +{"Finetune/Learning Rate": 5.421007015125046e-06, "Finetune/Loss": 0.8717836141586304, "Finetune/Loss (Raw)": 0.8611566424369812, "Finetune/Step": 3440, "Finetune/Step Time": 4.500380964949727} +{"Finetune/Learning Rate": 5.415469754778979e-06, "Finetune/Loss": 0.8740854263305664, "Finetune/Loss (Raw)": 0.7259371280670166, "Finetune/Step": 3441, "Finetune/Step Time": 4.490365801379085} +{"Finetune/Learning Rate": 5.409934273599594e-06, "Finetune/Loss": 0.873930811882019, "Finetune/Loss (Raw)": 0.7819738388061523, "Finetune/Step": 3442, "Finetune/Step Time": 4.492224181070924} +{"Finetune/Learning Rate": 5.404400573735105e-06, "Finetune/Loss": 0.8716623783111572, "Finetune/Loss (Raw)": 0.8251843452453613, "Finetune/Step": 3443, "Finetune/Step Time": 4.502301836386323} +{"Finetune/Learning Rate": 5.3988686573330316e-06, "Finetune/Loss": 0.8723773956298828, "Finetune/Loss (Raw)": 0.9215058088302612, "Finetune/Step": 3444, "Finetune/Step Time": 4.485622154548764} +{"Finetune/Learning Rate": 5.393338526540202e-06, "Finetune/Loss": 0.8709369897842407, "Finetune/Loss (Raw)": 0.7142549157142639, "Finetune/Step": 3445, "Finetune/Step Time": 4.503604672849178} +{"Finetune/Learning Rate": 5.387810183502749e-06, "Finetune/Loss": 0.8750101327896118, "Finetune/Loss (Raw)": 0.858076274394989, "Finetune/Step": 3446, "Finetune/Step Time": 4.474705390632153} +{"Finetune/Learning Rate": 5.382283630366123e-06, "Finetune/Loss": 0.8751652836799622, "Finetune/Loss (Raw)": 0.9973525404930115, "Finetune/Step": 3447, "Finetune/Step Time": 4.474810319021344} +{"Finetune/Learning Rate": 5.37675886927507e-06, "Finetune/Loss": 0.8753238916397095, "Finetune/Loss (Raw)": 0.8847017884254456, "Finetune/Step": 3448, "Finetune/Step Time": 4.480514606460929} +{"Finetune/Learning Rate": 5.371235902373632e-06, "Finetune/Loss": 0.8765357732772827, "Finetune/Loss (Raw)": 0.9828668832778931, "Finetune/Step": 3449, "Finetune/Step Time": 4.475997284054756} +{"Finetune/Learning Rate": 5.365714731805174e-06, "Finetune/Loss": 0.8770232200622559, "Finetune/Loss (Raw)": 0.9111799597740173, "Finetune/Step": 3450, "Finetune/Step Time": 4.496468111872673} +{"Finetune/Learning Rate": 5.360195359712353e-06, "Finetune/Loss": 0.8756577968597412, "Finetune/Loss (Raw)": 0.8422683477401733, "Finetune/Step": 3451, "Finetune/Step Time": 4.485176624730229} +{"Finetune/Learning Rate": 5.354677788237126e-06, "Finetune/Loss": 0.8766457438468933, "Finetune/Loss (Raw)": 0.9690678119659424, "Finetune/Step": 3452, "Finetune/Step Time": 4.47543834336102} +{"Finetune/Learning Rate": 5.349162019520759e-06, "Finetune/Loss": 0.8769416809082031, "Finetune/Loss (Raw)": 0.8500816822052002, "Finetune/Step": 3453, "Finetune/Step Time": 4.473559310659766} +{"Finetune/Learning Rate": 5.343648055703812e-06, "Finetune/Loss": 0.8761200904846191, "Finetune/Loss (Raw)": 0.8376991748809814, "Finetune/Step": 3454, "Finetune/Step Time": 4.476551774889231} +{"Finetune/Learning Rate": 5.338135898926147e-06, "Finetune/Loss": 0.8729952573776245, "Finetune/Loss (Raw)": 0.3417050540447235, "Finetune/Step": 3455, "Finetune/Step Time": 4.504524599760771} +{"Finetune/Learning Rate": 5.33262555132692e-06, "Finetune/Loss": 0.8723856210708618, "Finetune/Loss (Raw)": 0.9441616535186768, "Finetune/Step": 3456, "Finetune/Step Time": 4.501178070902824} +{"Finetune/Learning Rate": 5.327117015044599e-06, "Finetune/Loss": 0.8713831901550293, "Finetune/Loss (Raw)": 0.8900543451309204, "Finetune/Step": 3457, "Finetune/Step Time": 4.491567371413112} +{"Finetune/Learning Rate": 5.321610292216937e-06, "Finetune/Loss": 0.8722257614135742, "Finetune/Loss (Raw)": 0.9084787964820862, "Finetune/Step": 3458, "Finetune/Step Time": 4.499773845076561} +{"Finetune/Learning Rate": 5.316105384980974e-06, "Finetune/Loss": 0.8717302083969116, "Finetune/Loss (Raw)": 0.855144202709198, "Finetune/Step": 3459, "Finetune/Step Time": 4.489847861230373} +{"Finetune/Learning Rate": 5.3106022954730704e-06, "Finetune/Loss": 0.8724680542945862, "Finetune/Loss (Raw)": 0.9860681295394897, "Finetune/Step": 3460, "Finetune/Step Time": 4.485896499827504} +{"Finetune/Learning Rate": 5.3051010258288624e-06, "Finetune/Loss": 0.8677411675453186, "Finetune/Loss (Raw)": 0.3365026116371155, "Finetune/Step": 3461, "Finetune/Step Time": 4.5233239363878965} +{"Finetune/Learning Rate": 5.299601578183289e-06, "Finetune/Loss": 0.868119478225708, "Finetune/Loss (Raw)": 0.9358158707618713, "Finetune/Step": 3462, "Finetune/Step Time": 4.519612520933151} +{"Finetune/Learning Rate": 5.294103954670573e-06, "Finetune/Loss": 0.8683381080627441, "Finetune/Loss (Raw)": 0.8639798164367676, "Finetune/Step": 3463, "Finetune/Step Time": 4.527486091479659} +{"Finetune/Learning Rate": 5.288608157424238e-06, "Finetune/Loss": 0.8688943982124329, "Finetune/Loss (Raw)": 0.933494508266449, "Finetune/Step": 3464, "Finetune/Step Time": 4.528440341353416} +{"Finetune/Learning Rate": 5.283114188577094e-06, "Finetune/Loss": 0.8692903518676758, "Finetune/Loss (Raw)": 0.8380499482154846, "Finetune/Step": 3465, "Finetune/Step Time": 4.523674776777625} +{"Finetune/Learning Rate": 5.277622050261242e-06, "Finetune/Loss": 0.8687460422515869, "Finetune/Loss (Raw)": 0.8221601247787476, "Finetune/Step": 3466, "Finetune/Step Time": 4.522064909338951} +{"Finetune/Learning Rate": 5.272131744608078e-06, "Finetune/Loss": 0.867023766040802, "Finetune/Loss (Raw)": 0.7372353076934814, "Finetune/Step": 3467, "Finetune/Step Time": 4.520676163956523} +{"Finetune/Learning Rate": 5.2666432737482844e-06, "Finetune/Loss": 0.8657031655311584, "Finetune/Loss (Raw)": 0.7342672944068909, "Finetune/Step": 3468, "Finetune/Step Time": 4.529286559671164} +{"Finetune/Learning Rate": 5.261156639811817e-06, "Finetune/Loss": 0.8647778034210205, "Finetune/Loss (Raw)": 0.9180763959884644, "Finetune/Step": 3469, "Finetune/Step Time": 4.532619580626488} +{"Finetune/Learning Rate": 5.255671844927944e-06, "Finetune/Loss": 0.8635817766189575, "Finetune/Loss (Raw)": 0.8434844613075256, "Finetune/Step": 3470, "Finetune/Step Time": 4.533856885507703} +{"Finetune/Learning Rate": 5.250188891225203e-06, "Finetune/Loss": 0.8636289834976196, "Finetune/Loss (Raw)": 0.9393401741981506, "Finetune/Step": 3471, "Finetune/Step Time": 4.539583090692759} +{"Finetune/Learning Rate": 5.24470778083142e-06, "Finetune/Loss": 0.8642019033432007, "Finetune/Loss (Raw)": 0.805720865726471, "Finetune/Step": 3472, "Finetune/Step Time": 4.5452934224158525} +{"Finetune/Learning Rate": 5.2392285158737085e-06, "Finetune/Loss": 0.8694359064102173, "Finetune/Loss (Raw)": 1.021694540977478, "Finetune/Step": 3473, "Finetune/Step Time": 4.531948693096638} +{"Finetune/Learning Rate": 5.233751098478465e-06, "Finetune/Loss": 0.8676902651786804, "Finetune/Loss (Raw)": 0.7980180978775024, "Finetune/Step": 3474, "Finetune/Step Time": 4.537188094109297} +{"Finetune/Learning Rate": 5.228275530771366e-06, "Finetune/Loss": 0.8686110377311707, "Finetune/Loss (Raw)": 0.8875162601470947, "Finetune/Step": 3475, "Finetune/Step Time": 4.543411800637841} +{"Finetune/Learning Rate": 5.22280181487737e-06, "Finetune/Loss": 0.8670868277549744, "Finetune/Loss (Raw)": 0.7492137551307678, "Finetune/Step": 3476, "Finetune/Step Time": 4.548600371927023} +{"Finetune/Learning Rate": 5.217329952920727e-06, "Finetune/Loss": 0.8661434650421143, "Finetune/Loss (Raw)": 0.8217261433601379, "Finetune/Step": 3477, "Finetune/Step Time": 4.56659178622067} +{"Finetune/Learning Rate": 5.2118599470249575e-06, "Finetune/Loss": 0.8656317591667175, "Finetune/Loss (Raw)": 1.0001519918441772, "Finetune/Step": 3478, "Finetune/Step Time": 4.558933239430189} +{"Finetune/Learning Rate": 5.206391799312855e-06, "Finetune/Loss": 0.8672950267791748, "Finetune/Loss (Raw)": 1.0560884475708008, "Finetune/Step": 3479, "Finetune/Step Time": 4.5456419084221125} +{"Finetune/Learning Rate": 5.200925511906513e-06, "Finetune/Loss": 0.8671793341636658, "Finetune/Loss (Raw)": 0.926273763179779, "Finetune/Step": 3480, "Finetune/Step Time": 4.537294553592801} +{"Finetune/Learning Rate": 5.195461086927284e-06, "Finetune/Loss": 0.8670362830162048, "Finetune/Loss (Raw)": 0.9535694718360901, "Finetune/Step": 3481, "Finetune/Step Time": 4.514356814324856} +{"Finetune/Learning Rate": 5.189998526495808e-06, "Finetune/Loss": 0.8667904138565063, "Finetune/Loss (Raw)": 0.9008801579475403, "Finetune/Step": 3482, "Finetune/Step Time": 4.514091424643993} +{"Finetune/Learning Rate": 5.184537832731996e-06, "Finetune/Loss": 0.8661321401596069, "Finetune/Loss (Raw)": 0.8865125775337219, "Finetune/Step": 3483, "Finetune/Step Time": 4.552877666428685} +{"Finetune/Learning Rate": 5.1790790077550366e-06, "Finetune/Loss": 0.8660725355148315, "Finetune/Loss (Raw)": 0.9281982779502869, "Finetune/Step": 3484, "Finetune/Step Time": 4.56927340850234} +{"Finetune/Learning Rate": 5.173622053683395e-06, "Finetune/Loss": 0.8670879602432251, "Finetune/Loss (Raw)": 0.9947845935821533, "Finetune/Step": 3485, "Finetune/Step Time": 4.56659989990294} +{"Finetune/Learning Rate": 5.168166972634803e-06, "Finetune/Loss": 0.8681089282035828, "Finetune/Loss (Raw)": 0.9428308606147766, "Finetune/Step": 3486, "Finetune/Step Time": 4.560351200401783} +{"Finetune/Learning Rate": 5.1627137667262815e-06, "Finetune/Loss": 0.8658192157745361, "Finetune/Loss (Raw)": 0.7608505487442017, "Finetune/Step": 3487, "Finetune/Step Time": 4.558795362710953} +{"Finetune/Learning Rate": 5.1572624380741044e-06, "Finetune/Loss": 0.8639639019966125, "Finetune/Loss (Raw)": 0.7588514089584351, "Finetune/Step": 3488, "Finetune/Step Time": 4.558263376355171} +{"Finetune/Learning Rate": 5.151812988793825e-06, "Finetune/Loss": 0.8642732501029968, "Finetune/Loss (Raw)": 0.9856935143470764, "Finetune/Step": 3489, "Finetune/Step Time": 4.545600606128573} +{"Finetune/Learning Rate": 5.146365421000275e-06, "Finetune/Loss": 0.8640308380126953, "Finetune/Loss (Raw)": 0.7947263121604919, "Finetune/Step": 3490, "Finetune/Step Time": 4.543532870709896} +{"Finetune/Learning Rate": 5.140919736807546e-06, "Finetune/Loss": 0.8644512891769409, "Finetune/Loss (Raw)": 0.9953587651252747, "Finetune/Step": 3491, "Finetune/Step Time": 4.539224429056048} +{"Finetune/Learning Rate": 5.135475938329002e-06, "Finetune/Loss": 0.8640288710594177, "Finetune/Loss (Raw)": 0.9148139953613281, "Finetune/Step": 3492, "Finetune/Step Time": 4.549172375351191} +{"Finetune/Learning Rate": 5.1300340276772756e-06, "Finetune/Loss": 0.8628029227256775, "Finetune/Loss (Raw)": 0.7110427618026733, "Finetune/Step": 3493, "Finetune/Step Time": 4.541506592184305} +{"Finetune/Learning Rate": 5.124594006964263e-06, "Finetune/Loss": 0.864177942276001, "Finetune/Loss (Raw)": 0.9942227602005005, "Finetune/Step": 3494, "Finetune/Step Time": 4.5290387235581875} +{"Finetune/Learning Rate": 5.1191558783011315e-06, "Finetune/Loss": 0.8651385307312012, "Finetune/Loss (Raw)": 0.9393223524093628, "Finetune/Step": 3495, "Finetune/Step Time": 4.528653649613261} +{"Finetune/Learning Rate": 5.113719643798312e-06, "Finetune/Loss": 0.8691933155059814, "Finetune/Loss (Raw)": 0.9567625522613525, "Finetune/Step": 3496, "Finetune/Step Time": 4.498762853443623} +{"Finetune/Learning Rate": 5.108285305565508e-06, "Finetune/Loss": 0.8675416707992554, "Finetune/Loss (Raw)": 0.7798063158988953, "Finetune/Step": 3497, "Finetune/Step Time": 4.507947476580739} +{"Finetune/Learning Rate": 5.1028528657116715e-06, "Finetune/Loss": 0.8672060966491699, "Finetune/Loss (Raw)": 0.9129820466041565, "Finetune/Step": 3498, "Finetune/Step Time": 4.512114819139242} +{"Finetune/Learning Rate": 5.097422326345024e-06, "Finetune/Loss": 0.8693294525146484, "Finetune/Loss (Raw)": 0.9594211578369141, "Finetune/Step": 3499, "Finetune/Step Time": 4.497456718236208} +{"Finetune/Learning Rate": 5.091993689573062e-06, "Finetune/Loss": 0.8700049519538879, "Finetune/Loss (Raw)": 0.9564245343208313, "Finetune/Step": 3500, "Finetune/Step Time": 4.488214647397399} +{"Finetune/Learning Rate": 5.086566957502527e-06, "Finetune/Loss": 0.8699024319648743, "Finetune/Loss (Raw)": 0.9413151144981384, "Finetune/Step": 3501, "Finetune/Step Time": 4.474271537736058} +{"Finetune/Learning Rate": 5.081142132239429e-06, "Finetune/Loss": 0.8711783289909363, "Finetune/Loss (Raw)": 1.0637657642364502, "Finetune/Step": 3502, "Finetune/Step Time": 4.469647344201803} +{"Finetune/Learning Rate": 5.075719215889039e-06, "Finetune/Loss": 0.8698928952217102, "Finetune/Loss (Raw)": 0.961337149143219, "Finetune/Step": 3503, "Finetune/Step Time": 4.469993023201823} +{"Finetune/Learning Rate": 5.070298210555881e-06, "Finetune/Loss": 0.8685972094535828, "Finetune/Loss (Raw)": 0.8483299016952515, "Finetune/Step": 3504, "Finetune/Step Time": 4.477905610576272} +{"Finetune/Learning Rate": 5.064879118343744e-06, "Finetune/Loss": 0.8697683811187744, "Finetune/Loss (Raw)": 0.9560055732727051, "Finetune/Step": 3505, "Finetune/Step Time": 4.479240711778402} +{"Finetune/Learning Rate": 5.059461941355665e-06, "Finetune/Loss": 0.8685376644134521, "Finetune/Loss (Raw)": 0.8290529251098633, "Finetune/Step": 3506, "Finetune/Step Time": 4.476155513897538} +{"Finetune/Learning Rate": 5.05404668169396e-06, "Finetune/Loss": 0.8682672381401062, "Finetune/Loss (Raw)": 0.6776124835014343, "Finetune/Step": 3507, "Finetune/Step Time": 4.4856441877782345} +{"Finetune/Learning Rate": 5.048633341460174e-06, "Finetune/Loss": 0.8690381646156311, "Finetune/Loss (Raw)": 0.8870289921760559, "Finetune/Step": 3508, "Finetune/Step Time": 4.485694600269198} +{"Finetune/Learning Rate": 5.043221922755115e-06, "Finetune/Loss": 0.8697119951248169, "Finetune/Loss (Raw)": 0.9694268107414246, "Finetune/Step": 3509, "Finetune/Step Time": 4.493712058290839} +{"Finetune/Learning Rate": 5.03781242767886e-06, "Finetune/Loss": 0.8678860068321228, "Finetune/Loss (Raw)": 0.8027945756912231, "Finetune/Step": 3510, "Finetune/Step Time": 4.4865697007626295} +{"Finetune/Learning Rate": 5.032404858330721e-06, "Finetune/Loss": 0.8681102991104126, "Finetune/Loss (Raw)": 0.8794147968292236, "Finetune/Step": 3511, "Finetune/Step Time": 4.493821220472455} +{"Finetune/Learning Rate": 5.026999216809275e-06, "Finetune/Loss": 0.8679051399230957, "Finetune/Loss (Raw)": 0.9124153256416321, "Finetune/Step": 3512, "Finetune/Step Time": 4.486649923026562} +{"Finetune/Learning Rate": 5.021595505212342e-06, "Finetune/Loss": 0.8691246509552002, "Finetune/Loss (Raw)": 0.9404056668281555, "Finetune/Step": 3513, "Finetune/Step Time": 4.486590947955847} +{"Finetune/Learning Rate": 5.016193725637e-06, "Finetune/Loss": 0.8684985041618347, "Finetune/Loss (Raw)": 0.8049488067626953, "Finetune/Step": 3514, "Finetune/Step Time": 4.490782802924514} +{"Finetune/Learning Rate": 5.0107938801795695e-06, "Finetune/Loss": 0.8689287304878235, "Finetune/Loss (Raw)": 0.8550911545753479, "Finetune/Step": 3515, "Finetune/Step Time": 4.4977782014757395} +{"Finetune/Learning Rate": 5.005395970935627e-06, "Finetune/Loss": 0.8694053888320923, "Finetune/Loss (Raw)": 0.8782333731651306, "Finetune/Step": 3516, "Finetune/Step Time": 4.504673633724451} +{"Finetune/Learning Rate": 5.000000000000003e-06, "Finetune/Loss": 0.8695865273475647, "Finetune/Loss (Raw)": 0.9533843398094177, "Finetune/Step": 3517, "Finetune/Step Time": 4.502294883131981} +{"Finetune/Learning Rate": 4.994605969466758e-06, "Finetune/Loss": 0.8690063953399658, "Finetune/Loss (Raw)": 0.884231686592102, "Finetune/Step": 3518, "Finetune/Step Time": 4.527308797463775} +{"Finetune/Learning Rate": 4.989213881429211e-06, "Finetune/Loss": 0.8669911026954651, "Finetune/Loss (Raw)": 0.8051058650016785, "Finetune/Step": 3519, "Finetune/Step Time": 4.555331073701382} +{"Finetune/Learning Rate": 4.983823737979933e-06, "Finetune/Loss": 0.8674446940422058, "Finetune/Loss (Raw)": 0.9145627021789551, "Finetune/Step": 3520, "Finetune/Step Time": 4.556794006377459} +{"Finetune/Learning Rate": 4.97843554121073e-06, "Finetune/Loss": 0.8675951957702637, "Finetune/Loss (Raw)": 1.018230676651001, "Finetune/Step": 3521, "Finetune/Step Time": 4.5570589154958725} +{"Finetune/Learning Rate": 4.973049293212655e-06, "Finetune/Loss": 0.8666882514953613, "Finetune/Loss (Raw)": 0.7855435609817505, "Finetune/Step": 3522, "Finetune/Step Time": 4.553153682500124} +{"Finetune/Learning Rate": 4.967664996076007e-06, "Finetune/Loss": 0.8651516437530518, "Finetune/Loss (Raw)": 0.7907267808914185, "Finetune/Step": 3523, "Finetune/Step Time": 4.578504392877221} +{"Finetune/Learning Rate": 4.962282651890325e-06, "Finetune/Loss": 0.8653932213783264, "Finetune/Loss (Raw)": 0.791237473487854, "Finetune/Step": 3524, "Finetune/Step Time": 4.587892476469278} +{"Finetune/Learning Rate": 4.9569022627443944e-06, "Finetune/Loss": 0.8665916919708252, "Finetune/Loss (Raw)": 1.0471211671829224, "Finetune/Step": 3525, "Finetune/Step Time": 4.596889525651932} +{"Finetune/Learning Rate": 4.951523830726236e-06, "Finetune/Loss": 0.8663633465766907, "Finetune/Loss (Raw)": 0.8604899644851685, "Finetune/Step": 3526, "Finetune/Step Time": 4.60500063188374} +{"Finetune/Learning Rate": 4.946147357923116e-06, "Finetune/Loss": 0.8672752380371094, "Finetune/Loss (Raw)": 0.9414264559745789, "Finetune/Step": 3527, "Finetune/Step Time": 4.587465783581138} +{"Finetune/Learning Rate": 4.940772846421538e-06, "Finetune/Loss": 0.8676531314849854, "Finetune/Loss (Raw)": 0.8878971934318542, "Finetune/Step": 3528, "Finetune/Step Time": 4.586508106440306} +{"Finetune/Learning Rate": 4.935400298307241e-06, "Finetune/Loss": 0.8676543235778809, "Finetune/Loss (Raw)": 0.810930073261261, "Finetune/Step": 3529, "Finetune/Step Time": 4.596830433234572} +{"Finetune/Learning Rate": 4.930029715665215e-06, "Finetune/Loss": 0.8678668737411499, "Finetune/Loss (Raw)": 0.912338137626648, "Finetune/Step": 3530, "Finetune/Step Time": 4.592857202515006} +{"Finetune/Learning Rate": 4.9246611005796776e-06, "Finetune/Loss": 0.8690067529678345, "Finetune/Loss (Raw)": 0.972771406173706, "Finetune/Step": 3531, "Finetune/Step Time": 4.592363763600588} +{"Finetune/Learning Rate": 4.919294455134071e-06, "Finetune/Loss": 0.8683491945266724, "Finetune/Loss (Raw)": 0.821399450302124, "Finetune/Step": 3532, "Finetune/Step Time": 4.604582272469997} +{"Finetune/Learning Rate": 4.913929781411099e-06, "Finetune/Loss": 0.8650380969047546, "Finetune/Loss (Raw)": 0.32211658358573914, "Finetune/Step": 3533, "Finetune/Step Time": 4.5761641804128885} +{"Finetune/Learning Rate": 4.90856708149268e-06, "Finetune/Loss": 0.865037202835083, "Finetune/Loss (Raw)": 0.9265691637992859, "Finetune/Step": 3534, "Finetune/Step Time": 4.58417440392077} +{"Finetune/Learning Rate": 4.903206357459976e-06, "Finetune/Loss": 0.8605901002883911, "Finetune/Loss (Raw)": 0.33891114592552185, "Finetune/Step": 3535, "Finetune/Step Time": 4.626424491405487} +{"Finetune/Learning Rate": 4.897847611393378e-06, "Finetune/Loss": 0.8598482608795166, "Finetune/Loss (Raw)": 0.8482704758644104, "Finetune/Step": 3536, "Finetune/Step Time": 4.624205909669399} +{"Finetune/Learning Rate": 4.892490845372511e-06, "Finetune/Loss": 0.8609243631362915, "Finetune/Loss (Raw)": 1.0203132629394531, "Finetune/Step": 3537, "Finetune/Step Time": 4.6367960553616285} +{"Finetune/Learning Rate": 4.8871360614762295e-06, "Finetune/Loss": 0.8623011708259583, "Finetune/Loss (Raw)": 1.00150728225708, "Finetune/Step": 3538, "Finetune/Step Time": 4.6339981108903885} +{"Finetune/Learning Rate": 4.881783261782621e-06, "Finetune/Loss": 0.8617771863937378, "Finetune/Loss (Raw)": 0.9415619969367981, "Finetune/Step": 3539, "Finetune/Step Time": 4.638278715312481} +{"Finetune/Learning Rate": 4.876432448369006e-06, "Finetune/Loss": 0.8612751960754395, "Finetune/Loss (Raw)": 0.8838685750961304, "Finetune/Step": 3540, "Finetune/Step Time": 4.642924828454852} +{"Finetune/Learning Rate": 4.871083623311933e-06, "Finetune/Loss": 0.8602114915847778, "Finetune/Loss (Raw)": 0.8881340026855469, "Finetune/Step": 3541, "Finetune/Step Time": 4.633908096700907} +{"Finetune/Learning Rate": 4.865736788687164e-06, "Finetune/Loss": 0.858923614025116, "Finetune/Loss (Raw)": 0.8707633018493652, "Finetune/Step": 3542, "Finetune/Step Time": 4.643591510131955} +{"Finetune/Learning Rate": 4.860391946569714e-06, "Finetune/Loss": 0.859951376914978, "Finetune/Loss (Raw)": 0.8771558403968811, "Finetune/Step": 3543, "Finetune/Step Time": 4.65635115839541} +{"Finetune/Learning Rate": 4.855049099033806e-06, "Finetune/Loss": 0.8638715744018555, "Finetune/Loss (Raw)": 0.9259364008903503, "Finetune/Step": 3544, "Finetune/Step Time": 4.618061164394021} +{"Finetune/Learning Rate": 4.849708248152897e-06, "Finetune/Loss": 0.8600793480873108, "Finetune/Loss (Raw)": 0.4208128750324249, "Finetune/Step": 3545, "Finetune/Step Time": 4.650450725108385} +{"Finetune/Learning Rate": 4.8443693959996655e-06, "Finetune/Loss": 0.8595510721206665, "Finetune/Loss (Raw)": 0.8275661468505859, "Finetune/Step": 3546, "Finetune/Step Time": 4.647347757592797} +{"Finetune/Learning Rate": 4.839032544646016e-06, "Finetune/Loss": 0.8603256940841675, "Finetune/Loss (Raw)": 1.0082215070724487, "Finetune/Step": 3547, "Finetune/Step Time": 4.644353779032826} +{"Finetune/Learning Rate": 4.833697696163077e-06, "Finetune/Loss": 0.8625943660736084, "Finetune/Loss (Raw)": 0.9409439563751221, "Finetune/Step": 3548, "Finetune/Step Time": 4.632158324122429} +{"Finetune/Learning Rate": 4.828364852621195e-06, "Finetune/Loss": 0.8632184267044067, "Finetune/Loss (Raw)": 0.9998528957366943, "Finetune/Step": 3549, "Finetune/Step Time": 4.631040835753083} +{"Finetune/Learning Rate": 4.823034016089949e-06, "Finetune/Loss": 0.8590912818908691, "Finetune/Loss (Raw)": 0.3448442816734314, "Finetune/Step": 3550, "Finetune/Step Time": 4.647745043039322} +{"Finetune/Learning Rate": 4.8177051886381345e-06, "Finetune/Loss": 0.8588447570800781, "Finetune/Loss (Raw)": 0.787097692489624, "Finetune/Step": 3551, "Finetune/Step Time": 4.6658315025269985} +{"Finetune/Learning Rate": 4.812378372333751e-06, "Finetune/Loss": 0.8631774187088013, "Finetune/Loss (Raw)": 0.9076877236366272, "Finetune/Step": 3552, "Finetune/Step Time": 4.62541875615716} +{"Finetune/Learning Rate": 4.807053569244047e-06, "Finetune/Loss": 0.8621279001235962, "Finetune/Loss (Raw)": 0.7471199035644531, "Finetune/Step": 3553, "Finetune/Step Time": 4.633481461554766} +{"Finetune/Learning Rate": 4.801730781435469e-06, "Finetune/Loss": 0.863913357257843, "Finetune/Loss (Raw)": 0.9699267148971558, "Finetune/Step": 3554, "Finetune/Step Time": 4.636248989030719} +{"Finetune/Learning Rate": 4.796410010973686e-06, "Finetune/Loss": 0.8646317720413208, "Finetune/Loss (Raw)": 1.0696479082107544, "Finetune/Step": 3555, "Finetune/Step Time": 4.634085316210985} +{"Finetune/Learning Rate": 4.791091259923587e-06, "Finetune/Loss": 0.8603391647338867, "Finetune/Loss (Raw)": 0.3514828383922577, "Finetune/Step": 3556, "Finetune/Step Time": 4.663209995254874} +{"Finetune/Learning Rate": 4.785774530349275e-06, "Finetune/Loss": 0.8611903190612793, "Finetune/Loss (Raw)": 1.0358312129974365, "Finetune/Step": 3557, "Finetune/Step Time": 4.661064838990569} +{"Finetune/Learning Rate": 4.7804598243140664e-06, "Finetune/Loss": 0.8615792393684387, "Finetune/Loss (Raw)": 0.8928941488265991, "Finetune/Step": 3558, "Finetune/Step Time": 4.6629632115364075} +{"Finetune/Learning Rate": 4.775147143880496e-06, "Finetune/Loss": 0.8613688945770264, "Finetune/Loss (Raw)": 0.959252655506134, "Finetune/Step": 3559, "Finetune/Step Time": 4.661801433190703} +{"Finetune/Learning Rate": 4.769836491110314e-06, "Finetune/Loss": 0.8631703853607178, "Finetune/Loss (Raw)": 1.0054534673690796, "Finetune/Step": 3560, "Finetune/Step Time": 4.645500535145402} +{"Finetune/Learning Rate": 4.764527868064484e-06, "Finetune/Loss": 0.8646082282066345, "Finetune/Loss (Raw)": 1.0472921133041382, "Finetune/Step": 3561, "Finetune/Step Time": 4.629065943881869} +{"Finetune/Learning Rate": 4.759221276803172e-06, "Finetune/Loss": 0.863645076751709, "Finetune/Loss (Raw)": 0.7304475903511047, "Finetune/Step": 3562, "Finetune/Step Time": 4.6044247560203075} +{"Finetune/Learning Rate": 4.75391671938576e-06, "Finetune/Loss": 0.8628740310668945, "Finetune/Loss (Raw)": 0.7697914242744446, "Finetune/Step": 3563, "Finetune/Step Time": 4.590422602370381} +{"Finetune/Learning Rate": 4.748614197870852e-06, "Finetune/Loss": 0.8631353378295898, "Finetune/Loss (Raw)": 0.8224965929985046, "Finetune/Step": 3564, "Finetune/Step Time": 4.593064695596695} +{"Finetune/Learning Rate": 4.743313714316249e-06, "Finetune/Loss": 0.8626994490623474, "Finetune/Loss (Raw)": 0.8416669368743896, "Finetune/Step": 3565, "Finetune/Step Time": 4.596112936735153} +{"Finetune/Learning Rate": 4.738015270778965e-06, "Finetune/Loss": 0.8641250133514404, "Finetune/Loss (Raw)": 1.0125796794891357, "Finetune/Step": 3566, "Finetune/Step Time": 4.595467679202557} +{"Finetune/Learning Rate": 4.7327188693152235e-06, "Finetune/Loss": 0.8634284734725952, "Finetune/Loss (Raw)": 0.891460120677948, "Finetune/Step": 3567, "Finetune/Step Time": 4.599299976602197} +{"Finetune/Learning Rate": 4.727424511980453e-06, "Finetune/Loss": 0.8641044497489929, "Finetune/Loss (Raw)": 0.9476873874664307, "Finetune/Step": 3568, "Finetune/Step Time": 4.598715705797076} +{"Finetune/Learning Rate": 4.722132200829291e-06, "Finetune/Loss": 0.8652036190032959, "Finetune/Loss (Raw)": 0.8666240572929382, "Finetune/Step": 3569, "Finetune/Step Time": 4.581018367782235} +{"Finetune/Learning Rate": 4.716841937915582e-06, "Finetune/Loss": 0.8656479120254517, "Finetune/Loss (Raw)": 0.8388482928276062, "Finetune/Step": 3570, "Finetune/Step Time": 4.575326688587666} +{"Finetune/Learning Rate": 4.71155372529237e-06, "Finetune/Loss": 0.8664771318435669, "Finetune/Loss (Raw)": 0.9313200116157532, "Finetune/Step": 3571, "Finetune/Step Time": 4.578431826084852} +{"Finetune/Learning Rate": 4.706267565011909e-06, "Finetune/Loss": 0.8666204810142517, "Finetune/Loss (Raw)": 0.9398592114448547, "Finetune/Step": 3572, "Finetune/Step Time": 4.5802636574953794} +{"Finetune/Learning Rate": 4.700983459125652e-06, "Finetune/Loss": 0.86838698387146, "Finetune/Loss (Raw)": 0.940367579460144, "Finetune/Step": 3573, "Finetune/Step Time": 4.562095522880554} +{"Finetune/Learning Rate": 4.695701409684264e-06, "Finetune/Loss": 0.8693602681159973, "Finetune/Loss (Raw)": 0.982655942440033, "Finetune/Step": 3574, "Finetune/Step Time": 4.560259873047471} +{"Finetune/Learning Rate": 4.690421418737604e-06, "Finetune/Loss": 0.8683876395225525, "Finetune/Loss (Raw)": 0.8728545308113098, "Finetune/Step": 3575, "Finetune/Step Time": 4.556005664169788} +{"Finetune/Learning Rate": 4.685143488334723e-06, "Finetune/Loss": 0.8691696524620056, "Finetune/Loss (Raw)": 0.9848005175590515, "Finetune/Step": 3576, "Finetune/Step Time": 4.553528808057308} +{"Finetune/Learning Rate": 4.679867620523895e-06, "Finetune/Loss": 0.868433952331543, "Finetune/Loss (Raw)": 0.8886935114860535, "Finetune/Step": 3577, "Finetune/Step Time": 4.572269041091204} +{"Finetune/Learning Rate": 4.674593817352575e-06, "Finetune/Loss": 0.8688907623291016, "Finetune/Loss (Raw)": 0.9696539044380188, "Finetune/Step": 3578, "Finetune/Step Time": 4.55664848908782} +{"Finetune/Learning Rate": 4.669322080867427e-06, "Finetune/Loss": 0.8697763681411743, "Finetune/Loss (Raw)": 0.9556314945220947, "Finetune/Step": 3579, "Finetune/Step Time": 4.551693744957447} +{"Finetune/Learning Rate": 4.664052413114305e-06, "Finetune/Loss": 0.8694696426391602, "Finetune/Loss (Raw)": 0.9298081398010254, "Finetune/Step": 3580, "Finetune/Step Time": 4.558749673888087} +{"Finetune/Learning Rate": 4.658784816138266e-06, "Finetune/Loss": 0.8704637289047241, "Finetune/Loss (Raw)": 0.9773229360580444, "Finetune/Step": 3581, "Finetune/Step Time": 4.574245722964406} +{"Finetune/Learning Rate": 4.653519291983561e-06, "Finetune/Loss": 0.8705315589904785, "Finetune/Loss (Raw)": 0.8463798761367798, "Finetune/Step": 3582, "Finetune/Step Time": 4.5763177536427975} +{"Finetune/Learning Rate": 4.648255842693635e-06, "Finetune/Loss": 0.874301552772522, "Finetune/Loss (Raw)": 0.8242650032043457, "Finetune/Step": 3583, "Finetune/Step Time": 4.538447007536888} +{"Finetune/Learning Rate": 4.642994470311136e-06, "Finetune/Loss": 0.8742423057556152, "Finetune/Loss (Raw)": 0.9365753531455994, "Finetune/Step": 3584, "Finetune/Step Time": 4.540128668770194} +{"Finetune/Learning Rate": 4.6377351768779e-06, "Finetune/Loss": 0.8753643035888672, "Finetune/Loss (Raw)": 1.0336754322052002, "Finetune/Step": 3585, "Finetune/Step Time": 4.5404327511787415} +{"Finetune/Learning Rate": 4.632477964434944e-06, "Finetune/Loss": 0.874267578125, "Finetune/Loss (Raw)": 0.7680976390838623, "Finetune/Step": 3586, "Finetune/Step Time": 4.538709064945579} +{"Finetune/Learning Rate": 4.6272228350225026e-06, "Finetune/Loss": 0.8740614652633667, "Finetune/Loss (Raw)": 0.8287542462348938, "Finetune/Step": 3587, "Finetune/Step Time": 4.537703707814217} +{"Finetune/Learning Rate": 4.621969790679984e-06, "Finetune/Loss": 0.8718624114990234, "Finetune/Loss (Raw)": 0.7045947909355164, "Finetune/Step": 3588, "Finetune/Step Time": 4.538852773606777} +{"Finetune/Learning Rate": 4.6167188334459936e-06, "Finetune/Loss": 0.8760778903961182, "Finetune/Loss (Raw)": 0.8760846853256226, "Finetune/Step": 3589, "Finetune/Step Time": 4.510199652984738} +{"Finetune/Learning Rate": 4.611469965358323e-06, "Finetune/Loss": 0.8761905431747437, "Finetune/Loss (Raw)": 0.9502398371696472, "Finetune/Step": 3590, "Finetune/Step Time": 4.513678174465895} +{"Finetune/Learning Rate": 4.606223188453956e-06, "Finetune/Loss": 0.8757379055023193, "Finetune/Loss (Raw)": 0.8060353994369507, "Finetune/Step": 3591, "Finetune/Step Time": 4.506965594366193} +{"Finetune/Learning Rate": 4.6009785047690666e-06, "Finetune/Loss": 0.8757017850875854, "Finetune/Loss (Raw)": 0.9288666844367981, "Finetune/Step": 3592, "Finetune/Step Time": 4.5122692584991455} +{"Finetune/Learning Rate": 4.595735916339008e-06, "Finetune/Loss": 0.8770672082901001, "Finetune/Loss (Raw)": 1.0128282308578491, "Finetune/Step": 3593, "Finetune/Step Time": 4.508683241903782} +{"Finetune/Learning Rate": 4.590495425198335e-06, "Finetune/Loss": 0.8754711151123047, "Finetune/Loss (Raw)": 0.617861807346344, "Finetune/Step": 3594, "Finetune/Step Time": 4.520751986652613} +{"Finetune/Learning Rate": 4.585257033380779e-06, "Finetune/Loss": 0.8767601251602173, "Finetune/Loss (Raw)": 0.9022252559661865, "Finetune/Step": 3595, "Finetune/Step Time": 4.532614791765809} +{"Finetune/Learning Rate": 4.580020742919247e-06, "Finetune/Loss": 0.8790934681892395, "Finetune/Loss (Raw)": 1.0329372882843018, "Finetune/Step": 3596, "Finetune/Step Time": 4.518138293176889} +{"Finetune/Learning Rate": 4.5747865558458524e-06, "Finetune/Loss": 0.8739049434661865, "Finetune/Loss (Raw)": 0.25394558906555176, "Finetune/Step": 3597, "Finetune/Step Time": 4.549722425639629} +{"Finetune/Learning Rate": 4.569554474191878e-06, "Finetune/Loss": 0.8738114833831787, "Finetune/Loss (Raw)": 0.831516444683075, "Finetune/Step": 3598, "Finetune/Step Time": 4.563807152211666} +{"Finetune/Learning Rate": 4.56432449998779e-06, "Finetune/Loss": 0.8728105425834656, "Finetune/Loss (Raw)": 0.8112242817878723, "Finetune/Step": 3599, "Finetune/Step Time": 4.555255722254515} +{"Finetune/Learning Rate": 4.559096635263242e-06, "Finetune/Loss": 0.8699198961257935, "Finetune/Loss (Raw)": 0.4357199966907501, "Finetune/Step": 3600, "Finetune/Step Time": 4.5796571131795645} +{"Finetune/Learning Rate": 4.553870882047064e-06, "Finetune/Loss": 0.8692516684532166, "Finetune/Loss (Raw)": 0.9361627697944641, "Finetune/Step": 3601, "Finetune/Step Time": 4.557204242795706} +{"Finetune/Learning Rate": 4.548647242367271e-06, "Finetune/Loss": 0.8696552515029907, "Finetune/Loss (Raw)": 0.8496759533882141, "Finetune/Step": 3602, "Finetune/Step Time": 4.5522200632840395} +{"Finetune/Learning Rate": 4.543425718251051e-06, "Finetune/Loss": 0.8694108724594116, "Finetune/Loss (Raw)": 0.8562266826629639, "Finetune/Step": 3603, "Finetune/Step Time": 4.551156062632799} +{"Finetune/Learning Rate": 4.538206311724782e-06, "Finetune/Loss": 0.8706756830215454, "Finetune/Loss (Raw)": 0.9111171960830688, "Finetune/Step": 3604, "Finetune/Step Time": 4.54782972484827} +{"Finetune/Learning Rate": 4.532989024814015e-06, "Finetune/Loss": 0.8703700304031372, "Finetune/Loss (Raw)": 0.7826021909713745, "Finetune/Step": 3605, "Finetune/Step Time": 4.534115951508284} +{"Finetune/Learning Rate": 4.527773859543466e-06, "Finetune/Loss": 0.869371771812439, "Finetune/Loss (Raw)": 0.87237548828125, "Finetune/Step": 3606, "Finetune/Step Time": 4.551434716209769} +{"Finetune/Learning Rate": 4.52256081793705e-06, "Finetune/Loss": 0.8682806491851807, "Finetune/Loss (Raw)": 0.9164232611656189, "Finetune/Step": 3607, "Finetune/Step Time": 4.554962296038866} +{"Finetune/Learning Rate": 4.517349902017841e-06, "Finetune/Loss": 0.8679287433624268, "Finetune/Loss (Raw)": 0.8812340497970581, "Finetune/Step": 3608, "Finetune/Step Time": 4.582644442096353} +{"Finetune/Learning Rate": 4.512141113808097e-06, "Finetune/Loss": 0.8676110506057739, "Finetune/Loss (Raw)": 0.9129049777984619, "Finetune/Step": 3609, "Finetune/Step Time": 4.58330450579524} +{"Finetune/Learning Rate": 4.5069344553292436e-06, "Finetune/Loss": 0.8667820692062378, "Finetune/Loss (Raw)": 0.79476398229599, "Finetune/Step": 3610, "Finetune/Step Time": 4.582592699676752} +{"Finetune/Learning Rate": 4.501729928601887e-06, "Finetune/Loss": 0.862521767616272, "Finetune/Loss (Raw)": 0.3411983549594879, "Finetune/Step": 3611, "Finetune/Step Time": 4.577517895027995} +{"Finetune/Learning Rate": 4.4965275356458e-06, "Finetune/Loss": 0.8620003461837769, "Finetune/Loss (Raw)": 0.8614600896835327, "Finetune/Step": 3612, "Finetune/Step Time": 4.574674550443888} +{"Finetune/Learning Rate": 4.491327278479924e-06, "Finetune/Loss": 0.8604406118392944, "Finetune/Loss (Raw)": 0.7951266765594482, "Finetune/Step": 3613, "Finetune/Step Time": 4.575341938063502} +{"Finetune/Learning Rate": 4.4861291591223934e-06, "Finetune/Loss": 0.859857439994812, "Finetune/Loss (Raw)": 0.8681938052177429, "Finetune/Step": 3614, "Finetune/Step Time": 4.5796788688749075} +{"Finetune/Learning Rate": 4.4809331795904835e-06, "Finetune/Loss": 0.8610804080963135, "Finetune/Loss (Raw)": 0.9173858165740967, "Finetune/Step": 3615, "Finetune/Step Time": 4.584602365270257} +{"Finetune/Learning Rate": 4.47573934190065e-06, "Finetune/Loss": 0.8612006306648254, "Finetune/Loss (Raw)": 0.7742407321929932, "Finetune/Step": 3616, "Finetune/Step Time": 4.588817436248064} +{"Finetune/Learning Rate": 4.470547648068532e-06, "Finetune/Loss": 0.8602851629257202, "Finetune/Loss (Raw)": 0.8685213923454285, "Finetune/Step": 3617, "Finetune/Step Time": 4.597014052793384} +{"Finetune/Learning Rate": 4.465358100108916e-06, "Finetune/Loss": 0.8606100678443909, "Finetune/Loss (Raw)": 0.8363081216812134, "Finetune/Step": 3618, "Finetune/Step Time": 4.60458049736917} +{"Finetune/Learning Rate": 4.460170700035769e-06, "Finetune/Loss": 0.859484076499939, "Finetune/Loss (Raw)": 0.8512305617332458, "Finetune/Step": 3619, "Finetune/Step Time": 4.602312304079533} +{"Finetune/Learning Rate": 4.454985449862215e-06, "Finetune/Loss": 0.8589238524436951, "Finetune/Loss (Raw)": 0.8431054949760437, "Finetune/Step": 3620, "Finetune/Step Time": 4.5921768844127655} +{"Finetune/Learning Rate": 4.449802351600551e-06, "Finetune/Loss": 0.8597780466079712, "Finetune/Loss (Raw)": 0.8203876614570618, "Finetune/Step": 3621, "Finetune/Step Time": 4.586085623130202} +{"Finetune/Learning Rate": 4.444621407262237e-06, "Finetune/Loss": 0.8590579032897949, "Finetune/Loss (Raw)": 0.9020379781723022, "Finetune/Step": 3622, "Finetune/Step Time": 4.584970206022263} +{"Finetune/Learning Rate": 4.439442618857891e-06, "Finetune/Loss": 0.8589032888412476, "Finetune/Loss (Raw)": 0.9195291996002197, "Finetune/Step": 3623, "Finetune/Step Time": 4.585680045187473} +{"Finetune/Learning Rate": 4.434265988397312e-06, "Finetune/Loss": 0.8590967655181885, "Finetune/Loss (Raw)": 0.9815365672111511, "Finetune/Step": 3624, "Finetune/Step Time": 4.579485135152936} +{"Finetune/Learning Rate": 4.4290915178894355e-06, "Finetune/Loss": 0.8604505658149719, "Finetune/Loss (Raw)": 0.95308518409729, "Finetune/Step": 3625, "Finetune/Step Time": 4.569872733205557} +{"Finetune/Learning Rate": 4.423919209342376e-06, "Finetune/Loss": 0.8595675230026245, "Finetune/Loss (Raw)": 0.7999556660652161, "Finetune/Step": 3626, "Finetune/Step Time": 4.566093470901251} +{"Finetune/Learning Rate": 4.41874906476341e-06, "Finetune/Loss": 0.8592181205749512, "Finetune/Loss (Raw)": 0.914698600769043, "Finetune/Step": 3627, "Finetune/Step Time": 4.580789467319846} +{"Finetune/Learning Rate": 4.413581086158969e-06, "Finetune/Loss": 0.8587082624435425, "Finetune/Loss (Raw)": 0.8911588788032532, "Finetune/Step": 3628, "Finetune/Step Time": 4.586336152628064} +{"Finetune/Learning Rate": 4.408415275534641e-06, "Finetune/Loss": 0.8578490018844604, "Finetune/Loss (Raw)": 0.8313330411911011, "Finetune/Step": 3629, "Finetune/Step Time": 4.592938745394349} +{"Finetune/Learning Rate": 4.4032516348951795e-06, "Finetune/Loss": 0.856238842010498, "Finetune/Loss (Raw)": 0.8576605319976807, "Finetune/Step": 3630, "Finetune/Step Time": 4.59300229139626} +{"Finetune/Learning Rate": 4.3980901662444905e-06, "Finetune/Loss": 0.8543900847434998, "Finetune/Loss (Raw)": 0.7246983051300049, "Finetune/Step": 3631, "Finetune/Step Time": 4.607249215245247} +{"Finetune/Learning Rate": 4.39293087158564e-06, "Finetune/Loss": 0.8554675579071045, "Finetune/Loss (Raw)": 0.9862467050552368, "Finetune/Step": 3632, "Finetune/Step Time": 4.603937080129981} +{"Finetune/Learning Rate": 4.387773752920845e-06, "Finetune/Loss": 0.8558224439620972, "Finetune/Loss (Raw)": 1.0014344453811646, "Finetune/Step": 3633, "Finetune/Step Time": 4.5932382475584745} +{"Finetune/Learning Rate": 4.382618812251495e-06, "Finetune/Loss": 0.856177568435669, "Finetune/Loss (Raw)": 0.8745068907737732, "Finetune/Step": 3634, "Finetune/Step Time": 4.585949996486306} +{"Finetune/Learning Rate": 4.377466051578111e-06, "Finetune/Loss": 0.8569445610046387, "Finetune/Loss (Raw)": 0.7757915258407593, "Finetune/Step": 3635, "Finetune/Step Time": 4.575246332213283} +{"Finetune/Learning Rate": 4.372315472900377e-06, "Finetune/Loss": 0.8570963144302368, "Finetune/Loss (Raw)": 0.9064499139785767, "Finetune/Step": 3636, "Finetune/Step Time": 4.574431046843529} +{"Finetune/Learning Rate": 4.367167078217141e-06, "Finetune/Loss": 0.8571176528930664, "Finetune/Loss (Raw)": 0.9721652865409851, "Finetune/Step": 3637, "Finetune/Step Time": 4.560426376760006} +{"Finetune/Learning Rate": 4.36202086952639e-06, "Finetune/Loss": 0.858230710029602, "Finetune/Loss (Raw)": 0.9452601075172424, "Finetune/Step": 3638, "Finetune/Step Time": 4.558199496939778} +{"Finetune/Learning Rate": 4.356876848825266e-06, "Finetune/Loss": 0.8582907915115356, "Finetune/Loss (Raw)": 0.8870999813079834, "Finetune/Step": 3639, "Finetune/Step Time": 4.552789648994803} +{"Finetune/Learning Rate": 4.351735018110066e-06, "Finetune/Loss": 0.8535827994346619, "Finetune/Loss (Raw)": 0.30979666113853455, "Finetune/Step": 3640, "Finetune/Step Time": 4.590879505500197} +{"Finetune/Learning Rate": 4.346595379376232e-06, "Finetune/Loss": 0.8525243401527405, "Finetune/Loss (Raw)": 0.8049209713935852, "Finetune/Step": 3641, "Finetune/Step Time": 4.5937005411833525} +{"Finetune/Learning Rate": 4.341457934618357e-06, "Finetune/Loss": 0.8537171483039856, "Finetune/Loss (Raw)": 0.9576292037963867, "Finetune/Step": 3642, "Finetune/Step Time": 4.597336169332266} +{"Finetune/Learning Rate": 4.336322685830181e-06, "Finetune/Loss": 0.8542730808258057, "Finetune/Loss (Raw)": 0.9262537956237793, "Finetune/Step": 3643, "Finetune/Step Time": 4.590239642187953} +{"Finetune/Learning Rate": 4.331189635004604e-06, "Finetune/Loss": 0.8539797067642212, "Finetune/Loss (Raw)": 0.8406771421432495, "Finetune/Step": 3644, "Finetune/Step Time": 4.581548253074288} +{"Finetune/Learning Rate": 4.326058784133652e-06, "Finetune/Loss": 0.853017270565033, "Finetune/Loss (Raw)": 0.8301951885223389, "Finetune/Step": 3645, "Finetune/Step Time": 4.580864783376455} +{"Finetune/Learning Rate": 4.3209301352085075e-06, "Finetune/Loss": 0.8537800908088684, "Finetune/Loss (Raw)": 0.9818682074546814, "Finetune/Step": 3646, "Finetune/Step Time": 4.56523702852428} +{"Finetune/Learning Rate": 4.315803690219507e-06, "Finetune/Loss": 0.8546750545501709, "Finetune/Loss (Raw)": 0.9196604490280151, "Finetune/Step": 3647, "Finetune/Step Time": 4.544615343213081} +{"Finetune/Learning Rate": 4.310679451156122e-06, "Finetune/Loss": 0.8549953699111938, "Finetune/Loss (Raw)": 0.9555652141571045, "Finetune/Step": 3648, "Finetune/Step Time": 4.546338990330696} +{"Finetune/Learning Rate": 4.305557420006962e-06, "Finetune/Loss": 0.8545390367507935, "Finetune/Loss (Raw)": 0.9598245620727539, "Finetune/Step": 3649, "Finetune/Step Time": 4.55075497739017} +{"Finetune/Learning Rate": 4.300437598759794e-06, "Finetune/Loss": 0.8564085364341736, "Finetune/Loss (Raw)": 1.0248326063156128, "Finetune/Step": 3650, "Finetune/Step Time": 4.53641664981842} +{"Finetune/Learning Rate": 4.295319989401522e-06, "Finetune/Loss": 0.8561002612113953, "Finetune/Loss (Raw)": 0.7512752413749695, "Finetune/Step": 3651, "Finetune/Step Time": 4.5258401073515415} +{"Finetune/Learning Rate": 4.2902045939181855e-06, "Finetune/Loss": 0.8573211431503296, "Finetune/Loss (Raw)": 0.9475120902061462, "Finetune/Step": 3652, "Finetune/Step Time": 4.518490668386221} +{"Finetune/Learning Rate": 4.285091414294973e-06, "Finetune/Loss": 0.8566312789916992, "Finetune/Loss (Raw)": 0.9588122367858887, "Finetune/Step": 3653, "Finetune/Step Time": 4.510226780548692} +{"Finetune/Learning Rate": 4.2799804525162085e-06, "Finetune/Loss": 0.856903076171875, "Finetune/Loss (Raw)": 0.8952842950820923, "Finetune/Step": 3654, "Finetune/Step Time": 4.519944028928876} +{"Finetune/Learning Rate": 4.274871710565356e-06, "Finetune/Loss": 0.8577790260314941, "Finetune/Loss (Raw)": 1.0535496473312378, "Finetune/Step": 3655, "Finetune/Step Time": 4.52454636991024} +{"Finetune/Learning Rate": 4.269765190425015e-06, "Finetune/Loss": 0.8569753170013428, "Finetune/Loss (Raw)": 0.7850198745727539, "Finetune/Step": 3656, "Finetune/Step Time": 4.528664795681834} +{"Finetune/Learning Rate": 4.264660894076934e-06, "Finetune/Loss": 0.8574934005737305, "Finetune/Loss (Raw)": 0.877247154712677, "Finetune/Step": 3657, "Finetune/Step Time": 4.523091219365597} +{"Finetune/Learning Rate": 4.25955882350199e-06, "Finetune/Loss": 0.8573343753814697, "Finetune/Loss (Raw)": 0.8919801712036133, "Finetune/Step": 3658, "Finetune/Step Time": 4.5386730674654245} +{"Finetune/Learning Rate": 4.254458980680188e-06, "Finetune/Loss": 0.8568731546401978, "Finetune/Loss (Raw)": 0.913734495639801, "Finetune/Step": 3659, "Finetune/Step Time": 4.542024513706565} +{"Finetune/Learning Rate": 4.2493613675906865e-06, "Finetune/Loss": 0.8583037853240967, "Finetune/Loss (Raw)": 1.0045223236083984, "Finetune/Step": 3660, "Finetune/Step Time": 4.540200920775533} +{"Finetune/Learning Rate": 4.244265986211766e-06, "Finetune/Loss": 0.8626037240028381, "Finetune/Loss (Raw)": 0.8725011348724365, "Finetune/Step": 3661, "Finetune/Step Time": 4.501464989036322} +{"Finetune/Learning Rate": 4.239172838520846e-06, "Finetune/Loss": 0.8630024194717407, "Finetune/Loss (Raw)": 0.9776076078414917, "Finetune/Step": 3662, "Finetune/Step Time": 4.491945944726467} +{"Finetune/Learning Rate": 4.2340819264944755e-06, "Finetune/Loss": 0.868651270866394, "Finetune/Loss (Raw)": 1.06195867061615, "Finetune/Step": 3663, "Finetune/Step Time": 4.447438525035977} +{"Finetune/Learning Rate": 4.22899325210834e-06, "Finetune/Loss": 0.8685099482536316, "Finetune/Loss (Raw)": 0.8301844000816345, "Finetune/Step": 3664, "Finetune/Step Time": 4.459782445803285} +{"Finetune/Learning Rate": 4.223906817337253e-06, "Finetune/Loss": 0.8677238821983337, "Finetune/Loss (Raw)": 0.9196988344192505, "Finetune/Step": 3665, "Finetune/Step Time": 4.446438634768128} +{"Finetune/Learning Rate": 4.218822624155159e-06, "Finetune/Loss": 0.8671298027038574, "Finetune/Loss (Raw)": 0.9254669547080994, "Finetune/Step": 3666, "Finetune/Step Time": 4.441505823284388} +{"Finetune/Learning Rate": 4.213740674535141e-06, "Finetune/Loss": 0.8630931377410889, "Finetune/Loss (Raw)": 0.4248698353767395, "Finetune/Step": 3667, "Finetune/Step Time": 4.4759969599545} +{"Finetune/Learning Rate": 4.2086609704494015e-06, "Finetune/Loss": 0.8626254796981812, "Finetune/Loss (Raw)": 0.8240032196044922, "Finetune/Step": 3668, "Finetune/Step Time": 4.465648839250207} +{"Finetune/Learning Rate": 4.203583513869267e-06, "Finetune/Loss": 0.862409234046936, "Finetune/Loss (Raw)": 0.8604608178138733, "Finetune/Step": 3669, "Finetune/Step Time": 4.469140522181988} +{"Finetune/Learning Rate": 4.198508306765209e-06, "Finetune/Loss": 0.8627192378044128, "Finetune/Loss (Raw)": 0.9104418158531189, "Finetune/Step": 3670, "Finetune/Step Time": 4.4544885996729136} +{"Finetune/Learning Rate": 4.193435351106811e-06, "Finetune/Loss": 0.8635216951370239, "Finetune/Loss (Raw)": 0.9798669219017029, "Finetune/Step": 3671, "Finetune/Step Time": 4.4362340066581964} +{"Finetune/Learning Rate": 4.188364648862791e-06, "Finetune/Loss": 0.8628929853439331, "Finetune/Loss (Raw)": 0.8454654812812805, "Finetune/Step": 3672, "Finetune/Step Time": 4.444321375340223} +{"Finetune/Learning Rate": 4.183296202000988e-06, "Finetune/Loss": 0.8648238182067871, "Finetune/Loss (Raw)": 0.6679614782333374, "Finetune/Step": 3673, "Finetune/Step Time": 4.410737095400691} +{"Finetune/Learning Rate": 4.178230012488368e-06, "Finetune/Loss": 0.8649999499320984, "Finetune/Loss (Raw)": 0.8501096367835999, "Finetune/Step": 3674, "Finetune/Step Time": 4.406865920871496} +{"Finetune/Learning Rate": 4.173166082291019e-06, "Finetune/Loss": 0.86006760597229, "Finetune/Loss (Raw)": 0.376878947019577, "Finetune/Step": 3675, "Finetune/Step Time": 4.448832493275404} +{"Finetune/Learning Rate": 4.16810441337415e-06, "Finetune/Loss": 0.8601394891738892, "Finetune/Loss (Raw)": 0.9501421451568604, "Finetune/Step": 3676, "Finetune/Step Time": 4.4515501372516155} +{"Finetune/Learning Rate": 4.163045007702105e-06, "Finetune/Loss": 0.8599194884300232, "Finetune/Loss (Raw)": 0.9716900587081909, "Finetune/Step": 3677, "Finetune/Step Time": 4.451312739402056} +{"Finetune/Learning Rate": 4.157987867238338e-06, "Finetune/Loss": 0.8623220920562744, "Finetune/Loss (Raw)": 0.652388334274292, "Finetune/Step": 3678, "Finetune/Step Time": 4.41463428735733} +{"Finetune/Learning Rate": 4.152932993945418e-06, "Finetune/Loss": 0.8639276027679443, "Finetune/Loss (Raw)": 0.9925903677940369, "Finetune/Step": 3679, "Finetune/Step Time": 4.4024212416261435} +{"Finetune/Learning Rate": 4.147880389785051e-06, "Finetune/Loss": 0.8639697432518005, "Finetune/Loss (Raw)": 0.9130848050117493, "Finetune/Step": 3680, "Finetune/Step Time": 4.408490002155304} +{"Finetune/Learning Rate": 4.142830056718052e-06, "Finetune/Loss": 0.865719735622406, "Finetune/Loss (Raw)": 0.9711248278617859, "Finetune/Step": 3681, "Finetune/Step Time": 4.389253159984946} +{"Finetune/Learning Rate": 4.137781996704356e-06, "Finetune/Loss": 0.8666815757751465, "Finetune/Loss (Raw)": 1.0930371284484863, "Finetune/Step": 3682, "Finetune/Step Time": 4.386655921116471} +{"Finetune/Learning Rate": 4.1327362117030174e-06, "Finetune/Loss": 0.8653074502944946, "Finetune/Loss (Raw)": 0.8937655687332153, "Finetune/Step": 3683, "Finetune/Step Time": 4.387123975902796} +{"Finetune/Learning Rate": 4.127692703672207e-06, "Finetune/Loss": 0.8677643537521362, "Finetune/Loss (Raw)": 0.6659612655639648, "Finetune/Step": 3684, "Finetune/Step Time": 4.368016485124826} +{"Finetune/Learning Rate": 4.1226514745692115e-06, "Finetune/Loss": 0.8663945198059082, "Finetune/Loss (Raw)": 0.8604879975318909, "Finetune/Step": 3685, "Finetune/Step Time": 4.364585930481553} +{"Finetune/Learning Rate": 4.117612526350429e-06, "Finetune/Loss": 0.8665701150894165, "Finetune/Loss (Raw)": 0.9153748750686646, "Finetune/Step": 3686, "Finetune/Step Time": 4.363631462678313} +{"Finetune/Learning Rate": 4.112575860971384e-06, "Finetune/Loss": 0.8660123348236084, "Finetune/Loss (Raw)": 0.8878604173660278, "Finetune/Step": 3687, "Finetune/Step Time": 4.361819898709655} +{"Finetune/Learning Rate": 4.107541480386708e-06, "Finetune/Loss": 0.8646221160888672, "Finetune/Loss (Raw)": 0.8275008201599121, "Finetune/Step": 3688, "Finetune/Step Time": 4.370000718161464} +{"Finetune/Learning Rate": 4.102509386550141e-06, "Finetune/Loss": 0.8641715049743652, "Finetune/Loss (Raw)": 0.9896145462989807, "Finetune/Step": 3689, "Finetune/Step Time": 4.369873356074095} +{"Finetune/Learning Rate": 4.0974795814145384e-06, "Finetune/Loss": 0.8657456636428833, "Finetune/Loss (Raw)": 0.9319406151771545, "Finetune/Step": 3690, "Finetune/Step Time": 4.373052937909961} +{"Finetune/Learning Rate": 4.092452066931877e-06, "Finetune/Loss": 0.8621516227722168, "Finetune/Loss (Raw)": 0.30975741147994995, "Finetune/Step": 3691, "Finetune/Step Time": 4.409825408831239} +{"Finetune/Learning Rate": 4.087426845053236e-06, "Finetune/Loss": 0.8635622262954712, "Finetune/Loss (Raw)": 1.003047227859497, "Finetune/Step": 3692, "Finetune/Step Time": 4.403401656076312} +{"Finetune/Learning Rate": 4.082403917728796e-06, "Finetune/Loss": 0.8640685081481934, "Finetune/Loss (Raw)": 0.9064764976501465, "Finetune/Step": 3693, "Finetune/Step Time": 4.402703417465091} +{"Finetune/Learning Rate": 4.077383286907869e-06, "Finetune/Loss": 0.862230658531189, "Finetune/Loss (Raw)": 0.7773294448852539, "Finetune/Step": 3694, "Finetune/Step Time": 4.426360711455345} +{"Finetune/Learning Rate": 4.072364954538858e-06, "Finetune/Loss": 0.8640454411506653, "Finetune/Loss (Raw)": 1.1237547397613525, "Finetune/Step": 3695, "Finetune/Step Time": 4.405116569250822} +{"Finetune/Learning Rate": 4.067348922569281e-06, "Finetune/Loss": 0.8634686470031738, "Finetune/Loss (Raw)": 0.8738613128662109, "Finetune/Step": 3696, "Finetune/Step Time": 4.397576516494155} +{"Finetune/Learning Rate": 4.062335192945762e-06, "Finetune/Loss": 0.8624305725097656, "Finetune/Loss (Raw)": 0.7337460517883301, "Finetune/Step": 3697, "Finetune/Step Time": 4.413723716512322} +{"Finetune/Learning Rate": 4.057323767614032e-06, "Finetune/Loss": 0.8631119728088379, "Finetune/Loss (Raw)": 0.9260727763175964, "Finetune/Step": 3698, "Finetune/Step Time": 4.425430107861757} +{"Finetune/Learning Rate": 4.052314648518925e-06, "Finetune/Loss": 0.8630366325378418, "Finetune/Loss (Raw)": 0.9216670989990234, "Finetune/Step": 3699, "Finetune/Step Time": 4.411697871983051} +{"Finetune/Learning Rate": 4.047307837604383e-06, "Finetune/Loss": 0.8628544211387634, "Finetune/Loss (Raw)": 0.9165377020835876, "Finetune/Step": 3700, "Finetune/Step Time": 4.412298362702131} +{"Finetune/Learning Rate": 4.0423033368134545e-06, "Finetune/Loss": 0.8611452579498291, "Finetune/Loss (Raw)": 0.7216001152992249, "Finetune/Step": 3701, "Finetune/Step Time": 4.419677879661322} +{"Finetune/Learning Rate": 4.037301148088292e-06, "Finetune/Loss": 0.8607796430587769, "Finetune/Loss (Raw)": 0.9358583092689514, "Finetune/Step": 3702, "Finetune/Step Time": 4.408983878791332} +{"Finetune/Learning Rate": 4.032301273370132e-06, "Finetune/Loss": 0.8614183068275452, "Finetune/Loss (Raw)": 0.9546011090278625, "Finetune/Step": 3703, "Finetune/Step Time": 4.406608480960131} +{"Finetune/Learning Rate": 4.027303714599345e-06, "Finetune/Loss": 0.8609264492988586, "Finetune/Loss (Raw)": 0.9218383431434631, "Finetune/Step": 3704, "Finetune/Step Time": 4.412169201299548} +{"Finetune/Learning Rate": 4.022308473715379e-06, "Finetune/Loss": 0.8596854209899902, "Finetune/Loss (Raw)": 0.7298408150672913, "Finetune/Step": 3705, "Finetune/Step Time": 4.399079766124487} +{"Finetune/Learning Rate": 4.017315552656788e-06, "Finetune/Loss": 0.8588920831680298, "Finetune/Loss (Raw)": 0.8681111335754395, "Finetune/Step": 3706, "Finetune/Step Time": 4.401200197637081} +{"Finetune/Learning Rate": 4.01232495336123e-06, "Finetune/Loss": 0.8581370711326599, "Finetune/Loss (Raw)": 0.8589935302734375, "Finetune/Step": 3707, "Finetune/Step Time": 4.427367355674505} +{"Finetune/Learning Rate": 4.0073366777654574e-06, "Finetune/Loss": 0.8569809794425964, "Finetune/Loss (Raw)": 0.7818244695663452, "Finetune/Step": 3708, "Finetune/Step Time": 4.433257741853595} +{"Finetune/Learning Rate": 4.002350727805324e-06, "Finetune/Loss": 0.8572973012924194, "Finetune/Loss (Raw)": 1.0178066492080688, "Finetune/Step": 3709, "Finetune/Step Time": 4.417952049523592} +{"Finetune/Learning Rate": 3.997367105415775e-06, "Finetune/Loss": 0.8584592342376709, "Finetune/Loss (Raw)": 0.9951059818267822, "Finetune/Step": 3710, "Finetune/Step Time": 4.404859775677323} +{"Finetune/Learning Rate": 3.992385812530864e-06, "Finetune/Loss": 0.8582427501678467, "Finetune/Loss (Raw)": 0.796570360660553, "Finetune/Step": 3711, "Finetune/Step Time": 4.416599033400416} +{"Finetune/Learning Rate": 3.987406851083735e-06, "Finetune/Loss": 0.8588552474975586, "Finetune/Loss (Raw)": 1.0149651765823364, "Finetune/Step": 3712, "Finetune/Step Time": 4.415939340367913} +{"Finetune/Learning Rate": 3.982430223006614e-06, "Finetune/Loss": 0.8530042171478271, "Finetune/Loss (Raw)": 0.2847396731376648, "Finetune/Step": 3713, "Finetune/Step Time": 4.45738529600203} +{"Finetune/Learning Rate": 3.977455930230842e-06, "Finetune/Loss": 0.8541234731674194, "Finetune/Loss (Raw)": 0.911371648311615, "Finetune/Step": 3714, "Finetune/Step Time": 4.450330512598157} +{"Finetune/Learning Rate": 3.972483974686845e-06, "Finetune/Loss": 0.8550777435302734, "Finetune/Loss (Raw)": 0.9509015679359436, "Finetune/Step": 3715, "Finetune/Step Time": 4.448992360383272} +{"Finetune/Learning Rate": 3.967514358304139e-06, "Finetune/Loss": 0.8558590412139893, "Finetune/Loss (Raw)": 0.8045913577079773, "Finetune/Step": 3716, "Finetune/Step Time": 4.472140176221728} +{"Finetune/Learning Rate": 3.962547083011338e-06, "Finetune/Loss": 0.8551253080368042, "Finetune/Loss (Raw)": 0.7821664214134216, "Finetune/Step": 3717, "Finetune/Step Time": 4.485770480707288} +{"Finetune/Learning Rate": 3.957582150736141e-06, "Finetune/Loss": 0.8538820743560791, "Finetune/Loss (Raw)": 0.7911142110824585, "Finetune/Step": 3718, "Finetune/Step Time": 4.490280702710152} +{"Finetune/Learning Rate": 3.952619563405343e-06, "Finetune/Loss": 0.8529233932495117, "Finetune/Loss (Raw)": 0.6833195686340332, "Finetune/Step": 3719, "Finetune/Step Time": 4.5058709643781185} +{"Finetune/Learning Rate": 3.9476593229448245e-06, "Finetune/Loss": 0.8534806966781616, "Finetune/Loss (Raw)": 1.0002020597457886, "Finetune/Step": 3720, "Finetune/Step Time": 4.509814314544201} +{"Finetune/Learning Rate": 3.942701431279564e-06, "Finetune/Loss": 0.853222668170929, "Finetune/Loss (Raw)": 0.9798002243041992, "Finetune/Step": 3721, "Finetune/Step Time": 4.509304715320468} +{"Finetune/Learning Rate": 3.937745890333623e-06, "Finetune/Loss": 0.8556207418441772, "Finetune/Loss (Raw)": 0.9248179793357849, "Finetune/Step": 3722, "Finetune/Step Time": 4.5012705735862255} +{"Finetune/Learning Rate": 3.932792702030139e-06, "Finetune/Loss": 0.8561898469924927, "Finetune/Loss (Raw)": 0.9750695824623108, "Finetune/Step": 3723, "Finetune/Step Time": 4.499060304835439} +{"Finetune/Learning Rate": 3.92784186829136e-06, "Finetune/Loss": 0.8545274138450623, "Finetune/Loss (Raw)": 0.8201422691345215, "Finetune/Step": 3724, "Finetune/Step Time": 4.500004520639777} +{"Finetune/Learning Rate": 3.9228933910386014e-06, "Finetune/Loss": 0.8601393699645996, "Finetune/Loss (Raw)": 0.9722822904586792, "Finetune/Step": 3725, "Finetune/Step Time": 4.466114154085517} +{"Finetune/Learning Rate": 3.917947272192273e-06, "Finetune/Loss": 0.8566567897796631, "Finetune/Loss (Raw)": 0.3857426643371582, "Finetune/Step": 3726, "Finetune/Step Time": 4.485377183184028} +{"Finetune/Learning Rate": 3.913003513671866e-06, "Finetune/Loss": 0.8568214774131775, "Finetune/Loss (Raw)": 0.8323060274124146, "Finetune/Step": 3727, "Finetune/Step Time": 4.489968629553914} +{"Finetune/Learning Rate": 3.908062117395956e-06, "Finetune/Loss": 0.8588598370552063, "Finetune/Loss (Raw)": 0.6966301202774048, "Finetune/Step": 3728, "Finetune/Step Time": 4.459850683808327} +{"Finetune/Learning Rate": 3.903123085282204e-06, "Finetune/Loss": 0.858812153339386, "Finetune/Loss (Raw)": 0.9300599098205566, "Finetune/Step": 3729, "Finetune/Step Time": 4.468774627894163} +{"Finetune/Learning Rate": 3.898186419247347e-06, "Finetune/Loss": 0.8593610525131226, "Finetune/Loss (Raw)": 0.9199368357658386, "Finetune/Step": 3730, "Finetune/Step Time": 4.502436354756355} +{"Finetune/Learning Rate": 3.893252121207221e-06, "Finetune/Loss": 0.8600293397903442, "Finetune/Loss (Raw)": 0.9417598843574524, "Finetune/Step": 3731, "Finetune/Step Time": 4.502208925783634} +{"Finetune/Learning Rate": 3.888320193076719e-06, "Finetune/Loss": 0.8599661588668823, "Finetune/Loss (Raw)": 0.9030303955078125, "Finetune/Step": 3732, "Finetune/Step Time": 4.500464919954538} +{"Finetune/Learning Rate": 3.8833906367698245e-06, "Finetune/Loss": 0.8605101108551025, "Finetune/Loss (Raw)": 0.8522343635559082, "Finetune/Step": 3733, "Finetune/Step Time": 4.507762862369418} +{"Finetune/Learning Rate": 3.878463454199613e-06, "Finetune/Loss": 0.8612676858901978, "Finetune/Loss (Raw)": 0.9693470597267151, "Finetune/Step": 3734, "Finetune/Step Time": 4.4954270124435425} +{"Finetune/Learning Rate": 3.87353864727822e-06, "Finetune/Loss": 0.8596925139427185, "Finetune/Loss (Raw)": 0.7147986888885498, "Finetune/Step": 3735, "Finetune/Step Time": 4.496340142562985} +{"Finetune/Learning Rate": 3.8686162179168695e-06, "Finetune/Loss": 0.8612877130508423, "Finetune/Loss (Raw)": 1.0854156017303467, "Finetune/Step": 3736, "Finetune/Step Time": 4.466574586927891} +{"Finetune/Learning Rate": 3.863696168025859e-06, "Finetune/Loss": 0.8613130450248718, "Finetune/Loss (Raw)": 0.916150689125061, "Finetune/Step": 3737, "Finetune/Step Time": 4.484801506623626} +{"Finetune/Learning Rate": 3.858778499514565e-06, "Finetune/Loss": 0.8616208434104919, "Finetune/Loss (Raw)": 0.8341594338417053, "Finetune/Step": 3738, "Finetune/Step Time": 4.494012957438827} +{"Finetune/Learning Rate": 3.853863214291439e-06, "Finetune/Loss": 0.865523099899292, "Finetune/Loss (Raw)": 0.8406972289085388, "Finetune/Step": 3739, "Finetune/Step Time": 4.4561566691845655} +{"Finetune/Learning Rate": 3.848950314264001e-06, "Finetune/Loss": 0.8657032251358032, "Finetune/Loss (Raw)": 0.8845027685165405, "Finetune/Step": 3740, "Finetune/Step Time": 4.439697626978159} +{"Finetune/Learning Rate": 3.8440398013388655e-06, "Finetune/Loss": 0.8670656681060791, "Finetune/Loss (Raw)": 0.9695245027542114, "Finetune/Step": 3741, "Finetune/Step Time": 4.439483311027288} +{"Finetune/Learning Rate": 3.839131677421694e-06, "Finetune/Loss": 0.867154598236084, "Finetune/Loss (Raw)": 0.8795700073242188, "Finetune/Step": 3742, "Finetune/Step Time": 4.435125183314085} +{"Finetune/Learning Rate": 3.834225944417235e-06, "Finetune/Loss": 0.8672695159912109, "Finetune/Loss (Raw)": 0.932099461555481, "Finetune/Step": 3743, "Finetune/Step Time": 4.467960845679045} +{"Finetune/Learning Rate": 3.829322604229313e-06, "Finetune/Loss": 0.8640367984771729, "Finetune/Loss (Raw)": 0.3604568839073181, "Finetune/Step": 3744, "Finetune/Step Time": 4.50659659691155} +{"Finetune/Learning Rate": 3.824421658760815e-06, "Finetune/Loss": 0.8598018884658813, "Finetune/Loss (Raw)": 0.32645708322525024, "Finetune/Step": 3745, "Finetune/Step Time": 4.532318567857146} +{"Finetune/Learning Rate": 3.819523109913706e-06, "Finetune/Loss": 0.8591713905334473, "Finetune/Loss (Raw)": 0.7556048035621643, "Finetune/Step": 3746, "Finetune/Step Time": 4.523182258009911} +{"Finetune/Learning Rate": 3.814626959589012e-06, "Finetune/Loss": 0.8584381937980652, "Finetune/Loss (Raw)": 0.7573752999305725, "Finetune/Step": 3747, "Finetune/Step Time": 4.543589644134045} +{"Finetune/Learning Rate": 3.8097332096868368e-06, "Finetune/Loss": 0.8592749834060669, "Finetune/Loss (Raw)": 0.9502155184745789, "Finetune/Step": 3748, "Finetune/Step Time": 4.554411860182881} +{"Finetune/Learning Rate": 3.8048418621063475e-06, "Finetune/Loss": 0.8590091466903687, "Finetune/Loss (Raw)": 0.7863625288009644, "Finetune/Step": 3749, "Finetune/Step Time": 4.561211595311761} +{"Finetune/Learning Rate": 3.7999529187457775e-06, "Finetune/Loss": 0.8593531847000122, "Finetune/Loss (Raw)": 0.9460724592208862, "Finetune/Step": 3750, "Finetune/Step Time": 4.565494906157255} +{"Finetune/Learning Rate": 3.7950663815024424e-06, "Finetune/Loss": 0.8601061105728149, "Finetune/Loss (Raw)": 1.0159026384353638, "Finetune/Step": 3751, "Finetune/Step Time": 4.572697054594755} +{"Finetune/Learning Rate": 3.790182252272698e-06, "Finetune/Loss": 0.8590450286865234, "Finetune/Loss (Raw)": 0.8457161784172058, "Finetune/Step": 3752, "Finetune/Step Time": 4.575188597664237} +{"Finetune/Learning Rate": 3.785300532951983e-06, "Finetune/Loss": 0.8584241271018982, "Finetune/Loss (Raw)": 0.8736166954040527, "Finetune/Step": 3753, "Finetune/Step Time": 4.590276597067714} +{"Finetune/Learning Rate": 3.780421225434803e-06, "Finetune/Loss": 0.859249472618103, "Finetune/Loss (Raw)": 0.9055905342102051, "Finetune/Step": 3754, "Finetune/Step Time": 4.592770803719759} +{"Finetune/Learning Rate": 3.7755443316147177e-06, "Finetune/Loss": 0.8595921397209167, "Finetune/Loss (Raw)": 0.9585708379745483, "Finetune/Step": 3755, "Finetune/Step Time": 4.580775728449225} +{"Finetune/Learning Rate": 3.7706698533843556e-06, "Finetune/Loss": 0.8582267761230469, "Finetune/Loss (Raw)": 0.7163860201835632, "Finetune/Step": 3756, "Finetune/Step Time": 4.581849366426468} +{"Finetune/Learning Rate": 3.765797792635406e-06, "Finetune/Loss": 0.8591950535774231, "Finetune/Loss (Raw)": 0.955275297164917, "Finetune/Step": 3757, "Finetune/Step Time": 4.584301374852657} +{"Finetune/Learning Rate": 3.7609281512586203e-06, "Finetune/Loss": 0.8592404127120972, "Finetune/Loss (Raw)": 0.8634644150733948, "Finetune/Step": 3758, "Finetune/Step Time": 4.59904052875936} +{"Finetune/Learning Rate": 3.756060931143811e-06, "Finetune/Loss": 0.8610580563545227, "Finetune/Loss (Raw)": 0.9573595523834229, "Finetune/Step": 3759, "Finetune/Step Time": 4.583610245957971} +{"Finetune/Learning Rate": 3.7511961341798487e-06, "Finetune/Loss": 0.8599402904510498, "Finetune/Loss (Raw)": 0.8431748151779175, "Finetune/Step": 3760, "Finetune/Step Time": 4.58916430734098} +{"Finetune/Learning Rate": 3.746333762254677e-06, "Finetune/Loss": 0.8579559326171875, "Finetune/Loss (Raw)": 0.7474319934844971, "Finetune/Step": 3761, "Finetune/Step Time": 4.600129576399922} +{"Finetune/Learning Rate": 3.7414738172552744e-06, "Finetune/Loss": 0.8580126762390137, "Finetune/Loss (Raw)": 0.8817716240882874, "Finetune/Step": 3762, "Finetune/Step Time": 4.605709528550506} +{"Finetune/Learning Rate": 3.7366163010676937e-06, "Finetune/Loss": 0.859273374080658, "Finetune/Loss (Raw)": 0.937160074710846, "Finetune/Step": 3763, "Finetune/Step Time": 4.601810049265623} +{"Finetune/Learning Rate": 3.7317612155770467e-06, "Finetune/Loss": 0.8585785627365112, "Finetune/Loss (Raw)": 0.8175180554389954, "Finetune/Step": 3764, "Finetune/Step Time": 4.607723243534565} +{"Finetune/Learning Rate": 3.726908562667496e-06, "Finetune/Loss": 0.8576177954673767, "Finetune/Loss (Raw)": 0.849180281162262, "Finetune/Step": 3765, "Finetune/Step Time": 4.616662597283721} +{"Finetune/Learning Rate": 3.72205834422226e-06, "Finetune/Loss": 0.857620358467102, "Finetune/Loss (Raw)": 0.9455884099006653, "Finetune/Step": 3766, "Finetune/Step Time": 4.625648066401482} +{"Finetune/Learning Rate": 3.717210562123613e-06, "Finetune/Loss": 0.8580888509750366, "Finetune/Loss (Raw)": 0.947066605091095, "Finetune/Step": 3767, "Finetune/Step Time": 4.623609092086554} +{"Finetune/Learning Rate": 3.712365218252887e-06, "Finetune/Loss": 0.8616408705711365, "Finetune/Loss (Raw)": 0.7644609212875366, "Finetune/Step": 3768, "Finetune/Step Time": 4.594684908166528} +{"Finetune/Learning Rate": 3.7075223144904638e-06, "Finetune/Loss": 0.8572911024093628, "Finetune/Loss (Raw)": 0.24814878404140472, "Finetune/Step": 3769, "Finetune/Step Time": 4.624695913866162} +{"Finetune/Learning Rate": 3.702681852715777e-06, "Finetune/Loss": 0.8580005168914795, "Finetune/Loss (Raw)": 1.0484265089035034, "Finetune/Step": 3770, "Finetune/Step Time": 4.61600599437952} +{"Finetune/Learning Rate": 3.6978438348073253e-06, "Finetune/Loss": 0.8591736555099487, "Finetune/Loss (Raw)": 1.0764250755310059, "Finetune/Step": 3771, "Finetune/Step Time": 4.6180532313883305} +{"Finetune/Learning Rate": 3.6930082626426377e-06, "Finetune/Loss": 0.860127329826355, "Finetune/Loss (Raw)": 0.9627472162246704, "Finetune/Step": 3772, "Finetune/Step Time": 4.623922470957041} +{"Finetune/Learning Rate": 3.6881751380983066e-06, "Finetune/Loss": 0.8563359975814819, "Finetune/Loss (Raw)": 0.34489408135414124, "Finetune/Step": 3773, "Finetune/Step Time": 4.65967264585197} +{"Finetune/Learning Rate": 3.6833444630499804e-06, "Finetune/Loss": 0.8559352159500122, "Finetune/Loss (Raw)": 0.9305740594863892, "Finetune/Step": 3774, "Finetune/Step Time": 4.661565316841006} +{"Finetune/Learning Rate": 3.6785162393723482e-06, "Finetune/Loss": 0.8562955260276794, "Finetune/Loss (Raw)": 0.9657850861549377, "Finetune/Step": 3775, "Finetune/Step Time": 4.6659030392766} +{"Finetune/Learning Rate": 3.6736904689391417e-06, "Finetune/Loss": 0.856226921081543, "Finetune/Loss (Raw)": 0.9467796087265015, "Finetune/Step": 3776, "Finetune/Step Time": 4.669421190395951} +{"Finetune/Learning Rate": 3.6688671536231557e-06, "Finetune/Loss": 0.8508559465408325, "Finetune/Loss (Raw)": 0.27233827114105225, "Finetune/Step": 3777, "Finetune/Step Time": 4.707040715962648} +{"Finetune/Learning Rate": 3.6640462952962217e-06, "Finetune/Loss": 0.8501054048538208, "Finetune/Loss (Raw)": 0.9287622570991516, "Finetune/Step": 3778, "Finetune/Step Time": 4.709014572203159} +{"Finetune/Learning Rate": 3.6592278958292225e-06, "Finetune/Loss": 0.8508555889129639, "Finetune/Loss (Raw)": 0.847307562828064, "Finetune/Step": 3779, "Finetune/Step Time": 4.708799410611391} +{"Finetune/Learning Rate": 3.6544119570920845e-06, "Finetune/Loss": 0.8497422337532043, "Finetune/Loss (Raw)": 0.8049997091293335, "Finetune/Step": 3780, "Finetune/Step Time": 4.7094200644642115} +{"Finetune/Learning Rate": 3.6495984809537775e-06, "Finetune/Loss": 0.8496929407119751, "Finetune/Loss (Raw)": 0.9524956345558167, "Finetune/Step": 3781, "Finetune/Step Time": 4.706941394135356} +{"Finetune/Learning Rate": 3.6447874692823195e-06, "Finetune/Loss": 0.8492576479911804, "Finetune/Loss (Raw)": 0.8395713567733765, "Finetune/Step": 3782, "Finetune/Step Time": 4.689209369942546} +{"Finetune/Learning Rate": 3.639978923944766e-06, "Finetune/Loss": 0.8473226428031921, "Finetune/Loss (Raw)": 0.8058675527572632, "Finetune/Step": 3783, "Finetune/Step Time": 4.6970111932605505} +{"Finetune/Learning Rate": 3.6351728468072265e-06, "Finetune/Loss": 0.8475579619407654, "Finetune/Loss (Raw)": 0.8151437640190125, "Finetune/Step": 3784, "Finetune/Step Time": 4.708329293876886} +{"Finetune/Learning Rate": 3.6303692397348455e-06, "Finetune/Loss": 0.8475162982940674, "Finetune/Loss (Raw)": 0.8719155788421631, "Finetune/Step": 3785, "Finetune/Step Time": 4.70647805929184} +{"Finetune/Learning Rate": 3.6255681045918e-06, "Finetune/Loss": 0.8482421040534973, "Finetune/Loss (Raw)": 0.9848806262016296, "Finetune/Step": 3786, "Finetune/Step Time": 4.690387913957238} +{"Finetune/Learning Rate": 3.6207694432413255e-06, "Finetune/Loss": 0.8483576774597168, "Finetune/Loss (Raw)": 0.9285301566123962, "Finetune/Step": 3787, "Finetune/Step Time": 4.687521090731025} +{"Finetune/Learning Rate": 3.6159732575456862e-06, "Finetune/Loss": 0.8473683595657349, "Finetune/Loss (Raw)": 0.8778882026672363, "Finetune/Step": 3788, "Finetune/Step Time": 4.682375276461244} +{"Finetune/Learning Rate": 3.6111795493661885e-06, "Finetune/Loss": 0.8467589020729065, "Finetune/Loss (Raw)": 0.7944868803024292, "Finetune/Step": 3789, "Finetune/Step Time": 4.701069241389632} +{"Finetune/Learning Rate": 3.606388320563177e-06, "Finetune/Loss": 0.8465226888656616, "Finetune/Loss (Raw)": 0.9473775029182434, "Finetune/Step": 3790, "Finetune/Step Time": 4.7078843005001545} +{"Finetune/Learning Rate": 3.6015995729960316e-06, "Finetune/Loss": 0.8450841903686523, "Finetune/Loss (Raw)": 0.8778297901153564, "Finetune/Step": 3791, "Finetune/Step Time": 4.707539547234774} +{"Finetune/Learning Rate": 3.596813308523176e-06, "Finetune/Loss": 0.8413006067276001, "Finetune/Loss (Raw)": 0.34588921070098877, "Finetune/Step": 3792, "Finetune/Step Time": 4.733741458505392} +{"Finetune/Learning Rate": 3.592029529002059e-06, "Finetune/Loss": 0.8412749767303467, "Finetune/Loss (Raw)": 0.9164081811904907, "Finetune/Step": 3793, "Finetune/Step Time": 4.752650054171681} +{"Finetune/Learning Rate": 3.5872482362891802e-06, "Finetune/Loss": 0.8412825465202332, "Finetune/Loss (Raw)": 0.9264390468597412, "Finetune/Step": 3794, "Finetune/Step Time": 4.75190057978034} +{"Finetune/Learning Rate": 3.5824694322400653e-06, "Finetune/Loss": 0.8439210057258606, "Finetune/Loss (Raw)": 0.7625950574874878, "Finetune/Step": 3795, "Finetune/Step Time": 4.727682171389461} +{"Finetune/Learning Rate": 3.5776931187092667e-06, "Finetune/Loss": 0.844619631767273, "Finetune/Loss (Raw)": 0.9134243726730347, "Finetune/Step": 3796, "Finetune/Step Time": 4.73888298869133} +{"Finetune/Learning Rate": 3.5729192975503856e-06, "Finetune/Loss": 0.8437203764915466, "Finetune/Loss (Raw)": 0.7453588247299194, "Finetune/Step": 3797, "Finetune/Step Time": 4.7433743737638} +{"Finetune/Learning Rate": 3.5681479706160483e-06, "Finetune/Loss": 0.8431594371795654, "Finetune/Loss (Raw)": 0.8386366963386536, "Finetune/Step": 3798, "Finetune/Step Time": 4.739936001598835} +{"Finetune/Learning Rate": 3.563379139757911e-06, "Finetune/Loss": 0.8419408798217773, "Finetune/Loss (Raw)": 0.8238984942436218, "Finetune/Step": 3799, "Finetune/Step Time": 4.749735534191132} +{"Finetune/Learning Rate": 3.558612806826666e-06, "Finetune/Loss": 0.8428518772125244, "Finetune/Loss (Raw)": 0.9620721936225891, "Finetune/Step": 3800, "Finetune/Step Time": 4.745474996045232} +{"Finetune/Learning Rate": 3.553848973672033e-06, "Finetune/Loss": 0.8445160388946533, "Finetune/Loss (Raw)": 0.8809743523597717, "Finetune/Step": 3801, "Finetune/Step Time": 4.74613793939352} +{"Finetune/Learning Rate": 3.549087642142762e-06, "Finetune/Loss": 0.8454192876815796, "Finetune/Loss (Raw)": 0.9657242894172668, "Finetune/Step": 3802, "Finetune/Step Time": 4.742100378498435} +{"Finetune/Learning Rate": 3.544328814086632e-06, "Finetune/Loss": 0.850345253944397, "Finetune/Loss (Raw)": 1.0074020624160767, "Finetune/Step": 3803, "Finetune/Step Time": 4.700494857504964} +{"Finetune/Learning Rate": 3.5395724913504546e-06, "Finetune/Loss": 0.8501265645027161, "Finetune/Loss (Raw)": 0.9221500754356384, "Finetune/Step": 3804, "Finetune/Step Time": 4.6850337870419025} +{"Finetune/Learning Rate": 3.5348186757800683e-06, "Finetune/Loss": 0.8492128849029541, "Finetune/Loss (Raw)": 0.8547335267066956, "Finetune/Step": 3805, "Finetune/Step Time": 4.701954325661063} +{"Finetune/Learning Rate": 3.530067369220326e-06, "Finetune/Loss": 0.8510339260101318, "Finetune/Loss (Raw)": 0.8854870796203613, "Finetune/Step": 3806, "Finetune/Step Time": 4.713916501030326} +{"Finetune/Learning Rate": 3.525318573515125e-06, "Finetune/Loss": 0.8499058485031128, "Finetune/Loss (Raw)": 0.8481947779655457, "Finetune/Step": 3807, "Finetune/Step Time": 4.721654199063778} +{"Finetune/Learning Rate": 3.520572290507378e-06, "Finetune/Loss": 0.8488789796829224, "Finetune/Loss (Raw)": 0.7816466093063354, "Finetune/Step": 3808, "Finetune/Step Time": 4.72841203212738} +{"Finetune/Learning Rate": 3.5158285220390256e-06, "Finetune/Loss": 0.8479437828063965, "Finetune/Loss (Raw)": 0.8514215350151062, "Finetune/Step": 3809, "Finetune/Step Time": 4.742902435362339} +{"Finetune/Learning Rate": 3.5110872699510313e-06, "Finetune/Loss": 0.846760630607605, "Finetune/Loss (Raw)": 0.941589891910553, "Finetune/Step": 3810, "Finetune/Step Time": 4.739780940115452} +{"Finetune/Learning Rate": 3.5063485360833816e-06, "Finetune/Loss": 0.8463712930679321, "Finetune/Loss (Raw)": 0.8439404964447021, "Finetune/Step": 3811, "Finetune/Step Time": 4.745989039540291} +{"Finetune/Learning Rate": 3.5016123222750863e-06, "Finetune/Loss": 0.8474310636520386, "Finetune/Loss (Raw)": 0.8016027212142944, "Finetune/Step": 3812, "Finetune/Step Time": 4.740049218758941} +{"Finetune/Learning Rate": 3.4968786303641764e-06, "Finetune/Loss": 0.8475534319877625, "Finetune/Loss (Raw)": 0.8761588931083679, "Finetune/Step": 3813, "Finetune/Step Time": 4.737729547545314} +{"Finetune/Learning Rate": 3.492147462187714e-06, "Finetune/Loss": 0.8470147848129272, "Finetune/Loss (Raw)": 0.8464174866676331, "Finetune/Step": 3814, "Finetune/Step Time": 4.735833406448364} +{"Finetune/Learning Rate": 3.4874188195817626e-06, "Finetune/Loss": 0.8468343615531921, "Finetune/Loss (Raw)": 0.8647762537002563, "Finetune/Step": 3815, "Finetune/Step Time": 4.754037471488118} +{"Finetune/Learning Rate": 3.4826927043814197e-06, "Finetune/Loss": 0.8472815752029419, "Finetune/Loss (Raw)": 0.8847415447235107, "Finetune/Step": 3816, "Finetune/Step Time": 4.7495767045766115} +{"Finetune/Learning Rate": 3.4779691184207954e-06, "Finetune/Loss": 0.8458660840988159, "Finetune/Loss (Raw)": 0.8084287047386169, "Finetune/Step": 3817, "Finetune/Step Time": 4.746578939259052} +{"Finetune/Learning Rate": 3.473248063533028e-06, "Finetune/Loss": 0.8449676036834717, "Finetune/Loss (Raw)": 0.8169373869895935, "Finetune/Step": 3818, "Finetune/Step Time": 4.7394681591540575} +{"Finetune/Learning Rate": 3.4685295415502663e-06, "Finetune/Loss": 0.8488404154777527, "Finetune/Loss (Raw)": 0.8054758310317993, "Finetune/Step": 3819, "Finetune/Step Time": 4.703996500000358} +{"Finetune/Learning Rate": 3.4638135543036656e-06, "Finetune/Loss": 0.8466097116470337, "Finetune/Loss (Raw)": 0.7175153493881226, "Finetune/Step": 3820, "Finetune/Step Time": 4.705873629078269} +{"Finetune/Learning Rate": 3.4591001036234206e-06, "Finetune/Loss": 0.8459017872810364, "Finetune/Loss (Raw)": 0.8158610463142395, "Finetune/Step": 3821, "Finetune/Step Time": 4.711293982341886} +{"Finetune/Learning Rate": 3.4543891913387253e-06, "Finetune/Loss": 0.8459387421607971, "Finetune/Loss (Raw)": 0.7820627689361572, "Finetune/Step": 3822, "Finetune/Step Time": 4.69020733051002} +{"Finetune/Learning Rate": 3.449680819277792e-06, "Finetune/Loss": 0.8448654413223267, "Finetune/Loss (Raw)": 0.9863790273666382, "Finetune/Step": 3823, "Finetune/Step Time": 4.693059669807553} +{"Finetune/Learning Rate": 3.4449749892678486e-06, "Finetune/Loss": 0.8450406789779663, "Finetune/Loss (Raw)": 0.896287202835083, "Finetune/Step": 3824, "Finetune/Step Time": 4.692524256184697} +{"Finetune/Learning Rate": 3.4402717031351363e-06, "Finetune/Loss": 0.8452897667884827, "Finetune/Loss (Raw)": 0.7656283378601074, "Finetune/Step": 3825, "Finetune/Step Time": 4.671139653772116} +{"Finetune/Learning Rate": 3.4355709627049073e-06, "Finetune/Loss": 0.844403862953186, "Finetune/Loss (Raw)": 0.8126747012138367, "Finetune/Step": 3826, "Finetune/Step Time": 4.662371398881078} +{"Finetune/Learning Rate": 3.4308727698014243e-06, "Finetune/Loss": 0.8442943096160889, "Finetune/Loss (Raw)": 0.9076539278030396, "Finetune/Step": 3827, "Finetune/Step Time": 4.689881669357419} +{"Finetune/Learning Rate": 3.4261771262479724e-06, "Finetune/Loss": 0.8445589542388916, "Finetune/Loss (Raw)": 0.9503961801528931, "Finetune/Step": 3828, "Finetune/Step Time": 4.693147866055369} +{"Finetune/Learning Rate": 3.4214840338668376e-06, "Finetune/Loss": 0.8411022424697876, "Finetune/Loss (Raw)": 0.27915412187576294, "Finetune/Step": 3829, "Finetune/Step Time": 4.72468894906342} +{"Finetune/Learning Rate": 3.416793494479308e-06, "Finetune/Loss": 0.8407241106033325, "Finetune/Loss (Raw)": 0.8874490857124329, "Finetune/Step": 3830, "Finetune/Step Time": 4.733699658885598} +{"Finetune/Learning Rate": 3.412105509905701e-06, "Finetune/Loss": 0.8397856950759888, "Finetune/Loss (Raw)": 0.8344818949699402, "Finetune/Step": 3831, "Finetune/Step Time": 4.74085253290832} +{"Finetune/Learning Rate": 3.407420081965327e-06, "Finetune/Loss": 0.8406713008880615, "Finetune/Loss (Raw)": 1.0351965427398682, "Finetune/Step": 3832, "Finetune/Step Time": 4.743303803727031} +{"Finetune/Learning Rate": 3.402737212476509e-06, "Finetune/Loss": 0.8425145149230957, "Finetune/Loss (Raw)": 0.9657782316207886, "Finetune/Step": 3833, "Finetune/Step Time": 4.761144490912557} +{"Finetune/Learning Rate": 3.398056903256579e-06, "Finetune/Loss": 0.8433471322059631, "Finetune/Loss (Raw)": 0.9746831059455872, "Finetune/Step": 3834, "Finetune/Step Time": 4.755478339269757} +{"Finetune/Learning Rate": 3.393379156121873e-06, "Finetune/Loss": 0.8440053462982178, "Finetune/Loss (Raw)": 0.9432481527328491, "Finetune/Step": 3835, "Finetune/Step Time": 4.741785639896989} +{"Finetune/Learning Rate": 3.3887039728877336e-06, "Finetune/Loss": 0.844279944896698, "Finetune/Loss (Raw)": 0.8169666528701782, "Finetune/Step": 3836, "Finetune/Step Time": 4.719898683950305} +{"Finetune/Learning Rate": 3.3840313553685034e-06, "Finetune/Loss": 0.843006432056427, "Finetune/Loss (Raw)": 0.8548011779785156, "Finetune/Step": 3837, "Finetune/Step Time": 4.725693861022592} +{"Finetune/Learning Rate": 3.379361305377542e-06, "Finetune/Loss": 0.84162437915802, "Finetune/Loss (Raw)": 0.8182031512260437, "Finetune/Step": 3838, "Finetune/Step Time": 4.723598865792155} +{"Finetune/Learning Rate": 3.374693824727204e-06, "Finetune/Loss": 0.8423512578010559, "Finetune/Loss (Raw)": 0.8896095156669617, "Finetune/Step": 3839, "Finetune/Step Time": 4.742784569039941} +{"Finetune/Learning Rate": 3.3700289152288377e-06, "Finetune/Loss": 0.8410284519195557, "Finetune/Loss (Raw)": 0.8456448316574097, "Finetune/Step": 3840, "Finetune/Step Time": 4.743236223235726} +{"Finetune/Learning Rate": 3.3653665786928138e-06, "Finetune/Loss": 0.8466734886169434, "Finetune/Loss (Raw)": 1.0073068141937256, "Finetune/Step": 3841, "Finetune/Step Time": 4.7005261071026325} +{"Finetune/Learning Rate": 3.360706816928491e-06, "Finetune/Loss": 0.8459336161613464, "Finetune/Loss (Raw)": 0.8166689872741699, "Finetune/Step": 3842, "Finetune/Step Time": 4.692615048959851} +{"Finetune/Learning Rate": 3.3560496317442305e-06, "Finetune/Loss": 0.8450645208358765, "Finetune/Loss (Raw)": 0.8396584987640381, "Finetune/Step": 3843, "Finetune/Step Time": 4.700189463794231} +{"Finetune/Learning Rate": 3.351395024947395e-06, "Finetune/Loss": 0.8454359769821167, "Finetune/Loss (Raw)": 0.8521378636360168, "Finetune/Step": 3844, "Finetune/Step Time": 4.6813311614096165} +{"Finetune/Learning Rate": 3.3467429983443477e-06, "Finetune/Loss": 0.8459497690200806, "Finetune/Loss (Raw)": 0.8479276895523071, "Finetune/Step": 3845, "Finetune/Step Time": 4.665760291740298} +{"Finetune/Learning Rate": 3.34209355374045e-06, "Finetune/Loss": 0.8452531099319458, "Finetune/Loss (Raw)": 0.7019481658935547, "Finetune/Step": 3846, "Finetune/Step Time": 4.661207700148225} +{"Finetune/Learning Rate": 3.337446692940055e-06, "Finetune/Loss": 0.8476517796516418, "Finetune/Loss (Raw)": 0.9903410077095032, "Finetune/Step": 3847, "Finetune/Step Time": 4.644014347344637} +{"Finetune/Learning Rate": 3.332802417746527e-06, "Finetune/Loss": 0.8424351811408997, "Finetune/Loss (Raw)": 0.3324805796146393, "Finetune/Step": 3848, "Finetune/Step Time": 4.670972378924489} +{"Finetune/Learning Rate": 3.3281607299622175e-06, "Finetune/Loss": 0.8429398536682129, "Finetune/Loss (Raw)": 1.0444023609161377, "Finetune/Step": 3849, "Finetune/Step Time": 4.686845829710364} +{"Finetune/Learning Rate": 3.3235216313884657e-06, "Finetune/Loss": 0.8405658602714539, "Finetune/Loss (Raw)": 0.6209477186203003, "Finetune/Step": 3850, "Finetune/Step Time": 4.6993288192898035} +{"Finetune/Learning Rate": 3.3188851238256246e-06, "Finetune/Loss": 0.8389422297477722, "Finetune/Loss (Raw)": 0.7672451734542847, "Finetune/Step": 3851, "Finetune/Step Time": 4.690834084525704} +{"Finetune/Learning Rate": 3.3142512090730283e-06, "Finetune/Loss": 0.839758574962616, "Finetune/Loss (Raw)": 0.9246270060539246, "Finetune/Step": 3852, "Finetune/Step Time": 4.69138996489346} +{"Finetune/Learning Rate": 3.309619888929011e-06, "Finetune/Loss": 0.8357098698616028, "Finetune/Loss (Raw)": 0.45405030250549316, "Finetune/Step": 3853, "Finetune/Step Time": 4.724276022985578} +{"Finetune/Learning Rate": 3.304991165190895e-06, "Finetune/Loss": 0.8397458791732788, "Finetune/Loss (Raw)": 0.9023538827896118, "Finetune/Step": 3854, "Finetune/Step Time": 4.691301487386227} +{"Finetune/Learning Rate": 3.300365039655e-06, "Finetune/Loss": 0.8383775949478149, "Finetune/Loss (Raw)": 0.6571555137634277, "Finetune/Step": 3855, "Finetune/Step Time": 4.701919090002775} +{"Finetune/Learning Rate": 3.2957415141166327e-06, "Finetune/Loss": 0.8407187461853027, "Finetune/Loss (Raw)": 0.9963024258613586, "Finetune/Step": 3856, "Finetune/Step Time": 4.694833328947425} +{"Finetune/Learning Rate": 3.2911205903700916e-06, "Finetune/Loss": 0.8362866640090942, "Finetune/Loss (Raw)": 0.3627493977546692, "Finetune/Step": 3857, "Finetune/Step Time": 4.721215954050422} +{"Finetune/Learning Rate": 3.2865022702086756e-06, "Finetune/Loss": 0.8317326307296753, "Finetune/Loss (Raw)": 0.3370240032672882, "Finetune/Step": 3858, "Finetune/Step Time": 4.733765289187431} +{"Finetune/Learning Rate": 3.281886555424656e-06, "Finetune/Loss": 0.8326334357261658, "Finetune/Loss (Raw)": 1.057064414024353, "Finetune/Step": 3859, "Finetune/Step Time": 4.726268198341131} +{"Finetune/Learning Rate": 3.277273447809299e-06, "Finetune/Loss": 0.8325480222702026, "Finetune/Loss (Raw)": 0.8921001553535461, "Finetune/Step": 3860, "Finetune/Step Time": 4.71931929141283} +{"Finetune/Learning Rate": 3.2726629491528714e-06, "Finetune/Loss": 0.8325186371803284, "Finetune/Loss (Raw)": 0.8484749794006348, "Finetune/Step": 3861, "Finetune/Step Time": 4.70476839132607} +{"Finetune/Learning Rate": 3.268055061244614e-06, "Finetune/Loss": 0.8318767547607422, "Finetune/Loss (Raw)": 0.887184739112854, "Finetune/Step": 3862, "Finetune/Step Time": 4.70359031111002} +{"Finetune/Learning Rate": 3.2634497858727553e-06, "Finetune/Loss": 0.8328025937080383, "Finetune/Loss (Raw)": 0.833301305770874, "Finetune/Step": 3863, "Finetune/Step Time": 4.713661294430494} +{"Finetune/Learning Rate": 3.2588471248245157e-06, "Finetune/Loss": 0.8316057324409485, "Finetune/Loss (Raw)": 0.9322256445884705, "Finetune/Step": 3864, "Finetune/Step Time": 4.716708958148956} +{"Finetune/Learning Rate": 3.254247079886097e-06, "Finetune/Loss": 0.8313776254653931, "Finetune/Loss (Raw)": 0.8869500756263733, "Finetune/Step": 3865, "Finetune/Step Time": 4.734547756612301} +{"Finetune/Learning Rate": 3.2496496528426867e-06, "Finetune/Loss": 0.8329364061355591, "Finetune/Loss (Raw)": 1.0336819887161255, "Finetune/Step": 3866, "Finetune/Step Time": 4.726226229220629} +{"Finetune/Learning Rate": 3.245054845478454e-06, "Finetune/Loss": 0.8347911834716797, "Finetune/Loss (Raw)": 1.078108549118042, "Finetune/Step": 3867, "Finetune/Step Time": 4.726551592350006} +{"Finetune/Learning Rate": 3.240462659576562e-06, "Finetune/Loss": 0.8348702192306519, "Finetune/Loss (Raw)": 0.8946170210838318, "Finetune/Step": 3868, "Finetune/Step Time": 4.7298617493361235} +{"Finetune/Learning Rate": 3.23587309691914e-06, "Finetune/Loss": 0.833656370639801, "Finetune/Loss (Raw)": 0.8141539692878723, "Finetune/Step": 3869, "Finetune/Step Time": 4.736518735066056} +{"Finetune/Learning Rate": 3.231286159287308e-06, "Finetune/Loss": 0.8331334590911865, "Finetune/Loss (Raw)": 0.8126395344734192, "Finetune/Step": 3870, "Finetune/Step Time": 4.761058563366532} +{"Finetune/Learning Rate": 3.2267018484611725e-06, "Finetune/Loss": 0.8327260613441467, "Finetune/Loss (Raw)": 0.8799471855163574, "Finetune/Step": 3871, "Finetune/Step Time": 4.724408276379108} +{"Finetune/Learning Rate": 3.222120166219812e-06, "Finetune/Loss": 0.8339661359786987, "Finetune/Loss (Raw)": 0.5191940069198608, "Finetune/Step": 3872, "Finetune/Step Time": 4.71659885905683} +{"Finetune/Learning Rate": 3.217541114341288e-06, "Finetune/Loss": 0.8371553421020508, "Finetune/Loss (Raw)": 0.7346687912940979, "Finetune/Step": 3873, "Finetune/Step Time": 4.687975903972983} +{"Finetune/Learning Rate": 3.2129646946026407e-06, "Finetune/Loss": 0.838118851184845, "Finetune/Loss (Raw)": 0.8789335489273071, "Finetune/Step": 3874, "Finetune/Step Time": 4.687921401113272} +{"Finetune/Learning Rate": 3.2083909087798905e-06, "Finetune/Loss": 0.8390975594520569, "Finetune/Loss (Raw)": 0.8826521635055542, "Finetune/Step": 3875, "Finetune/Step Time": 4.669919915497303} +{"Finetune/Learning Rate": 3.2038197586480336e-06, "Finetune/Loss": 0.8386071920394897, "Finetune/Loss (Raw)": 0.8874518871307373, "Finetune/Step": 3876, "Finetune/Step Time": 4.6562895234674215} +{"Finetune/Learning Rate": 3.199251245981041e-06, "Finetune/Loss": 0.8393322229385376, "Finetune/Loss (Raw)": 0.8791648745536804, "Finetune/Step": 3877, "Finetune/Step Time": 4.663761990144849} +{"Finetune/Learning Rate": 3.1946853725518724e-06, "Finetune/Loss": 0.8356341123580933, "Finetune/Loss (Raw)": 0.472710520029068, "Finetune/Step": 3878, "Finetune/Step Time": 4.699983222410083} +{"Finetune/Learning Rate": 3.1901221401324446e-06, "Finetune/Loss": 0.8343497514724731, "Finetune/Loss (Raw)": 0.8515006303787231, "Finetune/Step": 3879, "Finetune/Step Time": 4.725398907437921} +{"Finetune/Learning Rate": 3.18556155049366e-06, "Finetune/Loss": 0.8345141410827637, "Finetune/Loss (Raw)": 0.8667652606964111, "Finetune/Step": 3880, "Finetune/Step Time": 4.722098207101226} +{"Finetune/Learning Rate": 3.1810036054054005e-06, "Finetune/Loss": 0.8348495364189148, "Finetune/Loss (Raw)": 0.9165464639663696, "Finetune/Step": 3881, "Finetune/Step Time": 4.714509965851903} +{"Finetune/Learning Rate": 3.1764483066365117e-06, "Finetune/Loss": 0.8351644277572632, "Finetune/Loss (Raw)": 0.9458935260772705, "Finetune/Step": 3882, "Finetune/Step Time": 4.718561075627804} +{"Finetune/Learning Rate": 3.1718956559548154e-06, "Finetune/Loss": 0.8353429436683655, "Finetune/Loss (Raw)": 0.9814234375953674, "Finetune/Step": 3883, "Finetune/Step Time": 4.711993880569935} +{"Finetune/Learning Rate": 3.1673456551271086e-06, "Finetune/Loss": 0.8362033367156982, "Finetune/Loss (Raw)": 0.8265111446380615, "Finetune/Step": 3884, "Finetune/Step Time": 4.705220218747854} +{"Finetune/Learning Rate": 3.162798305919157e-06, "Finetune/Loss": 0.834097146987915, "Finetune/Loss (Raw)": 0.6856926083564758, "Finetune/Step": 3885, "Finetune/Step Time": 4.711727868765593} +{"Finetune/Learning Rate": 3.1582536100956973e-06, "Finetune/Loss": 0.8332282900810242, "Finetune/Loss (Raw)": 0.7522493004798889, "Finetune/Step": 3886, "Finetune/Step Time": 4.698457797989249} +{"Finetune/Learning Rate": 3.1537115694204345e-06, "Finetune/Loss": 0.832055926322937, "Finetune/Loss (Raw)": 0.8072969913482666, "Finetune/Step": 3887, "Finetune/Step Time": 4.699688922613859} +{"Finetune/Learning Rate": 3.1491721856560555e-06, "Finetune/Loss": 0.8325598239898682, "Finetune/Loss (Raw)": 0.9076688289642334, "Finetune/Step": 3888, "Finetune/Step Time": 4.691752217710018} +{"Finetune/Learning Rate": 3.144635460564197e-06, "Finetune/Loss": 0.8336150646209717, "Finetune/Loss (Raw)": 0.8825016617774963, "Finetune/Step": 3889, "Finetune/Step Time": 4.682591715827584} +{"Finetune/Learning Rate": 3.1401013959054726e-06, "Finetune/Loss": 0.8339247703552246, "Finetune/Loss (Raw)": 0.9214186668395996, "Finetune/Step": 3890, "Finetune/Step Time": 4.675854064524174} +{"Finetune/Learning Rate": 3.1355699934394724e-06, "Finetune/Loss": 0.8335146903991699, "Finetune/Loss (Raw)": 0.8846725225448608, "Finetune/Step": 3891, "Finetune/Step Time": 4.690518531948328} +{"Finetune/Learning Rate": 3.1310412549247403e-06, "Finetune/Loss": 0.8338330388069153, "Finetune/Loss (Raw)": 0.8582624793052673, "Finetune/Step": 3892, "Finetune/Step Time": 4.688238091766834} +{"Finetune/Learning Rate": 3.1265151821187933e-06, "Finetune/Loss": 0.8300893306732178, "Finetune/Loss (Raw)": 0.36998414993286133, "Finetune/Step": 3893, "Finetune/Step Time": 4.71646886318922} +{"Finetune/Learning Rate": 3.121991776778109e-06, "Finetune/Loss": 0.8298107385635376, "Finetune/Loss (Raw)": 0.9099338054656982, "Finetune/Step": 3894, "Finetune/Step Time": 4.709464006125927} +{"Finetune/Learning Rate": 3.1174710406581355e-06, "Finetune/Loss": 0.8296369314193726, "Finetune/Loss (Raw)": 0.924821138381958, "Finetune/Step": 3895, "Finetune/Step Time": 4.707471452653408} +{"Finetune/Learning Rate": 3.1129529755132815e-06, "Finetune/Loss": 0.8310345411300659, "Finetune/Loss (Raw)": 0.943347692489624, "Finetune/Step": 3896, "Finetune/Step Time": 4.696177076548338} +{"Finetune/Learning Rate": 3.108437583096918e-06, "Finetune/Loss": 0.8359900116920471, "Finetune/Loss (Raw)": 0.8824507594108582, "Finetune/Step": 3897, "Finetune/Step Time": 4.658084018155932} +{"Finetune/Learning Rate": 3.103924865161383e-06, "Finetune/Loss": 0.8344874978065491, "Finetune/Loss (Raw)": 0.856106162071228, "Finetune/Step": 3898, "Finetune/Step Time": 4.674566682428122} +{"Finetune/Learning Rate": 3.0994148234579734e-06, "Finetune/Loss": 0.833893895149231, "Finetune/Loss (Raw)": 1.0004454851150513, "Finetune/Step": 3899, "Finetune/Step Time": 4.677818726748228} +{"Finetune/Learning Rate": 3.0949074597369444e-06, "Finetune/Loss": 0.8331044912338257, "Finetune/Loss (Raw)": 0.8617047071456909, "Finetune/Step": 3900, "Finetune/Step Time": 4.693493464961648} +{"Finetune/Learning Rate": 3.090402775747523e-06, "Finetune/Loss": 0.8377385139465332, "Finetune/Loss (Raw)": 0.938042402267456, "Finetune/Step": 3901, "Finetune/Step Time": 4.655252393335104} +{"Finetune/Learning Rate": 3.0859007732378897e-06, "Finetune/Loss": 0.8380851745605469, "Finetune/Loss (Raw)": 0.9749466180801392, "Finetune/Step": 3902, "Finetune/Step Time": 4.6397588066756725} +{"Finetune/Learning Rate": 3.0814014539551727e-06, "Finetune/Loss": 0.8380379676818848, "Finetune/Loss (Raw)": 0.9597427248954773, "Finetune/Step": 3903, "Finetune/Step Time": 4.629529481753707} +{"Finetune/Learning Rate": 3.076904819645481e-06, "Finetune/Loss": 0.8376367688179016, "Finetune/Loss (Raw)": 0.895423173904419, "Finetune/Step": 3904, "Finetune/Step Time": 4.638899093493819} +{"Finetune/Learning Rate": 3.072410872053867e-06, "Finetune/Loss": 0.8409085273742676, "Finetune/Loss (Raw)": 0.6911291480064392, "Finetune/Step": 3905, "Finetune/Step Time": 4.614688239991665} +{"Finetune/Learning Rate": 3.067919612924343e-06, "Finetune/Loss": 0.8409877419471741, "Finetune/Loss (Raw)": 0.9389039874076843, "Finetune/Step": 3906, "Finetune/Step Time": 4.6486039608716965} +{"Finetune/Learning Rate": 3.0634310439998803e-06, "Finetune/Loss": 0.8433656692504883, "Finetune/Loss (Raw)": 1.1516817808151245, "Finetune/Step": 3907, "Finetune/Step Time": 4.633570358157158} +{"Finetune/Learning Rate": 3.0589451670224037e-06, "Finetune/Loss": 0.8435255289077759, "Finetune/Loss (Raw)": 0.8254570364952087, "Finetune/Step": 3908, "Finetune/Step Time": 4.646082125604153} +{"Finetune/Learning Rate": 3.054461983732795e-06, "Finetune/Loss": 0.8425623178482056, "Finetune/Loss (Raw)": 0.8292035460472107, "Finetune/Step": 3909, "Finetune/Step Time": 4.657626777887344} +{"Finetune/Learning Rate": 3.0499814958708883e-06, "Finetune/Loss": 0.8434779644012451, "Finetune/Loss (Raw)": 0.9567773342132568, "Finetune/Step": 3910, "Finetune/Step Time": 4.658295031636953} +{"Finetune/Learning Rate": 3.045503705175478e-06, "Finetune/Loss": 0.8451724648475647, "Finetune/Loss (Raw)": 1.0227633714675903, "Finetune/Step": 3911, "Finetune/Step Time": 4.649090034887195} +{"Finetune/Learning Rate": 3.041028613384307e-06, "Finetune/Loss": 0.8451293110847473, "Finetune/Loss (Raw)": 0.8096153736114502, "Finetune/Step": 3912, "Finetune/Step Time": 4.6469350308179855} +{"Finetune/Learning Rate": 3.036556222234063e-06, "Finetune/Loss": 0.8453083038330078, "Finetune/Loss (Raw)": 0.8948294520378113, "Finetune/Step": 3913, "Finetune/Step Time": 4.656189011409879} +{"Finetune/Learning Rate": 3.032086533460402e-06, "Finetune/Loss": 0.8441541194915771, "Finetune/Loss (Raw)": 0.8371512293815613, "Finetune/Step": 3914, "Finetune/Step Time": 4.659396044909954} +{"Finetune/Learning Rate": 3.027619548797921e-06, "Finetune/Loss": 0.8439955711364746, "Finetune/Loss (Raw)": 0.9082269072532654, "Finetune/Step": 3915, "Finetune/Step Time": 4.656497407704592} +{"Finetune/Learning Rate": 3.023155269980169e-06, "Finetune/Loss": 0.8441247940063477, "Finetune/Loss (Raw)": 0.8944329023361206, "Finetune/Step": 3916, "Finetune/Step Time": 4.656891383230686} +{"Finetune/Learning Rate": 3.0186936987396442e-06, "Finetune/Loss": 0.8449939489364624, "Finetune/Loss (Raw)": 0.9057420492172241, "Finetune/Step": 3917, "Finetune/Step Time": 4.657520851120353} +{"Finetune/Learning Rate": 3.014234836807798e-06, "Finetune/Loss": 0.8450548648834229, "Finetune/Loss (Raw)": 0.9551673531532288, "Finetune/Step": 3918, "Finetune/Step Time": 4.6512961983680725} +{"Finetune/Learning Rate": 3.0097786859150247e-06, "Finetune/Loss": 0.8448784351348877, "Finetune/Loss (Raw)": 0.8552477359771729, "Finetune/Step": 3919, "Finetune/Step Time": 4.6675712913274765} +{"Finetune/Learning Rate": 3.005325247790668e-06, "Finetune/Loss": 0.8490955829620361, "Finetune/Loss (Raw)": 0.8856821060180664, "Finetune/Step": 3920, "Finetune/Step Time": 4.665001634508371} +{"Finetune/Learning Rate": 3.0008745241630266e-06, "Finetune/Loss": 0.847233235836029, "Finetune/Loss (Raw)": 0.6780298352241516, "Finetune/Step": 3921, "Finetune/Step Time": 4.674403242766857} +{"Finetune/Learning Rate": 2.9964265167593386e-06, "Finetune/Loss": 0.8473992347717285, "Finetune/Loss (Raw)": 0.9476900100708008, "Finetune/Step": 3922, "Finetune/Step Time": 4.677337044849992} +{"Finetune/Learning Rate": 2.9919812273057815e-06, "Finetune/Loss": 0.8470635414123535, "Finetune/Loss (Raw)": 0.7196237444877625, "Finetune/Step": 3923, "Finetune/Step Time": 4.676960580050945} +{"Finetune/Learning Rate": 2.987538657527491e-06, "Finetune/Loss": 0.8470228910446167, "Finetune/Loss (Raw)": 0.9082297086715698, "Finetune/Step": 3924, "Finetune/Step Time": 4.6807148195803165} +{"Finetune/Learning Rate": 2.9830988091485426e-06, "Finetune/Loss": 0.8483182787895203, "Finetune/Loss (Raw)": 0.9111643433570862, "Finetune/Step": 3925, "Finetune/Step Time": 4.677215052768588} +{"Finetune/Learning Rate": 2.9786616838919515e-06, "Finetune/Loss": 0.8481786251068115, "Finetune/Loss (Raw)": 0.8207615613937378, "Finetune/Step": 3926, "Finetune/Step Time": 4.681795539334416} +{"Finetune/Learning Rate": 2.9742272834796813e-06, "Finetune/Loss": 0.848484992980957, "Finetune/Loss (Raw)": 0.8631147742271423, "Finetune/Step": 3927, "Finetune/Step Time": 4.689032578840852} +{"Finetune/Learning Rate": 2.969795609632636e-06, "Finetune/Loss": 0.8486782908439636, "Finetune/Loss (Raw)": 0.9868119955062866, "Finetune/Step": 3928, "Finetune/Step Time": 4.689853582531214} +{"Finetune/Learning Rate": 2.9653666640706614e-06, "Finetune/Loss": 0.8489840626716614, "Finetune/Loss (Raw)": 0.9201141595840454, "Finetune/Step": 3929, "Finetune/Step Time": 4.6861574705690145} +{"Finetune/Learning Rate": 2.9609404485125414e-06, "Finetune/Loss": 0.8480848073959351, "Finetune/Loss (Raw)": 0.8506190776824951, "Finetune/Step": 3930, "Finetune/Step Time": 4.691949587315321} +{"Finetune/Learning Rate": 2.9565169646760108e-06, "Finetune/Loss": 0.8479019403457642, "Finetune/Loss (Raw)": 0.9839960336685181, "Finetune/Step": 3931, "Finetune/Step Time": 4.693171916529536} +{"Finetune/Learning Rate": 2.9520962142777367e-06, "Finetune/Loss": 0.8472416996955872, "Finetune/Loss (Raw)": 0.8376358151435852, "Finetune/Step": 3932, "Finetune/Step Time": 4.707874884828925} +{"Finetune/Learning Rate": 2.947678199033317e-06, "Finetune/Loss": 0.8457404971122742, "Finetune/Loss (Raw)": 0.6625843644142151, "Finetune/Step": 3933, "Finetune/Step Time": 4.7161212004721165} +{"Finetune/Learning Rate": 2.943262920657306e-06, "Finetune/Loss": 0.8450133800506592, "Finetune/Loss (Raw)": 0.7924158573150635, "Finetune/Step": 3934, "Finetune/Step Time": 4.703041004016995} +{"Finetune/Learning Rate": 2.938850380863184e-06, "Finetune/Loss": 0.8449344635009766, "Finetune/Loss (Raw)": 0.8380942344665527, "Finetune/Step": 3935, "Finetune/Step Time": 4.7000818736851215} +{"Finetune/Learning Rate": 2.9344405813633724e-06, "Finetune/Loss": 0.8462764024734497, "Finetune/Loss (Raw)": 0.9534087181091309, "Finetune/Step": 3936, "Finetune/Step Time": 4.691347312182188} +{"Finetune/Learning Rate": 2.930033523869228e-06, "Finetune/Loss": 0.8466898202896118, "Finetune/Loss (Raw)": 0.9043371677398682, "Finetune/Step": 3937, "Finetune/Step Time": 4.686059854924679} +{"Finetune/Learning Rate": 2.925629210091043e-06, "Finetune/Loss": 0.8460991978645325, "Finetune/Loss (Raw)": 0.86599200963974, "Finetune/Step": 3938, "Finetune/Step Time": 4.691396387293935} +{"Finetune/Learning Rate": 2.9212276417380468e-06, "Finetune/Loss": 0.8468426465988159, "Finetune/Loss (Raw)": 0.9390981197357178, "Finetune/Step": 3939, "Finetune/Step Time": 4.692464265972376} +{"Finetune/Learning Rate": 2.9168288205183983e-06, "Finetune/Loss": 0.8474125862121582, "Finetune/Loss (Raw)": 0.8745633959770203, "Finetune/Step": 3940, "Finetune/Step Time": 4.680221291258931} +{"Finetune/Learning Rate": 2.9124327481392023e-06, "Finetune/Loss": 0.8480700254440308, "Finetune/Loss (Raw)": 0.9603122472763062, "Finetune/Step": 3941, "Finetune/Step Time": 4.683395013213158} +{"Finetune/Learning Rate": 2.9080394263064815e-06, "Finetune/Loss": 0.8491109013557434, "Finetune/Loss (Raw)": 0.9796513319015503, "Finetune/Step": 3942, "Finetune/Step Time": 4.6900594756007195} +{"Finetune/Learning Rate": 2.9036488567252006e-06, "Finetune/Loss": 0.848494827747345, "Finetune/Loss (Raw)": 0.785912811756134, "Finetune/Step": 3943, "Finetune/Step Time": 4.6961975283920765} +{"Finetune/Learning Rate": 2.8992610410992506e-06, "Finetune/Loss": 0.8487652540206909, "Finetune/Loss (Raw)": 0.9193629026412964, "Finetune/Step": 3944, "Finetune/Step Time": 4.695928871631622} +{"Finetune/Learning Rate": 2.894875981131463e-06, "Finetune/Loss": 0.8509140610694885, "Finetune/Loss (Raw)": 1.0834702253341675, "Finetune/Step": 3945, "Finetune/Step Time": 4.718465816229582} +{"Finetune/Learning Rate": 2.890493678523595e-06, "Finetune/Loss": 0.8514531850814819, "Finetune/Loss (Raw)": 0.8859483003616333, "Finetune/Step": 3946, "Finetune/Step Time": 4.732992200180888} +{"Finetune/Learning Rate": 2.8861141349763224e-06, "Finetune/Loss": 0.8522647619247437, "Finetune/Loss (Raw)": 0.9093561172485352, "Finetune/Step": 3947, "Finetune/Step Time": 4.740037204697728} +{"Finetune/Learning Rate": 2.881737352189271e-06, "Finetune/Loss": 0.8494857549667358, "Finetune/Loss (Raw)": 0.3618044853210449, "Finetune/Step": 3948, "Finetune/Step Time": 4.778381682932377} +{"Finetune/Learning Rate": 2.8773633318609796e-06, "Finetune/Loss": 0.8510143160820007, "Finetune/Loss (Raw)": 1.0115182399749756, "Finetune/Step": 3949, "Finetune/Step Time": 4.774770259857178} +{"Finetune/Learning Rate": 2.872992075688922e-06, "Finetune/Loss": 0.8525667190551758, "Finetune/Loss (Raw)": 0.9807714819908142, "Finetune/Step": 3950, "Finetune/Step Time": 4.7724512834101915} +{"Finetune/Learning Rate": 2.868623585369497e-06, "Finetune/Loss": 0.852439284324646, "Finetune/Loss (Raw)": 0.9700613021850586, "Finetune/Step": 3951, "Finetune/Step Time": 4.77632949501276} +{"Finetune/Learning Rate": 2.864257862598029e-06, "Finetune/Loss": 0.8516172766685486, "Finetune/Loss (Raw)": 0.7910705804824829, "Finetune/Step": 3952, "Finetune/Step Time": 4.785212216898799} +{"Finetune/Learning Rate": 2.8598949090687723e-06, "Finetune/Loss": 0.8522731065750122, "Finetune/Loss (Raw)": 0.8495694398880005, "Finetune/Step": 3953, "Finetune/Step Time": 4.782613215968013} +{"Finetune/Learning Rate": 2.8555347264748988e-06, "Finetune/Loss": 0.8523310422897339, "Finetune/Loss (Raw)": 0.8200994729995728, "Finetune/Step": 3954, "Finetune/Step Time": 4.776158507913351} +{"Finetune/Learning Rate": 2.8511773165085164e-06, "Finetune/Loss": 0.8514758348464966, "Finetune/Loss (Raw)": 0.798182487487793, "Finetune/Step": 3955, "Finetune/Step Time": 4.744097009301186} +{"Finetune/Learning Rate": 2.8468226808606525e-06, "Finetune/Loss": 0.8523812294006348, "Finetune/Loss (Raw)": 1.0662915706634521, "Finetune/Step": 3956, "Finetune/Step Time": 4.74446289986372} +{"Finetune/Learning Rate": 2.842470821221246e-06, "Finetune/Loss": 0.8572218418121338, "Finetune/Loss (Raw)": 0.8987512588500977, "Finetune/Step": 3957, "Finetune/Step Time": 4.70710602030158} +{"Finetune/Learning Rate": 2.8381217392791784e-06, "Finetune/Loss": 0.85686856508255, "Finetune/Loss (Raw)": 0.8422278165817261, "Finetune/Step": 3958, "Finetune/Step Time": 4.703553261235356} +{"Finetune/Learning Rate": 2.833775436722238e-06, "Finetune/Loss": 0.8564763069152832, "Finetune/Loss (Raw)": 0.7842705249786377, "Finetune/Step": 3959, "Finetune/Step Time": 4.716415213420987} +{"Finetune/Learning Rate": 2.829431915237144e-06, "Finetune/Loss": 0.8548691868782043, "Finetune/Loss (Raw)": 0.829486608505249, "Finetune/Step": 3960, "Finetune/Step Time": 4.714542603120208} +{"Finetune/Learning Rate": 2.825091176509529e-06, "Finetune/Loss": 0.8543984889984131, "Finetune/Loss (Raw)": 0.9055243134498596, "Finetune/Step": 3961, "Finetune/Step Time": 4.696497147902846} +{"Finetune/Learning Rate": 2.8207532222239497e-06, "Finetune/Loss": 0.8528363108634949, "Finetune/Loss (Raw)": 0.7747289538383484, "Finetune/Step": 3962, "Finetune/Step Time": 4.7100000362843275} +{"Finetune/Learning Rate": 2.816418054063883e-06, "Finetune/Loss": 0.8526948690414429, "Finetune/Loss (Raw)": 0.9251420497894287, "Finetune/Step": 3963, "Finetune/Step Time": 4.708279486745596} +{"Finetune/Learning Rate": 2.8120856737117187e-06, "Finetune/Loss": 0.853162407875061, "Finetune/Loss (Raw)": 0.8768172860145569, "Finetune/Step": 3964, "Finetune/Step Time": 4.717162748798728} +{"Finetune/Learning Rate": 2.807756082848775e-06, "Finetune/Loss": 0.8542116284370422, "Finetune/Loss (Raw)": 0.9890971779823303, "Finetune/Step": 3965, "Finetune/Step Time": 4.72945062816143} +{"Finetune/Learning Rate": 2.803429283155281e-06, "Finetune/Loss": 0.8508452773094177, "Finetune/Loss (Raw)": 0.38731124997138977, "Finetune/Step": 3966, "Finetune/Step Time": 4.776684712618589} +{"Finetune/Learning Rate": 2.7991052763103767e-06, "Finetune/Loss": 0.8503551483154297, "Finetune/Loss (Raw)": 0.8268734216690063, "Finetune/Step": 3967, "Finetune/Step Time": 4.744837641716003} +{"Finetune/Learning Rate": 2.7947840639921308e-06, "Finetune/Loss": 0.8510693907737732, "Finetune/Loss (Raw)": 0.937066912651062, "Finetune/Step": 3968, "Finetune/Step Time": 4.743129866197705} +{"Finetune/Learning Rate": 2.7904656478775196e-06, "Finetune/Loss": 0.8503642082214355, "Finetune/Loss (Raw)": 0.9170433878898621, "Finetune/Step": 3969, "Finetune/Step Time": 4.773102540522814} +{"Finetune/Learning Rate": 2.7861500296424372e-06, "Finetune/Loss": 0.8513342142105103, "Finetune/Loss (Raw)": 0.9408237338066101, "Finetune/Step": 3970, "Finetune/Step Time": 4.773472206667066} +{"Finetune/Learning Rate": 2.781837210961689e-06, "Finetune/Loss": 0.8523651361465454, "Finetune/Loss (Raw)": 0.9716206789016724, "Finetune/Step": 3971, "Finetune/Step Time": 4.765022346749902} +{"Finetune/Learning Rate": 2.7775271935089964e-06, "Finetune/Loss": 0.8519566059112549, "Finetune/Loss (Raw)": 0.7998466491699219, "Finetune/Step": 3972, "Finetune/Step Time": 4.7620085049420595} +{"Finetune/Learning Rate": 2.7732199789569937e-06, "Finetune/Loss": 0.8518863916397095, "Finetune/Loss (Raw)": 0.8389438390731812, "Finetune/Step": 3973, "Finetune/Step Time": 4.7531432919204235} +{"Finetune/Learning Rate": 2.768915568977222e-06, "Finetune/Loss": 0.8527618646621704, "Finetune/Loss (Raw)": 0.8140038847923279, "Finetune/Step": 3974, "Finetune/Step Time": 4.756374955177307} +{"Finetune/Learning Rate": 2.7646139652401473e-06, "Finetune/Loss": 0.8524329662322998, "Finetune/Loss (Raw)": 0.9482437372207642, "Finetune/Step": 3975, "Finetune/Step Time": 4.760098464787006} +{"Finetune/Learning Rate": 2.7603151694151354e-06, "Finetune/Loss": 0.8566105365753174, "Finetune/Loss (Raw)": 0.867207944393158, "Finetune/Step": 3976, "Finetune/Step Time": 4.740291081368923} +{"Finetune/Learning Rate": 2.7560191831704584e-06, "Finetune/Loss": 0.854438304901123, "Finetune/Loss (Raw)": 0.7663539052009583, "Finetune/Step": 3977, "Finetune/Step Time": 4.73495071567595} +{"Finetune/Learning Rate": 2.751726008173312e-06, "Finetune/Loss": 0.8546202182769775, "Finetune/Loss (Raw)": 0.6442332863807678, "Finetune/Step": 3978, "Finetune/Step Time": 4.7211688086390495} +{"Finetune/Learning Rate": 2.747435646089791e-06, "Finetune/Loss": 0.8555620908737183, "Finetune/Loss (Raw)": 0.887811005115509, "Finetune/Step": 3979, "Finetune/Step Time": 4.724568519741297} +{"Finetune/Learning Rate": 2.7431480985849023e-06, "Finetune/Loss": 0.853487491607666, "Finetune/Loss (Raw)": 0.6590789556503296, "Finetune/Step": 3980, "Finetune/Step Time": 4.726682972162962} +{"Finetune/Learning Rate": 2.7388633673225586e-06, "Finetune/Loss": 0.8573729991912842, "Finetune/Loss (Raw)": 0.9513867497444153, "Finetune/Step": 3981, "Finetune/Step Time": 4.684471605345607} +{"Finetune/Learning Rate": 2.734581453965579e-06, "Finetune/Loss": 0.8568629622459412, "Finetune/Loss (Raw)": 0.8370746374130249, "Finetune/Step": 3982, "Finetune/Step Time": 4.6893731988966465} +{"Finetune/Learning Rate": 2.730302360175693e-06, "Finetune/Loss": 0.8582815527915955, "Finetune/Loss (Raw)": 0.8387336134910583, "Finetune/Step": 3983, "Finetune/Step Time": 4.673232449218631} +{"Finetune/Learning Rate": 2.726026087613528e-06, "Finetune/Loss": 0.8531609773635864, "Finetune/Loss (Raw)": 0.340870201587677, "Finetune/Step": 3984, "Finetune/Step Time": 4.7111716363579035} +{"Finetune/Learning Rate": 2.7217526379386327e-06, "Finetune/Loss": 0.8575179576873779, "Finetune/Loss (Raw)": 0.9204460978507996, "Finetune/Step": 3985, "Finetune/Step Time": 4.675925510004163} +{"Finetune/Learning Rate": 2.7174820128094393e-06, "Finetune/Loss": 0.8632072806358337, "Finetune/Loss (Raw)": 1.0652531385421753, "Finetune/Step": 3986, "Finetune/Step Time": 4.630980238318443} +{"Finetune/Learning Rate": 2.713214213883295e-06, "Finetune/Loss": 0.8611059188842773, "Finetune/Loss (Raw)": 0.7880885601043701, "Finetune/Step": 3987, "Finetune/Step Time": 4.638614568859339} +{"Finetune/Learning Rate": 2.7089492428164556e-06, "Finetune/Loss": 0.8615971803665161, "Finetune/Loss (Raw)": 0.9549853801727295, "Finetune/Step": 3988, "Finetune/Step Time": 4.642225708812475} +{"Finetune/Learning Rate": 2.7046871012640687e-06, "Finetune/Loss": 0.8625035285949707, "Finetune/Loss (Raw)": 0.9644919037818909, "Finetune/Step": 3989, "Finetune/Step Time": 4.641319630667567} +{"Finetune/Learning Rate": 2.7004277908801877e-06, "Finetune/Loss": 0.8627606630325317, "Finetune/Loss (Raw)": 0.9200894832611084, "Finetune/Step": 3990, "Finetune/Step Time": 4.645368620753288} +{"Finetune/Learning Rate": 2.6961713133177705e-06, "Finetune/Loss": 0.8638144731521606, "Finetune/Loss (Raw)": 0.9681882262229919, "Finetune/Step": 3991, "Finetune/Step Time": 4.637180723249912} +{"Finetune/Learning Rate": 2.6919176702286696e-06, "Finetune/Loss": 0.8624048233032227, "Finetune/Loss (Raw)": 0.7517939805984497, "Finetune/Step": 3992, "Finetune/Step Time": 4.65979952737689} +{"Finetune/Learning Rate": 2.687666863263644e-06, "Finetune/Loss": 0.8625683784484863, "Finetune/Loss (Raw)": 0.9078848958015442, "Finetune/Step": 3993, "Finetune/Step Time": 4.62612590752542} +{"Finetune/Learning Rate": 2.683418894072345e-06, "Finetune/Loss": 0.8605451583862305, "Finetune/Loss (Raw)": 0.7747077941894531, "Finetune/Step": 3994, "Finetune/Step Time": 4.638465546071529} +{"Finetune/Learning Rate": 2.6791737643033333e-06, "Finetune/Loss": 0.8584648966789246, "Finetune/Loss (Raw)": 0.8118371963500977, "Finetune/Step": 3995, "Finetune/Step Time": 4.638684064149857} +{"Finetune/Learning Rate": 2.6749314756040547e-06, "Finetune/Loss": 0.8577625751495361, "Finetune/Loss (Raw)": 0.8047218322753906, "Finetune/Step": 3996, "Finetune/Step Time": 4.637139115482569} +{"Finetune/Learning Rate": 2.6706920296208573e-06, "Finetune/Loss": 0.8579641580581665, "Finetune/Loss (Raw)": 0.8399498462677002, "Finetune/Step": 3997, "Finetune/Step Time": 4.6302221324294806} +{"Finetune/Learning Rate": 2.666455427998994e-06, "Finetune/Loss": 0.8595553636550903, "Finetune/Loss (Raw)": 1.0163136720657349, "Finetune/Step": 3998, "Finetune/Step Time": 4.613666890189052} +{"Finetune/Learning Rate": 2.6622216723826035e-06, "Finetune/Loss": 0.8598959445953369, "Finetune/Loss (Raw)": 0.9235422015190125, "Finetune/Step": 3999, "Finetune/Step Time": 4.630016041919589} +{"Finetune/Learning Rate": 2.6579907644147227e-06, "Finetune/Loss": 0.8623040318489075, "Finetune/Loss (Raw)": 0.8274335265159607, "Finetune/Step": 4000, "Finetune/Step Time": 4.600101632997394} +{"Finetune/Learning Rate": 2.653762705737287e-06, "Finetune/Loss": 0.8631414771080017, "Finetune/Loss (Raw)": 0.8418627977371216, "Finetune/Step": 4001, "Finetune/Step Time": 4.611228622496128} +{"Finetune/Learning Rate": 2.6495374979911215e-06, "Finetune/Loss": 0.8625686764717102, "Finetune/Loss (Raw)": 0.8056145310401917, "Finetune/Step": 4002, "Finetune/Step Time": 4.6216460932046175} +{"Finetune/Learning Rate": 2.6453151428159472e-06, "Finetune/Loss": 0.8624712824821472, "Finetune/Loss (Raw)": 0.8701839447021484, "Finetune/Step": 4003, "Finetune/Step Time": 4.633791785687208} +{"Finetune/Learning Rate": 2.641095641850375e-06, "Finetune/Loss": 0.8627631664276123, "Finetune/Loss (Raw)": 0.9248159527778625, "Finetune/Step": 4004, "Finetune/Step Time": 4.643421817570925} +{"Finetune/Learning Rate": 2.6368789967319196e-06, "Finetune/Loss": 0.8627786040306091, "Finetune/Loss (Raw)": 0.8811371922492981, "Finetune/Step": 4005, "Finetune/Step Time": 4.628374831750989} +{"Finetune/Learning Rate": 2.6326652090969697e-06, "Finetune/Loss": 0.8672700524330139, "Finetune/Loss (Raw)": 1.0476148128509521, "Finetune/Step": 4006, "Finetune/Step Time": 4.588141081854701} +{"Finetune/Learning Rate": 2.6284542805808144e-06, "Finetune/Loss": 0.8634419441223145, "Finetune/Loss (Raw)": 0.36149847507476807, "Finetune/Step": 4007, "Finetune/Step Time": 4.590270075947046} +{"Finetune/Learning Rate": 2.624246212817638e-06, "Finetune/Loss": 0.8627251982688904, "Finetune/Loss (Raw)": 0.7750301957130432, "Finetune/Step": 4008, "Finetune/Step Time": 4.588364668190479} +{"Finetune/Learning Rate": 2.620041007440508e-06, "Finetune/Loss": 0.8621166944503784, "Finetune/Loss (Raw)": 0.838660717010498, "Finetune/Step": 4009, "Finetune/Step Time": 4.584016680717468} +{"Finetune/Learning Rate": 2.6158386660813807e-06, "Finetune/Loss": 0.8610039949417114, "Finetune/Loss (Raw)": 0.8034661412239075, "Finetune/Step": 4010, "Finetune/Step Time": 4.592223754152656} +{"Finetune/Learning Rate": 2.6116391903711035e-06, "Finetune/Loss": 0.8610014915466309, "Finetune/Loss (Raw)": 0.9811006188392639, "Finetune/Step": 4011, "Finetune/Step Time": 4.597495725378394} +{"Finetune/Learning Rate": 2.607442581939411e-06, "Finetune/Loss": 0.8598514199256897, "Finetune/Loss (Raw)": 0.6793054342269897, "Finetune/Step": 4012, "Finetune/Step Time": 4.615156497806311} +{"Finetune/Learning Rate": 2.603248842414925e-06, "Finetune/Loss": 0.8612584471702576, "Finetune/Loss (Raw)": 0.8657881021499634, "Finetune/Step": 4013, "Finetune/Step Time": 4.60935989767313} +{"Finetune/Learning Rate": 2.599057973425151e-06, "Finetune/Loss": 0.861807107925415, "Finetune/Loss (Raw)": 0.8224722743034363, "Finetune/Step": 4014, "Finetune/Step Time": 4.6345556899905205} +{"Finetune/Learning Rate": 2.5948699765964915e-06, "Finetune/Loss": 0.8621684312820435, "Finetune/Loss (Raw)": 0.8535510301589966, "Finetune/Step": 4015, "Finetune/Step Time": 4.635112199932337} +{"Finetune/Learning Rate": 2.5906848535542196e-06, "Finetune/Loss": 0.862063467502594, "Finetune/Loss (Raw)": 0.894238293170929, "Finetune/Step": 4016, "Finetune/Step Time": 4.6366669498384} +{"Finetune/Learning Rate": 2.5865026059224974e-06, "Finetune/Loss": 0.8623228669166565, "Finetune/Loss (Raw)": 0.9157037138938904, "Finetune/Step": 4017, "Finetune/Step Time": 4.639098996296525} +{"Finetune/Learning Rate": 2.58232323532438e-06, "Finetune/Loss": 0.8576714992523193, "Finetune/Loss (Raw)": 0.3260382413864136, "Finetune/Step": 4018, "Finetune/Step Time": 4.678154267370701} +{"Finetune/Learning Rate": 2.5781467433817973e-06, "Finetune/Loss": 0.8585933446884155, "Finetune/Loss (Raw)": 1.0026652812957764, "Finetune/Step": 4019, "Finetune/Step Time": 4.6579729821532965} +{"Finetune/Learning Rate": 2.573973131715565e-06, "Finetune/Loss": 0.8593257665634155, "Finetune/Loss (Raw)": 0.9520164728164673, "Finetune/Step": 4020, "Finetune/Step Time": 4.656545039266348} +{"Finetune/Learning Rate": 2.569802401945378e-06, "Finetune/Loss": 0.8633832931518555, "Finetune/Loss (Raw)": 0.8893468379974365, "Finetune/Step": 4021, "Finetune/Step Time": 4.629382977262139} +{"Finetune/Learning Rate": 2.565634555689818e-06, "Finetune/Loss": 0.862575888633728, "Finetune/Loss (Raw)": 0.8065848350524902, "Finetune/Step": 4022, "Finetune/Step Time": 4.6321227084845304} +{"Finetune/Learning Rate": 2.5614695945663415e-06, "Finetune/Loss": 0.862484872341156, "Finetune/Loss (Raw)": 0.9131697416305542, "Finetune/Step": 4023, "Finetune/Step Time": 4.634839717298746} +{"Finetune/Learning Rate": 2.557307520191291e-06, "Finetune/Loss": 0.8626763224601746, "Finetune/Loss (Raw)": 0.9678615927696228, "Finetune/Step": 4024, "Finetune/Step Time": 4.64087850600481} +{"Finetune/Learning Rate": 2.5531483341798856e-06, "Finetune/Loss": 0.862562894821167, "Finetune/Loss (Raw)": 0.8679291605949402, "Finetune/Step": 4025, "Finetune/Step Time": 4.640648508444428} +{"Finetune/Learning Rate": 2.548992038146224e-06, "Finetune/Loss": 0.8634738922119141, "Finetune/Loss (Raw)": 0.9727238416671753, "Finetune/Step": 4026, "Finetune/Step Time": 4.627568865194917} +{"Finetune/Learning Rate": 2.5448386337032817e-06, "Finetune/Loss": 0.8635175228118896, "Finetune/Loss (Raw)": 1.006018042564392, "Finetune/Step": 4027, "Finetune/Step Time": 4.624026499688625} +{"Finetune/Learning Rate": 2.5406881224629175e-06, "Finetune/Loss": 0.8631609082221985, "Finetune/Loss (Raw)": 0.8160633444786072, "Finetune/Step": 4028, "Finetune/Step Time": 4.604124030098319} +{"Finetune/Learning Rate": 2.5365405060358663e-06, "Finetune/Loss": 0.8629090785980225, "Finetune/Loss (Raw)": 0.9058094024658203, "Finetune/Step": 4029, "Finetune/Step Time": 4.6044449135661125} +{"Finetune/Learning Rate": 2.532395786031726e-06, "Finetune/Loss": 0.8583801984786987, "Finetune/Loss (Raw)": 0.39525145292282104, "Finetune/Step": 4030, "Finetune/Step Time": 4.639182658866048} +{"Finetune/Learning Rate": 2.528253964058992e-06, "Finetune/Loss": 0.8577454090118408, "Finetune/Loss (Raw)": 0.878480851650238, "Finetune/Step": 4031, "Finetune/Step Time": 4.644586497917771} +{"Finetune/Learning Rate": 2.5241150417250216e-06, "Finetune/Loss": 0.857646644115448, "Finetune/Loss (Raw)": 0.882789134979248, "Finetune/Step": 4032, "Finetune/Step Time": 4.632024832069874} +{"Finetune/Learning Rate": 2.5199790206360486e-06, "Finetune/Loss": 0.85953289270401, "Finetune/Loss (Raw)": 0.9325651526451111, "Finetune/Step": 4033, "Finetune/Step Time": 4.620567390695214} +{"Finetune/Learning Rate": 2.5158459023971826e-06, "Finetune/Loss": 0.8589046001434326, "Finetune/Loss (Raw)": 0.8584809303283691, "Finetune/Step": 4034, "Finetune/Step Time": 4.585953524336219} +{"Finetune/Learning Rate": 2.511715688612405e-06, "Finetune/Loss": 0.8564716577529907, "Finetune/Loss (Raw)": 0.8402731418609619, "Finetune/Step": 4035, "Finetune/Step Time": 4.598472945392132} +{"Finetune/Learning Rate": 2.5075883808845736e-06, "Finetune/Loss": 0.8581568002700806, "Finetune/Loss (Raw)": 1.041143536567688, "Finetune/Step": 4036, "Finetune/Step Time": 4.579764474183321} +{"Finetune/Learning Rate": 2.5034639808154115e-06, "Finetune/Loss": 0.857868492603302, "Finetune/Loss (Raw)": 0.7923104763031006, "Finetune/Step": 4037, "Finetune/Step Time": 4.580733688548207} +{"Finetune/Learning Rate": 2.4993424900055228e-06, "Finetune/Loss": 0.853172779083252, "Finetune/Loss (Raw)": 0.3557174503803253, "Finetune/Step": 4038, "Finetune/Step Time": 4.616815103217959} +{"Finetune/Learning Rate": 2.4952239100543808e-06, "Finetune/Loss": 0.8525333404541016, "Finetune/Loss (Raw)": 0.9409200549125671, "Finetune/Step": 4039, "Finetune/Step Time": 4.637287857010961} +{"Finetune/Learning Rate": 2.491108242560313e-06, "Finetune/Loss": 0.8534425497055054, "Finetune/Loss (Raw)": 0.9259913563728333, "Finetune/Step": 4040, "Finetune/Step Time": 4.626012198626995} +{"Finetune/Learning Rate": 2.486995489120542e-06, "Finetune/Loss": 0.8528331518173218, "Finetune/Loss (Raw)": 0.8168331980705261, "Finetune/Step": 4041, "Finetune/Step Time": 4.615420950576663} +{"Finetune/Learning Rate": 2.4828856513311405e-06, "Finetune/Loss": 0.8530689477920532, "Finetune/Loss (Raw)": 0.8673271536827087, "Finetune/Step": 4042, "Finetune/Step Time": 4.6280834302306175} +{"Finetune/Learning Rate": 2.4787787307870605e-06, "Finetune/Loss": 0.8530721664428711, "Finetune/Loss (Raw)": 0.9086475968360901, "Finetune/Step": 4043, "Finetune/Step Time": 4.630926258862019} +{"Finetune/Learning Rate": 2.474674729082115e-06, "Finetune/Loss": 0.8530030250549316, "Finetune/Loss (Raw)": 0.885577917098999, "Finetune/Step": 4044, "Finetune/Step Time": 4.628058193251491} +{"Finetune/Learning Rate": 2.470573647808987e-06, "Finetune/Loss": 0.8482089042663574, "Finetune/Loss (Raw)": 0.29209351539611816, "Finetune/Step": 4045, "Finetune/Step Time": 4.651281215250492} +{"Finetune/Learning Rate": 2.466475488559227e-06, "Finetune/Loss": 0.8471097946166992, "Finetune/Loss (Raw)": 0.8144856095314026, "Finetune/Step": 4046, "Finetune/Step Time": 4.648095108568668} +{"Finetune/Learning Rate": 2.4623802529232476e-06, "Finetune/Loss": 0.8466999530792236, "Finetune/Loss (Raw)": 0.8027821183204651, "Finetune/Step": 4047, "Finetune/Step Time": 4.645111145451665} +{"Finetune/Learning Rate": 2.458287942490335e-06, "Finetune/Loss": 0.8426996469497681, "Finetune/Loss (Raw)": 0.37364017963409424, "Finetune/Step": 4048, "Finetune/Step Time": 4.646840091794729} +{"Finetune/Learning Rate": 2.454198558848635e-06, "Finetune/Loss": 0.8433287143707275, "Finetune/Loss (Raw)": 0.7585525512695312, "Finetune/Step": 4049, "Finetune/Step Time": 4.635505734011531} +{"Finetune/Learning Rate": 2.4501121035851494e-06, "Finetune/Loss": 0.8424746990203857, "Finetune/Loss (Raw)": 0.8383727073669434, "Finetune/Step": 4050, "Finetune/Step Time": 4.633623102679849} +{"Finetune/Learning Rate": 2.4460285782857606e-06, "Finetune/Loss": 0.8430399298667908, "Finetune/Loss (Raw)": 0.7919816374778748, "Finetune/Step": 4051, "Finetune/Step Time": 4.625323647633195} +{"Finetune/Learning Rate": 2.441947984535201e-06, "Finetune/Loss": 0.8425562977790833, "Finetune/Loss (Raw)": 0.8463166952133179, "Finetune/Step": 4052, "Finetune/Step Time": 4.640479879453778} +{"Finetune/Learning Rate": 2.4378703239170707e-06, "Finetune/Loss": 0.8383474349975586, "Finetune/Loss (Raw)": 0.3724346458911896, "Finetune/Step": 4053, "Finetune/Step Time": 4.6749521512538195} +{"Finetune/Learning Rate": 2.4337955980138293e-06, "Finetune/Loss": 0.8386458158493042, "Finetune/Loss (Raw)": 0.8589541912078857, "Finetune/Step": 4054, "Finetune/Step Time": 4.684453301131725} +{"Finetune/Learning Rate": 2.4297238084067988e-06, "Finetune/Loss": 0.8389843106269836, "Finetune/Loss (Raw)": 0.9064452648162842, "Finetune/Step": 4055, "Finetune/Step Time": 4.666119206696749} +{"Finetune/Learning Rate": 2.425654956676161e-06, "Finetune/Loss": 0.8363070487976074, "Finetune/Loss (Raw)": 0.6441161632537842, "Finetune/Step": 4056, "Finetune/Step Time": 4.681682210415602} +{"Finetune/Learning Rate": 2.421589044400955e-06, "Finetune/Loss": 0.8367434740066528, "Finetune/Loss (Raw)": 0.9759835004806519, "Finetune/Step": 4057, "Finetune/Step Time": 4.683882752433419} +{"Finetune/Learning Rate": 2.4175260731590878e-06, "Finetune/Loss": 0.8365541100502014, "Finetune/Loss (Raw)": 0.8263785243034363, "Finetune/Step": 4058, "Finetune/Step Time": 4.675584550946951} +{"Finetune/Learning Rate": 2.413466044527318e-06, "Finetune/Loss": 0.8358075618743896, "Finetune/Loss (Raw)": 0.8884366154670715, "Finetune/Step": 4059, "Finetune/Step Time": 4.6898287534713745} +{"Finetune/Learning Rate": 2.4094089600812576e-06, "Finetune/Loss": 0.8343624472618103, "Finetune/Loss (Raw)": 0.6526580452919006, "Finetune/Step": 4060, "Finetune/Step Time": 4.691937141120434} +{"Finetune/Learning Rate": 2.405354821395388e-06, "Finetune/Loss": 0.8354873657226562, "Finetune/Loss (Raw)": 0.8065739274024963, "Finetune/Step": 4061, "Finetune/Step Time": 4.684272704645991} +{"Finetune/Learning Rate": 2.4013036300430403e-06, "Finetune/Loss": 0.8372081518173218, "Finetune/Loss (Raw)": 1.0126755237579346, "Finetune/Step": 4062, "Finetune/Step Time": 4.687235126271844} +{"Finetune/Learning Rate": 2.3972553875964035e-06, "Finetune/Loss": 0.8334092497825623, "Finetune/Loss (Raw)": 0.3518349528312683, "Finetune/Step": 4063, "Finetune/Step Time": 4.713868714869022} +{"Finetune/Learning Rate": 2.393210095626515e-06, "Finetune/Loss": 0.828936755657196, "Finetune/Loss (Raw)": 0.3809289336204529, "Finetune/Step": 4064, "Finetune/Step Time": 4.753386622294784} +{"Finetune/Learning Rate": 2.38916775570328e-06, "Finetune/Loss": 0.8280178904533386, "Finetune/Loss (Raw)": 0.7867277264595032, "Finetune/Step": 4065, "Finetune/Step Time": 4.758592562749982} +{"Finetune/Learning Rate": 2.3851283693954506e-06, "Finetune/Loss": 0.8280651569366455, "Finetune/Loss (Raw)": 0.8720337748527527, "Finetune/Step": 4066, "Finetune/Step Time": 4.756993414834142} +{"Finetune/Learning Rate": 2.3810919382706297e-06, "Finetune/Loss": 0.8284053802490234, "Finetune/Loss (Raw)": 0.9826568961143494, "Finetune/Step": 4067, "Finetune/Step Time": 4.761297225952148} +{"Finetune/Learning Rate": 2.3770584638952874e-06, "Finetune/Loss": 0.8287672400474548, "Finetune/Loss (Raw)": 0.9208710789680481, "Finetune/Step": 4068, "Finetune/Step Time": 4.762650042772293} +{"Finetune/Learning Rate": 2.3730279478347275e-06, "Finetune/Loss": 0.8284715414047241, "Finetune/Loss (Raw)": 0.9224696755409241, "Finetune/Step": 4069, "Finetune/Step Time": 4.775686539709568} +{"Finetune/Learning Rate": 2.3690003916531157e-06, "Finetune/Loss": 0.8272351026535034, "Finetune/Loss (Raw)": 0.8213847279548645, "Finetune/Step": 4070, "Finetune/Step Time": 4.771207867190242} +{"Finetune/Learning Rate": 2.3649757969134667e-06, "Finetune/Loss": 0.8279495239257812, "Finetune/Loss (Raw)": 0.8773586750030518, "Finetune/Step": 4071, "Finetune/Step Time": 4.777747603133321} +{"Finetune/Learning Rate": 2.3609541651776537e-06, "Finetune/Loss": 0.8268436193466187, "Finetune/Loss (Raw)": 0.7778053283691406, "Finetune/Step": 4072, "Finetune/Step Time": 4.78746017254889} +{"Finetune/Learning Rate": 2.356935498006391e-06, "Finetune/Loss": 0.8253893852233887, "Finetune/Loss (Raw)": 0.8973339796066284, "Finetune/Step": 4073, "Finetune/Step Time": 4.769749196246266} +{"Finetune/Learning Rate": 2.3529197969592375e-06, "Finetune/Loss": 0.8261991739273071, "Finetune/Loss (Raw)": 0.989602267742157, "Finetune/Step": 4074, "Finetune/Step Time": 4.757207132875919} +{"Finetune/Learning Rate": 2.3489070635946175e-06, "Finetune/Loss": 0.8255503177642822, "Finetune/Loss (Raw)": 0.8263010382652283, "Finetune/Step": 4075, "Finetune/Step Time": 4.745594650506973} +{"Finetune/Learning Rate": 2.3448972994697906e-06, "Finetune/Loss": 0.8280248045921326, "Finetune/Loss (Raw)": 0.6785352826118469, "Finetune/Step": 4076, "Finetune/Step Time": 4.727150997146964} +{"Finetune/Learning Rate": 2.340890506140869e-06, "Finetune/Loss": 0.8268616795539856, "Finetune/Loss (Raw)": 0.8626407384872437, "Finetune/Step": 4077, "Finetune/Step Time": 4.740853412076831} +{"Finetune/Learning Rate": 2.3368866851628102e-06, "Finetune/Loss": 0.8255422711372375, "Finetune/Loss (Raw)": 0.8118851780891418, "Finetune/Step": 4078, "Finetune/Step Time": 4.743470868095756} +{"Finetune/Learning Rate": 2.3328858380894194e-06, "Finetune/Loss": 0.8261558413505554, "Finetune/Loss (Raw)": 1.0485962629318237, "Finetune/Step": 4079, "Finetune/Step Time": 4.754935000091791} +{"Finetune/Learning Rate": 2.328887966473348e-06, "Finetune/Loss": 0.82607102394104, "Finetune/Loss (Raw)": 0.780219554901123, "Finetune/Step": 4080, "Finetune/Step Time": 4.754210498183966} +{"Finetune/Learning Rate": 2.324893071866087e-06, "Finetune/Loss": 0.826213002204895, "Finetune/Loss (Raw)": 0.8677381277084351, "Finetune/Step": 4081, "Finetune/Step Time": 4.776289563626051} +{"Finetune/Learning Rate": 2.320901155817983e-06, "Finetune/Loss": 0.8261973857879639, "Finetune/Loss (Raw)": 0.8181045651435852, "Finetune/Step": 4082, "Finetune/Step Time": 4.784358058124781} +{"Finetune/Learning Rate": 2.316912219878222e-06, "Finetune/Loss": 0.8276287317276001, "Finetune/Loss (Raw)": 0.9813987016677856, "Finetune/Step": 4083, "Finetune/Step Time": 4.786543190479279} +{"Finetune/Learning Rate": 2.312926265594825e-06, "Finetune/Loss": 0.8229409456253052, "Finetune/Loss (Raw)": 0.4662436544895172, "Finetune/Step": 4084, "Finetune/Step Time": 4.818619705736637} +{"Finetune/Learning Rate": 2.308943294514667e-06, "Finetune/Loss": 0.8225717544555664, "Finetune/Loss (Raw)": 0.851500928401947, "Finetune/Step": 4085, "Finetune/Step Time": 4.830030320212245} +{"Finetune/Learning Rate": 2.3049633081834634e-06, "Finetune/Loss": 0.8229179978370667, "Finetune/Loss (Raw)": 0.8865440487861633, "Finetune/Step": 4086, "Finetune/Step Time": 4.837711380794644} +{"Finetune/Learning Rate": 2.3009863081457663e-06, "Finetune/Loss": 0.8246257305145264, "Finetune/Loss (Raw)": 1.0028619766235352, "Finetune/Step": 4087, "Finetune/Step Time": 4.824822066351771} +{"Finetune/Learning Rate": 2.2970122959449716e-06, "Finetune/Loss": 0.825552225112915, "Finetune/Loss (Raw)": 0.948076605796814, "Finetune/Step": 4088, "Finetune/Step Time": 4.820316327735782} +{"Finetune/Learning Rate": 2.2930412731233174e-06, "Finetune/Loss": 0.8250682353973389, "Finetune/Loss (Raw)": 0.8435835242271423, "Finetune/Step": 4089, "Finetune/Step Time": 4.821803251281381} +{"Finetune/Learning Rate": 2.2890732412218795e-06, "Finetune/Loss": 0.825525164604187, "Finetune/Loss (Raw)": 0.8332124948501587, "Finetune/Step": 4090, "Finetune/Step Time": 4.808716986328363} +{"Finetune/Learning Rate": 2.2851082017805704e-06, "Finetune/Loss": 0.8248387575149536, "Finetune/Loss (Raw)": 0.8372809290885925, "Finetune/Step": 4091, "Finetune/Step Time": 4.822077952325344} +{"Finetune/Learning Rate": 2.2811461563381497e-06, "Finetune/Loss": 0.8245104551315308, "Finetune/Loss (Raw)": 0.8347970843315125, "Finetune/Step": 4092, "Finetune/Step Time": 4.823635803535581} +{"Finetune/Learning Rate": 2.2771871064322117e-06, "Finetune/Loss": 0.824505090713501, "Finetune/Loss (Raw)": 0.9884028434753418, "Finetune/Step": 4093, "Finetune/Step Time": 4.807944618165493} +{"Finetune/Learning Rate": 2.2732310535991775e-06, "Finetune/Loss": 0.8297871351242065, "Finetune/Loss (Raw)": 1.0634187459945679, "Finetune/Step": 4094, "Finetune/Step Time": 4.765788214281201} +{"Finetune/Learning Rate": 2.2692779993743207e-06, "Finetune/Loss": 0.829437792301178, "Finetune/Loss (Raw)": 0.782150149345398, "Finetune/Step": 4095, "Finetune/Step Time": 4.773197676986456} +{"Finetune/Learning Rate": 2.265327945291742e-06, "Finetune/Loss": 0.830147922039032, "Finetune/Loss (Raw)": 1.0279678106307983, "Finetune/Step": 4096, "Finetune/Step Time": 4.777444103732705} +{"Finetune/Learning Rate": 2.2613808928843827e-06, "Finetune/Loss": 0.8305169343948364, "Finetune/Loss (Raw)": 0.9642750024795532, "Finetune/Step": 4097, "Finetune/Step Time": 4.763404298573732} +{"Finetune/Learning Rate": 2.2574368436840144e-06, "Finetune/Loss": 0.8285947442054749, "Finetune/Loss (Raw)": 0.6947886943817139, "Finetune/Step": 4098, "Finetune/Step Time": 4.771966114640236} +{"Finetune/Learning Rate": 2.2534957992212457e-06, "Finetune/Loss": 0.8282238841056824, "Finetune/Loss (Raw)": 0.9241496324539185, "Finetune/Step": 4099, "Finetune/Step Time": 4.775250148028135} +{"Finetune/Learning Rate": 2.2495577610255203e-06, "Finetune/Loss": 0.829465389251709, "Finetune/Loss (Raw)": 0.9587541818618774, "Finetune/Step": 4100, "Finetune/Step Time": 4.780969699844718} +{"Finetune/Learning Rate": 2.24562273062511e-06, "Finetune/Loss": 0.8298579454421997, "Finetune/Loss (Raw)": 0.8891915678977966, "Finetune/Step": 4101, "Finetune/Step Time": 4.807349795475602} +{"Finetune/Learning Rate": 2.2416907095471296e-06, "Finetune/Loss": 0.8301419019699097, "Finetune/Loss (Raw)": 0.8503528833389282, "Finetune/Step": 4102, "Finetune/Step Time": 4.803651450201869} +{"Finetune/Learning Rate": 2.237761699317519e-06, "Finetune/Loss": 0.8297061920166016, "Finetune/Loss (Raw)": 0.8924717307090759, "Finetune/Step": 4103, "Finetune/Step Time": 4.800651332363486} +{"Finetune/Learning Rate": 2.2338357014610425e-06, "Finetune/Loss": 0.8296996355056763, "Finetune/Loss (Raw)": 0.8663704991340637, "Finetune/Step": 4104, "Finetune/Step Time": 4.782420497387648} +{"Finetune/Learning Rate": 2.2299127175013113e-06, "Finetune/Loss": 0.8298481702804565, "Finetune/Loss (Raw)": 0.7853683233261108, "Finetune/Step": 4105, "Finetune/Step Time": 4.776601994410157} +{"Finetune/Learning Rate": 2.2259927489607568e-06, "Finetune/Loss": 0.8312782049179077, "Finetune/Loss (Raw)": 0.8272719979286194, "Finetune/Step": 4106, "Finetune/Step Time": 4.77499339915812} +{"Finetune/Learning Rate": 2.222075797360641e-06, "Finetune/Loss": 0.8309241533279419, "Finetune/Loss (Raw)": 0.8424921631813049, "Finetune/Step": 4107, "Finetune/Step Time": 4.777000838890672} +{"Finetune/Learning Rate": 2.2181618642210577e-06, "Finetune/Loss": 0.8332128524780273, "Finetune/Loss (Raw)": 0.9520359039306641, "Finetune/Step": 4108, "Finetune/Step Time": 4.776184804737568} +{"Finetune/Learning Rate": 2.2142509510609277e-06, "Finetune/Loss": 0.832181453704834, "Finetune/Loss (Raw)": 0.819366991519928, "Finetune/Step": 4109, "Finetune/Step Time": 4.7842296082526445} +{"Finetune/Learning Rate": 2.210343059397999e-06, "Finetune/Loss": 0.8327775597572327, "Finetune/Loss (Raw)": 0.9133783578872681, "Finetune/Step": 4110, "Finetune/Step Time": 4.7956839725375175} +{"Finetune/Learning Rate": 2.206438190748846e-06, "Finetune/Loss": 0.8340128660202026, "Finetune/Loss (Raw)": 0.9968463778495789, "Finetune/Step": 4111, "Finetune/Step Time": 4.801161605864763} +{"Finetune/Learning Rate": 2.2025363466288796e-06, "Finetune/Loss": 0.8373169302940369, "Finetune/Loss (Raw)": 0.7637928128242493, "Finetune/Step": 4112, "Finetune/Step Time": 4.776424765586853} +{"Finetune/Learning Rate": 2.1986375285523233e-06, "Finetune/Loss": 0.8357475996017456, "Finetune/Loss (Raw)": 0.7195731401443481, "Finetune/Step": 4113, "Finetune/Step Time": 4.774366918951273} +{"Finetune/Learning Rate": 2.194741738032229e-06, "Finetune/Loss": 0.8342679142951965, "Finetune/Loss (Raw)": 0.875856876373291, "Finetune/Step": 4114, "Finetune/Step Time": 4.776118718087673} +{"Finetune/Learning Rate": 2.1908489765804853e-06, "Finetune/Loss": 0.8349164724349976, "Finetune/Loss (Raw)": 0.8710954189300537, "Finetune/Step": 4115, "Finetune/Step Time": 4.772434299811721} +{"Finetune/Learning Rate": 2.1869592457077924e-06, "Finetune/Loss": 0.8324881196022034, "Finetune/Loss (Raw)": 0.6441644430160522, "Finetune/Step": 4116, "Finetune/Step Time": 4.787245547398925} +{"Finetune/Learning Rate": 2.183072546923679e-06, "Finetune/Loss": 0.8310176134109497, "Finetune/Loss (Raw)": 0.7762616872787476, "Finetune/Step": 4117, "Finetune/Step Time": 4.796136999502778} +{"Finetune/Learning Rate": 2.179188881736498e-06, "Finetune/Loss": 0.8320499062538147, "Finetune/Loss (Raw)": 1.0522288084030151, "Finetune/Step": 4118, "Finetune/Step Time": 4.794485237449408} +{"Finetune/Learning Rate": 2.1753082516534242e-06, "Finetune/Loss": 0.8321825265884399, "Finetune/Loss (Raw)": 0.9851582050323486, "Finetune/Step": 4119, "Finetune/Step Time": 4.789158707484603} +{"Finetune/Learning Rate": 2.1714306581804533e-06, "Finetune/Loss": 0.8332817554473877, "Finetune/Loss (Raw)": 0.8925008773803711, "Finetune/Step": 4120, "Finetune/Step Time": 4.779008245095611} +{"Finetune/Learning Rate": 2.1675561028224012e-06, "Finetune/Loss": 0.8330622911453247, "Finetune/Loss (Raw)": 0.8797886967658997, "Finetune/Step": 4121, "Finetune/Step Time": 4.784644769504666} +{"Finetune/Learning Rate": 2.163684587082915e-06, "Finetune/Loss": 0.8338715434074402, "Finetune/Loss (Raw)": 0.8782954812049866, "Finetune/Step": 4122, "Finetune/Step Time": 4.789390979334712} +{"Finetune/Learning Rate": 2.159816112464447e-06, "Finetune/Loss": 0.8354365825653076, "Finetune/Loss (Raw)": 1.0121623277664185, "Finetune/Step": 4123, "Finetune/Step Time": 4.790365988388658} +{"Finetune/Learning Rate": 2.1559506804682762e-06, "Finetune/Loss": 0.8342571258544922, "Finetune/Loss (Raw)": 0.6537501811981201, "Finetune/Step": 4124, "Finetune/Step Time": 4.792205495759845} +{"Finetune/Learning Rate": 2.1520882925945074e-06, "Finetune/Loss": 0.8348127603530884, "Finetune/Loss (Raw)": 0.9110668301582336, "Finetune/Step": 4125, "Finetune/Step Time": 4.795861225575209} +{"Finetune/Learning Rate": 2.1482289503420537e-06, "Finetune/Loss": 0.8337922096252441, "Finetune/Loss (Raw)": 0.8856912851333618, "Finetune/Step": 4126, "Finetune/Step Time": 4.790521619841456} +{"Finetune/Learning Rate": 2.144372655208653e-06, "Finetune/Loss": 0.8336465954780579, "Finetune/Loss (Raw)": 0.9048988223075867, "Finetune/Step": 4127, "Finetune/Step Time": 4.77038805373013} +{"Finetune/Learning Rate": 2.1405194086908555e-06, "Finetune/Loss": 0.8336264491081238, "Finetune/Loss (Raw)": 0.8248523473739624, "Finetune/Step": 4128, "Finetune/Step Time": 4.766065800562501} +{"Finetune/Learning Rate": 2.1366692122840317e-06, "Finetune/Loss": 0.8334484100341797, "Finetune/Loss (Raw)": 0.8190817832946777, "Finetune/Step": 4129, "Finetune/Step Time": 4.74665467441082} +{"Finetune/Learning Rate": 2.1328220674823686e-06, "Finetune/Loss": 0.8337095975875854, "Finetune/Loss (Raw)": 0.8390439748764038, "Finetune/Step": 4130, "Finetune/Step Time": 4.736989451572299} +{"Finetune/Learning Rate": 2.1289779757788655e-06, "Finetune/Loss": 0.8338354229927063, "Finetune/Loss (Raw)": 0.8862857222557068, "Finetune/Step": 4131, "Finetune/Step Time": 4.730029271915555} +{"Finetune/Learning Rate": 2.1251369386653454e-06, "Finetune/Loss": 0.8336344957351685, "Finetune/Loss (Raw)": 0.8991063237190247, "Finetune/Step": 4132, "Finetune/Step Time": 4.726799147203565} +{"Finetune/Learning Rate": 2.121298957632434e-06, "Finetune/Loss": 0.8333421945571899, "Finetune/Loss (Raw)": 0.8437122106552124, "Finetune/Step": 4133, "Finetune/Step Time": 4.734521523118019} +{"Finetune/Learning Rate": 2.117464034169575e-06, "Finetune/Loss": 0.8319097757339478, "Finetune/Loss (Raw)": 0.8642734289169312, "Finetune/Step": 4134, "Finetune/Step Time": 4.755883891135454} +{"Finetune/Learning Rate": 2.113632169765033e-06, "Finetune/Loss": 0.8352333307266235, "Finetune/Loss (Raw)": 0.7869054079055786, "Finetune/Step": 4135, "Finetune/Step Time": 4.718757774680853} +{"Finetune/Learning Rate": 2.1098033659058792e-06, "Finetune/Loss": 0.8357961177825928, "Finetune/Loss (Raw)": 0.847066342830658, "Finetune/Step": 4136, "Finetune/Step Time": 4.719127334654331} +{"Finetune/Learning Rate": 2.105977624077995e-06, "Finetune/Loss": 0.8354180455207825, "Finetune/Loss (Raw)": 0.790268063545227, "Finetune/Step": 4137, "Finetune/Step Time": 4.716429194435477} +{"Finetune/Learning Rate": 2.102154945766076e-06, "Finetune/Loss": 0.8366159200668335, "Finetune/Loss (Raw)": 0.9568042159080505, "Finetune/Step": 4138, "Finetune/Step Time": 4.702652622014284} +{"Finetune/Learning Rate": 2.0983353324536316e-06, "Finetune/Loss": 0.8350821733474731, "Finetune/Loss (Raw)": 0.78476881980896, "Finetune/Step": 4139, "Finetune/Step Time": 4.692643016576767} +{"Finetune/Learning Rate": 2.0945187856229778e-06, "Finetune/Loss": 0.8361425399780273, "Finetune/Loss (Raw)": 0.8150428533554077, "Finetune/Step": 4140, "Finetune/Step Time": 4.6819605100899935} +{"Finetune/Learning Rate": 2.0907053067552384e-06, "Finetune/Loss": 0.8323880434036255, "Finetune/Loss (Raw)": 0.38520607352256775, "Finetune/Step": 4141, "Finetune/Step Time": 4.70986745133996} +{"Finetune/Learning Rate": 2.0868948973303593e-06, "Finetune/Loss": 0.8320333957672119, "Finetune/Loss (Raw)": 0.7770752310752869, "Finetune/Step": 4142, "Finetune/Step Time": 4.686666086316109} +{"Finetune/Learning Rate": 2.0830875588270773e-06, "Finetune/Loss": 0.8315181732177734, "Finetune/Loss (Raw)": 0.7876028418540955, "Finetune/Step": 4143, "Finetune/Step Time": 4.68381436355412} +{"Finetune/Learning Rate": 2.0792832927229465e-06, "Finetune/Loss": 0.8268076777458191, "Finetune/Loss (Raw)": 0.29129573702812195, "Finetune/Step": 4144, "Finetune/Step Time": 4.717253370210528} +{"Finetune/Learning Rate": 2.0754821004943337e-06, "Finetune/Loss": 0.8254148960113525, "Finetune/Loss (Raw)": 0.7374311685562134, "Finetune/Step": 4145, "Finetune/Step Time": 4.7257595006376505} +{"Finetune/Learning Rate": 2.071683983616406e-06, "Finetune/Loss": 0.8299981355667114, "Finetune/Loss (Raw)": 0.9126935601234436, "Finetune/Step": 4146, "Finetune/Step Time": 4.682567210868001} +{"Finetune/Learning Rate": 2.0678889435631333e-06, "Finetune/Loss": 0.8287907838821411, "Finetune/Loss (Raw)": 0.8481242060661316, "Finetune/Step": 4147, "Finetune/Step Time": 4.688533041626215} +{"Finetune/Learning Rate": 2.064096981807302e-06, "Finetune/Loss": 0.8283541202545166, "Finetune/Loss (Raw)": 0.8961221575737, "Finetune/Step": 4148, "Finetune/Step Time": 4.688944034278393} +{"Finetune/Learning Rate": 2.060308099820496e-06, "Finetune/Loss": 0.8279808759689331, "Finetune/Loss (Raw)": 0.8415760397911072, "Finetune/Step": 4149, "Finetune/Step Time": 4.6778753735125065} +{"Finetune/Learning Rate": 2.056522299073108e-06, "Finetune/Loss": 0.8277826309204102, "Finetune/Loss (Raw)": 0.781207263469696, "Finetune/Step": 4150, "Finetune/Step Time": 4.70281789265573} +{"Finetune/Learning Rate": 2.052739581034332e-06, "Finetune/Loss": 0.8273993730545044, "Finetune/Loss (Raw)": 0.8641060590744019, "Finetune/Step": 4151, "Finetune/Step Time": 4.707546954974532} +{"Finetune/Learning Rate": 2.0489599471721678e-06, "Finetune/Loss": 0.8265860080718994, "Finetune/Loss (Raw)": 0.8637560606002808, "Finetune/Step": 4152, "Finetune/Step Time": 4.712209559977055} +{"Finetune/Learning Rate": 2.0451833989534165e-06, "Finetune/Loss": 0.8263365030288696, "Finetune/Loss (Raw)": 0.8359950184822083, "Finetune/Step": 4153, "Finetune/Step Time": 4.709120128303766} +{"Finetune/Learning Rate": 2.0414099378436812e-06, "Finetune/Loss": 0.8256044387817383, "Finetune/Loss (Raw)": 0.8790198564529419, "Finetune/Step": 4154, "Finetune/Step Time": 4.713196678087115} +{"Finetune/Learning Rate": 2.0376395653073735e-06, "Finetune/Loss": 0.8241280913352966, "Finetune/Loss (Raw)": 0.8170418739318848, "Finetune/Step": 4155, "Finetune/Step Time": 4.715368734672666} +{"Finetune/Learning Rate": 2.0338722828077007e-06, "Finetune/Loss": 0.8246515989303589, "Finetune/Loss (Raw)": 0.8830731511116028, "Finetune/Step": 4156, "Finetune/Step Time": 4.714459968730807} +{"Finetune/Learning Rate": 2.030108091806664e-06, "Finetune/Loss": 0.8252381086349487, "Finetune/Loss (Raw)": 0.9808836579322815, "Finetune/Step": 4157, "Finetune/Step Time": 4.71479051373899} +{"Finetune/Learning Rate": 2.0263469937650813e-06, "Finetune/Loss": 0.8286387324333191, "Finetune/Loss (Raw)": 0.8305268883705139, "Finetune/Step": 4158, "Finetune/Step Time": 4.687425348907709} +{"Finetune/Learning Rate": 2.0225889901425565e-06, "Finetune/Loss": 0.8290612697601318, "Finetune/Loss (Raw)": 0.9325599074363708, "Finetune/Step": 4159, "Finetune/Step Time": 4.681842058897018} +{"Finetune/Learning Rate": 2.0188340823975005e-06, "Finetune/Loss": 0.8298519849777222, "Finetune/Loss (Raw)": 0.9840009808540344, "Finetune/Step": 4160, "Finetune/Step Time": 4.6746355798095465} +{"Finetune/Learning Rate": 2.0150822719871167e-06, "Finetune/Loss": 0.8297152519226074, "Finetune/Loss (Raw)": 0.9150727391242981, "Finetune/Step": 4161, "Finetune/Step Time": 4.671952510252595} +{"Finetune/Learning Rate": 2.011333560367411e-06, "Finetune/Loss": 0.8304398059844971, "Finetune/Loss (Raw)": 0.9512199759483337, "Finetune/Step": 4162, "Finetune/Step Time": 4.668283024802804} +{"Finetune/Learning Rate": 2.007587948993185e-06, "Finetune/Loss": 0.8295103311538696, "Finetune/Loss (Raw)": 0.7212976813316345, "Finetune/Step": 4163, "Finetune/Step Time": 4.678777297958732} +{"Finetune/Learning Rate": 2.003845439318035e-06, "Finetune/Loss": 0.8260567784309387, "Finetune/Loss (Raw)": 0.5990879535675049, "Finetune/Step": 4164, "Finetune/Step Time": 4.691411644220352} +{"Finetune/Learning Rate": 2.000106032794361e-06, "Finetune/Loss": 0.8277955055236816, "Finetune/Loss (Raw)": 1.0148789882659912, "Finetune/Step": 4165, "Finetune/Step Time": 4.683734899386764} +{"Finetune/Learning Rate": 1.996369730873353e-06, "Finetune/Loss": 0.8324186205863953, "Finetune/Loss (Raw)": 0.947471022605896, "Finetune/Step": 4166, "Finetune/Step Time": 4.651305487379432} +{"Finetune/Learning Rate": 1.99263653500499e-06, "Finetune/Loss": 0.8312239646911621, "Finetune/Loss (Raw)": 0.7879975438117981, "Finetune/Step": 4167, "Finetune/Step Time": 4.641765175387263} +{"Finetune/Learning Rate": 1.9889064466380593e-06, "Finetune/Loss": 0.8294897675514221, "Finetune/Loss (Raw)": 0.7040175795555115, "Finetune/Step": 4168, "Finetune/Step Time": 4.640259867534041} +{"Finetune/Learning Rate": 1.985179467220133e-06, "Finetune/Loss": 0.8260496854782104, "Finetune/Loss (Raw)": 0.37650802731513977, "Finetune/Step": 4169, "Finetune/Step Time": 4.677902523428202} +{"Finetune/Learning Rate": 1.9814555981975803e-06, "Finetune/Loss": 0.8253219127655029, "Finetune/Loss (Raw)": 0.7741715908050537, "Finetune/Step": 4170, "Finetune/Step Time": 4.6694351229816675} +{"Finetune/Learning Rate": 1.9777348410155606e-06, "Finetune/Loss": 0.825312614440918, "Finetune/Loss (Raw)": 0.9074480533599854, "Finetune/Step": 4171, "Finetune/Step Time": 4.671922460198402} +{"Finetune/Learning Rate": 1.9740171971180278e-06, "Finetune/Loss": 0.8254839181900024, "Finetune/Loss (Raw)": 0.9075068831443787, "Finetune/Step": 4172, "Finetune/Step Time": 4.686868485063314} +{"Finetune/Learning Rate": 1.9703026679477253e-06, "Finetune/Loss": 0.8297016620635986, "Finetune/Loss (Raw)": 0.8319714665412903, "Finetune/Step": 4173, "Finetune/Step Time": 4.655282618477941} +{"Finetune/Learning Rate": 1.9665912549461898e-06, "Finetune/Loss": 0.8261281251907349, "Finetune/Loss (Raw)": 0.35707175731658936, "Finetune/Step": 4174, "Finetune/Step Time": 4.693685844540596} +{"Finetune/Learning Rate": 1.962882959553751e-06, "Finetune/Loss": 0.8259402513504028, "Finetune/Loss (Raw)": 0.7787261009216309, "Finetune/Step": 4175, "Finetune/Step Time": 4.683024514466524} +{"Finetune/Learning Rate": 1.959177783209527e-06, "Finetune/Loss": 0.8293080925941467, "Finetune/Loss (Raw)": 0.8047285079956055, "Finetune/Step": 4176, "Finetune/Step Time": 4.6584417801350355} +{"Finetune/Learning Rate": 1.955475727351417e-06, "Finetune/Loss": 0.8309419751167297, "Finetune/Loss (Raw)": 0.9676960706710815, "Finetune/Step": 4177, "Finetune/Step Time": 4.663198070600629} +{"Finetune/Learning Rate": 1.9517767934161224e-06, "Finetune/Loss": 0.8315415382385254, "Finetune/Loss (Raw)": 0.9151132702827454, "Finetune/Step": 4178, "Finetune/Step Time": 4.671544594690204} +{"Finetune/Learning Rate": 1.9480809828391266e-06, "Finetune/Loss": 0.8322702646255493, "Finetune/Loss (Raw)": 0.8852545022964478, "Finetune/Step": 4179, "Finetune/Step Time": 4.684305848553777} +{"Finetune/Learning Rate": 1.9443882970547003e-06, "Finetune/Loss": 0.8332085609436035, "Finetune/Loss (Raw)": 0.966425359249115, "Finetune/Step": 4180, "Finetune/Step Time": 4.6643270496279} +{"Finetune/Learning Rate": 1.940698737495904e-06, "Finetune/Loss": 0.8367002010345459, "Finetune/Loss (Raw)": 0.8193625807762146, "Finetune/Step": 4181, "Finetune/Step Time": 4.630918158218265} +{"Finetune/Learning Rate": 1.937012305594583e-06, "Finetune/Loss": 0.8353301286697388, "Finetune/Loss (Raw)": 0.6835787296295166, "Finetune/Step": 4182, "Finetune/Step Time": 4.630952792242169} +{"Finetune/Learning Rate": 1.93332900278137e-06, "Finetune/Loss": 0.8356121778488159, "Finetune/Loss (Raw)": 0.9425480961799622, "Finetune/Step": 4183, "Finetune/Step Time": 4.635956859216094} +{"Finetune/Learning Rate": 1.9296488304856785e-06, "Finetune/Loss": 0.8376257419586182, "Finetune/Loss (Raw)": 0.9018547534942627, "Finetune/Step": 4184, "Finetune/Step Time": 4.619435757398605} +{"Finetune/Learning Rate": 1.92597179013572e-06, "Finetune/Loss": 0.8375424146652222, "Finetune/Loss (Raw)": 0.9653239250183105, "Finetune/Step": 4185, "Finetune/Step Time": 4.616715362295508} +{"Finetune/Learning Rate": 1.9222978831584793e-06, "Finetune/Loss": 0.8384529948234558, "Finetune/Loss (Raw)": 0.9429280757904053, "Finetune/Step": 4186, "Finetune/Step Time": 4.629825757816434} +{"Finetune/Learning Rate": 1.9186271109797215e-06, "Finetune/Loss": 0.8377920389175415, "Finetune/Loss (Raw)": 0.8038283586502075, "Finetune/Step": 4187, "Finetune/Step Time": 4.617789998650551} +{"Finetune/Learning Rate": 1.9149594750240087e-06, "Finetune/Loss": 0.8389431834220886, "Finetune/Loss (Raw)": 0.8000120520591736, "Finetune/Step": 4188, "Finetune/Step Time": 4.6083546709269285} +{"Finetune/Learning Rate": 1.911294976714676e-06, "Finetune/Loss": 0.8389415740966797, "Finetune/Loss (Raw)": 0.8063745498657227, "Finetune/Step": 4189, "Finetune/Step Time": 4.590962678194046} +{"Finetune/Learning Rate": 1.9076336174738476e-06, "Finetune/Loss": 0.8364180326461792, "Finetune/Loss (Raw)": 0.6896556615829468, "Finetune/Step": 4190, "Finetune/Step Time": 4.591341000050306} +{"Finetune/Learning Rate": 1.9039753987224164e-06, "Finetune/Loss": 0.840876579284668, "Finetune/Loss (Raw)": 0.9225308895111084, "Finetune/Step": 4191, "Finetune/Step Time": 4.557986943051219} +{"Finetune/Learning Rate": 1.9003203218800748e-06, "Finetune/Loss": 0.8458360433578491, "Finetune/Loss (Raw)": 1.0157365798950195, "Finetune/Step": 4192, "Finetune/Step Time": 4.517849942669272} +{"Finetune/Learning Rate": 1.8966683883652826e-06, "Finetune/Loss": 0.8420737981796265, "Finetune/Loss (Raw)": 0.3051627278327942, "Finetune/Step": 4193, "Finetune/Step Time": 4.545643476769328} +{"Finetune/Learning Rate": 1.8930195995952827e-06, "Finetune/Loss": 0.8422813415527344, "Finetune/Loss (Raw)": 0.8985920548439026, "Finetune/Step": 4194, "Finetune/Step Time": 4.545287741348147} +{"Finetune/Learning Rate": 1.8893739569861058e-06, "Finetune/Loss": 0.8416550755500793, "Finetune/Loss (Raw)": 0.9025004506111145, "Finetune/Step": 4195, "Finetune/Step Time": 4.540673732757568} +{"Finetune/Learning Rate": 1.8857314619525469e-06, "Finetune/Loss": 0.8414902687072754, "Finetune/Loss (Raw)": 0.8997794985771179, "Finetune/Step": 4196, "Finetune/Step Time": 4.538353430107236} +{"Finetune/Learning Rate": 1.8820921159081896e-06, "Finetune/Loss": 0.8388398885726929, "Finetune/Loss (Raw)": 0.5832211971282959, "Finetune/Step": 4197, "Finetune/Step Time": 4.547584434971213} +{"Finetune/Learning Rate": 1.8784559202653907e-06, "Finetune/Loss": 0.8399372696876526, "Finetune/Loss (Raw)": 0.9618438482284546, "Finetune/Step": 4198, "Finetune/Step Time": 4.5459393709897995} +{"Finetune/Learning Rate": 1.8748228764352915e-06, "Finetune/Loss": 0.8407362103462219, "Finetune/Loss (Raw)": 0.9796207547187805, "Finetune/Step": 4199, "Finetune/Step Time": 4.5150963217020035} +{"Finetune/Learning Rate": 1.8711929858278066e-06, "Finetune/Loss": 0.8416856527328491, "Finetune/Loss (Raw)": 0.8993403315544128, "Finetune/Step": 4200, "Finetune/Step Time": 4.511836130172014} +{"Finetune/Learning Rate": 1.8675662498516167e-06, "Finetune/Loss": 0.8413106203079224, "Finetune/Loss (Raw)": 0.8493274450302124, "Finetune/Step": 4201, "Finetune/Step Time": 4.527358140796423} +{"Finetune/Learning Rate": 1.863942669914196e-06, "Finetune/Loss": 0.8399864435195923, "Finetune/Loss (Raw)": 0.8201084733009338, "Finetune/Step": 4202, "Finetune/Step Time": 4.530007241293788} +{"Finetune/Learning Rate": 1.8603222474217819e-06, "Finetune/Loss": 0.8396416902542114, "Finetune/Loss (Raw)": 0.7821763753890991, "Finetune/Step": 4203, "Finetune/Step Time": 4.531304839998484} +{"Finetune/Learning Rate": 1.8567049837793894e-06, "Finetune/Loss": 0.8406044840812683, "Finetune/Loss (Raw)": 0.8017719388008118, "Finetune/Step": 4204, "Finetune/Step Time": 4.511651698499918} +{"Finetune/Learning Rate": 1.8530908803908088e-06, "Finetune/Loss": 0.8408578634262085, "Finetune/Loss (Raw)": 0.8950712084770203, "Finetune/Step": 4205, "Finetune/Step Time": 4.493174452334642} +{"Finetune/Learning Rate": 1.8494799386586015e-06, "Finetune/Loss": 0.8410553336143494, "Finetune/Loss (Raw)": 0.8371559381484985, "Finetune/Step": 4206, "Finetune/Step Time": 4.491281278431416} +{"Finetune/Learning Rate": 1.8458721599841056e-06, "Finetune/Loss": 0.8400636911392212, "Finetune/Loss (Raw)": 0.9216697216033936, "Finetune/Step": 4207, "Finetune/Step Time": 4.485017456114292} +{"Finetune/Learning Rate": 1.8422675457674254e-06, "Finetune/Loss": 0.8404394388198853, "Finetune/Loss (Raw)": 0.8283185362815857, "Finetune/Step": 4208, "Finetune/Step Time": 4.474388157948852} +{"Finetune/Learning Rate": 1.8386660974074466e-06, "Finetune/Loss": 0.841934084892273, "Finetune/Loss (Raw)": 1.0590496063232422, "Finetune/Step": 4209, "Finetune/Step Time": 4.452345971018076} +{"Finetune/Learning Rate": 1.8350678163018221e-06, "Finetune/Loss": 0.8421180248260498, "Finetune/Loss (Raw)": 0.8416439294815063, "Finetune/Step": 4210, "Finetune/Step Time": 4.44749541580677} +{"Finetune/Learning Rate": 1.831472703846966e-06, "Finetune/Loss": 0.8417599201202393, "Finetune/Loss (Raw)": 0.9355660080909729, "Finetune/Step": 4211, "Finetune/Step Time": 4.452659765258431} +{"Finetune/Learning Rate": 1.827880761438079e-06, "Finetune/Loss": 0.8442342281341553, "Finetune/Loss (Raw)": 0.7829573750495911, "Finetune/Step": 4212, "Finetune/Step Time": 4.414658635854721} +{"Finetune/Learning Rate": 1.8242919904691214e-06, "Finetune/Loss": 0.8431874513626099, "Finetune/Loss (Raw)": 0.7175115942955017, "Finetune/Step": 4213, "Finetune/Step Time": 4.41323009878397} +{"Finetune/Learning Rate": 1.820706392332824e-06, "Finetune/Loss": 0.8432404398918152, "Finetune/Loss (Raw)": 0.8933277726173401, "Finetune/Step": 4214, "Finetune/Step Time": 4.403083425015211} +{"Finetune/Learning Rate": 1.8171239684206898e-06, "Finetune/Loss": 0.8422187566757202, "Finetune/Loss (Raw)": 0.8720810413360596, "Finetune/Step": 4215, "Finetune/Step Time": 4.429340921342373} +{"Finetune/Learning Rate": 1.8135447201229861e-06, "Finetune/Loss": 0.8415975570678711, "Finetune/Loss (Raw)": 0.8685660362243652, "Finetune/Step": 4216, "Finetune/Step Time": 4.424089716747403} +{"Finetune/Learning Rate": 1.8099686488287483e-06, "Finetune/Loss": 0.8423450589179993, "Finetune/Loss (Raw)": 0.9392677545547485, "Finetune/Step": 4217, "Finetune/Step Time": 4.42080937884748} +{"Finetune/Learning Rate": 1.8063957559257794e-06, "Finetune/Loss": 0.8424825668334961, "Finetune/Loss (Raw)": 0.8508103489875793, "Finetune/Step": 4218, "Finetune/Step Time": 4.42007446102798} +{"Finetune/Learning Rate": 1.8028260428006538e-06, "Finetune/Loss": 0.8418260812759399, "Finetune/Loss (Raw)": 0.7532538175582886, "Finetune/Step": 4219, "Finetune/Step Time": 4.402734162285924} +{"Finetune/Learning Rate": 1.7992595108387068e-06, "Finetune/Loss": 0.8421874642372131, "Finetune/Loss (Raw)": 0.8810524940490723, "Finetune/Step": 4220, "Finetune/Step Time": 4.392177851870656} +{"Finetune/Learning Rate": 1.795696161424032e-06, "Finetune/Loss": 0.8423035144805908, "Finetune/Loss (Raw)": 1.0032541751861572, "Finetune/Step": 4221, "Finetune/Step Time": 4.393786007538438} +{"Finetune/Learning Rate": 1.7921359959395046e-06, "Finetune/Loss": 0.8415400981903076, "Finetune/Loss (Raw)": 0.9656991362571716, "Finetune/Step": 4222, "Finetune/Step Time": 4.397860689088702} +{"Finetune/Learning Rate": 1.7885790157667526e-06, "Finetune/Loss": 0.8430111408233643, "Finetune/Loss (Raw)": 0.9704518914222717, "Finetune/Step": 4223, "Finetune/Step Time": 4.38959788903594} +{"Finetune/Learning Rate": 1.7850252222861697e-06, "Finetune/Loss": 0.8413004279136658, "Finetune/Loss (Raw)": 0.8089941143989563, "Finetune/Step": 4224, "Finetune/Step Time": 4.395002841949463} +{"Finetune/Learning Rate": 1.7814746168769138e-06, "Finetune/Loss": 0.8399868607521057, "Finetune/Loss (Raw)": 0.796136736869812, "Finetune/Step": 4225, "Finetune/Step Time": 4.382498310878873} +{"Finetune/Learning Rate": 1.7779272009169068e-06, "Finetune/Loss": 0.8422465920448303, "Finetune/Loss (Raw)": 0.9840340614318848, "Finetune/Step": 4226, "Finetune/Step Time": 4.375584846362472} +{"Finetune/Learning Rate": 1.77438297578283e-06, "Finetune/Loss": 0.8433636426925659, "Finetune/Loss (Raw)": 1.0671321153640747, "Finetune/Step": 4227, "Finetune/Step Time": 4.375355085358024} +{"Finetune/Learning Rate": 1.7708419428501256e-06, "Finetune/Loss": 0.8425601720809937, "Finetune/Loss (Raw)": 0.8559046983718872, "Finetune/Step": 4228, "Finetune/Step Time": 4.377422230318189} +{"Finetune/Learning Rate": 1.7673041034930083e-06, "Finetune/Loss": 0.8426557779312134, "Finetune/Loss (Raw)": 0.9014328122138977, "Finetune/Step": 4229, "Finetune/Step Time": 4.354904806241393} +{"Finetune/Learning Rate": 1.7637694590844357e-06, "Finetune/Loss": 0.838117778301239, "Finetune/Loss (Raw)": 0.2694917917251587, "Finetune/Step": 4230, "Finetune/Step Time": 4.38711323030293} +{"Finetune/Learning Rate": 1.7602380109961348e-06, "Finetune/Loss": 0.8362820148468018, "Finetune/Loss (Raw)": 0.6574983596801758, "Finetune/Step": 4231, "Finetune/Step Time": 4.432869864627719} +{"Finetune/Learning Rate": 1.7567097605985972e-06, "Finetune/Loss": 0.8365466594696045, "Finetune/Loss (Raw)": 0.9002373814582825, "Finetune/Step": 4232, "Finetune/Step Time": 4.441633397713304} +{"Finetune/Learning Rate": 1.7531847092610644e-06, "Finetune/Loss": 0.8365982174873352, "Finetune/Loss (Raw)": 0.7919692397117615, "Finetune/Step": 4233, "Finetune/Step Time": 4.447707204148173} +{"Finetune/Learning Rate": 1.7496628583515395e-06, "Finetune/Loss": 0.8387026786804199, "Finetune/Loss (Raw)": 1.0966389179229736, "Finetune/Step": 4234, "Finetune/Step Time": 4.4553159065544605} +{"Finetune/Learning Rate": 1.7461442092367864e-06, "Finetune/Loss": 0.839127779006958, "Finetune/Loss (Raw)": 0.8969165086746216, "Finetune/Step": 4235, "Finetune/Step Time": 4.455151712521911} +{"Finetune/Learning Rate": 1.7426287632823213e-06, "Finetune/Loss": 0.8373714089393616, "Finetune/Loss (Raw)": 0.7272166609764099, "Finetune/Step": 4236, "Finetune/Step Time": 4.463825274258852} +{"Finetune/Learning Rate": 1.7391165218524232e-06, "Finetune/Loss": 0.8387981653213501, "Finetune/Loss (Raw)": 1.0019901990890503, "Finetune/Step": 4237, "Finetune/Step Time": 4.461954591795802} +{"Finetune/Learning Rate": 1.735607486310119e-06, "Finetune/Loss": 0.8391519784927368, "Finetune/Loss (Raw)": 0.9586676359176636, "Finetune/Step": 4238, "Finetune/Step Time": 4.440091954544187} +{"Finetune/Learning Rate": 1.7321016580172068e-06, "Finetune/Loss": 0.8384817242622375, "Finetune/Loss (Raw)": 0.9110565185546875, "Finetune/Step": 4239, "Finetune/Step Time": 4.441570231691003} +{"Finetune/Learning Rate": 1.728599038334221e-06, "Finetune/Loss": 0.8390328884124756, "Finetune/Loss (Raw)": 0.8343431353569031, "Finetune/Step": 4240, "Finetune/Step Time": 4.4296266566962} +{"Finetune/Learning Rate": 1.7250996286204602e-06, "Finetune/Loss": 0.840052604675293, "Finetune/Loss (Raw)": 0.8500909209251404, "Finetune/Step": 4241, "Finetune/Step Time": 4.431465897709131} +{"Finetune/Learning Rate": 1.7216034302339824e-06, "Finetune/Loss": 0.8405743837356567, "Finetune/Loss (Raw)": 0.9426413774490356, "Finetune/Step": 4242, "Finetune/Step Time": 4.430081045255065} +{"Finetune/Learning Rate": 1.718110444531591e-06, "Finetune/Loss": 0.8406955003738403, "Finetune/Loss (Raw)": 0.8865987062454224, "Finetune/Step": 4243, "Finetune/Step Time": 4.450074136257172} +{"Finetune/Learning Rate": 1.7146206728688465e-06, "Finetune/Loss": 0.8425993919372559, "Finetune/Loss (Raw)": 0.8878624439239502, "Finetune/Step": 4244, "Finetune/Step Time": 4.43374290317297} +{"Finetune/Learning Rate": 1.7111341166000594e-06, "Finetune/Loss": 0.8434950709342957, "Finetune/Loss (Raw)": 0.8909139037132263, "Finetune/Step": 4245, "Finetune/Step Time": 4.441184125840664} +{"Finetune/Learning Rate": 1.7076507770782957e-06, "Finetune/Loss": 0.8420038223266602, "Finetune/Loss (Raw)": 0.8613497018814087, "Finetune/Step": 4246, "Finetune/Step Time": 4.472395459190011} +{"Finetune/Learning Rate": 1.7041706556553717e-06, "Finetune/Loss": 0.8411104679107666, "Finetune/Loss (Raw)": 0.8708029389381409, "Finetune/Step": 4247, "Finetune/Step Time": 4.479508752003312} +{"Finetune/Learning Rate": 1.7006937536818503e-06, "Finetune/Loss": 0.8364073038101196, "Finetune/Loss (Raw)": 0.29049891233444214, "Finetune/Step": 4248, "Finetune/Step Time": 4.504500672221184} +{"Finetune/Learning Rate": 1.6972200725070587e-06, "Finetune/Loss": 0.836780309677124, "Finetune/Loss (Raw)": 0.9275365471839905, "Finetune/Step": 4249, "Finetune/Step Time": 4.497853448614478} +{"Finetune/Learning Rate": 1.6937496134790554e-06, "Finetune/Loss": 0.8356066942214966, "Finetune/Loss (Raw)": 0.7280672192573547, "Finetune/Step": 4250, "Finetune/Step Time": 4.493563404306769} +{"Finetune/Learning Rate": 1.6902823779446586e-06, "Finetune/Loss": 0.8335214853286743, "Finetune/Loss (Raw)": 0.7452504634857178, "Finetune/Step": 4251, "Finetune/Step Time": 4.4932244066149} +{"Finetune/Learning Rate": 1.686818367249441e-06, "Finetune/Loss": 0.83655846118927, "Finetune/Loss (Raw)": 1.0424903631210327, "Finetune/Step": 4252, "Finetune/Step Time": 4.494963455945253} +{"Finetune/Learning Rate": 1.6833575827377135e-06, "Finetune/Loss": 0.834537148475647, "Finetune/Loss (Raw)": 0.6523396968841553, "Finetune/Step": 4253, "Finetune/Step Time": 4.5156314224004745} +{"Finetune/Learning Rate": 1.6799000257525388e-06, "Finetune/Loss": 0.8328808546066284, "Finetune/Loss (Raw)": 0.6736801266670227, "Finetune/Step": 4254, "Finetune/Step Time": 4.525914998725057} +{"Finetune/Learning Rate": 1.6764456976357279e-06, "Finetune/Loss": 0.8319098949432373, "Finetune/Loss (Raw)": 0.7806176543235779, "Finetune/Step": 4255, "Finetune/Step Time": 4.532123884186149} +{"Finetune/Learning Rate": 1.6729945997278386e-06, "Finetune/Loss": 0.8326061964035034, "Finetune/Loss (Raw)": 0.9139856696128845, "Finetune/Step": 4256, "Finetune/Step Time": 4.547977199777961} +{"Finetune/Learning Rate": 1.6695467333681748e-06, "Finetune/Loss": 0.8338083028793335, "Finetune/Loss (Raw)": 0.9729500412940979, "Finetune/Step": 4257, "Finetune/Step Time": 4.55230132304132} +{"Finetune/Learning Rate": 1.6661020998947818e-06, "Finetune/Loss": 0.8326208591461182, "Finetune/Loss (Raw)": 0.68704754114151, "Finetune/Step": 4258, "Finetune/Step Time": 4.553342653438449} +{"Finetune/Learning Rate": 1.6626607006444628e-06, "Finetune/Loss": 0.8336174488067627, "Finetune/Loss (Raw)": 1.0138448476791382, "Finetune/Step": 4259, "Finetune/Step Time": 4.557556971907616} +{"Finetune/Learning Rate": 1.6592225369527515e-06, "Finetune/Loss": 0.8331775069236755, "Finetune/Loss (Raw)": 0.8427993655204773, "Finetune/Step": 4260, "Finetune/Step Time": 4.554000575095415} +{"Finetune/Learning Rate": 1.6557876101539305e-06, "Finetune/Loss": 0.8338794708251953, "Finetune/Loss (Raw)": 0.9335587024688721, "Finetune/Step": 4261, "Finetune/Step Time": 4.547793721780181} +{"Finetune/Learning Rate": 1.6523559215810336e-06, "Finetune/Loss": 0.8337320685386658, "Finetune/Loss (Raw)": 0.845410943031311, "Finetune/Step": 4262, "Finetune/Step Time": 4.538751496002078} +{"Finetune/Learning Rate": 1.64892747256583e-06, "Finetune/Loss": 0.8356767296791077, "Finetune/Loss (Raw)": 1.0358237028121948, "Finetune/Step": 4263, "Finetune/Step Time": 4.536641595885158} +{"Finetune/Learning Rate": 1.645502264438833e-06, "Finetune/Loss": 0.8351666927337646, "Finetune/Loss (Raw)": 0.7817786931991577, "Finetune/Step": 4264, "Finetune/Step Time": 4.548306051641703} +{"Finetune/Learning Rate": 1.6420802985292995e-06, "Finetune/Loss": 0.836113691329956, "Finetune/Loss (Raw)": 0.911492645740509, "Finetune/Step": 4265, "Finetune/Step Time": 4.556054301559925} +{"Finetune/Learning Rate": 1.638661576165227e-06, "Finetune/Loss": 0.8360862731933594, "Finetune/Loss (Raw)": 0.9532855749130249, "Finetune/Step": 4266, "Finetune/Step Time": 4.5550172589719296} +{"Finetune/Learning Rate": 1.6352460986733564e-06, "Finetune/Loss": 0.8360731601715088, "Finetune/Loss (Raw)": 0.7830933332443237, "Finetune/Step": 4267, "Finetune/Step Time": 4.568810665979981} +{"Finetune/Learning Rate": 1.6318338673791655e-06, "Finetune/Loss": 0.8359524011611938, "Finetune/Loss (Raw)": 0.7995820641517639, "Finetune/Step": 4268, "Finetune/Step Time": 4.560155788436532} +{"Finetune/Learning Rate": 1.6284248836068772e-06, "Finetune/Loss": 0.8387687802314758, "Finetune/Loss (Raw)": 0.7457020878791809, "Finetune/Step": 4269, "Finetune/Step Time": 4.558291995897889} +{"Finetune/Learning Rate": 1.6250191486794497e-06, "Finetune/Loss": 0.8392386436462402, "Finetune/Loss (Raw)": 0.8372179865837097, "Finetune/Step": 4270, "Finetune/Step Time": 4.558692429214716} +{"Finetune/Learning Rate": 1.6216166639185804e-06, "Finetune/Loss": 0.8356407880783081, "Finetune/Loss (Raw)": 0.327078640460968, "Finetune/Step": 4271, "Finetune/Step Time": 4.597767312079668} +{"Finetune/Learning Rate": 1.618217430644713e-06, "Finetune/Loss": 0.8405154943466187, "Finetune/Loss (Raw)": 0.9152648448944092, "Finetune/Step": 4272, "Finetune/Step Time": 4.562362430617213} +{"Finetune/Learning Rate": 1.6148214501770221e-06, "Finetune/Loss": 0.8405575156211853, "Finetune/Loss (Raw)": 0.7428059577941895, "Finetune/Step": 4273, "Finetune/Step Time": 4.549940360710025} +{"Finetune/Learning Rate": 1.6114287238334148e-06, "Finetune/Loss": 0.8401172757148743, "Finetune/Loss (Raw)": 0.8563403487205505, "Finetune/Step": 4274, "Finetune/Step Time": 4.555563552305102} +{"Finetune/Learning Rate": 1.6080392529305489e-06, "Finetune/Loss": 0.8399045467376709, "Finetune/Loss (Raw)": 0.8208946585655212, "Finetune/Step": 4275, "Finetune/Step Time": 4.566424418240786} +{"Finetune/Learning Rate": 1.60465303878381e-06, "Finetune/Loss": 0.8401097059249878, "Finetune/Loss (Raw)": 0.9223807454109192, "Finetune/Step": 4276, "Finetune/Step Time": 4.567497044801712} +{"Finetune/Learning Rate": 1.6012700827073213e-06, "Finetune/Loss": 0.8401676416397095, "Finetune/Loss (Raw)": 0.8489966988563538, "Finetune/Step": 4277, "Finetune/Step Time": 4.589339561760426} +{"Finetune/Learning Rate": 1.5978903860139416e-06, "Finetune/Loss": 0.8413038849830627, "Finetune/Loss (Raw)": 0.9266420602798462, "Finetune/Step": 4278, "Finetune/Step Time": 4.562967527657747} +{"Finetune/Learning Rate": 1.5945139500152662e-06, "Finetune/Loss": 0.8408271670341492, "Finetune/Loss (Raw)": 0.8030929565429688, "Finetune/Step": 4279, "Finetune/Step Time": 4.56100551225245} +{"Finetune/Learning Rate": 1.5911407760216235e-06, "Finetune/Loss": 0.8414645791053772, "Finetune/Loss (Raw)": 0.9453414678573608, "Finetune/Step": 4280, "Finetune/Step Time": 4.554473033174872} +{"Finetune/Learning Rate": 1.5877708653420742e-06, "Finetune/Loss": 0.8427150249481201, "Finetune/Loss (Raw)": 0.996052622795105, "Finetune/Step": 4281, "Finetune/Step Time": 4.5602974854409695} +{"Finetune/Learning Rate": 1.58440421928442e-06, "Finetune/Loss": 0.8432913422584534, "Finetune/Loss (Raw)": 0.9527832865715027, "Finetune/Step": 4282, "Finetune/Step Time": 4.558483857661486} +{"Finetune/Learning Rate": 1.581040839155189e-06, "Finetune/Loss": 0.8437580466270447, "Finetune/Loss (Raw)": 0.8767850995063782, "Finetune/Step": 4283, "Finetune/Step Time": 4.563490992411971} +{"Finetune/Learning Rate": 1.5776807262596371e-06, "Finetune/Loss": 0.8440377116203308, "Finetune/Loss (Raw)": 0.9188669323921204, "Finetune/Step": 4284, "Finetune/Step Time": 4.5618279334157705} +{"Finetune/Learning Rate": 1.574323881901767e-06, "Finetune/Loss": 0.8416410088539124, "Finetune/Loss (Raw)": 0.6741052865982056, "Finetune/Step": 4285, "Finetune/Step Time": 4.596611067652702} +{"Finetune/Learning Rate": 1.5709703073843008e-06, "Finetune/Loss": 0.8422070741653442, "Finetune/Loss (Raw)": 0.9029805064201355, "Finetune/Step": 4286, "Finetune/Step Time": 4.587863773107529} +{"Finetune/Learning Rate": 1.5676200040086941e-06, "Finetune/Loss": 0.8420131206512451, "Finetune/Loss (Raw)": 0.9077425003051758, "Finetune/Step": 4287, "Finetune/Step Time": 4.590539151802659} +{"Finetune/Learning Rate": 1.564272973075136e-06, "Finetune/Loss": 0.8422052264213562, "Finetune/Loss (Raw)": 1.008587121963501, "Finetune/Step": 4288, "Finetune/Step Time": 4.6053884960711} +{"Finetune/Learning Rate": 1.560929215882544e-06, "Finetune/Loss": 0.8413989543914795, "Finetune/Loss (Raw)": 0.8118664026260376, "Finetune/Step": 4289, "Finetune/Step Time": 4.606622001156211} +{"Finetune/Learning Rate": 1.5575887337285644e-06, "Finetune/Loss": 0.8416334986686707, "Finetune/Loss (Raw)": 0.9812463521957397, "Finetune/Step": 4290, "Finetune/Step Time": 4.607707969844341} +{"Finetune/Learning Rate": 1.5542515279095704e-06, "Finetune/Loss": 0.8431957960128784, "Finetune/Loss (Raw)": 0.9212697744369507, "Finetune/Step": 4291, "Finetune/Step Time": 4.592589419335127} +{"Finetune/Learning Rate": 1.5509175997206705e-06, "Finetune/Loss": 0.8457629680633545, "Finetune/Loss (Raw)": 0.9276910424232483, "Finetune/Step": 4292, "Finetune/Step Time": 4.587858337908983} +{"Finetune/Learning Rate": 1.5475869504556996e-06, "Finetune/Loss": 0.8449233770370483, "Finetune/Loss (Raw)": 0.9074059128761292, "Finetune/Step": 4293, "Finetune/Step Time": 4.5960657969117165} +{"Finetune/Learning Rate": 1.5442595814072081e-06, "Finetune/Loss": 0.8443484902381897, "Finetune/Loss (Raw)": 0.8738837242126465, "Finetune/Step": 4294, "Finetune/Step Time": 4.612564940005541} +{"Finetune/Learning Rate": 1.540935493866491e-06, "Finetune/Loss": 0.8405451774597168, "Finetune/Loss (Raw)": 0.3011733889579773, "Finetune/Step": 4295, "Finetune/Step Time": 4.6414621863514185} +{"Finetune/Learning Rate": 1.53761468912356e-06, "Finetune/Loss": 0.8426966667175293, "Finetune/Loss (Raw)": 0.9794055223464966, "Finetune/Step": 4296, "Finetune/Step Time": 4.638338951393962} +{"Finetune/Learning Rate": 1.5342971684671537e-06, "Finetune/Loss": 0.846781849861145, "Finetune/Loss (Raw)": 0.899412214756012, "Finetune/Step": 4297, "Finetune/Step Time": 4.603067144751549} +{"Finetune/Learning Rate": 1.530982933184737e-06, "Finetune/Loss": 0.846206545829773, "Finetune/Loss (Raw)": 0.7005410194396973, "Finetune/Step": 4298, "Finetune/Step Time": 4.603633031249046} +{"Finetune/Learning Rate": 1.527671984562501e-06, "Finetune/Loss": 0.8464916944503784, "Finetune/Loss (Raw)": 0.9439482688903809, "Finetune/Step": 4299, "Finetune/Step Time": 4.6103874463588} +{"Finetune/Learning Rate": 1.524364323885359e-06, "Finetune/Loss": 0.8468140363693237, "Finetune/Loss (Raw)": 0.9487640261650085, "Finetune/Step": 4300, "Finetune/Step Time": 4.5951567478477955} +{"Finetune/Learning Rate": 1.5210599524369484e-06, "Finetune/Loss": 0.8431493043899536, "Finetune/Loss (Raw)": 0.36287930607795715, "Finetune/Step": 4301, "Finetune/Step Time": 4.621801283210516} +{"Finetune/Learning Rate": 1.517758871499635e-06, "Finetune/Loss": 0.847955584526062, "Finetune/Loss (Raw)": 0.9722791910171509, "Finetune/Step": 4302, "Finetune/Step Time": 4.588685501366854} +{"Finetune/Learning Rate": 1.5144610823545035e-06, "Finetune/Loss": 0.8496569395065308, "Finetune/Loss (Raw)": 0.9965029954910278, "Finetune/Step": 4303, "Finetune/Step Time": 4.588612977415323} +{"Finetune/Learning Rate": 1.511166586281355e-06, "Finetune/Loss": 0.8460257053375244, "Finetune/Loss (Raw)": 0.3399316668510437, "Finetune/Step": 4304, "Finetune/Step Time": 4.62330062687397} +{"Finetune/Learning Rate": 1.507875384558727e-06, "Finetune/Loss": 0.8445262908935547, "Finetune/Loss (Raw)": 0.7757718563079834, "Finetune/Step": 4305, "Finetune/Step Time": 4.620194608345628} +{"Finetune/Learning Rate": 1.504587478463867e-06, "Finetune/Loss": 0.8439068794250488, "Finetune/Loss (Raw)": 0.8358196020126343, "Finetune/Step": 4306, "Finetune/Step Time": 4.620292304083705} +{"Finetune/Learning Rate": 1.5013028692727483e-06, "Finetune/Loss": 0.8423043489456177, "Finetune/Loss (Raw)": 0.6801364421844482, "Finetune/Step": 4307, "Finetune/Step Time": 4.608572658151388} +{"Finetune/Learning Rate": 1.4980215582600621e-06, "Finetune/Loss": 0.8417341709136963, "Finetune/Loss (Raw)": 0.893447756767273, "Finetune/Step": 4308, "Finetune/Step Time": 4.608934074640274} +{"Finetune/Learning Rate": 1.4947435466992222e-06, "Finetune/Loss": 0.8420436382293701, "Finetune/Loss (Raw)": 0.858972430229187, "Finetune/Step": 4309, "Finetune/Step Time": 4.613586004823446} +{"Finetune/Learning Rate": 1.4914688358623608e-06, "Finetune/Loss": 0.8394598364830017, "Finetune/Loss (Raw)": 0.3528449237346649, "Finetune/Step": 4310, "Finetune/Step Time": 4.646695898845792} +{"Finetune/Learning Rate": 1.488197427020327e-06, "Finetune/Loss": 0.8389269113540649, "Finetune/Loss (Raw)": 0.8743348717689514, "Finetune/Step": 4311, "Finetune/Step Time": 4.652907777577639} +{"Finetune/Learning Rate": 1.4849293214426975e-06, "Finetune/Loss": 0.8395397067070007, "Finetune/Loss (Raw)": 0.9802912473678589, "Finetune/Step": 4312, "Finetune/Step Time": 4.654898313805461} +{"Finetune/Learning Rate": 1.481664520397753e-06, "Finetune/Loss": 0.8396691083908081, "Finetune/Loss (Raw)": 0.981896698474884, "Finetune/Step": 4313, "Finetune/Step Time": 4.665525684133172} +{"Finetune/Learning Rate": 1.4784030251524994e-06, "Finetune/Loss": 0.8395870327949524, "Finetune/Loss (Raw)": 0.932414174079895, "Finetune/Step": 4314, "Finetune/Step Time": 4.655590431764722} +{"Finetune/Learning Rate": 1.4751448369726652e-06, "Finetune/Loss": 0.8399365544319153, "Finetune/Loss (Raw)": 0.8485690951347351, "Finetune/Step": 4315, "Finetune/Step Time": 4.652089241892099} +{"Finetune/Learning Rate": 1.4718899571226842e-06, "Finetune/Loss": 0.8400609493255615, "Finetune/Loss (Raw)": 0.81593918800354, "Finetune/Step": 4316, "Finetune/Step Time": 4.655665388330817} +{"Finetune/Learning Rate": 1.468638386865716e-06, "Finetune/Loss": 0.840538740158081, "Finetune/Loss (Raw)": 0.8675225377082825, "Finetune/Step": 4317, "Finetune/Step Time": 4.6533545684069395} +{"Finetune/Learning Rate": 1.465390127463625e-06, "Finetune/Loss": 0.8413164615631104, "Finetune/Loss (Raw)": 0.789207398891449, "Finetune/Step": 4318, "Finetune/Step Time": 4.656902749091387} +{"Finetune/Learning Rate": 1.4621451801770038e-06, "Finetune/Loss": 0.8398867845535278, "Finetune/Loss (Raw)": 0.7395325899124146, "Finetune/Step": 4319, "Finetune/Step Time": 4.662836508825421} +{"Finetune/Learning Rate": 1.4589035462651502e-06, "Finetune/Loss": 0.8383876085281372, "Finetune/Loss (Raw)": 0.8238429427146912, "Finetune/Step": 4320, "Finetune/Step Time": 4.666354598477483} +{"Finetune/Learning Rate": 1.4556652269860772e-06, "Finetune/Loss": 0.8415826559066772, "Finetune/Loss (Raw)": 0.7141290903091431, "Finetune/Step": 4321, "Finetune/Step Time": 4.642198970541358} +{"Finetune/Learning Rate": 1.4524302235965203e-06, "Finetune/Loss": 0.8424720764160156, "Finetune/Loss (Raw)": 1.0124341249465942, "Finetune/Step": 4322, "Finetune/Step Time": 4.645183375105262} +{"Finetune/Learning Rate": 1.4491985373519146e-06, "Finetune/Loss": 0.8409569263458252, "Finetune/Loss (Raw)": 0.7085580825805664, "Finetune/Step": 4323, "Finetune/Step Time": 4.642157692462206} +{"Finetune/Learning Rate": 1.4459701695064155e-06, "Finetune/Loss": 0.8402692079544067, "Finetune/Loss (Raw)": 0.811758816242218, "Finetune/Step": 4324, "Finetune/Step Time": 4.641591047868133} +{"Finetune/Learning Rate": 1.4427451213128873e-06, "Finetune/Loss": 0.8409274816513062, "Finetune/Loss (Raw)": 0.6674740314483643, "Finetune/Step": 4325, "Finetune/Step Time": 4.631192982196808} +{"Finetune/Learning Rate": 1.4395233940229147e-06, "Finetune/Loss": 0.8392434120178223, "Finetune/Loss (Raw)": 0.7462844848632812, "Finetune/Step": 4326, "Finetune/Step Time": 4.639451807364821} +{"Finetune/Learning Rate": 1.4363049888867853e-06, "Finetune/Loss": 0.8372244238853455, "Finetune/Loss (Raw)": 0.7211954593658447, "Finetune/Step": 4327, "Finetune/Step Time": 4.656724914908409} +{"Finetune/Learning Rate": 1.4330899071534942e-06, "Finetune/Loss": 0.8369467258453369, "Finetune/Loss (Raw)": 0.8637863397598267, "Finetune/Step": 4328, "Finetune/Step Time": 4.655522396788001} +{"Finetune/Learning Rate": 1.4298781500707558e-06, "Finetune/Loss": 0.8369926810264587, "Finetune/Loss (Raw)": 0.8552189469337463, "Finetune/Step": 4329, "Finetune/Step Time": 4.6398352943360806} +{"Finetune/Learning Rate": 1.426669718884991e-06, "Finetune/Loss": 0.8380776643753052, "Finetune/Loss (Raw)": 0.9589781165122986, "Finetune/Step": 4330, "Finetune/Step Time": 4.639643341302872} +{"Finetune/Learning Rate": 1.4234646148413277e-06, "Finetune/Loss": 0.8393822908401489, "Finetune/Loss (Raw)": 0.9491733908653259, "Finetune/Step": 4331, "Finetune/Step Time": 4.639632133767009} +{"Finetune/Learning Rate": 1.420262839183605e-06, "Finetune/Loss": 0.8400883674621582, "Finetune/Loss (Raw)": 0.8921568393707275, "Finetune/Step": 4332, "Finetune/Step Time": 4.642300991341472} +{"Finetune/Learning Rate": 1.4170643931543693e-06, "Finetune/Loss": 0.8406760096549988, "Finetune/Loss (Raw)": 0.9702799916267395, "Finetune/Step": 4333, "Finetune/Step Time": 4.643874920904636} +{"Finetune/Learning Rate": 1.4138692779948749e-06, "Finetune/Loss": 0.8414918184280396, "Finetune/Loss (Raw)": 0.9415751695632935, "Finetune/Step": 4334, "Finetune/Step Time": 4.642118968069553} +{"Finetune/Learning Rate": 1.4106774949450808e-06, "Finetune/Loss": 0.8414270877838135, "Finetune/Loss (Raw)": 0.9133886694908142, "Finetune/Step": 4335, "Finetune/Step Time": 4.636862924322486} +{"Finetune/Learning Rate": 1.4074890452436607e-06, "Finetune/Loss": 0.8428786993026733, "Finetune/Loss (Raw)": 1.0141292810440063, "Finetune/Step": 4336, "Finetune/Step Time": 4.6531405337154865} +{"Finetune/Learning Rate": 1.4043039301279904e-06, "Finetune/Loss": 0.8398478031158447, "Finetune/Loss (Raw)": 0.6710964441299438, "Finetune/Step": 4337, "Finetune/Step Time": 4.675831213593483} +{"Finetune/Learning Rate": 1.4011221508341433e-06, "Finetune/Loss": 0.8400617837905884, "Finetune/Loss (Raw)": 0.8690274357795715, "Finetune/Step": 4338, "Finetune/Step Time": 4.675442162901163} +{"Finetune/Learning Rate": 1.3979437085969127e-06, "Finetune/Loss": 0.8389619588851929, "Finetune/Loss (Raw)": 0.7947842478752136, "Finetune/Step": 4339, "Finetune/Step Time": 4.6757886577397585} +{"Finetune/Learning Rate": 1.3947686046497878e-06, "Finetune/Loss": 0.8396298885345459, "Finetune/Loss (Raw)": 0.868465006351471, "Finetune/Step": 4340, "Finetune/Step Time": 4.690131472423673} +{"Finetune/Learning Rate": 1.3915968402249647e-06, "Finetune/Loss": 0.8416134119033813, "Finetune/Loss (Raw)": 0.9713923335075378, "Finetune/Step": 4341, "Finetune/Step Time": 4.681337088346481} +{"Finetune/Learning Rate": 1.3884284165533423e-06, "Finetune/Loss": 0.8410794734954834, "Finetune/Loss (Raw)": 0.824990451335907, "Finetune/Step": 4342, "Finetune/Step Time": 4.72768397256732} +{"Finetune/Learning Rate": 1.3852633348645262e-06, "Finetune/Loss": 0.8405433297157288, "Finetune/Loss (Raw)": 0.8034464716911316, "Finetune/Step": 4343, "Finetune/Step Time": 4.690755574032664} +{"Finetune/Learning Rate": 1.38210159638682e-06, "Finetune/Loss": 0.8408249616622925, "Finetune/Loss (Raw)": 0.9046141505241394, "Finetune/Step": 4344, "Finetune/Step Time": 4.700065026059747} +{"Finetune/Learning Rate": 1.3789432023472315e-06, "Finetune/Loss": 0.8401764631271362, "Finetune/Loss (Raw)": 0.8562712073326111, "Finetune/Step": 4345, "Finetune/Step Time": 4.699715256690979} +{"Finetune/Learning Rate": 1.3757881539714768e-06, "Finetune/Loss": 0.8403833508491516, "Finetune/Loss (Raw)": 0.877285897731781, "Finetune/Step": 4346, "Finetune/Step Time": 4.7031962685287} +{"Finetune/Learning Rate": 1.3726364524839687e-06, "Finetune/Loss": 0.8419762849807739, "Finetune/Loss (Raw)": 0.9571524262428284, "Finetune/Step": 4347, "Finetune/Step Time": 4.696001974865794} +{"Finetune/Learning Rate": 1.369488099107814e-06, "Finetune/Loss": 0.8422596454620361, "Finetune/Loss (Raw)": 0.9173177480697632, "Finetune/Step": 4348, "Finetune/Step Time": 4.701826184988022} +{"Finetune/Learning Rate": 1.3663430950648326e-06, "Finetune/Loss": 0.8420745134353638, "Finetune/Loss (Raw)": 0.9795624613761902, "Finetune/Step": 4349, "Finetune/Step Time": 4.698906870558858} +{"Finetune/Learning Rate": 1.363201441575539e-06, "Finetune/Loss": 0.8395719528198242, "Finetune/Loss (Raw)": 0.645363986492157, "Finetune/Step": 4350, "Finetune/Step Time": 4.710607264190912} +{"Finetune/Learning Rate": 1.360063139859147e-06, "Finetune/Loss": 0.8390429019927979, "Finetune/Loss (Raw)": 0.9027372002601624, "Finetune/Step": 4351, "Finetune/Step Time": 4.710184667259455} +{"Finetune/Learning Rate": 1.3569281911335686e-06, "Finetune/Loss": 0.8394448757171631, "Finetune/Loss (Raw)": 0.8604475855827332, "Finetune/Step": 4352, "Finetune/Step Time": 4.701860202476382} +{"Finetune/Learning Rate": 1.3537965966154177e-06, "Finetune/Loss": 0.840368390083313, "Finetune/Loss (Raw)": 0.9143542647361755, "Finetune/Step": 4353, "Finetune/Step Time": 4.699845662340522} +{"Finetune/Learning Rate": 1.3506683575200053e-06, "Finetune/Loss": 0.8385595083236694, "Finetune/Loss (Raw)": 0.7524814009666443, "Finetune/Step": 4354, "Finetune/Step Time": 4.698639875277877} +{"Finetune/Learning Rate": 1.347543475061337e-06, "Finetune/Loss": 0.8371177911758423, "Finetune/Loss (Raw)": 0.8826040029525757, "Finetune/Step": 4355, "Finetune/Step Time": 4.694293102249503} +{"Finetune/Learning Rate": 1.344421950452126e-06, "Finetune/Loss": 0.8373968601226807, "Finetune/Loss (Raw)": 0.8916262984275818, "Finetune/Step": 4356, "Finetune/Step Time": 4.686890389770269} +{"Finetune/Learning Rate": 1.341303784903767e-06, "Finetune/Loss": 0.8375459313392639, "Finetune/Loss (Raw)": 0.9205108880996704, "Finetune/Step": 4357, "Finetune/Step Time": 4.686915962025523} +{"Finetune/Learning Rate": 1.3381889796263602e-06, "Finetune/Loss": 0.8425954580307007, "Finetune/Loss (Raw)": 0.9158370494842529, "Finetune/Step": 4358, "Finetune/Step Time": 4.659243544563651} +{"Finetune/Learning Rate": 1.335077535828706e-06, "Finetune/Loss": 0.8444270491600037, "Finetune/Loss (Raw)": 0.89193195104599, "Finetune/Step": 4359, "Finetune/Step Time": 4.618617996573448} +{"Finetune/Learning Rate": 1.3319694547182916e-06, "Finetune/Loss": 0.8438057899475098, "Finetune/Loss (Raw)": 0.8207180500030518, "Finetune/Step": 4360, "Finetune/Step Time": 4.610977064818144} +{"Finetune/Learning Rate": 1.328864737501302e-06, "Finetune/Loss": 0.8459548354148865, "Finetune/Loss (Raw)": 1.0670493841171265, "Finetune/Step": 4361, "Finetune/Step Time": 4.619265254586935} +{"Finetune/Learning Rate": 1.325763385382618e-06, "Finetune/Loss": 0.8437206745147705, "Finetune/Loss (Raw)": 0.8106595873832703, "Finetune/Step": 4362, "Finetune/Step Time": 4.616798106580973} +{"Finetune/Learning Rate": 1.3226653995658144e-06, "Finetune/Loss": 0.8432416915893555, "Finetune/Loss (Raw)": 0.8356082439422607, "Finetune/Step": 4363, "Finetune/Step Time": 4.613610420376062} +{"Finetune/Learning Rate": 1.319570781253159e-06, "Finetune/Loss": 0.8436428308486938, "Finetune/Loss (Raw)": 0.778567910194397, "Finetune/Step": 4364, "Finetune/Step Time": 4.630309749394655} +{"Finetune/Learning Rate": 1.3164795316456091e-06, "Finetune/Loss": 0.841850996017456, "Finetune/Loss (Raw)": 0.7726296186447144, "Finetune/Step": 4365, "Finetune/Step Time": 4.645013762637973} +{"Finetune/Learning Rate": 1.313391651942827e-06, "Finetune/Loss": 0.8414556980133057, "Finetune/Loss (Raw)": 0.9080725908279419, "Finetune/Step": 4366, "Finetune/Step Time": 4.644703326746821} +{"Finetune/Learning Rate": 1.3103071433431501e-06, "Finetune/Loss": 0.8405113220214844, "Finetune/Loss (Raw)": 0.7901756763458252, "Finetune/Step": 4367, "Finetune/Step Time": 4.645660445094109} +{"Finetune/Learning Rate": 1.307226007043617e-06, "Finetune/Loss": 0.8411964178085327, "Finetune/Loss (Raw)": 0.9220325946807861, "Finetune/Step": 4368, "Finetune/Step Time": 4.647463746368885} +{"Finetune/Learning Rate": 1.30414824423996e-06, "Finetune/Loss": 0.8409918546676636, "Finetune/Loss (Raw)": 0.8239206075668335, "Finetune/Step": 4369, "Finetune/Step Time": 4.6541393380612135} +{"Finetune/Learning Rate": 1.301073856126598e-06, "Finetune/Loss": 0.8411526679992676, "Finetune/Loss (Raw)": 0.9632192254066467, "Finetune/Step": 4370, "Finetune/Step Time": 4.65626097843051} +{"Finetune/Learning Rate": 1.2980028438966396e-06, "Finetune/Loss": 0.8419818878173828, "Finetune/Loss (Raw)": 0.9927358031272888, "Finetune/Step": 4371, "Finetune/Step Time": 4.634452959522605} +{"Finetune/Learning Rate": 1.2949352087418864e-06, "Finetune/Loss": 0.8370015621185303, "Finetune/Loss (Raw)": 0.2503834366798401, "Finetune/Step": 4372, "Finetune/Step Time": 4.673603186383843} +{"Finetune/Learning Rate": 1.291870951852826e-06, "Finetune/Loss": 0.8359476923942566, "Finetune/Loss (Raw)": 0.7560197710990906, "Finetune/Step": 4373, "Finetune/Step Time": 4.6764181684702635} +{"Finetune/Learning Rate": 1.288810074418637e-06, "Finetune/Loss": 0.8357441425323486, "Finetune/Loss (Raw)": 0.8352912664413452, "Finetune/Step": 4374, "Finetune/Step Time": 4.641626896336675} +{"Finetune/Learning Rate": 1.2857525776271852e-06, "Finetune/Loss": 0.8352019190788269, "Finetune/Loss (Raw)": 0.8014030456542969, "Finetune/Step": 4375, "Finetune/Step Time": 4.643759772181511} +{"Finetune/Learning Rate": 1.2826984626650297e-06, "Finetune/Loss": 0.8401763439178467, "Finetune/Loss (Raw)": 0.9272158145904541, "Finetune/Step": 4376, "Finetune/Step Time": 4.625051360577345} +{"Finetune/Learning Rate": 1.2796477307174082e-06, "Finetune/Loss": 0.8397336006164551, "Finetune/Loss (Raw)": 0.8708713054656982, "Finetune/Step": 4377, "Finetune/Step Time": 4.626001823693514} +{"Finetune/Learning Rate": 1.2766003829682504e-06, "Finetune/Loss": 0.8408718705177307, "Finetune/Loss (Raw)": 0.8737647533416748, "Finetune/Step": 4378, "Finetune/Step Time": 4.630652561783791} +{"Finetune/Learning Rate": 1.273556420600175e-06, "Finetune/Loss": 0.8410409092903137, "Finetune/Loss (Raw)": 0.7668942809104919, "Finetune/Step": 4379, "Finetune/Step Time": 4.641839819028974} +{"Finetune/Learning Rate": 1.2705158447944843e-06, "Finetune/Loss": 0.8405730724334717, "Finetune/Loss (Raw)": 0.9826006293296814, "Finetune/Step": 4380, "Finetune/Step Time": 4.644589325413108} +{"Finetune/Learning Rate": 1.2674786567311648e-06, "Finetune/Loss": 0.8425394296646118, "Finetune/Loss (Raw)": 0.9040297865867615, "Finetune/Step": 4381, "Finetune/Step Time": 4.63180092535913} +{"Finetune/Learning Rate": 1.2644448575888913e-06, "Finetune/Loss": 0.8445663452148438, "Finetune/Loss (Raw)": 0.9331270456314087, "Finetune/Step": 4382, "Finetune/Step Time": 4.625779896974564} +{"Finetune/Learning Rate": 1.2614144485450208e-06, "Finetune/Loss": 0.8442096710205078, "Finetune/Loss (Raw)": 0.7349616885185242, "Finetune/Step": 4383, "Finetune/Step Time": 4.639848705381155} +{"Finetune/Learning Rate": 1.2583874307755961e-06, "Finetune/Loss": 0.8438882231712341, "Finetune/Loss (Raw)": 0.8728399872779846, "Finetune/Step": 4384, "Finetune/Step Time": 4.625781387090683} +{"Finetune/Learning Rate": 1.2553638054553418e-06, "Finetune/Loss": 0.8426080346107483, "Finetune/Loss (Raw)": 0.8090928196907043, "Finetune/Step": 4385, "Finetune/Step Time": 4.618780601769686} +{"Finetune/Learning Rate": 1.252343573757674e-06, "Finetune/Loss": 0.8439277410507202, "Finetune/Loss (Raw)": 0.8559696674346924, "Finetune/Step": 4386, "Finetune/Step Time": 4.636114533990622} +{"Finetune/Learning Rate": 1.2493267368546802e-06, "Finetune/Loss": 0.84428870677948, "Finetune/Loss (Raw)": 1.0600484609603882, "Finetune/Step": 4387, "Finetune/Step Time": 4.623778259381652} +{"Finetune/Learning Rate": 1.2463132959171343e-06, "Finetune/Loss": 0.8447703719139099, "Finetune/Loss (Raw)": 0.9044510722160339, "Finetune/Step": 4388, "Finetune/Step Time": 4.62869031354785} +{"Finetune/Learning Rate": 1.2433032521145006e-06, "Finetune/Loss": 0.8446685075759888, "Finetune/Loss (Raw)": 0.9205257296562195, "Finetune/Step": 4389, "Finetune/Step Time": 4.627676727250218} +{"Finetune/Learning Rate": 1.2402966066149135e-06, "Finetune/Loss": 0.8445389270782471, "Finetune/Loss (Raw)": 0.8288193941116333, "Finetune/Step": 4390, "Finetune/Step Time": 4.621430817991495} +{"Finetune/Learning Rate": 1.2372933605851944e-06, "Finetune/Loss": 0.8430531024932861, "Finetune/Loss (Raw)": 0.8456372618675232, "Finetune/Step": 4391, "Finetune/Step Time": 4.6291933208703995} +{"Finetune/Learning Rate": 1.2342935151908453e-06, "Finetune/Loss": 0.8397130966186523, "Finetune/Loss (Raw)": 0.3542550802230835, "Finetune/Step": 4392, "Finetune/Step Time": 4.659130722284317} +{"Finetune/Learning Rate": 1.231297071596046e-06, "Finetune/Loss": 0.8384050726890564, "Finetune/Loss (Raw)": 0.7440710067749023, "Finetune/Step": 4393, "Finetune/Step Time": 4.666919952258468} +{"Finetune/Learning Rate": 1.2283040309636596e-06, "Finetune/Loss": 0.8373696804046631, "Finetune/Loss (Raw)": 0.8207569718360901, "Finetune/Step": 4394, "Finetune/Step Time": 4.676869101822376} +{"Finetune/Learning Rate": 1.2253143944552258e-06, "Finetune/Loss": 0.8378900289535522, "Finetune/Loss (Raw)": 0.849702775478363, "Finetune/Step": 4395, "Finetune/Step Time": 4.666840700432658} +{"Finetune/Learning Rate": 1.2223281632309636e-06, "Finetune/Loss": 0.839188814163208, "Finetune/Loss (Raw)": 0.9658161997795105, "Finetune/Step": 4396, "Finetune/Step Time": 4.675130369141698} +{"Finetune/Learning Rate": 1.2193453384497722e-06, "Finetune/Loss": 0.8366379737854004, "Finetune/Loss (Raw)": 0.4191935062408447, "Finetune/Step": 4397, "Finetune/Step Time": 4.6791052389889956} +{"Finetune/Learning Rate": 1.2163659212692246e-06, "Finetune/Loss": 0.8370676040649414, "Finetune/Loss (Raw)": 0.8922054171562195, "Finetune/Step": 4398, "Finetune/Step Time": 4.681427633389831} +{"Finetune/Learning Rate": 1.2133899128455796e-06, "Finetune/Loss": 0.8408083915710449, "Finetune/Loss (Raw)": 0.805904746055603, "Finetune/Step": 4399, "Finetune/Step Time": 4.645774275064468} +{"Finetune/Learning Rate": 1.2104173143337672e-06, "Finetune/Loss": 0.8406018018722534, "Finetune/Loss (Raw)": 0.8888192772865295, "Finetune/Step": 4400, "Finetune/Step Time": 4.652884567156434} +{"Finetune/Learning Rate": 1.2074481268873894e-06, "Finetune/Loss": 0.8374149799346924, "Finetune/Loss (Raw)": 0.3348935842514038, "Finetune/Step": 4401, "Finetune/Step Time": 4.691731097176671} +{"Finetune/Learning Rate": 1.2044823516587345e-06, "Finetune/Loss": 0.8370806574821472, "Finetune/Loss (Raw)": 0.813550591468811, "Finetune/Step": 4402, "Finetune/Step Time": 4.71550733782351} +{"Finetune/Learning Rate": 1.2015199897987616e-06, "Finetune/Loss": 0.837855339050293, "Finetune/Loss (Raw)": 0.9200525283813477, "Finetune/Step": 4403, "Finetune/Step Time": 4.70024867542088} +{"Finetune/Learning Rate": 1.198561042457105e-06, "Finetune/Loss": 0.8385704755783081, "Finetune/Loss (Raw)": 1.0139204263687134, "Finetune/Step": 4404, "Finetune/Step Time": 4.701223174110055} +{"Finetune/Learning Rate": 1.195605510782073e-06, "Finetune/Loss": 0.8383575677871704, "Finetune/Loss (Raw)": 0.821744978427887, "Finetune/Step": 4405, "Finetune/Step Time": 4.691717483103275} +{"Finetune/Learning Rate": 1.192653395920652e-06, "Finetune/Loss": 0.8370110988616943, "Finetune/Loss (Raw)": 0.7542849779129028, "Finetune/Step": 4406, "Finetune/Step Time": 4.689328646287322} +{"Finetune/Learning Rate": 1.189704699018499e-06, "Finetune/Loss": 0.8368172645568848, "Finetune/Loss (Raw)": 0.7782893180847168, "Finetune/Step": 4407, "Finetune/Step Time": 4.6873928271234035} +{"Finetune/Learning Rate": 1.186759421219943e-06, "Finetune/Loss": 0.837399959564209, "Finetune/Loss (Raw)": 1.0199260711669922, "Finetune/Step": 4408, "Finetune/Step Time": 4.68683017231524} +{"Finetune/Learning Rate": 1.1838175636679938e-06, "Finetune/Loss": 0.8360075950622559, "Finetune/Loss (Raw)": 0.8178296089172363, "Finetune/Step": 4409, "Finetune/Step Time": 4.685946732759476} +{"Finetune/Learning Rate": 1.1808791275043297e-06, "Finetune/Loss": 0.835465669631958, "Finetune/Loss (Raw)": 0.8834174275398254, "Finetune/Step": 4410, "Finetune/Step Time": 4.682774726301432} +{"Finetune/Learning Rate": 1.1779441138692926e-06, "Finetune/Loss": 0.8352782726287842, "Finetune/Loss (Raw)": 0.8527956604957581, "Finetune/Step": 4411, "Finetune/Step Time": 4.675931168720126} +{"Finetune/Learning Rate": 1.1750125239019118e-06, "Finetune/Loss": 0.8345110416412354, "Finetune/Loss (Raw)": 0.8206701278686523, "Finetune/Step": 4412, "Finetune/Step Time": 4.67781325802207} +{"Finetune/Learning Rate": 1.1720843587398756e-06, "Finetune/Loss": 0.8377436399459839, "Finetune/Loss (Raw)": 1.0878697633743286, "Finetune/Step": 4413, "Finetune/Step Time": 4.6409729309380054} +{"Finetune/Learning Rate": 1.1691596195195498e-06, "Finetune/Loss": 0.8370051383972168, "Finetune/Loss (Raw)": 0.8084529638290405, "Finetune/Step": 4414, "Finetune/Step Time": 4.640001937747002} +{"Finetune/Learning Rate": 1.1662383073759686e-06, "Finetune/Loss": 0.8372551202774048, "Finetune/Loss (Raw)": 0.9397464990615845, "Finetune/Step": 4415, "Finetune/Step Time": 4.645542876794934} +{"Finetune/Learning Rate": 1.1633204234428352e-06, "Finetune/Loss": 0.8353251218795776, "Finetune/Loss (Raw)": 0.7615450024604797, "Finetune/Step": 4416, "Finetune/Step Time": 4.6466403640806675} +{"Finetune/Learning Rate": 1.1604059688525238e-06, "Finetune/Loss": 0.8364895582199097, "Finetune/Loss (Raw)": 0.9609069228172302, "Finetune/Step": 4417, "Finetune/Step Time": 4.647378772497177} +{"Finetune/Learning Rate": 1.157494944736075e-06, "Finetune/Loss": 0.8355170488357544, "Finetune/Loss (Raw)": 0.856767475605011, "Finetune/Step": 4418, "Finetune/Step Time": 4.647339982911944} +{"Finetune/Learning Rate": 1.1545873522232055e-06, "Finetune/Loss": 0.8355429172515869, "Finetune/Loss (Raw)": 0.9245842695236206, "Finetune/Step": 4419, "Finetune/Step Time": 4.646899446845055} +{"Finetune/Learning Rate": 1.1516831924422945e-06, "Finetune/Loss": 0.8342724442481995, "Finetune/Loss (Raw)": 0.765072762966156, "Finetune/Step": 4420, "Finetune/Step Time": 4.659002428874373} +{"Finetune/Learning Rate": 1.1487824665203828e-06, "Finetune/Loss": 0.8337365388870239, "Finetune/Loss (Raw)": 0.8388116359710693, "Finetune/Step": 4421, "Finetune/Step Time": 4.661905279383063} +{"Finetune/Learning Rate": 1.1458851755831934e-06, "Finetune/Loss": 0.833875834941864, "Finetune/Loss (Raw)": 0.8917080760002136, "Finetune/Step": 4422, "Finetune/Step Time": 4.644321177154779} +{"Finetune/Learning Rate": 1.1429913207551046e-06, "Finetune/Loss": 0.837702751159668, "Finetune/Loss (Raw)": 0.7910196185112, "Finetune/Step": 4423, "Finetune/Step Time": 4.6117806527763605} +{"Finetune/Learning Rate": 1.1401009031591658e-06, "Finetune/Loss": 0.8326354026794434, "Finetune/Loss (Raw)": 0.33078205585479736, "Finetune/Step": 4424, "Finetune/Step Time": 4.653160700574517} +{"Finetune/Learning Rate": 1.137213923917092e-06, "Finetune/Loss": 0.8320629000663757, "Finetune/Loss (Raw)": 0.8261358737945557, "Finetune/Step": 4425, "Finetune/Step Time": 4.669971102848649} +{"Finetune/Learning Rate": 1.1343303841492626e-06, "Finetune/Loss": 0.833712100982666, "Finetune/Loss (Raw)": 0.9116324186325073, "Finetune/Step": 4426, "Finetune/Step Time": 4.659925913438201} +{"Finetune/Learning Rate": 1.1314502849747244e-06, "Finetune/Loss": 0.8324307203292847, "Finetune/Loss (Raw)": 0.7799327969551086, "Finetune/Step": 4427, "Finetune/Step Time": 4.668537667021155} +{"Finetune/Learning Rate": 1.1285736275111836e-06, "Finetune/Loss": 0.8320672512054443, "Finetune/Loss (Raw)": 0.9022445678710938, "Finetune/Step": 4428, "Finetune/Step Time": 4.673994079232216} +{"Finetune/Learning Rate": 1.1257004128750205e-06, "Finetune/Loss": 0.8360115885734558, "Finetune/Loss (Raw)": 0.867755651473999, "Finetune/Step": 4429, "Finetune/Step Time": 4.642720436677337} +{"Finetune/Learning Rate": 1.122830642181273e-06, "Finetune/Loss": 0.8338292837142944, "Finetune/Loss (Raw)": 0.692937433719635, "Finetune/Step": 4430, "Finetune/Step Time": 4.682986283674836} +{"Finetune/Learning Rate": 1.119964316543637e-06, "Finetune/Loss": 0.832817554473877, "Finetune/Loss (Raw)": 0.8670049905776978, "Finetune/Step": 4431, "Finetune/Step Time": 4.69588659144938} +{"Finetune/Learning Rate": 1.1171014370744837e-06, "Finetune/Loss": 0.8370727300643921, "Finetune/Loss (Raw)": 0.8845981955528259, "Finetune/Step": 4432, "Finetune/Step Time": 4.654447486624122} +{"Finetune/Learning Rate": 1.114242004884839e-06, "Finetune/Loss": 0.8366228342056274, "Finetune/Loss (Raw)": 0.7181830406188965, "Finetune/Step": 4433, "Finetune/Step Time": 4.658055203035474} +{"Finetune/Learning Rate": 1.111386021084394e-06, "Finetune/Loss": 0.8376120924949646, "Finetune/Loss (Raw)": 0.9624440670013428, "Finetune/Step": 4434, "Finetune/Step Time": 4.652535008266568} +{"Finetune/Learning Rate": 1.1085334867814946e-06, "Finetune/Loss": 0.839125394821167, "Finetune/Loss (Raw)": 0.87384033203125, "Finetune/Step": 4435, "Finetune/Step Time": 4.647797172889113} +{"Finetune/Learning Rate": 1.1056844030831603e-06, "Finetune/Loss": 0.8381549715995789, "Finetune/Loss (Raw)": 0.7692363262176514, "Finetune/Step": 4436, "Finetune/Step Time": 4.664783298969269} +{"Finetune/Learning Rate": 1.1028387710950616e-06, "Finetune/Loss": 0.837920069694519, "Finetune/Loss (Raw)": 0.8289052844047546, "Finetune/Step": 4437, "Finetune/Step Time": 4.66482213139534} +{"Finetune/Learning Rate": 1.0999965919215327e-06, "Finetune/Loss": 0.8419944643974304, "Finetune/Loss (Raw)": 0.8743634819984436, "Finetune/Step": 4438, "Finetune/Step Time": 4.627909641712904} +{"Finetune/Learning Rate": 1.0971578666655713e-06, "Finetune/Loss": 0.8431659936904907, "Finetune/Loss (Raw)": 1.0242840051651, "Finetune/Step": 4439, "Finetune/Step Time": 4.618777211755514} +{"Finetune/Learning Rate": 1.0943225964288272e-06, "Finetune/Loss": 0.8433655500411987, "Finetune/Loss (Raw)": 1.0058428049087524, "Finetune/Step": 4440, "Finetune/Step Time": 4.617001285776496} +{"Finetune/Learning Rate": 1.091490782311614e-06, "Finetune/Loss": 0.8433452844619751, "Finetune/Loss (Raw)": 0.9793006777763367, "Finetune/Step": 4441, "Finetune/Step Time": 4.6108504962176085} +{"Finetune/Learning Rate": 1.088662425412903e-06, "Finetune/Loss": 0.8433628082275391, "Finetune/Loss (Raw)": 0.9346522688865662, "Finetune/Step": 4442, "Finetune/Step Time": 4.625611396506429} +{"Finetune/Learning Rate": 1.0858375268303267e-06, "Finetune/Loss": 0.843297004699707, "Finetune/Loss (Raw)": 0.8401519060134888, "Finetune/Step": 4443, "Finetune/Step Time": 4.625892965123057} +{"Finetune/Learning Rate": 1.0830160876601737e-06, "Finetune/Loss": 0.8433110117912292, "Finetune/Loss (Raw)": 0.8177330493927002, "Finetune/Step": 4444, "Finetune/Step Time": 4.621682237833738} +{"Finetune/Learning Rate": 1.0801981089973844e-06, "Finetune/Loss": 0.8422338962554932, "Finetune/Loss (Raw)": 0.7296594381332397, "Finetune/Step": 4445, "Finetune/Step Time": 4.624637704342604} +{"Finetune/Learning Rate": 1.0773835919355669e-06, "Finetune/Loss": 0.8437183499336243, "Finetune/Loss (Raw)": 0.979206383228302, "Finetune/Step": 4446, "Finetune/Step Time": 4.6162573248147964} +{"Finetune/Learning Rate": 1.0745725375669768e-06, "Finetune/Loss": 0.8445266485214233, "Finetune/Loss (Raw)": 0.8429918885231018, "Finetune/Step": 4447, "Finetune/Step Time": 4.60606586560607} +{"Finetune/Learning Rate": 1.0717649469825298e-06, "Finetune/Loss": 0.8449211120605469, "Finetune/Loss (Raw)": 0.8743365406990051, "Finetune/Step": 4448, "Finetune/Step Time": 4.605070190504193} +{"Finetune/Learning Rate": 1.0689608212718005e-06, "Finetune/Loss": 0.8471834659576416, "Finetune/Loss (Raw)": 1.0037157535552979, "Finetune/Step": 4449, "Finetune/Step Time": 4.591385534033179} +{"Finetune/Learning Rate": 1.0661601615230122e-06, "Finetune/Loss": 0.8453328609466553, "Finetune/Loss (Raw)": 0.7755522131919861, "Finetune/Step": 4450, "Finetune/Step Time": 4.585251161828637} +{"Finetune/Learning Rate": 1.0633629688230452e-06, "Finetune/Loss": 0.8458157181739807, "Finetune/Loss (Raw)": 0.7703661322593689, "Finetune/Step": 4451, "Finetune/Step Time": 4.5901401322335005} +{"Finetune/Learning Rate": 1.0605692442574356e-06, "Finetune/Loss": 0.8421920537948608, "Finetune/Loss (Raw)": 0.3479244112968445, "Finetune/Step": 4452, "Finetune/Step Time": 4.627077551558614} +{"Finetune/Learning Rate": 1.0577789889103763e-06, "Finetune/Loss": 0.8429078459739685, "Finetune/Loss (Raw)": 0.7591042518615723, "Finetune/Step": 4453, "Finetune/Step Time": 4.6188474129885435} +{"Finetune/Learning Rate": 1.054992203864712e-06, "Finetune/Loss": 0.8439521193504333, "Finetune/Loss (Raw)": 0.8799469470977783, "Finetune/Step": 4454, "Finetune/Step Time": 4.616961997002363} +{"Finetune/Learning Rate": 1.0522088902019324e-06, "Finetune/Loss": 0.8441307544708252, "Finetune/Loss (Raw)": 0.7440659403800964, "Finetune/Step": 4455, "Finetune/Step Time": 4.6139136385172606} +{"Finetune/Learning Rate": 1.049429049002193e-06, "Finetune/Loss": 0.843761682510376, "Finetune/Loss (Raw)": 0.8165374398231506, "Finetune/Step": 4456, "Finetune/Step Time": 4.609895199537277} +{"Finetune/Learning Rate": 1.0466526813442957e-06, "Finetune/Loss": 0.8432149291038513, "Finetune/Loss (Raw)": 0.7852343916893005, "Finetune/Step": 4457, "Finetune/Step Time": 4.621191363781691} +{"Finetune/Learning Rate": 1.0438797883056927e-06, "Finetune/Loss": 0.8422768115997314, "Finetune/Loss (Raw)": 0.8388976454734802, "Finetune/Step": 4458, "Finetune/Step Time": 4.63492576777935} +{"Finetune/Learning Rate": 1.0411103709624903e-06, "Finetune/Loss": 0.8414593935012817, "Finetune/Loss (Raw)": 0.8445557951927185, "Finetune/Step": 4459, "Finetune/Step Time": 4.640556029975414} +{"Finetune/Learning Rate": 1.0383444303894453e-06, "Finetune/Loss": 0.8419822454452515, "Finetune/Loss (Raw)": 0.959075927734375, "Finetune/Step": 4460, "Finetune/Step Time": 4.640959916636348} +{"Finetune/Learning Rate": 1.0355819676599655e-06, "Finetune/Loss": 0.8406128287315369, "Finetune/Loss (Raw)": 0.7949994802474976, "Finetune/Step": 4461, "Finetune/Step Time": 4.639918280765414} +{"Finetune/Learning Rate": 1.0328229838461069e-06, "Finetune/Loss": 0.8404121398925781, "Finetune/Loss (Raw)": 0.9158797860145569, "Finetune/Step": 4462, "Finetune/Step Time": 4.652489768341184} +{"Finetune/Learning Rate": 1.0300674800185795e-06, "Finetune/Loss": 0.8384844064712524, "Finetune/Loss (Raw)": 0.6666405200958252, "Finetune/Step": 4463, "Finetune/Step Time": 4.6621327847242355} +{"Finetune/Learning Rate": 1.0273154572467436e-06, "Finetune/Loss": 0.8372311592102051, "Finetune/Loss (Raw)": 0.8537132143974304, "Finetune/Step": 4464, "Finetune/Step Time": 4.66493277810514} +{"Finetune/Learning Rate": 1.0245669165985973e-06, "Finetune/Loss": 0.8382474780082703, "Finetune/Loss (Raw)": 0.8011815547943115, "Finetune/Step": 4465, "Finetune/Step Time": 4.641795307397842} +{"Finetune/Learning Rate": 1.0218218591408013e-06, "Finetune/Loss": 0.8379847407341003, "Finetune/Loss (Raw)": 0.8353955149650574, "Finetune/Step": 4466, "Finetune/Step Time": 4.640883568674326} +{"Finetune/Learning Rate": 1.0190802859386585e-06, "Finetune/Loss": 0.8388936519622803, "Finetune/Loss (Raw)": 0.9111282825469971, "Finetune/Step": 4467, "Finetune/Step Time": 4.635189864784479} +{"Finetune/Learning Rate": 1.0163421980561184e-06, "Finetune/Loss": 0.8387850522994995, "Finetune/Loss (Raw)": 0.854559063911438, "Finetune/Step": 4468, "Finetune/Step Time": 4.621841987594962} +{"Finetune/Learning Rate": 1.0136075965557812e-06, "Finetune/Loss": 0.8377613425254822, "Finetune/Loss (Raw)": 0.8403677344322205, "Finetune/Step": 4469, "Finetune/Step Time": 4.62758638523519} +{"Finetune/Learning Rate": 1.0108764824988903e-06, "Finetune/Loss": 0.8378863334655762, "Finetune/Loss (Raw)": 0.8409858345985413, "Finetune/Step": 4470, "Finetune/Step Time": 4.584044182673097} +{"Finetune/Learning Rate": 1.008148856945339e-06, "Finetune/Loss": 0.8378247022628784, "Finetune/Loss (Raw)": 0.7955628037452698, "Finetune/Step": 4471, "Finetune/Step Time": 4.597557881847024} +{"Finetune/Learning Rate": 1.0054247209536628e-06, "Finetune/Loss": 0.8375935554504395, "Finetune/Loss (Raw)": 0.8750258088111877, "Finetune/Step": 4472, "Finetune/Step Time": 4.588394306600094} +{"Finetune/Learning Rate": 1.0027040755810513e-06, "Finetune/Loss": 0.8390203714370728, "Finetune/Loss (Raw)": 1.0388991832733154, "Finetune/Step": 4473, "Finetune/Step Time": 4.586503576487303} +{"Finetune/Learning Rate": 9.999869218833313e-07, "Finetune/Loss": 0.8392603397369385, "Finetune/Loss (Raw)": 0.9080038666725159, "Finetune/Step": 4474, "Finetune/Step Time": 4.584707019850612} +{"Finetune/Learning Rate": 9.972732609149738e-07, "Finetune/Loss": 0.8378826379776001, "Finetune/Loss (Raw)": 0.7808074951171875, "Finetune/Step": 4475, "Finetune/Step Time": 4.605810044333339} +{"Finetune/Learning Rate": 9.945630937291006e-07, "Finetune/Loss": 0.8379453420639038, "Finetune/Loss (Raw)": 0.925335705280304, "Finetune/Step": 4476, "Finetune/Step Time": 4.607159566134214} +{"Finetune/Learning Rate": 9.918564213774762e-07, "Finetune/Loss": 0.8365142345428467, "Finetune/Loss (Raw)": 0.7963912487030029, "Finetune/Step": 4477, "Finetune/Step Time": 4.60777622461319} +{"Finetune/Learning Rate": 9.891532449105045e-07, "Finetune/Loss": 0.8388614654541016, "Finetune/Loss (Raw)": 0.9458067417144775, "Finetune/Step": 4478, "Finetune/Step Time": 4.5897304974496365} +{"Finetune/Learning Rate": 9.864535653772367e-07, "Finetune/Loss": 0.8386921882629395, "Finetune/Loss (Raw)": 0.8810672760009766, "Finetune/Step": 4479, "Finetune/Step Time": 4.5990061443299055} +{"Finetune/Learning Rate": 9.837573838253655e-07, "Finetune/Loss": 0.8385281562805176, "Finetune/Loss (Raw)": 0.8394544720649719, "Finetune/Step": 4480, "Finetune/Step Time": 4.603382993489504} +{"Finetune/Learning Rate": 9.810647013012265e-07, "Finetune/Loss": 0.8380159139633179, "Finetune/Loss (Raw)": 0.8487823009490967, "Finetune/Step": 4481, "Finetune/Step Time": 4.604677218943834} +{"Finetune/Learning Rate": 9.783755188497944e-07, "Finetune/Loss": 0.8384448289871216, "Finetune/Loss (Raw)": 0.8073853254318237, "Finetune/Step": 4482, "Finetune/Step Time": 4.619056932628155} +{"Finetune/Learning Rate": 9.75689837514696e-07, "Finetune/Loss": 0.8382906913757324, "Finetune/Loss (Raw)": 0.8628782033920288, "Finetune/Step": 4483, "Finetune/Step Time": 4.622030446305871} +{"Finetune/Learning Rate": 9.730076583381841e-07, "Finetune/Loss": 0.838448703289032, "Finetune/Loss (Raw)": 0.911849319934845, "Finetune/Step": 4484, "Finetune/Step Time": 4.6370490454137325} +{"Finetune/Learning Rate": 9.703289823611606e-07, "Finetune/Loss": 0.8379086852073669, "Finetune/Loss (Raw)": 0.8513908386230469, "Finetune/Step": 4485, "Finetune/Step Time": 4.637338979169726} +{"Finetune/Learning Rate": 9.676538106231725e-07, "Finetune/Loss": 0.837534487247467, "Finetune/Loss (Raw)": 0.8679355382919312, "Finetune/Step": 4486, "Finetune/Step Time": 4.634455671533942} +{"Finetune/Learning Rate": 9.649821441623986e-07, "Finetune/Loss": 0.8331037759780884, "Finetune/Loss (Raw)": 0.3248055577278137, "Finetune/Step": 4487, "Finetune/Step Time": 4.666028780862689} +{"Finetune/Learning Rate": 9.623139840156604e-07, "Finetune/Loss": 0.8341064453125, "Finetune/Loss (Raw)": 0.9490567445755005, "Finetune/Step": 4488, "Finetune/Step Time": 4.6721581146121025} +{"Finetune/Learning Rate": 9.596493312184197e-07, "Finetune/Loss": 0.8310609459877014, "Finetune/Loss (Raw)": 0.6772266030311584, "Finetune/Step": 4489, "Finetune/Step Time": 4.661957159638405} +{"Finetune/Learning Rate": 9.569881868047747e-07, "Finetune/Loss": 0.8305225372314453, "Finetune/Loss (Raw)": 0.7417464852333069, "Finetune/Step": 4490, "Finetune/Step Time": 4.666172966361046} +{"Finetune/Learning Rate": 9.543305518074652e-07, "Finetune/Loss": 0.8301913142204285, "Finetune/Loss (Raw)": 0.7932083606719971, "Finetune/Step": 4491, "Finetune/Step Time": 4.665449492633343} +{"Finetune/Learning Rate": 9.516764272578638e-07, "Finetune/Loss": 0.8312275409698486, "Finetune/Loss (Raw)": 0.9112055897712708, "Finetune/Step": 4492, "Finetune/Step Time": 4.642786948010325} +{"Finetune/Learning Rate": 9.49025814185992e-07, "Finetune/Loss": 0.8316116333007812, "Finetune/Loss (Raw)": 0.821793794631958, "Finetune/Step": 4493, "Finetune/Step Time": 4.632714277133346} +{"Finetune/Learning Rate": 9.463787136204938e-07, "Finetune/Loss": 0.8299399614334106, "Finetune/Loss (Raw)": 0.6941010355949402, "Finetune/Step": 4494, "Finetune/Step Time": 4.637644469738007} +{"Finetune/Learning Rate": 9.437351265886585e-07, "Finetune/Loss": 0.8308059573173523, "Finetune/Loss (Raw)": 0.9010170102119446, "Finetune/Step": 4495, "Finetune/Step Time": 4.643134620040655} +{"Finetune/Learning Rate": 9.410950541164143e-07, "Finetune/Loss": 0.8296309113502502, "Finetune/Loss (Raw)": 0.7716273069381714, "Finetune/Step": 4496, "Finetune/Step Time": 4.645313672721386} +{"Finetune/Learning Rate": 9.384584972283184e-07, "Finetune/Loss": 0.8305234313011169, "Finetune/Loss (Raw)": 0.9381641745567322, "Finetune/Step": 4497, "Finetune/Step Time": 4.648505041375756} +{"Finetune/Learning Rate": 9.358254569475689e-07, "Finetune/Loss": 0.8289350867271423, "Finetune/Loss (Raw)": 0.7599155902862549, "Finetune/Step": 4498, "Finetune/Step Time": 4.658646887168288} +{"Finetune/Learning Rate": 9.331959342959951e-07, "Finetune/Loss": 0.8285844326019287, "Finetune/Loss (Raw)": 0.9478505849838257, "Finetune/Step": 4499, "Finetune/Step Time": 4.6585620529949665} +{"Finetune/Learning Rate": 9.305699302940662e-07, "Finetune/Loss": 0.8333258628845215, "Finetune/Loss (Raw)": 0.8572885394096375, "Finetune/Step": 4500, "Finetune/Step Time": 4.632936358451843} +{"Finetune/Learning Rate": 9.279474459608806e-07, "Finetune/Loss": 0.8348231911659241, "Finetune/Loss (Raw)": 0.9476745128631592, "Finetune/Step": 4501, "Finetune/Step Time": 4.625941539183259} +{"Finetune/Learning Rate": 9.253284823141728e-07, "Finetune/Loss": 0.8339508771896362, "Finetune/Loss (Raw)": 0.7236316800117493, "Finetune/Step": 4502, "Finetune/Step Time": 4.630637884140015} +{"Finetune/Learning Rate": 9.227130403703178e-07, "Finetune/Loss": 0.8350968360900879, "Finetune/Loss (Raw)": 0.9480893015861511, "Finetune/Step": 4503, "Finetune/Step Time": 4.620969142764807} +{"Finetune/Learning Rate": 9.201011211443101e-07, "Finetune/Loss": 0.8344844579696655, "Finetune/Loss (Raw)": 0.848829984664917, "Finetune/Step": 4504, "Finetune/Step Time": 4.605726623907685} +{"Finetune/Learning Rate": 9.174927256497846e-07, "Finetune/Loss": 0.8341091871261597, "Finetune/Loss (Raw)": 0.8228382468223572, "Finetune/Step": 4505, "Finetune/Step Time": 4.605200024321675} +{"Finetune/Learning Rate": 9.148878548990137e-07, "Finetune/Loss": 0.8351612687110901, "Finetune/Loss (Raw)": 1.0084270238876343, "Finetune/Step": 4506, "Finetune/Step Time": 4.590234819799662} +{"Finetune/Learning Rate": 9.122865099028944e-07, "Finetune/Loss": 0.831739068031311, "Finetune/Loss (Raw)": 0.32885801792144775, "Finetune/Step": 4507, "Finetune/Step Time": 4.619039973244071} +{"Finetune/Learning Rate": 9.09688691670958e-07, "Finetune/Loss": 0.8308566808700562, "Finetune/Loss (Raw)": 0.869657576084137, "Finetune/Step": 4508, "Finetune/Step Time": 4.612593041732907} +{"Finetune/Learning Rate": 9.070944012113669e-07, "Finetune/Loss": 0.8299523591995239, "Finetune/Loss (Raw)": 0.7882734537124634, "Finetune/Step": 4509, "Finetune/Step Time": 4.6101051196455956} +{"Finetune/Learning Rate": 9.045036395309148e-07, "Finetune/Loss": 0.8302356600761414, "Finetune/Loss (Raw)": 0.9693868160247803, "Finetune/Step": 4510, "Finetune/Step Time": 4.6063021048903465} +{"Finetune/Learning Rate": 9.019164076350262e-07, "Finetune/Loss": 0.8324059247970581, "Finetune/Loss (Raw)": 1.0127516984939575, "Finetune/Step": 4511, "Finetune/Step Time": 4.5934995748102665} +{"Finetune/Learning Rate": 8.993327065277524e-07, "Finetune/Loss": 0.8284014463424683, "Finetune/Loss (Raw)": 0.3602665364742279, "Finetune/Step": 4512, "Finetune/Step Time": 4.626716969534755} +{"Finetune/Learning Rate": 8.967525372117825e-07, "Finetune/Loss": 0.8285995721817017, "Finetune/Loss (Raw)": 0.8344576954841614, "Finetune/Step": 4513, "Finetune/Step Time": 4.631650744006038} +{"Finetune/Learning Rate": 8.941759006884265e-07, "Finetune/Loss": 0.8278419971466064, "Finetune/Loss (Raw)": 0.7589994668960571, "Finetune/Step": 4514, "Finetune/Step Time": 4.613297402858734} +{"Finetune/Learning Rate": 8.916027979576247e-07, "Finetune/Loss": 0.8261377215385437, "Finetune/Loss (Raw)": 0.8419045209884644, "Finetune/Step": 4515, "Finetune/Step Time": 4.616507541388273} +{"Finetune/Learning Rate": 8.890332300179516e-07, "Finetune/Loss": 0.8250967264175415, "Finetune/Loss (Raw)": 0.7712046504020691, "Finetune/Step": 4516, "Finetune/Step Time": 4.610838795080781} +{"Finetune/Learning Rate": 8.864671978666084e-07, "Finetune/Loss": 0.823752760887146, "Finetune/Loss (Raw)": 0.7484960556030273, "Finetune/Step": 4517, "Finetune/Step Time": 4.612424589693546} +{"Finetune/Learning Rate": 8.839047024994152e-07, "Finetune/Loss": 0.8251210451126099, "Finetune/Loss (Raw)": 1.003961205482483, "Finetune/Step": 4518, "Finetune/Step Time": 4.60779689066112} +{"Finetune/Learning Rate": 8.813457449108309e-07, "Finetune/Loss": 0.8258795738220215, "Finetune/Loss (Raw)": 0.9427257776260376, "Finetune/Step": 4519, "Finetune/Step Time": 4.601505858823657} +{"Finetune/Learning Rate": 8.787903260939368e-07, "Finetune/Loss": 0.8303820490837097, "Finetune/Loss (Raw)": 0.9305745363235474, "Finetune/Step": 4520, "Finetune/Step Time": 4.56184889189899} +{"Finetune/Learning Rate": 8.762384470404417e-07, "Finetune/Loss": 0.8323013782501221, "Finetune/Loss (Raw)": 0.9897429347038269, "Finetune/Step": 4521, "Finetune/Step Time": 4.549493696540594} +{"Finetune/Learning Rate": 8.736901087406779e-07, "Finetune/Loss": 0.8324999809265137, "Finetune/Loss (Raw)": 0.8461736440658569, "Finetune/Step": 4522, "Finetune/Step Time": 4.54647602327168} +{"Finetune/Learning Rate": 8.711453121836066e-07, "Finetune/Loss": 0.8327051401138306, "Finetune/Loss (Raw)": 0.8759738206863403, "Finetune/Step": 4523, "Finetune/Step Time": 4.550815682858229} +{"Finetune/Learning Rate": 8.686040583568134e-07, "Finetune/Loss": 0.831820011138916, "Finetune/Loss (Raw)": 0.8525175452232361, "Finetune/Step": 4524, "Finetune/Step Time": 4.545066226273775} +{"Finetune/Learning Rate": 8.660663482465093e-07, "Finetune/Loss": 0.8350434899330139, "Finetune/Loss (Raw)": 0.8317965865135193, "Finetune/Step": 4525, "Finetune/Step Time": 4.50945408642292} +{"Finetune/Learning Rate": 8.63532182837531e-07, "Finetune/Loss": 0.8333216905593872, "Finetune/Loss (Raw)": 0.6718162894248962, "Finetune/Step": 4526, "Finetune/Step Time": 4.505157448351383} +{"Finetune/Learning Rate": 8.610015631133395e-07, "Finetune/Loss": 0.8313037157058716, "Finetune/Loss (Raw)": 0.5475984811782837, "Finetune/Step": 4527, "Finetune/Step Time": 4.523551534861326} +{"Finetune/Learning Rate": 8.584744900560149e-07, "Finetune/Loss": 0.8309139013290405, "Finetune/Loss (Raw)": 0.8389300107955933, "Finetune/Step": 4528, "Finetune/Step Time": 4.514245783910155} +{"Finetune/Learning Rate": 8.559509646462693e-07, "Finetune/Loss": 0.8354957103729248, "Finetune/Loss (Raw)": 0.9213621020317078, "Finetune/Step": 4529, "Finetune/Step Time": 4.480061981827021} +{"Finetune/Learning Rate": 8.534309878634317e-07, "Finetune/Loss": 0.8367063999176025, "Finetune/Loss (Raw)": 0.9685097336769104, "Finetune/Step": 4530, "Finetune/Step Time": 4.4580029640346766} +{"Finetune/Learning Rate": 8.509145606854552e-07, "Finetune/Loss": 0.8373225927352905, "Finetune/Loss (Raw)": 0.9989381432533264, "Finetune/Step": 4531, "Finetune/Step Time": 4.465054972097278} +{"Finetune/Learning Rate": 8.484016840889176e-07, "Finetune/Loss": 0.8345379829406738, "Finetune/Loss (Raw)": 0.6574823260307312, "Finetune/Step": 4532, "Finetune/Step Time": 4.475073235109448} +{"Finetune/Learning Rate": 8.458923590490165e-07, "Finetune/Loss": 0.8350365161895752, "Finetune/Loss (Raw)": 0.8855642676353455, "Finetune/Step": 4533, "Finetune/Step Time": 4.467342380434275} +{"Finetune/Learning Rate": 8.433865865395718e-07, "Finetune/Loss": 0.8317583203315735, "Finetune/Loss (Raw)": 0.3346700668334961, "Finetune/Step": 4534, "Finetune/Step Time": 4.50637437030673} +{"Finetune/Learning Rate": 8.408843675330225e-07, "Finetune/Loss": 0.8315354585647583, "Finetune/Loss (Raw)": 0.7497618794441223, "Finetune/Step": 4535, "Finetune/Step Time": 4.514704978093505} +{"Finetune/Learning Rate": 8.383857030004361e-07, "Finetune/Loss": 0.8303401470184326, "Finetune/Loss (Raw)": 0.866930365562439, "Finetune/Step": 4536, "Finetune/Step Time": 4.5246457029134035} +{"Finetune/Learning Rate": 8.358905939114948e-07, "Finetune/Loss": 0.8307726383209229, "Finetune/Loss (Raw)": 0.873184859752655, "Finetune/Step": 4537, "Finetune/Step Time": 4.5274391155689955} +{"Finetune/Learning Rate": 8.333990412344961e-07, "Finetune/Loss": 0.830916166305542, "Finetune/Loss (Raw)": 0.9017883539199829, "Finetune/Step": 4538, "Finetune/Step Time": 4.526832697913051} +{"Finetune/Learning Rate": 8.30911045936369e-07, "Finetune/Loss": 0.8303085565567017, "Finetune/Loss (Raw)": 0.7750222086906433, "Finetune/Step": 4539, "Finetune/Step Time": 4.534398155286908} +{"Finetune/Learning Rate": 8.284266089826531e-07, "Finetune/Loss": 0.8263301253318787, "Finetune/Loss (Raw)": 0.3114354610443115, "Finetune/Step": 4540, "Finetune/Step Time": 4.572910154238343} +{"Finetune/Learning Rate": 8.259457313375097e-07, "Finetune/Loss": 0.8238303661346436, "Finetune/Loss (Raw)": 0.7678977251052856, "Finetune/Step": 4541, "Finetune/Step Time": 4.594765121117234} +{"Finetune/Learning Rate": 8.234684139637205e-07, "Finetune/Loss": 0.8233588933944702, "Finetune/Loss (Raw)": 0.7481034398078918, "Finetune/Step": 4542, "Finetune/Step Time": 4.59632108733058} +{"Finetune/Learning Rate": 8.209946578226836e-07, "Finetune/Loss": 0.8226057291030884, "Finetune/Loss (Raw)": 0.843339204788208, "Finetune/Step": 4543, "Finetune/Step Time": 4.590141516178846} +{"Finetune/Learning Rate": 8.185244638744139e-07, "Finetune/Loss": 0.8237939476966858, "Finetune/Loss (Raw)": 0.9136380553245544, "Finetune/Step": 4544, "Finetune/Step Time": 4.581511301919818} +{"Finetune/Learning Rate": 8.160578330775459e-07, "Finetune/Loss": 0.8234437704086304, "Finetune/Loss (Raw)": 0.9160866141319275, "Finetune/Step": 4545, "Finetune/Step Time": 4.59366937354207} +{"Finetune/Learning Rate": 8.135947663893329e-07, "Finetune/Loss": 0.824118971824646, "Finetune/Loss (Raw)": 0.943194568157196, "Finetune/Step": 4546, "Finetune/Step Time": 4.598076952621341} +{"Finetune/Learning Rate": 8.111352647656423e-07, "Finetune/Loss": 0.8241268396377563, "Finetune/Loss (Raw)": 0.9255843162536621, "Finetune/Step": 4547, "Finetune/Step Time": 4.591436501592398} +{"Finetune/Learning Rate": 8.086793291609562e-07, "Finetune/Loss": 0.8226812481880188, "Finetune/Loss (Raw)": 0.5800395607948303, "Finetune/Step": 4548, "Finetune/Step Time": 4.637100636959076} +{"Finetune/Learning Rate": 8.062269605283779e-07, "Finetune/Loss": 0.8218080997467041, "Finetune/Loss (Raw)": 0.7270528078079224, "Finetune/Step": 4549, "Finetune/Step Time": 4.632414251565933} +{"Finetune/Learning Rate": 8.037781598196225e-07, "Finetune/Loss": 0.8225765228271484, "Finetune/Loss (Raw)": 0.9900684356689453, "Finetune/Step": 4550, "Finetune/Step Time": 4.629586633294821} +{"Finetune/Learning Rate": 8.013329279850235e-07, "Finetune/Loss": 0.823498547077179, "Finetune/Loss (Raw)": 0.9090330004692078, "Finetune/Step": 4551, "Finetune/Step Time": 4.626916494220495} +{"Finetune/Learning Rate": 7.988912659735249e-07, "Finetune/Loss": 0.8263591527938843, "Finetune/Loss (Raw)": 0.6969414353370667, "Finetune/Step": 4552, "Finetune/Step Time": 4.600496631115675} +{"Finetune/Learning Rate": 7.964531747326898e-07, "Finetune/Loss": 0.8271249532699585, "Finetune/Loss (Raw)": 0.9241623878479004, "Finetune/Step": 4553, "Finetune/Step Time": 4.580755924805999} +{"Finetune/Learning Rate": 7.940186552086937e-07, "Finetune/Loss": 0.8281570076942444, "Finetune/Loss (Raw)": 1.0437324047088623, "Finetune/Step": 4554, "Finetune/Step Time": 4.579479677602649} +{"Finetune/Learning Rate": 7.915877083463242e-07, "Finetune/Loss": 0.8288958668708801, "Finetune/Loss (Raw)": 0.8745068311691284, "Finetune/Step": 4555, "Finetune/Step Time": 4.580518310889602} +{"Finetune/Learning Rate": 7.891603350889888e-07, "Finetune/Loss": 0.8301899433135986, "Finetune/Loss (Raw)": 1.067886471748352, "Finetune/Step": 4556, "Finetune/Step Time": 4.57729566283524} +{"Finetune/Learning Rate": 7.867365363787005e-07, "Finetune/Loss": 0.829531192779541, "Finetune/Loss (Raw)": 0.7834358215332031, "Finetune/Step": 4557, "Finetune/Step Time": 4.574636019766331} +{"Finetune/Learning Rate": 7.843163131560871e-07, "Finetune/Loss": 0.8292511701583862, "Finetune/Loss (Raw)": 0.6570895314216614, "Finetune/Step": 4558, "Finetune/Step Time": 4.547728369012475} +{"Finetune/Learning Rate": 7.818996663603917e-07, "Finetune/Loss": 0.8300639390945435, "Finetune/Loss (Raw)": 0.9710443019866943, "Finetune/Step": 4559, "Finetune/Step Time": 4.548413019627333} +{"Finetune/Learning Rate": 7.794865969294685e-07, "Finetune/Loss": 0.8297606706619263, "Finetune/Loss (Raw)": 0.8457779288291931, "Finetune/Step": 4560, "Finetune/Step Time": 4.544100226834416} +{"Finetune/Learning Rate": 7.770771057997828e-07, "Finetune/Loss": 0.8312065601348877, "Finetune/Loss (Raw)": 0.9032597541809082, "Finetune/Step": 4561, "Finetune/Step Time": 4.533733207732439} +{"Finetune/Learning Rate": 7.746711939064066e-07, "Finetune/Loss": 0.8310372829437256, "Finetune/Loss (Raw)": 0.9407748579978943, "Finetune/Step": 4562, "Finetune/Step Time": 4.534292424097657} +{"Finetune/Learning Rate": 7.722688621830333e-07, "Finetune/Loss": 0.8265920281410217, "Finetune/Loss (Raw)": 0.3048458993434906, "Finetune/Step": 4563, "Finetune/Step Time": 4.570622993633151} +{"Finetune/Learning Rate": 7.698701115619578e-07, "Finetune/Loss": 0.8272944688796997, "Finetune/Loss (Raw)": 0.8591497540473938, "Finetune/Step": 4564, "Finetune/Step Time": 4.548920389264822} +{"Finetune/Learning Rate": 7.674749429740868e-07, "Finetune/Loss": 0.8269850015640259, "Finetune/Loss (Raw)": 0.7892938256263733, "Finetune/Step": 4565, "Finetune/Step Time": 4.54395149089396} +{"Finetune/Learning Rate": 7.650833573489436e-07, "Finetune/Loss": 0.8267018795013428, "Finetune/Loss (Raw)": 0.8381242752075195, "Finetune/Step": 4566, "Finetune/Step Time": 4.547005670145154} +{"Finetune/Learning Rate": 7.626953556146521e-07, "Finetune/Loss": 0.8262218236923218, "Finetune/Loss (Raw)": 0.9628390669822693, "Finetune/Step": 4567, "Finetune/Step Time": 4.545031363144517} +{"Finetune/Learning Rate": 7.603109386979501e-07, "Finetune/Loss": 0.8255975246429443, "Finetune/Loss (Raw)": 0.9259340763092041, "Finetune/Step": 4568, "Finetune/Step Time": 4.542302489280701} +{"Finetune/Learning Rate": 7.579301075241808e-07, "Finetune/Loss": 0.8257339000701904, "Finetune/Loss (Raw)": 0.9967553615570068, "Finetune/Step": 4569, "Finetune/Step Time": 4.54004175029695} +{"Finetune/Learning Rate": 7.555528630173014e-07, "Finetune/Loss": 0.8257323503494263, "Finetune/Loss (Raw)": 0.934454619884491, "Finetune/Step": 4570, "Finetune/Step Time": 4.5317158326506615} +{"Finetune/Learning Rate": 7.531792060998766e-07, "Finetune/Loss": 0.8256742358207703, "Finetune/Loss (Raw)": 0.8327121138572693, "Finetune/Step": 4571, "Finetune/Step Time": 4.549694266170263} +{"Finetune/Learning Rate": 7.508091376930682e-07, "Finetune/Loss": 0.8259557485580444, "Finetune/Loss (Raw)": 0.8537660837173462, "Finetune/Step": 4572, "Finetune/Step Time": 4.54939135722816} +{"Finetune/Learning Rate": 7.484426587166615e-07, "Finetune/Loss": 0.8265571594238281, "Finetune/Loss (Raw)": 0.8066418170928955, "Finetune/Step": 4573, "Finetune/Step Time": 4.569893214851618} +{"Finetune/Learning Rate": 7.460797700890365e-07, "Finetune/Loss": 0.8259369134902954, "Finetune/Loss (Raw)": 0.8998164534568787, "Finetune/Step": 4574, "Finetune/Step Time": 4.568721910938621} +{"Finetune/Learning Rate": 7.437204727271852e-07, "Finetune/Loss": 0.8257054090499878, "Finetune/Loss (Raw)": 0.8133491277694702, "Finetune/Step": 4575, "Finetune/Step Time": 4.58797224983573} +{"Finetune/Learning Rate": 7.41364767546705e-07, "Finetune/Loss": 0.8250772953033447, "Finetune/Loss (Raw)": 0.7939454913139343, "Finetune/Step": 4576, "Finetune/Step Time": 4.5951395239681005} +{"Finetune/Learning Rate": 7.390126554617982e-07, "Finetune/Loss": 0.8250000476837158, "Finetune/Loss (Raw)": 0.9938260316848755, "Finetune/Step": 4577, "Finetune/Step Time": 4.597962444648147} +{"Finetune/Learning Rate": 7.366641373852745e-07, "Finetune/Loss": 0.8263996839523315, "Finetune/Loss (Raw)": 0.954710841178894, "Finetune/Step": 4578, "Finetune/Step Time": 4.601892154663801} +{"Finetune/Learning Rate": 7.343192142285471e-07, "Finetune/Loss": 0.8236295580863953, "Finetune/Loss (Raw)": 0.4157869219779968, "Finetune/Step": 4579, "Finetune/Step Time": 4.628271836787462} +{"Finetune/Learning Rate": 7.319778869016358e-07, "Finetune/Loss": 0.8230007886886597, "Finetune/Loss (Raw)": 0.2674371898174286, "Finetune/Step": 4580, "Finetune/Step Time": 4.628322571516037} +{"Finetune/Learning Rate": 7.296401563131661e-07, "Finetune/Loss": 0.8236918449401855, "Finetune/Loss (Raw)": 0.8475654721260071, "Finetune/Step": 4581, "Finetune/Step Time": 4.637277092784643} +{"Finetune/Learning Rate": 7.273060233703599e-07, "Finetune/Loss": 0.8238799571990967, "Finetune/Loss (Raw)": 0.9040226936340332, "Finetune/Step": 4582, "Finetune/Step Time": 4.6406883392483} +{"Finetune/Learning Rate": 7.249754889790539e-07, "Finetune/Loss": 0.8256348371505737, "Finetune/Loss (Raw)": 0.9686907529830933, "Finetune/Step": 4583, "Finetune/Step Time": 4.626899976283312} +{"Finetune/Learning Rate": 7.2264855404368e-07, "Finetune/Loss": 0.825370192527771, "Finetune/Loss (Raw)": 0.7826695442199707, "Finetune/Step": 4584, "Finetune/Step Time": 4.628082402050495} +{"Finetune/Learning Rate": 7.203252194672783e-07, "Finetune/Loss": 0.8250173330307007, "Finetune/Loss (Raw)": 0.740054726600647, "Finetune/Step": 4585, "Finetune/Step Time": 4.616243747994304} +{"Finetune/Learning Rate": 7.180054861514885e-07, "Finetune/Loss": 0.8243077993392944, "Finetune/Loss (Raw)": 0.7480834126472473, "Finetune/Step": 4586, "Finetune/Step Time": 4.607673618942499} +{"Finetune/Learning Rate": 7.156893549965538e-07, "Finetune/Loss": 0.8248514533042908, "Finetune/Loss (Raw)": 0.9141458868980408, "Finetune/Step": 4587, "Finetune/Step Time": 4.603609271347523} +{"Finetune/Learning Rate": 7.133768269013186e-07, "Finetune/Loss": 0.825431227684021, "Finetune/Loss (Raw)": 1.033286690711975, "Finetune/Step": 4588, "Finetune/Step Time": 4.626854101195931} +{"Finetune/Learning Rate": 7.110679027632295e-07, "Finetune/Loss": 0.8257855772972107, "Finetune/Loss (Raw)": 0.8403546214103699, "Finetune/Step": 4589, "Finetune/Step Time": 4.627000404521823} +{"Finetune/Learning Rate": 7.087625834783385e-07, "Finetune/Loss": 0.8249981999397278, "Finetune/Loss (Raw)": 0.815093994140625, "Finetune/Step": 4590, "Finetune/Step Time": 4.617192333564162} +{"Finetune/Learning Rate": 7.064608699412944e-07, "Finetune/Loss": 0.8269025683403015, "Finetune/Loss (Raw)": 0.910398006439209, "Finetune/Step": 4591, "Finetune/Step Time": 4.601205620914698} +{"Finetune/Learning Rate": 7.041627630453418e-07, "Finetune/Loss": 0.82252037525177, "Finetune/Loss (Raw)": 0.2927982211112976, "Finetune/Step": 4592, "Finetune/Step Time": 4.620369533076882} +{"Finetune/Learning Rate": 7.018682636823371e-07, "Finetune/Loss": 0.8234120607376099, "Finetune/Loss (Raw)": 0.9153107404708862, "Finetune/Step": 4593, "Finetune/Step Time": 4.625582171604037} +{"Finetune/Learning Rate": 6.99577372742728e-07, "Finetune/Loss": 0.8201038837432861, "Finetune/Loss (Raw)": 0.4119518995285034, "Finetune/Step": 4594, "Finetune/Step Time": 4.662582792341709} +{"Finetune/Learning Rate": 6.972900911155656e-07, "Finetune/Loss": 0.8202043771743774, "Finetune/Loss (Raw)": 0.9239891767501831, "Finetune/Step": 4595, "Finetune/Step Time": 4.664923623204231} +{"Finetune/Learning Rate": 6.950064196884987e-07, "Finetune/Loss": 0.8209187984466553, "Finetune/Loss (Raw)": 0.9460073113441467, "Finetune/Step": 4596, "Finetune/Step Time": 4.671993790194392} +{"Finetune/Learning Rate": 6.927263593477751e-07, "Finetune/Loss": 0.8202620148658752, "Finetune/Loss (Raw)": 0.7562988996505737, "Finetune/Step": 4597, "Finetune/Step Time": 4.668398765847087} +{"Finetune/Learning Rate": 6.904499109782426e-07, "Finetune/Loss": 0.8204801678657532, "Finetune/Loss (Raw)": 0.8689063191413879, "Finetune/Step": 4598, "Finetune/Step Time": 4.670958835631609} +{"Finetune/Learning Rate": 6.881770754633444e-07, "Finetune/Loss": 0.8205512762069702, "Finetune/Loss (Raw)": 0.8046611547470093, "Finetune/Step": 4599, "Finetune/Step Time": 4.664813173934817} +{"Finetune/Learning Rate": 6.859078536851283e-07, "Finetune/Loss": 0.8202470541000366, "Finetune/Loss (Raw)": 0.8360992074012756, "Finetune/Step": 4600, "Finetune/Step Time": 4.668680179864168} +{"Finetune/Learning Rate": 6.836422465242298e-07, "Finetune/Loss": 0.8177995681762695, "Finetune/Loss (Raw)": 0.7256142497062683, "Finetune/Step": 4601, "Finetune/Step Time": 4.673491854220629} +{"Finetune/Learning Rate": 6.81380254859888e-07, "Finetune/Loss": 0.81879723072052, "Finetune/Loss (Raw)": 1.0357022285461426, "Finetune/Step": 4602, "Finetune/Step Time": 4.6752368826419115} +{"Finetune/Learning Rate": 6.791218795699406e-07, "Finetune/Loss": 0.819819986820221, "Finetune/Loss (Raw)": 0.9117210507392883, "Finetune/Step": 4603, "Finetune/Step Time": 4.6676484905183315} +{"Finetune/Learning Rate": 6.768671215308165e-07, "Finetune/Loss": 0.8191697597503662, "Finetune/Loss (Raw)": 0.8421056270599365, "Finetune/Step": 4604, "Finetune/Step Time": 4.661379557102919} +{"Finetune/Learning Rate": 6.746159816175435e-07, "Finetune/Loss": 0.8155404925346375, "Finetune/Loss (Raw)": 0.3318500220775604, "Finetune/Step": 4605, "Finetune/Step Time": 4.697896691039205} +{"Finetune/Learning Rate": 6.723684607037451e-07, "Finetune/Loss": 0.8151103258132935, "Finetune/Loss (Raw)": 0.8907395005226135, "Finetune/Step": 4606, "Finetune/Step Time": 4.705460010096431} +{"Finetune/Learning Rate": 6.701245596616402e-07, "Finetune/Loss": 0.8146471381187439, "Finetune/Loss (Raw)": 0.8217813968658447, "Finetune/Step": 4607, "Finetune/Step Time": 4.697274252772331} +{"Finetune/Learning Rate": 6.678842793620433e-07, "Finetune/Loss": 0.814744770526886, "Finetune/Loss (Raw)": 0.8519558310508728, "Finetune/Step": 4608, "Finetune/Step Time": 4.69336392916739} +{"Finetune/Learning Rate": 6.656476206743623e-07, "Finetune/Loss": 0.8162316679954529, "Finetune/Loss (Raw)": 1.0391038656234741, "Finetune/Step": 4609, "Finetune/Step Time": 4.6922337375581264} +{"Finetune/Learning Rate": 6.634145844666051e-07, "Finetune/Loss": 0.8163528442382812, "Finetune/Loss (Raw)": 0.8228907585144043, "Finetune/Step": 4610, "Finetune/Step Time": 4.674974367022514} +{"Finetune/Learning Rate": 6.611851716053641e-07, "Finetune/Loss": 0.8118467330932617, "Finetune/Loss (Raw)": 0.2861044406890869, "Finetune/Step": 4611, "Finetune/Step Time": 4.706851344555616} +{"Finetune/Learning Rate": 6.589593829558316e-07, "Finetune/Loss": 0.8116036057472229, "Finetune/Loss (Raw)": 0.8807262182235718, "Finetune/Step": 4612, "Finetune/Step Time": 4.688534941524267} +{"Finetune/Learning Rate": 6.567372193817967e-07, "Finetune/Loss": 0.8113043308258057, "Finetune/Loss (Raw)": 0.8130897879600525, "Finetune/Step": 4613, "Finetune/Step Time": 4.713039183989167} +{"Finetune/Learning Rate": 6.545186817456339e-07, "Finetune/Loss": 0.810515820980072, "Finetune/Loss (Raw)": 0.7669931054115295, "Finetune/Step": 4614, "Finetune/Step Time": 4.71931597776711} +{"Finetune/Learning Rate": 6.523037709083158e-07, "Finetune/Loss": 0.8154895305633545, "Finetune/Loss (Raw)": 0.9614413976669312, "Finetune/Step": 4615, "Finetune/Step Time": 4.681814305484295} +{"Finetune/Learning Rate": 6.500924877294057e-07, "Finetune/Loss": 0.8145284652709961, "Finetune/Loss (Raw)": 0.8260478377342224, "Finetune/Step": 4616, "Finetune/Step Time": 4.679114604368806} +{"Finetune/Learning Rate": 6.478848330670607e-07, "Finetune/Loss": 0.8152391910552979, "Finetune/Loss (Raw)": 0.7682027816772461, "Finetune/Step": 4617, "Finetune/Step Time": 4.6711594220250845} +{"Finetune/Learning Rate": 6.456808077780263e-07, "Finetune/Loss": 0.8163290619850159, "Finetune/Loss (Raw)": 0.8812413215637207, "Finetune/Step": 4618, "Finetune/Step Time": 4.662509860470891} +{"Finetune/Learning Rate": 6.434804127176409e-07, "Finetune/Loss": 0.8180907964706421, "Finetune/Loss (Raw)": 1.0187069177627563, "Finetune/Step": 4619, "Finetune/Step Time": 4.664499193429947} +{"Finetune/Learning Rate": 6.4128364873984e-07, "Finetune/Loss": 0.8174242973327637, "Finetune/Loss (Raw)": 0.825899064540863, "Finetune/Step": 4620, "Finetune/Step Time": 4.668572282418609} +{"Finetune/Learning Rate": 6.390905166971406e-07, "Finetune/Loss": 0.8188128471374512, "Finetune/Loss (Raw)": 0.999538242816925, "Finetune/Step": 4621, "Finetune/Step Time": 4.658208820968866} +{"Finetune/Learning Rate": 6.369010174406531e-07, "Finetune/Loss": 0.8203108310699463, "Finetune/Loss (Raw)": 0.8858358860015869, "Finetune/Step": 4622, "Finetune/Step Time": 4.654827049002051} +{"Finetune/Learning Rate": 6.347151518200834e-07, "Finetune/Loss": 0.8208838701248169, "Finetune/Loss (Raw)": 0.9743672609329224, "Finetune/Step": 4623, "Finetune/Step Time": 4.646744767203927} +{"Finetune/Learning Rate": 6.325329206837217e-07, "Finetune/Loss": 0.8214051723480225, "Finetune/Loss (Raw)": 0.8383477926254272, "Finetune/Step": 4624, "Finetune/Step Time": 4.639399413019419} +{"Finetune/Learning Rate": 6.303543248784483e-07, "Finetune/Loss": 0.8211226463317871, "Finetune/Loss (Raw)": 0.9020068645477295, "Finetune/Step": 4625, "Finetune/Step Time": 4.645364174619317} +{"Finetune/Learning Rate": 6.281793652497359e-07, "Finetune/Loss": 0.8179036974906921, "Finetune/Loss (Raw)": 0.34788423776626587, "Finetune/Step": 4626, "Finetune/Step Time": 4.672177519649267} +{"Finetune/Learning Rate": 6.260080426416415e-07, "Finetune/Loss": 0.8173805475234985, "Finetune/Loss (Raw)": 0.8808925151824951, "Finetune/Step": 4627, "Finetune/Step Time": 4.6821010652929544} +{"Finetune/Learning Rate": 6.238403578968144e-07, "Finetune/Loss": 0.8168333768844604, "Finetune/Loss (Raw)": 0.787246584892273, "Finetune/Step": 4628, "Finetune/Step Time": 4.6860610488802195} +{"Finetune/Learning Rate": 6.216763118564883e-07, "Finetune/Loss": 0.816074013710022, "Finetune/Loss (Raw)": 0.8504840731620789, "Finetune/Step": 4629, "Finetune/Step Time": 4.678732126951218} +{"Finetune/Learning Rate": 6.195159053604926e-07, "Finetune/Loss": 0.8166592717170715, "Finetune/Loss (Raw)": 0.7985363006591797, "Finetune/Step": 4630, "Finetune/Step Time": 4.686983708292246} +{"Finetune/Learning Rate": 6.173591392472334e-07, "Finetune/Loss": 0.8148019313812256, "Finetune/Loss (Raw)": 0.710351824760437, "Finetune/Step": 4631, "Finetune/Step Time": 4.705762542784214} +{"Finetune/Learning Rate": 6.152060143537109e-07, "Finetune/Loss": 0.8102642297744751, "Finetune/Loss (Raw)": 0.2680077850818634, "Finetune/Step": 4632, "Finetune/Step Time": 4.738254075869918} +{"Finetune/Learning Rate": 6.130565315155124e-07, "Finetune/Loss": 0.8103599548339844, "Finetune/Loss (Raw)": 0.835094690322876, "Finetune/Step": 4633, "Finetune/Step Time": 4.750749185681343} +{"Finetune/Learning Rate": 6.109106915668106e-07, "Finetune/Loss": 0.8095787763595581, "Finetune/Loss (Raw)": 0.9084217548370361, "Finetune/Step": 4634, "Finetune/Step Time": 4.752088602632284} +{"Finetune/Learning Rate": 6.087684953403616e-07, "Finetune/Loss": 0.8123456835746765, "Finetune/Loss (Raw)": 0.6830282211303711, "Finetune/Step": 4635, "Finetune/Step Time": 4.7393970768898726} +{"Finetune/Learning Rate": 6.066299436675117e-07, "Finetune/Loss": 0.8122230768203735, "Finetune/Loss (Raw)": 0.8539667129516602, "Finetune/Step": 4636, "Finetune/Step Time": 4.7414918672293425} +{"Finetune/Learning Rate": 6.04495037378191e-07, "Finetune/Loss": 0.8130726218223572, "Finetune/Loss (Raw)": 0.8970140814781189, "Finetune/Step": 4637, "Finetune/Step Time": 4.735683901235461} +{"Finetune/Learning Rate": 6.023637773009139e-07, "Finetune/Loss": 0.81402987241745, "Finetune/Loss (Raw)": 1.091910719871521, "Finetune/Step": 4638, "Finetune/Step Time": 4.739637657999992} +{"Finetune/Learning Rate": 6.002361642627807e-07, "Finetune/Loss": 0.8116897940635681, "Finetune/Loss (Raw)": 0.7132235765457153, "Finetune/Step": 4639, "Finetune/Step Time": 4.769648674875498} +{"Finetune/Learning Rate": 5.981121990894789e-07, "Finetune/Loss": 0.8164201974868774, "Finetune/Loss (Raw)": 0.9657541513442993, "Finetune/Step": 4640, "Finetune/Step Time": 4.737855589017272} +{"Finetune/Learning Rate": 5.959918826052757e-07, "Finetune/Loss": 0.8166933059692383, "Finetune/Loss (Raw)": 0.869424045085907, "Finetune/Step": 4641, "Finetune/Step Time": 4.7362691424787045} +{"Finetune/Learning Rate": 5.938752156330241e-07, "Finetune/Loss": 0.817618727684021, "Finetune/Loss (Raw)": 0.8774481415748596, "Finetune/Step": 4642, "Finetune/Step Time": 4.737393552437425} +{"Finetune/Learning Rate": 5.917621989941624e-07, "Finetune/Loss": 0.8183544874191284, "Finetune/Loss (Raw)": 0.9360835552215576, "Finetune/Step": 4643, "Finetune/Step Time": 4.740068459883332} +{"Finetune/Learning Rate": 5.896528335087137e-07, "Finetune/Loss": 0.8152427673339844, "Finetune/Loss (Raw)": 0.37290656566619873, "Finetune/Step": 4644, "Finetune/Step Time": 4.7793878726661205} +{"Finetune/Learning Rate": 5.875471199952754e-07, "Finetune/Loss": 0.8171493411064148, "Finetune/Loss (Raw)": 0.9925363659858704, "Finetune/Step": 4645, "Finetune/Step Time": 4.777582116425037} +{"Finetune/Learning Rate": 5.854450592710382e-07, "Finetune/Loss": 0.8152538537979126, "Finetune/Loss (Raw)": 0.7613440752029419, "Finetune/Step": 4646, "Finetune/Step Time": 4.790537416934967} +{"Finetune/Learning Rate": 5.833466521517695e-07, "Finetune/Loss": 0.8148705959320068, "Finetune/Loss (Raw)": 0.8936691880226135, "Finetune/Step": 4647, "Finetune/Step Time": 4.794766198843718} +{"Finetune/Learning Rate": 5.812518994518213e-07, "Finetune/Loss": 0.8148186206817627, "Finetune/Loss (Raw)": 0.9239147901535034, "Finetune/Step": 4648, "Finetune/Step Time": 4.799648331478238} +{"Finetune/Learning Rate": 5.791608019841244e-07, "Finetune/Loss": 0.8145252466201782, "Finetune/Loss (Raw)": 0.9521937370300293, "Finetune/Step": 4649, "Finetune/Step Time": 4.795944321900606} +{"Finetune/Learning Rate": 5.770733605601953e-07, "Finetune/Loss": 0.8140985369682312, "Finetune/Loss (Raw)": 0.791550874710083, "Finetune/Step": 4650, "Finetune/Step Time": 4.80361801572144} +{"Finetune/Learning Rate": 5.74989575990127e-07, "Finetune/Loss": 0.8140695095062256, "Finetune/Loss (Raw)": 0.8722614049911499, "Finetune/Step": 4651, "Finetune/Step Time": 4.79682058095932} +{"Finetune/Learning Rate": 5.729094490825948e-07, "Finetune/Loss": 0.8140097856521606, "Finetune/Loss (Raw)": 0.8448711037635803, "Finetune/Step": 4652, "Finetune/Step Time": 4.795090472325683} +{"Finetune/Learning Rate": 5.708329806448587e-07, "Finetune/Loss": 0.8098591566085815, "Finetune/Loss (Raw)": 0.30052268505096436, "Finetune/Step": 4653, "Finetune/Step Time": 4.83051985502243} +{"Finetune/Learning Rate": 5.687601714827551e-07, "Finetune/Loss": 0.8114088773727417, "Finetune/Loss (Raw)": 0.870176374912262, "Finetune/Step": 4654, "Finetune/Step Time": 4.835501682013273} +{"Finetune/Learning Rate": 5.666910224006971e-07, "Finetune/Loss": 0.8126357197761536, "Finetune/Loss (Raw)": 0.7046334147453308, "Finetune/Step": 4655, "Finetune/Step Time": 4.819936521351337} +{"Finetune/Learning Rate": 5.646255342016848e-07, "Finetune/Loss": 0.8094780445098877, "Finetune/Loss (Raw)": 0.4347406327724457, "Finetune/Step": 4656, "Finetune/Step Time": 4.859982939437032} +{"Finetune/Learning Rate": 5.625637076872914e-07, "Finetune/Loss": 0.8088616728782654, "Finetune/Loss (Raw)": 0.8424719572067261, "Finetune/Step": 4657, "Finetune/Step Time": 4.857125245034695} +{"Finetune/Learning Rate": 5.605055436576746e-07, "Finetune/Loss": 0.8086419105529785, "Finetune/Loss (Raw)": 0.9403784871101379, "Finetune/Step": 4658, "Finetune/Step Time": 4.852988580241799} +{"Finetune/Learning Rate": 5.584510429115641e-07, "Finetune/Loss": 0.8080482482910156, "Finetune/Loss (Raw)": 0.9229550361633301, "Finetune/Step": 4659, "Finetune/Step Time": 4.849685238674283} +{"Finetune/Learning Rate": 5.564002062462737e-07, "Finetune/Loss": 0.810285210609436, "Finetune/Loss (Raw)": 0.9438167810440063, "Finetune/Step": 4660, "Finetune/Step Time": 4.856420697644353} +{"Finetune/Learning Rate": 5.54353034457693e-07, "Finetune/Loss": 0.809587836265564, "Finetune/Loss (Raw)": 0.7962926626205444, "Finetune/Step": 4661, "Finetune/Step Time": 4.858100615441799} +{"Finetune/Learning Rate": 5.523095283402868e-07, "Finetune/Loss": 0.813988983631134, "Finetune/Loss (Raw)": 0.8980188369750977, "Finetune/Step": 4662, "Finetune/Step Time": 4.8194733168929815} +{"Finetune/Learning Rate": 5.502696886871039e-07, "Finetune/Loss": 0.8153179883956909, "Finetune/Loss (Raw)": 0.9198726415634155, "Finetune/Step": 4663, "Finetune/Step Time": 4.808383060619235} +{"Finetune/Learning Rate": 5.482335162897657e-07, "Finetune/Loss": 0.8152900338172913, "Finetune/Loss (Raw)": 0.8633483648300171, "Finetune/Step": 4664, "Finetune/Step Time": 4.796185849234462} +{"Finetune/Learning Rate": 5.462010119384665e-07, "Finetune/Loss": 0.8147218227386475, "Finetune/Loss (Raw)": 0.8004611730575562, "Finetune/Step": 4665, "Finetune/Step Time": 4.790967043489218} +{"Finetune/Learning Rate": 5.441721764219853e-07, "Finetune/Loss": 0.8134033679962158, "Finetune/Loss (Raw)": 0.7330257296562195, "Finetune/Step": 4666, "Finetune/Step Time": 4.810546822845936} +{"Finetune/Learning Rate": 5.42147010527675e-07, "Finetune/Loss": 0.814944863319397, "Finetune/Loss (Raw)": 0.972335159778595, "Finetune/Step": 4667, "Finetune/Step Time": 4.797431567683816} +{"Finetune/Learning Rate": 5.401255150414598e-07, "Finetune/Loss": 0.8191251754760742, "Finetune/Loss (Raw)": 0.8465126752853394, "Finetune/Step": 4668, "Finetune/Step Time": 4.775443539023399} +{"Finetune/Learning Rate": 5.381076907478433e-07, "Finetune/Loss": 0.8158619403839111, "Finetune/Loss (Raw)": 0.35020527243614197, "Finetune/Step": 4669, "Finetune/Step Time": 4.793547973036766} +{"Finetune/Learning Rate": 5.360935384299049e-07, "Finetune/Loss": 0.8170561790466309, "Finetune/Loss (Raw)": 0.9009670615196228, "Finetune/Step": 4670, "Finetune/Step Time": 4.792798954993486} +{"Finetune/Learning Rate": 5.34083058869297e-07, "Finetune/Loss": 0.8168354034423828, "Finetune/Loss (Raw)": 0.8150772452354431, "Finetune/Step": 4671, "Finetune/Step Time": 4.788244940340519} +{"Finetune/Learning Rate": 5.320762528462453e-07, "Finetune/Loss": 0.8165913820266724, "Finetune/Loss (Raw)": 0.8823949098587036, "Finetune/Step": 4672, "Finetune/Step Time": 4.8022720515728} +{"Finetune/Learning Rate": 5.30073121139556e-07, "Finetune/Loss": 0.8160040378570557, "Finetune/Loss (Raw)": 0.8409157395362854, "Finetune/Step": 4673, "Finetune/Step Time": 4.790989125147462} +{"Finetune/Learning Rate": 5.28073664526606e-07, "Finetune/Loss": 0.8160371780395508, "Finetune/Loss (Raw)": 0.9474349617958069, "Finetune/Step": 4674, "Finetune/Step Time": 4.809284634888172} +{"Finetune/Learning Rate": 5.260778837833391e-07, "Finetune/Loss": 0.8157446384429932, "Finetune/Loss (Raw)": 0.8881419897079468, "Finetune/Step": 4675, "Finetune/Step Time": 4.811302652582526} +{"Finetune/Learning Rate": 5.240857796842846e-07, "Finetune/Loss": 0.8177610039710999, "Finetune/Loss (Raw)": 0.8381359577178955, "Finetune/Step": 4676, "Finetune/Step Time": 4.748890979215503} +{"Finetune/Learning Rate": 5.220973530025375e-07, "Finetune/Loss": 0.819198727607727, "Finetune/Loss (Raw)": 0.9110766649246216, "Finetune/Step": 4677, "Finetune/Step Time": 4.73881028778851} +{"Finetune/Learning Rate": 5.201126045097671e-07, "Finetune/Loss": 0.8187476396560669, "Finetune/Loss (Raw)": 0.9323342442512512, "Finetune/Step": 4678, "Finetune/Step Time": 4.764376685023308} +{"Finetune/Learning Rate": 5.181315349762162e-07, "Finetune/Loss": 0.8174515962600708, "Finetune/Loss (Raw)": 0.7431389689445496, "Finetune/Step": 4679, "Finetune/Step Time": 4.763762721791863} +{"Finetune/Learning Rate": 5.161541451706975e-07, "Finetune/Loss": 0.8198753595352173, "Finetune/Loss (Raw)": 1.0071759223937988, "Finetune/Step": 4680, "Finetune/Step Time": 4.755257157608867} +{"Finetune/Learning Rate": 5.141804358605995e-07, "Finetune/Loss": 0.8195469975471497, "Finetune/Loss (Raw)": 0.8821347951889038, "Finetune/Step": 4681, "Finetune/Step Time": 4.772058621048927} +{"Finetune/Learning Rate": 5.122104078118773e-07, "Finetune/Loss": 0.8184500932693481, "Finetune/Loss (Raw)": 0.9033271670341492, "Finetune/Step": 4682, "Finetune/Step Time": 4.783177884295583} +{"Finetune/Learning Rate": 5.102440617890658e-07, "Finetune/Loss": 0.8188045024871826, "Finetune/Loss (Raw)": 0.9198674559593201, "Finetune/Step": 4683, "Finetune/Step Time": 4.767456714063883} +{"Finetune/Learning Rate": 5.082813985552604e-07, "Finetune/Loss": 0.8178518414497375, "Finetune/Loss (Raw)": 0.9459531903266907, "Finetune/Step": 4684, "Finetune/Step Time": 4.77115505002439} +{"Finetune/Learning Rate": 5.063224188721327e-07, "Finetune/Loss": 0.818915069103241, "Finetune/Loss (Raw)": 0.919522762298584, "Finetune/Step": 4685, "Finetune/Step Time": 4.773162085562944} +{"Finetune/Learning Rate": 5.043671234999271e-07, "Finetune/Loss": 0.819611132144928, "Finetune/Loss (Raw)": 0.7461894750595093, "Finetune/Step": 4686, "Finetune/Step Time": 4.771362509578466} +{"Finetune/Learning Rate": 5.02415513197454e-07, "Finetune/Loss": 0.818947434425354, "Finetune/Loss (Raw)": 0.8860946297645569, "Finetune/Step": 4687, "Finetune/Step Time": 4.757362497970462} +{"Finetune/Learning Rate": 5.004675887220978e-07, "Finetune/Loss": 0.8208873271942139, "Finetune/Loss (Raw)": 1.0940749645233154, "Finetune/Step": 4688, "Finetune/Step Time": 4.75459641776979} +{"Finetune/Learning Rate": 4.985233508298037e-07, "Finetune/Loss": 0.821588397026062, "Finetune/Loss (Raw)": 0.99300616979599, "Finetune/Step": 4689, "Finetune/Step Time": 4.746512455865741} +{"Finetune/Learning Rate": 4.965828002750994e-07, "Finetune/Loss": 0.8204268217086792, "Finetune/Loss (Raw)": 0.792090654373169, "Finetune/Step": 4690, "Finetune/Step Time": 4.760208772495389} +{"Finetune/Learning Rate": 4.946459378110701e-07, "Finetune/Loss": 0.8256352543830872, "Finetune/Loss (Raw)": 0.971523642539978, "Finetune/Step": 4691, "Finetune/Step Time": 4.723959231749177} +{"Finetune/Learning Rate": 4.927127641893759e-07, "Finetune/Loss": 0.8269098997116089, "Finetune/Loss (Raw)": 1.0223075151443481, "Finetune/Step": 4692, "Finetune/Step Time": 4.724858975037932} +{"Finetune/Learning Rate": 4.907832801602452e-07, "Finetune/Loss": 0.8277736902236938, "Finetune/Loss (Raw)": 0.8998531103134155, "Finetune/Step": 4693, "Finetune/Step Time": 4.736383635550737} +{"Finetune/Learning Rate": 4.888574864724715e-07, "Finetune/Loss": 0.8289680480957031, "Finetune/Loss (Raw)": 0.9910101890563965, "Finetune/Step": 4694, "Finetune/Step Time": 4.724766205996275} +{"Finetune/Learning Rate": 4.869353838734169e-07, "Finetune/Loss": 0.8291803598403931, "Finetune/Loss (Raw)": 0.9900104403495789, "Finetune/Step": 4695, "Finetune/Step Time": 4.729566102847457} +{"Finetune/Learning Rate": 4.850169731090127e-07, "Finetune/Loss": 0.8281388878822327, "Finetune/Loss (Raw)": 0.7926241755485535, "Finetune/Step": 4696, "Finetune/Step Time": 4.7522218599915504} +{"Finetune/Learning Rate": 4.831022549237574e-07, "Finetune/Loss": 0.8272192478179932, "Finetune/Loss (Raw)": 0.8790455460548401, "Finetune/Step": 4697, "Finetune/Step Time": 4.752768028527498} +{"Finetune/Learning Rate": 4.811912300607169e-07, "Finetune/Loss": 0.8271670341491699, "Finetune/Loss (Raw)": 0.9277668595314026, "Finetune/Step": 4698, "Finetune/Step Time": 4.7514400109648705} +{"Finetune/Learning Rate": 4.792838992615179e-07, "Finetune/Loss": 0.8282114267349243, "Finetune/Loss (Raw)": 0.9663942456245422, "Finetune/Step": 4699, "Finetune/Step Time": 4.74694306589663} +{"Finetune/Learning Rate": 4.773802632663627e-07, "Finetune/Loss": 0.8284807801246643, "Finetune/Loss (Raw)": 0.8882443308830261, "Finetune/Step": 4700, "Finetune/Step Time": 4.742417162284255} +{"Finetune/Learning Rate": 4.7548032281401367e-07, "Finetune/Loss": 0.8293189406394958, "Finetune/Loss (Raw)": 0.9139246344566345, "Finetune/Step": 4701, "Finetune/Step Time": 4.725725075230002} +{"Finetune/Learning Rate": 4.735840786418011e-07, "Finetune/Loss": 0.8288931846618652, "Finetune/Loss (Raw)": 0.8453211784362793, "Finetune/Step": 4702, "Finetune/Step Time": 4.733532253652811} +{"Finetune/Learning Rate": 4.716915314856196e-07, "Finetune/Loss": 0.830109715461731, "Finetune/Loss (Raw)": 0.9690716862678528, "Finetune/Step": 4703, "Finetune/Step Time": 4.711295301094651} +{"Finetune/Learning Rate": 4.698026820799306e-07, "Finetune/Loss": 0.8297263383865356, "Finetune/Loss (Raw)": 0.7448647022247314, "Finetune/Step": 4704, "Finetune/Step Time": 4.705553259700537} +{"Finetune/Learning Rate": 4.6791753115775884e-07, "Finetune/Loss": 0.8294762372970581, "Finetune/Loss (Raw)": 0.9618197679519653, "Finetune/Step": 4705, "Finetune/Step Time": 4.706287797540426} +{"Finetune/Learning Rate": 4.660360794506946e-07, "Finetune/Loss": 0.8284658193588257, "Finetune/Loss (Raw)": 0.8253742456436157, "Finetune/Step": 4706, "Finetune/Step Time": 4.699620760977268} +{"Finetune/Learning Rate": 4.641583276888939e-07, "Finetune/Loss": 0.8311042189598083, "Finetune/Loss (Raw)": 0.7534989714622498, "Finetune/Step": 4707, "Finetune/Step Time": 4.664217511191964} +{"Finetune/Learning Rate": 4.622842766010749e-07, "Finetune/Loss": 0.8350322246551514, "Finetune/Loss (Raw)": 0.7702184915542603, "Finetune/Step": 4708, "Finetune/Step Time": 4.627140322700143} +{"Finetune/Learning Rate": 4.604139269145191e-07, "Finetune/Loss": 0.8341997861862183, "Finetune/Loss (Raw)": 0.7410151958465576, "Finetune/Step": 4709, "Finetune/Step Time": 4.631499418988824} +{"Finetune/Learning Rate": 4.585472793550738e-07, "Finetune/Loss": 0.8349552154541016, "Finetune/Loss (Raw)": 1.0007272958755493, "Finetune/Step": 4710, "Finetune/Step Time": 4.622007969766855} +{"Finetune/Learning Rate": 4.5668433464714835e-07, "Finetune/Loss": 0.8340980410575867, "Finetune/Loss (Raw)": 0.8589650988578796, "Finetune/Step": 4711, "Finetune/Step Time": 4.628013972193003} +{"Finetune/Learning Rate": 4.548250935137144e-07, "Finetune/Loss": 0.8342435359954834, "Finetune/Loss (Raw)": 0.8012949228286743, "Finetune/Step": 4712, "Finetune/Step Time": 4.640897214412689} +{"Finetune/Learning Rate": 4.5296955667630926e-07, "Finetune/Loss": 0.8359099626541138, "Finetune/Loss (Raw)": 0.953359842300415, "Finetune/Step": 4713, "Finetune/Step Time": 4.643328422680497} +{"Finetune/Learning Rate": 4.51117724855028e-07, "Finetune/Loss": 0.8365191221237183, "Finetune/Loss (Raw)": 0.8260484337806702, "Finetune/Step": 4714, "Finetune/Step Time": 4.646497840061784} +{"Finetune/Learning Rate": 4.492695987685314e-07, "Finetune/Loss": 0.836530327796936, "Finetune/Loss (Raw)": 0.9155812859535217, "Finetune/Step": 4715, "Finetune/Step Time": 4.649990184232593} +{"Finetune/Learning Rate": 4.474251791340412e-07, "Finetune/Loss": 0.8351202607154846, "Finetune/Loss (Raw)": 0.8528004884719849, "Finetune/Step": 4716, "Finetune/Step Time": 4.626175429672003} +{"Finetune/Learning Rate": 4.4558446666734147e-07, "Finetune/Loss": 0.8355166912078857, "Finetune/Loss (Raw)": 0.8910970091819763, "Finetune/Step": 4717, "Finetune/Step Time": 4.623512499034405} +{"Finetune/Learning Rate": 4.437474620827764e-07, "Finetune/Loss": 0.8342664241790771, "Finetune/Loss (Raw)": 0.6550595164299011, "Finetune/Step": 4718, "Finetune/Step Time": 4.637032236903906} +{"Finetune/Learning Rate": 4.419141660932491e-07, "Finetune/Loss": 0.8330938220024109, "Finetune/Loss (Raw)": 0.7603042125701904, "Finetune/Step": 4719, "Finetune/Step Time": 4.636737244203687} +{"Finetune/Learning Rate": 4.4008457941022933e-07, "Finetune/Loss": 0.8354712724685669, "Finetune/Loss (Raw)": 0.5971102714538574, "Finetune/Step": 4720, "Finetune/Step Time": 4.635192269459367} +{"Finetune/Learning Rate": 4.382587027437435e-07, "Finetune/Loss": 0.8345745801925659, "Finetune/Loss (Raw)": 0.8005350828170776, "Finetune/Step": 4721, "Finetune/Step Time": 4.639482472091913} +{"Finetune/Learning Rate": 4.364365368023771e-07, "Finetune/Loss": 0.8379592299461365, "Finetune/Loss (Raw)": 0.8451870679855347, "Finetune/Step": 4722, "Finetune/Step Time": 4.603272467851639} +{"Finetune/Learning Rate": 4.3461808229327773e-07, "Finetune/Loss": 0.8376436829566956, "Finetune/Loss (Raw)": 0.8835982084274292, "Finetune/Step": 4723, "Finetune/Step Time": 4.601909661665559} +{"Finetune/Learning Rate": 4.3280333992215206e-07, "Finetune/Loss": 0.8366521596908569, "Finetune/Loss (Raw)": 0.819092333316803, "Finetune/Step": 4724, "Finetune/Step Time": 4.623586688190699} +{"Finetune/Learning Rate": 4.3099231039326785e-07, "Finetune/Loss": 0.8374450206756592, "Finetune/Loss (Raw)": 0.8577930331230164, "Finetune/Step": 4725, "Finetune/Step Time": 4.625888055190444} +{"Finetune/Learning Rate": 4.291849944094462e-07, "Finetune/Loss": 0.8365318179130554, "Finetune/Loss (Raw)": 0.752012312412262, "Finetune/Step": 4726, "Finetune/Step Time": 4.623119503259659} +{"Finetune/Learning Rate": 4.273813926720771e-07, "Finetune/Loss": 0.8367090225219727, "Finetune/Loss (Raw)": 0.8273417353630066, "Finetune/Step": 4727, "Finetune/Step Time": 4.619457274675369} +{"Finetune/Learning Rate": 4.2558150588109747e-07, "Finetune/Loss": 0.8371646404266357, "Finetune/Loss (Raw)": 0.8944147229194641, "Finetune/Step": 4728, "Finetune/Step Time": 4.622050570324063} +{"Finetune/Learning Rate": 4.237853347350107e-07, "Finetune/Loss": 0.8397473096847534, "Finetune/Loss (Raw)": 1.0561983585357666, "Finetune/Step": 4729, "Finetune/Step Time": 4.61885080486536} +{"Finetune/Learning Rate": 4.219928799308759e-07, "Finetune/Loss": 0.8385214805603027, "Finetune/Loss (Raw)": 0.878795325756073, "Finetune/Step": 4730, "Finetune/Step Time": 4.613860448822379} +{"Finetune/Learning Rate": 4.2020414216431106e-07, "Finetune/Loss": 0.8376493453979492, "Finetune/Loss (Raw)": 0.8000885844230652, "Finetune/Step": 4731, "Finetune/Step Time": 4.6176370326429605} +{"Finetune/Learning Rate": 4.184191221294875e-07, "Finetune/Loss": 0.8377184271812439, "Finetune/Loss (Raw)": 0.8509502410888672, "Finetune/Step": 4732, "Finetune/Step Time": 4.617861617356539} +{"Finetune/Learning Rate": 4.16637820519139e-07, "Finetune/Loss": 0.8423014879226685, "Finetune/Loss (Raw)": 0.9184831380844116, "Finetune/Step": 4733, "Finetune/Step Time": 4.575594594702125} +{"Finetune/Learning Rate": 4.1486023802455253e-07, "Finetune/Loss": 0.8415400385856628, "Finetune/Loss (Raw)": 0.7932709455490112, "Finetune/Step": 4734, "Finetune/Step Time": 4.582475945353508} +{"Finetune/Learning Rate": 4.1308637533557294e-07, "Finetune/Loss": 0.8425931930541992, "Finetune/Loss (Raw)": 0.9565810561180115, "Finetune/Step": 4735, "Finetune/Step Time": 4.582945272326469} +{"Finetune/Learning Rate": 4.1131623314060175e-07, "Finetune/Loss": 0.8414944410324097, "Finetune/Loss (Raw)": 0.711317777633667, "Finetune/Step": 4736, "Finetune/Step Time": 4.592856803908944} +{"Finetune/Learning Rate": 4.095498121265995e-07, "Finetune/Loss": 0.8395394682884216, "Finetune/Loss (Raw)": 0.7888684868812561, "Finetune/Step": 4737, "Finetune/Step Time": 4.602148523554206} +{"Finetune/Learning Rate": 4.077871129790767e-07, "Finetune/Loss": 0.840859055519104, "Finetune/Loss (Raw)": 0.9918034076690674, "Finetune/Step": 4738, "Finetune/Step Time": 4.611001331359148} +{"Finetune/Learning Rate": 4.060281363821017e-07, "Finetune/Loss": 0.844752311706543, "Finetune/Loss (Raw)": 0.7844407558441162, "Finetune/Step": 4739, "Finetune/Step Time": 4.576860288158059} +{"Finetune/Learning Rate": 4.042728830183018e-07, "Finetune/Loss": 0.8443262577056885, "Finetune/Loss (Raw)": 0.8261942267417908, "Finetune/Step": 4740, "Finetune/Step Time": 4.580116510391235} +{"Finetune/Learning Rate": 4.025213535688544e-07, "Finetune/Loss": 0.8434137105941772, "Finetune/Loss (Raw)": 0.6962800621986389, "Finetune/Step": 4741, "Finetune/Step Time": 4.562366021797061} +{"Finetune/Learning Rate": 4.007735487134956e-07, "Finetune/Loss": 0.8434758186340332, "Finetune/Loss (Raw)": 0.7749418020248413, "Finetune/Step": 4742, "Finetune/Step Time": 4.573135904967785} +{"Finetune/Learning Rate": 3.99029469130513e-07, "Finetune/Loss": 0.8416777849197388, "Finetune/Loss (Raw)": 0.7312942147254944, "Finetune/Step": 4743, "Finetune/Step Time": 4.588447978720069} +{"Finetune/Learning Rate": 3.972891154967506e-07, "Finetune/Loss": 0.8430908918380737, "Finetune/Loss (Raw)": 1.0069289207458496, "Finetune/Step": 4744, "Finetune/Step Time": 4.584305122494698} +{"Finetune/Learning Rate": 3.95552488487605e-07, "Finetune/Loss": 0.8431688547134399, "Finetune/Loss (Raw)": 0.7781720161437988, "Finetune/Step": 4745, "Finetune/Step Time": 4.603551143780351} +{"Finetune/Learning Rate": 3.9381958877702684e-07, "Finetune/Loss": 0.8424573540687561, "Finetune/Loss (Raw)": 0.7901730537414551, "Finetune/Step": 4746, "Finetune/Step Time": 4.627478279173374} +{"Finetune/Learning Rate": 3.920904170375239e-07, "Finetune/Loss": 0.842507004737854, "Finetune/Loss (Raw)": 1.0250663757324219, "Finetune/Step": 4747, "Finetune/Step Time": 4.627052808180451} +{"Finetune/Learning Rate": 3.903649739401494e-07, "Finetune/Loss": 0.8433681726455688, "Finetune/Loss (Raw)": 0.936121940612793, "Finetune/Step": 4748, "Finetune/Step Time": 4.622278798371553} +{"Finetune/Learning Rate": 3.8864326015451536e-07, "Finetune/Loss": 0.8421391248703003, "Finetune/Loss (Raw)": 0.8422225117683411, "Finetune/Step": 4749, "Finetune/Step Time": 4.618987655267119} +{"Finetune/Learning Rate": 3.8692527634878853e-07, "Finetune/Loss": 0.8419448137283325, "Finetune/Loss (Raw)": 0.8609657883644104, "Finetune/Step": 4750, "Finetune/Step Time": 4.627160018309951} +{"Finetune/Learning Rate": 3.8521102318968106e-07, "Finetune/Loss": 0.8412520885467529, "Finetune/Loss (Raw)": 0.8856968879699707, "Finetune/Step": 4751, "Finetune/Step Time": 4.642909279093146} +{"Finetune/Learning Rate": 3.835005013424631e-07, "Finetune/Loss": 0.8417159914970398, "Finetune/Loss (Raw)": 0.8977268934249878, "Finetune/Step": 4752, "Finetune/Step Time": 4.646444017067552} +{"Finetune/Learning Rate": 3.8179371147095354e-07, "Finetune/Loss": 0.8406672477722168, "Finetune/Loss (Raw)": 0.7677735686302185, "Finetune/Step": 4753, "Finetune/Step Time": 4.632563075050712} +{"Finetune/Learning Rate": 3.80090654237526e-07, "Finetune/Loss": 0.8451030850410461, "Finetune/Loss (Raw)": 0.9156666994094849, "Finetune/Step": 4754, "Finetune/Step Time": 4.600554434582591} +{"Finetune/Learning Rate": 3.7839133030310174e-07, "Finetune/Loss": 0.8455063104629517, "Finetune/Loss (Raw)": 0.9325065612792969, "Finetune/Step": 4755, "Finetune/Step Time": 4.600438332185149} +{"Finetune/Learning Rate": 3.766957403271554e-07, "Finetune/Loss": 0.846211314201355, "Finetune/Loss (Raw)": 0.8774839639663696, "Finetune/Step": 4756, "Finetune/Step Time": 4.593523815274239} +{"Finetune/Learning Rate": 3.750038849677162e-07, "Finetune/Loss": 0.8463841080665588, "Finetune/Loss (Raw)": 0.8726030588150024, "Finetune/Step": 4757, "Finetune/Step Time": 4.5921860337257385} +{"Finetune/Learning Rate": 3.7331576488135456e-07, "Finetune/Loss": 0.8467574715614319, "Finetune/Loss (Raw)": 0.846331000328064, "Finetune/Step": 4758, "Finetune/Step Time": 4.582284478470683} +{"Finetune/Learning Rate": 3.716313807231986e-07, "Finetune/Loss": 0.8477334380149841, "Finetune/Loss (Raw)": 0.8352695107460022, "Finetune/Step": 4759, "Finetune/Step Time": 4.5679054241627455} +{"Finetune/Learning Rate": 3.699507331469276e-07, "Finetune/Loss": 0.8524355888366699, "Finetune/Loss (Raw)": 0.8698855638504028, "Finetune/Step": 4760, "Finetune/Step Time": 4.532980736345053} +{"Finetune/Learning Rate": 3.682738228047655e-07, "Finetune/Loss": 0.8518472909927368, "Finetune/Loss (Raw)": 0.759795069694519, "Finetune/Step": 4761, "Finetune/Step Time": 4.540885886177421} +{"Finetune/Learning Rate": 3.666006503474895e-07, "Finetune/Loss": 0.8509212136268616, "Finetune/Loss (Raw)": 0.7898820638656616, "Finetune/Step": 4762, "Finetune/Step Time": 4.5393145233392715} +{"Finetune/Learning Rate": 3.6493121642442454e-07, "Finetune/Loss": 0.8524196743965149, "Finetune/Loss (Raw)": 0.8748335838317871, "Finetune/Step": 4763, "Finetune/Step Time": 4.519171396270394} +{"Finetune/Learning Rate": 3.6326552168344574e-07, "Finetune/Loss": 0.852392852306366, "Finetune/Loss (Raw)": 0.8505315184593201, "Finetune/Step": 4764, "Finetune/Step Time": 4.528555065393448} +{"Finetune/Learning Rate": 3.616035667709761e-07, "Finetune/Loss": 0.8514183759689331, "Finetune/Loss (Raw)": 0.7722839117050171, "Finetune/Step": 4765, "Finetune/Step Time": 4.528347758576274} +{"Finetune/Learning Rate": 3.5994535233198844e-07, "Finetune/Loss": 0.8509178161621094, "Finetune/Loss (Raw)": 1.0278325080871582, "Finetune/Step": 4766, "Finetune/Step Time": 4.5300883539021015} +{"Finetune/Learning Rate": 3.582908790100037e-07, "Finetune/Loss": 0.852264940738678, "Finetune/Loss (Raw)": 0.8856602907180786, "Finetune/Step": 4767, "Finetune/Step Time": 4.500618927180767} +{"Finetune/Learning Rate": 3.566401474470904e-07, "Finetune/Loss": 0.8510423302650452, "Finetune/Loss (Raw)": 0.809260904788971, "Finetune/Step": 4768, "Finetune/Step Time": 4.505423981696367} +{"Finetune/Learning Rate": 3.5499315828386393e-07, "Finetune/Loss": 0.8506565093994141, "Finetune/Loss (Raw)": 0.8200376033782959, "Finetune/Step": 4769, "Finetune/Step Time": 4.504390347748995} +{"Finetune/Learning Rate": 3.533499121594919e-07, "Finetune/Loss": 0.8487549424171448, "Finetune/Loss (Raw)": 0.6340466141700745, "Finetune/Step": 4770, "Finetune/Step Time": 4.508340816944838} +{"Finetune/Learning Rate": 3.5171040971168414e-07, "Finetune/Loss": 0.8491631746292114, "Finetune/Loss (Raw)": 0.9883452653884888, "Finetune/Step": 4771, "Finetune/Step Time": 4.5061244033277035} +{"Finetune/Learning Rate": 3.500746515766973e-07, "Finetune/Loss": 0.8522709608078003, "Finetune/Loss (Raw)": 0.7706936001777649, "Finetune/Step": 4772, "Finetune/Step Time": 4.480587800964713} +{"Finetune/Learning Rate": 3.4844263838934023e-07, "Finetune/Loss": 0.849366307258606, "Finetune/Loss (Raw)": 0.6207428574562073, "Finetune/Step": 4773, "Finetune/Step Time": 4.480385906994343} +{"Finetune/Learning Rate": 3.468143707829641e-07, "Finetune/Loss": 0.8503684997558594, "Finetune/Loss (Raw)": 0.88962322473526, "Finetune/Step": 4774, "Finetune/Step Time": 4.470877222716808} +{"Finetune/Learning Rate": 3.45189849389469e-07, "Finetune/Loss": 0.8509469032287598, "Finetune/Loss (Raw)": 0.9677047729492188, "Finetune/Step": 4775, "Finetune/Step Time": 4.467916492372751} +{"Finetune/Learning Rate": 3.4356907483929835e-07, "Finetune/Loss": 0.8499953150749207, "Finetune/Loss (Raw)": 0.8021122217178345, "Finetune/Step": 4776, "Finetune/Step Time": 4.480273714289069} +{"Finetune/Learning Rate": 3.419520477614435e-07, "Finetune/Loss": 0.8494779467582703, "Finetune/Loss (Raw)": 0.8859728574752808, "Finetune/Step": 4777, "Finetune/Step Time": 4.480939019471407} +{"Finetune/Learning Rate": 3.403387687834425e-07, "Finetune/Loss": 0.8455557227134705, "Finetune/Loss (Raw)": 0.28950515389442444, "Finetune/Step": 4778, "Finetune/Step Time": 4.508230067789555} +{"Finetune/Learning Rate": 3.3872923853137453e-07, "Finetune/Loss": 0.8466708660125732, "Finetune/Loss (Raw)": 1.0149973630905151, "Finetune/Step": 4779, "Finetune/Step Time": 4.509652700275183} +{"Finetune/Learning Rate": 3.371234576298699e-07, "Finetune/Loss": 0.8466286659240723, "Finetune/Loss (Raw)": 0.8394696116447449, "Finetune/Step": 4780, "Finetune/Step Time": 4.514651274308562} +{"Finetune/Learning Rate": 3.355214267021012e-07, "Finetune/Loss": 0.8505463600158691, "Finetune/Loss (Raw)": 0.8019861578941345, "Finetune/Step": 4781, "Finetune/Step Time": 4.485129043459892} +{"Finetune/Learning Rate": 3.3392314636978117e-07, "Finetune/Loss": 0.8502620458602905, "Finetune/Loss (Raw)": 0.8337888121604919, "Finetune/Step": 4782, "Finetune/Step Time": 4.486691968515515} +{"Finetune/Learning Rate": 3.323286172531748e-07, "Finetune/Loss": 0.8476132154464722, "Finetune/Loss (Raw)": 0.3655836582183838, "Finetune/Step": 4783, "Finetune/Step Time": 4.519717695191503} +{"Finetune/Learning Rate": 3.30737839971087e-07, "Finetune/Loss": 0.8510881066322327, "Finetune/Loss (Raw)": 0.8795279860496521, "Finetune/Step": 4784, "Finetune/Step Time": 4.48089543916285} +{"Finetune/Learning Rate": 3.2915081514086624e-07, "Finetune/Loss": 0.8526469469070435, "Finetune/Loss (Raw)": 1.0419970750808716, "Finetune/Step": 4785, "Finetune/Step Time": 4.482591554522514} +{"Finetune/Learning Rate": 3.2756754337840757e-07, "Finetune/Loss": 0.8522695302963257, "Finetune/Loss (Raw)": 0.8920762538909912, "Finetune/Step": 4786, "Finetune/Step Time": 4.48477103933692} +{"Finetune/Learning Rate": 3.2598802529814734e-07, "Finetune/Loss": 0.8517166972160339, "Finetune/Loss (Raw)": 0.8521921038627625, "Finetune/Step": 4787, "Finetune/Step Time": 4.4800967033952475} +{"Finetune/Learning Rate": 3.2441226151306403e-07, "Finetune/Loss": 0.8507921695709229, "Finetune/Loss (Raw)": 0.8254724144935608, "Finetune/Step": 4788, "Finetune/Step Time": 4.48907757550478} +{"Finetune/Learning Rate": 3.228402526346819e-07, "Finetune/Loss": 0.850162148475647, "Finetune/Loss (Raw)": 0.7156512141227722, "Finetune/Step": 4789, "Finetune/Step Time": 4.488814624026418} +{"Finetune/Learning Rate": 3.212719992730684e-07, "Finetune/Loss": 0.8505469560623169, "Finetune/Loss (Raw)": 0.9472687840461731, "Finetune/Step": 4790, "Finetune/Step Time": 4.4943742118775845} +{"Finetune/Learning Rate": 3.197075020368312e-07, "Finetune/Loss": 0.845725417137146, "Finetune/Loss (Raw)": 0.30271920561790466, "Finetune/Step": 4791, "Finetune/Step Time": 4.532833959907293} +{"Finetune/Learning Rate": 3.1814676153311907e-07, "Finetune/Loss": 0.8469488620758057, "Finetune/Loss (Raw)": 1.0199493169784546, "Finetune/Step": 4792, "Finetune/Step Time": 4.537209520116448} +{"Finetune/Learning Rate": 3.165897783676275e-07, "Finetune/Loss": 0.8478748798370361, "Finetune/Loss (Raw)": 0.9189929962158203, "Finetune/Step": 4793, "Finetune/Step Time": 4.5410313829779625} +{"Finetune/Learning Rate": 3.1503655314458983e-07, "Finetune/Loss": 0.8484976887702942, "Finetune/Loss (Raw)": 0.8127462863922119, "Finetune/Step": 4794, "Finetune/Step Time": 4.522144261747599} +{"Finetune/Learning Rate": 3.134870864667827e-07, "Finetune/Loss": 0.8470805883407593, "Finetune/Loss (Raw)": 0.7909463047981262, "Finetune/Step": 4795, "Finetune/Step Time": 4.5541819129139185} +{"Finetune/Learning Rate": 3.1194137893552523e-07, "Finetune/Loss": 0.8478143215179443, "Finetune/Loss (Raw)": 0.9404333233833313, "Finetune/Step": 4796, "Finetune/Step Time": 4.542394721880555} +{"Finetune/Learning Rate": 3.1039943115067417e-07, "Finetune/Loss": 0.8522238731384277, "Finetune/Loss (Raw)": 0.9146196246147156, "Finetune/Step": 4797, "Finetune/Step Time": 4.504900611937046} +{"Finetune/Learning Rate": 3.088612437106309e-07, "Finetune/Loss": 0.8506515026092529, "Finetune/Loss (Raw)": 0.6997010111808777, "Finetune/Step": 4798, "Finetune/Step Time": 4.5212560798972845} +{"Finetune/Learning Rate": 3.0732681721233446e-07, "Finetune/Loss": 0.8511276245117188, "Finetune/Loss (Raw)": 0.8760239481925964, "Finetune/Step": 4799, "Finetune/Step Time": 4.526214813813567} +{"Finetune/Learning Rate": 3.057961522512676e-07, "Finetune/Loss": 0.8524776101112366, "Finetune/Loss (Raw)": 1.0551979541778564, "Finetune/Step": 4800, "Finetune/Step Time": 4.509935023263097} +{"Finetune/Learning Rate": 3.0426924942145166e-07, "Finetune/Loss": 0.8531516790390015, "Finetune/Loss (Raw)": 0.9271964430809021, "Finetune/Step": 4801, "Finetune/Step Time": 4.502876445651054} +{"Finetune/Learning Rate": 3.027461093154449e-07, "Finetune/Loss": 0.8526370525360107, "Finetune/Loss (Raw)": 0.8815553188323975, "Finetune/Step": 4802, "Finetune/Step Time": 4.4769182857126} +{"Finetune/Learning Rate": 3.012267325243512e-07, "Finetune/Loss": 0.8528391718864441, "Finetune/Loss (Raw)": 0.9140188097953796, "Finetune/Step": 4803, "Finetune/Step Time": 4.481908995658159} +{"Finetune/Learning Rate": 2.9971111963780994e-07, "Finetune/Loss": 0.8533518314361572, "Finetune/Loss (Raw)": 0.9037619829177856, "Finetune/Step": 4804, "Finetune/Step Time": 4.484915418550372} +{"Finetune/Learning Rate": 2.981992712440007e-07, "Finetune/Loss": 0.85365891456604, "Finetune/Loss (Raw)": 0.9503746032714844, "Finetune/Step": 4805, "Finetune/Step Time": 4.4914609752595425} +{"Finetune/Learning Rate": 2.96691187929643e-07, "Finetune/Loss": 0.8533865213394165, "Finetune/Loss (Raw)": 0.8974677920341492, "Finetune/Step": 4806, "Finetune/Step Time": 4.472335835918784} +{"Finetune/Learning Rate": 2.9518687027999427e-07, "Finetune/Loss": 0.8541916012763977, "Finetune/Loss (Raw)": 0.8461936712265015, "Finetune/Step": 4807, "Finetune/Step Time": 4.473782705143094} +{"Finetune/Learning Rate": 2.936863188788497e-07, "Finetune/Loss": 0.8538423776626587, "Finetune/Loss (Raw)": 0.9624776840209961, "Finetune/Step": 4808, "Finetune/Step Time": 4.470312297344208} +{"Finetune/Learning Rate": 2.921895343085446e-07, "Finetune/Loss": 0.852164089679718, "Finetune/Loss (Raw)": 0.6673083305358887, "Finetune/Step": 4809, "Finetune/Step Time": 4.459166707471013} +{"Finetune/Learning Rate": 2.9069651714995426e-07, "Finetune/Loss": 0.851603090763092, "Finetune/Loss (Raw)": 0.8315244317054749, "Finetune/Step": 4810, "Finetune/Step Time": 4.449827091768384} +{"Finetune/Learning Rate": 2.8920726798248645e-07, "Finetune/Loss": 0.85140061378479, "Finetune/Loss (Raw)": 0.8939518928527832, "Finetune/Step": 4811, "Finetune/Step Time": 4.465957894921303} +{"Finetune/Learning Rate": 2.8772178738408985e-07, "Finetune/Loss": 0.8508456945419312, "Finetune/Loss (Raw)": 0.8749156594276428, "Finetune/Step": 4812, "Finetune/Step Time": 4.460066270083189} +{"Finetune/Learning Rate": 2.862400759312522e-07, "Finetune/Loss": 0.8473755717277527, "Finetune/Loss (Raw)": 0.475350946187973, "Finetune/Step": 4813, "Finetune/Step Time": 4.493246691301465} +{"Finetune/Learning Rate": 2.847621341989959e-07, "Finetune/Loss": 0.848362922668457, "Finetune/Loss (Raw)": 0.8725659847259521, "Finetune/Step": 4814, "Finetune/Step Time": 4.483609359711409} +{"Finetune/Learning Rate": 2.832879627608831e-07, "Finetune/Loss": 0.8488138914108276, "Finetune/Loss (Raw)": 0.9438251852989197, "Finetune/Step": 4815, "Finetune/Step Time": 4.481090290471911} +{"Finetune/Learning Rate": 2.8181756218900626e-07, "Finetune/Loss": 0.8466598987579346, "Finetune/Loss (Raw)": 0.8183560967445374, "Finetune/Step": 4816, "Finetune/Step Time": 4.502323875203729} +{"Finetune/Learning Rate": 2.803509330540033e-07, "Finetune/Loss": 0.8459123373031616, "Finetune/Loss (Raw)": 0.897322952747345, "Finetune/Step": 4817, "Finetune/Step Time": 4.50106381252408} +{"Finetune/Learning Rate": 2.788880759250423e-07, "Finetune/Loss": 0.8472696542739868, "Finetune/Loss (Raw)": 0.9658313393592834, "Finetune/Step": 4818, "Finetune/Step Time": 4.491442248225212} +{"Finetune/Learning Rate": 2.7742899136983026e-07, "Finetune/Loss": 0.8471325635910034, "Finetune/Loss (Raw)": 0.9539727568626404, "Finetune/Step": 4819, "Finetune/Step Time": 4.490711402148008} +{"Finetune/Learning Rate": 2.759736799546109e-07, "Finetune/Loss": 0.8462172150611877, "Finetune/Loss (Raw)": 0.9051443934440613, "Finetune/Step": 4820, "Finetune/Step Time": 4.4928463362157345} +{"Finetune/Learning Rate": 2.745221422441602e-07, "Finetune/Loss": 0.8465821146965027, "Finetune/Loss (Raw)": 0.9465590715408325, "Finetune/Step": 4821, "Finetune/Step Time": 4.480248177424073} +{"Finetune/Learning Rate": 2.73074378801792e-07, "Finetune/Loss": 0.8457396030426025, "Finetune/Loss (Raw)": 0.8831630945205688, "Finetune/Step": 4822, "Finetune/Step Time": 4.488686507567763} +{"Finetune/Learning Rate": 2.7163039018935357e-07, "Finetune/Loss": 0.8453471064567566, "Finetune/Loss (Raw)": 0.9397738575935364, "Finetune/Step": 4823, "Finetune/Step Time": 4.487343966960907} +{"Finetune/Learning Rate": 2.701901769672322e-07, "Finetune/Loss": 0.8414581418037415, "Finetune/Loss (Raw)": 0.2948382794857025, "Finetune/Step": 4824, "Finetune/Step Time": 4.504387181252241} +{"Finetune/Learning Rate": 2.687537396943451e-07, "Finetune/Loss": 0.841411828994751, "Finetune/Loss (Raw)": 0.8731104135513306, "Finetune/Step": 4825, "Finetune/Step Time": 4.508676148951054} +{"Finetune/Learning Rate": 2.673210789281433e-07, "Finetune/Loss": 0.8412676453590393, "Finetune/Loss (Raw)": 0.9093207716941833, "Finetune/Step": 4826, "Finetune/Step Time": 4.507649499922991} +{"Finetune/Learning Rate": 2.658921952246163e-07, "Finetune/Loss": 0.8410859704017639, "Finetune/Loss (Raw)": 0.9431352615356445, "Finetune/Step": 4827, "Finetune/Step Time": 4.502358365803957} +{"Finetune/Learning Rate": 2.6446708913828725e-07, "Finetune/Loss": 0.8415676355361938, "Finetune/Loss (Raw)": 0.9498988389968872, "Finetune/Step": 4828, "Finetune/Step Time": 4.509572541341186} +{"Finetune/Learning Rate": 2.6304576122221035e-07, "Finetune/Loss": 0.8409897089004517, "Finetune/Loss (Raw)": 0.8399554491043091, "Finetune/Step": 4829, "Finetune/Step Time": 4.515741027891636} +{"Finetune/Learning Rate": 2.616282120279745e-07, "Finetune/Loss": 0.8413248658180237, "Finetune/Loss (Raw)": 0.8882164359092712, "Finetune/Step": 4830, "Finetune/Step Time": 4.5137662924826145} +{"Finetune/Learning Rate": 2.602144421057051e-07, "Finetune/Loss": 0.8365278840065002, "Finetune/Loss (Raw)": 0.3550602197647095, "Finetune/Step": 4831, "Finetune/Step Time": 4.554677115753293} +{"Finetune/Learning Rate": 2.5880445200405667e-07, "Finetune/Loss": 0.8365890979766846, "Finetune/Loss (Raw)": 0.7526997923851013, "Finetune/Step": 4832, "Finetune/Step Time": 4.5779980812221766} +{"Finetune/Learning Rate": 2.573982422702193e-07, "Finetune/Loss": 0.8356741666793823, "Finetune/Loss (Raw)": 0.8447068929672241, "Finetune/Step": 4833, "Finetune/Step Time": 4.581167398020625} +{"Finetune/Learning Rate": 2.559958134499163e-07, "Finetune/Loss": 0.8369826078414917, "Finetune/Loss (Raw)": 0.9928502440452576, "Finetune/Step": 4834, "Finetune/Step Time": 4.585677105933428} +{"Finetune/Learning Rate": 2.545971660874036e-07, "Finetune/Loss": 0.8384279012680054, "Finetune/Loss (Raw)": 0.9385005235671997, "Finetune/Step": 4835, "Finetune/Step Time": 4.588191261515021} +{"Finetune/Learning Rate": 2.532023007254658e-07, "Finetune/Loss": 0.8399516940116882, "Finetune/Loss (Raw)": 0.9652662873268127, "Finetune/Step": 4836, "Finetune/Step Time": 4.586395241320133} +{"Finetune/Learning Rate": 2.518112179054266e-07, "Finetune/Loss": 0.8409352898597717, "Finetune/Loss (Raw)": 0.8669171929359436, "Finetune/Step": 4837, "Finetune/Step Time": 4.568406272679567} +{"Finetune/Learning Rate": 2.504239181671353e-07, "Finetune/Loss": 0.8406480550765991, "Finetune/Loss (Raw)": 0.9639559984207153, "Finetune/Step": 4838, "Finetune/Step Time": 4.569332551211119} +{"Finetune/Learning Rate": 2.490404020489767e-07, "Finetune/Loss": 0.8403522968292236, "Finetune/Loss (Raw)": 0.821111261844635, "Finetune/Step": 4839, "Finetune/Step Time": 4.565346969291568} +{"Finetune/Learning Rate": 2.476606700878681e-07, "Finetune/Loss": 0.8411194086074829, "Finetune/Loss (Raw)": 0.8994888663291931, "Finetune/Step": 4840, "Finetune/Step Time": 4.555741667747498} +{"Finetune/Learning Rate": 2.4628472281925443e-07, "Finetune/Loss": 0.8400522470474243, "Finetune/Loss (Raw)": 0.8167558312416077, "Finetune/Step": 4841, "Finetune/Step Time": 4.5617381278425455} +{"Finetune/Learning Rate": 2.449125607771141e-07, "Finetune/Loss": 0.8402644395828247, "Finetune/Loss (Raw)": 0.8532139658927917, "Finetune/Step": 4842, "Finetune/Step Time": 4.571979900822043} +{"Finetune/Learning Rate": 2.435441844939557e-07, "Finetune/Loss": 0.8412951231002808, "Finetune/Loss (Raw)": 1.0475006103515625, "Finetune/Step": 4843, "Finetune/Step Time": 4.568064749240875} +{"Finetune/Learning Rate": 2.4217959450082207e-07, "Finetune/Loss": 0.8411529064178467, "Finetune/Loss (Raw)": 0.8346050977706909, "Finetune/Step": 4844, "Finetune/Step Time": 4.593023527413607} +{"Finetune/Learning Rate": 2.408187913272841e-07, "Finetune/Loss": 0.8420647382736206, "Finetune/Loss (Raw)": 1.0078096389770508, "Finetune/Step": 4845, "Finetune/Step Time": 4.594985697418451} +{"Finetune/Learning Rate": 2.3946177550143835e-07, "Finetune/Loss": 0.8443595170974731, "Finetune/Loss (Raw)": 0.948788583278656, "Finetune/Step": 4846, "Finetune/Step Time": 4.60624866001308} +{"Finetune/Learning Rate": 2.3810854754992008e-07, "Finetune/Loss": 0.840944766998291, "Finetune/Loss (Raw)": 0.32321926951408386, "Finetune/Step": 4847, "Finetune/Step Time": 4.644898112863302} +{"Finetune/Learning Rate": 2.3675910799789036e-07, "Finetune/Loss": 0.842257559299469, "Finetune/Loss (Raw)": 0.765145480632782, "Finetune/Step": 4848, "Finetune/Step Time": 4.607478808611631} +{"Finetune/Learning Rate": 2.354134573690392e-07, "Finetune/Loss": 0.841932475566864, "Finetune/Loss (Raw)": 0.7589223980903625, "Finetune/Step": 4849, "Finetune/Step Time": 4.622841868549585} +{"Finetune/Learning Rate": 2.3407159618558772e-07, "Finetune/Loss": 0.8427965641021729, "Finetune/Loss (Raw)": 0.9557938575744629, "Finetune/Step": 4850, "Finetune/Step Time": 4.6316124722361565} +{"Finetune/Learning Rate": 2.3273352496828605e-07, "Finetune/Loss": 0.8430262804031372, "Finetune/Loss (Raw)": 0.9130025506019592, "Finetune/Step": 4851, "Finetune/Step Time": 4.636895380914211} +{"Finetune/Learning Rate": 2.3139924423641325e-07, "Finetune/Loss": 0.8430904150009155, "Finetune/Loss (Raw)": 0.8272984623908997, "Finetune/Step": 4852, "Finetune/Step Time": 4.62937786616385} +{"Finetune/Learning Rate": 2.3006875450777733e-07, "Finetune/Loss": 0.8442869186401367, "Finetune/Loss (Raw)": 1.0109459161758423, "Finetune/Step": 4853, "Finetune/Step Time": 4.636081857606769} +{"Finetune/Learning Rate": 2.287420562987175e-07, "Finetune/Loss": 0.8432633876800537, "Finetune/Loss (Raw)": 0.6209954619407654, "Finetune/Step": 4854, "Finetune/Step Time": 4.638979263603687} +{"Finetune/Learning Rate": 2.2741915012409744e-07, "Finetune/Loss": 0.8439129590988159, "Finetune/Loss (Raw)": 0.9104927778244019, "Finetune/Step": 4855, "Finetune/Step Time": 4.640380851924419} +{"Finetune/Learning Rate": 2.2610003649731093e-07, "Finetune/Loss": 0.8439378142356873, "Finetune/Loss (Raw)": 0.8975987434387207, "Finetune/Step": 4856, "Finetune/Step Time": 4.64063522964716} +{"Finetune/Learning Rate": 2.2478471593028073e-07, "Finetune/Loss": 0.8432214260101318, "Finetune/Loss (Raw)": 0.9644911289215088, "Finetune/Step": 4857, "Finetune/Step Time": 4.639266924932599} +{"Finetune/Learning Rate": 2.234731889334585e-07, "Finetune/Loss": 0.8441224098205566, "Finetune/Loss (Raw)": 0.9941267371177673, "Finetune/Step": 4858, "Finetune/Step Time": 4.649236541241407} +{"Finetune/Learning Rate": 2.221654560158204e-07, "Finetune/Loss": 0.8448238372802734, "Finetune/Loss (Raw)": 0.8898692727088928, "Finetune/Step": 4859, "Finetune/Step Time": 4.630412856116891} +{"Finetune/Learning Rate": 2.2086151768487273e-07, "Finetune/Loss": 0.8443617820739746, "Finetune/Loss (Raw)": 0.791806697845459, "Finetune/Step": 4860, "Finetune/Step Time": 4.629034917801619} +{"Finetune/Learning Rate": 2.195613744466485e-07, "Finetune/Loss": 0.8439126014709473, "Finetune/Loss (Raw)": 0.8609952330589294, "Finetune/Step": 4861, "Finetune/Step Time": 4.643937481567264} +{"Finetune/Learning Rate": 2.182650268057085e-07, "Finetune/Loss": 0.8452503085136414, "Finetune/Loss (Raw)": 0.9644913673400879, "Finetune/Step": 4862, "Finetune/Step Time": 4.628815395757556} +{"Finetune/Learning Rate": 2.1697247526513698e-07, "Finetune/Loss": 0.8454799652099609, "Finetune/Loss (Raw)": 0.9859849810600281, "Finetune/Step": 4863, "Finetune/Step Time": 4.633145933970809} +{"Finetune/Learning Rate": 2.1568372032655384e-07, "Finetune/Loss": 0.8471596240997314, "Finetune/Loss (Raw)": 0.9263104200363159, "Finetune/Step": 4864, "Finetune/Step Time": 4.638459989801049} +{"Finetune/Learning Rate": 2.143987624900945e-07, "Finetune/Loss": 0.848106861114502, "Finetune/Loss (Raw)": 0.9101201295852661, "Finetune/Step": 4865, "Finetune/Step Time": 4.632968723773956} +{"Finetune/Learning Rate": 2.1311760225442678e-07, "Finetune/Loss": 0.8463952541351318, "Finetune/Loss (Raw)": 0.7727084159851074, "Finetune/Step": 4866, "Finetune/Step Time": 4.624866483733058} +{"Finetune/Learning Rate": 2.118402401167452e-07, "Finetune/Loss": 0.8430130481719971, "Finetune/Loss (Raw)": 0.3515266478061676, "Finetune/Step": 4867, "Finetune/Step Time": 4.662972457706928} +{"Finetune/Learning Rate": 2.1056667657276876e-07, "Finetune/Loss": 0.8425010442733765, "Finetune/Loss (Raw)": 0.7606453895568848, "Finetune/Step": 4868, "Finetune/Step Time": 4.662209099158645} +{"Finetune/Learning Rate": 2.0929691211674319e-07, "Finetune/Loss": 0.8431280851364136, "Finetune/Loss (Raw)": 0.7765535712242126, "Finetune/Step": 4869, "Finetune/Step Time": 4.666776223108172} +{"Finetune/Learning Rate": 2.0803094724143879e-07, "Finetune/Loss": 0.843191385269165, "Finetune/Loss (Raw)": 0.7830353379249573, "Finetune/Step": 4870, "Finetune/Step Time": 4.652350354939699} +{"Finetune/Learning Rate": 2.0676878243815145e-07, "Finetune/Loss": 0.8448262214660645, "Finetune/Loss (Raw)": 0.9405589699745178, "Finetune/Step": 4871, "Finetune/Step Time": 4.647137263789773} +{"Finetune/Learning Rate": 2.055104181967027e-07, "Finetune/Loss": 0.8434848189353943, "Finetune/Loss (Raw)": 0.8352274298667908, "Finetune/Step": 4872, "Finetune/Step Time": 4.647223895415664} +{"Finetune/Learning Rate": 2.0425585500543854e-07, "Finetune/Loss": 0.843712568283081, "Finetune/Loss (Raw)": 0.8073278069496155, "Finetune/Step": 4873, "Finetune/Step Time": 4.64428541995585} +{"Finetune/Learning Rate": 2.0300509335123286e-07, "Finetune/Loss": 0.8427222967147827, "Finetune/Loss (Raw)": 0.6634119153022766, "Finetune/Step": 4874, "Finetune/Step Time": 4.628671070560813} +{"Finetune/Learning Rate": 2.0175813371948072e-07, "Finetune/Loss": 0.8412030935287476, "Finetune/Loss (Raw)": 0.8306042551994324, "Finetune/Step": 4875, "Finetune/Step Time": 4.6375211495906115} +{"Finetune/Learning Rate": 2.0051497659409947e-07, "Finetune/Loss": 0.8411608934402466, "Finetune/Loss (Raw)": 0.9307292699813843, "Finetune/Step": 4876, "Finetune/Step Time": 4.630654506385326} +{"Finetune/Learning Rate": 1.9927562245753874e-07, "Finetune/Loss": 0.8422623872756958, "Finetune/Loss (Raw)": 0.983219563961029, "Finetune/Step": 4877, "Finetune/Step Time": 4.637779233977199} +{"Finetune/Learning Rate": 1.9804007179076602e-07, "Finetune/Loss": 0.8421745300292969, "Finetune/Loss (Raw)": 0.8497055172920227, "Finetune/Step": 4878, "Finetune/Step Time": 4.6338155176490545} +{"Finetune/Learning Rate": 1.968083250732744e-07, "Finetune/Loss": 0.8424314260482788, "Finetune/Loss (Raw)": 0.9185925722122192, "Finetune/Step": 4879, "Finetune/Step Time": 4.61479832790792} +{"Finetune/Learning Rate": 1.9558038278308044e-07, "Finetune/Loss": 0.842835545539856, "Finetune/Loss (Raw)": 0.9494447112083435, "Finetune/Step": 4880, "Finetune/Step Time": 4.610986892133951} +{"Finetune/Learning Rate": 1.9435624539672515e-07, "Finetune/Loss": 0.8451439738273621, "Finetune/Loss (Raw)": 1.0632585287094116, "Finetune/Step": 4881, "Finetune/Step Time": 4.60535591468215} +{"Finetune/Learning Rate": 1.9313591338927185e-07, "Finetune/Loss": 0.8440283536911011, "Finetune/Loss (Raw)": 0.7728627324104309, "Finetune/Step": 4882, "Finetune/Step Time": 4.607690025120974} +{"Finetune/Learning Rate": 1.9191938723430614e-07, "Finetune/Loss": 0.8426175117492676, "Finetune/Loss (Raw)": 0.7519164085388184, "Finetune/Step": 4883, "Finetune/Step Time": 4.598428137600422} +{"Finetune/Learning Rate": 1.9070666740394151e-07, "Finetune/Loss": 0.8434815406799316, "Finetune/Loss (Raw)": 0.9880814552307129, "Finetune/Step": 4884, "Finetune/Step Time": 4.594805177301168} +{"Finetune/Learning Rate": 1.8949775436880924e-07, "Finetune/Loss": 0.8432668447494507, "Finetune/Loss (Raw)": 0.8451283574104309, "Finetune/Step": 4885, "Finetune/Step Time": 4.598398543894291} +{"Finetune/Learning Rate": 1.8829264859806296e-07, "Finetune/Loss": 0.8441931009292603, "Finetune/Loss (Raw)": 0.9648897051811218, "Finetune/Step": 4886, "Finetune/Step Time": 4.610377365723252} +{"Finetune/Learning Rate": 1.8709135055938298e-07, "Finetune/Loss": 0.844595193862915, "Finetune/Loss (Raw)": 0.8867421746253967, "Finetune/Step": 4887, "Finetune/Step Time": 4.616804594174027} +{"Finetune/Learning Rate": 1.8589386071896975e-07, "Finetune/Loss": 0.8451963663101196, "Finetune/Loss (Raw)": 0.9468297362327576, "Finetune/Step": 4888, "Finetune/Step Time": 4.626968381926417} +{"Finetune/Learning Rate": 1.8470017954154262e-07, "Finetune/Loss": 0.8461102247238159, "Finetune/Loss (Raw)": 0.876773476600647, "Finetune/Step": 4889, "Finetune/Step Time": 4.610439281910658} +{"Finetune/Learning Rate": 1.8351030749034992e-07, "Finetune/Loss": 0.8466266393661499, "Finetune/Loss (Raw)": 0.8559783697128296, "Finetune/Step": 4890, "Finetune/Step Time": 4.629958463832736} +{"Finetune/Learning Rate": 1.8232424502715672e-07, "Finetune/Loss": 0.8459253311157227, "Finetune/Loss (Raw)": 0.7850646376609802, "Finetune/Step": 4891, "Finetune/Step Time": 4.621401470154524} +{"Finetune/Learning Rate": 1.811419926122493e-07, "Finetune/Loss": 0.8461630940437317, "Finetune/Loss (Raw)": 0.8809669613838196, "Finetune/Step": 4892, "Finetune/Step Time": 4.617496971040964} +{"Finetune/Learning Rate": 1.7996355070443727e-07, "Finetune/Loss": 0.8479404449462891, "Finetune/Loss (Raw)": 0.9997888207435608, "Finetune/Step": 4893, "Finetune/Step Time": 4.622833553701639} +{"Finetune/Learning Rate": 1.787889197610515e-07, "Finetune/Loss": 0.8463039398193359, "Finetune/Loss (Raw)": 0.818350076675415, "Finetune/Step": 4894, "Finetune/Step Time": 4.629276989027858} +{"Finetune/Learning Rate": 1.776181002379429e-07, "Finetune/Loss": 0.8455045819282532, "Finetune/Loss (Raw)": 0.7833495140075684, "Finetune/Step": 4895, "Finetune/Step Time": 4.625434298068285} +{"Finetune/Learning Rate": 1.7645109258948245e-07, "Finetune/Loss": 0.8464361429214478, "Finetune/Loss (Raw)": 0.9284971356391907, "Finetune/Step": 4896, "Finetune/Step Time": 4.623357092961669} +{"Finetune/Learning Rate": 1.7528789726856566e-07, "Finetune/Loss": 0.845445990562439, "Finetune/Loss (Raw)": 0.6932966709136963, "Finetune/Step": 4897, "Finetune/Step Time": 4.6348069831728935} +{"Finetune/Learning Rate": 1.7412851472660585e-07, "Finetune/Loss": 0.846516489982605, "Finetune/Loss (Raw)": 0.7710691094398499, "Finetune/Step": 4898, "Finetune/Step Time": 4.636271696537733} +{"Finetune/Learning Rate": 1.7297294541353317e-07, "Finetune/Loss": 0.8463484048843384, "Finetune/Loss (Raw)": 0.9668404459953308, "Finetune/Step": 4899, "Finetune/Step Time": 4.633401418104768} +{"Finetune/Learning Rate": 1.7182118977780548e-07, "Finetune/Loss": 0.8469939827919006, "Finetune/Loss (Raw)": 0.853323221206665, "Finetune/Step": 4900, "Finetune/Step Time": 4.620362173765898} +{"Finetune/Learning Rate": 1.706732482663942e-07, "Finetune/Loss": 0.8485805988311768, "Finetune/Loss (Raw)": 0.8238239884376526, "Finetune/Step": 4901, "Finetune/Step Time": 4.618705531582236} +{"Finetune/Learning Rate": 1.6952912132479404e-07, "Finetune/Loss": 0.8480527400970459, "Finetune/Loss (Raw)": 0.8220603466033936, "Finetune/Step": 4902, "Finetune/Step Time": 4.6180435717105865} +{"Finetune/Learning Rate": 1.6838880939701875e-07, "Finetune/Loss": 0.847045361995697, "Finetune/Loss (Raw)": 0.838763415813446, "Finetune/Step": 4903, "Finetune/Step Time": 4.617799902334809} +{"Finetune/Learning Rate": 1.6725231292559985e-07, "Finetune/Loss": 0.8474657535552979, "Finetune/Loss (Raw)": 0.8559137582778931, "Finetune/Step": 4904, "Finetune/Step Time": 4.60296162776649} +{"Finetune/Learning Rate": 1.661196323515901e-07, "Finetune/Loss": 0.8478341698646545, "Finetune/Loss (Raw)": 0.9331352114677429, "Finetune/Step": 4905, "Finetune/Step Time": 4.607174098491669} +{"Finetune/Learning Rate": 1.6499076811456015e-07, "Finetune/Loss": 0.8521387577056885, "Finetune/Loss (Raw)": 0.8404974341392517, "Finetune/Step": 4906, "Finetune/Step Time": 4.567122718319297} +{"Finetune/Learning Rate": 1.6386572065260065e-07, "Finetune/Loss": 0.8502120971679688, "Finetune/Loss (Raw)": 0.7683805823326111, "Finetune/Step": 4907, "Finetune/Step Time": 4.574711972847581} +{"Finetune/Learning Rate": 1.6274449040232233e-07, "Finetune/Loss": 0.850566029548645, "Finetune/Loss (Raw)": 0.8847771286964417, "Finetune/Step": 4908, "Finetune/Step Time": 4.586641628295183} +{"Finetune/Learning Rate": 1.6162707779884933e-07, "Finetune/Loss": 0.8509825468063354, "Finetune/Loss (Raw)": 0.8553018569946289, "Finetune/Step": 4909, "Finetune/Step Time": 4.58801937289536} +{"Finetune/Learning Rate": 1.6051348327583038e-07, "Finetune/Loss": 0.8515161871910095, "Finetune/Loss (Raw)": 0.9020861983299255, "Finetune/Step": 4910, "Finetune/Step Time": 4.577510941773653} +{"Finetune/Learning Rate": 1.5940370726542864e-07, "Finetune/Loss": 0.8552590012550354, "Finetune/Loss (Raw)": 0.8446698784828186, "Finetune/Step": 4911, "Finetune/Step Time": 4.538805540651083} +{"Finetune/Learning Rate": 1.5829775019832628e-07, "Finetune/Loss": 0.8554880023002625, "Finetune/Loss (Raw)": 0.908836841583252, "Finetune/Step": 4912, "Finetune/Step Time": 4.545576993376017} +{"Finetune/Learning Rate": 1.571956125037244e-07, "Finetune/Loss": 0.8546438813209534, "Finetune/Loss (Raw)": 0.9339525699615479, "Finetune/Step": 4913, "Finetune/Step Time": 4.55381028726697} +{"Finetune/Learning Rate": 1.5609729460934087e-07, "Finetune/Loss": 0.8508613109588623, "Finetune/Loss (Raw)": 0.40790191292762756, "Finetune/Step": 4914, "Finetune/Step Time": 4.588778095319867} +{"Finetune/Learning Rate": 1.550027969414114e-07, "Finetune/Loss": 0.8500396013259888, "Finetune/Loss (Raw)": 0.7470229268074036, "Finetune/Step": 4915, "Finetune/Step Time": 4.59220752492547} +{"Finetune/Learning Rate": 1.5391211992468736e-07, "Finetune/Loss": 0.8502446413040161, "Finetune/Loss (Raw)": 0.8517118692398071, "Finetune/Step": 4916, "Finetune/Step Time": 4.566310603171587} +{"Finetune/Learning Rate": 1.5282526398244236e-07, "Finetune/Loss": 0.8478617072105408, "Finetune/Loss (Raw)": 0.41063618659973145, "Finetune/Step": 4917, "Finetune/Step Time": 4.6003307066857815} +{"Finetune/Learning Rate": 1.5174222953646344e-07, "Finetune/Loss": 0.8473451137542725, "Finetune/Loss (Raw)": 0.8811421990394592, "Finetune/Step": 4918, "Finetune/Step Time": 4.5940162762999535} +{"Finetune/Learning Rate": 1.5066301700705334e-07, "Finetune/Loss": 0.8517091274261475, "Finetune/Loss (Raw)": 0.8613128662109375, "Finetune/Step": 4919, "Finetune/Step Time": 4.55796535871923} +{"Finetune/Learning Rate": 1.4958762681303362e-07, "Finetune/Loss": 0.8508398532867432, "Finetune/Loss (Raw)": 0.9086782336235046, "Finetune/Step": 4920, "Finetune/Step Time": 4.553838791325688} +{"Finetune/Learning Rate": 1.4851605937174384e-07, "Finetune/Loss": 0.8508210182189941, "Finetune/Loss (Raw)": 0.9165865182876587, "Finetune/Step": 4921, "Finetune/Step Time": 4.561203956604004} +{"Finetune/Learning Rate": 1.474483150990369e-07, "Finetune/Loss": 0.8503592014312744, "Finetune/Loss (Raw)": 0.7536325454711914, "Finetune/Step": 4922, "Finetune/Step Time": 4.567695859819651} +{"Finetune/Learning Rate": 1.4638439440928354e-07, "Finetune/Loss": 0.850520133972168, "Finetune/Loss (Raw)": 0.8115488886833191, "Finetune/Step": 4923, "Finetune/Step Time": 4.5420772433280945} +{"Finetune/Learning Rate": 1.4532429771537237e-07, "Finetune/Loss": 0.8499729633331299, "Finetune/Loss (Raw)": 0.8703985214233398, "Finetune/Step": 4924, "Finetune/Step Time": 4.543679827824235} +{"Finetune/Learning Rate": 1.4426802542870433e-07, "Finetune/Loss": 0.8493281602859497, "Finetune/Loss (Raw)": 0.8320823311805725, "Finetune/Step": 4925, "Finetune/Step Time": 4.543145067989826} +{"Finetune/Learning Rate": 1.4321557795919705e-07, "Finetune/Loss": 0.850200891494751, "Finetune/Loss (Raw)": 0.8114036917686462, "Finetune/Step": 4926, "Finetune/Step Time": 4.52732584439218} +{"Finetune/Learning Rate": 1.4216695571528828e-07, "Finetune/Loss": 0.8496801853179932, "Finetune/Loss (Raw)": 0.8093854188919067, "Finetune/Step": 4927, "Finetune/Step Time": 4.525471979752183} +{"Finetune/Learning Rate": 1.411221591039269e-07, "Finetune/Loss": 0.846559464931488, "Finetune/Loss (Raw)": 0.6557371616363525, "Finetune/Step": 4928, "Finetune/Step Time": 4.548333605751395} +{"Finetune/Learning Rate": 1.4008118853057529e-07, "Finetune/Loss": 0.8458347320556641, "Finetune/Loss (Raw)": 0.8344323635101318, "Finetune/Step": 4929, "Finetune/Step Time": 4.5527316853404045} +{"Finetune/Learning Rate": 1.3904404439921582e-07, "Finetune/Loss": 0.8458439111709595, "Finetune/Loss (Raw)": 0.8827361464500427, "Finetune/Step": 4930, "Finetune/Step Time": 4.552208004519343} +{"Finetune/Learning Rate": 1.3801072711234541e-07, "Finetune/Loss": 0.8460413217544556, "Finetune/Loss (Raw)": 0.9392777681350708, "Finetune/Step": 4931, "Finetune/Step Time": 4.549859641119838} +{"Finetune/Learning Rate": 1.369812370709711e-07, "Finetune/Loss": 0.846031665802002, "Finetune/Loss (Raw)": 0.9025273323059082, "Finetune/Step": 4932, "Finetune/Step Time": 4.54713730327785} +{"Finetune/Learning Rate": 1.3595557467461994e-07, "Finetune/Loss": 0.844825267791748, "Finetune/Loss (Raw)": 0.7959597706794739, "Finetune/Step": 4933, "Finetune/Step Time": 4.557319980114698} +{"Finetune/Learning Rate": 1.3493374032133134e-07, "Finetune/Loss": 0.8450819253921509, "Finetune/Loss (Raw)": 0.9303115010261536, "Finetune/Step": 4934, "Finetune/Step Time": 4.552497535943985} +{"Finetune/Learning Rate": 1.3391573440765915e-07, "Finetune/Loss": 0.845792829990387, "Finetune/Loss (Raw)": 0.9371982216835022, "Finetune/Step": 4935, "Finetune/Step Time": 4.551698511466384} +{"Finetune/Learning Rate": 1.329015573286696e-07, "Finetune/Loss": 0.8444732427597046, "Finetune/Loss (Raw)": 0.7935636639595032, "Finetune/Step": 4936, "Finetune/Step Time": 4.562209773808718} +{"Finetune/Learning Rate": 1.3189120947794897e-07, "Finetune/Loss": 0.8471710085868835, "Finetune/Loss (Raw)": 1.0126303434371948, "Finetune/Step": 4937, "Finetune/Step Time": 4.5556965339928865} +{"Finetune/Learning Rate": 1.308846912475914e-07, "Finetune/Loss": 0.8478341102600098, "Finetune/Loss (Raw)": 0.9163964986801147, "Finetune/Step": 4938, "Finetune/Step Time": 4.5676326006650925} +{"Finetune/Learning Rate": 1.2988200302820552e-07, "Finetune/Loss": 0.8435012102127075, "Finetune/Loss (Raw)": 0.3393440246582031, "Finetune/Step": 4939, "Finetune/Step Time": 4.583926009014249} +{"Finetune/Learning Rate": 1.2888314520891676e-07, "Finetune/Loss": 0.8420677185058594, "Finetune/Loss (Raw)": 0.6914200782775879, "Finetune/Step": 4940, "Finetune/Step Time": 4.584188956767321} +{"Finetune/Learning Rate": 1.2788811817736168e-07, "Finetune/Loss": 0.8442278504371643, "Finetune/Loss (Raw)": 0.7518531680107117, "Finetune/Step": 4941, "Finetune/Step Time": 4.547414602711797} +{"Finetune/Learning Rate": 1.2689692231969364e-07, "Finetune/Loss": 0.8438389301300049, "Finetune/Loss (Raw)": 0.8227876424789429, "Finetune/Step": 4942, "Finetune/Step Time": 4.553984090685844} +{"Finetune/Learning Rate": 1.2590955802057047e-07, "Finetune/Loss": 0.8426422476768494, "Finetune/Loss (Raw)": 0.7906503677368164, "Finetune/Step": 4943, "Finetune/Step Time": 4.565596956759691} +{"Finetune/Learning Rate": 1.2492602566317457e-07, "Finetune/Loss": 0.8434433937072754, "Finetune/Loss (Raw)": 0.9208944439888, "Finetune/Step": 4944, "Finetune/Step Time": 4.546592965722084} +{"Finetune/Learning Rate": 1.2394632562919283e-07, "Finetune/Loss": 0.8438106775283813, "Finetune/Loss (Raw)": 0.944342851638794, "Finetune/Step": 4945, "Finetune/Step Time": 4.546368278563023} +{"Finetune/Learning Rate": 1.2297045829882892e-07, "Finetune/Loss": 0.8430076837539673, "Finetune/Loss (Raw)": 0.8630503416061401, "Finetune/Step": 4946, "Finetune/Step Time": 4.5473107770085335} +{"Finetune/Learning Rate": 1.2199842405079766e-07, "Finetune/Loss": 0.8427979946136475, "Finetune/Loss (Raw)": 0.9271314144134521, "Finetune/Step": 4947, "Finetune/Step Time": 4.554595550522208} +{"Finetune/Learning Rate": 1.2103022326232505e-07, "Finetune/Loss": 0.8421369194984436, "Finetune/Loss (Raw)": 0.8205223679542542, "Finetune/Step": 4948, "Finetune/Step Time": 4.552316177636385} +{"Finetune/Learning Rate": 1.2006585630915391e-07, "Finetune/Loss": 0.8410621881484985, "Finetune/Loss (Raw)": 0.8089954853057861, "Finetune/Step": 4949, "Finetune/Step Time": 4.550413174554706} +{"Finetune/Learning Rate": 1.1910532356553261e-07, "Finetune/Loss": 0.840528130531311, "Finetune/Loss (Raw)": 0.8148027062416077, "Finetune/Step": 4950, "Finetune/Step Time": 4.562182357534766} +{"Finetune/Learning Rate": 1.1814862540422744e-07, "Finetune/Loss": 0.8402771949768066, "Finetune/Loss (Raw)": 0.9076538681983948, "Finetune/Step": 4951, "Finetune/Step Time": 4.560438048094511} +{"Finetune/Learning Rate": 1.1719576219651585e-07, "Finetune/Loss": 0.8451812267303467, "Finetune/Loss (Raw)": 0.9225517511367798, "Finetune/Step": 4952, "Finetune/Step Time": 4.526686318218708} +{"Finetune/Learning Rate": 1.1624673431218092e-07, "Finetune/Loss": 0.8447803258895874, "Finetune/Loss (Raw)": 0.8217979669570923, "Finetune/Step": 4953, "Finetune/Step Time": 4.522121990099549} +{"Finetune/Learning Rate": 1.1530154211952583e-07, "Finetune/Loss": 0.8449058532714844, "Finetune/Loss (Raw)": 0.9253945350646973, "Finetune/Step": 4954, "Finetune/Step Time": 4.573352864012122} +{"Finetune/Learning Rate": 1.1436018598535937e-07, "Finetune/Loss": 0.8443219661712646, "Finetune/Loss (Raw)": 0.8683910965919495, "Finetune/Step": 4955, "Finetune/Step Time": 4.57097427174449} +{"Finetune/Learning Rate": 1.1342266627500375e-07, "Finetune/Loss": 0.8442131876945496, "Finetune/Loss (Raw)": 0.9359742999076843, "Finetune/Step": 4956, "Finetune/Step Time": 4.564314913004637} +{"Finetune/Learning Rate": 1.1248898335229241e-07, "Finetune/Loss": 0.8452348113059998, "Finetune/Loss (Raw)": 0.9707249402999878, "Finetune/Step": 4957, "Finetune/Step Time": 4.553914979100227} +{"Finetune/Learning Rate": 1.1155913757956881e-07, "Finetune/Loss": 0.8446308374404907, "Finetune/Loss (Raw)": 0.8109049201011658, "Finetune/Step": 4958, "Finetune/Step Time": 4.565452819690108} +{"Finetune/Learning Rate": 1.1063312931768877e-07, "Finetune/Loss": 0.8452968597412109, "Finetune/Loss (Raw)": 0.4403195083141327, "Finetune/Step": 4959, "Finetune/Step Time": 4.5658301170915365} +{"Finetune/Learning Rate": 1.0971095892601702e-07, "Finetune/Loss": 0.845904529094696, "Finetune/Loss (Raw)": 0.8304727673530579, "Finetune/Step": 4960, "Finetune/Step Time": 4.545114709064364} +{"Finetune/Learning Rate": 1.0879262676243174e-07, "Finetune/Loss": 0.845641016960144, "Finetune/Loss (Raw)": 0.8109800219535828, "Finetune/Step": 4961, "Finetune/Step Time": 4.543198758736253} +{"Finetune/Learning Rate": 1.0787813318331786e-07, "Finetune/Loss": 0.8450919985771179, "Finetune/Loss (Raw)": 0.9225817918777466, "Finetune/Step": 4962, "Finetune/Step Time": 4.547636674717069} +{"Finetune/Learning Rate": 1.0696747854357259e-07, "Finetune/Loss": 0.8443396091461182, "Finetune/Loss (Raw)": 0.8421941995620728, "Finetune/Step": 4963, "Finetune/Step Time": 4.546209942549467} +{"Finetune/Learning Rate": 1.0606066319660435e-07, "Finetune/Loss": 0.8442030549049377, "Finetune/Loss (Raw)": 0.9477794766426086, "Finetune/Step": 4964, "Finetune/Step Time": 4.549666488543153} +{"Finetune/Learning Rate": 1.0515768749432942e-07, "Finetune/Loss": 0.845055341720581, "Finetune/Loss (Raw)": 0.9760081171989441, "Finetune/Step": 4965, "Finetune/Step Time": 4.555990256369114} +{"Finetune/Learning Rate": 1.0425855178717636e-07, "Finetune/Loss": 0.843768835067749, "Finetune/Loss (Raw)": 0.7992914319038391, "Finetune/Step": 4966, "Finetune/Step Time": 4.559206668287516} +{"Finetune/Learning Rate": 1.0336325642408052e-07, "Finetune/Loss": 0.8446198105812073, "Finetune/Loss (Raw)": 0.9300349354743958, "Finetune/Step": 4967, "Finetune/Step Time": 4.556658614426851} +{"Finetune/Learning Rate": 1.024718017524895e-07, "Finetune/Loss": 0.8445127010345459, "Finetune/Loss (Raw)": 0.8857712745666504, "Finetune/Step": 4968, "Finetune/Step Time": 4.552045838907361} +{"Finetune/Learning Rate": 1.015841881183588e-07, "Finetune/Loss": 0.8449923396110535, "Finetune/Loss (Raw)": 0.8781555891036987, "Finetune/Step": 4969, "Finetune/Step Time": 4.55542080104351} +{"Finetune/Learning Rate": 1.0070041586615398e-07, "Finetune/Loss": 0.845700204372406, "Finetune/Loss (Raw)": 0.9438197612762451, "Finetune/Step": 4970, "Finetune/Step Time": 4.537492034956813} +{"Finetune/Learning Rate": 9.98204853388518e-08, "Finetune/Loss": 0.8454300165176392, "Finetune/Loss (Raw)": 1.0129191875457764, "Finetune/Step": 4971, "Finetune/Step Time": 4.549395760521293} +{"Finetune/Learning Rate": 9.894439687793466e-08, "Finetune/Loss": 0.8456615805625916, "Finetune/Loss (Raw)": 0.8642411828041077, "Finetune/Step": 4972, "Finetune/Step Time": 4.537977835163474} +{"Finetune/Learning Rate": 9.807215082339395e-08, "Finetune/Loss": 0.8456166386604309, "Finetune/Loss (Raw)": 1.0020599365234375, "Finetune/Step": 4973, "Finetune/Step Time": 4.54392222687602} +{"Finetune/Learning Rate": 9.720374751373329e-08, "Finetune/Loss": 0.840585470199585, "Finetune/Loss (Raw)": 0.3047977685928345, "Finetune/Step": 4974, "Finetune/Step Time": 4.5543590653687716} +{"Finetune/Learning Rate": 9.633918728596203e-08, "Finetune/Loss": 0.844420850276947, "Finetune/Loss (Raw)": 0.8141480088233948, "Finetune/Step": 4975, "Finetune/Step Time": 4.52075139246881} +{"Finetune/Learning Rate": 9.547847047559955e-08, "Finetune/Loss": 0.8461911678314209, "Finetune/Loss (Raw)": 0.9917467832565308, "Finetune/Step": 4976, "Finetune/Step Time": 4.523784166201949} +{"Finetune/Learning Rate": 9.462159741667309e-08, "Finetune/Loss": 0.8461292386054993, "Finetune/Loss (Raw)": 0.7509996891021729, "Finetune/Step": 4977, "Finetune/Step Time": 4.523231130093336} +{"Finetune/Learning Rate": 9.376856844171889e-08, "Finetune/Loss": 0.8449570536613464, "Finetune/Loss (Raw)": 0.8057515621185303, "Finetune/Step": 4978, "Finetune/Step Time": 4.5135525688529015} +{"Finetune/Learning Rate": 9.291938388178102e-08, "Finetune/Loss": 0.8442394733428955, "Finetune/Loss (Raw)": 0.8211499452590942, "Finetune/Step": 4979, "Finetune/Step Time": 4.515782721340656} +{"Finetune/Learning Rate": 9.207404406640919e-08, "Finetune/Loss": 0.8432989120483398, "Finetune/Loss (Raw)": 0.7069063782691956, "Finetune/Step": 4980, "Finetune/Step Time": 4.514552799984813} +{"Finetune/Learning Rate": 9.123254932366655e-08, "Finetune/Loss": 0.8414111137390137, "Finetune/Loss (Raw)": 0.7693116664886475, "Finetune/Step": 4981, "Finetune/Step Time": 4.50550976395607} +{"Finetune/Learning Rate": 9.039489998011853e-08, "Finetune/Loss": 0.8440464735031128, "Finetune/Loss (Raw)": 0.9583142399787903, "Finetune/Step": 4982, "Finetune/Step Time": 4.50044166110456} +{"Finetune/Learning Rate": 8.956109636084065e-08, "Finetune/Loss": 0.8401544094085693, "Finetune/Loss (Raw)": 0.41231003403663635, "Finetune/Step": 4983, "Finetune/Step Time": 4.536269577220082} +{"Finetune/Learning Rate": 8.873113878941631e-08, "Finetune/Loss": 0.8392807245254517, "Finetune/Loss (Raw)": 0.7857640385627747, "Finetune/Step": 4984, "Finetune/Step Time": 4.541614463552833} +{"Finetune/Learning Rate": 8.790502758793562e-08, "Finetune/Loss": 0.8397269248962402, "Finetune/Loss (Raw)": 1.0216104984283447, "Finetune/Step": 4985, "Finetune/Step Time": 4.538296222686768} +{"Finetune/Learning Rate": 8.70827630769977e-08, "Finetune/Loss": 0.8374489545822144, "Finetune/Loss (Raw)": 0.7025451064109802, "Finetune/Step": 4986, "Finetune/Step Time": 4.530050104483962} +{"Finetune/Learning Rate": 8.626434557570507e-08, "Finetune/Loss": 0.8376890420913696, "Finetune/Loss (Raw)": 0.9205948710441589, "Finetune/Step": 4987, "Finetune/Step Time": 4.537385230883956} +{"Finetune/Learning Rate": 8.544977540167032e-08, "Finetune/Loss": 0.8388814330101013, "Finetune/Loss (Raw)": 0.944441556930542, "Finetune/Step": 4988, "Finetune/Step Time": 4.545799745246768} +{"Finetune/Learning Rate": 8.463905287101282e-08, "Finetune/Loss": 0.8393970131874084, "Finetune/Loss (Raw)": 0.9269893169403076, "Finetune/Step": 4989, "Finetune/Step Time": 4.536848830059171} +{"Finetune/Learning Rate": 8.383217829835755e-08, "Finetune/Loss": 0.838589072227478, "Finetune/Loss (Raw)": 0.8610764145851135, "Finetune/Step": 4990, "Finetune/Step Time": 4.539896795526147} +{"Finetune/Learning Rate": 8.302915199683737e-08, "Finetune/Loss": 0.8373696208000183, "Finetune/Loss (Raw)": 0.829889178276062, "Finetune/Step": 4991, "Finetune/Step Time": 4.534282283857465} +{"Finetune/Learning Rate": 8.222997427809076e-08, "Finetune/Loss": 0.8378692865371704, "Finetune/Loss (Raw)": 0.9902709722518921, "Finetune/Step": 4992, "Finetune/Step Time": 4.534039776772261} +{"Finetune/Learning Rate": 8.143464545226298e-08, "Finetune/Loss": 0.8354200124740601, "Finetune/Loss (Raw)": 0.5966101288795471, "Finetune/Step": 4993, "Finetune/Step Time": 4.547452023252845} +{"Finetune/Learning Rate": 8.064316582800491e-08, "Finetune/Loss": 0.8359333276748657, "Finetune/Loss (Raw)": 0.8384198546409607, "Finetune/Step": 4994, "Finetune/Step Time": 4.554397756233811} +{"Finetune/Learning Rate": 7.98555357124764e-08, "Finetune/Loss": 0.8380417227745056, "Finetune/Loss (Raw)": 0.6213923692703247, "Finetune/Step": 4995, "Finetune/Step Time": 4.519569663330913} +{"Finetune/Learning Rate": 7.907175541133849e-08, "Finetune/Loss": 0.840064525604248, "Finetune/Loss (Raw)": 1.0195714235305786, "Finetune/Step": 4996, "Finetune/Step Time": 4.521018702536821} +{"Finetune/Learning Rate": 7.829182522876344e-08, "Finetune/Loss": 0.8412981033325195, "Finetune/Loss (Raw)": 0.9344491362571716, "Finetune/Step": 4997, "Finetune/Step Time": 4.505784286186099} +{"Finetune/Learning Rate": 7.75157454674258e-08, "Finetune/Loss": 0.8437520265579224, "Finetune/Loss (Raw)": 1.0971318483352661, "Finetune/Step": 4998, "Finetune/Step Time": 4.503577573224902} +{"Finetune/Learning Rate": 7.674351642850797e-08, "Finetune/Loss": 0.8441532850265503, "Finetune/Loss (Raw)": 0.9919206500053406, "Finetune/Step": 4999, "Finetune/Step Time": 4.493904998525977} +{"Finetune/Learning Rate": 7.597513841169469e-08, "Finetune/Loss": 0.840377926826477, "Finetune/Loss (Raw)": 0.35199013352394104, "Finetune/Step": 5000, "Finetune/Step Time": 4.532823000103235} +{"Finetune/Learning Rate": 7.521061171518184e-08, "Finetune/Loss": 0.8413864374160767, "Finetune/Loss (Raw)": 0.9364144206047058, "Finetune/Step": 5001, "Finetune/Step Time": 4.515083974227309} +{"Finetune/Learning Rate": 7.444993663566325e-08, "Finetune/Loss": 0.8415553569793701, "Finetune/Loss (Raw)": 0.6850289106369019, "Finetune/Step": 5002, "Finetune/Step Time": 4.516659447923303} +{"Finetune/Learning Rate": 7.369311346834496e-08, "Finetune/Loss": 0.8413248062133789, "Finetune/Loss (Raw)": 0.80109041929245, "Finetune/Step": 5003, "Finetune/Step Time": 4.505024479702115} +{"Finetune/Learning Rate": 7.294014250693538e-08, "Finetune/Loss": 0.8398171067237854, "Finetune/Loss (Raw)": 0.7377508282661438, "Finetune/Step": 5004, "Finetune/Step Time": 4.518673170357943} +{"Finetune/Learning Rate": 7.21910240436463e-08, "Finetune/Loss": 0.8389207124710083, "Finetune/Loss (Raw)": 0.8684831261634827, "Finetune/Step": 5005, "Finetune/Step Time": 4.514988198876381} +{"Finetune/Learning Rate": 7.144575836919632e-08, "Finetune/Loss": 0.8391874432563782, "Finetune/Loss (Raw)": 0.883844256401062, "Finetune/Step": 5006, "Finetune/Step Time": 4.5070690624415874} +{"Finetune/Learning Rate": 7.07043457728096e-08, "Finetune/Loss": 0.8382105231285095, "Finetune/Loss (Raw)": 0.7935453653335571, "Finetune/Step": 5007, "Finetune/Step Time": 4.517046209424734} +{"Finetune/Learning Rate": 6.996678654221267e-08, "Finetune/Loss": 0.8369684815406799, "Finetune/Loss (Raw)": 0.7904653549194336, "Finetune/Step": 5008, "Finetune/Step Time": 4.523972952738404} +{"Finetune/Learning Rate": 6.923308096363879e-08, "Finetune/Loss": 0.8365651369094849, "Finetune/Loss (Raw)": 1.0116329193115234, "Finetune/Step": 5009, "Finetune/Step Time": 4.5265389159321785} +{"Finetune/Learning Rate": 6.850322932182352e-08, "Finetune/Loss": 0.8379024863243103, "Finetune/Loss (Raw)": 0.9440404176712036, "Finetune/Step": 5010, "Finetune/Step Time": 4.517461223527789} +{"Finetune/Learning Rate": 6.777723190000918e-08, "Finetune/Loss": 0.8387920260429382, "Finetune/Loss (Raw)": 0.8657768964767456, "Finetune/Step": 5011, "Finetune/Step Time": 4.518733946606517} +{"Finetune/Learning Rate": 6.705508897994151e-08, "Finetune/Loss": 0.8376701474189758, "Finetune/Loss (Raw)": 0.8444828391075134, "Finetune/Step": 5012, "Finetune/Step Time": 4.522225642576814} +{"Finetune/Learning Rate": 6.633680084186856e-08, "Finetune/Loss": 0.8367864489555359, "Finetune/Loss (Raw)": 0.732013463973999, "Finetune/Step": 5013, "Finetune/Step Time": 4.521037448197603} +{"Finetune/Learning Rate": 6.562236776454512e-08, "Finetune/Loss": 0.8354362845420837, "Finetune/Loss (Raw)": 0.7920657992362976, "Finetune/Step": 5014, "Finetune/Step Time": 4.505363754928112} +{"Finetune/Learning Rate": 6.491179002522829e-08, "Finetune/Loss": 0.8345680236816406, "Finetune/Loss (Raw)": 0.775606632232666, "Finetune/Step": 5015, "Finetune/Step Time": 4.4945068173110485} +{"Finetune/Learning Rate": 6.420506789967973e-08, "Finetune/Loss": 0.8339245319366455, "Finetune/Loss (Raw)": 0.864464282989502, "Finetune/Step": 5016, "Finetune/Step Time": 4.509613132104278} +{"Finetune/Learning Rate": 6.350220166216337e-08, "Finetune/Loss": 0.8332226276397705, "Finetune/Loss (Raw)": 0.7869311571121216, "Finetune/Step": 5017, "Finetune/Step Time": 4.515555668622255} +{"Finetune/Learning Rate": 6.280319158544989e-08, "Finetune/Loss": 0.8323907256126404, "Finetune/Loss (Raw)": 0.7494959831237793, "Finetune/Step": 5018, "Finetune/Step Time": 4.50371135212481} +{"Finetune/Learning Rate": 6.21080379408101e-08, "Finetune/Loss": 0.8323865532875061, "Finetune/Loss (Raw)": 0.7845303416252136, "Finetune/Step": 5019, "Finetune/Step Time": 4.5024542063474655} +{"Finetune/Learning Rate": 6.14167409980193e-08, "Finetune/Loss": 0.8282065987586975, "Finetune/Loss (Raw)": 0.3459289073944092, "Finetune/Step": 5020, "Finetune/Step Time": 4.5328476298600435} +{"Finetune/Learning Rate": 6.072930102535734e-08, "Finetune/Loss": 0.8276687264442444, "Finetune/Loss (Raw)": 0.9309465289115906, "Finetune/Step": 5021, "Finetune/Step Time": 4.532344726845622} +{"Finetune/Learning Rate": 6.004571828960525e-08, "Finetune/Loss": 0.8282013535499573, "Finetune/Loss (Raw)": 0.8865240216255188, "Finetune/Step": 5022, "Finetune/Step Time": 4.51446146517992} +{"Finetune/Learning Rate": 5.936599305604862e-08, "Finetune/Loss": 0.8289409875869751, "Finetune/Loss (Raw)": 0.8780225515365601, "Finetune/Step": 5023, "Finetune/Step Time": 4.520909022539854} +{"Finetune/Learning Rate": 5.869012558847642e-08, "Finetune/Loss": 0.828130304813385, "Finetune/Loss (Raw)": 0.8247256278991699, "Finetune/Step": 5024, "Finetune/Step Time": 4.524862792342901} +{"Finetune/Learning Rate": 5.801811614917885e-08, "Finetune/Loss": 0.8292025327682495, "Finetune/Loss (Raw)": 0.8305436372756958, "Finetune/Step": 5025, "Finetune/Step Time": 4.524073721840978} +{"Finetune/Learning Rate": 5.73499649989484e-08, "Finetune/Loss": 0.8304287195205688, "Finetune/Loss (Raw)": 0.9280197024345398, "Finetune/Step": 5026, "Finetune/Step Time": 4.521194543689489} +{"Finetune/Learning Rate": 5.6685672397083226e-08, "Finetune/Loss": 0.8300642967224121, "Finetune/Loss (Raw)": 0.9202011227607727, "Finetune/Step": 5027, "Finetune/Step Time": 4.524269750341773} +{"Finetune/Learning Rate": 5.602523860138154e-08, "Finetune/Loss": 0.8309875130653381, "Finetune/Loss (Raw)": 0.9714915752410889, "Finetune/Step": 5028, "Finetune/Step Time": 4.526586268097162} +{"Finetune/Learning Rate": 5.536866386814499e-08, "Finetune/Loss": 0.8314818739891052, "Finetune/Loss (Raw)": 0.8870994448661804, "Finetune/Step": 5029, "Finetune/Step Time": 4.528386063873768} +{"Finetune/Learning Rate": 5.471594845217865e-08, "Finetune/Loss": 0.8314917087554932, "Finetune/Loss (Raw)": 0.8233168721199036, "Finetune/Step": 5030, "Finetune/Step Time": 4.531572621315718} +{"Finetune/Learning Rate": 5.4067092606785445e-08, "Finetune/Loss": 0.8319126963615417, "Finetune/Loss (Raw)": 0.8926528096199036, "Finetune/Step": 5031, "Finetune/Step Time": 4.538832636550069} +{"Finetune/Learning Rate": 5.342209658377728e-08, "Finetune/Loss": 0.8326385617256165, "Finetune/Loss (Raw)": 0.9488231539726257, "Finetune/Step": 5032, "Finetune/Step Time": 4.536844536662102} +{"Finetune/Learning Rate": 5.278096063346172e-08, "Finetune/Loss": 0.8319984674453735, "Finetune/Loss (Raw)": 0.8512083292007446, "Finetune/Step": 5033, "Finetune/Step Time": 4.536115912720561} +{"Finetune/Learning Rate": 5.214368500465305e-08, "Finetune/Loss": 0.8325795531272888, "Finetune/Loss (Raw)": 0.9148768186569214, "Finetune/Step": 5034, "Finetune/Step Time": 4.5333913043141365} +{"Finetune/Learning Rate": 5.151026994466457e-08, "Finetune/Loss": 0.8327778577804565, "Finetune/Loss (Raw)": 0.7937629222869873, "Finetune/Step": 5035, "Finetune/Step Time": 4.5307933278381824} +{"Finetune/Learning Rate": 5.088071569931185e-08, "Finetune/Loss": 0.8323186635971069, "Finetune/Loss (Raw)": 0.825999915599823, "Finetune/Step": 5036, "Finetune/Step Time": 4.5107464753091335} +{"Finetune/Learning Rate": 5.025502251291392e-08, "Finetune/Loss": 0.8332202434539795, "Finetune/Loss (Raw)": 0.9706994891166687, "Finetune/Step": 5037, "Finetune/Step Time": 4.500507544726133} +{"Finetune/Learning Rate": 4.963319062828875e-08, "Finetune/Loss": 0.8337879776954651, "Finetune/Loss (Raw)": 0.9747605323791504, "Finetune/Step": 5038, "Finetune/Step Time": 4.508425049483776} +{"Finetune/Learning Rate": 4.901522028675776e-08, "Finetune/Loss": 0.8355706930160522, "Finetune/Loss (Raw)": 1.0728541612625122, "Finetune/Step": 5039, "Finetune/Step Time": 4.513849835842848} +{"Finetune/Learning Rate": 4.8401111728143545e-08, "Finetune/Loss": 0.8341736793518066, "Finetune/Loss (Raw)": 0.7300207018852234, "Finetune/Step": 5040, "Finetune/Step Time": 4.523398535326123} +{"Finetune/Learning Rate": 4.779086519076992e-08, "Finetune/Loss": 0.8345451951026917, "Finetune/Loss (Raw)": 0.9815016388893127, "Finetune/Step": 5041, "Finetune/Step Time": 4.511816518381238} +{"Finetune/Learning Rate": 4.7184480911459665e-08, "Finetune/Loss": 0.8388918042182922, "Finetune/Loss (Raw)": 0.9642770886421204, "Finetune/Step": 5042, "Finetune/Step Time": 4.487878922373056} +{"Finetune/Learning Rate": 4.65819591255412e-08, "Finetune/Loss": 0.8398597240447998, "Finetune/Loss (Raw)": 0.8709183931350708, "Finetune/Step": 5043, "Finetune/Step Time": 4.489693351089954} +{"Finetune/Learning Rate": 4.5983300066839706e-08, "Finetune/Loss": 0.8398709893226624, "Finetune/Loss (Raw)": 0.8531472682952881, "Finetune/Step": 5044, "Finetune/Step Time": 4.49283878877759} +{"Finetune/Learning Rate": 4.538850396768379e-08, "Finetune/Loss": 0.8389261960983276, "Finetune/Loss (Raw)": 0.2897028625011444, "Finetune/Step": 5045, "Finetune/Step Time": 4.491470236331224} +{"Finetune/Learning Rate": 4.479757105890104e-08, "Finetune/Loss": 0.8346366882324219, "Finetune/Loss (Raw)": 0.3320888578891754, "Finetune/Step": 5046, "Finetune/Step Time": 4.533434679731727} +{"Finetune/Learning Rate": 4.4210501569823536e-08, "Finetune/Loss": 0.8349407911300659, "Finetune/Loss (Raw)": 0.9002343416213989, "Finetune/Step": 5047, "Finetune/Step Time": 4.5331254582852125} +{"Finetune/Learning Rate": 4.362729572827906e-08, "Finetune/Loss": 0.8353978991508484, "Finetune/Loss (Raw)": 0.967184841632843, "Finetune/Step": 5048, "Finetune/Step Time": 4.53384443372488} +{"Finetune/Learning Rate": 4.304795376059989e-08, "Finetune/Loss": 0.8349432349205017, "Finetune/Loss (Raw)": 0.8583886027336121, "Finetune/Step": 5049, "Finetune/Step Time": 4.52640145458281} +{"Finetune/Learning Rate": 4.2472475891617295e-08, "Finetune/Loss": 0.8355967402458191, "Finetune/Loss (Raw)": 0.8372865915298462, "Finetune/Step": 5050, "Finetune/Step Time": 4.532474394887686} +{"Finetune/Learning Rate": 4.1900862344661506e-08, "Finetune/Loss": 0.8353166580200195, "Finetune/Loss (Raw)": 0.7756964564323425, "Finetune/Step": 5051, "Finetune/Step Time": 4.546483976766467} +{"Finetune/Learning Rate": 4.133311334156509e-08, "Finetune/Loss": 0.831423282623291, "Finetune/Loss (Raw)": 0.37204793095588684, "Finetune/Step": 5052, "Finetune/Step Time": 4.578838523477316} +{"Finetune/Learning Rate": 4.076922910266179e-08, "Finetune/Loss": 0.8312406539916992, "Finetune/Loss (Raw)": 0.808699905872345, "Finetune/Step": 5053, "Finetune/Step Time": 4.587452407926321} +{"Finetune/Learning Rate": 4.020920984678323e-08, "Finetune/Loss": 0.8327531814575195, "Finetune/Loss (Raw)": 1.005020260810852, "Finetune/Step": 5054, "Finetune/Step Time": 4.5936956740915775} +{"Finetune/Learning Rate": 3.965305579126111e-08, "Finetune/Loss": 0.8329430818557739, "Finetune/Loss (Raw)": 0.8336809873580933, "Finetune/Step": 5055, "Finetune/Step Time": 4.593887763097882} +{"Finetune/Learning Rate": 3.9100767151928345e-08, "Finetune/Loss": 0.8347578048706055, "Finetune/Loss (Raw)": 0.8880202174186707, "Finetune/Step": 5056, "Finetune/Step Time": 4.568651534616947} +{"Finetune/Learning Rate": 3.855234414311904e-08, "Finetune/Loss": 0.8351140022277832, "Finetune/Loss (Raw)": 0.8800313472747803, "Finetune/Step": 5057, "Finetune/Step Time": 4.572427827864885} +{"Finetune/Learning Rate": 3.800778697766294e-08, "Finetune/Loss": 0.8351370692253113, "Finetune/Loss (Raw)": 0.8856955170631409, "Finetune/Step": 5058, "Finetune/Step Time": 4.576089141890407} +{"Finetune/Learning Rate": 3.746709586689323e-08, "Finetune/Loss": 0.8352752923965454, "Finetune/Loss (Raw)": 0.9569642543792725, "Finetune/Step": 5059, "Finetune/Step Time": 4.573650045320392} +{"Finetune/Learning Rate": 3.693027102064206e-08, "Finetune/Loss": 0.8358355760574341, "Finetune/Loss (Raw)": 0.974249005317688, "Finetune/Step": 5060, "Finetune/Step Time": 4.574031572788954} +{"Finetune/Learning Rate": 3.639731264724056e-08, "Finetune/Loss": 0.8367261290550232, "Finetune/Loss (Raw)": 0.9099432826042175, "Finetune/Step": 5061, "Finetune/Step Time": 4.556672701612115} +{"Finetune/Learning Rate": 3.586822095351772e-08, "Finetune/Loss": 0.8356485366821289, "Finetune/Loss (Raw)": 0.7923769354820251, "Finetune/Step": 5062, "Finetune/Step Time": 4.561248999089003} +{"Finetune/Learning Rate": 3.5342996144805966e-08, "Finetune/Loss": 0.8356977105140686, "Finetune/Loss (Raw)": 0.943494975566864, "Finetune/Step": 5063, "Finetune/Step Time": 4.558149481192231} +{"Finetune/Learning Rate": 3.482163842493447e-08, "Finetune/Loss": 0.8354300856590271, "Finetune/Loss (Raw)": 0.7593101263046265, "Finetune/Step": 5064, "Finetune/Step Time": 4.55331963673234} +{"Finetune/Learning Rate": 3.4304147996230275e-08, "Finetune/Loss": 0.8350297212600708, "Finetune/Loss (Raw)": 0.9613779783248901, "Finetune/Step": 5065, "Finetune/Step Time": 4.559677744284272} +{"Finetune/Learning Rate": 3.379052505952274e-08, "Finetune/Loss": 0.835177481174469, "Finetune/Loss (Raw)": 0.9353166222572327, "Finetune/Step": 5066, "Finetune/Step Time": 4.550457090139389} +{"Finetune/Learning Rate": 3.328076981413908e-08, "Finetune/Loss": 0.8389517664909363, "Finetune/Loss (Raw)": 0.8224489688873291, "Finetune/Step": 5067, "Finetune/Step Time": 4.516146784648299} +{"Finetune/Learning Rate": 3.2774882457904386e-08, "Finetune/Loss": 0.8406113982200623, "Finetune/Loss (Raw)": 0.9038544297218323, "Finetune/Step": 5068, "Finetune/Step Time": 4.5089955143630505} +{"Finetune/Learning Rate": 3.227286318714384e-08, "Finetune/Loss": 0.8408377766609192, "Finetune/Loss (Raw)": 0.7808278799057007, "Finetune/Step": 5069, "Finetune/Step Time": 4.5109536070376635} +{"Finetune/Learning Rate": 3.177471219668271e-08, "Finetune/Loss": 0.8407005071640015, "Finetune/Loss (Raw)": 0.8052180409431458, "Finetune/Step": 5070, "Finetune/Step Time": 4.508230781182647} +{"Finetune/Learning Rate": 3.1280429679841905e-08, "Finetune/Loss": 0.8410289287567139, "Finetune/Loss (Raw)": 0.832684338092804, "Finetune/Step": 5071, "Finetune/Step Time": 4.514799784868956} +{"Finetune/Learning Rate": 3.079001582844354e-08, "Finetune/Loss": 0.8422133922576904, "Finetune/Loss (Raw)": 1.0725127458572388, "Finetune/Step": 5072, "Finetune/Step Time": 4.523136306554079} +{"Finetune/Learning Rate": 3.0303470832806494e-08, "Finetune/Loss": 0.8418782949447632, "Finetune/Loss (Raw)": 0.9014502167701721, "Finetune/Step": 5073, "Finetune/Step Time": 4.527401514351368} +{"Finetune/Learning Rate": 2.9820794881751934e-08, "Finetune/Loss": 0.8425097465515137, "Finetune/Loss (Raw)": 0.9438770413398743, "Finetune/Step": 5074, "Finetune/Step Time": 4.523068366572261} +{"Finetune/Learning Rate": 2.9341988162595593e-08, "Finetune/Loss": 0.8381221890449524, "Finetune/Loss (Raw)": 0.3655209243297577, "Finetune/Step": 5075, "Finetune/Step Time": 4.553712833672762} +{"Finetune/Learning Rate": 2.886705086115216e-08, "Finetune/Loss": 0.8390206098556519, "Finetune/Loss (Raw)": 0.9355214834213257, "Finetune/Step": 5076, "Finetune/Step Time": 4.552174691110849} +{"Finetune/Learning Rate": 2.8395983161736418e-08, "Finetune/Loss": 0.8393526077270508, "Finetune/Loss (Raw)": 0.8514935970306396, "Finetune/Step": 5077, "Finetune/Step Time": 4.573760783299804} +{"Finetune/Learning Rate": 2.7928785247161027e-08, "Finetune/Loss": 0.8397500514984131, "Finetune/Loss (Raw)": 0.8656786680221558, "Finetune/Step": 5078, "Finetune/Step Time": 4.552092123776674} +{"Finetune/Learning Rate": 2.7465457298736508e-08, "Finetune/Loss": 0.840583086013794, "Finetune/Loss (Raw)": 1.014279842376709, "Finetune/Step": 5079, "Finetune/Step Time": 4.552677683532238} +{"Finetune/Learning Rate": 2.700599949626903e-08, "Finetune/Loss": 0.8409074544906616, "Finetune/Loss (Raw)": 0.9640753865242004, "Finetune/Step": 5080, "Finetune/Step Time": 4.545118885114789} +{"Finetune/Learning Rate": 2.6550412018067072e-08, "Finetune/Loss": 0.8400862812995911, "Finetune/Loss (Raw)": 0.7166787385940552, "Finetune/Step": 5081, "Finetune/Step Time": 4.541546808555722} +{"Finetune/Learning Rate": 2.6098695040935874e-08, "Finetune/Loss": 0.8355472087860107, "Finetune/Loss (Raw)": 0.34440019726753235, "Finetune/Step": 5082, "Finetune/Step Time": 4.523649174720049} +{"Finetune/Learning Rate": 2.5650848740177425e-08, "Finetune/Loss": 0.8355206847190857, "Finetune/Loss (Raw)": 0.8649884462356567, "Finetune/Step": 5083, "Finetune/Step Time": 4.528453214094043} +{"Finetune/Learning Rate": 2.5206873289590484e-08, "Finetune/Loss": 0.8341681361198425, "Finetune/Loss (Raw)": 0.7628514170646667, "Finetune/Step": 5084, "Finetune/Step Time": 4.533012775704265} +{"Finetune/Learning Rate": 2.4766768861475e-08, "Finetune/Loss": 0.8333454728126526, "Finetune/Loss (Raw)": 0.8654208779335022, "Finetune/Step": 5085, "Finetune/Step Time": 4.536507917568088} +{"Finetune/Learning Rate": 2.4330535626627684e-08, "Finetune/Loss": 0.8347153663635254, "Finetune/Loss (Raw)": 0.98625648021698, "Finetune/Step": 5086, "Finetune/Step Time": 4.528781579807401} +{"Finetune/Learning Rate": 2.389817375433978e-08, "Finetune/Loss": 0.8383879661560059, "Finetune/Loss (Raw)": 0.9104135036468506, "Finetune/Step": 5087, "Finetune/Step Time": 4.513671707361937} +{"Finetune/Learning Rate": 2.3469683412403745e-08, "Finetune/Loss": 0.8397687673568726, "Finetune/Loss (Raw)": 1.0072160959243774, "Finetune/Step": 5088, "Finetune/Step Time": 4.507461074739695} +{"Finetune/Learning Rate": 2.304506476710877e-08, "Finetune/Loss": 0.840697169303894, "Finetune/Loss (Raw)": 0.9298092126846313, "Finetune/Step": 5089, "Finetune/Step Time": 4.511799190193415} +{"Finetune/Learning Rate": 2.262431798323972e-08, "Finetune/Loss": 0.8397656083106995, "Finetune/Loss (Raw)": 0.803341805934906, "Finetune/Step": 5090, "Finetune/Step Time": 4.507571659982204} +{"Finetune/Learning Rate": 2.2207443224081528e-08, "Finetune/Loss": 0.8406285047531128, "Finetune/Loss (Raw)": 0.9526512026786804, "Finetune/Step": 5091, "Finetune/Step Time": 4.515269920229912} +{"Finetune/Learning Rate": 2.1794440651414782e-08, "Finetune/Loss": 0.8357570171356201, "Finetune/Loss (Raw)": 0.32422757148742676, "Finetune/Step": 5092, "Finetune/Step Time": 4.54911402054131} +{"Finetune/Learning Rate": 2.1385310425517945e-08, "Finetune/Loss": 0.8351424932479858, "Finetune/Loss (Raw)": 0.8973434567451477, "Finetune/Step": 5093, "Finetune/Step Time": 4.550475209951401} +{"Finetune/Learning Rate": 2.0980052705165123e-08, "Finetune/Loss": 0.8353022933006287, "Finetune/Loss (Raw)": 0.819745659828186, "Finetune/Step": 5094, "Finetune/Step Time": 4.5653898902237415} +{"Finetune/Learning Rate": 2.05786676476305e-08, "Finetune/Loss": 0.8335316181182861, "Finetune/Loss (Raw)": 0.7033880949020386, "Finetune/Step": 5095, "Finetune/Step Time": 4.572536269202828} +{"Finetune/Learning Rate": 2.0181155408682816e-08, "Finetune/Loss": 0.8328343033790588, "Finetune/Loss (Raw)": 0.7965179681777954, "Finetune/Step": 5096, "Finetune/Step Time": 4.600417796522379} +{"Finetune/Learning Rate": 1.978751614258978e-08, "Finetune/Loss": 0.8310100436210632, "Finetune/Loss (Raw)": 0.6446460485458374, "Finetune/Step": 5097, "Finetune/Step Time": 4.5962960831820965} +{"Finetune/Learning Rate": 1.9397750002113634e-08, "Finetune/Loss": 0.8292823433876038, "Finetune/Loss (Raw)": 0.7226765751838684, "Finetune/Step": 5098, "Finetune/Step Time": 4.5979132652282715} +{"Finetune/Learning Rate": 1.901185713851672e-08, "Finetune/Loss": 0.8274962902069092, "Finetune/Loss (Raw)": 0.7843096256256104, "Finetune/Step": 5099, "Finetune/Step Time": 4.590878808870912} +{"Finetune/Learning Rate": 1.8629837701554797e-08, "Finetune/Loss": 0.826796293258667, "Finetune/Loss (Raw)": 0.7746339440345764, "Finetune/Step": 5100, "Finetune/Step Time": 4.577012475579977} +{"Finetune/Learning Rate": 1.825169183948372e-08, "Finetune/Loss": 0.8251415491104126, "Finetune/Loss (Raw)": 0.7902572751045227, "Finetune/Step": 5101, "Finetune/Step Time": 4.57315781712532} +{"Finetune/Learning Rate": 1.7877419699053876e-08, "Finetune/Loss": 0.829742431640625, "Finetune/Loss (Raw)": 0.8937081694602966, "Finetune/Step": 5102, "Finetune/Step Time": 4.538100028410554} +{"Finetune/Learning Rate": 1.7507021425512417e-08, "Finetune/Loss": 0.8308420777320862, "Finetune/Loss (Raw)": 0.9549049139022827, "Finetune/Step": 5103, "Finetune/Step Time": 4.533134568482637} +{"Finetune/Learning Rate": 1.7140497162605464e-08, "Finetune/Loss": 0.828103244304657, "Finetune/Loss (Raw)": 0.641177237033844, "Finetune/Step": 5104, "Finetune/Step Time": 4.535351250320673} +{"Finetune/Learning Rate": 1.677784705257146e-08, "Finetune/Loss": 0.8291398286819458, "Finetune/Loss (Raw)": 0.8836779594421387, "Finetune/Step": 5105, "Finetune/Step Time": 4.513822041451931} +{"Finetune/Learning Rate": 1.6419071236148943e-08, "Finetune/Loss": 0.830570638179779, "Finetune/Loss (Raw)": 0.9889007210731506, "Finetune/Step": 5106, "Finetune/Step Time": 4.520577369257808} +{"Finetune/Learning Rate": 1.6064169852572088e-08, "Finetune/Loss": 0.8316938877105713, "Finetune/Loss (Raw)": 0.9649208784103394, "Finetune/Step": 5107, "Finetune/Step Time": 4.5195978581905365} +{"Finetune/Learning Rate": 1.5713143039571833e-08, "Finetune/Loss": 0.8328404426574707, "Finetune/Loss (Raw)": 0.8536704778671265, "Finetune/Step": 5108, "Finetune/Step Time": 4.49805941991508} +{"Finetune/Learning Rate": 1.5365990933373653e-08, "Finetune/Loss": 0.834353506565094, "Finetune/Loss (Raw)": 0.9629767537117004, "Finetune/Step": 5109, "Finetune/Step Time": 4.506349267438054} +{"Finetune/Learning Rate": 1.5022713668699785e-08, "Finetune/Loss": 0.8337610960006714, "Finetune/Loss (Raw)": 0.8824856877326965, "Finetune/Step": 5110, "Finetune/Step Time": 4.518509041517973} +{"Finetune/Learning Rate": 1.4683311378771437e-08, "Finetune/Loss": 0.8364206552505493, "Finetune/Loss (Raw)": 0.7527324557304382, "Finetune/Step": 5111, "Finetune/Step Time": 4.5064343474805355} +{"Finetune/Learning Rate": 1.434778419530325e-08, "Finetune/Loss": 0.8330034613609314, "Finetune/Loss (Raw)": 0.34836697578430176, "Finetune/Step": 5112, "Finetune/Step Time": 4.533348688855767} +{"Finetune/Learning Rate": 1.401613224850551e-08, "Finetune/Loss": 0.8272964358329773, "Finetune/Loss (Raw)": 0.291111022233963, "Finetune/Step": 5113, "Finetune/Step Time": 4.576060665771365} +{"Finetune/Learning Rate": 1.3688355667087483e-08, "Finetune/Loss": 0.8285031318664551, "Finetune/Loss (Raw)": 0.8569998145103455, "Finetune/Step": 5114, "Finetune/Step Time": 4.5777767188847065} +{"Finetune/Learning Rate": 1.3364454578252972e-08, "Finetune/Loss": 0.8273850679397583, "Finetune/Loss (Raw)": 0.7774908542633057, "Finetune/Step": 5115, "Finetune/Step Time": 4.569558762013912} +{"Finetune/Learning Rate": 1.3044429107700319e-08, "Finetune/Loss": 0.8220300078392029, "Finetune/Loss (Raw)": 0.2589874863624573, "Finetune/Step": 5116, "Finetune/Step Time": 4.600249882787466} +{"Finetune/Learning Rate": 1.272827937962573e-08, "Finetune/Loss": 0.8229403495788574, "Finetune/Loss (Raw)": 1.0435175895690918, "Finetune/Step": 5117, "Finetune/Step Time": 4.601992815732956} +{"Finetune/Learning Rate": 1.2416005516722174e-08, "Finetune/Loss": 0.8239713907241821, "Finetune/Loss (Raw)": 0.9930425882339478, "Finetune/Step": 5118, "Finetune/Step Time": 4.604707196354866} +{"Finetune/Learning Rate": 1.2107607640174934e-08, "Finetune/Loss": 0.8254832625389099, "Finetune/Loss (Raw)": 1.0234119892120361, "Finetune/Step": 5119, "Finetune/Step Time": 4.604793580248952} +{"Finetune/Learning Rate": 1.1803085869667164e-08, "Finetune/Loss": 0.820428729057312, "Finetune/Loss (Raw)": 0.34329739212989807, "Finetune/Step": 5120, "Finetune/Step Time": 4.626382976770401} +{"Finetune/Learning Rate": 1.1502440323379882e-08, "Finetune/Loss": 0.8221218585968018, "Finetune/Loss (Raw)": 0.8133291602134705, "Finetune/Step": 5121, "Finetune/Step Time": 4.612711234018207} +{"Finetune/Learning Rate": 1.1205671117985318e-08, "Finetune/Loss": 0.821666955947876, "Finetune/Loss (Raw)": 0.7801941633224487, "Finetune/Step": 5122, "Finetune/Step Time": 4.604250526055694} +{"Finetune/Learning Rate": 1.0912778368655786e-08, "Finetune/Loss": 0.8238216042518616, "Finetune/Loss (Raw)": 0.8971843719482422, "Finetune/Step": 5123, "Finetune/Step Time": 4.60649568028748} +{"Finetune/Learning Rate": 1.0623762189057029e-08, "Finetune/Loss": 0.8232104778289795, "Finetune/Loss (Raw)": 0.9413402080535889, "Finetune/Step": 5124, "Finetune/Step Time": 4.607576912268996} +{"Finetune/Learning Rate": 1.0338622691349332e-08, "Finetune/Loss": 0.8236750364303589, "Finetune/Loss (Raw)": 0.9939080476760864, "Finetune/Step": 5125, "Finetune/Step Time": 4.61217724531889} +{"Finetune/Learning Rate": 1.005735998619084e-08, "Finetune/Loss": 0.8181347250938416, "Finetune/Loss (Raw)": 0.3879791498184204, "Finetune/Step": 5126, "Finetune/Step Time": 4.64830001629889} +{"Finetune/Learning Rate": 9.779974182733131e-09, "Finetune/Loss": 0.8171400427818298, "Finetune/Loss (Raw)": 0.8646010160446167, "Finetune/Step": 5127, "Finetune/Step Time": 4.6615541353821754} +{"Finetune/Learning Rate": 9.506465388624541e-09, "Finetune/Loss": 0.8211151957511902, "Finetune/Loss (Raw)": 0.8608118891716003, "Finetune/Step": 5128, "Finetune/Step Time": 4.627301953732967} +{"Finetune/Learning Rate": 9.236833710010163e-09, "Finetune/Loss": 0.8160953521728516, "Finetune/Loss (Raw)": 0.2938733696937561, "Finetune/Step": 5129, "Finetune/Step Time": 4.666144831106067} +{"Finetune/Learning Rate": 8.971079251526293e-09, "Finetune/Loss": 0.8172929286956787, "Finetune/Loss (Raw)": 0.8383169770240784, "Finetune/Step": 5130, "Finetune/Step Time": 4.661061763763428} +{"Finetune/Learning Rate": 8.709202116308214e-09, "Finetune/Loss": 0.8167185187339783, "Finetune/Loss (Raw)": 0.7275708913803101, "Finetune/Step": 5131, "Finetune/Step Time": 4.666798433288932} +{"Finetune/Learning Rate": 8.451202405985736e-09, "Finetune/Loss": 0.8168915510177612, "Finetune/Loss (Raw)": 0.7598973512649536, "Finetune/Step": 5132, "Finetune/Step Time": 4.661248134449124} +{"Finetune/Learning Rate": 8.197080220683217e-09, "Finetune/Loss": 0.8169374465942383, "Finetune/Loss (Raw)": 0.8743560910224915, "Finetune/Step": 5133, "Finetune/Step Time": 4.661983700469136} +{"Finetune/Learning Rate": 7.946835659020657e-09, "Finetune/Loss": 0.8169752359390259, "Finetune/Loss (Raw)": 0.8886807560920715, "Finetune/Step": 5134, "Finetune/Step Time": 4.665429875254631} +{"Finetune/Learning Rate": 7.700468818111483e-09, "Finetune/Loss": 0.8164646029472351, "Finetune/Loss (Raw)": 0.7281843423843384, "Finetune/Step": 5135, "Finetune/Step Time": 4.659140923991799} +{"Finetune/Learning Rate": 7.457979793568105e-09, "Finetune/Loss": 0.8183885216712952, "Finetune/Loss (Raw)": 1.0367281436920166, "Finetune/Step": 5136, "Finetune/Step Time": 4.6638964135199785} +{"Finetune/Learning Rate": 7.219368679495242e-09, "Finetune/Loss": 0.8169125914573669, "Finetune/Loss (Raw)": 0.8227138519287109, "Finetune/Step": 5137, "Finetune/Step Time": 4.66693994961679} +{"Finetune/Learning Rate": 6.984635568492159e-09, "Finetune/Loss": 0.8160313963890076, "Finetune/Loss (Raw)": 0.8312481045722961, "Finetune/Step": 5138, "Finetune/Step Time": 4.680870965123177} +{"Finetune/Learning Rate": 6.753780551654876e-09, "Finetune/Loss": 0.8119337558746338, "Finetune/Loss (Raw)": 0.34127920866012573, "Finetune/Step": 5139, "Finetune/Step Time": 4.716981632634997} +{"Finetune/Learning Rate": 6.526803718572838e-09, "Finetune/Loss": 0.8125636577606201, "Finetune/Loss (Raw)": 0.9251086115837097, "Finetune/Step": 5140, "Finetune/Step Time": 4.711815293878317} +{"Finetune/Learning Rate": 6.3037051573333615e-09, "Finetune/Loss": 0.8144631385803223, "Finetune/Loss (Raw)": 0.9751397967338562, "Finetune/Step": 5141, "Finetune/Step Time": 4.709172781556845} +{"Finetune/Learning Rate": 6.0844849545149685e-09, "Finetune/Loss": 0.8147205114364624, "Finetune/Loss (Raw)": 0.8250183463096619, "Finetune/Step": 5142, "Finetune/Step Time": 4.713570652529597} +{"Finetune/Learning Rate": 5.869143195192939e-09, "Finetune/Loss": 0.8161175847053528, "Finetune/Loss (Raw)": 0.9544312357902527, "Finetune/Step": 5143, "Finetune/Step Time": 4.716582242399454} +{"Finetune/Learning Rate": 5.657679962938201e-09, "Finetune/Loss": 0.8145787715911865, "Finetune/Loss (Raw)": 0.6674931049346924, "Finetune/Step": 5144, "Finetune/Step Time": 4.7221290320158005} +{"Finetune/Learning Rate": 5.4500953398139985e-09, "Finetune/Loss": 0.8148623108863831, "Finetune/Loss (Raw)": 0.823225736618042, "Finetune/Step": 5145, "Finetune/Step Time": 4.717090051621199} +{"Finetune/Learning Rate": 5.246389406381447e-09, "Finetune/Loss": 0.8168666362762451, "Finetune/Loss (Raw)": 1.0060553550720215, "Finetune/Step": 5146, "Finetune/Step Time": 4.712196171283722} +{"Finetune/Learning Rate": 5.046562241693975e-09, "Finetune/Loss": 0.8174524307250977, "Finetune/Loss (Raw)": 0.8595134615898132, "Finetune/Step": 5147, "Finetune/Step Time": 4.7143084444105625} +{"Finetune/Learning Rate": 4.850613923300662e-09, "Finetune/Loss": 0.8199818134307861, "Finetune/Loss (Raw)": 0.6696807146072388, "Finetune/Step": 5148, "Finetune/Step Time": 4.682695064693689} +{"Finetune/Learning Rate": 4.658544527244013e-09, "Finetune/Loss": 0.8191180229187012, "Finetune/Loss (Raw)": 0.8203850984573364, "Finetune/Step": 5149, "Finetune/Step Time": 4.6834083292633295} +{"Finetune/Learning Rate": 4.470354128065513e-09, "Finetune/Loss": 0.8186467885971069, "Finetune/Loss (Raw)": 0.8262009620666504, "Finetune/Step": 5150, "Finetune/Step Time": 4.690213585272431} +{"Finetune/Learning Rate": 4.286042798794521e-09, "Finetune/Loss": 0.8180590867996216, "Finetune/Loss (Raw)": 0.8028003573417664, "Finetune/Step": 5151, "Finetune/Step Time": 4.686870591714978} +{"Finetune/Learning Rate": 4.105610610961597e-09, "Finetune/Loss": 0.8177010416984558, "Finetune/Loss (Raw)": 0.7788915634155273, "Finetune/Step": 5152, "Finetune/Step Time": 4.688561489805579} +{"Finetune/Learning Rate": 3.929057634587397e-09, "Finetune/Loss": 0.8145145773887634, "Finetune/Loss (Raw)": 0.42267948389053345, "Finetune/Step": 5153, "Finetune/Step Time": 4.7141847014427185} +{"Finetune/Learning Rate": 3.756383938189334e-09, "Finetune/Loss": 0.810171365737915, "Finetune/Loss (Raw)": 0.3720909059047699, "Finetune/Step": 5154, "Finetune/Step Time": 4.747102376073599} +{"Finetune/Learning Rate": 3.587589588778251e-09, "Finetune/Loss": 0.8089275360107422, "Finetune/Loss (Raw)": 0.7609914541244507, "Finetune/Step": 5155, "Finetune/Step Time": 4.7477653454989195} +{"Finetune/Learning Rate": 3.4226746518606356e-09, "Finetune/Loss": 0.8088311553001404, "Finetune/Loss (Raw)": 0.9591543674468994, "Finetune/Step": 5156, "Finetune/Step Time": 4.760331194847822} +{"Finetune/Learning Rate": 3.2616391914364056e-09, "Finetune/Loss": 0.808634877204895, "Finetune/Loss (Raw)": 0.8619716763496399, "Finetune/Step": 5157, "Finetune/Step Time": 4.765398474410176} +{"Finetune/Learning Rate": 3.1044832699989035e-09, "Finetune/Loss": 0.8088358640670776, "Finetune/Loss (Raw)": 0.8490462303161621, "Finetune/Step": 5158, "Finetune/Step Time": 4.762538863345981} +{"Finetune/Learning Rate": 2.9512069485393426e-09, "Finetune/Loss": 0.8092714548110962, "Finetune/Loss (Raw)": 0.9484085440635681, "Finetune/Step": 5159, "Finetune/Step Time": 4.765029326081276} +{"Finetune/Learning Rate": 2.801810286540141e-09, "Finetune/Loss": 0.8097012042999268, "Finetune/Loss (Raw)": 1.003827452659607, "Finetune/Step": 5160, "Finetune/Step Time": 4.764062894508243} +{"Finetune/Learning Rate": 2.656293341980476e-09, "Finetune/Loss": 0.8102667331695557, "Finetune/Loss (Raw)": 0.9235995411872864, "Finetune/Step": 5161, "Finetune/Step Time": 4.785664198920131} +{"Finetune/Learning Rate": 2.5146561713307315e-09, "Finetune/Loss": 0.8093655109405518, "Finetune/Loss (Raw)": 0.7995260953903198, "Finetune/Step": 5162, "Finetune/Step Time": 4.790686318650842} +{"Finetune/Learning Rate": 2.3768988295580497e-09, "Finetune/Loss": 0.8102402687072754, "Finetune/Loss (Raw)": 0.9057254791259766, "Finetune/Step": 5163, "Finetune/Step Time": 4.786763735115528} +{"Finetune/Learning Rate": 2.2430213701252203e-09, "Finetune/Loss": 0.8113843202590942, "Finetune/Loss (Raw)": 0.972445011138916, "Finetune/Step": 5164, "Finetune/Step Time": 4.800449077039957} +{"Finetune/Learning Rate": 2.1130238449851292e-09, "Finetune/Loss": 0.8094314932823181, "Finetune/Loss (Raw)": 0.7207319736480713, "Finetune/Step": 5165, "Finetune/Step Time": 4.810684870928526} +{"Finetune/Learning Rate": 1.986906304588532e-09, "Finetune/Loss": 0.8089358806610107, "Finetune/Loss (Raw)": 0.911317765712738, "Finetune/Step": 5166, "Finetune/Step Time": 4.823130948469043} +{"Finetune/Learning Rate": 1.8646687978784992e-09, "Finetune/Loss": 0.8067365288734436, "Finetune/Loss (Raw)": 0.7913397550582886, "Finetune/Step": 5167, "Finetune/Step Time": 4.815750142559409} +{"Finetune/Learning Rate": 1.7463113722937518e-09, "Finetune/Loss": 0.8083350658416748, "Finetune/Loss (Raw)": 0.9346392154693604, "Finetune/Step": 5168, "Finetune/Step Time": 4.800921907648444} +{"Finetune/Learning Rate": 1.631834073765326e-09, "Finetune/Loss": 0.8078809976577759, "Finetune/Loss (Raw)": 0.9233787655830383, "Finetune/Step": 5169, "Finetune/Step Time": 4.8020276837050915} +{"Finetune/Learning Rate": 1.5212369467210164e-09, "Finetune/Loss": 0.806023359298706, "Finetune/Loss (Raw)": 0.7264970541000366, "Finetune/Step": 5170, "Finetune/Step Time": 4.798842066898942} +{"Finetune/Learning Rate": 1.4145200340809352e-09, "Finetune/Loss": 0.8070243000984192, "Finetune/Loss (Raw)": 0.9990417957305908, "Finetune/Step": 5171, "Finetune/Step Time": 4.799951883032918} +{"Finetune/Learning Rate": 1.3116833772597315e-09, "Finetune/Loss": 0.8070493340492249, "Finetune/Loss (Raw)": 0.8563520312309265, "Finetune/Step": 5172, "Finetune/Step Time": 4.796361718326807} +{"Finetune/Learning Rate": 1.2127270161654824e-09, "Finetune/Loss": 0.8109406232833862, "Finetune/Loss (Raw)": 0.7877905368804932, "Finetune/Step": 5173, "Finetune/Step Time": 4.758662478998303} +{"Finetune/Learning Rate": 1.1176509892030229e-09, "Finetune/Loss": 0.8169038891792297, "Finetune/Loss (Raw)": 1.095379114151001, "Finetune/Step": 5174, "Finetune/Step Time": 4.716827597469091} +{"Finetune/Learning Rate": 1.0264553332683946e-09, "Finetune/Loss": 0.8167940974235535, "Finetune/Loss (Raw)": 0.8861856460571289, "Finetune/Step": 5175, "Finetune/Step Time": 4.723278550431132} +{"Finetune/Learning Rate": 9.39140083752177e-10, "Finetune/Loss": 0.8117865324020386, "Finetune/Loss (Raw)": 0.3262117803096771, "Finetune/Step": 5176, "Finetune/Step Time": 4.759124925360084} +{"Finetune/Learning Rate": 8.557052745405969e-10, "Finetune/Loss": 0.8127683401107788, "Finetune/Loss (Raw)": 0.9840594530105591, "Finetune/Step": 5177, "Finetune/Step Time": 4.75282434746623} +{"Finetune/Learning Rate": 7.761509380133092e-10, "Finetune/Loss": 0.8121607303619385, "Finetune/Loss (Raw)": 0.7595160007476807, "Finetune/Step": 5178, "Finetune/Step Time": 4.737157680094242} +{"Finetune/Learning Rate": 7.004771050433956e-10, "Finetune/Loss": 0.8122681379318237, "Finetune/Loss (Raw)": 0.7894446849822998, "Finetune/Step": 5179, "Finetune/Step Time": 4.733074180781841} +{"Finetune/Learning Rate": 6.286838049984755e-10, "Finetune/Loss": 0.8154939413070679, "Finetune/Loss (Raw)": 0.7849586009979248, "Finetune/Step": 5180, "Finetune/Step Time": 4.703750293701887} +{"Finetune/Learning Rate": 5.607710657407061e-10, "Finetune/Loss": 0.8161880970001221, "Finetune/Loss (Raw)": 0.8975403904914856, "Finetune/Step": 5181, "Finetune/Step Time": 4.69875431060791} +{"Finetune/Learning Rate": 4.967389136245615e-10, "Finetune/Loss": 0.81394362449646, "Finetune/Loss (Raw)": 0.7177363038063049, "Finetune/Step": 5182, "Finetune/Step Time": 4.7019178085029125} +{"Finetune/Learning Rate": 4.3658737350016354e-10, "Finetune/Loss": 0.8144258260726929, "Finetune/Loss (Raw)": 0.8953983187675476, "Finetune/Step": 5183, "Finetune/Step Time": 4.706363391131163} +{"Finetune/Learning Rate": 3.8031646871106166e-10, "Finetune/Loss": 0.8142085075378418, "Finetune/Loss (Raw)": 0.8602074980735779, "Finetune/Step": 5184, "Finetune/Step Time": 4.716550527140498} +{"Finetune/Learning Rate": 3.279262210953427e-10, "Finetune/Loss": 0.8150578737258911, "Finetune/Loss (Raw)": 0.9887391924858093, "Finetune/Step": 5185, "Finetune/Step Time": 4.727621972560883} +{"Finetune/Learning Rate": 2.794166509845209e-10, "Finetune/Loss": 0.8111138343811035, "Finetune/Loss (Raw)": 0.38085901737213135, "Finetune/Step": 5186, "Finetune/Step Time": 4.76259039901197} +{"Finetune/Learning Rate": 2.3478777720353785e-10, "Finetune/Loss": 0.8094869256019592, "Finetune/Loss (Raw)": 0.748725414276123, "Finetune/Step": 5187, "Finetune/Step Time": 4.780921475961804} +{"Finetune/Learning Rate": 1.940396170718728e-10, "Finetune/Loss": 0.8082288503646851, "Finetune/Loss (Raw)": 0.8132195472717285, "Finetune/Step": 5188, "Finetune/Step Time": 4.78179619833827} +{"Finetune/Learning Rate": 1.5717218640465272e-10, "Finetune/Loss": 0.807079553604126, "Finetune/Loss (Raw)": 0.762830913066864, "Finetune/Step": 5189, "Finetune/Step Time": 4.794712234288454} +{"Finetune/Learning Rate": 1.2418549950821145e-10, "Finetune/Loss": 0.8066836595535278, "Finetune/Loss (Raw)": 0.7417012453079224, "Finetune/Step": 5190, "Finetune/Step Time": 4.792918341234326} +{"Finetune/Learning Rate": 9.507956918453076e-11, "Finetune/Loss": 0.8050257563591003, "Finetune/Loss (Raw)": 0.7312816381454468, "Finetune/Step": 5191, "Finetune/Step Time": 4.805547337979078} +{"Finetune/Learning Rate": 6.985440672790945e-11, "Finetune/Loss": 0.8056857585906982, "Finetune/Loss (Raw)": 0.8437871336936951, "Finetune/Step": 5192, "Finetune/Step Time": 4.801188215613365} +{"Finetune/Learning Rate": 4.851002192940435e-11, "Finetune/Loss": 0.8039178848266602, "Finetune/Loss (Raw)": 0.7350906133651733, "Finetune/Step": 5193, "Finetune/Step Time": 4.812956498935819} +{"Finetune/Learning Rate": 3.1046423071279254e-11, "Finetune/Loss": 0.7992229461669922, "Finetune/Loss (Raw)": 0.3343689441680908, "Finetune/Step": 5194, "Finetune/Step Time": 4.849966801702976} +{"Finetune/Learning Rate": 1.7463616930335493e-11, "Finetune/Loss": 0.7994292974472046, "Finetune/Loss (Raw)": 0.8488621115684509, "Finetune/Step": 5195, "Finetune/Step Time": 4.848378017544746} +{"Finetune/Learning Rate": 7.761608780132434e-12, "Finetune/Loss": 0.7986112833023071, "Finetune/Loss (Raw)": 0.7991523742675781, "Finetune/Step": 5196, "Finetune/Step Time": 4.854516291990876} +{"Finetune/Learning Rate": 1.9404023832159115e-12, "Finetune/Loss": 0.7993015050888062, "Finetune/Loss (Raw)": 0.8691728115081787, "Finetune/Step": 5197, "Finetune/Step Time": 4.846884090453386} +{"Finetune/Learning Rate": 0.0, "Finetune/Loss": 0.7997910976409912, "Finetune/Loss (Raw)": 0.8678842782974243, "Finetune/Step": 5198, "Finetune/Step Time": 4.842910025268793} diff --git a/qformer2_64/run-metrics.jsonl b/qformer2_64/run-metrics.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..c44d646487ad3ab367718d1a878e50b5b1f6930e --- /dev/null +++ b/qformer2_64/run-metrics.jsonl @@ -0,0 +1 @@ +{"hparams": {"dataset": {"align_stage_components": ["download/llava-laion-cc-sbu-558k/chat.json", "download/llava-laion-cc-sbu-558k"], "dataset_id": "llava-v15", "dataset_root_dir": "data", "finetune_stage_components": ["download/llava-v1.5-instruct/llava_v1_5_mix665k.json", "download/llava-v1.5-instruct"], "type": "llava-v15"}, "hf_token": ".hf_token", "max_length": 4096, "model": {"align_epochs": 1, "align_global_batch_size": 256, "align_learning_rate": 0.001, "align_lr_scheduler_type": "linear-warmup+cosine-decay", "align_max_grad_norm": 1.0, "align_max_steps": null, "align_per_device_batch_size": 16, "align_train_strategy": "fsdp-shard-grad-op", "align_warmup_ratio": 0.03, "align_weight_decay": 0.0, "arch_specifier": "qformer2_64", "enable_gradient_checkpointing": true, "enable_mixed_precision_training": true, "finetune_epochs": 1, "finetune_global_batch_size": 128, "finetune_learning_rate": 2e-05, "finetune_lr_scheduler_type": "linear-warmup+cosine-decay", "finetune_max_grad_norm": 1.0, "finetune_max_steps": null, "finetune_per_device_batch_size": 16, "finetune_train_strategy": "fsdp-full-shard", "finetune_warmup_ratio": 0.03, "finetune_weight_decay": 0.1, "image_resize_strategy": "letterbox", "llm_backbone_id": "vicuna-v15-7b", "llm_max_length": 2048, "model_id": "one-stage+7b_qformer2_64", "reduce_in_full_precision": false, "type": "one-stage+7b", "vision_backbone_id": "clip-vit-l-336px"}, "pretrained_checkpoint": null, "run_id": "qformer2_64", "run_root_dir": "runs", "seed": 7, "stage": "finetune", "trackers": ["jsonl", "wandb"], "wandb_entity": "lilei_stones", "wandb_project": "hf-vlms"}, "run_id": "qformer2_64"} diff --git a/qformer2_64/wandb/debug-internal.log b/qformer2_64/wandb/debug-internal.log new file mode 120000 index 0000000000000000000000000000000000000000..85a43f7dddfa264778a55d3b071a7aee6de7fffa --- /dev/null +++ b/qformer2_64/wandb/debug-internal.log @@ -0,0 +1 @@ +run-20240312_142831-b1zz0q9t/logs/debug-internal.log \ No newline at end of file diff --git a/qformer2_64/wandb/debug.log b/qformer2_64/wandb/debug.log new file mode 120000 index 0000000000000000000000000000000000000000..341e625bfe5cc5da5bf8c104fddfba387c69817c --- /dev/null +++ b/qformer2_64/wandb/debug.log @@ -0,0 +1 @@ +run-20240312_142831-b1zz0q9t/logs/debug.log \ No newline at end of file diff --git a/qformer2_64/wandb/latest-run b/qformer2_64/wandb/latest-run new file mode 120000 index 0000000000000000000000000000000000000000..822bc5131e3680fb388acc284c30bcbd509b9f5d --- /dev/null +++ b/qformer2_64/wandb/latest-run @@ -0,0 +1 @@ +run-20240312_142831-b1zz0q9t \ No newline at end of file diff --git a/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/conda-environment.yaml b/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/conda-environment.yaml new file mode 100644 index 0000000000000000000000000000000000000000..799a6f3a49522423b58a3ae238b95584ecfe64c1 --- /dev/null +++ b/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/conda-environment.yaml @@ -0,0 +1,194 @@ +name: pvlm +channels: + - pytorch + - nvidia + - defaults +dependencies: + - _libgcc_mutex=0.1=main + - _openmp_mutex=5.1=1_gnu + - blas=1.0=mkl + - bzip2=1.0.8=h7b6447c_0 + - ca-certificates=2023.12.12=h06a4308_0 + - certifi=2024.2.2=py310h06a4308_0 + - charset-normalizer=2.0.4=pyhd3eb1b0_0 + - cuda-cudart=11.8.89=0 + - cuda-cupti=11.8.87=0 + - cuda-libraries=11.8.0=0 + - cuda-nvrtc=11.8.89=0 + - cuda-nvtx=11.8.86=0 + - cuda-runtime=11.8.0=0 + - ffmpeg=4.3=hf484d3e_0 + - filelock=3.13.1=py310h06a4308_0 + - freetype=2.12.1=h4a9f257_0 + - gmp=6.2.1=h295c915_3 + - gmpy2=2.1.2=py310heeb90bb_0 + - gnutls=3.6.15=he1e5248_0 + - idna=3.4=py310h06a4308_0 + - intel-openmp=2023.1.0=hdb19cb5_46306 + - jinja2=3.1.3=py310h06a4308_0 + - jpeg=9e=h5eee18b_1 + - lame=3.100=h7b6447c_0 + - lcms2=2.12=h3be6417_0 + - ld_impl_linux-64=2.38=h1181459_1 + - lerc=3.0=h295c915_0 + - libcublas=11.11.3.6=0 + - libcufft=10.9.0.58=0 + - libcufile=1.8.1.2=0 + - libcurand=10.3.4.107=0 + - libcusolver=11.4.1.48=0 + - libcusparse=11.7.5.86=0 + - libdeflate=1.17=h5eee18b_1 + - libffi=3.4.4=h6a678d5_0 + - libgcc-ng=11.2.0=h1234567_1 + - libgomp=11.2.0=h1234567_1 + - libiconv=1.16=h7f8727e_2 + - libidn2=2.3.4=h5eee18b_0 + - libjpeg-turbo=2.0.0=h9bf148f_0 + - libnpp=11.8.0.86=0 + - libnvjpeg=11.9.0.86=0 + - libpng=1.6.39=h5eee18b_0 + - libstdcxx-ng=11.2.0=h1234567_1 + - libtasn1=4.19.0=h5eee18b_0 + - libtiff=4.5.1=h6a678d5_0 + - libunistring=0.9.10=h27cfd23_0 + - libuuid=1.41.5=h5eee18b_0 + - libwebp-base=1.3.2=h5eee18b_0 + - llvm-openmp=14.0.6=h9e868ea_0 + - lz4-c=1.9.4=h6a678d5_0 + - markupsafe=2.1.3=py310h5eee18b_0 + - mkl=2023.1.0=h213fc3f_46344 + - mkl-service=2.4.0=py310h5eee18b_1 + - mkl_fft=1.3.8=py310h5eee18b_0 + - mkl_random=1.2.4=py310hdb19cb5_0 + - mpc=1.1.0=h10f8cd9_1 + - mpfr=4.0.2=hb69a4c5_1 + - mpmath=1.3.0=py310h06a4308_0 + - ncurses=6.4=h6a678d5_0 + - nettle=3.7.3=hbbd107a_1 + - networkx=3.1=py310h06a4308_0 + - numpy=1.26.4=py310h5f9d8c6_0 + - numpy-base=1.26.4=py310hb5e798b_0 + - openh264=2.1.1=h4ff587b_0 + - openjpeg=2.4.0=h3ad879b_0 + - openssl=3.0.13=h7f8727e_0 + - pillow=10.2.0=py310h5eee18b_0 + - pip=23.3.1=py310h06a4308_0 + - python=3.10.13=h955ad1f_0 + - pytorch=2.2.1=py3.10_cuda11.8_cudnn8.7.0_0 + - pytorch-cuda=11.8=h7e8668a_5 + - pytorch-mutex=1.0=cuda + - pyyaml=6.0.1=py310h5eee18b_0 + - readline=8.2=h5eee18b_0 + - requests=2.31.0=py310h06a4308_1 + - setuptools=68.2.2=py310h06a4308_0 + - sqlite=3.41.2=h5eee18b_0 + - sympy=1.12=py310h06a4308_0 + - tbb=2021.8.0=hdb19cb5_0 + - tk=8.6.12=h1ccaba5_0 + - torchaudio=2.2.1=py310_cu118 + - torchtriton=2.2.0=py310 + - torchvision=0.17.1=py310_cu118 + - typing_extensions=4.9.0=py310h06a4308_1 + - urllib3=2.1.0=py310h06a4308_0 + - wheel=0.41.2=py310h06a4308_0 + - xz=5.4.5=h5eee18b_0 + - yaml=0.2.5=h7b6447c_0 + - zlib=1.2.13=h5eee18b_0 + - zstd=1.5.5=hc292b87_0 + - pip: + - accelerate==0.27.2 + - aiohttp==3.9.3 + - aiosignal==1.3.1 + - annotated-types==0.6.0 + - appdirs==1.4.4 + - async-timeout==4.0.3 + - attrs==23.2.0 + - beautifulsoup4==4.12.3 + - blis==0.7.11 + - catalogue==2.0.10 + - click==8.1.7 + - cloudpathlib==0.16.0 + - confection==0.1.4 + - contourpy==1.2.0 + - cycler==0.12.1 + - cymem==2.0.8 + - datasets==2.18.0 + - deepspeed==0.13.4 + - dill==0.3.8 + - docker-pycreds==0.4.0 + - draccus==0.7.1 + - einops==0.7.0 + - flash-attn==2.5.5 + - fonttools==4.49.0 + - frozenlist==1.4.1 + - fsspec==2024.2.0 + - gdown==5.1.0 + - gitdb==4.0.11 + - gitpython==3.1.42 + - hjson==3.1.0 + - huggingface-hub==0.20.3 + - install==1.3.5 + - jsonlines==4.0.0 + - kiwisolver==1.4.5 + - langcodes==3.3.0 + - markdown-it-py==3.0.0 + - matplotlib==3.8.3 + - mdurl==0.1.2 + - mergedeep==1.3.4 + - multidict==6.0.5 + - multiprocess==0.70.16 + - murmurhash==1.0.10 + - mypy-extensions==1.0.0 + - ninja==1.11.1.1 + - packaging==23.2 + - pandas==2.2.1 + - preshed==3.0.9 + - prismatic==0.0.1 + - protobuf==4.25.3 + - psutil==5.9.8 + - py-cpuinfo==9.0.0 + - pyarrow==15.0.0 + - pyarrow-hotfix==0.6 + - pycocoevalcap==1.2 + - pycocotools==2.0.7 + - pydantic==2.6.3 + - pydantic-core==2.16.3 + - pygments==2.17.2 + - pynvml==11.5.0 + - pyparsing==3.1.1 + - pysocks==1.7.1 + - pythainlp==5.0.1 + - python-dateutil==2.9.0.post0 + - pytz==2024.1 + - pyyaml-include==1.3.2 + - regex==2023.12.25 + - rich==13.7.0 + - safetensors==0.4.2 + - sentencepiece==0.2.0 + - sentry-sdk==1.40.5 + - setproctitle==1.3.3 + - six==1.16.0 + - smart-open==6.4.0 + - smmap==5.0.1 + - soupsieve==2.5 + - spacy==3.7.4 + - spacy-legacy==3.0.12 + - spacy-loggers==1.0.5 + - srsly==2.4.8 + - sudachidict-core==20240109 + - sudachipy==0.6.8 + - thinc==8.2.3 + - timm==0.9.16 + - tokenizers==0.15.2 + - toml==0.10.2 + - tqdm==4.66.2 + - transformers==4.38.1 + - typer==0.9.0 + - typing-inspect==0.9.0 + - tzdata==2024.1 + - wandb==0.16.3 + - wasabi==1.1.2 + - weasel==0.3.4 + - xxhash==3.4.1 + - yarl==1.9.4 +prefix: /home/lilei/anaconda3/envs/pvlm diff --git a/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/config.yaml b/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..4a1e465233b8b95d0d87969108548d2f0e206d93 --- /dev/null +++ b/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/config.yaml @@ -0,0 +1,119 @@ +wandb_version: 1 + +model: + desc: null + value: + type: one-stage+7b + model_id: one-stage+7b_qformer2_64 + arch_specifier: qformer2_64 + vision_backbone_id: clip-vit-l-336px + llm_backbone_id: vicuna-v15-7b + image_resize_strategy: letterbox + llm_max_length: 2048 + align_epochs: 1 + align_max_steps: null + align_global_batch_size: 256 + align_per_device_batch_size: 16 + align_learning_rate: 0.001 + align_weight_decay: 0.0 + align_max_grad_norm: 1.0 + align_lr_scheduler_type: linear-warmup+cosine-decay + align_warmup_ratio: 0.03 + align_train_strategy: fsdp-shard-grad-op + finetune_epochs: 1 + finetune_max_steps: null + finetune_global_batch_size: 128 + finetune_per_device_batch_size: 16 + finetune_learning_rate: 2.0e-05 + finetune_weight_decay: 0.1 + finetune_max_grad_norm: 1.0 + finetune_lr_scheduler_type: linear-warmup+cosine-decay + finetune_warmup_ratio: 0.03 + finetune_train_strategy: fsdp-full-shard + enable_gradient_checkpointing: true + enable_mixed_precision_training: true + reduce_in_full_precision: false +dataset: + desc: null + value: + type: llava-v15 + dataset_id: llava-v15 + align_stage_components: + - download/llava-laion-cc-sbu-558k/chat.json + - download/llava-laion-cc-sbu-558k + finetune_stage_components: + - download/llava-v1.5-instruct/llava_v1_5_mix665k.json + - download/llava-v1.5-instruct + dataset_root_dir: data +stage: + desc: null + value: finetune +pretrained_checkpoint: + desc: null + value: null +run_id: + desc: null + value: qformer2_64 +run_root_dir: + desc: null + value: runs +seed: + desc: null + value: 7 +hf_token: + desc: null + value: .hf_token +trackers: + desc: null + value: + - jsonl + - wandb +wandb_project: + desc: null + value: hf-vlms +wandb_entity: + desc: null + value: lilei_stones +max_length: + desc: null + value: 4096 +_wandb: + desc: null + value: + python_version: 3.10.13 + cli_version: 0.16.3 + framework: huggingface + huggingface_version: 4.38.1 + is_jupyter_run: false + is_kaggle_kernel: false + start_time: 1710253711.46141 + t: + 1: + - 1 + - 11 + - 41 + - 49 + - 51 + - 55 + - 63 + - 71 + 2: + - 1 + - 11 + - 41 + - 49 + - 51 + - 55 + - 63 + - 71 + 3: + - 2 + - 13 + - 16 + - 23 + 4: 3.10.13 + 5: 0.16.3 + 6: 4.38.1 + 8: + - 5 + 13: linux-x86_64 diff --git a/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log b/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..8295237d257e97063a8f1712002b176c23ff7635 --- /dev/null +++ b/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log @@ -0,0 +1,5203 @@ + +03/12 [14:28:36] INFO  | >> [*] Starting Training Loop ]8;id=72103;file:///home/lilei/prismatic-vlms/scripts/pretrain.py\pretrain.py]8;;\:]8;id=880770;file:///home/lilei/prismatic-vlms/scripts/pretrain.py#226\226]8;;\ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +=>> [Global Step] 005198 =>> LR :: 0.000000 -- Loss :: 0.7998: 100%|█████████████████████████████████████████████| 5198/5198 [6:43:06<00:00, 4.26s/it] + +03/12 [21:16:03] INFO  | >> [*] Done with Training =>> Finalizing Metrics ]8;id=696414;file:///home/lilei/prismatic-vlms/scripts/pretrain.py\pretrain.py]8;;\:]8;id=68157;file:///home/lilei/prismatic-vlms/scripts/pretrain.py#230\230]8;;\ \ No newline at end of file diff --git a/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/requirements.txt b/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..52215be56573f749927d1265d23e43a5fe36d5b1 --- /dev/null +++ b/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/requirements.txt @@ -0,0 +1,122 @@ +accelerate==0.27.2 +aiohttp==3.9.3 +aiosignal==1.3.1 +annotated-types==0.6.0 +appdirs==1.4.4 +async-timeout==4.0.3 +attrs==23.2.0 +beautifulsoup4==4.12.3 +blis==0.7.11 +catalogue==2.0.10 +certifi==2024.2.2 +charset-normalizer==2.0.4 +click==8.1.7 +cloudpathlib==0.16.0 +confection==0.1.4 +contourpy==1.2.0 +cycler==0.12.1 +cymem==2.0.8 +datasets==2.18.0 +deepspeed==0.13.4 +dill==0.3.8 +docker-pycreds==0.4.0 +draccus==0.7.1 +einops==0.7.0 +filelock==3.13.1 +flash-attn==2.5.5 +fonttools==4.49.0 +frozenlist==1.4.1 +fsspec==2024.2.0 +gdown==5.1.0 +gitdb==4.0.11 +gitpython==3.1.42 +gmpy2==2.1.2 +hjson==3.1.0 +huggingface-hub==0.20.3 +idna==3.4 +install==1.3.5 +jinja2==3.1.3 +jsonlines==4.0.0 +kiwisolver==1.4.5 +langcodes==3.3.0 +markdown-it-py==3.0.0 +markupsafe==2.1.3 +matplotlib==3.8.3 +mdurl==0.1.2 +mergedeep==1.3.4 +mkl-fft==1.3.8 +mkl-random==1.2.4 +mkl-service==2.4.0 +mpmath==1.3.0 +multidict==6.0.5 +multiprocess==0.70.16 +murmurhash==1.0.10 +mypy-extensions==1.0.0 +networkx==3.1 +ninja==1.11.1.1 +numpy==1.26.4 +ocifs==1.3.1 +packaging==23.2 +pandas==2.2.1 +pillow==10.2.0 +pip==23.3.1 +preshed==3.0.9 +prismatic==0.0.1 +protobuf==4.25.3 +psutil==5.9.8 +py-cpuinfo==9.0.0 +pyarrow-hotfix==0.6 +pyarrow==15.0.0 +pycocoevalcap==1.2 +pycocotools==2.0.7 +pydantic-core==2.16.3 +pydantic==2.6.3 +pygments==2.17.2 +pynvml==11.5.0 +pyparsing==3.1.1 +pysocks==1.7.1 +pythainlp==5.0.1 +python-dateutil==2.9.0.post0 +pytz==2024.1 +pyyaml-include==1.3.2 +pyyaml==6.0.1 +regex==2023.12.25 +requests==2.31.0 +rich==13.7.0 +safetensors==0.4.2 +sentencepiece==0.2.0 +sentry-sdk==1.40.5 +setproctitle==1.3.3 +setuptools==68.2.2 +six==1.16.0 +smart-open==6.4.0 +smmap==5.0.1 +soupsieve==2.5 +spacy-legacy==3.0.12 +spacy-loggers==1.0.5 +spacy==3.7.4 +srsly==2.4.8 +sudachidict-core==20240109 +sudachipy==0.6.8 +sympy==1.12 +thinc==8.2.3 +timm==0.9.16 +tokenizers==0.15.2 +toml==0.10.2 +torch==2.2.1 +torchaudio==2.2.1 +torchvision==0.17.1 +tqdm==4.66.2 +transformers==4.38.1 +triton==2.2.0 +typer==0.9.0 +typing-extensions==4.9.0 +typing-inspect==0.9.0 +tzdata==2024.1 +urllib3==2.1.0 +wandb==0.16.3 +wasabi==1.1.2 +weasel==0.3.4 +wheel==0.41.2 +xxhash==3.4.1 +yarl==1.9.4 \ No newline at end of file diff --git a/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-metadata.json b/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..6174a88cc728428767da9b5871d2152f84b32c3a --- /dev/null +++ b/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-metadata.json @@ -0,0 +1,1360 @@ +{ + "os": "Linux-5.15.0-1027-oracle-x86_64-with-glibc2.31", + "python": "3.10.13", + "heartbeatAt": "2024-03-12T14:28:32.197467", + "startedAt": "2024-03-12T14:28:31.407967", + "docker": null, + "cuda": null, + "args": [ + "--model.type", + "one-stage+7b", + "--model.model_id", + "one-stage+7b_qformer2_64", + "--run_id", + "qformer2_64", + "--model.arch_specifier", + "qformer2_64" + ], + "state": "running", + "program": "/home/lilei/prismatic-vlms/scripts/pretrain.py", + "codePathLocal": "scripts/pretrain.py", + "codePath": "scripts/pretrain.py", + "git": { + "remote": "https://github.com/TRI-ML/prismatic-vlms.git", + "commit": "b1c33c284c6e76d908c29e5545b00f5b8b4c601f" + }, + "email": "tobiaslee@qq.com", + "root": "/home/lilei/prismatic-vlms", + "host": "compute-permanent-node-88", + "username": "lilei", + "executable": "/home/lilei/anaconda3/envs/pvlm/bin/python", + "cpu_count": 128, + "cpu_count_logical": 255, + "cpu_freq": { + "current": 2575.3999333333336, + "min": 1500.0, + "max": 2550.0 + }, + "cpu_freq_per_core": [ + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 3249.002, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 3248.267, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 3247.033, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2716.886, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 3249.005, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 3249.239, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 3249.276, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 3249.389, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 3249.335, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 3245.888, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + }, + { + "current": 2550.0, + "min": 1500.0, + "max": 2550.0 + } + ], + "disk": { + "/": { + "total": 242.1086883544922, + "used": 49.482757568359375 + } + }, + "gpu": "NVIDIA A100-SXM4-80GB", + "gpu_count": 8, + "gpu_devices": [ + { + "name": "NVIDIA A100-SXM4-80GB", + "memory_total": 85899345920 + }, + { + "name": "NVIDIA A100-SXM4-80GB", + "memory_total": 85899345920 + }, + { + "name": "NVIDIA A100-SXM4-80GB", + "memory_total": 85899345920 + }, + { + "name": "NVIDIA A100-SXM4-80GB", + "memory_total": 85899345920 + }, + { + "name": "NVIDIA A100-SXM4-80GB", + "memory_total": 85899345920 + }, + { + "name": "NVIDIA A100-SXM4-80GB", + "memory_total": 85899345920 + }, + { + "name": "NVIDIA A100-SXM4-80GB", + "memory_total": 85899345920 + }, + { + "name": "NVIDIA A100-SXM4-80GB", + "memory_total": 85899345920 + } + ], + "memory": { + "total": 2015.6942672729492 + } +} diff --git a/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json b/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json new file mode 100644 index 0000000000000000000000000000000000000000..a30f59f3dff91676a8fc4db4758db98b7a34d50e --- /dev/null +++ b/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json @@ -0,0 +1 @@ +{"Finetune/Step": 5198, "Finetune/Loss": 0.7997910976409912, "Finetune/Loss (Raw)": 0.8678842782974243, "Finetune/Learning Rate": 0.0, "Finetune/Step Time": 4.842910025268793, "_timestamp": 1710277906.839119, "_runtime": 24195.377708911896, "_step": 5198, "_wandb": {"runtime": 24451}} \ No newline at end of file diff --git a/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/logs/debug-internal.log b/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..641d08dbe27fbfd07d599be5a3e0c42b9b59e24a --- /dev/null +++ b/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/logs/debug-internal.log @@ -0,0 +1,41561 @@ +2024-03-12 14:28:31,461 INFO StreamThr :2571781 [internal.py:wandb_internal():86] W&B internal server running at pid: 2571781, started at: 2024-03-12 14:28:31.460168 +2024-03-12 14:28:31,462 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status +2024-03-12 14:28:31,474 INFO WriterThread:2571781 [datastore.py:open_for_write():87] open: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/run-b1zz0q9t.wandb +2024-03-12 14:28:31,476 DEBUG SenderThread:2571781 [sender.py:send():382] send: header +2024-03-12 14:28:31,493 DEBUG SenderThread:2571781 [sender.py:send():382] send: run +2024-03-12 14:28:32,006 INFO SenderThread:2571781 [dir_watcher.py:__init__():211] watching files in: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files +2024-03-12 14:28:32,006 INFO SenderThread:2571781 [sender.py:_start_run_threads():1136] run started: b1zz0q9t with start time 1710253711.46141 +2024-03-12 14:28:32,020 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: check_version +2024-03-12 14:28:32,020 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: check_version +2024-03-12 14:28:32,072 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: run_start +2024-03-12 14:28:32,083 DEBUG HandlerThread:2571781 [system_info.py:__init__():27] System info init +2024-03-12 14:28:32,083 DEBUG HandlerThread:2571781 [system_info.py:__init__():42] System info init done +2024-03-12 14:28:32,083 INFO HandlerThread:2571781 [system_monitor.py:start():194] Starting system monitor +2024-03-12 14:28:32,083 INFO SystemMonitor:2571781 [system_monitor.py:_start():158] Starting system asset monitoring threads +2024-03-12 14:28:32,083 INFO HandlerThread:2571781 [system_monitor.py:probe():214] Collecting system info +2024-03-12 14:28:32,084 INFO SystemMonitor:2571781 [interfaces.py:start():190] Started cpu monitoring +2024-03-12 14:28:32,084 INFO SystemMonitor:2571781 [interfaces.py:start():190] Started disk monitoring +2024-03-12 14:28:32,085 INFO SystemMonitor:2571781 [interfaces.py:start():190] Started gpu monitoring +2024-03-12 14:28:32,085 INFO SystemMonitor:2571781 [interfaces.py:start():190] Started memory monitoring +2024-03-12 14:28:32,085 INFO SystemMonitor:2571781 [interfaces.py:start():190] Started network monitoring +2024-03-12 14:28:32,197 DEBUG HandlerThread:2571781 [system_info.py:probe():151] Probing system +2024-03-12 14:28:32,202 DEBUG HandlerThread:2571781 [system_info.py:_probe_git():136] Probing git +2024-03-12 14:28:32,223 DEBUG HandlerThread:2571781 [system_info.py:_probe_git():144] Probing git done +2024-03-12 14:28:32,223 DEBUG HandlerThread:2571781 [system_info.py:probe():199] Probing system done +2024-03-12 14:28:32,223 DEBUG HandlerThread:2571781 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-1027-oracle-x86_64-with-glibc2.31', 'python': '3.10.13', 'heartbeatAt': '2024-03-12T14:28:32.197467', 'startedAt': '2024-03-12T14:28:31.407967', 'docker': None, 'cuda': None, 'args': ('--model.type', 'one-stage+7b', '--model.model_id', 'one-stage+7b_qformer2_64', '--run_id', 'qformer2_64', '--model.arch_specifier', 'qformer2_64'), 'state': 'running', 'program': '/home/lilei/prismatic-vlms/scripts/pretrain.py', 'codePathLocal': 'scripts/pretrain.py', 'codePath': 'scripts/pretrain.py', 'git': {'remote': 'https://github.com/TRI-ML/prismatic-vlms.git', 'commit': 'b1c33c284c6e76d908c29e5545b00f5b8b4c601f'}, 'email': 'tobiaslee@qq.com', 'root': '/home/lilei/prismatic-vlms', 'host': 'compute-permanent-node-88', 'username': 'lilei', 'executable': '/home/lilei/anaconda3/envs/pvlm/bin/python', 'cpu_count': 128, 'cpu_count_logical': 255, 'cpu_freq': {'current': 2575.3999333333336, 'min': 1500.0, 'max': 2550.0}, 'cpu_freq_per_core': [{'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 3249.002, 'min': 1500.0, 'max': 2550.0}, {'current': 3248.267, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 3247.033, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2716.886, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 3249.005, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 3249.239, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 3249.276, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 3249.389, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 3249.335, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 3245.888, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}, {'current': 2550.0, 'min': 1500.0, 'max': 2550.0}], 'disk': {'/': {'total': 242.1086883544922, 'used': 49.482757568359375}}, 'gpu': 'NVIDIA A100-SXM4-80GB', 'gpu_count': 8, 'gpu_devices': [{'name': 'NVIDIA A100-SXM4-80GB', 'memory_total': 85899345920}, {'name': 'NVIDIA A100-SXM4-80GB', 'memory_total': 85899345920}, {'name': 'NVIDIA A100-SXM4-80GB', 'memory_total': 85899345920}, {'name': 'NVIDIA A100-SXM4-80GB', 'memory_total': 85899345920}, {'name': 'NVIDIA A100-SXM4-80GB', 'memory_total': 85899345920}, {'name': 'NVIDIA A100-SXM4-80GB', 'memory_total': 85899345920}, {'name': 'NVIDIA A100-SXM4-80GB', 'memory_total': 85899345920}, {'name': 'NVIDIA A100-SXM4-80GB', 'memory_total': 85899345920}], 'memory': {'total': 2015.6942672729492}} +2024-03-12 14:28:32,223 INFO HandlerThread:2571781 [system_monitor.py:probe():224] Finished collecting system info +2024-03-12 14:28:32,223 INFO HandlerThread:2571781 [system_monitor.py:probe():227] Publishing system info +2024-03-12 14:28:32,224 DEBUG HandlerThread:2571781 [system_info.py:_save_conda():208] Saving list of conda packages installed into the current environment +2024-03-12 14:28:33,009 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_created():271] file/dir created: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/conda-environment.yaml +2024-03-12 14:28:36,901 DEBUG HandlerThread:2571781 [system_info.py:_save_conda():220] Saving conda packages done +2024-03-12 14:28:36,910 INFO HandlerThread:2571781 [system_monitor.py:probe():229] Finished publishing system info +2024-03-12 14:28:36,915 DEBUG SenderThread:2571781 [sender.py:send():382] send: files +2024-03-12 14:28:36,915 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-metadata.json with policy now +2024-03-12 14:28:36,915 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:28:36,926 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: python_packages +2024-03-12 14:28:36,926 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: python_packages +2024-03-12 14:28:36,927 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:28:36,927 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:28:36,930 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:28:37,014 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/conda-environment.yaml +2024-03-12 14:28:37,015 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_created():271] file/dir created: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/requirements.txt +2024-03-12 14:28:37,015 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_created():271] file/dir created: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-metadata.json +2024-03-12 14:28:37,151 DEBUG SenderThread:2571781 [sender.py:send():382] send: telemetry +2024-03-12 14:28:37,928 INFO wandb-upload_0:2571781 [upload_job.py:push():131] Uploaded file /tmp/tmp7_m3238twandb/3hoz07t5-wandb-metadata.json +2024-03-12 14:28:38,016 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_created():271] file/dir created: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 14:28:40,020 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 14:28:42,485 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:28:43,024 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 14:28:47,486 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:28:51,927 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:28:51,927 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:28:51,928 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:28:53,133 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:28:58,133 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:29:03,136 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:29:04,041 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/config.yaml +2024-03-12 14:29:06,927 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:29:06,927 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:29:06,970 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:29:09,179 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:29:14,179 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:29:19,179 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:29:21,927 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:29:21,927 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:29:21,970 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:29:22,740 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:29:24,783 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:29:25,056 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 14:29:29,783 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:29:32,085 DEBUG SystemMonitor:2571781 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2024-03-12 14:29:32,088 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 14:29:35,090 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:29:36,927 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:29:36,927 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:29:36,929 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:29:37,662 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:29:37,663 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 14:29:37,663 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:29:37,667 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:29:38,066 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_created():271] file/dir created: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 14:29:39,069 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 14:29:40,707 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:29:45,577 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:29:45,578 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 14:29:45,578 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:29:45,584 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:29:46,073 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 14:29:46,584 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:29:47,076 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 14:29:51,585 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:29:51,927 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:29:51,927 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:29:51,929 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:29:53,440 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:29:53,441 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 14:29:53,441 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:29:53,447 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:29:54,081 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 14:29:55,083 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 14:29:57,483 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:30:02,090 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 14:30:03,092 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:30:03,596 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:30:03,596 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 14:30:03,597 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:30:03,603 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:30:04,089 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 14:30:05,091 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 14:30:06,927 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:30:06,927 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:30:06,929 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:30:08,201 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:30:12,761 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:30:12,762 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 14:30:12,762 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:30:12,767 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:30:13,096 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 14:30:13,803 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:30:15,099 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 14:30:18,803 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:30:21,824 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:30:21,825 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 14:30:21,825 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:30:21,830 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:30:21,927 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:30:21,927 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:30:21,929 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:30:22,104 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 14:30:23,107 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 14:30:24,170 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:30:29,171 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:30:32,099 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 14:30:33,651 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:30:33,652 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 14:30:33,652 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:30:33,657 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:30:34,114 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 14:30:34,695 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:30:35,117 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 14:30:36,927 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:30:36,928 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:30:36,929 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:30:39,701 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:30:42,994 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:30:42,995 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 14:30:42,995 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:30:43,000 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:30:43,123 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 14:30:45,039 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:30:45,126 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 14:30:50,039 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:30:51,927 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:30:51,928 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:30:51,929 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:30:55,195 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:30:57,513 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:30:57,514 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 14:30:57,514 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:30:57,518 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:30:58,134 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 14:30:59,136 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 14:31:00,555 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:31:02,095 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 14:31:06,071 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:31:06,072 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 14:31:06,072 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:31:06,073 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:31:06,078 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:31:06,142 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 14:31:06,927 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:31:06,928 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:31:06,929 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:31:07,144 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 14:31:11,566 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:31:14,946 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:31:14,947 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 14:31:14,947 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:31:14,952 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:31:15,149 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 14:31:16,991 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:31:17,152 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 14:31:21,927 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:31:21,928 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:31:21,928 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:31:22,133 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:31:23,734 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:31:23,735 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 14:31:23,735 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:31:23,740 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:31:24,157 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 14:31:25,159 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 14:31:27,228 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:31:27,229 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 14:31:27,229 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:31:27,230 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:31:27,234 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:31:28,162 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 14:31:29,165 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 14:31:32,097 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 14:31:32,225 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:31:32,226 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 14:31:32,226 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:31:32,231 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:31:32,231 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:31:33,167 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 14:31:35,170 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 14:31:36,927 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:31:36,928 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:31:36,929 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:31:37,392 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:31:38,078 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 14:31:38,078 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:31:38,079 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:31:38,085 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:31:38,172 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 14:31:39,175 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 14:31:41,476 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:31:41,477 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 14:31:41,477 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:31:41,482 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:31:42,177 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 14:31:43,179 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 14:31:43,483 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:31:46,016 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:31:46,017 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 14:31:46,017 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:31:46,022 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:31:46,181 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 14:31:47,183 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 14:31:49,022 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:31:49,582 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:31:49,583 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 14:31:49,584 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:31:49,589 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:31:50,185 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 14:31:51,187 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 14:31:51,928 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:31:51,928 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:31:51,929 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:31:52,906 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:31:52,906 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 14:31:52,907 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:31:52,911 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:31:53,189 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 14:31:54,959 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:31:55,192 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 14:31:58,089 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:31:58,089 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 14:31:58,090 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:31:58,095 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:31:58,194 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 14:31:59,194 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 14:32:00,095 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:32:02,099 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 14:32:02,103 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:32:02,104 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 14:32:02,104 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:32:02,109 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:32:02,196 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 14:32:03,198 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 14:32:05,110 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:32:06,928 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:32:06,928 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:32:06,929 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:32:10,367 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:32:10,368 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 14:32:10,368 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:32:10,368 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:32:10,373 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:32:11,204 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 14:32:13,207 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 14:32:13,744 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:32:13,745 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 14:32:13,745 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:32:13,750 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:32:14,208 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 14:32:15,211 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 14:32:15,750 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:32:17,813 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:32:17,814 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 14:32:17,814 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:32:17,819 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:32:18,212 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 14:32:19,215 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 14:32:20,820 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:32:21,928 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:32:21,928 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:32:21,929 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:32:26,217 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:32:26,387 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:32:26,388 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 14:32:26,388 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:32:26,393 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:32:27,222 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 14:32:29,225 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 14:32:29,509 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:32:29,510 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 14:32:29,510 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:32:29,515 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:32:30,225 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 14:32:31,228 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 14:32:31,515 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:32:32,101 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 14:32:32,932 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:32:32,932 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 14:32:32,933 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:32:32,937 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:32:33,229 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 14:32:35,232 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 14:32:36,928 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:32:36,928 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:32:36,929 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:32:37,222 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:32:37,868 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:32:37,869 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 14:32:37,869 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:32:37,874 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:32:38,234 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 14:32:39,237 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 14:32:42,859 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:32:42,860 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 14:32:42,861 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:32:42,861 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:32:42,865 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:32:43,239 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 14:32:45,242 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 14:32:47,034 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:32:47,035 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 14:32:47,035 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:32:47,040 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:32:47,243 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 14:32:48,040 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:32:49,246 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 14:32:51,928 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:32:51,928 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:32:51,930 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:32:53,011 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:32:53,012 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 14:32:53,012 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:32:53,017 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:32:53,055 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:32:53,249 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 14:32:55,252 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 14:32:57,500 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:32:57,501 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 14:32:57,501 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:32:57,506 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:32:58,254 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 14:32:58,507 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:32:59,256 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 14:33:01,370 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:33:01,371 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 14:33:01,371 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:33:01,376 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:33:02,108 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 14:33:02,258 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 14:33:03,261 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 14:33:04,110 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:33:05,522 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:33:05,523 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 14:33:05,523 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:33:05,528 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:33:06,264 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 14:33:06,928 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:33:06,928 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:33:06,930 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:33:07,267 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 14:33:09,223 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:33:10,961 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:33:10,962 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 14:33:10,962 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:33:10,968 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:33:11,269 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 14:33:13,272 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 14:33:15,003 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:33:17,919 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:33:17,919 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 14:33:17,920 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:33:17,925 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:33:18,275 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 14:33:19,277 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 14:33:20,925 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:33:21,373 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:33:21,374 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 14:33:21,374 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:33:21,379 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:33:21,928 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:33:21,928 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:33:21,930 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:33:22,279 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 14:33:23,282 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 14:33:24,826 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:33:24,827 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 14:33:24,827 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:33:24,832 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:33:25,284 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 14:33:26,871 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:33:27,288 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 14:33:30,154 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:33:30,154 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 14:33:30,155 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:33:30,159 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:33:30,290 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 14:33:31,292 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 14:33:32,105 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 14:33:32,106 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:33:33,064 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:33:33,065 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 14:33:33,065 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:33:33,070 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:33:33,293 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 14:33:35,297 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 14:33:36,516 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:33:36,518 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 14:33:36,518 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:33:36,523 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:33:36,928 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:33:36,929 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:33:36,930 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:33:37,298 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 14:33:37,916 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:33:39,301 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 14:33:41,421 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:33:41,422 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 14:33:41,422 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:33:41,427 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:33:42,303 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 14:33:43,306 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 14:33:43,463 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:33:46,062 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:33:46,063 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 14:33:46,064 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:33:46,068 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:33:46,307 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 14:33:47,310 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 14:33:49,069 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:33:51,928 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:33:51,929 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:33:51,930 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:33:53,611 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:33:53,612 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 14:33:53,612 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:33:53,617 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:33:54,314 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 14:33:54,655 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:33:55,316 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 14:33:59,655 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:34:02,113 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 14:34:03,264 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:34:03,265 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 14:34:03,265 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:34:03,270 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:34:03,322 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 14:34:05,270 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:34:05,325 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 14:34:06,928 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:34:06,929 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:34:06,930 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:34:07,874 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:34:07,874 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 14:34:07,874 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:34:07,879 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:34:08,327 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 14:34:09,330 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 14:34:10,919 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:34:13,283 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:34:13,284 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 14:34:13,285 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:34:13,289 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:34:13,332 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 14:34:15,335 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 14:34:16,290 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:34:17,204 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:34:17,204 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 14:34:17,205 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:34:17,210 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:34:17,337 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 14:34:19,340 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 14:34:21,273 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:34:21,274 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 14:34:21,274 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:34:21,279 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:34:21,341 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 14:34:21,928 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:34:21,929 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:34:21,930 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:34:22,178 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:34:23,345 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 14:34:25,010 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:34:25,010 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 14:34:25,011 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:34:25,016 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:34:25,346 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 14:34:27,350 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 14:34:28,055 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:34:29,603 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:34:29,604 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 14:34:29,605 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:34:29,610 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:34:30,352 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 14:34:31,354 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 14:34:32,110 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 14:34:33,111 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:34:35,360 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:34:35,361 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 14:34:35,362 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:34:35,367 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:34:36,357 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 14:34:36,928 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:34:36,929 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:34:36,930 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:34:37,360 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 14:34:38,217 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:34:40,110 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:34:40,110 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 14:34:40,111 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:34:40,116 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:34:40,362 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 14:34:41,364 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 14:34:44,155 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:34:45,265 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:34:45,266 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 14:34:45,266 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:34:45,271 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:34:45,366 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 14:34:47,370 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 14:34:49,272 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:34:50,865 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:34:50,866 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 14:34:50,866 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:34:50,871 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:34:51,372 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 14:34:51,929 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:34:51,929 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:34:51,930 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:34:53,375 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 14:34:54,629 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:34:54,629 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 14:34:54,630 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:34:54,630 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:34:54,635 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:34:55,376 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 14:34:57,379 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 14:34:59,671 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:35:01,202 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:35:01,203 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 14:35:01,203 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:35:01,208 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:35:01,381 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 14:35:02,111 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 14:35:03,384 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 14:35:05,113 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:35:06,675 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:35:06,676 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 14:35:06,676 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:35:06,681 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:35:06,929 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:35:06,929 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:35:06,931 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:35:07,388 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 14:35:09,391 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 14:35:10,214 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:35:10,251 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:35:10,251 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 14:35:10,252 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:35:10,257 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:35:10,392 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 14:35:11,395 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 14:35:14,403 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:35:14,404 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 14:35:14,404 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:35:14,409 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:35:15,399 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 14:35:15,399 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 14:35:15,410 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:35:17,894 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:35:17,895 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 14:35:17,895 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:35:17,900 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:35:18,400 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 14:35:19,403 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 14:35:20,900 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:35:21,617 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:35:21,618 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 14:35:21,619 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:35:21,623 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:35:21,929 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:35:21,929 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:35:21,931 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:35:22,404 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 14:35:23,407 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 14:35:26,214 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:35:28,830 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:35:28,830 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 14:35:28,831 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:35:28,835 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:35:29,412 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 14:35:31,415 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 14:35:31,875 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:35:32,114 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 14:35:33,443 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:35:33,444 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 14:35:33,444 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:35:33,449 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:35:34,417 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 14:35:35,419 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 14:35:36,929 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:35:36,929 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:35:36,931 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:35:37,549 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:35:37,742 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 14:35:37,742 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:35:37,743 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:35:37,748 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:35:38,422 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 14:35:39,424 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 14:35:40,913 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:35:40,914 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 14:35:40,914 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:35:40,919 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:35:41,425 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 14:35:42,920 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:35:43,428 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 14:35:45,712 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:35:45,713 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 14:35:45,713 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:35:45,718 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:35:46,430 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 14:35:47,433 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 14:35:48,719 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:35:49,065 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:35:49,066 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 14:35:49,066 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:35:49,071 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:35:49,434 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 14:35:51,437 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 14:35:51,929 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:35:51,929 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:35:51,930 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:35:52,663 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:35:52,663 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 14:35:52,664 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:35:52,669 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:35:53,438 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 14:35:54,707 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:35:55,441 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 14:35:56,409 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:35:56,409 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 14:35:56,410 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:35:56,415 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:35:56,442 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 14:35:57,444 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 14:35:59,689 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:35:59,690 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 14:35:59,690 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:35:59,695 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:36:00,446 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 14:36:00,696 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:36:01,449 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 14:36:02,116 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 14:36:03,260 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:36:03,260 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 14:36:03,261 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:36:03,266 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:36:03,450 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 14:36:05,454 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 14:36:06,266 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:36:06,929 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:36:06,929 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:36:06,931 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:36:06,960 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:36:07,215 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 14:36:07,215 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:36:07,220 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:36:07,455 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 14:36:09,459 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 14:36:11,372 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:36:11,373 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 14:36:11,373 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:36:11,373 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:36:11,378 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:36:11,460 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 14:36:13,463 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 14:36:15,770 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:36:15,771 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 14:36:15,771 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:36:15,776 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:36:16,465 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 14:36:16,776 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:36:17,467 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 14:36:19,011 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:36:19,012 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 14:36:19,012 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:36:19,017 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:36:19,468 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 14:36:21,471 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 14:36:21,929 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:36:21,930 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:36:21,930 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:36:22,277 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:36:22,297 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:36:22,297 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 14:36:22,297 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:36:22,302 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:36:22,472 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 14:36:23,475 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 14:36:25,917 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:36:25,918 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 14:36:25,918 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:36:25,923 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:36:26,478 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 14:36:27,480 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 14:36:27,923 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:36:31,204 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:36:31,205 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 14:36:31,206 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:36:31,211 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:36:31,483 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 14:36:32,123 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 14:36:33,125 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:36:33,486 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 14:36:35,812 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:36:35,812 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 14:36:35,813 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:36:35,818 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:36:36,487 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 14:36:36,929 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:36:36,930 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:36:36,930 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:36:37,490 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 14:36:38,161 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:36:39,339 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:36:39,340 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 14:36:39,340 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:36:39,345 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:36:39,491 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 14:36:41,495 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 14:36:43,383 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:36:44,792 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:36:44,793 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 14:36:44,793 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:36:44,798 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:36:45,497 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 14:36:47,500 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 14:36:48,798 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:36:49,772 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:36:49,773 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 14:36:49,774 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:36:49,778 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:36:50,502 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 14:36:51,505 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 14:36:51,929 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:36:51,930 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:36:51,930 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:36:53,431 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:36:53,432 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 14:36:53,432 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:36:53,437 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:36:53,506 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 14:36:54,475 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:36:55,508 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 14:36:59,475 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:37:02,120 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 14:37:02,630 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:37:02,630 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 14:37:02,631 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:37:02,635 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:37:03,513 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 14:37:04,636 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:37:05,517 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 14:37:06,213 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:37:06,214 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 14:37:06,215 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:37:06,219 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:37:06,518 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 14:37:06,929 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:37:06,930 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:37:06,930 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:37:07,521 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 14:37:10,313 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:37:11,757 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:37:11,757 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 14:37:11,757 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:37:11,763 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:37:12,524 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 14:37:13,527 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 14:37:15,424 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:37:15,424 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 14:37:15,425 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:37:15,425 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:37:15,429 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:37:15,528 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 14:37:17,531 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 14:37:20,430 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:37:21,258 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:37:21,259 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 14:37:21,259 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:37:21,264 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:37:21,534 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 14:37:21,929 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:37:21,930 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:37:21,932 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:37:23,537 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 14:37:26,200 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:37:26,447 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:37:26,448 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 14:37:26,448 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:37:26,453 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:37:26,541 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 14:37:27,543 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 14:37:30,239 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:37:30,240 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 14:37:30,240 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:37:30,245 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:37:30,545 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 14:37:31,246 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:37:31,547 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 14:37:32,129 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 14:37:34,373 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:37:34,373 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 14:37:34,374 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:37:34,379 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:37:34,549 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 14:37:35,552 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 14:37:36,380 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:37:36,929 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:37:36,930 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:37:36,931 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:37:37,613 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:37:37,909 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 14:37:37,909 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:37:37,914 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:37:38,554 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 14:37:39,557 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 14:37:41,313 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:37:41,314 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 14:37:41,315 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:37:41,319 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:37:41,558 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 14:37:42,320 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:37:43,561 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 14:37:46,356 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:37:46,357 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 14:37:46,357 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:37:46,362 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:37:46,563 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 14:37:47,363 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:37:47,566 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 14:37:50,716 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:37:50,717 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 14:37:50,718 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:37:50,722 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:37:51,568 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 14:37:51,929 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:37:51,930 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:37:51,931 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:37:53,180 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:37:53,571 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 14:37:54,376 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:37:54,377 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 14:37:54,377 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:37:54,382 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:37:54,572 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 14:37:55,574 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 14:37:58,419 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:38:02,125 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 14:38:02,640 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:38:02,640 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 14:38:02,641 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:38:02,646 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:38:03,579 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 14:38:03,646 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:38:05,583 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 14:38:06,770 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:38:06,771 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 14:38:06,771 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:38:06,776 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:38:06,930 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:38:06,930 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:38:06,932 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:38:07,585 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 14:38:09,191 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:38:09,588 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 14:38:11,722 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:38:11,723 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 14:38:11,723 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:38:11,728 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:38:12,590 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 14:38:13,592 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 14:38:14,767 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:38:14,813 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:38:14,814 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 14:38:14,814 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:38:14,819 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:38:15,594 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 14:38:17,597 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 14:38:19,820 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:38:21,930 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:38:21,930 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:38:21,931 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:38:23,231 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:38:23,231 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 14:38:23,232 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:38:23,236 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:38:23,601 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 14:38:25,275 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:38:25,604 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 14:38:26,680 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:38:26,681 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 14:38:26,681 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:38:26,686 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:38:27,606 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 14:38:29,609 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 14:38:29,726 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:38:29,727 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 14:38:29,727 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:38:29,732 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:38:30,610 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 14:38:30,733 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:38:31,613 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 14:38:32,133 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 14:38:33,210 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:38:33,211 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 14:38:33,212 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:38:33,217 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:38:33,614 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 14:38:35,617 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 14:38:36,218 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:38:36,791 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:38:36,792 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 14:38:36,793 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:38:36,797 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:38:36,930 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:38:36,930 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:38:36,932 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:38:37,619 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 14:38:39,622 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 14:38:40,611 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:38:40,611 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 14:38:40,612 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:38:40,617 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:38:40,622 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 14:38:41,625 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 14:38:41,655 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:38:44,131 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:38:44,131 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 14:38:44,132 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:38:44,137 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:38:44,627 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 14:38:45,629 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 14:38:47,137 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:38:47,514 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:38:47,515 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 14:38:47,515 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:38:47,520 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:38:47,630 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 14:38:49,634 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 14:38:51,930 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:38:51,930 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:38:51,932 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:38:52,191 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:38:52,863 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:38:52,864 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 14:38:52,864 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:38:52,869 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:38:53,637 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 14:38:55,640 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 14:38:57,907 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:38:58,479 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:38:58,480 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 14:38:58,480 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:38:58,485 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:38:58,642 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 14:38:59,644 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 14:39:02,129 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 14:39:03,131 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:39:03,511 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:39:03,511 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 14:39:03,512 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:39:03,516 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:39:03,647 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 14:39:05,651 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 14:39:06,930 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:39:06,930 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:39:06,932 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:39:08,162 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:39:12,705 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:39:12,705 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 14:39:12,706 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:39:12,711 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:39:13,657 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 14:39:13,747 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:39:15,660 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 14:39:17,005 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:39:17,006 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 14:39:17,007 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:39:17,013 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:39:17,662 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 14:39:19,014 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:39:19,665 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 14:39:21,930 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:39:21,930 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:39:21,932 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:39:23,121 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:39:23,122 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 14:39:23,122 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:39:23,127 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:39:23,668 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 14:39:24,163 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:39:25,672 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 14:39:27,288 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:39:27,289 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 14:39:27,290 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:39:27,295 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:39:27,674 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 14:39:29,295 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:39:29,676 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 14:39:30,820 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:39:30,821 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 14:39:30,821 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:39:30,826 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:39:31,677 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 14:39:32,132 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 14:39:33,680 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 14:39:34,544 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:39:34,545 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 14:39:34,545 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:39:34,545 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:39:34,551 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:39:34,681 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 14:39:35,684 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 14:39:36,930 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:39:36,931 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:39:36,932 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:39:38,836 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:39:38,837 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 14:39:38,837 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:39:38,842 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:39:39,687 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 14:39:39,879 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:39:41,690 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 14:39:44,879 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:39:45,348 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:39:45,349 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 14:39:45,350 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:39:45,355 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:39:45,693 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 14:39:47,696 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 14:39:48,878 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:39:48,879 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 14:39:48,879 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:39:48,884 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:39:49,697 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 14:39:49,885 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:39:51,701 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 14:39:51,930 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:39:51,931 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:39:51,932 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:39:52,446 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:39:52,447 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 14:39:52,447 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:39:52,452 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:39:52,702 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 14:39:53,704 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 14:39:55,491 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:39:56,356 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:39:56,357 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 14:39:56,357 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:39:56,362 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:39:56,706 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 14:39:57,709 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 14:40:00,252 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:40:00,253 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 14:40:00,253 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:40:00,258 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:40:00,711 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 14:40:01,259 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:40:01,714 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 14:40:02,134 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 14:40:06,427 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:40:06,427 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 14:40:06,428 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:40:06,429 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:40:06,434 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:40:06,718 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 14:40:06,930 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:40:06,931 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:40:06,932 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:40:07,721 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 14:40:09,892 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:40:09,893 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 14:40:09,893 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:40:09,899 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:40:10,723 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 14:40:11,726 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 14:40:11,935 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:40:13,683 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:40:13,684 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 14:40:13,685 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:40:13,689 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:40:13,727 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 14:40:15,730 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 14:40:17,287 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:40:17,287 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 14:40:17,288 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:40:17,288 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:40:17,293 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:40:17,731 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 14:40:19,734 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 14:40:21,930 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:40:21,931 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:40:21,932 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:40:22,013 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:40:22,369 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 14:40:22,369 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:40:22,369 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:40:22,375 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:40:22,736 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 14:40:23,739 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 14:40:25,332 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:40:25,333 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 14:40:25,333 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:40:25,338 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:40:25,740 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 14:40:27,744 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 14:40:28,339 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:40:28,870 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:40:28,871 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 14:40:28,871 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:40:28,876 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:40:29,746 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 14:40:31,749 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 14:40:32,136 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 14:40:34,137 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:40:34,633 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:40:34,634 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 14:40:34,634 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:40:34,642 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:40:34,751 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 14:40:35,758 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 14:40:36,930 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:40:36,931 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:40:36,932 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:40:38,239 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:40:38,240 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 14:40:38,240 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:40:38,248 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:40:38,761 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 14:40:39,283 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:40:39,771 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 14:40:42,500 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:40:42,501 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 14:40:42,501 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:40:42,515 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:40:42,773 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 14:40:43,778 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 14:40:44,515 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:40:45,930 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:40:45,931 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 14:40:45,931 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:40:45,939 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:40:46,780 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 14:40:47,788 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 14:40:49,939 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:40:50,333 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:40:50,334 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 14:40:50,335 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:40:50,342 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:40:50,790 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 14:40:51,798 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 14:40:51,930 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:40:51,931 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:40:51,932 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:40:53,277 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:40:53,277 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 14:40:53,278 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:40:53,286 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:40:53,800 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 14:40:55,319 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:40:55,806 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 14:40:57,227 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:40:57,237 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 14:40:57,237 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:40:57,243 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:40:57,808 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 14:40:59,814 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 14:41:00,387 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:41:00,388 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 14:41:00,388 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:41:00,389 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:41:00,400 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:41:00,814 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 14:41:01,821 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 14:41:02,138 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 14:41:03,469 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:41:03,470 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 14:41:03,470 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:41:03,483 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:41:03,823 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 14:41:05,483 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:41:05,829 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 14:41:06,931 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:41:06,931 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:41:06,933 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:41:11,202 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:41:12,224 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:41:12,225 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 14:41:12,225 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:41:12,244 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:41:12,835 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 14:41:13,838 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 14:41:16,267 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:41:19,073 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:41:19,074 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 14:41:19,074 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:41:19,085 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:41:19,843 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 14:41:21,847 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 14:41:21,931 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:41:21,931 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:41:21,932 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:41:22,207 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:41:27,207 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:41:28,443 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:41:28,444 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 14:41:28,444 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:41:28,452 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:41:28,854 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 14:41:29,860 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 14:41:32,140 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 14:41:33,142 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:41:36,855 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:41:36,856 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 14:41:36,856 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:41:36,868 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:41:36,868 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 14:41:36,931 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:41:36,931 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:41:36,932 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:41:37,875 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 14:41:38,150 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:41:40,142 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:41:40,142 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 14:41:40,142 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:41:40,151 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:41:40,877 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 14:41:41,882 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 14:41:43,187 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:41:44,516 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:41:44,517 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 14:41:44,517 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:41:44,529 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:41:44,885 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 14:41:45,893 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 14:41:48,214 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:41:48,215 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 14:41:48,215 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:41:48,215 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:41:48,225 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:41:48,895 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 14:41:49,901 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 14:41:51,931 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:41:51,931 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:41:51,932 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:41:52,000 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:41:52,216 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 14:41:52,217 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:41:52,225 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:41:52,903 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 14:41:53,226 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:41:53,907 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 14:41:56,544 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:41:56,545 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 14:41:56,545 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:41:56,554 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:41:56,910 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 14:41:57,919 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 14:41:58,555 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:42:00,772 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:42:00,773 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 14:42:00,774 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:42:00,781 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:42:00,922 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 14:42:01,928 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 14:42:02,150 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 14:42:04,152 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:42:04,864 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:42:04,865 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 14:42:04,865 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:42:04,896 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:42:04,931 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 14:42:05,936 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 14:42:06,931 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:42:06,931 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:42:06,933 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:42:09,194 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:42:10,105 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:42:10,106 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 14:42:10,106 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:42:10,118 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:42:10,941 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 14:42:11,951 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 14:42:13,856 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:42:13,857 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 14:42:13,857 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:42:13,863 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:42:13,953 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 14:42:14,864 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:42:15,964 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 14:42:18,053 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:42:18,054 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 14:42:18,054 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:42:18,067 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:42:18,967 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 14:42:19,972 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 14:42:20,067 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:42:21,931 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:42:21,931 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:42:21,933 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:42:25,203 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:42:26,562 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:42:26,563 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 14:42:26,563 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:42:26,572 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:42:26,980 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 14:42:27,990 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 14:42:30,230 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:42:30,231 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 14:42:30,231 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:42:30,231 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:42:30,241 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:42:30,993 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 14:42:31,999 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 14:42:32,145 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 14:42:34,130 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:42:34,131 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 14:42:34,131 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:42:34,138 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:42:35,002 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 14:42:36,005 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 14:42:36,139 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:42:36,931 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:42:36,931 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:42:36,933 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:42:37,330 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:42:37,330 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 14:42:37,331 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:42:37,337 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:42:38,008 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 14:42:40,016 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 14:42:41,375 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:42:41,922 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:42:41,923 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 14:42:41,923 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:42:41,931 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:42:42,018 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 14:42:44,024 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 14:42:45,873 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:42:45,874 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 14:42:45,874 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:42:45,880 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:42:46,025 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 14:42:46,881 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:42:48,035 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 14:42:49,519 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:42:49,519 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 14:42:49,520 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:42:49,526 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:42:50,037 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 14:42:51,931 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:42:51,932 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:42:51,933 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:42:52,043 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 14:42:52,188 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:42:53,784 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:42:53,784 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 14:42:53,785 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:42:53,792 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:42:54,045 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 14:42:56,048 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 14:42:57,827 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:42:58,974 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:42:58,975 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 14:42:58,975 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:42:58,982 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:42:59,051 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 14:43:02,058 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 14:43:02,156 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 14:43:03,002 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:43:03,003 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 14:43:03,003 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:43:03,003 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:43:03,012 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:43:03,059 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 14:43:06,066 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 14:43:06,716 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:43:06,716 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 14:43:06,717 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:43:06,724 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:43:06,931 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:43:06,932 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:43:06,933 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:43:07,067 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 14:43:08,072 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 14:43:08,117 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:43:10,398 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:43:10,399 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 14:43:10,399 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:43:10,409 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:43:11,075 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 14:43:12,078 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 14:43:13,443 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:43:14,213 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:43:14,214 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 14:43:14,215 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:43:14,222 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:43:15,081 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 14:43:16,086 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 14:43:17,780 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:43:17,781 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 14:43:17,781 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:43:17,787 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:43:18,088 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 14:43:18,787 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:43:20,092 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 14:43:21,931 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:43:21,932 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:43:21,933 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:43:24,203 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:43:25,900 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:43:25,900 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 14:43:25,901 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:43:25,913 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:43:26,097 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 14:43:28,106 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 14:43:29,943 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:43:30,509 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:43:30,509 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 14:43:30,510 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:43:30,518 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:43:31,109 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 14:43:32,110 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 14:43:32,149 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 14:43:35,150 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:43:35,453 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:43:35,454 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 14:43:35,454 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:43:35,461 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:43:36,115 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 14:43:36,931 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:43:36,932 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:43:36,933 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:43:38,121 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 14:43:38,969 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:43:38,970 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 14:43:38,970 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:43:38,976 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:43:39,122 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 14:43:40,126 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 14:43:41,015 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:43:43,283 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:43:43,284 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 14:43:43,284 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:43:43,290 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:43:44,130 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 14:43:46,135 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 14:43:46,291 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:43:46,739 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:43:46,740 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 14:43:46,741 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:43:46,750 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:43:47,136 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 14:43:48,140 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 14:43:51,751 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:43:51,931 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:43:51,932 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:43:51,933 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:43:55,154 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:43:55,154 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 14:43:55,155 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:43:55,162 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:43:56,146 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 14:43:57,199 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:43:58,155 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 14:43:59,404 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:43:59,405 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 14:43:59,405 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:43:59,411 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:44:00,157 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 14:44:02,151 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 14:44:02,162 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 14:44:03,152 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:44:03,732 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:44:03,732 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 14:44:03,733 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:44:03,742 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:44:04,163 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 14:44:06,170 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 14:44:06,932 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:44:06,932 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:44:06,934 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:44:08,043 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:44:08,044 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 14:44:08,044 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:44:08,052 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:44:08,173 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 14:44:09,087 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:44:10,179 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 14:44:13,640 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:44:13,641 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 14:44:13,641 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:44:13,649 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:44:14,182 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 14:44:14,649 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:44:16,189 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 14:44:17,591 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:44:17,592 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 14:44:17,592 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:44:17,599 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:44:18,190 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 14:44:20,195 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 14:44:20,600 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:44:21,932 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:44:21,932 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:44:21,934 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:44:23,055 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:44:23,064 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 14:44:23,064 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:44:23,070 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:44:23,197 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 14:44:25,984 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:44:25,985 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 14:44:25,986 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:44:25,986 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:44:25,995 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:44:26,203 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 14:44:26,204 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 14:44:28,210 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 14:44:29,896 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:44:29,897 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 14:44:29,897 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:44:29,904 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:44:30,211 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 14:44:31,905 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:44:32,153 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 14:44:32,218 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 14:44:33,401 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:44:33,402 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 14:44:33,402 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:44:33,409 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:44:34,220 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 14:44:36,224 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 14:44:36,932 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:44:36,932 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:44:36,934 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:44:37,167 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:44:38,759 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:44:38,759 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 14:44:38,760 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:44:38,767 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:44:39,228 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 14:44:40,231 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 14:44:42,477 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:44:42,478 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 14:44:42,479 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:44:42,479 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:44:42,487 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:44:43,234 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 14:44:44,235 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 14:44:45,888 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:44:45,889 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 14:44:45,889 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:44:45,895 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:44:46,236 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 14:44:47,895 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:44:48,242 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 14:44:51,356 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:44:51,357 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 14:44:51,357 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:44:51,364 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:44:51,932 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:44:51,932 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:44:51,934 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:44:52,246 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 14:44:53,218 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:44:54,250 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 14:44:55,620 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:44:55,621 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 14:44:55,621 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:44:55,628 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:44:56,252 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 14:44:58,256 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 14:44:58,663 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:44:59,988 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:44:59,989 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 14:44:59,989 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:44:59,995 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:45:00,257 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 14:45:02,156 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 14:45:02,262 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 14:45:04,157 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:45:04,597 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:45:04,598 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 14:45:04,598 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:45:04,604 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:45:05,265 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 14:45:06,268 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 14:45:06,932 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:45:06,932 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:45:06,934 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:45:08,371 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:45:08,372 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 14:45:08,372 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:45:08,378 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:45:09,271 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 14:45:09,415 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:45:10,274 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 14:45:12,325 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:45:12,325 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 14:45:12,326 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:45:12,331 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:45:13,276 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 14:45:14,279 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 14:45:15,332 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:45:16,602 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:45:16,603 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 14:45:16,603 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:45:16,609 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:45:17,281 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 14:45:18,284 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 14:45:20,610 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:45:20,849 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:45:20,850 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 14:45:20,851 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:45:20,856 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:45:21,286 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 14:45:21,932 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:45:21,932 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:45:21,934 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:45:22,290 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 14:45:25,150 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:45:25,151 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 14:45:25,151 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:45:25,156 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:45:25,293 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 14:45:26,195 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:45:28,298 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 14:45:28,642 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:45:28,643 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 14:45:28,643 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:45:28,648 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:45:29,299 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 14:45:30,302 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 14:45:31,649 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:45:32,163 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 14:45:32,986 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:45:32,986 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 14:45:32,987 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:45:32,992 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:45:33,304 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 14:45:34,307 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 14:45:36,708 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:45:36,709 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 14:45:36,710 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:45:36,710 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:45:36,715 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:45:36,932 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:45:36,932 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:45:36,934 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:45:37,309 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 14:45:38,313 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 14:45:40,989 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:45:40,989 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 14:45:40,990 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:45:40,995 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:45:41,315 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 14:45:42,031 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:45:42,318 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 14:45:44,957 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:45:44,957 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 14:45:44,957 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:45:44,963 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:45:45,320 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 14:45:46,322 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 14:45:47,964 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:45:48,770 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:45:48,771 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 14:45:48,771 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:45:48,776 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:45:49,325 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 14:45:50,327 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 14:45:51,932 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:45:51,933 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:45:51,934 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:45:52,405 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:45:52,406 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 14:45:52,406 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:45:52,411 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:45:53,329 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 14:45:53,447 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:45:54,332 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 14:45:58,447 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:46:02,018 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:46:02,019 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 14:46:02,019 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:46:02,024 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:46:02,160 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 14:46:02,337 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 14:46:04,161 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:46:04,340 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 14:46:06,554 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:46:06,555 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 14:46:06,555 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:46:06,561 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:46:06,932 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:46:06,933 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:46:06,934 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:46:07,344 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 14:46:08,346 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 14:46:09,252 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:46:13,018 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:46:13,019 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 14:46:13,019 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:46:13,024 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:46:13,349 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 14:46:14,352 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 14:46:15,063 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:46:16,442 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:46:16,443 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 14:46:16,443 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:46:16,448 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:46:17,354 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 14:46:18,357 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 14:46:20,330 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:46:20,330 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 14:46:20,331 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:46:20,331 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:46:20,335 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:46:20,358 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 14:46:21,932 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:46:21,933 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:46:21,934 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:46:22,361 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 14:46:26,216 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:46:28,545 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:46:28,546 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 14:46:28,546 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:46:28,551 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:46:29,367 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 14:46:30,369 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 14:46:31,587 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:46:32,168 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 14:46:33,059 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:46:33,059 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 14:46:33,059 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:46:33,064 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:46:33,371 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 14:46:34,374 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 14:46:36,932 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:46:36,933 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:46:36,934 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:46:37,229 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:46:37,752 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:46:37,753 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 14:46:37,753 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:46:37,758 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:46:38,377 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 14:46:40,380 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 14:46:41,826 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:46:41,827 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 14:46:41,827 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:46:41,832 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:46:42,381 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 14:46:42,832 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:46:44,384 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 14:46:47,689 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:46:47,690 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 14:46:47,690 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:46:47,695 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:46:48,387 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 14:46:48,695 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:46:50,390 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 14:46:51,350 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:46:51,351 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 14:46:51,351 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:46:51,356 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:46:51,391 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 14:46:51,932 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:46:51,933 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:46:51,934 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:46:54,162 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:46:54,395 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 14:46:56,669 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:46:56,669 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 14:46:56,669 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:46:56,675 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:46:57,397 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 14:46:58,399 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 14:46:59,711 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:47:00,087 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:47:00,088 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 14:47:00,088 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:47:00,093 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:47:00,400 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 14:47:02,164 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 14:47:02,404 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 14:47:04,294 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:47:04,294 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 14:47:04,295 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:47:04,300 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:47:04,405 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 14:47:05,300 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:47:06,409 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 14:47:06,933 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:47:06,933 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:47:06,934 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:47:07,861 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:47:07,861 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 14:47:07,861 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:47:07,866 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:47:08,410 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 14:47:10,414 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 14:47:10,903 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:47:11,353 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:47:11,354 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 14:47:11,354 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:47:11,359 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:47:11,414 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 14:47:14,418 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 14:47:15,748 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:47:15,749 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 14:47:15,749 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:47:15,754 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:47:16,419 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 14:47:16,754 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:47:18,423 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 14:47:19,869 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:47:19,870 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 14:47:19,870 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:47:19,875 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:47:20,424 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 14:47:21,876 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:47:21,933 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:47:21,933 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:47:21,934 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:47:22,427 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 14:47:23,653 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:47:23,653 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 14:47:23,653 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:47:23,658 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:47:24,429 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 14:47:26,432 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 14:47:27,695 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:47:32,166 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:47:32,167 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 14:47:32,168 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 14:47:32,168 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:47:32,173 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:47:32,437 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 14:47:33,174 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:47:34,440 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 14:47:36,523 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:47:36,529 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 14:47:36,529 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:47:36,534 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:47:36,933 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:47:36,933 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:47:36,934 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:47:37,442 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 14:47:38,445 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 14:47:39,156 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:47:39,764 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:47:39,765 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 14:47:39,765 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:47:39,770 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:47:40,446 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 14:47:42,449 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 14:47:43,259 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:47:43,260 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 14:47:43,260 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:47:43,265 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:47:43,450 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 14:47:44,266 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:47:44,452 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 14:47:46,631 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:47:46,632 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 14:47:46,632 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:47:46,637 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:47:47,454 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 14:47:48,455 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 14:47:49,637 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:47:50,770 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:47:50,770 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 14:47:50,771 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:47:50,775 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:47:51,457 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 14:47:51,933 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:47:51,933 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:47:51,935 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:47:52,460 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 14:47:54,165 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:47:54,166 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 14:47:54,166 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:47:54,172 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:47:54,461 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 14:47:55,211 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:47:56,465 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 14:47:57,926 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:47:57,926 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 14:47:57,927 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:47:57,932 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:47:58,466 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 14:48:00,469 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 14:48:00,932 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:48:01,785 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:48:01,786 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 14:48:01,786 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:48:01,791 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:48:02,169 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 14:48:02,470 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 14:48:04,474 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 14:48:06,170 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:48:06,933 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:48:06,933 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:48:06,934 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:48:10,136 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:48:10,137 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 14:48:10,137 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:48:10,142 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:48:10,479 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 14:48:11,179 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:48:12,482 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 14:48:16,035 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:48:16,035 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 14:48:16,036 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:48:16,041 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:48:16,485 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 14:48:17,041 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:48:18,488 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 14:48:21,933 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:48:21,933 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:48:21,935 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:48:22,214 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:48:24,784 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:48:24,784 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 14:48:24,784 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:48:24,790 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:48:25,493 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 14:48:26,495 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 14:48:27,827 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:48:28,647 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:48:28,647 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 14:48:28,648 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:48:28,653 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:48:29,499 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 14:48:30,501 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 14:48:32,171 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 14:48:33,172 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:48:36,491 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:48:36,492 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 14:48:36,492 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:48:36,497 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:48:36,505 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 14:48:36,933 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:48:36,933 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:48:36,935 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:48:38,509 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 14:48:38,562 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:48:40,372 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:48:40,373 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 14:48:40,373 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:48:40,378 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:48:40,510 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 14:48:42,513 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 14:48:43,677 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:48:43,678 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 14:48:43,679 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:48:43,679 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:48:43,683 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:48:44,514 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 14:48:46,517 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 14:48:47,645 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:48:47,646 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 14:48:47,646 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:48:47,652 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:48:48,519 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 14:48:49,652 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:48:50,522 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 14:48:51,062 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:48:51,063 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 14:48:51,063 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:48:51,069 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:48:51,523 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 14:48:51,933 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:48:51,934 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:48:51,935 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:48:52,525 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 14:48:55,228 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:48:55,753 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:48:55,754 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 14:48:55,754 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:48:55,759 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:48:56,527 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 14:48:58,531 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 14:49:00,803 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:49:02,090 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:49:02,091 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 14:49:02,091 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:49:02,096 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:49:02,173 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 14:49:02,533 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 14:49:04,536 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 14:49:05,487 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:49:05,488 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 14:49:05,488 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:49:05,494 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:49:05,537 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 14:49:06,495 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:49:06,933 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:49:06,933 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:49:06,935 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:49:08,541 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 14:49:09,407 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:49:09,408 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 14:49:09,408 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:49:09,413 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:49:09,541 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 14:49:12,451 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:49:12,545 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 14:49:14,093 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:49:14,094 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 14:49:14,094 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:49:14,099 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:49:14,547 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 14:49:16,550 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 14:49:18,100 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:49:18,666 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:49:18,666 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 14:49:18,667 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:49:18,671 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:49:19,552 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 14:49:20,554 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 14:49:21,933 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:49:21,934 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:49:21,935 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:49:23,218 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:49:24,031 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:49:24,032 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 14:49:24,032 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:49:24,036 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:49:24,557 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 14:49:26,560 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 14:49:27,300 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:49:27,301 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 14:49:27,301 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:49:27,306 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:49:27,562 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 14:49:28,306 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:49:28,564 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 14:49:30,845 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:49:30,846 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 14:49:30,846 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:49:30,853 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:49:31,566 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 14:49:32,175 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 14:49:32,569 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 14:49:34,177 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:49:36,228 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:49:36,228 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 14:49:36,229 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:49:36,234 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:49:36,571 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 14:49:36,933 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:49:36,934 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:49:36,935 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:49:38,575 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 14:49:39,194 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:49:40,668 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:49:40,669 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 14:49:40,669 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:49:40,673 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:49:41,577 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 14:49:42,579 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 14:49:43,930 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:49:43,931 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 14:49:43,931 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:49:43,936 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:49:44,581 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 14:49:44,936 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:49:46,584 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 14:49:47,660 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:49:47,661 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 14:49:47,661 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:49:47,666 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:49:48,585 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 14:49:50,589 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 14:49:50,666 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:49:51,933 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:49:51,934 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:49:51,935 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:49:55,948 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:49:55,949 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 14:49:55,949 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:49:55,949 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:49:55,954 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:49:56,592 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 14:49:58,595 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 14:49:59,817 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:49:59,818 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 14:49:59,818 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:49:59,823 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:50:00,597 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 14:50:01,824 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:50:02,181 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 14:50:02,600 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 14:50:03,376 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:50:03,377 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 14:50:03,377 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:50:03,382 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:50:03,600 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 14:50:04,603 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 14:50:06,933 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:50:06,934 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:50:06,935 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:50:07,165 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:50:08,086 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:50:08,087 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 14:50:08,087 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:50:08,092 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:50:08,607 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 14:50:10,610 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 14:50:11,700 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:50:11,701 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 14:50:11,701 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:50:11,705 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:50:12,611 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 14:50:12,706 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:50:14,614 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 14:50:17,706 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:50:17,771 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:50:17,772 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 14:50:17,772 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:50:17,777 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:50:18,617 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 14:50:20,620 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 14:50:21,394 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:50:21,395 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 14:50:21,395 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:50:21,400 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:50:21,621 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 14:50:21,933 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:50:21,934 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:50:21,935 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:50:22,623 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 14:50:23,222 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:50:26,364 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:50:26,364 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 14:50:26,365 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:50:26,370 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:50:26,626 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 14:50:28,407 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:50:28,631 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 14:50:30,370 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:50:30,371 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 14:50:30,371 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:50:30,376 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:50:30,632 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 14:50:32,180 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 14:50:32,635 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 14:50:34,181 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:50:35,614 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:50:35,615 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 14:50:35,615 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:50:35,620 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:50:35,637 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 14:50:36,934 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:50:36,934 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:50:36,935 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:50:38,641 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 14:50:39,559 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:50:39,560 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 14:50:39,560 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:50:39,560 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:50:39,565 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:50:39,642 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 14:50:42,646 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 14:50:43,933 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:50:43,934 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 14:50:43,934 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:50:43,939 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:50:44,647 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 14:50:44,940 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:50:46,650 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 14:50:47,591 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:50:47,592 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 14:50:47,592 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:50:47,597 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:50:47,651 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 14:50:50,597 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:50:50,654 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 14:50:51,206 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:50:51,207 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 14:50:51,207 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:50:51,212 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:50:51,655 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 14:50:51,934 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:50:51,934 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:50:51,935 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:50:52,658 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 14:50:55,121 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:50:55,122 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 14:50:55,122 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:50:55,127 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:50:55,660 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 14:50:56,167 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:50:56,662 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 14:50:59,308 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:50:59,309 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 14:50:59,309 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:50:59,314 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:50:59,664 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 14:51:00,667 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 14:51:01,315 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:51:02,185 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 14:51:04,938 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:51:04,939 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 14:51:04,939 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:51:04,944 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:51:05,671 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 14:51:06,673 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 14:51:06,934 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:51:06,934 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:51:06,935 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:51:07,232 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:51:09,026 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:51:09,027 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 14:51:09,027 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:51:09,032 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:51:09,676 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 14:51:10,678 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 14:51:13,013 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:51:13,014 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 14:51:13,015 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:51:13,015 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:51:13,019 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:51:13,680 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 14:51:14,683 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 14:51:17,244 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:51:17,245 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 14:51:17,245 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:51:17,250 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:51:17,685 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 14:51:18,250 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:51:18,687 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 14:51:21,181 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:51:21,182 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 14:51:21,182 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:51:21,187 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:51:21,689 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 14:51:21,934 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:51:21,934 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:51:21,936 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:51:22,692 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 14:51:24,223 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:51:29,055 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:51:29,055 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 14:51:29,055 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:51:29,060 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:51:29,698 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 14:51:30,098 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:51:30,700 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 14:51:32,184 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 14:51:32,784 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:51:32,784 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 14:51:32,784 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:51:32,790 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:51:33,702 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 14:51:34,704 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 14:51:35,790 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:51:36,399 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:51:36,400 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 14:51:36,400 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:51:36,405 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:51:36,706 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 14:51:36,934 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:51:36,934 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:51:36,936 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:51:38,710 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 14:51:41,140 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:51:41,315 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:51:41,315 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 14:51:41,315 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:51:41,320 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:51:41,711 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 14:51:42,714 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 14:51:45,394 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:51:45,395 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 14:51:45,396 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:51:45,400 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:51:45,716 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 14:51:46,401 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:51:46,718 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 14:51:48,715 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:51:48,716 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 14:51:48,716 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:51:48,721 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:51:48,721 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 14:51:50,728 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 14:51:51,722 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:51:51,934 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:51:51,934 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:51:51,936 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:51:52,371 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:51:52,372 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 14:51:52,372 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:51:52,377 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:51:52,729 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 14:51:54,733 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 14:51:56,297 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:51:56,297 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 14:51:56,298 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:51:56,302 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:51:56,734 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 14:51:57,303 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:51:58,737 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 14:51:59,669 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:51:59,670 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 14:51:59,670 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:51:59,675 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:51:59,738 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 14:52:02,190 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 14:52:02,742 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 14:52:03,191 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:52:05,574 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:52:05,575 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 14:52:05,575 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:52:05,580 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:52:05,745 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 14:52:06,747 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 14:52:06,934 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:52:06,934 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:52:06,936 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:52:09,191 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:52:11,316 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:52:11,317 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 14:52:11,317 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:52:11,322 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:52:11,751 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 14:52:12,753 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 14:52:14,358 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:52:15,582 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:52:15,583 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 14:52:15,583 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:52:15,588 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:52:15,755 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 14:52:16,758 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 14:52:19,589 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:52:21,125 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:52:21,126 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 14:52:21,126 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:52:21,131 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:52:21,761 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 14:52:21,934 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:52:21,934 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:52:21,936 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:52:22,763 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 14:52:25,149 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:52:25,625 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:52:25,626 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 14:52:25,626 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:52:25,631 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:52:25,765 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 14:52:28,771 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 14:52:30,670 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:52:30,740 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:52:30,741 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 14:52:30,741 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:52:30,746 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:52:30,772 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 14:52:32,189 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 14:52:32,775 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 14:52:34,226 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:52:34,227 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 14:52:34,227 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:52:34,232 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:52:34,776 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 14:52:36,232 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:52:36,779 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 14:52:36,934 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:52:36,935 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:52:36,936 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:52:37,958 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:52:37,958 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 14:52:37,959 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:52:37,964 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:52:38,781 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 14:52:40,782 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 14:52:41,463 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:52:41,464 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 14:52:41,464 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:52:41,465 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:52:41,469 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:52:41,783 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 14:52:42,785 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 14:52:45,905 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:52:45,906 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 14:52:45,906 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:52:45,911 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:52:46,788 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 14:52:46,912 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:52:48,792 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 14:52:50,424 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:52:50,425 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 14:52:50,425 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:52:50,430 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:52:50,793 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 14:52:51,934 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:52:51,935 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:52:51,936 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:52:52,155 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:52:52,796 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 14:52:54,832 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:52:54,832 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 14:52:54,833 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:52:54,838 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:52:55,798 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 14:52:56,801 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 14:52:57,874 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:52:58,774 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:52:58,774 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 14:52:58,775 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:52:58,780 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:52:58,802 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 14:53:00,805 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 14:53:02,191 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 14:53:03,192 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:53:05,398 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:53:05,399 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 14:53:05,399 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:53:05,404 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:53:05,809 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 14:53:06,811 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 14:53:06,934 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:53:06,935 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:53:06,936 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:53:09,178 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:53:14,179 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:53:14,465 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:53:14,466 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 14:53:14,466 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:53:14,471 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:53:14,817 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 14:53:16,820 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 14:53:19,510 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:53:21,934 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:53:21,935 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:53:21,936 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:53:22,785 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:53:22,785 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 14:53:22,786 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:53:22,791 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:53:22,823 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 14:53:24,827 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 14:53:24,829 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:53:29,830 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:53:31,189 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:53:31,190 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 14:53:31,190 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:53:31,195 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:53:31,833 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 14:53:32,193 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 14:53:32,835 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 14:53:35,194 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:53:36,535 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:53:36,535 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 14:53:36,536 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:53:36,541 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:53:36,838 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 14:53:36,935 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:53:36,935 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:53:36,936 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:53:38,841 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 14:53:41,132 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:53:44,862 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:53:44,862 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 14:53:44,863 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:53:44,868 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:53:45,846 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 14:53:46,846 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 14:53:46,906 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:53:48,609 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:53:48,610 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 14:53:48,610 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:53:48,615 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:53:48,847 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 14:53:50,851 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 14:53:51,935 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:53:51,935 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:53:51,936 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:53:52,229 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:53:52,598 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:53:52,599 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 14:53:52,599 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:53:52,604 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:53:52,852 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 14:53:54,855 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 14:53:57,642 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:53:57,837 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:53:57,838 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 14:53:57,838 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:53:57,843 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:53:57,857 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 14:54:00,861 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 14:54:02,195 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 14:54:03,197 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:54:03,752 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:54:03,753 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 14:54:03,753 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:54:03,759 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:54:03,863 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 14:54:04,866 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 14:54:06,935 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:54:06,935 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:54:06,937 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:54:08,058 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:54:08,059 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 14:54:08,059 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:54:08,064 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:54:08,869 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 14:54:09,102 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:54:10,872 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 14:54:13,172 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:54:13,172 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 14:54:13,173 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:54:13,178 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:54:13,874 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 14:54:14,178 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:54:14,877 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 14:54:17,934 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:54:17,934 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 14:54:17,935 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:54:17,940 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:54:18,879 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 14:54:19,940 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:54:20,882 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 14:54:21,708 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:54:21,709 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 14:54:21,709 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:54:21,714 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:54:21,883 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 14:54:21,935 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:54:21,935 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:54:21,936 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:54:22,884 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 14:54:25,158 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:54:25,490 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:54:25,491 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 14:54:25,491 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:54:25,496 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:54:25,886 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 14:54:26,889 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 14:54:30,040 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:54:30,041 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 14:54:30,041 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:54:30,047 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:54:30,893 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 14:54:31,047 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:54:32,201 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 14:54:32,896 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 14:54:33,238 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:54:33,238 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 14:54:33,239 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:54:33,244 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:54:33,896 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 14:54:34,899 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 14:54:36,244 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:54:36,935 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:54:36,935 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:54:36,936 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:54:37,657 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:54:37,657 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 14:54:37,657 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:54:37,662 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:54:37,901 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 14:54:38,904 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 14:54:41,698 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:54:46,057 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:54:46,058 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 14:54:46,058 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:54:46,064 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:54:46,909 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 14:54:47,064 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:54:48,912 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 14:54:51,048 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:54:51,049 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 14:54:51,049 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:54:51,054 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:54:51,914 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 14:54:51,935 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:54:51,935 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:54:51,937 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:54:52,162 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:54:52,917 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 14:54:54,333 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:54:54,333 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 14:54:54,334 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:54:54,339 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:54:54,918 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 14:54:56,921 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 14:54:57,374 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:55:00,324 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:55:00,325 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 14:55:00,325 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:55:00,330 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:55:00,924 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 14:55:02,200 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 14:55:02,927 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 14:55:03,201 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:55:04,471 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:55:04,472 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 14:55:04,472 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:55:04,477 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:55:04,928 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 14:55:06,932 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 14:55:06,935 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:55:06,935 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:55:06,937 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:55:09,164 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:55:12,939 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:55:12,939 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 14:55:12,939 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:55:12,944 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:55:13,937 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 14:55:14,939 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 14:55:14,982 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:55:16,999 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:55:17,000 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 14:55:17,000 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:55:17,005 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:55:17,941 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 14:55:18,944 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 14:55:20,005 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:55:20,531 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:55:20,531 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 14:55:20,532 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:55:20,536 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:55:20,945 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 14:55:21,935 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:55:21,936 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:55:21,937 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:55:22,948 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 14:55:24,804 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:55:24,804 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 14:55:24,805 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:55:24,810 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:55:24,950 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 14:55:25,846 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:55:26,953 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 14:55:28,544 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:55:28,545 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 14:55:28,545 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:55:28,550 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:55:28,956 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 14:55:30,959 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 14:55:31,551 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:55:32,148 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:55:32,149 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 14:55:32,149 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:55:32,154 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:55:32,205 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 14:55:32,960 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 14:55:34,961 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 14:55:35,665 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:55:35,666 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 14:55:35,666 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:55:35,671 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:55:35,962 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 14:55:36,671 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:55:36,935 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:55:36,936 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:55:36,937 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:55:36,964 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 14:55:40,470 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:55:40,470 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 14:55:40,471 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:55:40,475 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:55:40,967 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 14:55:42,514 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:55:42,971 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 14:55:46,631 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:55:46,631 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 14:55:46,632 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:55:46,636 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:55:46,973 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 14:55:47,637 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:55:48,977 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 14:55:51,311 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:55:51,312 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 14:55:51,312 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:55:51,317 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:55:51,935 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:55:51,936 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:55:51,937 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:55:51,978 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 14:55:52,981 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 14:55:53,252 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:55:55,629 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:55:55,630 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 14:55:55,630 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:55:55,635 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:55:55,983 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 14:55:56,985 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 14:55:58,674 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:56:01,249 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:56:01,250 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 14:56:01,250 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:56:01,255 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:56:01,988 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 14:56:02,204 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 14:56:02,991 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 14:56:04,205 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:56:05,574 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:56:05,574 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 14:56:05,575 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:56:05,580 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:56:05,994 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 14:56:06,935 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:56:06,936 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:56:06,937 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:56:06,996 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 14:56:09,305 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:56:09,305 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 14:56:09,306 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:56:09,306 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:56:09,311 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:56:09,999 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 14:56:11,001 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 14:56:14,287 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:56:14,288 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 14:56:14,288 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:56:14,294 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:56:15,004 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 14:56:15,294 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:56:17,007 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 14:56:18,098 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:56:18,099 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 14:56:18,099 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:56:18,104 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:56:19,008 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 14:56:21,012 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 14:56:21,105 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:56:21,382 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:56:21,383 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 14:56:21,383 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:56:21,388 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:56:21,935 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:56:21,936 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:56:21,937 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:56:22,012 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 14:56:23,015 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 14:56:25,988 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:56:25,989 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 14:56:25,989 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:56:25,994 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:56:26,017 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 14:56:27,030 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:56:29,023 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 14:56:30,944 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:56:30,945 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 14:56:30,945 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:56:30,951 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:56:31,024 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 14:56:32,206 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 14:56:32,207 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:56:33,027 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 14:56:35,377 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:56:35,378 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 14:56:35,378 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:56:35,383 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:56:36,029 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 14:56:36,935 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:56:36,936 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:56:36,937 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:56:37,031 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 14:56:38,090 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:56:39,572 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:56:39,573 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 14:56:39,573 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:56:39,578 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:56:40,034 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 14:56:41,034 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 14:56:43,114 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:56:43,115 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 14:56:43,115 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:56:43,115 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:56:43,120 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:56:44,036 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 14:56:45,039 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 14:56:46,480 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:56:46,481 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 14:56:46,481 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:56:46,486 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:56:47,040 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 14:56:48,486 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:56:49,043 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 14:56:49,946 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:56:49,947 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 14:56:49,947 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:56:49,952 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:56:50,044 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 14:56:51,046 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 14:56:51,936 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:56:51,936 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:56:51,937 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:56:53,682 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:56:53,682 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 14:56:53,682 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:56:53,683 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:56:53,688 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:56:54,048 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 14:56:55,051 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 14:56:58,709 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:56:58,710 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 14:56:58,710 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:56:58,710 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:56:58,715 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:56:59,053 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 14:57:01,056 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 14:57:02,208 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 14:57:02,865 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:57:02,866 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 14:57:02,866 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:57:02,871 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:57:03,057 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 14:57:03,871 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:57:05,061 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 14:57:06,936 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:57:06,936 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:57:06,938 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:57:08,784 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:57:08,785 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 14:57:08,785 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:57:08,790 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:57:09,064 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 14:57:09,826 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:57:11,068 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 14:57:14,340 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:57:14,340 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 14:57:14,341 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:57:14,346 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:57:15,070 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 14:57:15,346 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:57:17,071 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 14:57:20,302 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:57:20,303 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 14:57:20,303 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:57:20,308 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:57:21,074 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 14:57:21,309 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:57:21,936 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:57:21,936 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:57:21,937 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:57:23,077 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 14:57:25,864 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:57:25,865 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 14:57:25,865 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:57:25,871 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:57:26,079 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 14:57:26,906 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:57:27,082 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 14:57:30,773 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:57:30,774 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 14:57:30,774 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:57:30,779 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:57:31,086 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 14:57:32,210 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 14:57:32,212 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:57:33,089 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 14:57:34,966 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:57:34,966 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 14:57:34,966 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:57:34,971 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:57:35,090 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 14:57:36,936 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:57:36,936 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:57:36,938 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:57:37,093 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 14:57:38,183 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:57:39,235 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:57:39,236 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 14:57:39,236 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:57:39,241 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:57:40,095 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 14:57:41,098 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 14:57:43,278 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:57:43,886 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:57:43,887 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 14:57:43,887 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:57:43,892 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:57:44,100 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 14:57:45,102 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 14:57:48,249 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:57:48,250 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 14:57:48,250 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:57:48,255 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:57:49,105 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 14:57:49,256 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:57:51,108 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 14:57:51,936 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:57:51,936 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:57:51,937 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:57:54,153 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:57:54,153 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 14:57:54,153 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:57:54,158 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:57:55,111 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 14:57:55,194 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:57:57,114 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 14:58:00,194 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:58:02,216 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 14:58:02,357 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:58:02,358 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 14:58:02,358 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:58:02,363 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:58:03,117 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 14:58:05,121 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 14:58:05,363 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:58:06,172 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:58:06,173 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 14:58:06,173 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:58:06,178 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:58:06,936 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:58:06,936 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:58:06,938 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:58:07,123 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 14:58:09,126 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 14:58:10,073 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:58:10,074 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 14:58:10,074 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:58:10,079 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:58:10,127 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 14:58:11,118 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:58:13,131 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 14:58:13,567 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:58:13,568 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 14:58:13,568 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:58:13,573 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:58:14,132 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 14:58:15,134 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 14:58:16,573 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:58:18,527 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:58:18,528 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 14:58:18,528 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:58:18,533 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:58:19,137 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 14:58:21,140 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 14:58:21,936 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:58:21,936 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:58:21,938 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:58:22,002 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:58:22,186 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 14:58:22,186 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:58:22,186 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:58:22,192 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:58:23,141 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 14:58:25,144 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 14:58:25,707 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:58:25,708 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 14:58:25,708 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:58:25,713 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:58:26,145 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 14:58:27,148 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 14:58:27,714 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:58:29,629 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:58:29,630 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 14:58:29,630 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:58:29,635 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:58:30,151 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 14:58:31,153 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 14:58:32,215 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 14:58:33,217 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:58:34,179 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:58:34,180 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 14:58:34,180 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:58:34,185 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:58:35,156 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 14:58:36,936 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:58:36,937 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:58:36,938 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:58:37,159 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 14:58:38,180 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:58:38,181 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 14:58:38,181 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:58:38,186 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:58:38,222 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:58:39,161 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 14:58:41,164 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 14:58:42,841 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:58:42,842 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 14:58:42,842 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:58:42,847 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:58:43,165 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 14:58:43,848 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:58:45,168 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 14:58:47,021 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:58:47,022 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 14:58:47,022 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:58:47,027 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:58:47,170 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 14:58:49,028 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:58:49,173 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 14:58:50,903 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:58:50,903 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 14:58:50,903 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:58:50,909 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:58:51,174 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 14:58:51,936 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:58:51,937 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:58:51,938 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:58:53,177 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 14:58:54,196 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:58:54,647 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:58:54,648 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 14:58:54,648 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:58:54,653 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:58:55,179 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 14:58:57,182 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 14:58:59,275 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:58:59,276 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 14:58:59,276 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:58:59,276 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:58:59,281 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:59:00,184 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 14:59:01,186 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 14:59:02,220 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 14:59:05,222 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:59:06,936 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:59:06,937 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:59:06,938 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:59:07,690 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:59:07,691 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 14:59:07,691 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:59:07,696 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:59:08,192 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 14:59:09,194 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 14:59:10,734 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:59:15,734 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:59:16,049 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:59:16,050 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 14:59:16,050 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:59:16,055 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:59:16,198 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 14:59:17,201 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 14:59:20,143 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:59:20,143 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 14:59:20,144 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:59:20,149 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:59:20,203 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 14:59:21,150 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:59:21,205 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 14:59:21,936 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:59:21,937 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:59:21,938 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:59:25,474 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:59:25,475 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 14:59:25,475 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:59:25,480 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:59:26,209 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 14:59:26,518 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:59:27,212 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 14:59:29,579 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:59:29,580 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 14:59:29,580 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:59:29,585 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:59:30,215 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 14:59:31,218 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 14:59:31,586 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:59:32,220 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 14:59:32,938 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:59:32,938 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 14:59:32,939 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:59:32,943 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:59:33,222 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 14:59:35,225 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 14:59:36,741 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:59:36,742 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 14:59:36,742 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:59:36,742 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:59:36,747 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:59:36,936 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:59:36,937 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:59:36,938 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:59:37,226 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 14:59:39,230 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 14:59:40,134 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:59:40,135 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 14:59:40,135 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:59:40,140 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:59:40,230 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 14:59:42,178 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:59:43,234 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 14:59:44,447 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:59:44,448 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 14:59:44,448 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:59:44,455 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:59:45,235 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 14:59:47,239 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 14:59:47,456 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:59:51,937 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 14:59:51,937 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 14:59:51,938 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 14:59:53,190 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 14:59:53,838 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 14:59:53,838 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 14:59:53,839 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 14:59:53,844 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 14:59:54,243 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 14:59:55,245 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 14:59:58,882 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:00:01,551 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:00:01,552 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:00:01,552 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:00:01,557 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:00:02,221 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 15:00:02,250 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:00:03,252 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:00:04,223 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:00:06,937 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:00:06,937 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:00:06,938 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:00:09,758 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:00:09,758 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:00:09,758 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:00:09,759 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:00:09,764 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:00:10,258 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:00:11,260 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:00:13,713 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:00:13,716 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:00:13,717 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:00:13,721 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:00:14,262 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:00:15,265 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:00:15,722 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:00:18,320 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:00:18,321 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:00:18,321 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:00:18,327 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:00:19,267 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:00:21,270 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:00:21,327 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:00:21,937 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:00:21,937 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:00:21,938 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:00:22,190 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:00:22,190 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:00:22,190 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:00:22,196 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:00:22,271 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:00:25,275 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:00:26,179 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:00:26,180 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:00:26,180 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:00:26,185 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:00:26,276 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:00:27,185 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:00:27,278 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:00:29,694 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:00:29,694 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:00:29,695 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:00:29,700 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:00:30,281 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:00:31,284 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:00:32,224 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 15:00:32,225 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:00:32,987 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:00:32,988 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:00:32,988 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:00:32,993 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:00:33,285 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:00:35,288 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:00:36,416 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:00:36,417 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:00:36,417 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:00:36,422 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:00:36,937 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:00:36,937 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:00:36,939 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:00:37,290 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:00:37,493 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:00:39,293 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:00:41,275 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:00:41,275 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:00:41,275 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:00:41,280 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:00:41,294 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:00:43,298 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:00:43,318 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:00:46,777 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:00:46,778 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:00:46,778 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:00:46,783 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:00:47,300 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:00:48,784 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:00:49,303 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:00:51,937 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:00:51,937 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:00:51,939 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:00:53,123 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:00:53,123 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:00:53,123 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:00:53,128 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:00:53,306 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:00:54,166 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:00:55,309 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:00:56,641 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:00:56,642 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:00:56,642 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:00:56,647 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:00:57,311 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:00:59,314 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:00:59,648 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:01:00,375 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:01:00,376 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:01:00,376 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:01:00,381 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:01:01,316 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:01:02,226 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 15:01:03,319 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:01:05,227 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:01:06,200 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:01:06,201 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:01:06,201 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:01:06,206 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:01:06,321 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:01:06,937 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:01:06,937 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:01:06,939 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:01:07,324 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:01:09,558 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:01:09,559 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:01:09,559 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:01:09,564 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:01:10,326 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:01:10,602 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:01:11,329 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:01:13,670 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:01:13,671 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:01:13,671 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:01:13,676 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:01:14,331 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:01:15,333 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:01:15,676 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:01:17,417 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:01:17,418 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:01:17,418 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:01:17,424 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:01:18,335 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:01:19,338 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:01:21,424 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:01:21,879 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:01:21,880 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:01:21,880 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:01:21,885 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:01:21,937 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:01:21,937 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:01:21,939 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:01:22,340 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:01:23,343 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:01:27,219 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:01:27,458 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:01:27,458 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:01:27,459 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:01:27,464 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:01:28,347 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:01:29,350 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:01:31,399 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:01:31,400 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:01:31,400 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:01:31,405 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:01:32,228 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 15:01:32,229 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:01:32,352 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:01:33,354 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:01:35,104 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:01:35,104 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:01:35,104 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:01:35,109 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:01:35,356 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:01:36,937 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:01:36,937 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:01:36,939 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:01:37,359 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:01:38,150 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:01:40,371 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:01:40,371 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:01:40,371 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:01:40,377 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:01:41,362 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:01:43,365 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:01:43,414 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:01:48,414 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:01:48,839 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:01:48,840 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:01:48,840 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:01:48,846 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:01:49,369 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:01:51,372 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:01:51,937 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:01:51,938 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:01:51,939 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:01:54,152 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:01:57,104 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:01:57,105 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:01:57,105 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:01:57,110 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:01:57,376 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:01:59,379 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:02:00,146 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:02:01,133 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:02:01,134 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:02:01,134 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:02:01,139 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:02:01,380 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:02:02,230 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 15:02:03,381 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:02:04,610 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:02:04,611 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:02:04,611 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:02:04,616 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:02:05,383 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:02:05,617 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:02:06,937 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:02:06,938 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:02:06,939 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:02:07,387 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:02:07,967 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:02:07,967 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:02:07,967 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:02:07,972 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:02:08,388 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:02:09,391 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:02:11,011 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:02:12,245 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:02:12,246 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:02:12,246 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:02:12,251 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:02:12,392 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:02:13,395 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:02:16,252 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:02:21,252 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:02:21,582 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:02:21,583 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:02:21,583 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:02:21,588 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:02:21,937 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:02:21,938 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:02:21,939 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:02:22,401 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:02:23,404 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:02:24,601 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:02:24,602 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:02:24,602 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:02:24,607 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:02:25,405 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:02:26,647 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:02:27,409 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:02:28,404 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:02:28,405 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:02:28,405 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:02:28,410 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:02:28,412 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:02:31,416 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:02:32,173 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:02:32,174 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:02:32,174 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:02:32,174 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:02:32,179 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:02:32,236 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 15:02:32,417 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:02:33,419 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:02:35,921 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:02:35,922 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:02:35,922 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:02:35,927 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:02:36,421 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:02:36,937 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:02:36,938 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:02:36,939 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:02:37,424 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:02:38,140 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:02:39,781 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:02:39,781 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:02:39,781 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:02:39,786 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:02:40,426 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:02:41,429 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:02:43,270 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:02:43,270 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:02:43,271 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:02:43,271 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:02:43,276 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:02:43,430 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:02:45,433 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:02:48,277 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:02:49,011 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:02:49,012 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:02:49,012 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:02:49,019 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:02:49,436 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:02:51,439 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:02:51,938 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:02:51,938 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:02:51,939 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:02:54,227 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:02:57,485 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:02:57,486 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:02:57,486 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:02:57,492 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:02:58,443 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:02:59,446 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:02:59,531 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:03:00,876 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:03:00,876 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:03:00,876 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:03:00,882 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:03:01,447 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:03:02,235 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 15:03:03,451 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:03:05,236 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:03:06,107 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:03:06,108 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:03:06,108 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:03:06,114 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:03:06,453 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:03:06,938 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:03:06,938 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:03:06,939 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:03:07,456 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:03:09,753 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:03:09,754 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:03:09,754 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:03:09,759 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:03:10,459 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:03:10,799 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:03:11,461 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:03:14,113 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:03:14,114 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:03:14,114 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:03:14,119 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:03:14,463 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:03:15,464 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:03:16,119 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:03:21,120 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:03:21,938 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:03:21,938 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:03:21,939 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:03:22,592 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:03:22,592 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:03:22,592 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:03:22,597 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:03:23,469 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:03:25,473 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:03:26,590 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:03:26,591 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:03:26,591 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:03:26,591 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:03:26,597 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:03:27,474 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:03:29,479 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:03:31,246 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:03:31,247 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:03:31,247 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:03:31,252 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:03:31,480 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:03:32,240 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 15:03:32,241 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:03:33,483 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:03:34,618 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:03:34,619 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:03:34,619 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:03:34,624 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:03:35,485 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:03:36,938 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:03:36,938 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:03:36,940 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:03:37,488 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:03:37,689 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:03:38,730 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:03:38,730 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:03:38,731 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:03:38,736 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:03:39,490 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:03:41,493 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:03:42,775 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:03:43,347 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:03:43,347 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:03:43,348 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:03:43,353 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:03:43,494 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:03:45,497 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:03:47,926 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:03:47,927 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:03:47,928 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:03:47,928 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:03:47,933 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:03:48,499 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:03:49,502 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:03:51,938 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:03:51,938 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:03:51,939 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:03:52,597 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:03:52,598 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:03:52,598 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:03:52,603 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:03:53,504 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:03:53,639 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:03:55,508 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:03:56,173 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:03:56,174 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:03:56,174 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:03:56,180 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:03:56,508 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:03:57,511 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:03:59,180 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:04:00,706 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:04:00,707 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:04:00,707 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:04:00,713 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:04:01,513 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:04:02,239 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 15:04:03,517 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:04:04,044 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:04:04,045 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:04:04,045 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:04:04,050 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:04:04,517 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:04:05,051 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:04:05,520 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:04:06,938 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:04:06,938 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:04:06,940 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:04:07,917 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:04:07,917 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:04:07,917 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:04:07,923 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:04:08,523 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:04:09,526 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:04:10,959 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:04:13,246 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:04:13,246 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:04:13,247 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:04:13,252 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:04:13,529 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:04:15,532 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:04:16,253 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:04:17,642 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:04:17,643 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:04:17,643 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:04:17,649 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:04:18,534 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:04:19,537 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:04:21,650 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:04:21,938 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:04:21,939 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:04:21,940 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:04:22,203 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:04:22,204 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:04:22,204 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:04:22,210 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:04:22,539 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:04:23,542 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:04:26,842 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:04:26,843 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:04:26,843 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:04:26,843 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:04:26,848 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:04:27,545 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:04:29,549 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:04:30,989 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:04:30,990 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:04:30,990 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:04:30,995 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:04:31,551 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:04:31,995 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:04:32,241 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 15:04:33,554 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:04:36,938 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:04:36,938 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:04:36,940 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:04:37,153 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:04:39,276 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:04:39,277 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:04:39,277 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:04:39,282 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:04:39,558 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:04:41,562 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:04:42,322 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:04:44,401 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:04:44,402 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:04:44,402 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:04:44,407 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:04:44,564 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:04:45,566 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:04:47,408 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:04:47,915 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:04:47,916 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:04:47,916 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:04:47,921 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:04:48,569 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:04:49,571 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:04:51,572 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:04:51,573 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:04:51,573 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:04:51,578 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:04:51,938 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:04:51,939 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:04:51,940 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:04:52,574 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:04:53,209 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:04:53,577 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:04:56,006 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:04:56,006 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:04:56,006 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:04:56,011 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:04:56,579 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:04:57,582 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:04:59,051 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:05:00,096 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:05:00,097 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:05:00,097 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:05:00,102 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:05:00,584 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:05:01,587 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:05:02,244 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 15:05:04,195 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:05:04,196 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:05:04,196 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:05:04,196 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:05:04,202 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:05:04,589 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:05:05,591 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:05:06,938 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:05:06,939 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:05:06,940 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:05:09,832 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:05:09,833 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:05:09,833 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:05:09,833 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:05:09,838 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:05:10,596 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:05:11,598 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:05:13,772 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:05:13,773 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:05:13,773 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:05:13,778 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:05:14,600 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:05:15,603 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:05:15,778 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:05:17,781 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:05:17,782 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:05:17,782 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:05:17,790 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:05:18,605 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:05:19,610 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:05:20,791 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:05:21,113 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:05:21,114 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:05:21,114 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:05:21,122 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:05:21,611 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:05:21,939 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:05:21,939 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:05:21,939 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:05:23,619 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:05:24,355 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:05:24,355 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:05:24,355 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:05:24,364 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:05:24,620 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:05:25,628 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:05:26,399 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:05:29,789 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:05:29,790 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:05:29,790 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:05:29,800 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:05:30,634 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:05:31,637 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:05:31,801 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:05:32,246 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 15:05:33,129 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:05:33,129 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:05:33,129 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:05:33,140 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:05:33,638 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:05:35,643 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:05:36,939 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:05:36,939 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:05:36,940 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:05:37,233 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:05:41,075 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:05:41,075 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:05:41,075 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:05:41,082 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:05:41,648 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:05:43,119 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:05:43,650 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:05:44,847 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:05:44,848 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:05:44,848 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:05:44,857 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:05:45,651 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:05:47,658 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:05:48,857 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:05:49,085 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:05:49,086 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:05:49,086 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:05:49,096 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:05:49,659 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:05:51,668 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:05:51,939 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:05:51,939 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:05:51,940 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:05:53,708 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:05:53,709 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:05:53,709 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:05:53,716 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:05:54,670 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:05:54,751 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:05:55,675 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:05:59,323 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:05:59,332 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:05:59,332 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:05:59,339 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:05:59,678 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:06:00,339 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:06:01,684 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:06:02,248 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 15:06:03,449 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:06:03,450 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:06:03,450 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:06:03,458 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:06:03,685 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:06:05,459 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:06:05,691 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:06:06,939 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:06:06,939 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:06:06,940 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:06:07,162 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:06:07,218 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:06:07,218 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:06:07,228 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:06:07,693 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:06:09,700 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:06:11,228 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:06:11,424 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:06:11,425 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:06:11,425 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:06:11,434 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:06:11,701 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:06:13,707 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:06:16,467 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:06:19,877 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:06:19,878 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:06:19,878 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:06:19,887 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:06:20,713 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:06:21,717 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:06:21,887 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:06:21,939 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:06:21,939 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:06:21,940 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:06:24,172 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:06:24,173 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:06:24,173 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:06:24,181 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:06:24,720 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:06:25,725 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:06:27,215 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:06:29,365 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:06:29,366 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:06:29,366 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:06:29,373 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:06:29,731 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:06:31,736 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:06:32,250 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 15:06:32,252 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:06:34,311 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:06:34,312 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:06:34,312 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:06:34,319 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:06:34,738 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:06:35,747 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:06:36,939 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:06:36,939 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:06:36,941 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:06:38,137 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:06:40,294 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:06:40,295 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:06:40,295 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:06:40,303 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:06:40,751 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:06:41,755 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:06:43,339 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:06:48,339 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:06:48,716 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:06:48,716 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:06:48,716 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:06:48,722 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:06:48,761 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:06:51,769 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:06:51,939 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:06:51,939 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:06:51,941 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:06:52,247 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:06:52,248 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:06:52,248 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:06:52,256 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:06:52,770 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:06:53,776 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:06:54,291 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:06:57,555 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:06:57,556 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:06:57,556 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:06:57,563 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:06:57,779 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:06:59,563 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:06:59,785 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:07:01,261 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:07:01,261 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:07:01,262 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:07:01,268 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:07:01,787 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:07:02,260 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 15:07:03,792 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:07:04,722 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:07:04,723 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:07:04,723 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:07:04,723 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:07:04,730 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:07:04,793 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:07:06,939 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:07:06,939 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:07:06,941 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:07:07,802 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:07:09,276 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:07:09,277 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:07:09,277 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:07:09,284 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:07:09,805 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:07:10,319 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:07:11,809 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:07:13,219 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:07:13,220 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:07:13,220 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:07:13,227 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:07:13,811 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:07:15,818 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:07:16,227 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:07:17,262 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:07:17,263 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:07:17,263 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:07:17,269 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:07:17,819 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:07:19,823 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:07:21,270 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:07:21,939 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:07:21,939 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:07:21,941 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:07:23,486 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:07:23,486 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:07:23,486 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:07:23,494 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:07:23,826 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:07:25,831 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:07:26,425 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:07:26,426 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:07:26,426 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:07:26,426 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:07:26,432 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:07:26,832 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:07:27,835 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:07:30,013 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:07:30,014 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:07:30,014 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:07:30,019 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:07:30,839 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:07:31,840 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:07:32,020 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:07:32,254 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 15:07:33,946 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:07:33,947 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:07:33,947 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:07:33,952 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:07:34,842 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:07:35,845 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:07:36,939 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:07:36,940 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:07:36,941 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:07:37,160 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:07:38,701 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:07:38,702 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:07:38,702 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:07:38,708 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:07:38,847 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:07:41,854 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:07:42,743 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:07:44,048 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:07:44,049 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:07:44,049 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:07:44,056 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:07:44,857 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:07:45,860 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:07:48,057 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:07:50,084 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:07:50,085 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:07:50,085 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:07:50,091 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:07:50,864 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:07:51,867 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:07:51,939 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:07:51,940 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:07:51,941 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:07:53,147 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:07:54,977 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:07:54,978 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:07:54,978 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:07:54,984 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:07:55,870 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:07:57,874 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:07:59,023 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:08:01,402 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:08:01,403 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:08:01,403 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:08:01,409 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:08:01,877 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:08:02,260 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 15:08:03,881 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:08:04,262 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:08:06,434 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:08:06,435 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:08:06,435 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:08:06,441 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:08:06,884 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:08:06,939 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:08:06,940 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:08:06,941 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:08:07,886 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:08:10,186 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:08:10,187 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:08:10,187 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:08:10,187 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:08:10,192 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:08:10,889 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:08:11,892 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:08:13,690 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:08:13,691 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:08:13,691 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:08:13,696 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:08:13,894 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:08:15,697 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:08:15,898 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:08:17,799 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:08:17,800 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:08:17,800 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:08:17,805 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:08:17,899 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:08:19,903 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:08:20,806 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:08:21,939 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:08:21,940 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:08:21,941 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:08:23,549 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:08:23,550 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:08:23,550 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:08:23,555 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:08:23,905 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:08:25,909 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:08:26,595 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:08:30,627 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:08:30,628 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:08:30,628 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:08:30,633 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:08:30,913 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:08:31,634 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:08:31,916 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:08:32,259 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 15:08:35,024 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:08:35,024 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:08:35,025 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:08:35,030 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:08:35,919 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:08:36,940 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:08:36,940 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:08:36,941 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:08:37,228 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:08:37,920 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:08:39,304 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:08:39,304 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:08:39,304 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:08:39,310 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:08:39,922 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:08:41,925 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:08:42,347 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:08:45,224 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:08:45,224 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:08:45,225 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:08:45,229 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:08:45,928 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:08:47,931 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:08:48,230 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:08:49,232 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:08:49,233 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:08:49,233 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:08:49,239 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:08:49,932 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:08:51,936 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:08:51,940 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:08:51,940 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:08:51,941 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:08:54,225 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:08:54,872 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:08:54,873 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:08:54,873 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:08:54,878 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:08:54,938 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:08:57,942 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:08:58,629 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:08:58,629 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:08:58,630 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:08:58,635 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:08:58,942 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:08:59,635 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:08:59,945 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:09:02,261 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 15:09:03,050 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:09:03,050 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:09:03,051 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:09:03,056 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:09:03,947 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:09:05,056 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:09:05,950 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:09:06,940 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:09:06,940 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:09:06,941 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:09:07,016 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:09:07,181 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:09:07,181 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:09:07,186 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:09:07,953 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:09:09,956 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:09:10,187 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:09:12,803 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:09:12,804 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:09:12,804 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:09:12,809 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:09:12,958 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:09:13,959 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:09:15,810 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:09:17,815 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:09:17,816 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:09:17,816 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:09:17,821 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:09:17,961 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:09:19,965 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:09:20,822 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:09:21,940 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:09:21,940 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:09:21,941 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:09:23,674 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:09:23,678 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:09:23,678 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:09:23,683 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:09:23,967 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:09:25,971 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:09:26,719 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:09:29,139 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:09:29,140 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:09:29,140 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:09:29,145 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:09:29,975 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:09:31,978 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:09:32,146 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:09:32,264 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 15:09:33,236 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:09:33,236 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:09:33,236 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:09:33,241 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:09:33,979 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:09:35,982 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:09:36,940 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:09:36,940 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:09:36,942 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:09:37,182 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:09:37,474 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:09:37,475 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:09:37,475 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:09:37,480 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:09:37,984 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:09:39,987 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:09:42,417 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:09:42,418 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:09:42,418 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:09:42,418 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:09:42,423 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:09:42,989 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:09:43,992 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:09:45,587 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:09:45,588 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:09:45,588 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:09:45,593 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:09:45,994 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:09:47,593 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:09:47,997 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:09:49,051 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:09:49,052 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:09:49,052 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:09:49,057 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:09:49,999 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:09:51,940 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:09:51,940 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:09:51,942 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:09:52,002 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:09:53,182 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:09:53,870 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:09:53,871 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:09:53,871 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:09:53,876 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:09:54,003 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:09:56,007 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:09:58,915 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:09:59,110 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:09:59,111 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:09:59,111 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:09:59,116 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:10:00,009 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:10:02,012 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:10:02,266 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 15:10:03,205 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:10:03,206 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:10:03,206 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:10:03,212 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:10:04,014 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:10:04,212 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:10:06,017 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:10:06,861 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:10:06,861 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:10:06,862 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:10:06,867 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:10:06,940 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:10:06,940 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:10:06,941 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:10:07,019 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:10:08,021 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:10:09,226 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:10:10,606 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:10:10,606 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:10:10,606 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:10:10,611 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:10:11,024 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:10:12,026 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:10:14,651 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:10:16,906 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:10:16,906 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:10:16,907 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:10:16,912 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:10:17,029 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:10:18,032 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:10:19,912 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:10:21,940 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:10:21,940 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:10:21,942 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:10:25,207 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:10:25,378 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:10:25,379 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:10:25,379 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:10:25,384 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:10:26,037 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:10:28,041 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:10:29,775 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:10:29,776 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:10:29,777 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:10:29,781 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:10:30,043 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:10:30,782 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:10:32,046 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:10:32,271 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 15:10:33,135 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:10:33,136 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:10:33,136 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:10:33,141 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:10:34,048 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:10:36,051 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:10:36,142 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:10:36,799 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:10:36,799 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:10:36,799 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:10:36,804 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:10:36,940 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:10:36,942 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:10:36,943 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:10:37,052 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:10:38,054 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:10:41,669 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:10:42,075 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:10:42,076 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:10:42,076 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:10:42,082 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:10:43,058 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:10:44,060 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:10:46,886 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:10:46,887 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:10:46,887 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:10:46,887 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:10:46,893 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:10:47,062 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:10:48,065 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:10:51,382 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:10:51,383 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:10:51,383 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:10:51,388 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:10:51,940 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:10:51,941 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:10:51,942 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:10:52,068 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:10:52,155 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:10:54,071 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:10:55,352 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:10:55,352 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:10:55,352 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:10:55,358 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:10:56,073 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:10:57,076 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:10:57,395 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:10:59,704 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:10:59,705 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:10:59,705 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:10:59,710 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:11:00,078 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:11:01,081 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:11:02,270 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 15:11:03,272 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:11:03,648 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:11:03,648 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:11:03,649 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:11:03,654 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:11:04,083 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:11:05,085 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:11:06,940 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:11:06,941 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:11:06,942 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:11:07,376 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:11:07,377 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:11:07,377 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:11:07,382 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:11:08,088 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:11:08,419 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:11:09,091 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:11:12,387 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:11:12,388 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:11:12,388 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:11:12,393 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:11:13,096 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:11:13,096 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:11:14,394 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:11:15,720 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:11:15,721 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:11:15,722 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:11:15,727 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:11:16,098 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:11:17,100 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:11:18,849 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:11:18,850 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:11:18,850 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:11:18,855 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:11:19,104 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:11:19,104 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:11:19,856 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:11:21,940 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:11:21,941 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:11:21,942 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:11:23,173 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:11:23,174 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:11:23,174 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:11:23,179 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:11:24,107 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:11:25,110 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:11:25,219 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:11:26,886 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:11:26,887 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:11:26,887 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:11:26,892 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:11:27,114 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:11:27,114 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:11:30,345 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:11:30,346 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:11:30,346 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:11:30,346 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:11:30,351 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:11:31,119 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:11:31,120 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:11:32,276 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 15:11:33,750 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:11:33,751 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:11:33,751 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:11:33,756 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:11:34,121 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:11:35,124 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:11:35,757 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:11:36,941 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:11:36,941 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:11:36,942 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:11:38,558 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:11:38,558 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:11:38,559 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:11:38,564 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:11:39,128 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:11:39,129 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:11:41,603 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:11:43,173 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:11:43,174 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:11:43,174 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:11:43,179 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:11:44,132 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:11:45,135 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:11:47,180 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:11:48,161 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:11:48,162 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:11:48,163 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:11:48,167 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:11:49,139 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:11:49,139 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:11:51,941 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:11:51,941 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:11:51,942 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:11:52,225 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:11:54,981 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:11:54,982 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:11:54,982 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:11:54,987 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:11:55,145 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:11:55,145 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:11:58,023 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:11:58,677 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:11:58,678 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:11:58,678 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:11:58,683 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:11:59,150 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:11:59,150 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:12:02,275 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 15:12:03,276 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:12:03,325 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:12:03,326 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:12:03,326 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:12:03,331 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:12:04,153 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:12:05,156 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:12:06,692 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:12:06,693 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:12:06,693 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:12:06,698 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:12:06,941 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:12:06,941 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:12:06,943 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:12:07,160 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:12:07,160 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:12:09,129 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:12:10,085 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:12:10,086 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:12:10,086 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:12:10,091 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:12:10,163 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:12:11,165 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:12:14,132 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:12:14,133 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:12:14,133 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:12:14,134 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:12:14,139 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:12:14,167 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:12:15,170 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:12:17,829 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:12:17,830 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:12:17,830 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:12:17,834 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:12:18,172 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:12:19,174 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:12:19,835 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:12:21,589 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:12:21,590 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:12:21,590 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:12:21,595 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:12:21,941 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:12:21,941 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:12:21,943 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:12:22,176 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:12:23,179 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:12:25,127 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:12:25,520 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:12:25,521 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:12:25,521 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:12:25,527 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:12:26,181 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:12:27,184 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:12:29,034 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:12:29,034 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:12:29,035 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:12:29,040 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:12:29,188 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:12:29,188 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:12:31,040 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:12:32,280 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 15:12:32,723 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:12:32,723 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:12:32,724 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:12:32,729 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:12:33,193 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:12:33,193 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:12:36,729 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:12:36,941 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:12:36,941 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:12:36,942 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:12:37,335 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:12:37,335 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:12:37,335 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:12:37,341 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:12:38,196 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:12:39,198 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:12:42,379 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:12:42,740 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:12:42,740 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:12:42,741 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:12:42,746 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:12:43,203 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:12:43,203 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:12:47,352 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:12:47,353 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:12:47,353 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:12:47,359 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:12:48,206 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:12:48,360 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:12:49,208 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:12:50,306 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:12:50,307 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:12:50,307 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:12:50,312 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:12:51,212 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:12:51,212 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:12:51,941 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:12:51,941 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:12:51,943 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:12:54,049 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:12:54,049 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:12:54,050 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:12:54,050 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:12:54,055 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:12:54,213 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:12:55,216 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:12:57,902 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:12:57,903 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:12:57,903 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:12:57,908 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:12:58,218 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:12:59,220 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:12:59,909 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:13:01,673 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:13:01,674 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:13:01,674 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:13:01,678 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:13:02,222 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:13:02,279 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 15:13:03,225 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:13:05,280 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:13:05,369 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:13:05,370 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:13:05,370 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:13:05,375 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:13:06,227 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:13:06,941 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:13:06,941 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:13:06,943 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:13:07,230 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:13:10,666 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:13:11,779 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:13:11,780 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:13:11,780 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:13:11,785 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:13:12,233 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:13:13,236 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:13:15,823 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:13:16,806 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:13:16,807 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:13:16,808 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:13:16,812 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:13:17,240 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:13:17,241 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:13:20,483 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:13:20,484 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:13:20,484 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:13:20,490 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:13:21,245 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:13:21,245 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:13:21,490 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:13:21,941 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:13:21,941 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:13:21,943 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:13:27,158 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:13:29,029 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:13:29,029 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:13:29,029 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:13:29,035 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:13:29,253 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:13:29,254 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:13:32,281 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 15:13:32,282 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:13:32,865 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:13:32,865 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:13:32,866 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:13:32,871 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:13:33,258 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:13:33,259 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:13:36,797 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:13:36,798 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:13:36,798 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:13:36,803 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:13:36,941 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:13:36,942 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:13:36,943 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:13:37,263 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:13:37,263 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:13:38,202 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:13:40,630 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:13:40,631 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:13:40,631 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:13:40,642 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:13:41,268 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:13:41,268 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:13:43,677 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:13:44,827 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:13:44,827 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:13:44,827 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:13:44,832 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:13:45,272 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:13:45,272 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:13:48,833 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:13:49,024 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:13:49,024 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:13:49,024 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:13:49,029 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:13:49,277 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:13:49,277 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:13:51,941 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:13:51,942 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:13:51,943 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:13:54,072 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:13:54,073 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:13:54,073 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:13:54,073 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:13:54,078 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:13:54,280 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:13:55,282 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:13:59,114 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:13:59,946 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:13:59,947 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:13:59,947 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:13:59,952 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:14:00,285 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:14:01,288 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:14:02,284 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 15:14:04,285 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:14:04,493 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:14:04,493 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:14:04,493 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:14:04,498 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:14:05,292 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:14:05,292 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:14:06,941 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:14:06,942 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:14:06,943 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:14:09,271 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:14:09,272 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:14:09,272 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:14:09,277 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:14:09,295 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:14:09,315 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:14:11,299 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:14:14,315 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:14:17,483 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:14:17,484 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:14:17,485 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:14:17,490 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:14:18,303 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:14:19,305 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:14:19,490 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:14:21,941 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:14:21,942 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:14:21,943 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:14:22,400 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:14:22,400 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:14:22,401 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:14:22,406 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:14:23,310 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:14:23,310 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:14:25,443 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:14:26,515 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:14:26,515 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:14:26,516 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:14:26,521 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:14:27,315 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:14:27,315 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:14:30,521 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:14:32,199 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:14:32,199 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:14:32,199 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:14:32,204 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:14:32,286 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 15:14:32,319 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:14:33,322 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:14:36,287 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:14:36,389 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:14:36,390 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:14:36,390 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:14:36,394 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:14:36,941 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:14:36,942 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:14:36,943 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:14:37,326 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:14:37,326 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:14:40,763 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:14:40,764 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:14:40,764 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:14:40,769 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:14:41,331 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:14:41,331 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:14:41,807 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:14:44,076 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:14:44,077 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:14:44,077 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:14:44,082 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:14:44,333 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:14:45,335 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:14:47,083 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:14:50,505 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:14:50,506 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:14:50,506 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:14:50,511 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:14:51,341 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:14:51,341 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:14:51,942 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:14:51,942 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:14:51,943 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:14:52,192 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:14:54,802 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:14:54,803 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:14:54,803 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:14:54,808 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:14:55,345 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:14:55,346 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:14:57,847 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:14:58,398 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:14:58,399 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:14:58,399 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:14:58,404 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:14:59,350 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:14:59,350 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:15:01,826 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:15:01,826 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:15:01,827 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:15:01,831 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:15:02,294 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 15:15:02,352 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:15:03,295 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:15:03,355 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:15:06,942 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:15:06,942 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:15:06,943 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:15:07,516 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:15:07,516 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:15:07,517 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:15:07,521 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:15:08,359 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:15:08,559 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:15:09,362 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:15:11,119 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:15:11,120 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:15:11,120 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:15:11,125 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:15:11,365 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:15:11,365 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:15:14,125 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:15:15,563 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:15:15,564 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:15:15,564 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:15:15,569 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:15:16,368 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:15:17,370 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:15:19,570 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:15:21,274 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:15:21,274 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:15:21,275 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:15:21,279 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:15:21,373 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:15:21,942 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:15:21,942 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:15:21,944 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:15:23,376 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:15:25,137 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:15:25,261 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:15:25,261 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:15:25,262 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:15:25,266 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:15:25,377 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:15:27,381 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:15:29,772 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:15:29,773 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:15:29,773 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:15:29,777 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:15:30,384 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:15:30,778 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:15:31,387 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:15:32,290 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 15:15:33,839 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:15:33,839 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:15:33,839 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:15:33,844 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:15:34,389 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:15:35,391 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:15:35,845 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:15:36,942 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:15:36,942 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:15:36,944 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:15:41,500 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:15:43,037 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:15:43,037 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:15:43,037 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:15:43,042 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:15:43,398 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:15:43,399 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:15:46,498 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:15:46,499 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:15:46,499 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:15:46,504 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:15:46,504 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:15:47,403 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:15:47,403 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:15:51,505 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:15:51,942 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:15:51,942 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:15:51,944 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:15:53,126 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:15:53,127 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:15:53,127 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:15:53,133 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:15:53,409 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:15:53,409 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:15:56,629 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:15:56,630 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:15:56,630 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:15:56,630 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:15:56,635 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:15:57,413 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:15:57,414 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:16:00,446 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:16:00,447 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:16:00,447 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:16:00,452 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:16:01,418 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:16:01,418 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:16:02,295 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 15:16:02,297 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:16:04,930 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:16:04,931 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:16:04,931 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:16:04,936 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:16:05,422 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:16:05,423 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:16:06,942 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:16:06,942 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:16:06,944 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:16:08,147 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:16:08,492 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:16:08,492 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:16:08,492 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:16:08,497 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:16:09,428 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:16:09,428 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:16:13,535 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:16:14,521 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:16:14,522 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:16:14,522 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:16:14,527 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:16:15,434 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:16:15,434 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:16:18,190 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:16:18,191 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:16:18,191 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:16:18,196 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:16:18,436 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:16:19,196 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:16:19,438 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:16:21,827 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:16:21,828 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:16:21,828 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:16:21,833 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:16:21,942 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:16:21,942 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:16:21,943 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:16:22,440 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:16:23,441 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:16:25,187 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:16:25,573 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:16:25,574 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:16:25,574 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:16:25,579 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:16:26,443 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:16:27,446 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:16:29,632 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:16:29,632 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:16:29,632 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:16:29,638 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:16:30,449 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:16:30,638 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:16:31,452 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:16:32,294 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 15:16:35,910 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:16:35,911 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:16:35,911 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:16:35,912 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:16:35,917 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:16:36,455 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:16:36,942 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:16:36,943 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:16:36,944 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:16:37,457 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:16:38,916 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:16:38,916 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:16:38,917 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:16:38,922 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:16:39,460 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:16:39,461 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:16:40,959 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:16:43,104 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:16:43,105 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:16:43,105 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:16:43,110 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:16:43,466 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:16:43,466 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:16:46,110 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:16:49,611 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:16:49,612 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:16:49,612 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:16:49,617 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:16:50,470 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:16:51,473 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:16:51,618 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:16:51,942 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:16:51,943 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:16:51,944 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:16:53,264 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:16:53,265 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:16:53,265 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:16:53,270 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:16:53,476 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:16:53,477 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:16:57,307 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:16:57,823 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:16:57,823 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:16:57,824 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:16:57,828 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:16:58,480 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:16:59,483 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:17:01,781 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:17:01,781 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:17:01,782 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:17:01,787 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:17:02,300 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 15:17:02,486 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:17:03,301 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:17:03,488 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:17:06,942 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:17:06,943 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:17:06,944 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:17:07,389 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:17:07,753 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:17:07,753 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:17:07,758 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:17:08,493 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:17:08,759 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:17:09,495 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:17:12,548 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:17:12,548 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:17:12,549 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:17:12,553 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:17:13,500 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:17:13,500 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:17:14,554 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:17:19,555 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:17:19,972 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:17:19,973 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:17:19,973 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:17:19,978 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:17:20,504 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:17:21,506 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:17:21,942 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:17:21,943 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:17:21,945 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:17:23,495 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:17:23,495 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:17:23,495 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:17:23,500 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:17:23,508 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:17:25,511 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:17:25,539 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:17:27,651 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:17:27,651 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:17:27,652 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:17:27,657 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:17:28,513 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:17:29,517 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:17:30,657 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:17:31,794 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:17:31,795 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:17:31,795 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:17:31,801 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:17:32,299 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 15:17:32,519 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:17:33,521 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:17:35,032 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:17:35,033 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:17:35,034 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:17:35,039 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:17:35,524 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:17:35,525 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:17:36,039 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:17:36,943 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:17:36,943 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:17:36,944 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:17:39,665 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:17:39,666 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:17:39,666 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:17:39,671 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:17:40,528 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:17:41,530 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:17:41,707 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:17:45,010 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:17:45,011 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:17:45,011 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:17:45,016 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:17:45,535 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:17:45,535 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:17:47,017 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:17:48,237 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:17:48,238 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:17:48,238 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:17:48,243 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:17:48,537 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:17:49,539 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:17:51,943 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:17:51,943 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:17:51,945 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:17:52,160 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:17:52,801 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:17:52,802 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:17:52,802 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:17:52,807 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:17:53,543 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:17:53,544 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:17:56,566 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:17:56,567 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:17:56,568 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:17:56,573 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:17:57,548 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:17:57,548 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:17:57,573 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:17:59,663 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:17:59,663 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:17:59,664 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:17:59,669 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:18:00,550 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:18:01,552 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:18:02,301 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 15:18:03,303 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:18:04,652 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:18:04,653 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:18:04,653 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:18:04,659 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:18:05,557 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:18:05,557 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:18:06,943 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:18:06,943 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:18:06,945 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:18:09,124 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:18:11,081 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:18:11,082 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:18:11,082 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:18:11,087 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:18:11,563 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:18:11,563 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:18:14,339 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:18:14,340 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:18:14,340 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:18:14,340 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:18:14,346 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:18:14,565 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:18:15,567 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:18:19,346 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:18:20,970 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:18:20,971 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:18:20,971 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:18:20,976 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:18:21,573 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:18:21,573 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:18:21,943 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:18:21,943 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:18:21,945 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:18:25,201 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:18:25,341 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:18:25,342 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:18:25,342 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:18:25,347 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:18:25,577 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:18:25,578 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:18:29,617 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:18:29,618 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:18:29,619 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:18:29,624 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:18:30,582 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:18:30,624 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:18:31,584 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:18:32,304 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 15:18:33,489 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:18:33,490 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:18:33,491 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:18:33,496 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:18:33,587 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:18:33,588 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:18:36,496 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:18:36,943 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:18:36,943 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:18:36,945 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:18:37,283 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:18:37,764 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:18:37,764 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:18:37,769 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:18:38,591 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:18:39,593 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:18:40,635 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:18:40,636 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:18:40,636 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:18:40,641 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:18:41,596 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:18:41,597 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:18:41,642 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:18:44,882 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:18:44,883 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:18:44,883 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:18:44,888 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:18:45,601 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:18:45,601 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:18:46,889 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:18:48,836 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:18:48,837 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:18:48,837 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:18:48,842 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:18:49,605 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:18:49,606 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:18:51,943 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:18:51,943 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:18:51,945 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:18:52,158 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:18:52,535 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:18:52,535 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:18:52,536 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:18:52,540 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:18:52,607 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:18:53,610 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:18:56,786 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:18:56,786 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:18:56,786 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:18:56,791 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:18:57,614 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:18:57,614 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:18:57,792 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:19:01,222 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:19:01,223 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:19:01,223 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:19:01,228 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:19:01,619 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:19:01,619 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:19:02,306 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 15:19:03,307 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:19:04,981 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:19:04,982 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:19:04,982 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:19:04,987 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:19:05,623 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:19:05,623 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:19:06,943 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:19:06,943 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:19:06,945 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:19:08,722 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:19:08,722 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:19:08,723 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:19:08,723 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:19:08,728 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:19:09,629 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:19:09,629 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:19:13,120 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:19:13,120 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:19:13,121 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:19:13,125 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:19:13,633 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:19:13,634 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:19:14,126 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:19:16,846 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:19:16,846 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:19:16,846 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:19:16,852 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:19:17,638 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:19:17,638 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:19:19,852 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:19:21,943 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:19:21,944 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:19:21,945 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:19:23,475 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:19:23,475 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:19:23,475 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:19:23,480 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:19:23,643 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:19:23,644 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:19:25,519 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:19:26,842 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:19:26,842 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:19:26,842 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:19:26,847 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:19:27,648 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:19:27,648 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:19:30,848 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:19:31,413 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:19:31,414 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:19:31,414 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:19:31,419 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:19:31,654 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:19:31,654 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:19:32,308 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 15:19:34,948 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:19:34,949 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:19:34,949 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:19:34,954 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:19:35,659 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:19:35,659 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:19:35,954 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:19:36,943 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:19:36,944 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:19:36,945 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:19:38,648 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:19:38,649 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:19:38,649 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:19:38,654 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:19:38,661 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:19:39,664 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:19:41,691 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:19:44,029 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:19:44,030 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:19:44,030 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:19:44,035 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:19:44,667 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:19:45,669 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:19:47,036 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:19:51,943 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:19:51,944 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:19:51,945 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:19:52,200 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:19:52,426 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:19:52,427 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:19:52,427 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:19:52,432 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:19:52,674 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:19:53,676 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:19:56,014 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:19:56,015 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:19:56,015 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:19:56,020 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:19:56,678 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:19:57,681 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:19:58,021 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:20:00,361 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:20:00,362 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:20:00,362 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:20:00,367 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:20:00,683 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:20:01,685 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:20:02,310 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 15:20:03,311 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:20:03,532 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:20:03,532 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:20:03,533 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:20:03,537 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:20:03,689 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:20:03,689 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:20:06,943 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:20:06,944 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:20:06,945 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:20:08,596 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:20:08,597 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:20:08,597 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:20:08,597 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:20:08,602 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:20:08,693 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:20:09,696 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:20:13,639 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:20:13,818 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:20:13,819 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:20:13,820 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:20:13,824 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:20:14,699 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:20:15,702 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:20:18,825 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:20:19,192 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:20:19,193 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:20:19,193 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:20:19,198 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:20:19,706 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:20:19,706 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:20:21,943 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:20:21,944 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:20:21,946 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:20:22,981 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:20:22,981 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:20:22,981 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:20:22,986 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:20:23,710 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:20:23,711 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:20:24,023 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:20:28,714 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:20:28,715 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:20:28,715 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:20:28,720 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:20:29,718 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:20:29,718 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:20:29,720 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:20:32,316 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 15:20:34,155 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:20:34,156 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:20:34,156 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:20:34,161 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:20:34,721 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:20:35,162 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:20:35,723 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:20:36,944 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:20:36,944 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:20:36,945 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:20:38,624 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:20:38,624 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:20:38,624 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:20:38,629 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:20:38,725 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:20:39,728 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:20:40,667 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:20:42,678 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:20:42,679 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:20:42,679 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:20:42,684 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:20:42,730 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:20:43,733 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:20:45,684 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:20:46,627 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:20:46,627 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:20:46,628 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:20:46,633 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:20:46,735 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:20:47,737 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:20:51,130 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:20:51,131 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:20:51,132 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:20:51,132 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:20:51,137 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:20:51,742 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:20:51,742 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:20:51,944 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:20:51,944 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:20:51,946 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:20:54,714 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:20:54,714 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:20:54,715 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:20:54,720 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:20:54,744 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:20:55,746 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:20:56,759 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:21:01,605 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:21:01,605 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:21:01,606 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:21:01,611 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:21:01,750 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:21:02,315 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 15:21:02,316 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:21:03,753 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:21:05,541 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:21:05,541 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:21:05,542 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:21:05,546 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:21:05,754 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:21:06,944 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:21:06,944 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:21:06,946 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:21:07,758 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:21:08,167 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:21:11,759 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:21:11,760 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:21:11,760 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:21:11,765 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:21:12,762 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:21:13,764 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:21:13,803 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:21:15,882 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:21:15,883 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:21:15,883 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:21:15,888 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:21:16,766 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:21:17,768 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:21:18,889 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:21:19,621 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:21:19,622 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:21:19,622 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:21:19,627 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:21:19,770 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:21:21,773 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:21:21,944 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:21:21,944 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:21:21,946 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:21:22,880 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:21:22,880 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:21:22,881 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:21:22,885 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:21:23,776 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:21:23,776 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:21:23,923 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:21:26,422 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:21:26,423 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:21:26,424 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:21:26,428 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:21:26,778 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:21:27,781 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:21:29,429 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:21:31,594 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:21:31,595 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:21:31,595 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:21:31,600 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:21:31,785 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:21:32,320 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 15:21:33,788 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:21:35,322 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:21:36,329 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:21:36,330 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:21:36,331 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:21:36,336 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:21:36,790 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:21:36,944 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:21:36,944 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:21:36,946 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:21:37,792 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:21:39,954 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:21:39,954 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:21:39,955 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:21:39,960 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:21:40,795 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:21:40,999 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:21:41,797 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:21:45,120 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:21:45,121 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:21:45,121 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:21:45,126 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:21:45,802 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:21:45,802 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:21:46,126 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:21:49,210 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:21:49,211 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:21:49,211 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:21:49,216 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:21:49,806 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:21:49,806 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:21:51,217 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:21:51,944 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:21:51,944 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:21:51,946 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:21:52,457 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:21:52,457 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:21:52,457 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:21:52,462 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:21:52,808 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:21:53,810 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:21:56,499 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:21:56,529 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:21:56,530 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:21:56,530 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:21:56,535 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:21:56,812 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:21:57,815 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:22:00,082 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:22:00,083 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:22:00,083 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:22:00,088 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:22:00,817 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:22:01,819 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:22:02,089 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:22:02,319 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 15:22:03,558 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:22:03,558 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:22:03,559 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:22:03,563 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:22:03,820 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:22:05,824 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:22:06,944 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:22:06,944 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:22:06,946 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:22:07,226 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:22:07,243 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:22:07,243 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:22:07,244 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:22:07,248 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:22:07,826 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:22:09,829 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:22:10,918 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:22:10,918 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:22:10,919 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:22:10,923 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:22:11,833 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:22:11,833 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:22:12,924 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:22:15,850 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:22:15,850 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:22:15,851 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:22:15,856 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:22:16,836 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:22:17,839 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:22:18,857 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:22:20,650 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:22:20,651 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:22:20,651 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:22:20,656 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:22:20,841 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:22:21,843 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:22:21,944 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:22:21,944 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:22:21,946 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:22:24,244 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:22:29,245 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:22:29,316 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:22:29,316 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:22:29,316 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:22:29,321 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:22:29,850 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:22:31,853 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:22:32,325 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 15:22:33,921 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:22:33,922 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:22:33,922 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:22:33,927 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:22:34,855 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:22:34,927 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:22:35,858 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:22:36,944 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:22:36,945 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:22:36,946 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:22:38,295 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:22:38,296 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:22:38,296 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:22:38,301 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:22:38,860 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:22:39,862 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:22:40,339 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:22:41,830 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:22:41,831 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:22:41,831 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:22:41,836 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:22:41,864 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:22:43,867 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:22:45,497 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:22:45,498 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:22:45,498 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:22:45,499 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:22:45,503 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:22:45,869 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:22:47,872 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:22:48,781 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:22:48,782 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:22:48,782 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:22:48,787 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:22:48,873 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:22:49,875 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:22:50,788 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:22:51,944 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:22:51,945 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:22:51,946 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:22:52,823 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:22:52,824 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:22:52,824 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:22:52,829 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:22:52,877 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:22:53,880 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:22:55,867 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:22:56,392 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:22:56,393 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:22:56,393 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:22:56,398 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:22:56,882 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:22:57,884 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:23:01,399 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:23:02,003 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:23:02,004 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:23:02,004 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:23:02,009 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:23:02,324 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 15:23:02,888 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:23:03,890 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:23:06,944 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:23:06,945 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:23:06,946 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:23:07,231 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:23:07,396 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:23:07,397 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:23:07,397 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:23:07,402 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:23:07,894 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:23:09,898 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:23:10,882 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:23:10,883 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:23:10,883 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:23:10,889 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:23:10,899 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:23:11,906 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:23:12,889 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:23:14,695 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:23:14,696 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:23:14,697 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:23:14,708 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:23:14,908 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:23:15,912 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:23:18,709 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:23:20,116 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:23:20,117 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:23:20,117 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:23:20,123 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:23:20,915 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:23:21,925 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:23:21,944 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:23:21,945 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:23:21,946 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:23:24,245 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:23:29,245 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:23:30,372 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:23:30,373 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:23:30,373 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:23:30,383 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:23:30,933 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:23:31,940 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:23:32,326 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 15:23:34,029 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:23:34,030 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:23:34,031 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:23:34,040 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:23:34,942 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:23:35,041 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:23:35,946 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:23:36,945 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:23:36,945 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:23:36,946 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:23:40,246 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:23:41,994 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:23:41,995 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:23:41,995 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:23:42,001 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:23:42,952 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:23:43,955 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:23:46,039 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:23:47,036 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:23:47,036 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:23:47,037 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:23:47,042 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:23:47,960 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:23:47,960 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:23:51,043 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:23:51,945 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:23:51,945 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:23:51,947 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:23:55,473 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:23:55,473 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:23:55,473 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:23:55,479 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:23:55,965 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:23:56,515 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:23:57,969 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:24:01,515 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:24:01,780 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:24:01,789 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:24:01,789 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:24:01,795 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:24:01,972 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:24:02,328 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 15:24:06,104 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:24:06,105 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:24:06,105 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:24:06,106 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:24:06,111 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:24:06,945 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:24:06,945 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:24:06,948 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:24:07,105 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:24:07,165 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:24:08,108 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:24:12,165 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:24:12,418 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:24:12,418 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:24:12,419 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:24:12,424 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:24:13,112 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:24:14,115 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:24:15,676 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:24:15,677 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:24:15,677 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:24:15,684 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:24:16,116 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:24:17,684 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:24:18,122 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:24:20,973 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:24:20,974 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:24:20,974 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:24:20,980 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:24:21,123 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:24:21,945 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:24:21,945 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:24:21,947 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:24:22,126 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:24:23,234 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:24:26,954 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:24:26,954 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:24:26,955 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:24:26,961 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:24:27,130 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:24:28,133 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:24:28,999 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:24:32,331 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 15:24:34,332 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:24:35,439 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:24:35,440 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:24:35,440 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:24:35,446 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:24:36,140 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:24:37,247 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:24:37,248 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:24:37,248 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:24:38,143 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:24:39,495 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:24:40,510 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:24:40,511 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:24:40,511 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:24:40,517 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:24:41,146 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:24:42,149 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:24:44,555 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:24:45,653 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:24:45,654 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:24:45,654 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:24:45,660 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:24:46,152 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:24:48,155 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:24:49,661 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:24:52,169 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:24:52,170 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:24:52,170 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:24:52,175 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:24:52,247 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:24:52,247 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:24:52,249 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:24:53,159 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:24:54,161 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:24:55,494 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:24:55,715 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:24:55,716 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:24:55,716 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:24:55,723 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:24:56,163 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:24:58,166 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:24:59,427 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:24:59,428 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:24:59,429 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:24:59,434 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:25:00,170 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:25:00,170 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:25:01,434 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:25:02,336 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 15:25:03,231 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:25:03,231 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:25:03,232 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:25:03,237 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:25:04,174 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:25:04,175 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:25:07,237 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:25:07,247 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:25:07,247 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:25:07,249 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:25:08,339 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:25:08,340 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:25:08,340 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:25:08,346 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:25:09,179 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:25:10,182 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:25:12,383 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:25:12,863 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:25:12,863 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:25:12,864 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:25:12,869 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:25:13,184 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:25:14,186 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:25:16,306 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:25:16,307 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:25:16,307 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:25:16,313 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:25:17,188 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:25:18,191 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:25:18,314 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:25:22,127 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:25:22,127 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:25:22,128 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:25:22,133 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:25:22,193 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:25:22,247 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:25:22,247 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:25:22,249 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:25:23,442 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:25:24,196 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:25:27,926 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:25:27,926 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:25:27,927 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:25:27,932 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:25:28,199 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:25:28,971 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:25:30,203 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:25:32,057 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:25:32,058 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:25:32,058 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:25:32,063 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:25:32,205 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:25:32,335 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 15:25:34,208 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:25:34,336 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:25:36,373 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:25:36,373 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:25:36,374 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:25:36,379 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:25:37,210 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:25:37,247 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:25:37,248 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:25:37,248 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:25:38,212 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:25:39,834 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:25:39,835 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:25:39,835 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:25:39,835 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:25:39,840 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:25:40,214 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:25:42,217 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:25:43,252 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:25:43,253 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:25:43,253 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:25:43,259 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:25:44,221 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:25:44,221 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:25:45,259 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:25:48,291 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:25:48,292 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:25:48,292 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:25:48,297 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:25:49,224 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:25:50,226 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:25:50,298 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:25:52,247 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:25:52,248 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:25:52,248 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:25:52,651 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:25:52,651 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:25:52,651 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:25:52,656 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:25:53,228 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:25:54,231 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:25:55,695 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:25:57,196 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:25:57,197 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:25:57,197 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:25:57,202 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:25:57,233 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:25:58,236 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:26:01,203 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:26:01,736 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:26:01,737 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:26:01,737 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:26:01,742 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:26:02,238 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:26:02,340 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 15:26:04,241 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:26:06,342 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:26:07,247 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:26:07,248 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:26:07,249 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:26:08,295 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:26:08,295 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:26:08,295 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:26:08,300 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:26:09,246 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:26:10,248 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:26:12,339 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:26:15,340 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:26:15,341 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:26:15,341 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:26:15,346 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:26:16,254 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:26:16,255 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:26:17,347 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:26:18,624 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:26:18,625 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:26:18,626 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:26:18,631 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:26:19,256 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:26:20,259 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:26:22,247 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:26:22,248 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:26:22,248 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:26:22,471 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:26:24,357 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:26:24,358 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:26:24,358 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:26:24,363 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:26:25,262 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:26:26,265 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:26:28,183 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:26:28,184 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:26:28,184 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:26:28,184 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:26:28,189 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:26:28,266 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:26:30,271 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:26:32,339 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 15:26:32,961 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:26:32,961 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:26:32,962 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:26:32,966 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:26:33,273 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:26:33,967 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:26:34,275 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:26:36,847 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:26:36,847 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:26:36,848 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:26:36,853 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:26:37,247 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:26:37,248 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:26:37,248 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:26:37,277 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:26:38,280 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:26:39,490 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:26:40,982 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:26:40,982 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:26:40,983 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:26:40,987 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:26:41,282 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:26:42,285 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:26:45,027 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:26:46,143 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:26:46,144 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:26:46,144 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:26:46,149 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:26:46,288 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:26:48,291 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:26:50,150 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:26:50,270 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:26:50,271 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:26:50,271 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:26:50,276 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:26:50,293 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:26:52,248 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:26:52,248 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:26:52,248 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:26:52,296 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:26:54,374 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:26:54,374 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:26:54,374 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:26:54,380 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:26:55,298 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:26:55,419 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:26:56,301 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:27:00,419 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:27:02,307 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:27:02,308 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:27:02,308 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:27:02,313 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:27:02,341 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 15:27:03,305 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:27:04,309 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:27:06,343 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:27:07,248 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:27:07,248 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:27:07,248 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:27:07,692 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:27:07,692 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:27:07,693 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:27:07,697 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:27:08,312 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:27:10,316 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:27:11,330 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:27:11,331 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:27:11,331 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:27:11,336 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:27:12,319 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:27:12,319 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:27:12,336 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:27:15,767 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:27:15,768 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:27:15,768 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:27:15,773 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:27:16,322 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:27:17,773 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:27:18,325 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:27:22,248 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:27:22,248 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:27:22,248 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:27:23,472 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:27:23,897 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:27:23,897 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:27:23,898 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:27:23,902 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:27:24,329 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:27:26,332 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:27:27,550 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:27:27,551 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:27:27,551 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:27:27,556 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:27:28,336 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:27:28,336 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:27:28,557 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:27:31,279 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:27:31,280 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:27:31,280 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:27:31,285 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:27:31,339 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:27:32,344 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:27:32,344 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 15:27:34,346 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:27:37,248 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:27:37,248 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:27:37,248 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:27:39,524 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:27:39,895 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:27:39,896 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:27:39,896 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:27:39,901 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:27:40,349 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:27:42,353 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:27:44,690 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:27:44,691 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:27:44,691 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:27:44,692 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:27:44,696 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:27:45,355 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:27:46,358 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:27:49,697 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:27:52,248 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:27:52,248 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:27:52,250 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:27:52,445 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:27:52,520 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:27:52,520 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:27:52,525 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:27:53,362 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:27:54,365 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:27:55,526 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:27:56,350 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:27:56,350 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:27:56,351 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:27:56,356 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:27:56,367 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:27:58,370 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:28:00,150 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:28:00,151 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:28:00,151 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:28:00,156 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:28:00,371 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:28:01,157 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:28:02,346 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 15:28:02,374 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:28:04,042 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:28:04,043 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:28:04,043 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:28:04,048 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:28:04,375 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:28:06,378 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:28:07,049 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:28:07,248 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:28:07,248 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:28:07,250 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:28:09,434 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:28:09,435 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:28:09,435 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:28:09,440 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:28:10,384 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:28:10,384 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:28:12,478 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:28:13,186 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:28:13,187 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:28:13,187 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:28:13,192 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:28:13,386 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:28:14,388 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:28:16,960 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:28:16,961 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:28:16,961 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:28:16,966 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:28:17,390 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:28:17,966 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:28:18,393 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:28:20,417 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:28:20,418 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:28:20,418 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:28:20,423 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:28:21,395 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:28:22,248 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:28:22,249 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:28:22,249 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:28:22,397 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:28:23,528 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:28:25,709 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:28:25,709 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:28:25,709 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:28:25,714 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:28:26,400 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:28:28,403 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:28:28,750 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:28:30,821 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:28:30,821 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:28:30,822 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:28:30,827 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:28:31,406 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:28:32,352 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 15:28:32,409 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:28:34,353 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:28:35,378 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:28:35,379 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:28:35,379 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:28:35,385 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:28:35,411 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:28:36,413 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:28:37,248 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:28:37,249 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:28:37,249 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:28:39,453 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:28:39,454 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:28:39,454 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:28:39,454 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:28:39,459 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:28:40,418 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:28:40,418 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:28:44,494 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:28:47,802 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:28:47,803 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:28:47,803 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:28:47,808 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:28:48,423 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:28:49,809 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:28:50,426 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:28:51,150 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:28:51,150 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:28:51,151 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:28:51,156 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:28:51,427 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:28:52,248 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:28:52,249 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:28:52,250 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:28:52,429 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:28:54,848 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:28:58,158 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:28:58,159 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:28:58,159 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:28:58,164 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:28:58,433 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:29:00,202 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:29:00,436 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:29:01,631 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:29:01,632 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:29:01,632 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:29:01,637 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:29:02,350 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 15:29:02,439 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:29:02,440 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:29:05,352 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:29:06,868 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:29:06,869 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:29:06,869 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:29:06,874 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:29:07,248 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:29:07,249 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:29:07,250 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:29:07,442 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:29:08,446 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:29:10,493 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:29:11,446 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:29:11,447 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:29:11,447 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:29:11,452 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:29:12,451 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:29:12,451 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:29:15,007 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:29:15,008 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:29:15,008 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:29:15,013 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:29:15,452 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:29:16,014 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:29:16,455 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:29:18,639 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:29:18,640 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:29:18,641 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:29:18,646 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:29:19,457 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:29:20,459 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:29:21,647 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:29:22,248 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:29:22,249 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:29:22,250 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:29:23,525 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:29:23,526 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:29:23,526 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:29:23,531 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:29:24,464 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:29:24,464 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:29:27,566 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:29:32,356 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 15:29:33,063 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:29:33,064 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:29:33,064 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:29:33,064 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:29:33,069 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:29:33,471 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:29:34,473 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:29:37,229 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:29:37,230 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:29:37,230 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:29:37,235 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:29:37,248 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:29:37,249 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:29:37,250 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:29:37,475 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:29:38,478 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:29:38,546 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:29:41,011 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:29:41,012 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:29:41,012 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:29:41,017 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:29:41,480 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:29:42,483 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:29:44,054 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:29:44,820 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:29:44,821 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:29:44,821 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:29:44,826 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:29:45,485 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:29:46,487 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:29:49,827 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:29:50,926 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:29:50,927 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:29:50,927 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:29:50,932 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:29:51,490 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:29:52,248 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:29:52,249 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:29:52,249 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:29:52,493 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:29:55,478 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:29:55,642 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:29:55,643 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:29:55,643 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:29:55,648 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:29:56,497 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:29:56,497 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:30:00,686 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:30:01,972 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:30:01,973 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:30:01,973 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:30:01,978 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:30:02,355 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 15:30:02,501 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:30:04,504 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:30:06,356 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:30:07,249 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:30:07,249 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:30:07,249 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:30:07,633 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:30:07,634 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:30:07,634 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:30:07,639 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:30:08,509 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:30:08,509 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:30:11,674 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:30:12,928 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:30:12,928 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:30:12,929 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:30:12,934 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:30:13,512 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:30:14,515 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:30:16,934 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:30:17,196 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:30:17,197 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:30:17,197 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:30:17,203 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:30:17,517 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:30:18,519 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:30:21,479 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:30:21,480 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:30:21,480 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:30:21,485 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:30:21,521 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:30:22,249 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:30:22,249 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:30:22,250 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:30:22,524 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:30:22,563 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:30:24,568 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:30:24,569 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:30:24,569 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:30:24,574 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:30:25,526 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:30:26,529 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:30:27,610 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:30:30,942 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:30:30,942 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:30:30,943 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:30:30,947 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:30:31,533 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:30:32,357 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 15:30:32,536 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:30:33,358 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:30:36,576 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:30:36,576 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:30:36,577 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:30:36,582 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:30:37,249 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:30:37,249 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:30:37,249 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:30:37,539 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:30:38,540 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:30:39,171 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:30:41,351 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:30:41,355 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:30:41,356 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:30:41,361 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:30:41,541 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:30:42,544 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:30:44,394 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:30:47,339 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:30:47,340 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:30:47,340 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:30:47,345 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:30:47,547 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:30:48,550 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:30:50,346 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:30:51,096 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:30:51,097 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:30:51,097 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:30:51,102 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:30:51,551 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:30:52,249 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:30:52,249 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:30:52,251 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:30:52,554 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:30:55,480 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:30:56,132 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:30:56,133 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:30:56,133 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:30:56,138 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:30:56,557 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:30:58,560 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:31:01,174 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:31:02,359 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 15:31:04,598 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:31:04,599 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:31:04,600 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:31:04,605 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:31:05,564 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:31:06,567 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:31:06,605 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:31:07,249 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:31:07,249 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:31:07,251 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:31:08,691 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:31:08,692 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:31:08,692 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:31:08,698 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:31:09,569 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:31:10,572 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:31:11,734 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:31:16,475 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:31:16,476 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:31:16,476 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:31:16,482 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:31:16,576 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:31:17,482 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:31:18,579 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:31:22,249 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:31:22,249 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:31:22,251 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:31:23,482 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:31:27,228 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:31:27,229 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:31:27,229 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:31:27,234 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:31:27,585 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:31:28,587 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:31:29,270 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:31:32,361 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 15:31:32,982 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:31:32,983 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:31:32,983 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:31:32,988 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:31:33,592 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:31:34,595 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:31:34,989 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:31:37,249 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:31:37,250 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:31:37,251 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:31:39,267 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:31:39,267 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:31:39,267 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:31:39,272 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:31:39,601 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:31:40,310 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:31:40,603 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:31:42,831 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:31:42,832 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:31:42,832 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:31:42,837 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:31:43,607 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:31:44,610 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:31:45,838 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:31:49,675 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:31:49,676 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:31:49,676 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:31:49,681 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:31:50,618 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:31:50,618 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:31:51,681 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:31:52,249 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:31:52,250 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:31:52,251 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:31:53,310 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:31:53,311 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:31:53,311 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:31:53,316 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:31:53,621 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:31:54,624 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:31:56,900 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:31:56,901 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:31:56,902 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:31:56,902 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:31:56,907 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:31:57,627 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:31:58,630 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:32:01,908 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:32:02,367 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 15:32:03,986 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:32:03,987 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:32:03,987 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:32:03,992 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:32:04,636 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:32:06,640 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:32:06,993 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:32:07,249 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:32:07,250 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:32:07,251 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:32:07,703 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:32:08,017 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:32:08,017 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:32:08,022 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:32:08,643 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:32:10,646 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:32:11,697 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:32:11,698 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:32:11,698 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:32:11,703 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:32:12,650 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:32:12,650 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:32:12,704 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:32:16,732 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:32:16,733 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:32:16,733 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:32:16,738 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:32:17,655 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:32:17,739 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:32:18,658 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:32:20,633 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:32:20,634 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:32:20,634 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:32:20,639 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:32:20,659 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:32:22,249 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:32:22,250 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:32:22,250 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:32:22,664 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:32:23,492 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:32:26,228 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:32:26,229 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:32:26,229 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:32:26,234 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:32:26,667 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:32:28,672 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:32:29,270 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:32:29,801 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:32:29,802 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:32:29,803 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:32:29,808 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:32:30,677 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:32:30,677 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:32:32,365 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 15:32:33,952 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:32:33,952 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:32:33,953 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:32:33,957 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:32:34,682 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:32:34,682 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:32:34,958 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:32:37,249 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:32:37,250 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:32:37,250 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:32:38,120 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:32:38,121 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:32:38,121 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:32:38,126 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:32:38,685 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:32:40,162 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:32:40,689 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:32:42,689 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:32:42,690 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:32:42,690 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:32:42,696 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:32:43,693 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:32:44,696 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:32:45,696 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:32:46,809 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:32:46,810 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:32:46,810 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:32:46,815 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:32:47,699 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:32:48,702 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:32:50,816 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:32:52,250 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:32:52,250 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:32:52,251 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:32:53,566 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:32:53,566 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:32:53,567 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:32:53,572 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:32:53,707 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:32:54,710 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:32:56,610 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:32:57,743 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:32:57,744 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:32:57,744 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:32:57,749 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:32:58,715 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:32:58,715 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:33:01,750 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:33:02,373 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 15:33:03,353 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:33:03,354 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:33:03,354 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:33:03,359 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:33:03,721 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:33:04,724 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:33:06,986 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:33:06,987 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:33:06,987 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:33:06,987 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:33:06,992 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:33:07,250 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:33:07,250 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:33:07,251 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:33:07,727 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:33:08,729 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:33:11,911 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:33:11,911 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:33:11,911 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:33:11,916 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:33:12,736 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:33:12,736 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:33:12,954 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:33:17,954 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:33:18,178 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:33:18,179 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:33:18,179 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:33:18,184 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:33:18,742 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:33:20,747 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:33:21,544 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:33:21,545 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:33:21,546 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:33:21,551 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:33:21,748 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:33:22,250 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:33:22,250 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:33:22,250 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:33:22,750 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:33:23,548 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:33:27,643 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:33:27,644 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:33:27,644 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:33:27,649 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:33:27,756 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:33:28,686 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:33:28,756 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:33:31,778 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:33:31,779 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:33:31,780 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:33:31,784 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:33:32,370 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 15:33:32,764 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:33:32,764 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:33:34,371 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:33:35,482 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:33:35,483 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:33:35,484 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:33:35,489 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:33:35,767 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:33:36,770 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:33:37,250 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:33:37,250 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:33:37,252 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:33:39,102 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:33:39,102 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:33:39,102 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:33:39,108 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:33:39,773 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:33:40,146 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:33:40,776 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:33:42,380 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:33:42,381 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:33:42,381 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:33:42,386 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:33:42,777 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:33:44,782 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:33:45,387 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:33:46,450 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:33:46,451 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:33:46,451 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:33:46,456 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:33:46,783 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:33:48,787 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:33:50,400 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:33:50,401 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:33:50,401 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:33:50,401 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:33:50,406 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:33:50,788 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:33:52,250 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:33:52,250 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:33:52,251 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:33:52,793 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:33:54,471 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:33:54,472 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:33:54,472 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:33:54,477 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:33:54,794 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:33:55,514 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:33:56,799 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:33:58,193 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:33:58,194 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:33:58,194 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:33:58,200 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:33:58,800 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:34:00,805 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:34:01,200 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:34:02,294 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:34:02,294 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:34:02,295 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:34:02,300 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:34:02,375 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 15:34:02,807 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:34:04,810 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:34:06,377 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:34:07,250 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:34:07,250 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:34:07,252 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:34:10,689 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:34:10,690 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:34:10,690 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:34:10,695 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:34:10,818 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:34:11,734 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:34:12,822 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:34:15,048 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:34:15,049 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:34:15,049 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:34:15,054 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:34:15,825 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:34:16,828 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:34:17,054 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:34:19,969 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:34:19,970 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:34:19,970 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:34:19,975 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:34:20,833 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:34:20,834 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:34:22,250 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:34:22,250 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:34:22,252 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:34:22,598 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:34:25,096 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:34:25,097 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:34:25,097 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:34:25,102 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:34:25,839 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:34:26,841 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:34:28,138 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:34:29,289 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:34:29,290 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:34:29,290 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:34:29,295 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:34:29,844 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:34:30,848 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:34:32,374 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 15:34:33,148 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:34:33,148 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:34:33,149 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:34:33,149 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:34:33,154 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:34:33,850 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:34:34,853 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:34:36,976 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:34:36,977 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:34:36,977 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:34:36,982 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:34:37,250 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:34:37,251 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:34:37,252 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:34:37,856 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:34:38,472 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:34:38,859 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:34:42,203 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:34:42,204 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:34:42,204 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:34:42,209 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:34:42,863 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:34:44,246 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:34:44,868 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:34:46,162 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:34:46,163 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:34:46,163 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:34:46,168 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:34:46,869 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:34:48,874 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:34:49,813 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:34:49,814 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:34:49,814 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:34:49,814 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:34:49,820 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:34:49,874 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:34:50,877 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:34:52,250 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:34:52,251 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:34:52,252 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:34:55,552 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:34:58,002 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:34:58,002 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:34:58,003 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:34:58,008 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:34:58,887 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:34:58,888 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:35:01,046 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:35:02,376 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 15:35:03,338 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:35:03,339 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:35:03,339 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:35:03,344 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:35:03,893 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:35:04,896 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:35:06,345 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:35:07,250 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:35:07,251 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:35:07,252 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:35:08,087 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:35:08,088 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:35:08,088 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:35:08,094 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:35:08,903 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:35:08,903 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:35:12,130 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:35:14,552 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:35:14,553 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:35:14,553 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:35:14,559 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:35:14,909 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:35:16,914 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:35:17,560 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:35:18,011 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:35:18,012 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:35:18,012 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:35:18,017 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:35:18,917 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:35:18,917 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:35:22,250 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:35:22,251 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:35:22,252 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:35:23,464 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:35:26,476 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:35:26,477 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:35:26,477 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:35:26,482 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:35:26,926 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:35:28,518 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:35:28,930 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:35:31,251 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:35:31,252 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:35:31,252 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:35:31,257 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:35:31,934 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:35:32,382 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 15:35:32,936 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:35:34,384 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:35:36,438 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:35:36,439 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:35:36,439 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:35:36,445 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:35:36,940 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:35:37,250 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:35:37,251 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:35:37,252 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:35:38,943 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:35:40,063 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:35:40,659 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:35:40,659 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:35:40,660 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:35:40,665 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:35:40,944 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:35:42,948 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:35:45,702 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:35:45,703 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:35:45,704 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:35:45,704 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:35:45,710 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:35:45,950 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:35:46,952 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:35:49,693 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:35:49,694 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:35:49,694 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:35:49,699 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:35:49,954 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:35:50,955 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:35:51,700 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:35:52,250 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:35:52,251 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:35:52,252 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:35:54,465 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:35:54,466 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:35:54,466 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:35:54,472 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:35:54,958 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:35:56,961 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:35:57,510 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:35:58,802 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:35:58,803 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:35:58,803 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:35:58,808 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:35:58,962 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:36:00,965 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:36:02,381 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 15:36:03,361 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:36:03,362 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:36:03,363 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:36:03,363 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:36:03,368 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:36:03,967 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:36:04,970 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:36:07,251 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:36:07,251 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:36:07,252 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:36:07,679 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:36:07,680 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:36:07,680 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:36:07,685 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:36:07,972 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:36:08,722 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:36:08,975 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:36:12,839 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:36:12,840 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:36:12,841 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:36:12,846 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:36:12,978 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:36:13,846 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:36:14,981 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:36:16,461 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:36:16,462 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:36:16,462 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:36:16,468 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:36:16,983 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:36:18,986 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:36:19,468 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:36:19,781 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:36:19,782 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:36:19,782 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:36:19,787 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:36:19,986 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:36:20,990 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:36:22,251 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:36:22,251 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:36:22,252 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:36:23,933 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:36:23,934 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:36:23,934 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:36:23,940 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:36:23,991 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:36:24,974 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:36:24,994 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:36:29,974 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:36:32,387 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 15:36:32,543 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:36:32,544 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:36:32,544 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:36:32,549 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:36:33,001 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:36:35,004 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:36:35,550 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:36:36,839 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:36:36,840 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:36:36,841 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:36:36,846 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:36:37,005 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:36:37,251 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:36:37,251 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:36:37,252 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:36:39,008 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:36:40,493 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:36:40,494 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:36:40,494 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:36:40,499 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:36:41,010 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:36:41,534 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:36:43,013 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:36:46,193 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:36:46,194 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:36:46,194 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:36:46,199 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:36:47,018 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:36:47,018 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:36:47,200 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:36:50,443 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:36:50,443 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:36:50,444 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:36:50,449 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:36:51,020 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:36:52,251 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:36:52,251 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:36:52,252 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:36:52,523 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:36:53,024 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:36:54,083 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:36:54,084 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:36:54,084 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:36:54,089 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:36:55,027 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:36:55,027 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:36:57,491 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:36:57,492 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:36:57,492 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:36:57,497 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:36:58,029 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:36:58,497 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:36:59,032 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:37:02,386 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 15:37:02,615 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:37:02,615 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:37:02,615 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:37:02,620 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:37:03,034 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:37:03,621 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:37:05,038 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:37:07,251 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:37:07,251 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:37:07,253 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:37:09,508 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:37:11,139 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:37:11,139 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:37:11,139 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:37:11,144 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:37:12,044 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:37:13,046 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:37:15,006 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:37:15,007 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:37:15,007 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:37:15,007 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:37:15,012 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:37:15,047 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:37:17,051 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:37:18,910 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:37:18,911 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:37:18,911 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:37:18,916 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:37:19,052 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:37:20,916 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:37:21,055 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:37:22,251 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:37:22,251 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:37:22,253 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:37:22,655 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:37:22,656 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:37:22,656 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:37:22,661 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:37:23,056 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:37:25,059 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:37:26,698 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:37:28,606 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:37:28,607 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:37:28,607 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:37:28,612 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:37:29,062 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:37:31,067 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:37:32,391 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 15:37:32,393 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:37:32,549 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:37:32,549 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:37:32,550 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:37:32,555 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:37:33,068 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:37:35,071 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:37:36,574 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:37:36,575 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:37:36,575 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:37:36,580 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:37:37,072 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:37:37,251 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:37:37,252 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:37:37,253 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:37:37,477 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:37:39,075 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:37:40,522 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:37:40,523 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:37:40,523 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:37:40,528 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:37:41,077 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:37:42,567 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:37:43,081 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:37:45,728 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:37:45,728 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:37:45,729 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:37:45,734 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:37:46,082 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:37:47,085 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:37:47,734 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:37:49,654 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:37:49,655 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:37:49,655 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:37:49,660 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:37:50,087 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:37:51,087 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:37:52,251 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:37:52,252 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:37:52,253 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:37:53,549 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:37:56,102 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:37:56,103 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:37:56,103 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:37:56,108 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:37:57,093 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:37:57,093 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:37:59,147 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:38:00,048 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:38:00,049 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:38:00,049 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:38:00,054 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:38:00,095 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:38:01,098 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:38:02,390 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 15:38:03,765 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:38:03,766 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:38:03,766 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:38:03,771 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:38:04,099 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:38:04,771 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:38:05,102 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:38:07,251 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:38:07,252 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:38:07,253 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:38:07,677 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:38:07,678 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:38:07,678 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:38:07,683 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:38:08,104 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:38:09,107 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:38:10,719 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:38:14,256 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:38:14,257 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:38:14,258 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:38:14,263 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:38:15,114 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:38:15,114 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:38:16,264 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:38:18,581 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:38:18,582 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:38:18,582 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:38:18,588 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:38:19,116 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:38:21,120 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:38:21,589 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:38:22,251 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:38:22,252 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:38:22,253 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:38:22,766 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:38:22,767 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:38:22,767 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:38:22,772 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:38:23,121 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:38:25,124 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:38:26,729 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:38:26,730 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:38:26,730 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:38:26,730 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:38:26,735 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:38:27,125 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:38:29,129 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:38:29,971 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:38:29,972 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:38:29,972 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:38:29,977 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:38:30,129 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:38:31,133 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:38:31,978 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:38:32,392 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 15:38:34,395 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:38:34,396 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:38:34,396 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:38:34,401 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:38:35,138 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:38:35,138 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:38:37,251 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:38:37,252 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:38:37,253 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:38:37,455 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:38:39,657 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:38:39,657 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:38:39,658 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:38:39,663 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:38:40,141 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:38:41,144 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:38:42,699 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:38:43,297 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:38:43,298 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:38:43,298 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:38:43,303 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:38:44,146 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:38:45,149 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:38:48,304 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:38:48,347 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:38:48,348 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:38:48,348 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:38:48,354 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:38:49,153 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:38:49,153 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:38:52,252 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:38:52,252 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:38:52,253 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:38:52,603 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:38:52,604 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:38:52,604 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:38:52,609 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:38:53,156 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:38:53,647 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:38:55,159 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:38:58,647 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:38:59,724 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:38:59,724 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:38:59,725 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:38:59,730 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:39:00,162 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:39:01,164 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:39:02,395 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 15:39:04,245 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:39:04,246 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:39:04,246 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:39:04,246 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:39:04,251 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:39:05,167 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:39:05,167 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:39:07,252 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:39:07,252 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:39:07,253 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:39:09,548 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:39:09,595 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:39:09,596 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:39:09,596 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:39:09,601 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:39:10,171 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:39:11,174 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:39:12,911 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:39:12,912 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:39:12,913 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:39:12,917 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:39:13,175 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:39:14,918 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:39:15,179 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:39:16,258 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:39:16,259 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:39:16,259 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:39:16,264 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:39:17,182 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:39:17,182 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:39:19,771 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:39:19,771 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:39:19,772 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:39:19,777 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:39:20,184 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:39:20,778 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:39:21,186 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:39:22,252 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:39:22,252 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:39:22,252 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:39:26,515 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:39:28,831 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:39:28,832 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:39:28,832 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:39:28,837 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:39:29,192 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:39:31,196 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:39:31,875 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:39:32,397 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 15:39:33,833 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:39:33,833 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:39:33,834 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:39:33,839 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:39:34,198 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:39:35,200 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:39:37,252 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:39:37,252 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:39:37,254 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:39:37,532 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:39:42,221 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:39:42,221 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:39:42,221 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:39:42,227 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:39:43,207 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:39:43,208 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:39:43,263 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:39:46,550 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:39:46,551 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:39:46,551 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:39:46,556 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:39:47,210 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:39:48,557 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:39:49,213 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:39:51,365 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:39:51,366 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:39:51,366 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:39:51,371 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:39:52,215 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:39:52,252 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:39:52,252 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:39:52,253 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:39:53,217 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:39:54,502 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:39:55,271 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:39:55,272 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:39:55,272 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:39:55,277 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:39:56,219 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:39:57,222 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:39:59,991 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:39:59,992 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:39:59,992 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:39:59,993 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:39:59,997 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:40:00,224 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:40:01,226 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:40:02,403 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 15:40:04,281 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:40:04,282 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:40:04,282 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:40:04,287 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:40:05,228 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:40:05,229 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:40:05,287 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:40:07,252 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:40:07,252 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:40:07,254 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:40:07,629 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:40:08,470 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:40:08,470 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:40:08,475 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:40:09,234 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:40:09,234 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:40:10,476 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:40:12,504 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:40:12,505 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:40:12,505 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:40:12,510 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:40:13,239 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:40:13,239 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:40:15,511 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:40:18,055 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:40:18,056 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:40:18,056 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:40:18,061 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:40:18,242 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:40:19,244 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:40:21,062 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:40:21,995 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:40:21,995 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:40:21,996 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:40:22,001 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:40:22,246 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:40:22,252 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:40:22,252 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:40:22,254 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:40:23,248 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:40:26,533 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:40:28,748 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:40:28,749 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:40:28,749 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:40:28,754 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:40:29,252 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:40:31,257 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:40:31,791 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:40:32,401 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 15:40:37,252 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:40:37,252 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:40:37,254 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:40:37,493 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:40:37,746 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:40:37,747 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:40:37,747 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:40:37,752 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:40:38,261 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:40:39,264 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:40:42,791 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:40:47,791 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:40:51,312 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:40:51,313 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:40:51,313 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:40:51,318 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:40:52,252 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:40:52,253 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:40:52,254 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:40:52,272 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:40:53,274 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:40:53,527 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:40:55,467 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:40:55,467 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:40:55,467 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:40:55,473 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:40:56,276 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:40:57,279 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:40:59,222 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:40:59,223 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:40:59,223 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:40:59,223 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:40:59,228 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:40:59,280 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:41:01,283 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:41:02,406 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 15:41:03,364 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:41:03,365 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:41:03,365 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:41:03,370 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:41:04,285 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:41:04,371 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:41:05,288 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:41:07,252 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:41:07,253 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:41:07,254 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:41:09,462 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:41:10,223 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:41:10,224 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:41:10,224 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:41:10,229 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:41:10,291 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:41:11,294 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:41:14,483 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:41:14,483 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:41:14,484 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:41:14,484 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:41:14,489 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:41:15,299 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:41:15,299 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:41:17,830 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:41:17,831 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:41:17,831 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:41:17,836 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:41:18,301 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:41:19,303 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:41:19,837 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:41:21,301 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:41:21,301 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:41:21,302 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:41:21,307 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:41:21,307 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:41:22,252 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:41:22,253 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:41:22,254 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:41:23,310 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:41:24,702 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:41:24,703 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:41:24,703 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:41:24,708 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:41:25,311 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:41:25,747 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:41:27,314 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:41:28,431 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:41:28,432 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:41:28,432 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:41:28,439 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:41:29,318 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:41:29,318 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:41:31,440 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:41:32,406 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 15:41:33,022 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:41:33,023 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:41:33,023 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:41:33,028 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:41:33,322 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:41:35,325 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:41:37,029 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:41:37,159 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:41:37,159 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:41:37,159 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:41:37,164 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:41:37,252 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:41:37,253 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:41:37,254 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:41:37,326 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:41:39,329 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:41:41,831 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:41:41,831 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:41:41,832 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:41:41,837 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:41:42,332 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:41:42,875 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:41:43,334 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:41:46,555 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:41:46,555 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:41:46,556 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:41:46,561 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:41:47,339 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:41:47,339 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:41:48,561 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:41:52,125 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:41:52,126 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:41:52,126 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:41:52,131 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:41:52,253 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:41:52,253 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:41:52,254 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:41:52,342 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:41:53,344 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:41:54,561 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:41:56,980 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:41:56,980 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:41:56,980 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:41:56,986 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:41:57,347 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:41:59,350 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:42:00,023 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:42:00,273 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:42:00,274 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:42:00,274 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:42:00,279 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:42:00,351 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:42:01,354 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:42:02,414 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 15:42:04,762 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:42:04,763 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:42:04,763 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:42:04,769 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:42:05,356 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:42:05,769 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:42:07,253 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:42:07,253 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:42:07,254 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:42:07,360 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:42:11,486 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:42:13,301 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:42:13,302 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:42:13,302 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:42:13,307 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:42:13,364 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:42:15,368 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:42:17,343 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:42:17,454 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:42:17,454 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:42:17,454 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:42:17,459 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:42:18,370 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:42:19,372 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:42:22,253 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:42:22,253 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:42:22,254 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:42:22,499 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:42:23,013 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:42:23,014 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:42:23,014 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:42:23,019 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:42:23,375 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:42:25,378 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:42:27,961 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:42:27,962 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:42:27,962 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:42:27,963 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:42:27,968 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:42:28,381 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:42:29,383 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:42:31,677 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:42:31,678 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:42:31,678 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:42:31,683 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:42:32,386 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:42:32,410 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 15:42:33,389 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:42:33,412 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:42:35,440 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:42:35,441 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:42:35,441 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:42:35,447 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:42:36,391 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:42:37,253 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:42:37,253 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:42:37,255 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:42:37,394 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:42:38,464 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:42:39,960 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:42:39,961 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:42:39,961 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:42:39,966 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:42:40,395 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:42:41,398 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:42:44,003 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:42:44,668 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:42:44,669 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:42:44,669 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:42:44,674 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:42:45,403 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:42:45,403 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:42:49,675 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:42:51,221 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:42:51,222 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:42:51,222 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:42:51,227 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:42:51,406 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:42:52,253 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:42:52,253 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:42:52,255 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:42:53,410 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:42:55,477 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:42:56,001 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:42:56,002 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:42:56,002 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:42:56,007 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:42:56,412 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:42:57,414 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:42:59,656 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:42:59,657 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:42:59,658 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:42:59,663 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:43:00,416 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:43:00,663 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:43:01,419 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:43:02,412 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 15:43:02,716 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:43:02,717 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:43:02,718 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:43:02,723 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:43:03,420 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:43:05,424 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:43:05,723 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:43:06,492 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:43:06,493 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:43:06,493 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:43:06,500 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:43:07,253 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:43:07,253 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:43:07,255 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:43:07,427 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:43:07,427 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:43:11,211 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:43:11,212 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:43:11,212 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:43:11,212 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:43:11,217 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:43:11,431 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:43:13,434 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:43:14,474 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:43:14,475 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:43:14,476 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:43:14,480 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:43:15,438 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:43:15,438 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:43:16,481 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:43:18,882 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:43:18,883 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:43:18,883 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:43:18,888 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:43:19,440 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:43:21,444 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:43:21,889 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:43:22,253 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:43:22,253 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:43:22,255 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:43:25,045 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:43:25,046 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:43:25,046 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:43:25,051 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:43:25,446 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:43:27,087 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:43:27,450 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:43:29,730 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:43:29,731 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:43:29,731 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:43:29,736 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:43:30,452 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:43:31,456 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:43:32,415 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 15:43:32,416 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:43:33,109 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:43:33,110 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:43:33,110 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:43:33,115 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:43:33,457 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:43:35,460 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:43:37,253 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:43:37,254 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:43:37,255 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:43:37,509 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:43:38,205 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:43:38,206 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:43:38,206 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:43:38,211 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:43:38,462 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:43:39,465 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:43:41,710 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:43:41,711 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:43:41,711 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:43:41,716 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:43:42,467 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:43:42,717 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:43:43,470 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:43:45,972 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:43:45,973 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:43:45,973 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:43:45,978 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:43:46,472 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:43:47,475 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:43:47,979 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:43:49,671 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:43:49,671 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:43:49,672 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:43:49,677 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:43:50,477 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:43:51,480 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:43:52,253 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:43:52,254 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:43:52,255 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:43:53,257 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:43:53,257 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:43:53,257 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:43:53,258 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:43:53,263 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:43:53,481 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:43:55,485 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:43:57,047 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:43:57,047 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:43:57,048 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:43:57,053 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:43:57,486 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:43:59,054 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:43:59,489 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:44:01,848 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:44:01,849 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:44:01,849 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:44:01,854 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:44:02,417 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 15:44:02,491 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:44:03,494 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:44:04,418 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:44:05,788 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:44:05,789 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:44:05,789 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:44:05,794 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:44:06,496 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:44:07,253 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:44:07,254 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:44:07,255 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:44:07,499 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:44:08,882 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:44:08,883 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:44:08,883 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:44:08,888 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:44:09,501 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:44:09,927 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:44:11,504 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:44:14,223 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:44:14,224 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:44:14,225 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:44:14,230 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:44:14,506 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:44:15,230 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:44:15,507 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:44:20,021 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:44:20,022 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:44:20,022 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:44:20,027 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:44:20,510 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:44:21,028 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:44:21,512 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:44:22,253 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:44:22,254 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:44:22,255 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:44:26,103 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:44:26,104 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:44:26,104 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:44:26,104 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:44:26,109 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:44:26,516 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:44:27,518 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:44:31,147 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:44:32,422 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 15:44:34,544 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:44:34,546 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:44:34,546 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:44:34,551 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:44:35,527 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:44:35,528 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:44:36,551 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:44:37,253 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:44:37,254 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:44:37,255 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:44:37,779 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:44:37,997 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:44:37,997 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:44:38,002 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:44:38,529 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:44:39,532 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:44:41,659 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:44:41,660 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:44:41,660 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:44:41,660 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:44:41,665 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:44:42,534 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:44:43,537 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:44:45,335 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:44:45,336 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:44:45,337 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:44:45,342 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:44:45,538 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:44:47,342 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:44:47,541 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:44:49,848 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:44:49,849 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:44:49,849 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:44:49,854 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:44:50,543 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:44:51,544 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:44:52,253 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:44:52,254 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:44:52,255 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:44:52,553 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:44:53,395 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:44:53,396 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:44:53,396 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:44:53,401 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:44:53,545 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:44:55,548 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:44:57,735 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:44:57,735 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:44:57,736 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:44:57,737 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:44:57,742 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:44:58,550 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:44:59,553 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:45:01,254 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:45:01,255 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:45:01,255 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:45:01,261 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:45:01,554 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:45:02,421 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 15:45:03,423 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:45:03,558 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:45:04,991 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:45:04,992 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:45:04,992 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:45:04,997 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:45:05,559 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:45:07,254 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:45:07,254 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:45:07,255 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:45:07,562 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:45:08,470 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:45:09,958 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:45:09,959 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:45:09,959 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:45:09,964 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:45:10,565 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:45:11,568 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:45:14,003 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:45:14,930 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:45:14,931 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:45:14,931 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:45:14,936 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:45:15,571 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:45:17,574 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:45:18,416 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:45:18,417 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:45:18,417 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:45:18,422 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:45:18,574 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:45:19,423 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:45:19,577 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:45:22,254 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:45:22,254 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:45:22,256 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:45:22,909 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:45:22,910 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:45:22,910 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:45:22,915 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:45:23,579 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:45:24,955 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:45:25,583 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:45:26,652 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:45:26,653 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:45:26,653 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:45:26,658 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:45:27,587 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:45:27,587 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:45:30,659 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:45:31,010 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:45:31,011 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:45:31,011 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:45:31,017 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:45:31,591 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:45:32,427 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 15:45:33,595 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:45:35,352 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:45:35,353 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:45:35,353 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:45:35,358 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:45:35,596 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:45:36,359 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:45:37,254 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:45:37,254 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:45:37,255 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:45:37,599 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:45:38,763 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:45:38,764 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:45:38,764 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:45:38,769 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:45:39,603 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:45:39,603 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:45:41,807 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:45:43,828 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:45:43,829 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:45:43,829 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:45:43,835 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:45:44,606 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:45:45,609 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:45:46,835 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:45:48,089 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:45:48,090 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:45:48,090 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:45:48,095 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:45:48,611 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:45:49,614 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:45:52,096 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:45:52,254 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:45:52,254 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:45:52,256 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:45:53,381 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:45:53,382 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:45:53,382 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:45:53,387 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:45:53,616 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:45:55,619 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:45:57,427 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:45:58,203 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:45:58,204 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:45:58,204 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:45:58,210 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:45:58,621 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:45:59,624 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:46:02,426 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 15:46:03,427 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:46:03,770 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:46:03,770 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:46:03,771 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:46:03,776 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:46:04,627 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:46:05,629 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:46:07,254 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:46:07,254 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:46:07,256 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:46:08,516 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:46:08,829 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:46:08,829 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:46:08,829 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:46:08,835 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:46:09,635 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:46:09,635 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:46:13,871 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:46:15,733 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:46:15,734 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:46:15,734 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:46:15,739 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:46:16,639 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:46:17,642 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:46:19,739 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:46:22,254 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:46:22,254 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:46:22,256 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:46:24,005 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:46:24,005 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:46:24,006 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:46:24,010 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:46:24,646 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:46:25,046 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:46:25,649 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:46:27,959 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:46:27,960 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:46:27,960 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:46:27,965 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:46:28,651 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:46:29,654 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:46:30,966 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:46:32,428 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 15:46:34,144 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:46:34,144 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:46:34,145 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:46:34,150 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:46:34,658 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:46:35,661 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:46:36,151 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:46:37,254 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:46:37,254 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:46:37,256 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:46:40,209 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:46:40,210 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:46:40,210 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:46:40,215 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:46:40,664 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:46:41,250 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:46:41,666 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:46:44,712 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:46:44,713 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:46:44,713 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:46:44,718 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:46:45,672 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:46:45,672 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:46:46,719 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:46:49,273 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:46:49,274 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:46:49,274 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:46:49,279 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:46:49,674 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:46:51,677 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:46:52,254 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:46:52,255 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:46:52,256 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:46:52,271 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:46:52,539 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:46:52,540 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:46:52,540 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:46:52,545 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:46:52,678 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:46:53,681 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:46:56,561 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:46:56,562 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:46:56,562 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:46:56,567 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:46:56,683 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:46:57,568 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:46:57,685 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:47:00,056 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:47:00,057 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:47:00,057 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:47:00,062 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:47:00,687 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:47:01,690 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:47:02,430 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 15:47:03,431 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:47:03,567 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:47:03,568 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:47:03,568 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:47:03,574 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:47:03,691 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:47:05,694 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:47:07,154 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:47:07,155 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:47:07,155 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:47:07,160 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:47:07,254 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:47:07,255 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:47:07,256 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:47:07,696 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:47:08,732 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:47:09,698 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:47:11,003 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:47:11,003 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:47:11,004 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:47:11,009 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:47:11,700 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:47:13,703 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:47:14,046 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:47:16,164 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:47:16,165 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:47:16,166 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:47:16,171 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:47:16,705 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:47:17,707 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:47:19,171 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:47:19,878 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:47:19,879 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:47:19,879 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:47:19,884 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:47:20,709 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:47:21,712 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:47:22,254 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:47:22,255 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:47:22,256 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:47:24,115 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:47:24,116 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:47:24,116 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:47:24,121 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:47:24,714 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:47:25,158 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:47:25,717 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:47:29,694 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:47:29,694 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:47:29,695 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:47:29,700 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:47:29,719 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:47:30,700 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:47:31,724 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:47:32,432 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 15:47:33,942 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:47:33,943 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:47:33,943 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:47:33,948 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:47:34,726 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:47:35,728 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:47:35,949 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:47:37,254 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:47:37,255 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:47:37,256 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:47:38,553 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:47:38,554 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:47:38,554 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:47:38,559 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:47:38,730 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:47:39,733 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:47:41,598 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:47:43,083 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:47:43,084 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:47:43,084 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:47:43,089 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:47:43,736 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:47:45,739 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:47:47,090 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:47:47,171 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:47:47,172 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:47:47,172 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:47:47,178 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:47:47,740 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:47:49,743 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:47:50,796 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:47:50,797 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:47:50,797 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:47:50,802 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:47:51,746 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:47:51,747 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:47:52,254 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:47:52,255 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:47:52,256 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:47:52,531 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:47:57,532 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:47:58,565 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:47:58,566 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:47:58,566 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:47:58,571 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:47:58,751 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:47:59,753 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:48:02,435 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 15:48:03,436 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:48:03,515 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:48:03,516 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:48:03,516 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:48:03,521 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:48:03,756 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:48:05,759 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:48:07,031 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:48:07,032 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:48:07,032 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:48:07,037 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:48:07,255 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:48:07,256 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:48:07,256 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:48:07,762 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:48:07,763 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:48:09,130 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:48:11,510 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:48:11,510 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:48:11,511 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:48:11,516 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:48:11,766 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:48:13,770 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:48:14,554 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:48:15,458 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:48:15,458 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:48:15,459 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:48:15,464 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:48:15,771 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:48:17,774 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:48:20,465 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:48:21,436 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:48:21,437 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:48:21,437 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:48:21,442 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:48:21,776 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:48:22,255 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:48:22,255 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:48:22,257 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:48:23,780 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:48:24,859 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:48:24,860 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:48:24,860 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:48:24,866 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:48:25,783 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:48:25,783 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:48:25,902 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:48:27,915 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:48:27,916 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:48:27,916 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:48:27,921 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:48:28,785 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:48:29,788 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:48:30,922 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:48:32,437 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 15:48:36,438 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:48:36,470 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:48:36,471 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:48:36,471 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:48:36,476 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:48:36,793 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:48:37,255 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:48:37,255 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:48:37,256 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:48:37,796 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:48:40,977 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:48:40,978 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:48:40,978 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:48:40,983 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:48:41,800 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:48:41,801 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:48:42,022 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:48:46,619 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:48:46,620 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:48:46,620 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:48:46,625 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:48:46,804 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:48:47,626 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:48:47,806 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:48:50,175 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:48:50,175 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:48:50,176 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:48:50,181 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:48:50,808 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:48:51,811 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:48:52,255 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:48:52,255 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:48:52,257 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:48:53,625 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:48:54,374 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:48:54,375 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:48:54,375 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:48:54,380 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:48:54,813 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:48:55,815 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:48:59,418 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:49:00,479 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:49:00,480 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:49:00,480 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:49:00,485 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:49:00,818 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:49:01,821 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:49:02,443 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 15:49:04,444 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:49:05,267 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:49:05,268 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:49:05,269 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:49:05,274 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:49:05,823 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:49:07,255 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:49:07,255 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:49:07,257 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:49:07,826 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:49:09,543 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:49:09,659 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:49:09,659 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:49:09,660 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:49:09,665 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:49:09,829 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:49:11,832 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:49:14,702 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:49:18,086 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:49:18,087 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:49:18,087 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:49:18,091 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:49:18,836 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:49:19,839 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:49:20,092 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:49:21,655 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:49:21,656 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:49:21,656 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:49:21,661 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:49:21,840 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:49:22,255 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:49:22,255 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:49:22,257 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:49:23,843 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:49:24,906 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:49:24,906 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:49:24,907 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:49:24,912 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:49:25,847 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:49:25,847 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:49:25,950 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:49:29,208 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:49:29,209 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:49:29,209 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:49:29,214 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:49:29,849 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:49:31,215 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:49:31,854 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:49:32,441 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 15:49:32,691 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:49:32,692 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:49:32,692 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:49:32,698 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:49:32,854 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:49:33,857 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:49:36,699 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:49:37,110 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:49:37,111 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:49:37,111 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:49:37,116 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:49:37,255 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:49:37,255 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:49:37,257 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:49:37,862 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:49:37,862 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:49:42,527 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:49:45,715 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:49:45,716 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:49:45,716 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:49:45,721 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:49:45,867 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:49:47,758 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:49:47,870 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:49:49,208 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:49:49,208 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:49:49,209 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:49:49,213 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:49:49,871 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:49:51,874 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:49:52,255 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:49:52,255 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:49:52,257 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:49:52,980 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:49:52,980 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:49:52,981 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:49:52,981 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:49:52,985 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:49:53,878 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:49:53,878 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:49:58,022 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:50:02,447 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 15:50:02,563 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:50:02,564 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:50:02,565 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:50:02,570 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:50:02,883 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:50:03,571 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:50:03,884 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:50:06,893 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:50:06,894 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:50:06,894 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:50:06,899 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:50:07,255 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:50:07,256 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:50:07,257 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:50:07,889 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:50:07,889 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:50:09,499 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:50:10,598 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:50:10,598 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:50:10,599 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:50:10,603 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:50:10,891 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:50:11,894 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:50:14,642 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:50:15,527 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:50:15,527 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:50:15,528 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:50:15,532 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:50:15,896 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:50:17,900 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:50:20,471 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:50:20,472 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:50:20,473 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:50:20,473 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:50:20,477 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:50:20,902 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:50:21,904 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:50:22,255 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:50:22,256 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:50:22,257 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:50:25,516 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:50:26,491 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:50:26,492 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:50:26,492 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:50:26,497 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:50:26,908 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:50:27,910 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:50:30,534 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:50:30,671 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:50:30,672 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:50:30,672 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:50:30,677 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:50:30,913 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:50:31,917 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:50:32,445 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 15:50:36,338 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:50:36,339 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:50:36,339 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:50:36,340 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:50:36,345 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:50:36,920 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:50:37,255 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:50:37,256 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:50:37,257 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:50:37,922 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:50:41,583 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:50:41,583 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:50:41,583 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:50:41,583 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:50:41,588 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:50:41,925 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:50:43,929 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:50:46,626 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:50:50,913 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:50:50,914 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:50:50,914 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:50:50,919 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:50:50,933 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:50:51,920 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:50:51,935 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:50:52,256 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:50:52,256 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:50:52,257 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:50:54,824 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:50:54,825 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:50:54,825 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:50:54,830 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:50:54,937 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:50:55,940 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:50:57,866 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:50:59,583 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:50:59,584 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:50:59,584 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:50:59,589 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:50:59,942 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:51:01,946 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:51:02,448 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 15:51:03,449 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:51:04,942 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:51:04,943 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:51:04,943 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:51:04,948 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:51:04,949 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:51:05,952 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:51:07,256 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:51:07,256 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:51:07,258 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:51:08,529 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:51:08,689 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:51:08,689 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:51:08,690 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:51:08,695 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:51:08,953 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:51:09,957 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:51:13,730 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:51:17,336 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:51:17,337 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:51:17,337 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:51:17,342 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:51:17,962 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:51:19,343 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:51:19,965 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:51:22,256 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:51:22,256 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:51:22,258 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:51:24,117 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:51:24,117 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:51:24,118 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:51:24,123 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:51:24,968 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:51:25,158 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:51:25,971 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:51:28,512 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:51:28,513 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:51:28,513 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:51:28,518 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:51:28,973 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:51:29,976 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:51:30,519 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:51:32,450 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 15:51:33,513 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:51:33,514 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:51:33,514 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:51:33,519 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:51:33,979 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:51:35,520 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:51:35,983 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:51:37,256 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:51:37,256 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:51:37,258 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:51:37,867 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:51:37,868 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:51:37,868 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:51:37,873 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:51:37,984 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:51:39,987 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:51:40,910 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:51:42,855 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:51:42,856 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:51:42,856 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:51:42,861 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:51:42,989 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:51:43,991 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:51:46,183 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:51:46,184 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:51:46,185 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:51:46,185 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:51:46,189 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:51:46,993 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:51:47,996 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:51:49,923 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:51:49,923 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:51:49,924 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:51:49,928 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:51:49,997 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:51:51,929 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:51:52,000 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:51:52,256 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:51:52,256 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:51:52,258 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:51:54,030 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:51:54,031 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:51:54,031 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:51:54,036 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:51:55,002 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:51:56,005 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:51:57,074 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:51:57,349 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:51:57,350 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:51:57,350 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:51:57,355 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:51:58,006 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:52:00,009 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:52:01,542 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:52:01,543 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:52:01,543 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:52:01,548 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:52:02,011 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:52:02,452 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 15:52:02,453 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:52:04,014 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:52:07,256 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:52:07,256 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:52:07,258 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:52:07,529 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:52:10,917 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:52:10,918 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:52:10,918 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:52:10,923 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:52:11,019 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:52:12,022 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:52:12,958 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:52:16,172 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:52:16,172 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:52:16,173 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:52:16,178 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:52:17,025 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:52:18,028 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:52:18,178 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:52:22,256 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:52:22,256 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:52:22,258 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:52:22,748 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:52:22,749 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:52:22,749 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:52:22,754 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:52:23,031 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:52:23,790 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:52:24,034 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:52:27,220 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:52:27,221 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:52:27,222 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:52:27,226 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:52:28,038 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:52:28,038 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:52:29,227 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:52:31,082 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:52:31,082 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:52:31,083 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:52:31,088 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:52:32,045 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:52:32,045 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:52:32,458 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 15:52:34,459 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:52:34,851 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:52:34,852 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:52:34,852 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:52:34,857 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:52:35,047 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:52:36,049 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:52:37,256 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:52:37,256 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:52:37,258 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:52:38,225 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:52:38,226 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:52:38,226 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:52:38,231 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:52:39,051 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:52:40,054 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:52:40,267 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:52:43,536 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:52:43,537 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:52:43,538 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:52:43,543 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:52:44,057 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:52:45,544 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:52:46,060 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:52:48,585 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:52:48,586 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:52:48,586 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:52:48,591 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:52:49,062 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:52:50,065 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:52:50,591 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:52:52,256 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:52:52,256 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:52:52,258 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:52:53,012 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:52:53,012 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:52:53,013 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:52:53,018 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:52:53,066 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:52:54,069 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:52:56,055 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:52:57,950 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:52:57,951 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:52:57,951 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:52:57,956 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:52:58,071 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:53:00,075 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:53:01,957 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:53:02,277 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:53:02,278 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:53:02,278 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:53:02,283 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:53:02,456 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 15:53:03,077 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:53:04,079 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:53:05,845 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:53:05,846 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:53:05,846 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:53:05,851 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:53:06,080 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:53:07,256 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:53:07,257 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:53:07,258 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:53:07,474 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:53:08,084 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:53:09,137 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:53:09,137 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:53:09,138 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:53:09,142 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:53:10,088 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:53:10,088 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:53:12,183 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:53:12,184 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:53:12,184 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:53:12,189 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:53:13,090 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:53:13,190 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:53:14,093 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:53:16,057 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:53:16,057 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:53:16,058 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:53:16,063 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:53:16,094 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:53:18,097 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:53:19,063 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:53:21,628 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:53:21,629 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:53:21,629 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:53:21,634 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:53:22,099 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:53:22,256 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:53:22,257 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:53:22,258 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:53:24,103 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:53:24,534 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:53:25,811 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:53:25,811 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:53:25,811 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:53:25,816 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:53:26,104 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:53:28,107 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:53:29,502 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:53:29,503 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:53:29,503 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:53:29,508 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:53:30,109 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:53:30,509 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:53:32,113 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:53:32,461 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 15:53:33,463 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:53:33,464 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:53:33,465 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:53:33,469 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:53:34,114 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:53:36,118 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:53:36,470 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:53:37,256 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:53:37,257 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:53:37,258 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:53:37,940 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:53:37,940 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:53:37,941 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:53:37,946 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:53:38,119 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:53:40,122 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:53:41,512 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:53:41,513 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:53:41,514 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:53:41,514 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:53:41,519 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:53:42,123 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:53:44,127 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:53:45,063 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:53:45,063 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:53:45,064 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:53:45,069 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:53:45,128 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:53:46,130 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:53:47,069 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:53:48,532 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:53:48,533 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:53:48,533 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:53:48,538 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:53:49,132 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:53:50,135 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:53:52,256 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:53:52,257 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:53:52,258 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:53:52,455 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:53:53,270 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:53:53,270 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:53:53,270 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:53:53,276 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:53:54,140 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:53:54,140 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:53:57,374 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:53:57,375 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:53:57,375 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:53:57,380 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:53:58,144 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:53:58,144 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:53:58,381 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:54:02,461 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 15:54:03,462 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:54:04,591 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:54:04,591 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:54:04,592 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:54:04,596 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:54:05,148 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:54:06,151 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:54:07,257 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:54:07,257 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:54:07,259 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:54:08,026 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:54:08,027 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:54:08,027 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:54:08,032 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:54:08,152 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:54:09,070 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:54:10,156 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:54:11,615 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:54:11,616 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:54:11,616 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:54:11,621 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:54:12,158 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:54:14,161 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:54:14,622 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:54:15,992 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:54:15,993 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:54:15,994 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:54:15,999 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:54:16,163 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:54:18,166 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:54:19,147 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:54:19,148 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:54:19,148 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:54:19,153 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:54:19,167 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:54:20,154 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:54:20,169 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:54:22,257 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:54:22,257 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:54:22,259 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:54:23,069 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:54:23,070 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:54:23,070 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:54:23,075 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:54:23,171 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:54:24,174 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:54:26,110 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:54:28,491 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:54:28,492 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:54:28,492 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:54:28,497 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:54:29,178 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:54:30,180 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:54:31,498 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:54:32,186 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:54:32,187 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:54:32,187 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:54:32,192 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:54:32,466 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 15:54:33,183 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:54:34,186 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:54:35,930 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:54:35,930 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:54:35,931 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:54:35,935 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:54:36,187 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:54:36,936 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:54:37,257 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:54:37,257 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:54:37,258 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:54:38,190 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:54:40,179 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:54:40,180 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:54:40,180 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:54:40,185 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:54:40,191 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:54:42,195 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:54:42,222 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:54:43,558 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:54:43,559 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:54:43,559 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:54:43,564 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:54:44,196 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:54:46,200 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:54:47,565 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:54:48,054 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:54:48,055 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:54:48,055 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:54:48,061 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:54:48,201 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:54:50,204 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:54:52,175 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:54:52,175 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:54:52,176 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:54:52,181 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:54:52,205 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:54:52,257 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:54:52,257 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:54:52,259 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:54:53,531 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:54:54,209 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:54:58,382 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:54:58,382 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:54:58,383 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:54:58,388 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:54:59,212 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:54:59,426 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:55:00,214 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:55:01,724 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:55:01,725 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:55:01,725 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:55:01,730 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:55:02,216 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:55:02,465 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 15:55:04,219 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:55:04,467 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:55:07,257 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:55:07,257 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:55:07,259 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:55:07,957 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:55:07,958 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:55:07,958 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:55:07,963 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:55:08,221 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:55:09,998 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:55:10,225 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:55:12,376 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:55:12,376 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:55:12,377 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:55:12,381 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:55:13,228 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:55:14,230 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:55:15,382 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:55:15,698 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:55:15,698 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:55:15,699 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:55:15,704 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:55:16,232 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:55:18,235 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:55:19,565 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:55:19,566 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:55:19,567 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:55:19,572 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:55:20,238 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:55:20,239 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:55:20,572 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:55:22,257 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:55:22,257 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:55:22,259 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:55:24,212 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:55:24,213 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:55:24,213 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:55:24,218 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:55:24,241 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:55:26,244 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:55:26,254 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:55:28,236 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:55:28,237 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:55:28,237 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:55:28,242 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:55:28,245 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:55:30,249 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:55:32,243 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:55:32,471 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 15:55:33,842 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:55:33,842 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:55:33,843 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:55:33,848 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:55:34,253 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:55:36,256 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:55:37,257 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:55:37,257 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:55:37,259 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:55:37,550 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:55:38,005 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:55:38,005 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:55:38,006 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:55:38,010 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:55:38,257 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:55:40,261 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:55:41,342 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:55:41,343 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:55:41,343 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:55:41,348 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:55:42,264 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:55:42,264 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:55:43,349 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:55:46,141 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:55:46,142 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:55:46,142 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:55:46,147 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:55:46,267 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:55:48,270 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:55:49,148 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:55:49,573 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:55:49,574 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:55:49,574 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:55:49,579 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:55:50,273 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:55:50,274 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:55:52,257 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:55:52,258 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:55:52,259 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:55:53,520 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:55:53,520 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:55:53,521 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:55:53,526 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:55:54,278 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:55:54,278 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:55:54,562 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:55:57,103 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:55:57,104 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:55:57,105 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:55:57,110 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:55:57,279 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:55:58,282 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:56:00,110 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:56:02,470 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 15:56:02,709 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:56:02,710 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:56:02,710 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:56:02,715 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:56:03,285 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:56:04,288 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:56:05,716 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:56:07,257 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:56:07,258 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:56:07,259 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:56:08,787 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:56:08,787 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:56:08,788 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:56:08,793 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:56:09,291 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:56:10,294 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:56:10,830 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:56:12,269 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:56:12,270 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:56:12,270 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:56:12,275 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:56:12,295 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:56:14,299 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:56:16,276 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:56:16,402 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:56:16,403 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:56:16,403 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:56:16,409 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:56:17,300 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:56:18,303 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:56:20,397 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:56:20,398 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:56:20,398 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:56:20,403 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:56:21,305 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:56:21,404 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:56:22,257 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:56:22,258 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:56:22,259 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:56:22,308 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:56:24,039 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:56:24,040 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:56:24,040 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:56:24,049 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:56:24,309 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:56:26,313 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:56:27,082 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:56:28,408 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:56:28,409 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:56:28,409 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:56:28,414 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:56:29,315 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:56:30,318 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:56:32,414 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:56:32,472 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 15:56:33,359 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:56:33,360 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:56:33,360 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:56:33,365 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:56:34,324 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:56:34,324 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:56:37,257 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:56:37,258 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:56:37,259 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:56:38,155 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:56:38,339 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:56:38,339 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:56:38,339 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:56:38,344 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:56:39,327 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:56:40,329 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:56:43,012 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:56:43,012 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:56:43,012 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:56:43,017 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:56:43,332 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:56:44,018 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:56:44,334 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:56:47,261 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:56:47,262 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:56:47,262 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:56:47,267 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:56:47,336 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:56:48,339 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:56:49,268 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:56:50,883 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:56:50,884 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:56:50,884 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:56:50,889 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:56:51,340 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:56:52,258 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:56:52,258 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:56:52,259 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:56:52,343 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:56:54,511 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:56:55,854 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:56:55,854 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:56:55,854 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:56:55,859 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:56:56,346 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:56:58,349 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:56:59,312 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:56:59,313 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:56:59,313 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:56:59,318 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:56:59,349 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:57:00,318 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:57:00,352 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:57:02,474 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 15:57:03,373 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:57:03,373 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:57:03,374 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:57:03,379 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:57:04,356 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:57:04,357 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:57:05,379 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:57:07,077 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:57:07,078 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:57:07,078 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:57:07,083 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:57:07,258 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:57:07,258 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:57:07,259 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:57:07,358 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:57:08,361 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:57:10,315 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:57:10,315 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:57:10,316 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:57:10,321 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:57:10,362 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:57:11,358 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:57:12,366 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:57:16,358 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:57:17,554 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:57:17,554 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:57:17,554 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:57:17,560 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:57:18,371 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:57:18,372 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:57:21,561 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:57:22,258 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:57:22,258 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:57:22,259 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:57:22,734 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:57:22,734 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:57:22,734 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:57:22,740 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:57:23,375 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:57:24,377 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:57:26,375 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:57:26,376 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:57:26,376 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:57:26,381 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:57:26,381 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:57:27,382 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:57:28,384 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:57:30,749 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:57:30,749 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:57:30,750 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:57:30,754 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:57:31,387 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:57:32,390 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:57:32,476 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 15:57:32,477 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:57:33,686 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:57:33,687 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:57:33,687 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:57:33,695 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:57:34,392 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:57:34,392 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:57:37,258 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:57:37,258 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:57:37,260 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:57:37,561 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:57:39,007 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:57:39,008 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:57:39,008 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:57:39,013 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:57:39,395 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:57:40,398 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:57:42,986 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:57:42,987 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:57:42,987 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:57:42,987 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:57:42,993 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:57:43,400 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:57:44,403 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:57:46,781 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:57:46,782 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:57:46,782 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:57:46,787 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:57:47,404 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:57:48,407 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:57:48,787 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:57:52,258 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:57:52,258 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:57:52,259 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:57:52,535 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:57:52,536 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:57:52,536 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:57:52,541 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:57:53,411 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:57:54,413 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:57:54,578 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:57:58,347 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:57:58,348 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:57:58,348 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:57:58,353 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:57:58,416 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:58:00,354 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:58:00,419 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:58:02,415 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:58:02,415 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:58:02,416 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:58:02,420 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:58:02,421 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:58:02,478 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 15:58:04,424 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:58:05,479 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:58:05,875 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:58:05,876 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:58:05,876 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:58:05,881 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:58:06,425 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:58:07,258 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:58:07,258 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:58:07,260 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:58:08,429 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:58:09,545 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:58:09,546 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:58:09,546 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:58:09,551 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:58:10,431 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:58:10,431 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:58:10,590 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:58:12,820 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:58:12,821 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:58:12,821 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:58:12,826 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:58:13,433 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:58:14,436 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:58:15,826 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:58:17,489 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:58:17,489 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:58:17,489 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:58:17,494 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:58:18,440 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:58:18,441 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:58:20,449 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:58:20,450 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:58:20,450 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:58:20,455 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:58:21,442 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:58:21,455 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:58:22,258 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:58:22,258 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:58:22,260 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:58:22,445 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:58:26,013 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:58:26,014 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:58:26,014 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:58:26,019 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:58:26,448 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:58:27,054 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:58:28,451 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:58:29,474 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:58:29,475 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:58:29,475 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:58:29,481 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:58:30,455 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:58:30,455 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:58:32,480 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 15:58:32,481 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:58:33,243 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:58:33,244 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:58:33,244 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:58:33,251 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:58:33,458 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:58:34,461 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:58:36,792 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:58:36,792 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:58:36,793 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:58:36,797 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:58:37,258 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:58:37,259 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:58:37,260 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:58:37,463 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:58:38,465 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:58:38,473 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:58:40,596 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:58:40,596 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:58:40,597 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:58:40,602 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:58:41,467 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:58:42,470 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:58:43,638 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:58:44,773 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:58:44,774 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:58:44,774 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:58:44,779 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:58:45,472 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:58:46,473 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:58:48,780 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:58:48,867 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:58:48,868 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:58:48,868 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:58:48,873 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:58:49,475 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:58:50,477 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:58:52,258 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:58:52,259 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:58:52,260 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:58:52,558 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:58:52,558 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:58:52,559 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:58:52,563 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:58:53,479 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:58:54,482 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:58:54,602 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:58:57,358 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:58:57,358 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:58:57,358 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:58:57,363 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:58:57,483 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:58:58,486 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:59:00,364 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:59:02,342 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:59:02,343 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:59:02,343 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:59:02,348 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:59:02,486 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 15:59:02,488 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:59:04,492 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:59:05,487 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:59:05,550 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:59:05,551 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:59:05,551 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:59:05,556 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:59:06,495 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:59:06,495 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:59:07,258 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:59:07,259 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:59:07,260 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:59:09,889 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:59:09,889 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:59:09,889 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:59:09,894 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:59:10,498 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:59:10,931 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:59:12,502 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:59:15,931 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:59:18,348 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:59:18,349 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:59:18,349 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:59:18,354 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:59:18,505 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:59:20,509 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:59:21,354 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:59:22,258 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:59:22,259 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:59:22,260 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:59:23,675 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:59:23,675 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:59:23,675 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:59:23,680 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:59:24,513 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:59:24,513 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:59:26,719 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:59:27,704 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:59:27,705 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:59:27,705 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:59:27,710 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:59:28,517 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:59:28,518 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:59:31,511 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:59:31,512 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:59:31,512 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:59:31,517 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:59:31,519 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:59:32,485 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 15:59:32,486 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:59:32,523 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:59:35,883 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:59:35,884 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:59:35,884 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:59:35,889 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:59:36,525 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:59:37,258 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:59:37,259 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:59:37,260 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:59:37,541 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:59:38,529 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:59:40,546 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:59:40,546 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:59:40,546 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:59:40,551 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:59:41,530 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:59:42,533 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:59:42,591 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:59:47,591 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:59:50,246 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:59:50,247 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:59:50,247 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:59:50,252 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:59:50,538 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:59:52,259 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 15:59:52,259 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 15:59:52,260 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 15:59:52,542 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:59:53,495 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:59:55,829 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:59:55,829 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:59:55,830 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:59:55,835 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:59:56,546 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 15:59:56,546 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 15:59:58,871 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 15:59:59,541 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 15:59:59,541 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 15:59:59,541 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 15:59:59,546 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 15:59:59,548 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:00:00,550 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:00:02,493 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 16:00:03,612 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:00:03,612 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:00:03,613 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:00:03,618 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:00:04,553 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:00:04,553 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:00:04,618 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:00:07,259 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:00:07,259 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:00:07,260 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:00:09,906 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:00:09,968 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:00:09,969 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:00:09,969 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:00:09,974 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:00:10,557 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:00:12,561 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:00:15,011 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:00:15,932 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:00:15,933 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:00:15,933 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:00:15,938 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:00:16,563 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:00:18,566 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:00:20,939 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:00:22,259 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:00:22,259 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:00:22,261 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:00:24,384 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:00:24,384 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:00:24,384 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:00:24,389 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:00:24,570 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:00:26,427 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:00:26,574 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:00:31,427 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:00:32,489 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 16:00:33,260 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:00:33,261 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:00:33,261 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:00:33,266 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:00:33,580 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:00:34,582 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:00:36,706 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:00:36,707 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:00:36,707 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:00:36,707 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:00:36,712 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:00:37,259 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:00:37,259 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:00:37,260 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:00:37,584 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:00:38,587 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:00:41,123 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:00:41,124 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:00:41,124 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:00:41,129 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:00:41,589 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:00:42,167 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:00:42,592 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:00:44,578 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:00:44,578 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:00:44,579 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:00:44,584 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:00:44,593 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:00:46,597 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:00:47,584 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:00:49,699 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:00:49,700 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:00:49,700 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:00:49,705 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:00:50,601 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:00:50,601 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:00:52,259 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:00:52,259 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:00:52,261 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:00:53,572 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:00:56,291 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:00:56,291 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:00:56,291 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:00:56,296 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:00:56,605 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:00:58,608 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:00:59,335 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:01:02,127 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:01:02,128 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:01:02,128 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:01:02,133 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:01:02,491 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 16:01:02,610 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:01:04,493 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:01:04,612 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:01:06,238 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:01:06,239 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:01:06,239 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:01:06,244 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:01:06,613 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:01:07,259 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:01:07,260 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:01:07,261 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:01:08,616 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:01:10,472 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:01:12,915 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:01:12,920 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:01:12,920 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:01:12,925 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:01:13,621 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:01:14,623 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:01:15,959 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:01:17,054 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:01:17,054 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:01:17,054 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:01:17,059 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:01:17,626 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:01:18,628 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:01:21,060 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:01:22,259 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:01:22,260 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:01:22,261 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:01:25,356 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:01:25,357 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:01:25,357 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:01:25,362 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:01:25,632 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:01:26,399 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:01:26,635 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:01:29,785 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:01:29,786 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:01:29,786 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:01:29,791 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:01:30,640 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:01:30,640 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:01:31,791 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:01:32,494 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 16:01:35,503 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:01:35,504 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:01:35,504 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:01:35,509 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:01:35,644 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:01:36,647 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:01:37,259 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:01:37,260 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:01:37,261 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:01:37,526 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:01:39,438 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:01:39,438 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:01:39,439 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:01:39,444 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:01:39,649 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:01:40,649 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:01:43,021 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:01:43,022 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:01:43,022 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:01:43,022 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:01:43,027 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:01:43,652 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:01:44,654 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:01:48,028 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:01:52,176 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:01:52,177 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:01:52,177 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:01:52,182 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:01:52,259 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:01:52,260 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:01:52,261 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:01:52,659 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:01:53,533 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:01:54,663 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:01:55,398 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:01:55,398 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:01:55,399 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:01:55,404 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:01:55,663 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:01:56,666 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:01:59,157 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:01:59,158 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:01:59,159 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:01:59,159 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:01:59,163 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:01:59,668 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:02:00,670 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:02:02,496 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 16:02:03,045 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:02:03,045 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:02:03,046 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:02:03,050 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:02:03,673 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:02:04,675 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:02:05,051 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:02:07,259 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:02:07,260 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:02:07,261 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:02:11,004 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:02:12,143 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:02:12,143 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:02:12,143 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:02:12,149 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:02:12,682 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:02:14,685 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:02:16,187 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:02:17,763 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:02:17,763 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:02:17,763 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:02:17,768 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:02:18,689 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:02:18,690 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:02:20,888 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:02:20,888 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:02:20,889 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:02:20,894 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:02:21,691 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:02:21,894 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:02:22,260 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:02:22,260 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:02:22,261 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:02:22,694 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:02:27,466 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:02:29,408 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:02:29,408 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:02:29,408 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:02:29,413 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:02:29,699 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:02:30,701 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:02:32,502 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 16:02:32,503 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:02:37,260 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:02:37,260 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:02:37,260 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:02:37,742 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:02:37,742 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:02:37,742 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:02:37,742 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:02:37,748 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:02:38,710 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:02:38,710 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:02:41,912 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:02:41,913 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:02:41,913 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:02:41,918 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:02:42,712 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:02:42,712 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:02:42,919 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:02:47,919 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:02:48,370 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:02:48,371 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:02:48,371 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:02:48,376 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:02:48,716 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:02:50,719 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:02:51,699 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:02:51,700 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:02:51,700 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:02:51,705 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:02:51,720 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:02:52,260 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:02:52,260 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:02:52,261 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:02:52,723 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:02:53,551 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:02:56,107 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:02:56,108 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:02:56,108 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:02:56,113 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:02:56,725 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:02:58,729 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:02:59,151 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:03:00,015 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:03:00,016 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:03:00,016 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:03:00,021 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:03:00,732 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:03:00,732 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:03:02,500 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 16:03:04,501 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:03:07,066 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:03:07,067 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:03:07,067 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:03:07,072 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:03:07,260 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:03:07,260 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:03:07,262 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:03:07,736 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:03:08,739 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:03:10,456 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:03:10,456 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:03:10,457 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:03:10,457 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:03:10,462 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:03:10,741 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:03:12,745 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:03:14,327 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:03:14,327 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:03:14,328 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:03:14,333 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:03:14,746 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:03:16,333 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:03:16,749 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:03:21,334 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:03:22,260 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:03:22,260 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:03:22,261 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:03:22,967 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:03:22,967 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:03:22,968 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:03:22,973 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:03:23,753 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:03:24,756 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:03:27,011 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:03:27,084 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:03:27,084 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:03:27,085 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:03:27,090 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:03:27,758 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:03:28,760 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:03:31,469 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:03:31,470 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:03:31,470 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:03:31,475 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:03:31,763 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:03:32,476 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:03:32,506 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 16:03:32,766 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:03:35,114 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:03:35,115 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:03:35,115 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:03:35,120 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:03:35,768 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:03:36,771 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:03:37,260 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:03:37,261 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:03:37,262 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:03:38,400 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:03:38,791 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:03:38,791 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:03:38,791 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:03:38,796 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:03:39,773 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:03:40,776 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:03:43,334 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:03:43,335 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:03:43,335 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:03:43,340 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:03:43,833 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:03:44,341 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:03:44,837 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:03:46,812 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:03:46,812 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:03:46,813 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:03:46,817 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:03:46,838 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:03:48,841 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:03:49,818 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:03:51,941 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:03:51,942 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:03:51,942 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:03:51,947 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:03:52,260 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:03:52,261 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:03:52,261 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:03:52,846 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:03:52,846 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:03:55,253 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:03:55,253 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:03:55,253 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:03:55,254 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:03:55,259 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:03:55,847 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:03:56,850 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:03:59,005 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:03:59,006 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:03:59,006 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:03:59,011 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:03:59,852 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:04:00,853 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:04:01,011 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:04:01,976 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:04:01,977 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:04:01,977 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:04:01,984 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:04:02,504 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 16:04:02,856 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:04:02,856 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:04:06,098 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:04:06,099 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:04:06,099 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:04:06,099 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:04:06,104 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:04:06,860 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:04:06,861 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:04:07,260 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:04:07,261 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:04:07,261 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:04:09,203 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:04:09,204 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:04:09,204 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:04:09,209 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:04:09,862 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:04:10,866 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:04:11,247 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:04:13,167 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:04:13,167 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:04:13,167 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:04:13,172 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:04:13,868 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:04:14,870 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:04:17,173 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:04:18,043 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:04:18,043 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:04:18,043 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:04:18,048 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:04:18,875 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:04:18,875 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:04:22,260 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:04:22,261 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:04:22,262 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:04:22,551 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:04:22,555 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:04:22,555 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:04:22,555 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:04:22,560 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:04:22,878 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:04:24,881 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:04:26,698 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:04:26,698 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:04:26,698 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:04:26,703 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:04:26,882 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:04:27,743 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:04:28,886 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:04:31,415 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:04:31,416 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:04:31,416 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:04:31,421 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:04:31,888 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:04:32,510 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 16:04:32,892 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:04:33,511 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:04:37,260 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:04:37,261 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:04:37,261 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:04:37,393 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:04:37,484 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:04:37,484 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:04:37,489 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:04:37,895 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:04:38,895 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:04:39,490 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:04:40,616 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:04:40,617 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:04:40,617 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:04:40,622 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:04:40,897 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:04:42,900 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:04:44,623 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:04:45,028 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:04:45,029 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:04:45,029 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:04:45,034 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:04:45,903 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:04:46,905 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:04:49,563 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:04:49,564 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:04:49,564 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:04:49,569 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:04:49,907 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:04:50,570 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:04:50,909 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:04:52,260 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:04:52,261 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:04:52,261 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:04:53,315 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:04:53,316 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:04:53,316 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:04:53,321 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:04:53,911 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:04:54,914 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:04:56,359 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:04:56,705 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:04:56,706 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:04:56,706 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:04:56,711 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:04:56,915 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:04:58,918 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:05:00,475 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:05:00,476 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:05:00,476 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:05:00,481 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:05:00,920 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:05:01,481 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:05:02,509 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 16:05:02,923 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:05:04,519 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:05:04,520 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:05:04,520 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:05:04,525 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:05:04,924 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:05:06,526 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:05:06,927 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:05:07,261 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:05:07,261 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:05:07,262 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:05:08,102 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:05:08,103 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:05:08,103 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:05:08,108 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:05:08,931 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:05:08,931 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:05:12,147 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:05:13,237 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:05:13,238 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:05:13,238 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:05:13,243 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:05:13,935 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:05:14,938 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:05:17,118 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:05:17,119 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:05:17,119 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:05:17,124 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:05:17,940 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:05:18,124 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:05:18,942 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:05:22,261 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:05:22,261 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:05:22,262 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:05:23,558 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:05:26,126 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:05:26,127 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:05:26,127 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:05:26,132 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:05:26,949 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:05:26,949 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:05:29,171 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:05:29,807 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:05:29,807 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:05:29,807 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:05:29,813 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:05:29,951 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:05:30,954 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:05:32,511 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 16:05:33,546 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:05:33,547 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:05:33,547 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:05:33,552 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:05:33,957 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:05:34,553 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:05:34,960 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:05:37,261 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:05:37,261 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:05:37,263 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:05:40,018 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:05:42,652 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:05:42,653 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:05:42,653 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:05:42,659 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:05:42,964 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:05:44,968 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:05:45,695 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:05:46,598 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:05:46,599 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:05:46,599 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:05:46,604 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:05:46,969 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:05:48,973 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:05:51,605 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:05:52,261 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:05:52,261 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:05:52,262 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:05:52,697 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:05:52,702 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:05:52,702 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:05:52,707 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:05:52,975 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:05:54,978 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:05:56,743 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:05:57,451 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:05:57,452 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:05:57,452 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:05:57,457 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:05:57,980 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:05:58,982 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:06:00,847 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:06:00,848 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:06:00,848 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:06:00,853 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:06:00,984 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:06:01,854 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:06:02,513 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 16:06:02,987 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:06:04,587 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:06:04,588 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:06:04,588 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:06:04,593 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:06:04,988 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:06:06,991 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:06:07,261 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:06:07,261 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:06:07,263 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:06:07,977 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:06:09,696 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:06:09,697 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:06:09,697 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:06:09,702 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:06:09,993 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:06:10,997 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:06:13,739 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:06:14,579 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:06:14,580 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:06:14,580 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:06:14,585 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:06:15,000 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:06:17,003 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:06:19,111 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:06:19,111 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:06:19,111 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:06:19,112 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:06:19,118 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:06:20,005 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:06:21,008 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:06:22,261 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:06:22,261 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:06:22,263 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:06:24,007 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:06:24,008 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:06:24,008 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:06:24,013 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:06:25,012 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:06:25,012 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:06:25,051 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:06:27,485 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:06:27,485 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:06:27,486 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:06:27,491 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:06:28,014 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:06:29,016 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:06:30,491 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:06:32,516 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 16:06:35,517 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:06:35,962 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:06:35,963 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:06:35,963 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:06:35,968 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:06:36,022 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:06:37,025 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:06:37,261 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:06:37,262 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:06:37,263 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:06:39,517 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:06:39,518 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:06:39,518 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:06:39,523 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:06:40,027 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:06:40,563 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:06:41,030 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:06:42,981 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:06:42,982 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:06:42,982 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:06:42,987 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:06:43,031 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:06:45,035 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:06:45,988 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:06:46,250 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:06:46,250 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:06:46,250 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:06:46,255 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:06:47,038 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:06:47,038 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:06:51,256 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:06:52,050 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:06:52,051 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:06:52,051 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:06:52,056 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:06:52,261 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:06:52,262 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:06:52,263 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:06:53,045 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:06:53,045 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:06:55,403 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:06:55,404 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:06:55,404 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:06:55,410 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:06:56,047 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:06:56,447 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:06:57,050 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:06:58,542 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:06:58,543 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:06:58,543 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:06:58,548 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:06:59,052 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:07:01,055 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:07:01,548 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:07:02,522 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 16:07:03,777 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:07:03,778 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:07:03,778 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:07:03,783 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:07:04,057 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:07:05,060 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:07:06,784 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:07:07,261 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:07:07,262 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:07:07,263 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:07:08,325 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:07:08,325 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:07:08,325 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:07:08,331 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:07:09,064 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:07:09,065 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:07:12,367 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:07:12,574 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:07:12,575 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:07:12,575 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:07:12,581 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:07:13,068 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:07:15,072 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:07:16,753 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:07:16,754 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:07:16,755 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:07:16,760 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:07:17,073 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:07:17,760 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:07:19,076 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:07:20,496 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:07:20,497 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:07:20,497 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:07:20,502 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:07:21,078 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:07:22,261 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:07:22,262 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:07:22,263 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:07:23,081 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:07:23,495 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:07:24,365 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:07:24,366 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:07:24,366 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:07:24,371 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:07:25,084 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:07:25,085 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:07:28,807 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:07:28,808 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:07:28,808 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:07:28,808 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:07:28,813 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:07:29,087 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:07:31,090 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:07:32,520 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 16:07:32,764 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:07:32,765 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:07:32,765 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:07:32,770 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:07:33,093 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:07:34,771 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:07:35,096 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:07:35,942 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:07:35,942 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:07:35,943 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:07:35,948 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:07:36,097 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:07:37,099 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:07:37,262 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:07:37,262 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:07:37,263 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:07:40,507 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:07:41,738 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:07:41,738 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:07:41,738 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:07:41,743 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:07:42,102 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:07:43,105 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:07:45,265 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:07:45,266 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:07:45,266 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:07:45,270 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:07:46,107 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:07:46,271 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:07:47,110 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:07:49,953 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:07:49,954 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:07:49,954 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:07:49,959 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:07:50,112 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:07:51,114 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:07:51,960 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:07:52,262 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:07:52,262 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:07:52,262 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:07:53,783 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:07:53,784 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:07:53,784 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:07:53,790 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:07:54,116 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:07:55,119 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:07:57,005 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:07:57,005 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:07:57,006 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:07:57,006 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:07:57,010 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:07:57,120 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:07:59,123 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:08:01,115 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:08:01,116 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:08:01,116 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:08:01,121 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:08:01,124 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:08:02,122 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:08:02,525 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 16:08:03,128 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:08:04,535 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:08:04,536 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:08:04,536 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:08:04,541 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:08:05,129 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:08:07,132 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:08:07,262 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:08:07,262 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:08:07,263 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:08:08,506 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:08:10,388 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:08:10,389 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:08:10,389 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:08:10,394 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:08:11,139 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:08:11,139 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:08:14,431 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:08:14,566 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:08:14,567 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:08:14,567 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:08:14,572 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:08:15,142 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:08:17,145 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:08:18,089 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:08:18,090 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:08:18,090 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:08:18,095 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:08:18,145 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:08:19,148 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:08:20,095 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:08:22,253 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:08:22,254 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:08:22,254 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:08:22,259 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:08:22,262 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:08:22,262 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:08:22,262 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:08:23,152 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:08:23,152 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:08:25,545 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:08:28,534 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:08:28,535 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:08:28,535 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:08:28,541 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:08:29,156 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:08:30,579 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:08:31,160 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:08:32,524 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 16:08:32,635 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:08:32,635 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:08:32,636 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:08:32,640 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:08:33,162 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:08:35,165 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:08:35,641 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:08:37,262 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:08:37,262 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:08:37,263 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:08:37,454 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:08:37,467 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:08:37,467 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:08:37,472 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:08:38,167 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:08:39,170 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:08:41,473 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:08:41,573 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:08:41,574 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:08:41,574 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:08:41,579 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:08:42,172 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:08:43,174 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:08:44,962 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:08:44,963 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:08:44,963 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:08:44,968 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:08:45,176 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:08:46,969 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:08:47,177 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:08:48,252 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:08:48,253 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:08:48,253 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:08:48,258 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:08:49,180 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:08:49,181 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:08:52,006 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:08:52,007 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:08:52,007 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:08:52,007 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:08:52,012 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:08:52,182 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:08:52,262 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:08:52,263 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:08:52,263 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:08:53,185 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:08:55,832 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:08:55,833 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:08:55,833 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:08:55,838 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:08:56,187 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:08:57,187 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:08:57,875 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:09:02,533 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 16:09:03,535 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:09:04,310 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:09:04,311 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:09:04,311 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:09:04,316 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:09:05,194 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:09:05,194 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:09:07,262 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:09:07,262 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:09:07,263 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:09:08,559 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:09:09,774 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:09:09,774 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:09:09,774 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:09:09,779 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:09:10,200 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:09:11,204 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:09:12,881 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:09:12,882 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:09:12,882 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:09:12,887 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:09:13,205 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:09:13,887 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:09:15,209 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:09:16,638 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:09:16,639 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:09:16,639 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:09:16,643 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:09:17,210 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:09:19,211 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:09:19,644 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:09:19,904 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:09:19,904 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:09:19,905 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:09:19,910 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:09:20,212 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:09:21,215 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:09:22,262 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:09:22,263 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:09:22,263 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:09:24,672 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:09:24,673 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:09:24,673 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:09:24,673 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:09:24,678 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:09:25,217 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:09:27,220 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:09:28,142 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:09:28,143 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:09:28,143 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:09:28,148 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:09:28,221 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:09:29,224 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:09:30,148 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:09:32,528 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 16:09:33,209 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:09:33,210 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:09:33,210 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:09:33,215 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:09:33,228 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:09:35,216 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:09:35,231 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:09:37,262 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:09:37,263 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:09:37,263 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:09:38,233 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:09:38,234 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:09:38,234 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:09:38,239 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:09:39,235 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:09:39,236 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:09:40,275 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:09:41,493 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:09:41,494 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:09:41,494 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:09:41,499 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:09:42,237 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:09:43,240 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:09:45,126 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:09:45,127 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:09:45,127 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:09:45,132 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:09:45,242 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:09:46,132 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:09:47,245 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:09:49,037 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:09:49,038 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:09:49,038 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:09:49,043 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:09:49,246 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:09:51,249 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:09:52,043 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:09:52,262 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:09:52,263 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:09:52,264 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:09:54,391 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:09:54,392 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:09:54,392 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:09:54,397 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:09:55,254 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:09:55,254 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:09:57,435 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:09:58,920 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:09:58,921 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:09:58,921 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:09:58,926 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:09:59,256 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:10:01,260 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:10:02,534 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 16:10:02,535 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:10:03,172 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:10:03,173 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:10:03,173 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:10:03,178 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:10:03,261 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:10:05,264 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:10:06,919 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:10:06,920 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:10:06,920 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:10:06,925 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:10:07,262 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:10:07,263 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:10:07,264 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:10:07,266 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:10:08,515 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:10:09,269 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:10:10,866 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:10:10,866 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:10:10,866 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:10:10,871 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:10:11,271 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:10:13,275 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:10:13,911 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:10:14,642 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:10:14,642 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:10:14,642 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:10:14,647 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:10:15,276 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:10:17,279 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:10:19,648 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:10:20,348 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:10:20,349 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:10:20,349 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:10:20,353 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:10:21,284 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:10:21,284 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:10:22,262 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:10:22,263 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:10:22,264 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:10:24,222 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:10:24,223 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:10:24,223 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:10:24,229 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:10:24,285 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:10:25,267 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:10:25,288 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:10:28,663 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:10:28,664 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:10:28,664 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:10:28,669 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:10:29,291 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:10:30,669 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:10:31,294 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:10:32,533 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 16:10:34,073 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:10:34,074 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:10:34,074 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:10:34,079 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:10:34,298 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:10:35,300 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:10:36,080 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:10:37,263 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:10:37,263 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:10:37,263 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:10:38,415 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:10:38,415 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:10:38,415 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:10:38,420 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:10:39,305 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:10:39,305 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:10:41,459 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:10:43,235 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:10:43,235 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:10:43,236 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:10:43,240 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:10:43,307 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:10:45,311 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:10:47,241 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:10:48,552 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:10:48,553 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:10:48,553 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:10:48,558 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:10:49,316 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:10:49,316 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:10:51,864 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:10:51,865 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:10:51,865 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:10:51,870 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:10:52,263 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:10:52,263 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:10:52,263 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:10:52,318 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:10:52,473 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:10:53,321 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:10:57,473 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:10:57,609 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:10:57,610 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:10:57,610 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:10:57,615 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:10:58,324 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:10:59,327 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:11:00,959 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:11:00,959 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:11:00,959 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:11:00,964 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:11:01,328 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:11:02,535 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 16:11:02,536 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:11:03,331 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:11:04,512 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:11:04,513 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:11:04,513 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:11:04,518 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:11:05,334 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:11:05,334 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:11:07,263 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:11:07,263 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:11:07,263 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:11:08,244 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:11:09,380 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:11:09,381 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:11:09,381 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:11:09,386 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:11:10,337 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:11:11,341 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:11:13,423 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:11:14,788 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:11:14,789 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:11:14,789 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:11:14,794 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:11:15,344 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:11:17,347 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:11:18,795 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:11:19,334 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:11:19,335 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:11:19,335 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:11:19,340 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:11:19,348 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:11:21,351 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:11:22,263 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:11:22,263 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:11:22,264 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:11:24,522 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:11:24,704 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:11:24,705 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:11:24,705 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:11:24,710 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:11:25,354 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:11:27,358 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:11:27,844 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:11:27,845 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:11:27,845 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:11:27,851 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:11:28,358 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:11:29,361 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:11:29,851 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:11:31,621 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:11:31,622 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:11:31,622 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:11:31,627 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:11:32,363 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:11:32,537 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 16:11:33,367 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:11:35,102 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:11:35,103 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:11:35,103 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:11:35,104 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:11:35,109 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:11:35,369 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:11:37,263 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:11:37,263 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:11:37,264 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:11:37,372 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:11:40,558 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:11:41,276 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:11:41,277 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:11:41,277 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:11:41,282 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:11:41,374 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:11:43,377 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:11:46,319 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:11:47,366 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:11:47,367 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:11:47,367 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:11:47,372 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:11:47,380 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:11:49,384 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:11:51,373 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:11:51,610 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:11:51,611 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:11:51,611 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:11:51,616 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:11:52,263 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:11:52,264 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:11:52,264 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:11:52,385 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:11:53,388 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:11:55,453 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:11:55,454 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:11:55,454 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:11:55,459 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:11:56,390 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:11:56,494 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:11:57,391 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:11:59,649 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:11:59,650 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:11:59,650 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:11:59,655 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:12:00,392 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:12:01,395 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:12:01,656 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:12:02,540 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 16:12:05,144 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:12:05,145 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:12:05,145 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:12:05,150 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:12:05,398 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:12:07,150 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:12:07,263 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:12:07,263 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:12:07,264 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:12:07,401 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:12:08,543 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:12:08,543 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:12:08,543 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:12:08,548 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:12:09,404 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:12:09,404 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:12:12,587 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:12:12,885 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:12:12,886 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:12:12,887 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:12:12,891 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:12:13,408 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:12:15,411 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:12:17,892 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:12:18,525 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:12:18,526 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:12:18,526 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:12:18,531 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:12:19,416 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:12:19,416 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:12:22,263 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:12:22,264 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:12:22,265 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:12:22,694 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:12:22,695 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:12:22,695 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:12:22,700 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:12:23,420 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:12:23,420 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:12:23,739 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:12:26,923 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:12:26,923 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:12:26,923 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:12:26,928 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:12:27,423 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:12:28,929 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:12:29,426 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:12:32,545 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 16:12:33,196 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:12:33,197 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:12:33,197 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:12:33,202 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:12:33,429 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:12:34,202 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:12:35,432 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:12:37,263 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:12:37,264 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:12:37,264 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:12:37,510 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:12:37,511 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:12:37,511 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:12:37,516 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:12:38,434 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:12:39,437 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:12:39,555 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:12:41,817 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:12:41,817 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:12:41,818 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:12:41,823 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:12:42,439 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:12:43,441 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:12:44,823 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:12:45,834 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:12:45,835 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:12:45,836 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:12:45,841 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:12:46,444 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:12:47,455 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:12:49,841 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:12:50,514 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:12:50,514 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:12:50,515 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:12:50,519 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:12:51,460 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:12:51,460 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:12:52,263 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:12:52,264 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:12:52,265 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:12:53,888 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:12:53,888 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:12:53,889 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:12:53,894 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:12:54,462 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:12:54,931 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:12:55,464 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:12:57,639 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:12:57,640 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:12:57,640 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:12:57,645 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:12:58,466 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:12:59,469 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:13:00,645 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:13:00,993 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:13:00,994 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:13:00,994 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:13:00,999 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:13:01,470 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:13:02,544 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 16:13:03,473 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:13:05,624 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:13:05,625 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:13:05,625 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:13:05,630 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:13:06,475 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:13:06,630 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:13:07,263 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:13:07,264 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:13:07,264 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:13:07,478 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:13:09,497 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:13:09,497 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:13:09,497 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:13:09,502 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:13:10,479 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:13:11,483 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:13:12,539 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:13:13,282 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:13:13,282 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:13:13,283 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:13:13,287 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:13:13,485 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:13:15,488 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:13:16,683 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:13:16,684 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:13:16,684 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:13:16,689 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:13:17,491 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:13:17,491 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:13:17,689 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:13:20,714 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:13:20,715 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:13:20,715 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:13:20,720 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:13:21,495 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:13:21,496 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:13:22,264 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:13:22,264 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:13:22,265 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:13:23,560 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:13:25,921 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:13:25,921 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:13:25,922 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:13:25,926 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:13:26,498 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:13:27,501 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:13:28,963 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:13:29,180 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:13:29,182 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:13:29,182 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:13:29,187 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:13:29,502 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:13:31,504 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:13:32,550 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 16:13:34,551 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:13:37,054 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:13:37,055 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:13:37,056 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:13:37,061 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:13:37,264 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:13:37,264 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:13:37,265 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:13:37,508 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:13:39,512 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:13:40,414 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:13:40,415 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:13:40,415 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:13:40,415 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:13:40,421 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:13:40,512 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:13:41,515 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:13:44,355 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:13:44,356 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:13:44,356 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:13:44,361 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:13:44,517 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:13:45,520 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:13:46,362 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:13:47,848 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:13:47,849 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:13:47,849 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:13:47,854 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:13:48,521 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:13:49,522 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:13:51,228 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:13:51,229 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:13:51,229 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:13:51,234 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:13:51,523 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:13:52,235 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:13:52,264 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:13:52,264 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:13:52,265 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:13:53,526 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:13:56,365 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:13:56,365 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:13:56,365 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:13:56,374 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:13:56,528 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:13:57,407 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:13:57,531 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:14:00,628 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:14:00,629 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:14:00,629 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:14:00,634 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:14:01,533 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:14:01,533 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:14:02,549 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 16:14:02,550 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:14:04,327 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:14:04,327 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:14:04,327 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:14:04,332 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:14:04,535 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:14:05,537 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:14:07,264 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:14:07,264 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:14:07,266 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:14:07,685 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:14:07,686 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:14:07,686 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:14:07,686 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:14:07,691 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:14:08,539 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:14:09,542 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:14:11,698 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:14:11,699 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:14:11,699 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:14:11,704 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:14:12,544 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:14:12,705 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:14:13,547 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:14:15,792 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:14:15,793 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:14:15,793 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:14:15,798 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:14:16,549 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:14:17,552 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:14:17,798 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:14:22,264 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:14:22,264 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:14:22,266 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:14:23,467 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:14:24,301 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:14:24,301 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:14:24,301 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:14:24,306 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:14:24,556 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:14:25,558 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:14:28,784 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:14:28,785 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:14:28,785 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:14:28,786 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:14:28,790 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:14:29,563 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:14:29,563 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:14:32,554 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 16:14:33,593 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:14:33,594 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:14:33,594 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:14:33,599 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:14:34,567 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:14:34,599 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:14:35,570 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:14:37,264 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:14:37,264 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:14:37,266 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:14:37,355 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:14:37,733 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:14:37,733 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:14:37,738 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:14:38,572 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:14:39,574 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:14:39,738 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:14:42,125 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:14:42,126 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:14:42,126 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:14:42,132 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:14:42,576 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:14:43,579 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:14:45,132 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:14:48,825 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:14:48,826 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:14:48,826 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:14:48,831 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:14:49,585 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:14:49,585 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:14:50,831 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:14:52,264 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:14:52,265 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:14:52,266 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:14:53,127 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:14:53,128 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:14:53,128 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:14:53,133 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:14:53,587 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:14:55,590 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:14:56,171 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:14:56,830 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:14:56,831 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:14:56,831 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:14:56,836 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:14:57,593 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:14:57,593 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:15:00,783 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:15:00,783 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:15:00,784 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:15:00,789 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:15:01,596 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:15:01,596 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:15:01,789 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:15:02,553 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 16:15:06,609 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:15:06,610 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:15:06,610 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:15:06,615 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:15:07,264 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:15:07,265 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:15:07,266 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:15:07,491 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:15:07,601 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:15:07,601 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:15:12,491 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:15:15,709 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:15:15,710 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:15:15,710 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:15:15,715 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:15:16,608 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:15:17,608 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:15:17,751 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:15:18,595 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:15:18,595 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:15:18,595 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:15:18,600 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:15:18,609 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:15:19,611 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:15:22,264 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:15:22,265 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:15:22,266 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:15:23,561 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:15:23,825 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:15:23,825 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:15:23,825 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:15:23,830 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:15:24,614 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:15:25,617 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:15:28,407 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:15:28,408 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:15:28,408 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:15:28,413 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:15:28,618 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:15:29,413 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:15:29,621 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:15:32,558 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 16:15:34,559 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:15:36,979 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:15:36,980 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:15:36,980 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:15:36,985 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:15:37,264 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:15:37,265 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:15:37,266 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:15:37,629 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:15:37,630 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:15:40,542 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:15:40,708 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:15:40,709 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:15:40,709 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:15:40,714 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:15:41,634 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:15:41,634 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:15:45,209 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:15:45,210 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:15:45,210 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:15:45,215 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:15:45,636 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:15:46,215 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:15:47,640 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:15:48,862 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:15:48,863 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:15:48,863 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:15:48,868 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:15:49,641 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:15:51,644 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:15:51,868 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:15:52,265 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:15:52,265 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:15:52,266 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:15:52,953 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:15:52,954 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:15:52,954 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:15:52,959 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:15:53,646 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:15:55,649 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:15:56,999 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:15:59,392 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:15:59,394 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:15:59,395 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:15:59,400 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:15:59,651 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:16:01,654 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:16:02,401 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:16:02,557 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 16:16:02,974 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:16:02,975 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:16:02,975 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:16:02,980 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:16:03,655 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:16:05,659 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:16:07,265 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:16:07,265 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:16:07,266 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:16:08,132 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:16:08,339 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:16:08,339 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:16:08,339 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:16:08,345 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:16:08,660 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:16:09,663 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:16:12,864 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:16:12,865 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:16:12,866 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:16:12,870 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:16:13,667 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:16:13,871 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:16:15,670 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:16:16,184 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:16:16,185 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:16:16,185 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:16:16,190 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:16:16,670 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:16:17,673 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:16:19,190 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:16:19,757 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:16:19,758 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:16:19,758 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:16:19,763 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:16:20,675 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:16:21,677 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:16:22,265 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:16:22,265 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:16:22,267 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:16:23,250 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:16:23,250 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:16:23,250 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:16:23,255 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:16:23,678 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:16:24,295 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:16:25,682 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:16:29,295 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:16:30,708 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:16:30,709 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:16:30,709 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:16:30,714 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:16:31,686 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:16:32,560 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 16:16:33,690 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:16:34,561 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:16:34,837 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:16:34,837 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:16:34,838 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:16:34,843 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:16:35,691 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:16:37,265 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:16:37,265 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:16:37,267 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:16:37,694 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:16:37,935 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:16:39,005 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:16:39,005 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:16:39,010 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:16:39,696 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:16:40,011 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:16:41,699 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:16:44,631 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:16:44,632 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:16:44,632 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:16:44,637 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:16:44,700 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:16:45,638 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:16:47,704 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:16:48,034 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:16:48,034 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:16:48,035 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:16:48,040 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:16:48,705 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:16:49,708 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:16:51,040 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:16:52,265 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:16:52,265 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:16:52,267 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:16:53,125 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:16:53,125 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:16:53,126 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:16:53,130 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:16:53,710 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:16:55,713 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:16:56,170 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:16:57,246 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:16:57,247 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:16:57,247 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:16:57,252 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:16:57,714 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:16:59,717 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:17:01,252 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:17:01,784 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:17:01,785 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:17:01,785 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:17:01,790 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:17:02,562 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 16:17:02,719 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:17:03,722 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:17:06,564 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:17:07,265 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:17:07,265 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:17:07,267 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:17:10,164 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:17:10,164 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:17:10,165 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:17:10,170 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:17:10,726 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:17:11,729 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:17:12,207 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:17:13,385 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:17:13,385 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:17:13,386 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:17:13,390 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:17:13,731 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:17:15,734 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:17:17,391 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:17:18,072 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:17:18,073 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:17:18,073 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:17:18,078 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:17:18,736 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:17:19,737 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:17:21,300 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:17:21,301 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:17:21,301 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:17:21,306 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:17:21,738 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:17:22,265 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:17:22,266 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:17:22,266 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:17:22,567 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:17:23,741 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:17:27,568 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:17:29,709 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:17:29,710 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:17:29,710 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:17:29,715 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:17:29,745 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:17:31,748 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:17:32,564 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 16:17:33,565 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:17:34,681 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:17:34,682 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:17:34,682 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:17:34,687 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:17:34,751 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:17:37,265 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:17:37,266 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:17:37,267 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:17:37,755 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:17:39,045 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:17:40,258 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:17:40,259 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:17:40,259 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:17:40,264 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:17:40,757 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:17:41,760 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:17:44,303 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:17:47,071 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:17:47,072 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:17:47,072 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:17:47,077 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:17:47,764 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:17:49,769 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:17:50,078 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:17:50,468 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:17:50,468 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:17:50,469 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:17:50,474 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:17:50,770 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:17:51,772 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:17:52,265 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:17:52,266 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:17:52,267 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:17:54,537 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:17:54,538 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:17:54,538 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:17:54,543 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:17:54,774 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:17:55,579 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:17:55,774 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:17:58,635 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:17:58,636 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:17:58,636 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:17:58,641 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:17:58,776 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:17:59,779 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:18:00,641 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:18:02,572 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 16:18:02,762 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:18:02,763 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:18:02,763 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:18:02,767 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:18:02,781 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:18:05,768 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:18:05,784 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:18:07,133 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:18:07,134 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:18:07,134 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:18:07,138 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:18:07,265 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:18:07,266 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:18:07,267 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:18:07,786 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:18:09,789 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:18:10,445 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:18:10,446 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:18:10,446 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:18:10,451 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:18:10,789 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:18:11,491 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:18:11,793 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:18:14,607 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:18:14,608 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:18:14,608 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:18:14,613 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:18:14,795 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:18:15,798 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:18:16,613 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:18:18,762 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:18:18,762 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:18:18,762 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:18:18,767 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:18:18,799 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:18:21,768 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:18:21,803 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:18:22,152 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:18:22,153 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:18:22,153 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:18:22,158 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:18:22,266 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:18:22,266 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:18:22,267 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:18:22,804 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:18:23,806 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:18:25,726 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:18:25,727 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:18:25,727 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:18:25,732 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:18:25,807 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:18:26,771 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:18:27,810 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:18:30,186 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:18:30,187 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:18:30,187 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:18:30,192 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:18:30,813 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:18:31,815 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:18:32,192 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:18:32,569 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 16:18:35,979 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:18:35,980 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:18:35,980 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:18:35,985 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:18:36,819 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:18:37,266 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:18:37,266 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:18:37,266 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:18:37,599 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:18:37,822 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:18:42,599 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:18:43,944 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:18:43,944 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:18:43,944 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:18:43,949 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:18:44,826 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:18:45,829 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:18:47,987 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:18:52,266 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:18:52,266 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:18:52,266 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:18:52,441 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:18:52,572 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:18:52,572 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:18:52,577 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:18:52,833 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:18:53,578 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:18:53,836 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:18:56,214 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:18:56,214 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:18:56,214 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:18:56,219 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:18:56,838 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:18:57,840 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:18:59,220 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:18:59,621 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:18:59,621 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:18:59,622 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:18:59,627 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:18:59,842 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:19:01,843 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:19:02,574 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 16:19:04,388 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:19:04,389 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:19:04,389 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:19:04,389 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:19:04,394 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:19:04,844 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:19:05,847 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:19:07,266 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:19:07,266 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:19:07,266 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:19:09,745 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:19:10,626 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:19:10,627 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:19:10,627 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:19:10,632 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:19:10,850 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:19:11,853 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:19:14,107 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:19:14,108 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:19:14,108 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:19:14,113 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:19:14,855 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:19:15,114 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:19:15,858 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:19:18,384 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:19:18,385 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:19:18,385 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:19:18,390 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:19:18,860 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:19:19,862 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:19:20,391 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:19:22,266 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:19:22,266 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:19:22,266 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:19:22,885 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:19:22,886 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:19:22,886 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:19:22,891 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:19:23,865 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:19:25,868 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:19:25,927 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:19:27,643 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:19:27,643 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:19:27,643 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:19:27,649 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:19:27,869 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:19:29,873 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:19:31,649 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:19:32,573 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 16:19:35,944 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:19:35,944 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:19:35,944 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:19:35,949 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:19:36,879 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:19:36,950 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:19:37,266 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:19:37,266 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:19:37,268 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:19:37,879 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:19:39,626 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:19:39,627 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:19:39,627 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:19:39,632 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:19:39,880 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:19:41,884 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:19:42,671 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:19:43,084 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:19:43,085 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:19:43,085 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:19:43,090 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:19:43,885 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:19:45,888 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:19:48,090 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:19:48,238 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:19:48,239 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:19:48,239 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:19:48,244 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:19:48,890 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:19:49,893 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:19:51,896 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:19:51,897 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:19:51,897 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:19:51,903 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:19:52,266 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:19:52,266 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:19:52,267 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:19:52,895 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:19:53,519 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:19:53,897 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:19:58,520 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:20:00,299 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:20:00,300 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:20:00,300 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:20:00,305 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:20:00,902 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:20:01,904 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:20:02,576 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 16:20:03,577 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:20:04,371 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:20:04,375 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:20:04,375 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:20:04,380 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:20:04,906 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:20:05,909 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:20:07,266 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:20:07,267 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:20:07,267 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:20:08,037 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:20:08,038 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:20:08,038 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:20:08,043 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:20:08,910 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:20:09,083 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:20:09,913 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:20:14,083 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:20:15,496 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:20:15,497 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:20:15,497 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:20:15,501 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:20:15,918 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:20:17,921 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:20:19,109 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:20:19,110 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:20:19,110 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:20:19,110 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:20:19,115 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:20:19,922 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:20:21,925 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:20:22,266 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:20:22,267 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:20:22,267 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:20:22,879 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:20:22,880 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:20:22,880 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:20:22,885 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:20:22,926 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:20:24,925 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:20:25,930 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:20:27,821 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:20:27,822 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:20:27,822 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:20:27,827 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:20:27,931 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:20:29,934 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:20:30,828 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:20:31,944 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:20:31,945 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:20:31,945 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:20:31,950 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:20:32,578 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 16:20:32,937 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:20:33,941 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:20:36,069 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:20:36,070 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:20:36,071 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:20:36,071 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:20:36,076 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:20:36,943 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:20:37,388 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:20:37,388 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:20:37,388 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:20:37,945 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:20:40,731 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:20:40,732 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:20:40,732 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:20:40,737 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:20:40,947 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:20:41,775 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:20:41,950 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:20:44,427 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:20:44,428 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:20:44,429 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:20:44,434 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:20:44,952 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:20:45,955 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:20:47,434 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:20:48,692 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:20:48,693 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:20:48,693 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:20:48,697 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:20:48,956 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:20:49,959 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:20:52,388 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:20:52,388 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:20:52,388 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:20:52,663 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:20:52,980 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:20:52,981 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:20:52,981 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:20:52,986 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:20:53,961 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:20:55,965 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:20:57,807 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:20:57,808 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:20:57,808 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:20:57,808 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:20:57,813 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:20:57,966 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:20:59,969 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:21:01,402 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:21:01,403 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:21:01,403 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:21:01,408 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:21:01,970 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:21:02,580 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 16:21:03,582 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:21:03,974 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:21:05,760 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:21:05,760 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:21:05,761 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:21:05,765 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:21:05,975 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:21:07,388 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:21:07,388 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:21:07,388 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:21:07,978 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:21:09,226 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:21:09,969 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:21:09,970 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:21:09,970 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:21:09,975 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:21:09,979 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:21:11,983 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:21:13,751 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:21:13,752 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:21:13,752 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:21:13,757 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:21:13,985 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:21:14,758 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:21:15,988 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:21:18,387 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:21:18,388 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:21:18,388 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:21:18,393 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:21:18,990 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:21:19,992 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:21:20,394 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:21:22,388 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:21:22,388 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:21:22,388 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:21:23,201 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:21:23,202 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:21:23,202 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:21:23,207 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:21:23,995 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:21:25,998 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:21:26,246 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:21:27,577 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:21:27,578 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:21:27,578 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:21:27,583 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:21:27,999 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:21:30,003 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:21:31,584 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:21:31,937 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:21:31,938 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:21:31,938 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:21:31,943 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:21:32,004 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:21:32,583 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 16:21:34,009 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:21:36,584 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:21:36,870 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:21:36,871 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:21:36,871 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:21:36,883 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:21:37,011 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:21:37,388 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:21:37,388 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:21:37,389 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:21:38,013 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:21:41,646 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:21:41,855 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:21:41,855 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:21:41,855 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:21:41,860 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:21:42,015 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:21:44,019 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:21:46,898 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:21:47,446 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:21:47,446 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:21:47,447 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:21:47,451 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:21:48,022 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:21:50,025 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:21:51,728 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:21:51,729 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:21:51,729 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:21:51,734 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:21:52,026 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:21:52,388 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:21:52,388 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:21:52,388 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:21:52,632 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:21:54,029 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:21:56,058 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:21:56,059 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:21:56,059 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:21:56,064 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:21:57,031 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:21:58,034 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:21:58,102 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:22:02,585 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 16:22:03,586 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:22:04,417 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:22:04,417 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:22:04,418 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:22:04,423 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:22:05,039 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:22:06,041 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:22:07,388 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:22:07,389 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:22:07,389 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:22:08,685 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:22:08,797 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:22:08,798 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:22:08,798 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:22:08,803 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:22:09,043 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:22:10,045 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:22:13,842 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:22:14,401 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:22:14,402 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:22:14,402 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:22:14,407 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:22:15,050 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:22:16,052 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:22:17,943 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:22:17,944 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:22:17,944 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:22:17,949 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:22:18,053 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:22:18,950 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:22:20,057 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:22:21,478 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:22:21,479 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:22:21,479 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:22:21,484 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:22:22,058 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:22:22,388 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:22:22,389 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:22:22,389 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:22:24,061 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:22:24,644 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:22:25,227 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:22:25,227 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:22:25,227 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:22:25,232 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:22:26,062 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:22:28,065 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:22:30,270 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:22:32,205 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:22:32,206 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:22:32,206 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:22:32,211 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:22:32,590 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 16:22:33,069 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:22:34,072 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:22:35,591 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:22:36,037 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:22:36,038 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:22:36,038 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:22:36,047 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:22:36,074 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:22:37,388 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:22:37,389 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:22:37,389 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:22:38,077 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:22:40,218 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:22:40,218 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:22:40,218 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:22:40,223 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:22:41,079 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:22:41,262 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:22:42,081 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:22:44,620 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:22:44,621 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:22:44,621 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:22:44,626 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:22:45,084 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:22:46,086 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:22:46,627 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:22:49,010 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:22:49,011 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:22:49,011 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:22:49,016 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:22:49,088 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:22:52,016 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:22:52,092 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:22:52,388 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:22:52,389 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:22:52,389 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:22:52,726 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:22:52,726 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:22:52,727 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:22:52,732 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:22:53,092 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:22:54,095 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:22:57,770 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:22:58,402 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:22:58,402 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:22:58,403 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:22:58,408 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:22:59,098 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:23:00,101 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:23:02,389 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:23:02,390 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:23:02,390 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:23:02,395 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:23:02,589 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 16:23:03,103 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:23:03,591 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:23:04,105 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:23:05,868 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:23:05,869 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:23:05,869 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:23:05,874 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:23:06,106 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:23:07,388 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:23:07,389 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:23:07,389 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:23:08,108 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:23:08,790 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:23:10,142 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:23:10,143 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:23:10,143 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:23:10,148 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:23:11,109 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:23:12,113 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:23:14,186 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:23:19,186 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:23:19,192 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:23:19,192 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:23:19,192 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:23:19,197 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:23:20,118 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:23:22,121 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:23:22,388 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:23:22,389 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:23:22,389 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:23:22,983 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:23:22,984 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:23:22,984 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:23:22,989 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:23:23,122 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:23:24,124 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:23:25,026 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:23:26,006 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:23:26,007 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:23:26,007 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:23:26,012 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:23:26,125 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:23:28,129 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:23:30,573 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:23:30,574 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:23:30,574 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:23:30,574 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:23:30,579 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:23:31,131 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:23:32,133 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:23:32,594 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 16:23:34,210 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:23:34,211 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:23:34,211 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:23:34,216 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:23:35,136 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:23:36,139 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:23:36,217 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:23:37,389 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:23:37,389 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:23:37,390 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:23:37,846 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:23:37,847 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:23:37,847 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:23:37,852 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:23:38,140 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:23:40,144 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:23:41,465 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:23:41,466 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:23:41,466 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:23:41,467 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:23:41,471 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:23:42,145 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:23:44,148 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:23:46,472 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:23:50,001 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:23:50,002 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:23:50,002 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:23:50,008 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:23:50,152 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:23:52,008 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:23:52,155 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:23:52,389 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:23:52,389 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:23:52,390 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:23:53,134 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:23:53,134 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:23:53,134 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:23:53,139 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:23:53,156 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:23:56,160 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:23:57,178 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:23:57,635 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:23:57,636 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:23:57,636 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:23:57,641 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:23:58,161 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:24:00,164 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:24:01,461 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:24:01,462 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:24:01,462 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:24:01,467 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:24:02,165 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:24:02,467 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:24:02,593 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 16:24:04,169 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:24:04,869 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:24:04,870 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:24:04,870 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:24:04,875 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:24:05,169 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:24:06,172 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:24:07,389 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:24:07,389 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:24:07,390 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:24:08,266 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:24:08,826 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:24:08,826 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:24:08,826 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:24:08,831 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:24:09,174 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:24:10,176 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:24:12,363 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:24:12,364 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:24:12,364 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:24:12,369 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:24:13,179 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:24:13,370 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:24:14,180 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:24:16,896 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:24:16,896 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:24:16,897 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:24:16,901 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:24:17,182 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:24:18,184 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:24:18,902 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:24:20,146 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:24:20,147 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:24:20,147 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:24:20,152 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:24:20,186 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:24:22,189 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:24:22,389 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:24:22,389 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:24:22,390 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:24:24,478 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:24:24,479 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:24:24,479 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:24:24,479 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:24:24,484 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:24:25,191 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:24:26,193 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:24:29,522 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:24:31,194 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:24:31,195 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:24:31,195 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:24:31,200 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:24:32,197 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:24:32,599 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 16:24:34,202 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:24:34,601 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:24:35,097 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:24:35,098 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:24:35,098 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:24:35,103 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:24:35,202 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:24:36,205 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:24:37,389 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:24:37,389 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:24:37,391 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:24:40,167 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:24:44,130 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:24:44,130 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:24:44,131 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:24:44,135 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:24:44,210 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:24:45,174 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:24:46,213 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:24:47,399 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:24:47,400 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:24:47,400 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:24:47,404 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:24:48,215 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:24:50,216 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:24:50,405 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:24:50,826 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:24:50,827 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:24:50,827 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:24:50,832 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:24:51,216 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:24:52,219 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:24:52,389 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:24:52,389 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:24:52,391 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:24:54,195 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:24:54,195 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:24:54,196 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:24:54,201 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:24:54,220 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:24:56,223 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:24:56,238 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:24:58,705 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:24:58,706 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:24:58,706 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:24:58,711 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:24:59,225 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:25:00,228 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:25:01,712 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:25:01,987 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:25:01,988 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:25:01,988 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:25:01,993 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:25:02,229 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:25:02,598 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 16:25:04,232 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:25:06,900 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:25:06,901 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:25:06,901 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:25:06,901 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:25:06,906 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:25:07,234 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:25:07,389 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:25:07,389 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:25:07,391 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:25:08,236 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:25:10,844 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:25:10,845 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:25:10,845 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:25:10,850 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:25:11,238 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:25:12,241 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:25:12,886 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:25:17,886 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:25:19,176 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:25:19,176 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:25:19,177 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:25:19,182 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:25:19,246 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:25:20,249 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:25:22,389 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:25:22,390 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:25:22,391 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:25:22,819 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:25:22,820 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:25:22,820 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:25:22,825 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:25:23,250 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:25:23,862 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:25:24,253 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:25:28,446 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:25:28,447 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:25:28,447 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:25:28,452 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:25:29,256 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:25:29,453 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:25:30,259 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:25:32,600 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 16:25:32,767 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:25:32,768 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:25:32,768 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:25:32,773 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:25:33,261 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:25:34,265 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:25:34,773 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:25:37,389 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:25:37,390 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:25:37,391 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:25:37,968 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:25:37,969 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:25:37,969 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:25:37,974 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:25:38,268 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:25:40,010 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:25:40,271 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:25:42,133 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:25:42,137 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:25:42,137 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:25:42,142 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:25:42,272 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:25:44,276 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:25:45,142 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:25:49,635 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:25:49,635 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:25:49,636 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:25:49,640 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:25:50,280 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:25:50,641 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:25:52,283 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:25:52,389 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:25:52,390 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:25:52,391 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:25:53,244 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:25:53,244 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:25:53,244 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:25:53,249 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:25:53,284 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:25:56,286 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:25:56,289 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:25:56,813 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:25:56,814 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:25:56,815 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:25:56,820 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:25:57,290 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:25:58,292 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:26:01,821 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:26:02,603 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 16:26:05,299 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:26:05,300 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:26:05,300 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:26:05,305 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:26:06,297 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:26:07,305 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:26:07,389 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:26:07,390 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:26:07,391 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:26:08,300 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:26:10,645 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:26:10,646 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:26:10,646 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:26:10,651 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:26:11,302 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:26:12,305 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:26:12,690 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:26:14,261 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:26:14,262 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:26:14,262 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:26:14,267 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:26:14,307 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:26:16,310 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:26:18,268 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:26:19,322 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:26:19,323 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:26:19,323 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:26:19,328 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:26:20,312 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:26:22,315 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:26:22,389 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:26:22,390 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:26:22,391 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:26:23,686 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:26:25,617 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:26:25,617 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:26:25,618 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:26:25,623 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:26:26,318 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:26:28,321 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:26:29,658 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:26:31,480 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:26:31,481 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:26:31,481 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:26:31,486 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:26:32,324 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:26:32,605 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 16:26:34,328 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:26:34,681 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:26:34,682 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:26:34,682 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:26:34,682 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:26:34,687 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:26:35,329 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:26:36,332 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:26:37,390 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:26:37,390 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:26:37,390 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:26:39,832 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:26:43,120 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:26:43,120 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:26:43,121 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:26:43,126 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:26:43,336 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:26:44,339 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:26:45,162 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:26:46,777 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:26:46,777 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:26:46,778 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:26:46,783 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:26:47,342 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:26:48,344 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:26:50,783 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:26:52,310 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:26:52,311 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:26:52,311 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:26:52,316 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:26:52,347 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:26:52,390 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:26:52,391 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:26:52,392 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:26:54,350 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:26:55,787 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:26:55,787 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:26:55,788 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:26:55,788 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:26:55,793 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:26:56,352 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:26:58,355 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:27:00,100 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:27:00,101 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:27:00,101 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:27:00,106 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:27:00,356 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:27:01,107 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:27:02,359 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:27:02,612 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 16:27:04,546 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:27:04,547 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:27:04,547 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:27:04,552 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:27:05,361 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:27:06,364 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:27:06,552 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:27:07,390 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:27:07,390 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:27:07,391 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:27:11,970 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:27:13,049 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:27:13,049 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:27:13,050 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:27:13,055 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:27:13,369 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:27:14,372 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:27:17,090 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:27:21,516 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:27:21,517 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:27:21,517 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:27:21,522 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:27:22,377 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:27:22,390 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:27:22,390 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:27:22,390 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:27:22,642 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:27:24,380 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:27:27,642 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:27:28,882 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:27:28,882 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:27:28,883 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:27:28,887 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:27:29,383 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:27:30,386 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:27:32,609 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 16:27:32,645 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:27:32,646 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:27:32,646 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:27:32,646 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:27:32,651 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:27:33,388 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:27:34,393 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:27:36,116 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:27:36,117 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:27:36,117 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:27:36,122 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:27:36,394 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:27:37,390 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:27:37,390 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:27:37,392 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:27:38,397 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:27:38,616 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:27:41,277 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:27:41,278 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:27:41,278 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:27:41,284 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:27:41,399 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:27:42,401 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:27:44,322 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:27:44,370 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:27:44,371 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:27:44,371 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:27:44,376 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:27:44,403 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:27:46,406 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:27:48,664 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:27:48,665 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:27:48,665 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:27:48,670 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:27:49,408 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:27:49,670 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:27:50,411 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:27:51,972 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:27:51,973 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:27:51,973 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:27:51,978 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:27:52,390 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:27:52,390 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:27:52,391 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:27:52,412 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:27:54,413 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:27:55,636 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:27:55,908 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:27:55,909 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:27:55,909 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:27:55,914 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:27:56,414 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:27:58,419 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:27:59,163 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:27:59,164 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:27:59,164 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:27:59,169 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:27:59,420 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:28:00,422 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:28:01,170 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:28:02,615 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 16:28:03,037 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:28:03,038 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:28:03,038 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:28:03,043 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:28:03,424 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:28:04,428 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:28:07,044 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:28:07,131 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:28:07,131 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:28:07,132 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:28:07,137 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:28:07,390 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:28:07,390 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:28:07,391 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:28:07,429 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:28:08,433 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:28:11,970 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:28:11,970 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:28:11,971 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:28:11,976 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:28:12,437 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:28:13,014 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:28:14,442 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:28:15,517 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:28:15,517 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:28:15,518 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:28:15,522 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:28:16,444 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:28:18,448 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:28:18,523 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:28:20,927 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:28:20,928 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:28:20,928 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:28:20,933 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:28:21,450 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:28:22,390 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:28:22,391 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:28:22,392 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:28:22,455 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:28:23,685 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:28:24,803 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:28:24,804 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:28:24,804 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:28:24,809 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:28:25,457 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:28:26,460 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:28:28,691 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:28:28,692 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:28:28,692 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:28:28,692 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:28:28,697 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:28:29,462 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:28:30,464 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:28:32,613 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 16:28:33,218 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:28:33,219 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:28:33,219 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:28:33,224 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:28:33,467 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:28:34,224 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:28:34,473 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:28:36,758 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:28:36,758 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:28:36,759 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:28:36,764 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:28:37,390 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:28:37,391 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:28:37,392 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:28:37,475 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:28:38,477 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:28:39,577 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:28:40,712 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:28:40,712 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:28:40,713 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:28:40,718 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:28:41,479 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:28:42,482 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:28:44,754 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:28:49,008 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:28:49,008 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:28:49,009 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:28:49,014 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:28:49,491 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:28:50,014 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:28:50,495 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:28:52,390 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:28:52,391 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:28:52,392 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:28:52,681 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:28:52,685 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:28:52,685 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:28:52,690 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:28:53,497 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:28:54,500 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:28:55,691 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:28:57,917 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:28:57,918 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:28:57,918 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:28:57,923 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:28:58,504 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:29:00,509 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:29:00,958 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:29:02,619 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 16:29:03,136 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:29:03,137 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:29:03,137 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:29:03,142 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:29:03,511 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:29:04,515 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:29:06,143 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:29:07,390 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:29:07,391 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:29:07,392 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:29:10,195 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:29:10,196 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:29:10,196 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:29:10,202 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:29:10,521 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:29:11,238 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:29:12,527 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:29:14,939 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:29:14,940 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:29:14,940 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:29:14,945 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:29:15,529 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:29:16,532 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:29:16,946 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:29:18,301 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:29:18,302 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:29:18,302 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:29:18,307 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:29:18,534 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:29:20,538 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:29:21,880 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:29:21,880 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:29:21,881 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:29:21,885 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:29:22,391 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:29:22,391 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:29:22,392 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:29:22,540 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:29:22,941 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:29:24,545 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:29:27,942 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:29:30,365 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:29:30,366 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:29:30,367 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:29:30,372 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:29:30,551 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:29:32,555 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:29:32,618 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 16:29:33,619 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:29:34,023 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:29:34,024 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:29:34,024 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:29:34,030 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:29:34,558 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:29:36,561 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:29:37,391 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:29:37,391 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:29:37,393 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:29:39,382 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:29:39,897 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:29:39,898 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:29:39,898 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:29:39,903 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:29:40,565 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:29:42,570 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:29:44,134 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:29:44,135 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:29:44,135 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:29:44,140 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:29:44,572 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:29:45,140 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:29:46,576 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:29:48,742 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:29:48,743 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:29:48,743 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:29:48,748 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:29:49,578 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:29:50,582 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:29:50,749 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:29:52,391 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:29:52,391 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:29:52,392 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:29:53,070 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:29:53,071 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:29:53,071 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:29:53,076 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:29:53,584 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:29:54,587 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:29:56,114 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:29:57,700 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:29:57,701 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:29:57,701 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:29:57,706 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:29:58,591 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:30:00,596 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:30:01,486 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:30:01,487 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:30:01,487 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:30:01,487 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:30:01,492 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:30:01,597 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:30:02,599 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:30:02,620 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 16:30:06,279 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:30:06,280 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:30:06,280 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:30:06,285 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:30:06,602 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:30:07,286 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:30:07,391 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:30:07,391 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:30:07,392 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:30:08,607 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:30:10,003 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:30:10,003 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:30:10,003 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:30:10,008 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:30:10,608 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:30:12,613 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:30:13,046 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:30:18,046 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:30:18,452 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:30:18,453 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:30:18,453 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:30:18,458 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:30:18,620 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:30:20,623 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:30:22,027 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:30:22,031 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:30:22,031 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:30:22,035 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:30:22,391 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:30:22,391 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:30:22,392 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:30:22,625 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:30:23,668 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:30:24,630 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:30:25,782 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:30:25,783 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:30:25,783 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:30:25,788 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:30:26,632 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:30:28,636 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:30:28,826 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:30:30,160 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:30:30,161 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:30:30,161 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:30:30,166 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:30:30,637 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:30:32,622 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 16:30:32,642 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:30:34,623 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:30:35,146 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:30:35,147 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:30:35,147 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:30:35,152 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:30:35,645 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:30:36,647 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:30:37,391 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:30:37,391 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:30:37,393 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:30:38,793 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:30:38,794 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:30:38,794 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:30:38,799 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:30:39,649 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:30:39,838 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:30:40,653 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:30:42,545 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:30:42,546 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:30:42,546 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:30:42,551 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:30:42,655 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:30:44,660 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:30:45,552 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:30:46,568 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:30:46,569 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:30:46,569 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:30:46,574 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:30:46,661 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:30:48,665 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:30:50,575 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:30:51,390 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:30:51,391 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:30:51,391 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:30:51,396 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:30:51,667 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:30:52,391 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:30:52,391 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:30:52,393 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:30:52,671 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:30:55,663 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:30:55,664 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:30:55,664 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:30:55,664 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:30:55,669 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:30:55,673 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:30:58,678 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:31:00,706 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:31:01,706 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:31:01,707 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:31:01,707 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:31:01,712 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:31:02,624 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 16:31:02,683 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:31:04,687 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:31:06,625 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:31:07,391 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:31:07,391 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:31:07,393 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:31:10,106 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:31:10,106 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:31:10,106 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:31:10,111 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:31:10,694 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:31:12,150 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:31:12,699 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:31:14,855 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:31:14,855 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:31:14,856 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:31:14,861 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:31:15,701 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:31:16,706 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:31:17,861 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:31:18,391 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:31:18,392 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:31:18,392 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:31:18,397 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:31:18,707 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:31:20,711 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:31:22,278 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:31:22,279 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:31:22,279 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:31:22,284 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:31:22,391 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:31:22,391 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:31:22,393 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:31:22,712 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:31:23,619 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:31:24,717 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:31:28,620 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:31:29,001 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:31:29,002 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:31:29,002 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:31:29,007 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:31:29,721 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:31:30,725 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:31:32,629 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 16:31:32,663 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:31:32,663 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:31:32,663 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:31:32,669 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:31:32,726 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:31:33,669 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:31:34,730 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:31:37,302 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:31:37,303 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:31:37,303 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:31:37,308 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:31:37,391 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:31:37,392 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:31:37,393 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:31:37,732 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:31:38,737 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:31:39,646 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:31:42,419 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:31:42,419 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:31:42,419 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:31:42,424 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:31:42,741 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:31:44,745 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:31:45,462 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:31:47,900 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:31:47,900 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:31:47,901 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:31:47,906 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:31:48,749 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:31:50,754 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:31:50,906 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:31:51,597 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:31:51,597 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:31:51,598 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:31:51,602 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:31:51,755 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:31:52,391 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:31:52,392 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:31:52,393 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:31:52,757 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:31:55,328 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:31:55,329 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:31:55,329 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:31:55,334 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:31:55,759 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:31:56,370 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:31:56,762 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:32:00,269 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:32:00,270 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:32:00,270 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:32:00,275 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:32:00,766 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:32:02,276 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:32:02,628 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 16:32:02,771 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:32:04,204 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:32:04,205 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:32:04,205 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:32:04,210 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:32:04,772 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:32:06,776 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:32:07,391 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:32:07,392 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:32:07,393 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:32:08,100 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:32:08,924 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:32:08,924 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:32:08,924 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:32:08,929 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:32:09,778 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:32:10,780 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:32:13,966 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:32:15,517 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:32:15,517 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:32:15,518 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:32:15,522 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:32:15,786 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:32:16,791 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:32:19,323 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:32:19,324 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:32:19,324 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:32:19,325 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:32:19,330 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:32:19,792 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:32:20,797 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:32:22,392 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:32:22,392 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:32:22,393 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:32:22,752 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:32:22,752 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:32:22,752 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:32:22,757 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:32:22,798 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:32:24,794 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:32:24,802 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:32:27,205 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:32:27,206 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:32:27,206 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:32:27,211 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:32:27,804 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:32:28,808 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:32:30,212 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:32:31,770 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:32:31,771 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:32:31,771 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:32:31,776 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:32:31,809 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:32:32,634 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 16:32:34,818 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:32:35,154 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:32:35,155 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:32:35,155 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:32:35,160 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:32:35,818 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:32:36,160 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:32:36,821 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:32:37,392 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:32:37,392 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:32:37,393 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:32:40,536 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:32:40,536 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:32:40,537 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:32:40,542 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:32:40,823 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:32:41,578 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:32:42,828 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:32:44,468 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:32:44,469 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:32:44,469 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:32:44,474 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:32:44,829 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:32:46,832 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:32:47,475 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:32:48,280 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:32:48,280 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:32:48,281 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:32:48,286 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:32:48,835 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:32:50,839 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:32:51,872 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:32:51,873 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:32:51,874 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:32:51,879 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:32:52,392 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:32:52,392 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:32:52,393 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:32:52,641 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:32:52,840 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:32:54,845 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:32:57,634 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:32:57,635 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:32:57,635 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:32:57,640 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:32:57,678 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:32:57,847 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:32:58,850 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:33:01,198 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:33:01,198 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:33:01,199 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:33:01,204 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:33:01,852 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:33:02,633 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 16:33:02,856 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:33:03,634 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:33:06,399 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:33:06,400 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:33:06,400 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:33:06,405 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:33:06,860 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:33:07,392 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:33:07,392 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:33:07,392 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:33:08,864 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:33:09,611 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:33:10,309 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:33:10,309 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:33:10,309 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:33:10,314 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:33:10,866 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:33:12,871 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:33:13,379 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:33:13,379 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:33:13,379 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:33:13,384 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:33:13,871 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:33:14,874 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:33:15,385 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:33:16,437 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:33:16,438 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:33:16,438 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:33:16,443 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:33:16,876 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:33:18,880 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:33:20,443 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:33:21,010 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:33:21,011 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:33:21,011 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:33:21,016 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:33:21,881 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:33:22,392 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:33:22,392 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:33:22,394 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:33:22,884 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:33:25,688 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:33:28,500 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:33:28,500 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:33:28,501 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:33:28,505 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:33:28,892 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:33:30,897 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:33:31,542 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:33:32,517 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:33:32,518 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:33:32,518 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:33:32,523 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:33:32,637 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 16:33:32,898 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:33:34,904 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:33:36,193 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:33:36,193 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:33:36,193 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:33:36,199 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:33:36,905 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:33:37,199 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:33:37,392 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:33:37,392 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:33:37,394 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:33:38,908 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:33:41,081 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:33:41,081 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:33:41,081 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:33:41,086 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:33:41,910 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:33:42,913 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:33:43,122 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:33:44,879 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:33:44,880 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:33:44,880 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:33:44,885 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:33:44,916 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:33:46,921 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:33:48,606 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:33:48,607 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:33:48,607 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:33:48,608 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:33:48,612 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:33:48,922 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:33:50,927 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:33:52,392 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:33:52,393 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:33:52,394 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:33:53,615 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:33:58,615 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:33:58,678 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:33:58,679 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:33:58,679 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:33:58,684 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:33:58,936 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:34:00,941 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:34:02,211 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:34:02,212 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:34:02,212 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:34:02,217 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:34:02,637 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 16:34:02,942 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:34:03,639 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:34:04,947 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:34:07,392 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:34:07,393 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:34:07,394 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:34:07,426 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:34:08,213 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:34:08,214 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:34:08,219 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:34:08,950 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:34:09,219 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:34:10,955 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:34:12,778 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:34:12,779 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:34:12,779 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:34:12,784 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:34:12,957 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:34:14,785 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:34:14,962 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:34:17,491 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:34:17,492 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:34:17,492 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:34:17,497 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:34:17,963 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:34:18,967 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:34:20,497 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:34:22,392 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:34:22,393 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:34:22,394 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:34:22,953 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:34:22,953 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:34:22,954 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:34:22,959 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:34:22,971 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:34:24,975 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:34:25,994 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:34:27,747 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:34:27,747 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:34:27,748 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:34:27,753 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:34:27,977 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:34:28,981 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:34:31,364 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:34:31,365 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:34:31,365 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:34:31,366 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:34:31,370 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:34:31,983 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:34:32,643 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 16:34:32,987 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:34:36,372 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:34:36,372 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:34:36,373 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:34:36,373 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:34:36,378 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:34:36,990 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:34:37,392 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:34:37,393 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:34:37,394 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:34:38,993 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:34:40,270 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:34:40,271 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:34:40,271 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:34:40,275 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:34:40,996 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:34:42,314 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:34:42,999 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:34:44,760 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:34:44,760 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:34:44,761 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:34:44,766 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:34:45,002 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:34:47,007 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:34:47,767 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:34:49,160 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:34:49,161 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:34:49,161 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:34:49,166 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:34:50,008 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:34:51,013 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:34:52,392 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:34:52,393 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:34:52,394 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:34:53,640 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:34:54,105 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:34:54,105 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:34:54,105 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:34:54,110 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:34:55,016 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:34:57,021 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:34:59,146 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:35:02,623 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:35:02,624 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:35:02,624 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:35:02,629 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:35:02,641 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 16:35:03,028 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:35:04,642 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:35:05,033 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:35:06,465 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:35:06,466 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:35:06,466 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:35:06,471 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:35:07,034 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:35:07,392 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:35:07,393 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:35:07,394 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:35:09,039 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:35:09,824 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:35:14,824 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:35:15,851 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:35:15,851 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:35:15,852 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:35:15,859 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:35:16,046 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:35:19,053 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:35:19,894 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:35:20,286 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:35:20,287 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:35:20,287 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:35:20,292 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:35:21,054 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:35:22,393 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:35:22,393 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:35:22,394 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:35:23,059 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:35:24,626 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:35:24,627 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:35:24,627 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:35:24,632 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:35:25,062 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:35:25,670 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:35:27,065 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:35:28,571 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:35:28,571 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:35:28,572 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:35:28,577 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:35:29,066 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:35:31,070 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:35:31,577 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:35:32,643 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 16:35:33,196 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:35:33,197 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:35:33,197 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:35:33,202 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:35:34,072 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:35:35,077 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:35:37,203 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:35:37,393 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:35:37,393 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:35:37,394 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:35:41,360 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:35:41,361 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:35:41,361 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:35:41,366 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:35:42,083 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:35:42,402 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:35:43,085 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:35:45,415 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:35:45,416 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:35:45,416 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:35:45,421 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:35:46,089 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:35:47,093 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:35:47,422 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:35:52,393 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:35:52,393 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:35:52,394 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:35:52,605 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:35:53,837 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:35:53,838 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:35:53,838 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:35:53,843 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:35:54,101 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:35:55,105 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:35:57,589 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:35:57,590 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:35:57,590 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:35:57,595 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:35:58,106 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:35:58,595 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:35:59,110 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:36:01,237 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:36:01,238 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:36:01,238 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:36:01,243 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:36:02,112 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:36:02,651 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 16:36:03,116 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:36:03,653 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:36:07,005 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:36:07,006 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:36:07,006 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:36:07,011 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:36:07,119 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:36:07,393 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:36:07,393 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:36:07,395 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:36:08,712 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:36:09,124 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:36:10,946 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:36:10,947 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:36:10,947 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:36:10,952 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:36:11,126 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:36:13,131 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:36:13,990 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:36:15,996 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:36:15,997 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:36:15,997 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:36:16,003 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:36:16,133 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:36:19,003 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:36:19,139 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:36:19,502 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:36:19,503 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:36:19,503 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:36:19,508 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:36:20,140 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:36:21,142 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:36:22,393 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:36:22,393 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:36:22,395 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:36:23,419 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:36:23,420 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:36:23,420 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:36:23,425 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:36:24,144 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:36:24,462 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:36:25,148 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:36:26,932 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:36:26,933 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:36:26,933 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:36:26,938 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:36:27,149 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:36:29,155 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:36:29,938 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:36:32,256 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:36:32,257 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:36:32,257 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:36:32,262 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:36:32,648 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 16:36:33,157 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:36:35,163 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:36:35,649 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:36:36,625 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:36:36,625 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:36:36,626 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:36:36,630 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:36:37,164 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:36:37,393 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:36:37,393 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:36:37,395 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:36:39,167 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:36:40,654 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:36:41,194 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:36:41,194 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:36:41,194 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:36:41,199 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:36:42,169 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:36:43,174 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:36:45,366 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:36:45,367 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:36:45,367 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:36:45,372 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:36:46,176 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:36:46,372 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:36:47,179 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:36:49,631 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:36:49,632 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:36:49,632 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:36:49,637 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:36:50,181 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:36:51,185 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:36:51,637 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:36:52,393 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:36:52,393 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:36:52,395 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:36:52,986 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:36:52,986 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:36:52,987 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:36:52,991 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:36:53,186 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:36:55,191 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:36:57,030 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:36:57,134 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:36:57,134 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:36:57,135 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:36:57,139 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:36:57,192 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:36:59,197 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:37:01,500 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:37:01,501 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:37:01,501 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:37:01,506 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:37:02,199 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:37:02,507 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:37:02,653 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 16:37:03,200 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:37:06,498 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:37:06,499 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:37:06,499 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:37:06,504 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:37:07,206 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:37:07,393 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:37:07,394 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:37:07,395 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:37:08,053 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:37:09,210 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:37:09,764 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:37:09,764 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:37:09,764 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:37:09,769 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:37:10,211 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:37:11,214 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:37:13,090 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:37:13,091 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:37:13,091 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:37:13,091 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:37:13,096 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:37:13,216 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:37:15,220 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:37:16,567 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:37:16,568 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:37:16,568 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:37:16,573 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:37:17,222 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:37:18,574 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:37:19,225 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:37:20,218 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:37:20,219 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:37:20,219 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:37:20,224 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:37:20,226 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:37:22,393 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:37:22,394 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:37:22,395 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:37:23,231 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:37:23,601 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:37:23,602 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:37:23,602 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:37:23,602 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:37:23,607 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:37:24,232 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:37:25,234 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:37:27,480 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:37:27,481 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:37:27,481 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:37:27,486 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:37:28,236 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:37:29,240 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:37:29,487 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:37:31,387 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:37:31,388 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:37:31,388 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:37:31,393 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:37:32,242 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:37:32,652 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 16:37:33,245 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:37:34,651 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:37:34,652 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:37:34,652 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:37:34,652 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:37:34,657 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:37:35,248 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:37:37,251 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:37:37,393 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:37:37,394 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:37:37,395 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:37:37,910 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:37:38,302 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:37:38,302 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:37:38,308 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:37:39,253 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:37:40,308 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:37:41,257 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:37:44,086 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:37:44,087 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:37:44,087 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:37:44,092 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:37:44,259 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:37:46,092 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:37:47,266 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:37:47,596 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:37:47,596 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:37:47,597 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:37:47,602 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:37:48,266 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:37:49,269 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:37:50,983 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:37:50,984 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:37:50,984 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:37:50,989 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:37:51,270 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:37:51,989 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:37:52,393 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:37:52,394 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:37:52,395 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:37:53,274 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:37:55,096 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:37:55,097 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:37:55,097 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:37:55,102 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:37:55,276 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:37:57,139 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:37:57,280 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:37:58,741 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:37:58,741 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:37:58,742 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:37:58,747 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:37:59,281 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:38:01,286 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:38:02,222 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:38:02,223 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:38:02,223 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:38:02,223 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:38:02,228 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:38:02,286 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:38:02,658 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 16:38:05,291 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:38:05,478 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:38:05,479 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:38:05,479 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:38:05,484 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:38:06,292 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:38:07,294 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:38:07,393 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:38:07,394 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:38:07,395 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:38:07,633 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:38:09,592 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:38:09,592 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:38:09,593 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:38:09,598 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:38:10,296 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:38:11,300 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:38:12,635 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:38:13,577 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:38:13,578 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:38:13,578 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:38:13,583 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:38:14,303 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:38:15,306 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:38:18,584 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:38:21,343 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:38:21,344 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:38:21,344 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:38:21,352 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:38:22,313 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:38:22,394 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:38:22,394 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:38:22,395 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:38:23,317 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:38:23,687 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:38:25,289 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:38:25,290 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:38:25,290 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:38:25,295 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:38:25,318 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:38:27,322 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:38:29,305 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:38:29,306 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:38:29,306 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:38:29,306 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:38:29,311 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:38:29,323 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:38:31,328 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:38:32,657 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 16:38:34,658 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:38:34,789 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:38:34,790 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:38:34,790 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:38:34,796 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:38:35,333 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:38:37,394 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:38:37,394 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:38:37,395 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:38:38,130 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:38:38,910 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:38:38,910 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:38:38,910 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:38:38,915 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:38:39,130 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:38:39,955 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:38:41,133 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:38:42,827 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:38:42,828 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:38:42,828 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:38:42,833 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:38:43,134 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:38:45,138 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:38:45,833 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:38:46,532 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:38:46,533 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:38:46,533 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:38:46,538 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:38:47,139 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:38:49,142 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:38:51,539 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:38:51,965 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:38:51,966 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:38:51,967 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:38:51,972 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:38:52,144 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:38:52,394 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:38:52,394 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:38:52,396 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:38:53,146 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:38:56,633 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:38:56,940 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:38:56,941 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:38:56,941 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:38:56,946 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:38:57,149 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:38:59,152 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:39:00,530 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:39:00,531 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:39:00,531 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:39:00,536 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:39:01,153 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:39:02,537 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:39:02,662 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 16:39:03,156 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:39:05,581 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:39:05,581 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:39:05,582 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:39:05,587 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:39:06,159 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:39:07,161 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:39:07,394 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:39:07,394 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:39:07,395 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:39:08,475 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:39:13,476 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:39:13,872 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:39:13,873 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:39:13,873 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:39:13,879 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:39:14,166 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:39:15,168 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:39:17,867 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:39:17,867 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:39:17,867 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:39:17,873 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:39:18,171 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:39:18,873 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:39:19,174 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:39:21,328 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:39:21,329 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:39:21,329 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:39:21,334 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:39:22,176 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:39:22,394 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:39:22,394 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:39:22,395 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:39:23,178 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:39:24,679 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:39:25,258 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:39:25,259 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:39:25,259 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:39:25,264 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:39:26,180 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:39:27,183 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:39:29,427 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:39:29,428 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:39:29,428 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:39:29,433 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:39:30,185 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:39:30,434 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:39:31,185 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:39:32,662 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 16:39:33,761 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:39:33,762 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:39:33,762 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:39:33,767 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:39:34,188 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:39:35,190 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:39:35,767 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:39:37,394 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:39:37,394 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:39:37,395 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:39:37,415 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:39:37,659 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:39:37,659 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:39:37,664 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:39:38,194 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:39:39,196 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:39:41,665 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:39:41,972 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:39:41,973 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:39:41,973 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:39:41,978 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:39:42,198 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:39:43,201 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:39:46,978 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:39:47,007 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:39:47,008 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:39:47,008 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:39:47,014 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:39:47,203 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:39:49,206 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:39:50,894 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:39:50,895 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:39:50,895 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:39:50,900 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:39:51,208 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:39:52,394 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:39:52,394 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:39:52,396 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:39:52,622 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:39:53,211 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:39:54,592 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:39:54,592 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:39:54,593 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:39:54,597 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:39:55,212 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:39:57,215 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:39:57,488 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:39:57,489 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:39:57,489 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:39:57,495 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:39:58,216 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:39:58,495 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:39:59,219 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:40:00,923 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:40:00,924 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:40:00,924 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:40:00,929 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:40:01,220 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:40:02,667 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 16:40:03,223 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:40:03,669 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:40:05,307 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:40:05,308 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:40:05,308 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:40:05,313 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:40:06,225 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:40:07,226 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:40:07,394 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:40:07,394 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:40:07,395 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:40:08,687 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:40:09,180 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:40:09,181 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:40:09,181 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:40:09,186 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:40:09,227 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:40:11,230 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:40:12,562 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:40:12,563 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:40:12,564 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:40:12,568 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:40:13,231 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:40:14,569 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:40:15,235 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:40:17,183 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:40:17,184 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:40:17,184 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:40:17,190 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:40:17,237 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:40:19,240 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:40:20,190 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:40:22,394 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:40:22,395 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:40:22,396 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:40:23,030 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:40:23,030 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:40:23,031 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:40:23,036 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:40:23,243 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:40:25,247 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:40:26,075 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:40:27,787 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:40:27,788 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:40:27,788 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:40:27,793 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:40:28,249 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:40:29,251 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:40:31,794 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:40:32,666 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 16:40:32,752 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:40:32,752 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:40:32,752 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:40:32,758 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:40:33,254 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:40:35,257 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:40:37,394 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:40:37,395 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:40:37,396 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:40:38,093 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:40:38,200 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:40:38,201 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:40:38,201 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:40:38,207 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:40:38,261 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:40:39,264 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:40:42,214 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:40:42,215 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:40:42,215 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:40:42,220 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:40:42,265 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:40:43,220 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:40:43,268 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:40:45,717 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:40:45,718 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:40:45,718 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:40:45,723 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:40:46,270 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:40:47,273 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:40:48,723 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:40:49,506 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:40:49,506 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:40:49,507 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:40:49,512 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:40:50,275 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:40:51,277 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:40:52,394 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:40:52,395 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:40:52,396 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:40:54,618 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:40:58,158 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:40:58,158 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:40:58,158 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:40:58,163 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:40:58,282 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:40:59,285 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:41:00,202 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:41:02,668 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 16:41:03,484 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:41:03,485 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:41:03,485 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:41:03,490 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:41:04,288 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:41:05,291 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:41:05,491 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:41:07,247 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:41:07,248 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:41:07,248 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:41:07,253 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:41:07,292 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:41:07,394 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:41:07,395 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:41:07,396 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:41:09,295 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:41:11,087 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:41:11,934 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:41:11,935 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:41:11,935 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:41:11,940 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:41:12,297 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:41:13,297 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:41:16,889 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:41:16,890 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:41:16,890 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:41:16,890 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:41:16,895 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:41:17,302 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:41:19,305 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:41:21,643 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:41:21,644 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:41:21,645 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:41:21,649 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:41:22,306 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:41:22,395 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:41:22,395 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:41:22,396 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:41:22,653 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:41:23,309 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:41:26,878 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:41:26,878 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:41:26,878 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:41:26,883 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:41:27,311 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:41:27,925 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:41:29,314 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:41:32,670 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 16:41:32,706 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:41:32,707 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:41:32,707 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:41:32,712 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:41:33,318 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:41:33,712 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:41:35,321 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:41:37,395 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:41:37,395 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:41:37,396 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:41:37,562 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:41:37,935 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:41:37,935 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:41:37,940 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:41:38,324 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:41:38,941 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:41:39,327 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:41:42,644 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:41:42,645 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:41:42,645 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:41:42,651 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:41:43,329 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:41:44,651 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:41:45,333 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:41:49,652 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:41:51,057 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:41:51,058 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:41:51,059 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:41:51,064 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:41:51,337 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:41:52,395 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:41:52,395 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:41:52,396 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:41:53,340 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:41:54,573 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:41:54,573 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:41:54,574 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:41:54,579 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:41:55,342 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:41:55,615 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:41:57,345 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:41:58,905 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:41:58,906 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:41:58,906 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:41:58,911 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:41:59,346 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:42:00,912 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:42:01,350 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:42:02,672 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 16:42:05,511 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:42:05,512 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:42:05,512 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:42:05,518 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:42:06,353 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:42:06,518 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:42:07,356 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:42:07,395 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:42:07,395 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:42:07,397 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:42:11,333 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:42:11,333 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:42:11,334 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:42:11,339 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:42:11,358 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:42:12,374 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:42:13,362 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:42:16,104 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:42:16,105 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:42:16,105 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:42:16,110 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:42:16,365 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:42:17,368 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:42:18,111 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:42:21,387 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:42:21,388 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:42:21,388 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:42:21,393 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:42:22,371 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:42:22,395 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:42:22,395 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:42:22,397 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:42:23,373 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:42:23,593 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:42:24,581 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:42:24,582 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:42:24,582 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:42:24,587 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:42:25,375 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:42:27,378 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:42:28,152 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:42:28,153 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:42:28,153 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:42:28,158 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:42:28,378 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:42:29,159 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:42:29,381 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:42:32,393 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:42:32,394 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:42:32,395 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:42:32,399 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:42:32,678 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 16:42:33,383 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:42:34,679 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:42:35,387 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:42:36,065 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:42:36,066 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:42:36,067 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:42:36,072 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:42:36,388 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:42:37,391 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:42:37,395 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:42:37,395 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:42:37,397 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:42:40,672 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:42:40,869 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:42:40,870 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:42:40,870 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:42:40,875 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:42:41,393 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:42:43,397 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:42:44,659 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:42:44,660 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:42:44,661 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:42:44,666 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:42:45,398 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:42:46,666 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:42:47,402 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:42:48,184 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:42:48,184 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:42:48,185 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:42:48,190 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:42:48,402 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:42:49,406 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:42:52,191 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:42:52,395 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:42:52,395 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:42:52,397 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:42:54,283 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:42:54,284 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:42:54,284 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:42:54,290 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:42:54,409 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:42:55,411 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:42:57,326 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:43:02,327 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:43:02,612 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:43:02,613 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:43:02,613 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:43:02,621 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:43:02,677 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 16:43:03,416 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:43:05,420 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:43:05,862 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:43:05,863 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:43:05,863 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:43:05,868 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:43:06,420 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:43:07,395 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:43:07,396 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:43:07,397 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:43:07,423 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:43:08,615 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:43:09,199 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:43:09,199 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:43:09,200 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:43:09,205 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:43:09,424 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:43:11,427 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:43:13,326 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:43:13,327 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:43:13,328 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:43:13,333 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:43:13,429 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:43:14,333 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:43:15,433 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:43:18,279 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:43:18,280 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:43:18,280 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:43:18,285 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:43:18,435 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:43:19,437 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:43:20,286 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:43:22,395 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:43:22,396 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:43:22,397 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:43:22,731 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:43:22,732 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:43:22,732 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:43:22,737 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:43:23,440 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:43:25,448 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:43:25,775 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:43:27,625 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:43:27,626 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:43:27,626 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:43:27,631 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:43:28,450 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:43:29,453 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:43:31,632 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:43:32,159 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:43:32,160 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:43:32,161 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:43:32,165 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:43:32,455 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:43:32,682 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 16:43:33,458 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:43:36,103 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:43:36,104 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:43:36,104 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:43:36,109 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:43:36,462 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:43:37,109 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:43:37,395 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:43:37,396 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:43:37,397 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:43:37,464 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:43:39,693 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:43:39,693 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:43:39,694 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:43:39,699 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:43:40,466 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:43:41,468 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:43:42,735 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:43:43,922 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:43:43,922 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:43:43,923 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:43:43,927 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:43:44,470 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:43:45,473 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:43:47,928 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:43:48,654 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:43:48,655 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:43:48,655 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:43:48,660 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:43:49,476 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:43:51,480 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:43:52,343 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:43:52,344 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:43:52,345 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:43:52,350 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:43:52,395 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:43:52,396 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:43:52,397 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:43:52,480 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:43:53,483 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:43:53,683 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:43:57,720 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:43:57,721 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:43:57,721 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:43:57,726 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:43:58,486 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:43:58,762 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:43:59,489 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:44:01,174 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:44:01,175 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:44:01,175 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:44:01,180 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:44:01,490 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:44:02,682 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 16:44:03,493 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:44:04,683 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:44:04,927 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:44:04,927 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:44:04,928 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:44:04,932 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:44:05,495 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:44:07,396 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:44:07,396 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:44:07,398 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:44:07,496 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:44:09,619 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:44:09,619 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:44:09,620 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:44:09,624 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:44:10,498 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:44:10,662 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:44:11,501 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:44:13,385 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:44:13,386 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:44:13,386 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:44:13,391 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:44:13,503 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:44:15,507 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:44:16,392 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:44:17,194 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:44:17,194 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:44:17,195 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:44:17,199 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:44:17,509 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:44:19,512 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:44:22,200 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:44:22,396 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:44:22,396 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:44:22,397 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:44:23,137 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:44:23,137 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:44:23,138 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:44:23,142 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:44:23,515 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:44:25,518 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:44:26,669 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:44:26,670 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:44:26,670 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:44:26,675 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:44:27,519 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:44:27,676 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:44:29,527 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:44:29,817 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:44:29,818 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:44:29,818 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:44:29,824 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:44:30,528 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:44:31,531 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:44:32,689 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 16:44:32,691 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:44:33,758 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:44:33,759 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:44:33,759 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:44:33,764 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:44:34,533 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:44:35,536 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:44:37,396 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:44:37,396 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:44:37,398 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:44:37,806 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:44:39,371 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:44:39,371 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:44:39,371 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:44:39,377 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:44:39,539 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:44:41,542 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:44:42,829 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:44:42,830 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:44:42,830 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:44:42,830 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:44:42,835 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:44:43,544 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:44:45,547 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:44:46,891 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:44:46,892 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:44:46,892 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:44:46,897 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:44:47,549 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:44:47,897 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:44:49,552 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:44:50,367 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:44:50,368 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:44:50,368 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:44:50,373 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:44:50,553 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:44:51,556 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:44:52,396 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:44:52,396 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:44:52,398 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:44:53,680 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:44:54,304 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:44:54,305 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:44:54,305 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:44:54,310 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:44:54,558 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:44:55,561 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:44:59,346 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:45:00,086 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:45:00,087 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:45:00,087 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:45:00,092 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:45:00,564 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:45:01,567 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:45:02,686 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 16:45:03,662 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:45:03,663 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:45:03,663 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:45:03,668 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:45:04,569 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:45:04,669 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:45:05,572 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:45:07,396 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:45:07,396 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:45:07,398 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:45:09,108 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:45:09,109 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:45:09,109 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:45:09,114 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:45:09,574 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:45:10,150 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:45:11,578 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:45:13,183 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:45:13,184 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:45:13,185 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:45:13,190 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:45:13,580 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:45:15,190 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:45:15,584 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:45:16,813 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:45:16,814 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:45:16,814 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:45:16,819 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:45:17,586 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:45:19,587 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:45:20,381 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:45:20,381 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:45:20,382 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:45:20,382 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:45:20,387 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:45:20,588 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:45:21,591 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:45:22,396 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:45:22,396 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:45:22,398 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:45:25,590 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:45:29,206 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:45:29,207 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:45:29,207 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:45:29,212 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:45:29,596 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:45:31,250 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:45:31,599 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:45:32,155 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:45:32,156 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:45:32,156 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:45:32,161 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:45:32,600 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:45:32,688 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 16:45:33,604 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:45:36,690 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:45:37,396 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:45:37,397 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:45:37,398 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:45:38,120 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:45:38,124 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:45:38,124 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:45:38,129 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:45:38,609 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:45:39,612 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:45:42,008 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:45:42,008 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:45:42,009 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:45:42,009 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:45:42,014 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:45:42,614 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:45:43,617 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:45:46,567 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:45:46,568 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:45:46,568 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:45:46,573 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:45:46,619 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:45:47,573 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:45:47,622 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:45:52,396 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:45:52,397 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:45:52,398 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:45:52,696 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:45:53,193 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:45:53,194 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:45:53,194 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:45:53,199 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:45:53,626 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:45:55,629 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:45:56,759 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:45:56,759 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:45:56,760 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:45:56,765 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:45:57,633 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:45:57,765 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:45:59,636 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:46:01,698 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:46:01,698 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:46:01,699 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:46:01,704 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:46:02,638 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:46:02,691 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 16:46:03,641 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:46:03,692 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:46:05,947 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:46:05,948 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:46:05,948 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:46:05,954 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:46:06,643 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:46:07,396 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:46:07,397 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:46:07,398 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:46:07,646 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:46:09,587 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:46:14,588 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:46:15,081 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:46:15,082 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:46:15,082 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:46:15,087 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:46:15,653 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:46:17,658 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:46:19,960 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:46:19,961 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:46:19,961 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:46:19,961 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:46:19,966 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:46:20,660 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:46:21,663 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:46:22,396 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:46:22,397 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:46:22,398 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:46:24,084 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:46:24,084 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:46:24,085 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:46:24,090 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:46:24,665 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:46:25,126 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:46:25,665 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:46:29,797 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:46:29,798 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:46:29,798 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:46:29,803 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:46:30,669 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:46:30,804 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:46:31,671 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:46:32,693 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 16:46:34,400 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:46:34,400 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:46:34,401 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:46:34,406 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:46:34,674 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:46:35,677 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:46:36,407 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:46:37,396 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:46:37,397 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:46:37,399 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:46:38,041 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:46:38,042 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:46:38,042 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:46:38,047 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:46:38,680 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:46:39,683 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:46:41,976 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:46:41,977 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:46:41,977 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:46:41,977 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:46:41,982 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:46:42,685 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:46:43,687 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:46:46,320 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:46:46,321 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:46:46,321 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:46:46,326 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:46:46,689 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:46:47,327 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:46:47,693 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:46:50,490 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:46:50,491 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:46:50,491 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:46:50,496 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:46:50,694 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:46:51,697 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:46:52,397 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:46:52,397 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:46:52,399 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:46:52,680 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:46:55,093 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:46:55,093 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:46:55,093 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:46:55,098 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:46:55,699 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:46:57,702 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:46:58,138 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:47:02,695 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 16:47:03,697 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:47:04,238 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:47:04,238 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:47:04,239 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:47:04,244 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:47:04,707 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:47:05,710 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:47:07,397 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:47:07,397 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:47:07,399 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:47:08,243 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:47:08,346 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:47:08,346 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:47:08,351 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:47:08,712 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:47:09,351 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:47:09,715 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:47:12,809 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:47:12,810 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:47:12,810 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:47:12,815 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:47:13,717 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:47:14,816 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:47:15,721 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:47:16,357 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:47:16,358 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:47:16,358 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:47:16,363 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:47:16,722 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:47:17,725 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:47:20,178 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:47:20,178 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:47:20,179 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:47:20,179 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:47:20,184 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:47:20,727 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:47:21,729 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:47:22,397 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:47:22,397 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:47:22,399 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:47:24,736 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:47:24,737 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:47:24,737 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:47:24,742 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:47:25,732 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:47:25,779 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:47:27,735 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:47:30,139 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:47:30,139 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:47:30,140 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:47:30,144 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:47:30,738 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:47:31,145 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:47:31,740 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:47:32,697 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 16:47:36,216 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:47:36,217 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:47:36,217 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:47:36,217 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:47:36,222 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:47:36,745 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:47:37,397 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:47:37,397 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:47:37,398 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:47:37,746 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:47:41,268 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:47:41,269 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:47:41,269 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:47:41,269 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:47:41,274 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:47:41,749 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:47:43,752 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:47:45,634 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:47:45,635 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:47:45,635 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:47:45,640 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:47:45,753 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:47:46,641 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:47:47,757 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:47:49,355 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:47:49,356 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:47:49,356 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:47:49,362 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:47:49,758 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:47:51,761 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:47:52,363 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:47:52,397 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:47:52,397 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:47:52,399 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:47:52,910 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:47:52,911 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:47:52,911 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:47:52,916 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:47:53,763 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:47:55,766 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:47:57,955 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:47:58,057 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:47:58,058 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:47:58,058 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:47:58,063 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:47:58,768 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:47:59,770 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:48:02,228 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:48:02,229 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:48:02,229 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:48:02,234 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:48:02,702 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 16:48:02,772 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:48:03,704 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:48:03,775 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:48:06,148 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:48:06,149 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:48:06,150 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:48:06,154 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:48:06,777 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:48:07,397 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:48:07,398 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:48:07,399 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:48:07,779 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:48:09,635 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:48:09,878 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:48:09,878 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:48:09,879 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:48:09,883 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:48:10,781 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:48:11,784 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:48:14,922 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:48:14,923 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:48:14,923 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:48:14,923 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:48:14,928 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:48:15,788 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:48:17,791 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:48:19,929 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:48:20,584 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:48:20,585 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:48:20,585 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:48:20,590 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:48:20,793 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:48:21,796 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:48:22,397 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:48:22,398 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:48:22,398 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:48:24,530 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:48:24,531 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:48:24,531 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:48:24,536 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:48:24,798 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:48:25,575 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:48:25,800 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:48:28,326 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:48:28,327 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:48:28,328 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:48:28,332 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:48:28,802 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:48:29,805 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:48:31,333 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:48:32,412 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:48:32,412 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:48:32,413 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:48:32,417 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:48:32,701 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 16:48:32,807 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:48:33,810 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:48:36,475 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:48:36,476 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:48:36,477 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:48:36,477 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:48:36,481 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:48:36,813 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:48:37,398 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:48:37,398 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:48:37,399 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:48:37,816 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:48:39,956 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:48:39,956 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:48:39,957 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:48:39,962 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:48:40,818 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:48:41,820 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:48:41,999 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:48:43,761 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:48:43,762 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:48:43,762 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:48:43,767 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:48:43,822 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:48:45,825 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:48:47,664 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:48:47,665 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:48:47,666 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:48:47,666 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:48:47,670 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:48:47,826 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:48:49,829 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:48:51,145 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:48:51,146 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:48:51,146 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:48:51,152 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:48:51,830 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:48:52,398 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:48:52,399 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:48:52,399 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:48:53,656 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:48:53,834 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:48:54,693 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:48:54,694 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:48:54,694 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:48:54,699 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:48:54,834 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:48:55,837 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:48:58,735 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:48:59,379 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:48:59,379 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:48:59,380 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:48:59,384 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:48:59,839 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:49:01,842 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:49:02,706 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 16:49:04,567 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:49:04,568 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:49:04,569 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:49:04,569 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:49:04,573 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:49:04,844 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:49:05,847 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:49:07,398 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:49:07,398 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:49:07,399 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:49:08,157 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:49:08,185 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:49:08,185 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:49:08,190 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:49:08,849 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:49:09,851 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:49:10,190 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:49:13,071 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:49:13,072 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:49:13,072 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:49:13,077 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:49:13,854 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:49:15,858 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:49:16,078 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:49:16,759 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:49:16,759 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:49:16,760 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:49:16,765 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:49:16,859 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:49:17,861 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:49:21,766 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:49:22,398 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:49:22,398 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:49:22,399 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:49:25,363 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:49:25,364 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:49:25,364 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:49:25,369 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:49:25,866 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:49:27,407 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:49:27,870 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:49:30,279 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:49:30,280 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:49:30,280 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:49:30,285 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:49:30,871 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:49:31,874 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:49:32,706 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 16:49:32,707 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:49:34,371 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:49:34,371 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:49:34,372 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:49:34,376 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:49:34,876 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:49:35,879 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:49:37,398 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:49:37,399 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:49:37,399 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:49:37,752 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:49:38,416 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:49:38,417 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:49:38,417 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:49:38,422 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:49:38,882 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:49:39,884 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:49:43,459 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:49:43,893 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:49:43,894 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:49:43,894 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:49:43,899 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:49:44,888 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:49:45,890 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:49:47,598 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:49:47,599 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:49:47,599 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:49:47,604 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:49:47,891 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:49:48,604 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:49:49,895 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:49:52,398 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:49:52,399 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:49:52,399 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:49:53,623 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:49:56,034 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:49:56,034 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:49:56,035 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:49:56,039 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:49:56,899 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:49:57,902 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:49:59,079 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:49:59,405 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:49:59,406 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:49:59,407 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:49:59,411 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:49:59,903 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:50:01,906 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:50:02,711 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 16:50:03,214 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:50:03,215 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:50:03,215 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:50:03,220 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:50:03,907 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:50:04,220 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:50:05,910 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:50:06,997 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:50:06,997 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:50:06,998 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:50:07,003 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:50:07,399 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:50:07,399 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:50:07,399 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:50:07,912 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:50:09,652 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:50:09,915 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:50:10,572 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:50:10,573 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:50:10,573 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:50:10,578 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:50:10,916 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:50:11,918 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:50:15,615 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:50:16,286 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:50:16,287 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:50:16,288 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:50:16,293 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:50:16,923 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:50:17,926 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:50:21,294 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:50:21,579 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:50:21,580 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:50:21,580 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:50:21,585 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:50:21,928 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:50:22,399 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:50:22,399 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:50:22,399 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:50:23,932 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:50:26,608 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:50:26,811 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:50:26,811 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:50:26,812 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:50:26,817 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:50:26,933 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:50:27,936 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:50:31,855 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:50:32,710 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 16:50:33,155 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:50:33,156 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:50:33,157 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:50:33,161 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:50:33,940 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:50:35,943 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:50:37,162 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:50:37,399 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:50:37,399 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:50:37,399 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:50:37,547 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:50:37,625 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:50:37,625 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:50:37,630 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:50:37,946 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:50:39,949 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:50:42,105 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:50:42,106 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:50:42,106 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:50:42,111 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:50:42,951 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:50:43,151 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:50:43,953 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:50:47,421 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:50:47,422 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:50:47,422 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:50:47,427 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:50:47,956 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:50:48,428 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:50:49,959 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:50:51,090 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:50:51,091 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:50:51,091 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:50:51,096 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:50:51,961 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:50:52,399 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:50:52,399 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:50:52,399 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:50:53,642 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:50:53,964 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:50:54,521 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:50:54,522 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:50:54,522 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:50:54,527 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:50:54,965 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:50:55,967 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:50:58,190 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:50:58,190 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:50:58,191 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:50:58,196 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:50:58,969 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:50:59,196 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:50:59,972 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:51:02,712 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 16:51:03,199 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:51:03,200 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:51:03,200 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:51:03,205 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:51:03,974 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:51:04,206 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:51:05,977 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:51:07,356 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:51:07,357 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:51:07,357 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:51:07,362 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:51:07,399 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:51:07,399 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:51:07,399 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:51:07,979 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:51:09,241 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:51:09,982 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:51:12,255 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:51:12,255 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:51:12,256 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:51:12,261 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:51:12,984 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:51:13,987 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:51:14,299 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:51:16,196 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:51:16,200 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:51:16,200 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:51:16,205 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:51:16,990 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:51:17,993 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:51:19,831 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:51:19,832 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:51:19,832 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:51:19,832 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:51:19,837 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:51:19,994 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:51:21,997 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:51:22,399 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:51:22,399 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:51:22,399 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:51:24,433 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:51:24,433 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:51:24,434 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:51:24,439 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:51:24,999 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:51:25,475 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:51:26,002 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:51:29,028 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:51:29,029 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:51:29,029 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:51:29,034 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:51:30,004 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:51:31,035 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:51:32,007 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:51:32,715 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 16:51:33,581 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:51:33,582 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:51:33,582 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:51:33,587 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:51:34,009 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:51:36,012 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:51:36,587 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:51:37,399 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:51:37,399 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:51:37,400 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:51:38,602 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:51:38,603 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:51:38,603 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:51:38,608 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:51:39,015 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:51:40,018 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:51:41,647 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:51:42,662 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:51:42,663 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:51:42,663 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:51:42,668 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:51:43,020 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:51:44,022 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:51:46,417 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:51:46,418 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:51:46,418 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:51:46,423 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:51:47,024 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:51:47,424 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:51:48,028 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:51:50,466 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:51:50,467 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:51:50,467 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:51:50,472 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:51:51,030 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:51:52,032 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:51:52,399 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:51:52,399 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:51:52,399 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:51:52,660 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:51:54,133 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:51:54,134 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:51:54,134 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:51:54,139 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:51:55,034 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:51:56,037 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:51:58,179 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:51:58,436 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:51:58,436 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:51:58,437 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:51:58,442 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:51:59,039 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:52:00,041 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:52:02,059 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:52:02,060 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:52:02,061 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:52:02,065 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:52:02,717 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 16:52:03,043 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:52:03,718 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:52:04,046 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:52:05,806 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:52:05,807 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:52:05,807 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:52:05,812 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:52:06,047 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:52:07,399 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:52:07,399 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:52:07,400 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:52:08,050 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:52:08,880 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:52:10,102 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:52:10,103 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:52:10,103 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:52:10,108 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:52:11,052 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:52:12,054 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:52:14,147 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:52:17,888 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:52:17,889 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:52:17,889 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:52:17,894 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:52:18,059 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:52:19,895 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:52:20,063 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:52:22,399 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:52:22,399 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:52:22,400 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:52:22,450 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:52:22,630 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:52:22,630 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:52:22,635 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:52:23,064 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:52:24,067 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:52:25,635 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:52:26,797 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:52:26,797 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:52:26,798 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:52:26,803 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:52:27,069 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:52:28,071 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:52:30,804 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:52:31,796 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:52:31,796 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:52:31,797 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:52:31,802 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:52:32,074 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:52:32,719 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 16:52:34,078 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:52:36,064 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:52:36,065 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:52:36,065 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:52:36,065 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:52:36,070 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:52:36,079 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:52:37,399 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:52:37,400 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:52:37,400 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:52:38,083 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:52:39,972 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:52:39,972 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:52:39,973 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:52:39,977 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:52:40,085 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:52:42,015 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:52:42,088 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:52:45,748 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:52:45,749 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:52:45,750 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:52:45,755 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:52:46,091 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:52:47,756 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:52:48,094 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:52:48,976 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:52:48,976 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:52:48,977 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:52:48,982 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:52:49,095 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:52:50,096 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:52:52,399 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:52:52,400 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:52:52,400 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:52:53,619 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:52:54,276 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:52:54,277 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:52:54,277 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:52:54,282 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:52:55,099 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:52:56,101 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:52:58,596 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:52:58,597 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:52:58,597 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:52:58,602 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:52:59,103 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:52:59,602 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:53:00,106 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:53:02,499 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:53:02,499 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:53:02,500 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:53:02,505 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:53:02,721 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 16:53:03,108 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:53:04,111 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:53:04,723 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:53:06,419 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:53:06,420 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:53:06,420 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:53:06,425 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:53:07,112 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:53:07,400 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:53:07,400 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:53:07,400 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:53:08,115 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:53:10,602 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:53:11,202 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:53:11,203 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:53:11,203 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:53:11,208 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:53:12,118 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:53:14,121 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:53:16,246 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:53:16,354 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:53:16,354 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:53:16,355 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:53:16,359 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:53:17,124 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:53:18,126 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:53:20,331 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:53:20,332 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:53:20,332 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:53:20,337 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:53:21,128 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:53:21,338 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:53:22,130 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:53:22,400 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:53:22,400 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:53:22,400 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:53:25,164 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:53:25,165 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:53:25,165 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:53:25,170 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:53:26,133 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:53:27,206 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:53:28,136 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:53:30,519 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:53:30,520 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:53:30,520 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:53:30,525 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:53:31,138 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:53:32,140 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:53:32,526 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:53:32,728 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 16:53:35,548 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:53:35,549 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:53:35,549 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:53:35,554 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:53:36,143 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:53:37,400 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:53:37,400 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:53:37,400 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:53:37,672 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:53:38,150 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:53:40,168 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:53:40,169 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:53:40,169 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:53:40,175 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:53:41,151 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:53:42,154 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:53:43,210 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:53:43,820 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:53:43,820 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:53:43,821 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:53:43,826 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:53:44,155 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:53:46,158 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:53:48,617 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:53:48,618 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:53:48,618 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:53:48,619 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:53:48,623 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:53:49,161 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:53:50,163 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:53:52,225 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:53:52,226 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:53:52,226 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:53:52,231 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:53:52,400 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:53:52,400 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:53:52,442 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:53:53,165 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:53:54,169 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:53:54,611 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:53:55,168 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:53:55,169 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:53:55,169 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:53:55,174 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:53:56,170 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:53:58,174 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:53:59,737 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:53:59,738 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:53:59,739 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:53:59,739 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:53:59,744 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:54:00,175 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:54:02,178 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:54:02,725 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 16:54:05,726 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:54:06,125 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:54:06,126 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:54:06,126 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:54:06,131 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:54:06,181 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:54:07,400 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:54:07,400 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:54:07,400 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:54:08,184 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:54:09,837 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:54:09,837 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:54:09,838 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:54:09,842 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:54:10,185 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:54:10,878 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:54:12,189 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:54:13,096 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:54:13,097 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:54:13,097 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:54:13,102 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:54:13,189 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:54:14,192 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:54:16,103 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:54:16,245 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:54:16,246 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:54:16,246 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:54:16,251 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:54:17,195 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:54:18,198 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:54:20,016 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:54:20,017 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:54:20,017 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:54:20,022 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:54:20,199 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:54:22,022 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:54:22,202 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:54:22,400 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:54:22,400 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:54:22,400 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:54:23,379 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:54:23,380 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:54:23,380 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:54:23,385 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:54:24,203 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:54:26,206 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:54:27,422 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:54:32,101 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:54:32,102 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:54:32,102 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:54:32,107 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:54:32,210 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:54:32,730 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 16:54:32,731 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:54:34,213 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:54:36,641 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:54:36,641 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:54:36,642 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:54:36,647 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:54:37,217 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:54:37,402 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:54:37,402 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:54:37,402 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:54:37,823 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:54:38,219 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:54:39,875 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:54:39,875 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:54:39,876 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:54:39,881 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:54:40,220 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:54:42,223 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:54:42,918 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:54:44,178 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:54:44,178 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:54:44,179 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:54:44,183 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:54:44,225 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:54:46,228 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:54:47,814 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:54:47,815 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:54:47,815 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:54:47,820 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:54:48,229 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:54:48,821 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:54:50,232 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:54:51,276 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:54:51,277 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:54:51,277 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:54:51,282 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:54:52,234 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:54:52,400 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:54:52,401 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:54:52,401 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:54:54,237 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:54:54,648 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:54:55,271 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:54:55,272 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:54:55,272 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:54:55,277 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:54:56,238 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:54:58,241 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:54:59,434 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:54:59,435 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:54:59,435 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:54:59,440 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:55:00,242 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:55:00,440 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:55:02,245 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:55:02,729 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 16:55:04,173 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:55:04,174 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:55:04,174 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:55:04,178 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:55:04,246 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:55:06,179 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:55:06,250 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:55:07,400 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:55:07,401 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:55:07,401 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:55:09,101 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:55:09,101 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:55:09,102 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:55:09,107 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:55:09,252 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:55:10,254 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:55:12,142 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:55:12,925 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:55:12,926 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:55:12,926 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:55:12,931 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:55:13,256 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:55:14,259 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:55:17,932 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:55:19,093 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:55:19,094 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:55:19,094 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:55:19,099 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:55:19,263 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:55:20,266 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:55:22,401 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:55:22,401 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:55:22,401 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:55:22,690 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:55:22,704 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:55:22,705 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:55:22,709 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:55:23,268 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:55:23,710 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:55:24,270 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:55:26,197 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:55:26,197 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:55:26,198 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:55:26,202 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:55:26,271 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:55:28,275 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:55:29,203 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:55:32,736 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 16:55:33,882 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:55:33,883 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:55:33,883 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:55:33,888 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:55:34,279 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:55:34,888 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:55:36,282 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:55:37,401 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:55:37,401 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:55:37,401 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:55:38,136 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:55:38,137 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:55:38,137 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:55:38,142 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:55:38,284 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:55:40,178 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:55:40,288 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:55:45,178 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:55:46,369 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:55:46,370 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:55:46,370 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:55:46,375 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:55:47,292 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:55:48,295 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:55:50,227 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:55:50,227 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:55:50,228 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:55:50,228 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:55:50,233 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:55:50,296 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:55:52,300 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:55:52,401 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:55:52,401 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:55:52,401 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:55:55,254 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:55:55,254 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:55:55,255 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:55:55,255 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:55:55,260 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:55:55,301 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:55:56,304 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:55:59,098 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:55:59,099 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:55:59,099 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:55:59,104 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:55:59,306 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:56:00,309 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:56:01,104 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:56:02,734 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 16:56:03,848 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:56:03,849 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:56:03,849 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:56:03,854 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:56:04,311 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:56:06,314 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:56:06,855 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:56:07,401 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:56:07,401 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:56:07,401 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:56:08,599 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:56:08,599 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:56:08,599 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:56:08,605 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:56:09,317 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:56:10,319 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:56:12,112 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:56:12,113 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:56:12,113 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:56:12,113 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:56:12,118 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:56:12,321 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:56:14,325 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:56:17,119 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:56:18,377 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:56:18,378 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:56:18,378 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:56:18,383 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:56:19,329 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:56:20,332 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:56:22,383 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:56:22,401 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:56:22,401 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:56:22,401 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:56:23,317 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:56:23,317 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:56:23,318 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:56:23,322 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:56:23,334 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:56:24,337 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:56:27,169 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:56:27,169 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:56:27,170 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:56:27,175 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:56:27,339 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:56:28,175 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:56:28,341 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:56:32,736 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 16:56:33,738 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:56:34,405 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:56:34,406 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:56:34,407 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:56:34,412 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:56:35,346 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:56:36,349 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:56:37,401 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:56:37,401 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:56:37,401 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:56:39,631 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:56:43,559 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:56:43,560 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:56:43,560 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:56:43,566 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:56:44,355 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:56:45,602 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:56:46,358 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:56:48,270 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:56:48,271 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:56:48,271 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:56:48,276 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:56:48,360 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:56:50,361 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:56:51,277 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:56:52,401 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:56:52,401 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:56:52,401 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:56:52,793 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:56:52,797 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:56:52,797 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:56:52,802 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:56:53,363 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:56:54,366 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:56:56,710 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:56:56,711 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:56:56,712 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:56:56,712 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:56:56,717 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:56:57,368 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:56:58,370 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:57:00,977 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:57:00,978 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:57:00,978 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:57:00,983 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:57:01,372 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:57:01,983 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:57:02,375 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:57:02,738 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 16:57:05,551 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:57:05,551 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:57:05,552 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:57:05,557 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:57:06,377 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:57:07,401 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:57:07,402 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:57:07,402 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:57:07,402 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:57:08,381 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:57:09,196 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:57:09,196 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:57:09,197 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:57:09,201 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:57:09,382 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:57:10,384 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:57:12,667 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:57:12,668 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:57:12,669 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:57:12,669 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:57:12,674 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:57:13,387 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:57:14,390 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:57:16,232 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:57:16,233 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:57:16,233 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:57:16,238 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:57:16,392 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:57:18,239 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:57:18,396 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:57:21,503 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:57:21,504 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:57:21,504 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:57:21,510 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:57:22,398 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:57:22,401 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:57:22,402 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:57:22,402 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:57:23,637 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:57:24,402 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:57:24,761 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:57:24,762 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:57:24,762 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:57:24,767 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:57:25,403 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:57:26,403 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:57:28,448 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:57:28,449 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:57:28,449 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:57:28,454 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:57:29,405 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:57:29,455 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:57:30,407 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:57:32,236 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:57:32,237 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:57:32,237 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:57:32,242 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:57:32,409 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:57:32,740 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 16:57:34,412 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:57:34,742 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:57:35,692 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:57:35,693 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:57:35,693 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:57:35,699 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:57:36,413 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:57:37,402 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:57:37,402 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:57:37,402 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:57:38,418 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:57:40,503 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:57:40,504 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:57:40,504 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:57:40,504 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:57:40,509 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:57:41,420 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:57:42,423 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:57:45,546 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:57:46,781 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:57:46,781 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:57:46,781 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:57:46,787 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:57:47,426 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:57:48,429 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:57:50,718 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:57:50,718 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:57:50,719 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:57:50,719 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:57:50,724 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:57:51,431 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:57:52,402 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:57:52,402 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:57:52,402 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:57:52,436 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:57:55,252 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:57:55,253 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:57:55,253 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:57:55,258 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:57:55,438 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:57:56,294 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:57:56,441 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:57:59,783 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:57:59,784 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:57:59,784 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:57:59,789 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:58:00,444 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:58:01,789 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:58:02,445 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:58:02,742 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 16:58:06,920 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:58:06,921 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:58:06,922 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:58:06,922 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:58:06,927 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:58:07,402 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:58:07,402 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:58:07,402 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:58:07,449 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:58:08,452 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:58:11,513 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:58:11,514 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:58:11,514 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:58:11,519 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:58:12,455 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:58:12,554 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:58:14,458 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:58:17,554 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:58:19,858 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:58:19,859 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:58:19,859 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:58:19,864 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:58:20,463 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:58:22,402 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:58:22,402 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:58:22,402 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:58:22,466 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:58:22,615 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:58:25,436 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:58:25,436 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:58:25,437 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:58:25,442 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:58:25,468 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:58:26,471 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:58:28,478 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:58:30,873 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:58:30,874 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:58:30,874 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:58:30,879 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:58:31,474 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:58:32,477 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:58:32,744 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 16:58:33,746 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:58:34,426 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:58:34,426 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:58:34,427 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:58:34,432 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:58:34,478 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:58:36,481 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:58:37,402 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:58:37,402 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:58:37,402 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:58:38,357 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:58:38,670 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:58:38,670 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:58:38,675 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:58:39,486 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:58:39,675 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:58:40,489 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:58:42,082 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:58:42,083 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:58:42,083 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:58:42,098 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:58:42,490 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:58:44,494 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:58:45,099 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:58:46,446 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:58:46,447 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:58:46,447 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:58:46,452 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:58:46,495 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:58:48,499 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:58:50,094 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:58:50,094 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:58:50,095 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:58:50,101 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:58:50,101 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:58:50,500 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:58:52,402 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:58:52,402 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:58:52,402 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:58:52,503 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:58:53,409 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:58:53,409 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:58:53,409 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:58:53,414 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:58:53,504 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:58:54,507 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:58:55,450 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:58:56,699 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:58:56,700 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:58:56,700 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:58:56,706 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:58:57,508 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:58:58,511 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:59:00,706 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:59:02,750 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 16:59:04,437 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:59:04,438 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:59:04,438 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:59:04,443 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:59:04,515 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:59:06,443 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:59:06,518 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:59:07,402 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:59:07,402 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:59:07,402 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:59:08,081 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:59:08,082 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:59:08,082 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:59:08,088 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:59:08,520 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:59:10,523 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:59:11,805 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:59:11,806 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:59:11,806 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:59:11,806 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:59:11,811 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:59:12,524 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:59:14,528 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:59:15,151 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:59:15,152 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:59:15,152 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:59:15,157 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:59:15,529 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:59:16,532 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:59:17,158 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:59:18,722 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:59:18,723 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:59:18,723 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:59:18,728 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:59:19,535 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:59:20,537 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:59:22,402 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:59:22,402 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:59:22,403 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:59:22,403 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:59:23,917 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:59:23,918 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:59:23,918 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:59:23,923 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:59:24,540 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:59:26,543 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:59:27,439 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:59:27,440 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:59:27,440 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:59:27,440 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:59:27,445 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:59:27,544 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:59:28,546 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:59:32,446 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:59:32,749 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 16:59:35,851 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:59:35,852 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:59:35,852 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:59:35,857 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:59:36,552 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:59:37,402 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:59:37,402 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:59:37,446 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:59:37,643 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:59:38,557 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:59:39,390 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:59:39,391 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:59:39,391 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:59:39,396 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:59:39,557 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:59:40,560 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:59:42,666 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:59:42,667 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:59:42,667 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:59:42,667 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:59:42,672 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:59:43,562 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:59:44,562 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:59:46,139 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:59:46,139 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:59:46,140 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:59:46,145 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:59:46,564 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:59:48,145 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:59:48,567 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:59:52,402 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 16:59:52,402 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 16:59:52,403 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 16:59:53,635 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 16:59:54,194 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 16:59:54,195 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 16:59:54,195 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 16:59:54,200 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 16:59:54,571 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 16:59:56,574 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 16:59:59,201 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:00:02,753 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 17:00:02,789 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:00:02,790 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:00:02,791 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:00:02,796 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:00:03,579 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:00:04,581 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:00:04,796 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:00:06,331 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:00:06,332 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:00:06,332 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:00:06,337 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:00:06,583 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:00:07,402 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:00:07,403 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:00:07,446 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:00:08,586 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:00:10,529 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:00:10,530 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:00:10,530 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:00:10,530 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:00:10,535 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:00:10,587 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:00:12,592 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:00:13,907 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:00:13,908 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:00:13,908 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:00:13,913 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:00:14,594 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:00:15,914 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:00:16,598 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:00:17,486 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:00:17,486 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:00:17,487 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:00:17,492 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:00:17,599 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:00:18,602 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:00:21,492 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:00:22,402 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:00:22,403 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:00:22,446 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:00:23,576 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:00:23,576 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:00:23,577 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:00:23,582 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:00:23,605 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:00:24,608 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:00:26,618 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:00:27,330 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:00:27,331 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:00:27,331 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:00:27,336 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:00:27,610 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:00:28,613 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:00:30,963 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:00:30,963 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:00:30,964 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:00:30,968 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:00:31,615 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:00:31,969 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:00:32,618 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:00:32,753 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 17:00:35,479 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:00:35,479 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:00:35,480 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:00:35,485 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:00:35,620 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:00:36,623 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:00:37,403 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:00:37,403 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:00:37,403 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:00:37,403 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:00:40,493 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:00:40,494 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:00:40,494 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:00:40,499 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:00:40,627 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:00:42,539 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:00:42,630 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:00:44,121 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:00:44,122 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:00:44,122 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:00:44,127 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:00:44,632 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:00:46,635 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:00:48,128 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:00:52,403 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:00:52,403 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:00:52,445 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:00:52,500 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:00:52,616 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:00:52,616 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:00:52,622 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:00:52,640 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:00:53,622 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:00:54,643 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:00:56,025 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:00:56,026 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:00:56,027 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:00:56,032 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:00:56,644 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:00:58,647 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:00:59,032 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:01:01,041 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:01:01,042 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:01:01,042 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:01:01,047 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:01:01,649 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:01:02,653 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:01:02,755 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 17:01:04,756 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:01:05,621 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:01:05,622 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:01:05,622 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:01:05,627 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:01:05,655 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:01:06,657 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:01:07,403 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:01:07,403 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:01:07,446 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:01:10,114 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:01:10,117 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:01:10,118 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:01:10,118 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:01:10,123 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:01:10,660 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:01:12,664 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:01:13,896 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:01:13,897 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:01:13,897 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:01:13,903 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:01:14,665 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:01:15,904 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:01:16,669 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:01:18,490 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:01:18,491 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:01:18,491 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:01:18,496 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:01:18,671 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:01:20,674 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:01:21,497 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:01:22,403 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:01:22,403 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:01:22,445 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:01:26,668 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:01:27,111 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:01:27,111 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:01:27,111 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:01:27,117 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:01:27,678 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:01:28,681 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:01:32,155 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:01:32,757 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 17:01:33,114 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:01:33,114 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:01:33,114 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:01:33,120 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:01:33,685 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:01:34,688 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:01:36,674 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:01:36,674 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:01:36,675 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:01:36,680 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:01:36,689 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:01:37,403 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:01:37,403 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:01:37,403 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:01:37,403 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:01:38,694 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:01:40,493 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:01:40,494 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:01:40,494 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:01:40,499 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:01:40,695 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:01:42,535 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:01:42,698 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:01:44,214 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:01:44,215 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:01:44,215 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:01:44,222 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:01:44,699 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:01:46,703 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:01:47,489 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:01:47,489 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:01:47,489 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:01:47,495 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:01:47,703 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:01:48,495 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:01:48,709 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:01:52,244 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:01:52,244 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:01:52,245 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:01:52,250 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:01:52,403 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:01:52,403 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:01:52,446 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:01:52,711 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:01:53,665 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:01:54,714 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:01:55,241 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:01:55,242 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:01:55,242 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:01:55,247 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:01:55,715 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:01:56,718 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:01:59,283 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:02:00,479 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:02:00,480 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:02:00,480 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:02:00,485 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:02:00,720 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:02:02,723 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:02:02,759 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 17:02:04,761 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:02:06,520 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:02:06,521 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:02:06,521 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:02:06,526 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:02:06,726 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:02:07,403 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:02:07,403 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:02:07,446 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:02:08,730 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:02:10,636 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:02:12,350 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:02:12,351 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:02:12,351 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:02:12,356 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:02:12,732 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:02:14,736 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:02:16,242 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:02:16,243 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:02:16,243 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:02:16,243 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:02:16,248 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:02:16,739 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:02:18,743 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:02:19,603 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:02:19,604 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:02:19,604 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:02:19,609 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:02:19,743 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:02:20,746 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:02:21,610 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:02:22,403 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:02:22,403 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:02:22,403 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:02:24,203 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:02:24,203 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:02:24,203 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:02:24,209 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:02:24,748 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:02:26,752 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:02:27,250 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:02:28,760 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:02:28,761 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:02:28,761 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:02:28,766 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:02:29,754 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:02:30,757 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:02:32,761 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 17:02:32,764 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:02:33,256 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:02:33,257 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:02:33,257 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:02:33,262 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:02:33,759 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:02:34,762 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:02:37,403 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:02:37,403 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:02:37,446 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:02:37,524 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:02:38,249 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:02:38,249 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:02:38,249 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:02:38,254 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:02:38,767 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:02:40,771 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:02:43,255 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:02:45,983 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:02:45,983 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:02:45,984 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:02:45,989 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:02:46,775 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:02:48,779 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:02:48,990 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:02:52,403 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:02:52,404 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:02:52,405 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:02:54,296 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:02:54,296 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:02:54,297 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:02:54,297 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:02:54,302 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:02:54,783 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:02:56,786 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:02:59,339 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:02:59,490 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:02:59,490 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:02:59,491 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:02:59,496 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:02:59,789 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:03:00,792 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:03:02,763 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 17:03:03,240 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:03:03,241 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:03:03,241 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:03:03,246 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:03:03,794 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:03:04,797 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:03:05,246 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:03:06,533 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:03:06,534 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:03:06,534 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:03:06,539 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:03:06,798 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:03:07,403 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:03:07,404 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:03:07,446 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:03:08,801 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:03:09,770 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:03:09,771 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:03:09,771 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:03:09,776 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:03:09,802 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:03:10,805 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:03:10,815 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:03:13,192 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:03:13,193 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:03:13,193 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:03:13,199 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:03:13,807 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:03:14,808 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:03:16,199 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:03:21,200 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:03:21,598 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:03:21,598 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:03:21,599 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:03:21,604 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:03:21,814 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:03:22,403 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:03:22,404 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:03:22,445 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:03:22,817 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:03:24,969 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:03:24,970 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:03:24,970 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:03:24,975 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:03:25,819 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:03:26,822 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:03:27,015 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:03:32,015 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:03:32,769 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 17:03:33,126 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:03:33,127 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:03:33,127 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:03:33,132 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:03:33,828 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:03:34,831 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:03:36,653 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:03:36,654 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:03:36,654 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:03:36,660 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:03:36,833 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:03:37,403 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:03:37,404 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:03:37,404 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:03:37,404 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:03:38,838 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:03:42,623 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:03:44,758 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:03:44,758 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:03:44,758 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:03:44,764 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:03:44,842 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:03:46,846 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:03:47,803 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:03:48,280 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:03:48,281 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:03:48,281 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:03:48,286 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:03:48,848 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:03:50,851 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:03:51,813 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:03:51,814 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:03:51,814 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:03:51,819 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:03:51,852 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:03:52,404 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:03:52,404 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:03:52,445 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:03:52,857 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:03:53,667 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:03:56,602 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:03:56,603 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:03:56,603 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:03:56,608 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:03:56,860 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:03:58,863 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:03:59,647 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:04:00,574 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:04:00,575 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:04:00,575 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:04:00,580 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:04:00,865 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:04:02,768 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 17:04:02,868 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:04:04,298 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:04:04,299 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:04:04,299 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:04:04,305 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:04:04,870 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:04:05,306 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:04:06,873 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:04:07,404 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:04:07,404 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:04:07,445 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:04:10,724 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:04:12,526 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:04:12,527 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:04:12,527 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:04:12,532 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:04:12,877 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:04:14,881 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:04:16,571 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:04:17,890 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:04:17,891 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:04:17,891 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:04:17,896 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:04:18,887 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:04:18,887 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:04:21,224 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:04:21,225 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:04:21,225 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:04:21,231 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:04:21,889 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:04:22,231 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:04:22,404 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:04:22,404 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:04:22,446 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:04:22,892 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:04:25,437 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:04:25,437 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:04:25,437 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:04:25,442 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:04:25,894 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:04:26,897 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:04:27,479 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:04:28,893 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:04:28,894 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:04:28,894 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:04:28,900 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:04:28,900 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:04:30,904 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:04:32,336 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:04:32,337 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:04:32,337 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:04:32,342 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:04:32,774 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 17:04:32,775 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:04:32,906 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:04:34,910 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:04:35,952 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:04:35,953 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:04:35,953 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:04:35,960 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:04:36,911 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:04:37,404 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:04:37,404 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:04:37,404 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:04:38,691 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:04:38,917 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:04:39,238 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:04:39,239 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:04:39,239 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:04:39,244 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:04:39,918 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:04:40,920 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:04:44,283 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:04:45,854 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:04:45,854 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:04:45,855 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:04:45,859 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:04:45,923 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:04:46,926 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:04:49,860 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:04:49,980 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:04:49,981 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:04:49,981 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:04:49,986 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:04:50,930 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:04:52,404 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:04:52,404 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:04:52,446 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:04:52,933 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:04:53,857 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:04:53,857 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:04:53,857 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:04:53,863 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:04:53,934 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:04:54,899 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:04:54,936 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:04:57,306 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:04:57,307 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:04:57,308 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:04:57,312 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:04:57,939 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:04:58,942 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:05:00,313 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:05:00,696 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:05:00,697 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:05:00,697 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:05:00,702 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:05:00,944 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:05:02,773 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 17:05:02,947 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:05:04,345 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:05:04,346 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:05:04,347 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:05:04,352 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:05:04,949 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:05:05,353 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:05:06,953 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:05:07,404 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:05:07,404 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:05:07,445 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:05:08,848 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:05:08,848 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:05:08,849 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:05:08,855 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:05:08,954 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:05:10,891 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:05:10,957 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:05:13,344 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:05:13,344 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:05:13,345 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:05:13,350 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:05:13,960 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:05:14,963 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:05:16,350 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:05:17,606 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:05:17,606 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:05:17,607 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:05:17,612 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:05:17,966 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:05:18,969 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:05:21,613 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:05:22,153 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:05:22,153 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:05:22,154 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:05:22,159 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:05:22,404 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:05:22,404 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:05:22,446 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:05:22,971 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:05:24,975 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:05:26,022 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:05:26,023 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:05:26,023 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:05:26,029 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:05:26,977 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:05:27,067 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:05:28,980 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:05:29,644 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:05:29,645 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:05:29,645 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:05:29,651 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:05:29,982 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:05:30,985 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:05:32,652 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:05:32,778 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 17:05:37,404 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:05:37,404 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:05:37,446 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:05:37,884 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:05:38,758 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:05:38,759 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:05:38,759 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:05:38,764 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:05:38,993 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:05:40,997 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:05:43,803 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:05:45,892 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:05:45,893 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:05:45,893 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:05:45,898 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:05:46,000 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:05:47,003 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:05:48,899 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:05:49,993 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:05:49,993 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:05:49,993 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:05:49,999 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:05:50,005 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:05:51,008 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:05:52,404 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:05:52,404 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:05:52,445 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:05:53,503 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:05:53,504 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:05:53,504 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:05:53,509 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:05:54,011 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:05:54,547 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:05:55,013 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:05:58,442 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:05:58,443 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:05:58,443 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:05:58,449 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:05:59,016 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:06:00,449 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:06:01,020 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:06:02,203 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:06:02,203 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:06:02,204 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:06:02,209 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:06:02,777 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 17:06:03,022 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:06:05,026 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:06:05,779 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:06:06,085 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:06:06,086 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:06:06,086 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:06:06,092 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:06:07,027 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:06:07,404 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:06:07,405 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:06:07,446 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:06:09,031 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:06:09,460 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:06:09,461 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:06:09,461 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:06:09,467 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:06:10,032 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:06:11,035 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:06:11,503 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:06:13,020 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:06:13,021 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:06:13,021 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:06:13,027 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:06:13,036 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:06:15,040 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:06:17,028 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:06:17,980 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:06:17,980 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:06:17,980 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:06:17,987 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:06:18,043 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:06:19,047 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:06:21,517 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:06:21,517 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:06:21,518 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:06:21,523 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:06:22,049 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:06:22,404 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:06:22,405 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:06:22,405 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:06:22,405 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:06:23,051 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:06:25,975 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:06:25,976 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:06:25,976 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:06:25,981 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:06:26,053 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:06:27,056 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:06:28,019 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:06:30,325 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:06:30,326 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:06:30,327 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:06:30,332 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:06:31,058 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:06:32,779 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 17:06:33,062 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:06:33,780 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:06:33,820 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:06:33,821 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:06:33,821 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:06:33,826 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:06:34,062 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:06:35,066 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:06:37,405 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:06:37,405 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:06:37,446 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:06:39,169 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:06:39,797 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:06:39,797 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:06:39,797 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:06:39,803 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:06:40,071 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:06:41,074 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:06:44,839 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:06:45,408 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:06:45,409 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:06:45,409 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:06:45,414 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:06:46,077 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:06:47,080 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:06:50,414 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:06:52,405 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:06:52,405 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:06:52,446 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:06:54,551 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:06:54,551 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:06:54,551 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:06:54,556 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:06:55,085 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:06:55,595 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:06:57,088 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:07:00,595 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:07:02,781 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 17:07:03,070 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:07:03,071 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:07:03,071 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:07:03,077 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:07:03,092 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:07:05,096 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:07:06,077 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:07:07,405 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:07:07,405 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:07:07,446 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:07:08,277 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:07:09,554 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:07:09,554 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:07:09,560 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:07:10,100 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:07:11,102 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:07:11,561 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:07:12,788 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:07:12,788 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:07:12,789 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:07:12,794 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:07:13,104 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:07:15,107 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:07:16,794 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:07:18,656 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:07:18,656 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:07:18,656 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:07:18,662 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:07:19,111 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:07:21,115 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:07:22,405 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:07:22,405 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:07:22,405 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:07:22,446 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:07:22,693 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:07:22,694 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:07:22,694 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:07:22,699 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:07:23,116 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:07:25,119 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:07:27,422 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:07:27,423 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:07:27,423 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:07:27,423 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:07:27,429 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:07:28,121 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:07:29,124 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:07:31,583 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:07:31,584 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:07:31,584 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:07:31,589 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:07:32,126 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:07:32,590 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:07:32,784 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 17:07:33,129 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:07:37,405 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:07:37,405 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:07:37,406 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:07:37,614 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:07:38,394 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:07:38,394 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:07:38,395 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:07:38,400 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:07:39,134 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:07:41,138 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:07:42,146 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:07:42,146 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:07:42,147 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:07:42,152 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:07:43,141 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:07:43,141 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:07:43,152 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:07:47,085 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:07:47,085 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:07:47,086 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:07:47,091 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:07:47,143 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:07:49,091 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:07:49,147 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:07:50,186 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:07:50,187 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:07:50,187 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:07:50,192 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:07:51,148 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:07:52,405 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:07:52,405 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:07:52,406 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:07:53,152 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:07:53,898 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:07:53,899 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:07:53,899 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:07:53,904 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:07:54,153 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:07:54,943 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:07:55,155 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:07:59,565 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:07:59,566 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:07:59,567 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:07:59,572 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:08:00,158 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:08:00,572 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:08:01,161 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:08:02,789 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 17:08:03,294 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:08:03,295 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:08:03,295 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:08:03,300 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:08:04,163 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:08:05,166 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:08:06,300 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:08:07,405 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:08:07,405 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:08:07,406 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:08:11,656 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:08:11,734 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:08:11,734 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:08:11,734 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:08:11,739 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:08:12,170 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:08:13,173 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:08:15,500 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:08:15,501 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:08:15,501 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:08:15,506 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:08:16,176 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:08:17,178 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:08:17,507 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:08:19,438 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:08:19,438 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:08:19,439 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:08:19,443 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:08:20,181 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:08:21,183 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:08:22,405 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:08:22,405 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:08:22,405 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:08:22,703 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:08:23,385 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:08:23,385 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:08:23,385 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:08:23,390 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:08:24,185 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:08:25,188 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:08:27,502 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:08:27,502 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:08:27,502 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:08:27,507 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:08:28,189 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:08:28,508 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:08:29,192 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:08:32,788 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 17:08:33,789 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:08:34,827 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:08:34,828 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:08:34,828 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:08:34,833 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:08:35,198 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:08:37,202 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:08:37,405 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:08:37,405 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:08:37,406 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:08:39,347 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:08:40,062 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:08:40,063 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:08:40,063 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:08:40,068 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:08:40,205 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:08:41,207 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:08:45,107 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:08:45,893 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:08:45,894 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:08:45,894 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:08:45,900 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:08:46,211 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:08:47,213 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:08:49,887 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:08:49,888 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:08:49,888 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:08:49,893 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:08:50,215 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:08:50,893 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:08:51,218 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:08:52,405 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:08:52,405 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:08:52,407 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:08:53,470 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:08:53,471 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:08:53,471 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:08:53,476 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:08:54,221 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:08:55,223 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:08:56,515 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:08:57,105 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:08:57,106 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:08:57,106 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:08:57,111 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:08:57,224 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:08:59,227 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:09:02,112 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:09:02,793 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 17:09:05,939 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:09:05,939 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:09:05,940 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:09:05,945 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:09:06,232 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:09:07,235 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:09:07,405 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:09:07,405 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:09:07,406 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:09:07,407 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:09:09,451 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:09:09,452 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:09:09,452 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:09:09,457 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:09:10,237 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:09:11,239 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:09:12,495 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:09:13,315 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:09:13,315 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:09:13,316 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:09:13,321 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:09:14,241 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:09:15,244 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:09:17,009 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:09:17,009 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:09:17,010 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:09:17,015 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:09:17,246 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:09:18,016 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:09:19,250 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:09:22,196 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:09:22,196 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:09:22,197 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:09:22,202 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:09:22,252 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:09:22,405 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:09:22,406 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:09:22,407 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:09:23,255 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:09:23,699 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:09:25,562 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:09:25,562 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:09:25,563 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:09:25,567 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:09:26,257 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:09:27,259 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:09:29,561 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:09:29,562 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:09:29,562 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:09:29,563 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:09:29,567 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:09:30,261 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:09:31,264 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:09:32,793 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 17:09:33,344 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:09:33,345 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:09:33,345 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:09:33,350 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:09:34,266 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:09:35,269 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:09:35,350 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:09:37,405 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:09:37,406 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:09:37,407 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:09:38,736 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:09:38,736 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:09:38,736 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:09:38,741 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:09:39,274 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:09:40,779 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:09:41,277 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:09:43,910 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:09:43,911 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:09:43,911 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:09:43,916 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:09:44,279 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:09:45,280 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:09:45,917 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:09:48,071 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:09:48,072 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:09:48,072 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:09:48,077 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:09:48,281 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:09:49,285 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:09:51,078 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:09:52,406 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:09:52,406 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:09:52,407 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:09:52,898 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:09:52,898 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:09:52,898 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:09:52,903 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:09:53,287 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:09:55,291 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:09:56,598 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:09:56,599 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:09:56,600 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:09:56,600 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:09:56,605 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:09:57,292 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:09:59,295 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:10:00,162 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:10:00,163 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:10:00,163 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:10:00,168 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:10:00,296 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:10:01,298 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:10:02,169 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:10:02,798 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 17:10:04,517 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:10:04,517 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:10:04,517 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:10:04,522 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:10:05,301 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:10:07,305 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:10:07,406 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:10:07,406 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:10:07,406 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:10:07,406 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:10:08,053 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:10:08,059 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:10:08,059 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:10:08,064 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:10:08,305 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:10:09,309 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:10:13,065 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:10:13,395 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:10:13,395 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:10:13,396 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:10:13,401 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:10:14,312 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:10:15,315 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:10:18,218 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:10:18,219 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:10:18,219 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:10:18,220 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:10:18,224 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:10:18,318 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:10:19,321 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:10:21,967 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:10:21,968 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:10:21,968 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:10:21,973 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:10:22,323 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:10:22,406 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:10:22,406 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:10:22,406 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:10:23,325 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:10:23,643 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:10:25,522 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:10:25,523 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:10:25,523 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:10:25,528 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:10:26,327 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:10:27,330 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:10:29,150 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:10:29,151 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:10:29,152 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:10:29,152 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:10:29,157 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:10:29,331 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:10:31,335 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:10:32,674 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:10:32,675 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:10:32,675 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:10:32,680 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:10:32,797 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 17:10:33,336 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:10:34,798 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:10:35,340 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:10:36,134 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:10:36,135 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:10:36,135 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:10:36,140 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:10:36,341 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:10:37,343 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:10:37,406 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:10:37,406 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:10:37,406 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:10:40,531 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:10:42,357 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:10:42,358 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:10:42,358 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:10:42,363 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:10:43,349 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:10:45,352 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:10:46,399 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:10:46,935 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:10:46,935 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:10:46,935 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:10:46,940 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:10:47,353 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:10:49,357 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:10:51,941 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:10:52,406 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:10:52,406 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:10:52,407 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:10:55,316 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:10:55,317 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:10:55,317 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:10:55,322 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:10:55,361 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:10:57,359 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:10:57,364 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:10:59,057 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:10:59,058 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:10:59,059 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:10:59,064 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:10:59,365 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:11:01,369 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:11:02,721 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:11:02,722 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:11:02,722 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:11:02,722 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:11:02,727 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:11:02,803 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 17:11:03,370 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:11:05,378 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:11:06,726 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:11:06,727 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:11:06,727 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:11:06,732 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:11:07,379 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:11:07,406 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:11:07,406 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:11:07,406 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:11:08,688 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:11:09,382 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:11:12,092 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:11:12,092 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:11:12,093 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:11:12,097 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:11:12,384 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:11:13,387 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:11:14,135 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:11:16,744 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:11:16,745 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:11:16,745 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:11:16,751 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:11:17,391 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:11:19,396 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:11:19,752 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:11:20,877 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:11:20,878 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:11:20,878 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:11:20,883 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:11:21,398 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:11:22,406 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:11:22,406 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:11:22,407 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:11:23,401 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:11:25,642 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:11:25,974 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:11:25,974 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:11:25,974 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:11:25,980 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:11:26,403 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:11:27,404 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:11:30,057 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:11:30,057 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:11:30,058 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:11:30,062 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:11:30,406 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:11:31,063 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:11:31,409 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:11:32,802 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 17:11:33,690 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:11:33,690 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:11:33,691 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:11:33,696 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:11:34,411 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:11:35,413 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:11:36,696 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:11:37,406 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:11:37,406 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:11:37,406 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:11:39,747 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:11:39,748 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:11:39,748 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:11:39,753 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:11:40,418 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:11:41,420 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:11:41,791 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:11:43,479 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:11:43,480 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:11:43,480 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:11:43,485 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:11:44,422 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:11:45,425 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:11:46,734 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:11:46,735 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:11:46,735 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:11:46,740 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:11:47,426 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:11:47,740 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:11:49,430 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:11:51,557 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:11:51,558 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:11:51,558 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:11:51,563 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:11:52,406 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:11:52,407 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:11:52,407 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:11:52,432 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:11:53,434 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:11:53,692 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:11:55,622 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:11:55,622 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:11:55,623 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:11:55,628 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:11:56,436 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:11:57,439 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:11:59,667 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:12:00,789 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:12:00,789 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:12:00,790 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:12:00,795 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:12:01,441 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:12:02,804 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 17:12:03,443 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:12:04,805 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:12:05,186 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:12:05,186 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:12:05,187 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:12:05,192 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:12:05,444 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:12:07,406 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:12:07,407 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:12:07,407 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:12:07,448 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:12:08,832 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:12:08,833 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:12:08,833 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:12:08,838 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:12:09,449 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:12:09,875 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:12:11,452 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:12:12,092 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:12:12,093 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:12:12,093 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:12:12,097 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:12:12,453 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:12:15,098 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:12:15,457 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:12:20,099 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:12:21,154 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:12:21,155 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:12:21,156 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:12:21,160 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:12:21,462 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:12:22,406 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:12:22,407 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:12:22,407 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:12:23,465 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:12:25,515 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:12:25,515 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:12:25,516 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:12:25,516 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:12:25,521 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:12:26,467 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:12:27,470 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:12:28,711 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:12:28,712 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:12:28,712 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:12:28,717 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:12:29,471 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:12:30,718 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:12:31,475 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:12:32,020 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:12:32,021 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:12:32,021 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:12:32,026 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:12:32,475 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:12:32,806 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 17:12:35,480 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:12:35,685 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:12:35,686 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:12:35,686 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:12:35,691 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:12:36,481 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:12:36,692 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:12:37,406 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:12:37,407 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:12:37,407 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:12:37,484 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:12:41,706 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:12:44,165 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:12:44,165 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:12:44,166 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:12:44,171 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:12:44,490 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:12:47,207 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:12:47,494 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:12:48,259 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:12:48,259 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:12:48,259 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:12:48,265 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:12:48,495 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:12:51,499 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:12:51,762 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:12:51,763 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:12:51,763 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:12:51,769 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:12:52,407 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:12:52,407 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:12:52,407 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:12:52,407 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:12:52,500 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:12:53,503 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:12:55,252 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:12:55,253 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:12:55,253 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:12:55,258 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:12:55,504 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:12:57,508 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:12:58,295 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:12:58,704 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:12:58,705 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:12:58,705 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:12:58,710 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:12:59,509 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:13:01,513 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:13:02,459 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:13:02,460 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:13:02,460 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:13:02,465 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:13:02,513 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:13:02,808 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 17:13:03,809 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:13:05,518 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:13:06,125 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:13:06,126 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:13:06,126 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:13:06,131 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:13:06,518 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:13:07,407 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:13:07,407 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:13:07,407 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:13:09,522 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:13:09,630 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:13:09,630 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:13:09,631 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:13:09,631 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:13:09,636 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:13:10,523 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:13:11,525 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:13:14,162 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:13:14,162 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:13:14,162 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:13:14,168 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:13:14,527 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:13:15,168 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:13:17,532 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:13:20,169 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:13:22,407 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:13:22,407 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:13:22,407 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:13:22,649 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:13:22,676 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:13:22,676 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:13:22,681 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:13:23,537 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:13:25,540 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:13:25,682 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:13:26,950 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:13:26,951 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:13:26,951 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:13:26,956 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:13:27,541 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:13:29,545 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:13:30,493 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:13:30,494 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:13:30,494 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:13:30,500 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:13:30,546 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:13:31,500 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:13:32,813 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 17:13:33,550 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:13:35,215 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:13:35,216 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:13:35,216 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:13:35,221 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:13:35,551 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:13:37,221 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:13:37,407 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:13:37,407 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:13:37,407 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:13:37,555 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:13:42,458 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:13:43,761 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:13:43,762 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:13:43,762 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:13:43,767 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:13:44,562 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:13:45,565 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:13:47,803 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:13:48,690 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:13:48,691 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:13:48,691 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:13:48,696 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:13:49,568 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:13:51,571 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:13:52,407 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:13:52,407 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:13:52,408 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:13:53,602 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:13:54,033 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:13:54,034 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:13:54,034 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:13:54,040 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:13:54,573 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:13:55,574 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:13:57,647 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:13:57,648 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:13:57,648 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:13:57,653 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:13:58,576 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:13:58,654 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:13:59,578 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:14:00,895 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:14:00,896 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:14:00,897 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:14:00,901 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:14:01,580 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:14:02,812 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 17:14:03,583 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:14:03,813 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:14:04,689 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:14:04,690 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:14:04,691 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:14:04,695 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:14:05,585 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:14:07,407 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:14:07,407 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:14:07,407 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:14:07,588 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:14:08,392 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:14:08,393 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:14:08,393 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:14:08,398 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:14:08,589 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:14:09,435 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:14:11,593 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:14:13,572 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:14:13,573 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:14:13,573 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:14:13,578 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:14:13,594 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:14:14,579 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:14:14,598 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:14:18,957 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:14:18,958 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:14:18,958 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:14:18,963 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:14:19,603 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:14:19,964 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:14:20,605 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:14:22,407 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:14:22,407 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:14:22,407 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:14:25,620 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:14:27,233 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:14:27,234 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:14:27,234 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:14:27,239 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:14:27,610 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:14:28,612 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:14:30,568 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:14:30,569 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:14:30,569 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:14:30,575 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:14:30,614 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:14:31,575 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:14:32,617 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:14:32,817 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 17:14:36,140 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:14:36,141 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:14:36,141 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:14:36,147 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:14:36,620 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:14:37,147 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:14:37,407 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:14:37,407 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:14:37,408 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:14:38,625 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:14:39,694 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:14:39,694 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:14:39,695 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:14:39,700 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:14:40,628 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:14:40,629 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:14:42,739 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:14:47,739 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:14:48,150 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:14:48,151 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:14:48,151 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:14:48,157 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:14:48,636 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:14:48,636 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:14:52,407 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:14:52,408 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:14:52,408 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:14:53,693 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:14:54,074 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:14:54,075 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:14:54,075 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:14:54,080 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:14:54,642 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:14:54,643 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:14:57,026 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:14:57,027 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:14:57,027 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:14:57,032 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:14:57,644 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:14:58,647 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:14:59,032 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:15:02,134 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:15:02,135 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:15:02,135 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:15:02,140 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:15:02,652 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:15:02,652 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:15:02,816 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 17:15:04,818 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:15:07,407 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:15:07,408 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:15:07,408 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:15:10,634 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:15:10,650 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:15:10,650 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:15:10,651 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:15:10,657 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:15:10,657 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:15:12,661 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:15:14,134 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:15:14,135 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:15:14,135 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:15:14,140 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:15:14,664 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:15:14,664 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:15:16,140 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:15:17,846 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:15:17,847 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:15:17,847 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:15:17,852 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:15:18,670 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:15:18,670 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:15:20,990 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:15:20,991 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:15:20,991 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:15:20,997 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:15:21,672 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:15:21,997 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:15:22,407 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:15:22,407 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:15:22,408 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:15:22,675 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:15:25,177 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:15:25,178 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:15:25,178 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:15:25,184 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:15:25,677 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:15:26,680 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:15:27,223 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:15:30,645 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:15:30,646 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:15:30,646 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:15:30,651 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:15:30,682 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:15:32,652 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:15:32,686 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:15:32,821 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 17:15:33,953 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:15:33,953 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:15:33,954 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:15:33,958 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:15:34,689 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:15:34,689 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:15:37,408 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:15:37,408 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:15:37,408 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:15:37,657 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:15:37,840 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:15:37,841 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:15:37,841 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:15:37,846 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:15:38,695 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:15:38,695 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:15:42,883 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:15:47,419 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:15:47,420 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:15:47,421 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:15:47,426 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:15:47,701 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:15:48,426 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:15:48,704 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:15:52,408 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:15:52,408 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:15:52,409 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:15:53,617 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:15:55,822 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:15:55,823 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:15:55,823 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:15:55,828 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:15:56,711 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:15:56,712 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:15:58,867 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:15:59,045 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:15:59,046 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:15:59,046 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:15:59,051 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:15:59,713 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:16:00,716 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:16:02,671 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:16:02,671 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:16:02,672 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:16:02,677 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:16:02,717 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:16:02,821 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 17:16:04,720 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:16:04,822 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:16:06,948 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:16:06,949 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:16:06,949 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:16:06,954 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:16:07,408 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:16:07,408 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:16:07,408 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:16:07,722 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:16:08,725 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:16:10,484 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:16:10,484 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:16:10,485 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:16:10,485 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:16:10,489 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:16:10,726 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:16:12,729 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:16:15,281 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:16:15,282 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:16:15,282 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:16:15,287 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:16:15,731 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:16:16,287 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:16:16,734 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:16:18,266 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:16:18,267 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:16:18,267 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:16:18,272 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:16:18,736 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:16:20,739 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:16:22,273 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:16:22,288 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:16:22,288 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:16:22,289 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:16:22,293 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:16:22,408 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:16:22,408 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:16:22,408 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:16:22,741 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:16:24,745 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:16:27,640 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:16:31,572 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:16:31,572 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:16:31,573 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:16:31,577 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:16:31,749 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:16:32,752 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:16:32,823 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 17:16:32,824 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:16:35,749 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:16:35,750 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:16:35,750 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:16:35,755 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:16:35,755 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:16:36,758 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:16:37,408 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:16:37,408 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:16:37,409 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:16:38,689 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:16:41,963 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:16:41,963 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:16:41,964 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:16:41,969 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:16:42,765 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:16:42,765 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:16:44,010 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:16:45,883 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:16:45,884 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:16:45,884 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:16:45,889 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:16:46,770 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:16:46,770 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:16:49,890 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:16:50,819 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:16:50,820 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:16:50,820 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:16:50,825 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:16:51,774 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:16:52,408 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:16:52,409 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:16:52,409 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:16:52,776 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:16:54,997 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:16:54,997 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:16:54,997 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:16:54,998 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:16:55,002 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:16:55,778 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:16:56,781 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:17:00,039 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:17:00,637 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:17:00,638 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:17:00,638 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:17:00,643 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:17:00,783 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:17:02,787 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:17:02,825 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 17:17:05,827 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:17:06,419 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:17:06,420 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:17:06,420 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:17:06,425 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:17:06,789 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:17:07,408 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:17:07,408 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:17:07,409 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:17:08,792 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:17:11,183 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:17:11,713 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:17:11,714 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:17:11,714 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:17:11,719 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:17:11,794 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:17:12,797 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:17:16,755 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:17:17,471 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:17:17,472 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:17:17,472 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:17:17,477 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:17:17,801 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:17:18,804 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:17:22,045 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:17:22,046 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:17:22,046 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:17:22,046 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:17:22,051 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:17:22,408 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:17:22,408 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:17:22,409 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:17:22,808 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:17:22,809 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:17:25,571 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:17:25,571 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:17:25,572 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:17:25,576 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:17:25,810 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:17:26,813 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:17:27,615 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:17:28,951 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:17:28,952 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:17:28,952 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:17:28,957 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:17:29,815 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:17:30,817 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:17:32,828 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 17:17:32,830 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:17:32,842 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:17:32,842 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:17:32,843 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:17:32,847 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:17:33,819 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:17:34,822 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:17:37,408 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:17:37,409 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:17:37,409 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:17:37,618 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:17:37,700 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:17:37,700 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:17:37,705 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:17:37,824 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:17:38,706 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:17:38,828 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:17:42,134 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:17:42,135 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:17:42,135 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:17:42,140 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:17:42,833 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:17:42,833 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:17:44,179 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:17:45,794 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:17:45,795 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:17:45,795 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:17:45,800 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:17:45,835 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:17:46,837 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:17:49,800 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:17:51,483 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:17:51,484 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:17:51,484 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:17:51,488 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:17:51,841 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:17:52,409 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:17:52,409 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:17:52,409 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:17:52,844 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:17:55,600 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:17:56,064 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:17:56,065 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:17:56,065 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:17:56,070 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:17:56,849 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:17:56,849 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:18:01,107 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:18:01,139 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:18:01,143 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:18:01,143 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:18:01,148 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:18:01,852 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:18:02,830 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 17:18:02,942 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:18:06,085 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:18:06,086 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:18:06,086 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:18:06,091 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:18:06,947 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:18:06,947 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:18:07,092 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:18:07,409 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:18:07,409 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:18:07,409 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:18:09,735 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:18:09,736 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:18:09,736 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:18:09,741 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:18:09,949 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:18:10,951 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:18:12,779 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:18:15,511 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:18:15,512 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:18:15,512 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:18:15,518 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:18:15,954 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:18:16,958 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:18:18,518 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:18:18,848 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:18:18,849 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:18:18,850 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:18:18,855 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:18:18,959 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:18:20,962 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:18:22,146 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:18:22,147 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:18:22,147 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:18:22,152 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:18:22,409 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:18:22,409 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:18:22,410 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:18:22,965 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:18:22,966 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:18:23,703 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:18:26,761 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:18:26,761 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:18:26,762 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:18:26,766 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:18:26,968 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:18:28,803 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:18:28,971 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:18:31,462 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:18:31,463 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:18:31,463 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:18:31,468 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:18:31,973 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:18:32,832 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 17:18:32,976 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:18:33,833 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:18:34,983 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:18:34,984 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:18:34,984 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:18:34,989 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:18:35,978 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:18:36,981 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:18:37,409 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:18:37,409 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:18:37,409 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:18:39,669 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:18:42,063 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:18:42,064 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:18:42,064 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:18:42,069 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:18:42,988 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:18:42,989 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:18:45,107 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:18:46,151 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:18:46,151 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:18:46,151 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:18:46,157 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:18:46,996 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:18:46,996 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:18:49,642 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:18:49,643 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:18:49,643 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:18:49,648 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:18:49,998 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:18:50,648 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:18:51,001 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:18:52,409 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:18:52,409 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:18:52,409 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:18:53,098 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:18:53,099 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:18:53,099 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:18:53,104 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:18:54,003 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:18:55,005 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:18:56,143 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:18:56,467 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:18:56,468 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:18:56,468 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:18:56,473 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:18:57,007 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:18:59,010 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:19:00,887 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:19:00,887 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:19:00,888 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:19:00,892 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:19:01,011 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:19:01,893 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:19:02,837 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 17:19:03,015 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:19:06,008 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:19:06,009 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:19:06,010 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:19:06,014 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:19:06,016 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:19:07,015 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:19:07,019 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:19:07,409 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:19:07,409 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:19:07,409 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:19:09,635 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:19:09,636 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:19:09,636 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:19:09,641 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:19:10,021 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:19:11,024 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:19:12,679 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:19:15,365 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:19:15,366 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:19:15,367 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:19:15,372 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:19:16,027 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:19:17,030 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:19:18,373 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:19:20,380 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:19:20,381 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:19:20,381 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:19:20,386 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:19:21,035 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:19:21,035 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:19:22,409 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:19:22,409 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:19:22,410 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:19:23,649 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:19:26,415 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:19:26,416 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:19:26,416 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:19:26,421 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:19:27,041 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:19:27,041 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:19:29,459 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:19:30,154 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:19:30,155 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:19:30,155 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:19:30,160 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:19:31,045 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:19:31,046 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:19:32,836 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 17:19:34,204 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:19:34,205 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:19:34,206 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:19:34,211 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:19:35,050 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:19:35,050 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:19:35,211 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:19:37,409 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:19:37,409 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:19:37,410 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:19:37,757 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:19:37,952 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:19:37,952 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:19:37,957 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:19:38,052 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:19:39,056 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:19:40,957 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:19:43,150 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:19:43,151 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:19:43,151 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:19:43,156 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:19:44,059 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:19:45,062 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:19:46,157 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:19:47,136 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:19:47,137 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:19:47,137 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:19:47,142 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:19:48,064 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:19:49,066 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:19:50,641 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:19:50,642 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:19:50,642 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:19:50,647 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:19:51,068 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:19:51,648 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:19:52,409 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:19:52,409 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:19:52,410 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:19:53,072 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:19:54,703 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:19:54,704 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:19:54,704 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:19:54,709 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:19:55,073 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:19:56,747 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:19:57,076 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:20:01,099 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:20:01,100 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:20:01,100 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:20:01,105 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:20:02,079 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:20:02,106 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:20:02,841 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 17:20:03,082 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:20:06,570 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:20:06,571 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:20:06,571 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:20:06,576 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:20:07,084 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:20:07,409 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:20:07,409 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:20:07,410 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:20:07,612 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:20:09,088 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:20:10,141 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:20:10,141 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:20:10,142 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:20:10,147 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:20:11,092 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:20:11,092 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:20:13,052 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:20:13,053 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:20:13,053 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:20:13,053 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:20:13,058 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:20:13,093 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:20:15,096 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:20:18,059 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:20:21,021 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:20:21,022 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:20:21,022 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:20:21,027 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:20:21,101 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:20:22,410 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:20:22,410 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:20:22,410 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:20:23,104 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:20:23,695 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:20:28,695 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:20:28,842 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:20:28,843 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:20:28,843 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:20:28,848 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:20:29,108 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:20:31,111 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:20:32,840 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 17:20:33,435 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:20:33,436 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:20:33,437 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:20:33,441 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:20:34,113 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:20:34,442 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:20:35,116 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:20:37,409 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:20:37,409 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:20:37,410 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:20:38,207 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:20:38,208 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:20:38,208 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:20:38,213 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:20:39,122 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:20:39,122 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:20:40,251 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:20:43,593 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:20:43,594 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:20:43,594 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:20:43,599 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:20:44,125 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:20:45,128 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:20:45,600 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:20:47,402 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:20:47,403 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:20:47,403 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:20:47,408 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:20:48,130 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:20:49,132 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:20:51,405 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:20:51,406 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:20:51,406 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:20:51,407 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:20:51,411 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:20:52,135 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:20:52,410 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:20:52,410 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:20:52,410 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:20:53,137 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:20:55,303 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:20:55,304 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:20:55,304 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:20:55,309 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:20:56,139 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:20:57,142 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:20:57,347 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:20:58,851 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:20:58,852 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:20:58,852 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:20:58,857 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:20:59,143 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:21:01,146 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:21:02,846 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 17:21:02,847 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:21:03,649 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:21:03,650 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:21:03,650 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:21:03,655 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:21:04,148 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:21:05,151 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:21:07,280 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:21:07,281 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:21:07,281 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:21:07,286 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:21:07,410 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:21:07,410 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:21:07,410 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:21:08,153 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:21:08,689 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:21:09,155 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:21:11,798 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:21:11,799 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:21:11,799 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:21:11,804 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:21:12,157 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:21:13,160 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:21:13,843 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:21:15,276 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:21:15,277 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:21:15,277 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:21:15,281 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:21:16,162 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:21:17,165 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:21:19,282 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:21:20,979 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:21:20,980 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:21:20,981 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:21:20,985 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:21:21,168 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:21:22,410 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:21:22,410 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:21:22,410 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:21:23,171 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:21:24,838 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:21:29,388 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:21:29,389 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:21:29,389 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:21:29,395 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:21:30,176 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:21:30,431 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:21:31,176 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:21:32,845 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 17:21:33,135 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:21:33,136 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:21:33,136 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:21:33,141 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:21:33,177 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:21:35,181 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:21:36,142 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:21:36,873 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:21:36,873 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:21:36,874 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:21:36,879 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:21:37,182 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:21:37,410 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:21:37,410 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:21:37,411 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:21:39,187 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:21:41,675 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:21:42,635 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:21:42,635 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:21:42,636 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:21:42,641 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:21:43,189 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:21:45,192 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:21:46,679 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:21:51,680 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:21:51,697 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:21:51,697 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:21:51,698 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:21:51,703 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:21:52,197 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:21:52,410 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:21:52,410 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:21:52,411 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:21:53,200 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:21:57,669 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:22:01,943 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:22:01,944 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:22:01,944 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:22:01,949 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:22:02,205 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:22:02,847 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 17:22:02,848 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:22:03,208 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:22:06,109 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:22:06,110 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:22:06,110 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:22:06,115 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:22:06,210 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:22:07,212 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:22:07,410 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:22:07,410 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:22:07,410 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:22:10,686 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:22:12,137 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:22:12,137 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:22:12,137 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:22:12,143 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:22:12,215 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:22:13,218 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:22:14,478 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:22:14,479 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:22:14,479 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:22:14,484 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:22:15,221 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:22:15,222 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:22:17,485 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:22:22,410 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:22:22,410 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:22:22,411 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:22:22,696 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:22:22,877 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:22:22,878 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:22:22,878 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:22:22,883 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:22:23,228 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:22:25,231 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:22:27,919 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:22:29,582 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:22:29,583 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:22:29,583 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:22:29,588 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:22:30,234 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:22:31,236 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:22:32,849 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 17:22:33,851 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:22:35,786 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:22:35,787 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:22:35,787 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:22:35,792 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:22:36,240 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:22:37,243 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:22:37,410 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:22:37,410 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:22:37,411 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:22:39,621 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:22:39,980 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:22:39,981 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:22:39,981 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:22:39,986 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:22:40,246 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:22:41,249 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:22:44,584 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:22:44,585 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:22:44,585 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:22:44,591 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:22:45,253 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:22:45,254 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:22:45,591 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:22:47,785 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:22:47,786 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:22:47,786 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:22:47,791 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:22:48,255 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:22:49,258 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:22:50,791 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:22:51,459 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:22:51,460 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:22:51,460 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:22:51,465 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:22:52,260 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:22:52,410 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:22:52,410 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:22:52,411 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:22:53,263 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:22:55,143 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:22:55,144 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:22:55,144 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:22:55,148 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:22:55,264 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:22:56,187 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:22:57,268 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:22:59,389 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:22:59,390 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:22:59,391 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:22:59,395 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:23:00,270 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:23:01,273 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:23:01,396 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:23:02,852 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 17:23:06,853 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:23:07,410 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:23:07,411 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:23:07,411 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:23:07,899 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:23:08,828 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:23:08,829 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:23:08,833 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:23:09,278 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:23:11,281 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:23:11,459 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:23:11,459 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:23:11,460 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:23:11,466 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:23:12,282 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:23:12,466 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:23:13,285 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:23:17,079 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:23:17,080 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:23:17,080 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:23:17,085 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:23:17,288 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:23:18,085 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:23:19,292 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:23:21,632 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:23:21,632 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:23:21,633 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:23:21,637 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:23:22,294 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:23:22,410 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:23:22,411 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:23:22,411 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:23:23,297 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:23:23,640 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:23:26,067 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:23:26,068 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:23:26,068 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:23:26,073 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:23:26,299 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:23:27,302 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:23:29,111 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:23:29,818 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:23:29,819 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:23:29,819 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:23:29,824 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:23:30,304 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:23:31,307 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:23:32,858 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 17:23:34,860 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:23:35,420 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:23:35,421 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:23:35,421 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:23:35,426 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:23:36,310 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:23:37,313 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:23:37,411 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:23:37,411 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:23:37,411 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:23:39,959 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:23:39,960 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:23:39,960 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:23:39,961 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:23:39,965 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:23:40,317 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:23:41,319 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:23:44,145 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:23:44,146 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:23:44,146 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:23:44,151 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:23:44,321 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:23:45,151 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:23:45,324 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:23:48,132 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:23:48,132 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:23:48,133 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:23:48,138 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:23:48,326 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:23:49,329 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:23:51,139 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:23:51,582 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:23:51,582 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:23:51,583 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:23:51,588 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:23:52,331 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:23:52,411 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:23:52,411 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:23:52,411 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:23:53,334 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:23:55,564 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:23:55,565 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:23:55,565 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:23:55,571 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:23:56,336 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:23:56,607 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:23:57,338 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:23:59,242 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:23:59,243 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:23:59,243 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:23:59,248 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:23:59,340 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:24:01,343 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:24:02,249 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:24:02,856 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 17:24:03,041 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:24:03,042 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:24:03,042 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:24:03,047 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:24:03,344 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:24:05,347 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:24:07,411 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:24:07,411 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:24:07,411 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:24:07,411 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:24:08,081 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:24:08,082 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:24:08,082 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:24:08,087 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:24:08,350 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:24:09,352 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:24:12,061 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:24:12,062 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:24:12,062 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:24:12,067 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:24:12,354 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:24:13,067 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:24:13,357 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:24:16,807 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:24:16,808 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:24:16,808 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:24:16,813 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:24:17,362 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:24:17,363 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:24:18,813 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:24:20,724 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:24:20,725 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:24:20,725 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:24:20,730 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:24:21,367 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:24:21,367 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:24:22,411 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:24:22,411 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:24:22,411 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:24:24,673 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:24:25,398 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:24:25,399 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:24:25,399 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:24:25,405 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:24:26,370 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:24:27,373 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:24:30,443 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:24:31,045 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:24:31,046 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:24:31,046 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:24:31,052 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:24:31,376 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:24:32,861 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 17:24:33,379 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:24:34,705 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:24:34,706 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:24:34,706 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:24:34,711 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:24:35,382 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:24:35,382 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:24:35,712 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:24:37,411 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:24:37,411 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:24:37,411 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:24:38,415 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:24:38,416 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:24:38,416 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:24:38,421 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:24:39,389 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:24:39,389 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:24:41,459 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:24:42,924 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:24:42,925 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:24:42,925 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:24:42,930 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:24:43,392 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:24:45,395 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:24:46,290 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:24:46,291 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:24:46,291 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:24:46,296 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:24:46,395 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:24:47,296 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:24:47,398 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:24:50,180 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:24:50,181 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:24:50,181 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:24:50,186 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:24:50,400 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:24:51,403 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:24:52,411 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:24:52,411 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:24:52,411 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:24:52,411 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:24:53,516 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:24:53,516 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:24:53,517 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:24:53,522 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:24:54,405 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:24:55,407 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:24:56,907 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:24:56,908 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:24:56,908 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:24:56,913 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:24:57,408 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:24:57,913 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:24:59,412 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:25:00,229 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:25:00,230 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:25:00,230 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:25:00,236 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:25:00,412 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:25:01,415 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:25:02,860 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 17:25:03,862 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:25:05,198 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:25:05,199 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:25:05,199 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:25:05,204 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:25:05,418 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:25:07,411 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:25:07,411 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:25:07,411 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:25:07,421 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:25:08,354 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:25:08,380 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:25:08,381 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:25:08,385 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:25:08,422 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:25:09,386 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:25:09,424 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:25:11,831 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:25:11,832 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:25:11,832 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:25:11,837 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:25:12,426 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:25:13,429 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:25:14,838 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:25:18,252 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:25:18,252 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:25:18,253 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:25:18,257 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:25:18,434 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:25:19,436 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:25:20,258 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:25:21,563 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:25:21,564 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:25:21,564 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:25:21,569 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:25:22,411 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:25:22,411 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:25:22,412 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:25:22,438 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:25:23,444 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:25:25,609 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:25:29,908 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:25:29,909 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:25:29,909 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:25:29,914 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:25:30,448 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:25:30,951 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:25:31,450 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:25:32,865 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 17:25:33,057 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:25:33,058 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:25:33,059 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:25:33,063 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:25:33,452 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:25:35,455 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:25:36,064 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:25:37,143 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:25:37,144 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:25:37,144 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:25:37,149 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:25:37,411 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:25:37,411 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:25:37,412 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:25:37,457 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:25:39,462 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:25:40,854 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:25:40,855 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:25:40,855 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:25:40,860 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:25:41,465 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:25:41,465 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:25:41,899 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:25:44,754 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:25:44,755 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:25:44,755 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:25:44,760 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:25:45,469 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:25:45,470 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:25:47,760 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:25:48,571 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:25:48,572 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:25:48,572 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:25:48,577 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:25:49,474 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:25:49,474 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:25:51,906 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:25:51,907 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:25:51,907 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:25:51,912 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:25:52,411 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:25:52,411 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:25:52,412 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:25:52,476 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:25:53,479 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:25:53,677 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:25:55,234 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:25:55,235 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:25:55,235 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:25:55,240 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:25:55,480 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:25:57,483 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:25:59,279 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:26:00,223 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:26:00,224 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:26:00,224 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:26:00,229 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:26:00,485 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:26:01,487 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:26:02,865 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 17:26:03,933 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:26:03,933 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:26:03,934 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:26:03,939 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:26:04,489 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:26:04,939 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:26:05,492 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:26:07,411 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:26:07,412 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:26:07,412 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:26:08,020 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:26:08,021 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:26:08,021 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:26:08,026 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:26:08,494 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:26:09,496 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:26:10,063 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:26:11,556 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:26:11,557 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:26:11,557 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:26:11,562 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:26:12,498 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:26:13,500 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:26:15,308 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:26:15,309 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:26:15,309 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:26:15,309 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:26:15,314 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:26:15,502 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:26:17,505 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:26:19,284 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:26:19,285 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:26:19,285 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:26:19,290 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:26:19,507 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:26:21,291 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:26:21,510 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:26:22,411 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:26:22,412 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:26:22,412 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:26:24,305 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:26:24,306 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:26:24,306 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:26:24,313 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:26:24,512 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:26:25,515 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:26:26,351 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:26:28,001 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:26:28,002 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:26:28,002 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:26:28,008 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:26:28,516 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:26:29,519 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:26:31,493 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:26:31,494 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:26:31,494 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:26:31,494 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:26:31,499 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:26:31,520 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:26:32,870 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 17:26:33,523 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:26:34,767 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:26:34,767 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:26:34,768 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:26:34,773 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:26:35,527 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:26:35,527 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:26:36,774 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:26:37,411 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:26:37,412 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:26:37,412 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:26:40,761 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:26:40,762 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:26:40,762 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:26:40,767 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:26:41,534 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:26:41,534 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:26:41,803 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:26:43,898 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:26:43,899 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:26:43,899 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:26:43,904 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:26:44,535 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:26:45,538 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:26:46,904 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:26:48,966 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:26:48,967 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:26:48,967 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:26:48,972 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:26:49,542 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:26:49,542 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:26:51,973 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:26:52,412 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:26:52,412 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:26:52,412 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:26:53,083 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:26:53,084 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:26:53,084 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:26:53,089 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:26:53,545 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:26:55,548 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:26:57,127 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:27:00,719 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:27:00,720 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:27:00,720 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:27:00,726 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:27:01,554 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:27:01,555 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:27:02,726 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:27:02,869 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 17:27:04,234 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:27:04,235 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:27:04,235 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:27:04,240 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:27:04,556 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:27:05,558 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:27:07,412 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:27:07,412 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:27:07,412 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:27:07,731 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:27:08,047 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:27:08,048 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:27:08,048 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:27:08,053 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:27:08,560 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:27:09,563 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:27:12,161 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:27:12,162 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:27:12,163 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:27:12,170 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:27:12,565 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:27:13,170 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:27:13,567 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:27:15,683 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:27:15,684 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:27:15,684 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:27:15,689 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:27:16,569 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:27:17,572 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:27:18,690 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:27:19,127 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:27:19,127 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:27:19,128 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:27:19,133 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:27:19,574 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:27:21,577 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:27:22,412 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:27:22,412 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:27:22,412 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:27:22,941 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:27:22,942 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:27:22,942 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:27:22,947 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:27:23,580 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:27:23,581 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:27:23,983 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:27:26,740 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:27:26,741 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:27:26,741 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:27:26,746 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:27:27,585 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:27:27,585 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:27:29,747 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:27:30,977 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:27:30,978 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:27:30,979 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:27:30,983 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:27:31,589 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:27:31,590 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:27:32,871 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 17:27:33,900 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:27:33,904 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:27:33,904 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:27:33,909 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:27:34,591 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:27:34,909 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:27:35,594 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:27:37,412 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:27:37,412 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:27:37,412 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:27:40,780 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:27:40,800 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:27:40,801 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:27:40,801 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:27:40,806 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:27:41,601 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:27:41,602 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:27:45,722 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:27:45,723 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:27:45,723 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:27:45,728 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:27:46,604 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:27:46,729 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:27:47,607 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:27:50,599 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:27:50,600 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:27:50,600 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:27:50,605 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:27:50,609 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:27:51,611 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:27:52,412 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:27:52,412 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:27:52,412 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:27:52,412 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:27:54,495 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:27:54,496 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:27:54,496 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:27:54,501 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:27:54,613 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:27:55,616 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:27:57,539 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:27:58,381 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:27:58,382 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:27:58,382 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:27:58,387 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:27:58,618 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:27:59,620 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:28:02,874 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 17:28:02,875 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:28:03,373 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:28:03,374 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:28:03,375 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:28:03,379 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:28:03,623 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:28:05,626 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:28:07,412 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:28:07,412 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:28:07,413 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:28:07,754 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:28:07,755 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:28:07,755 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:28:07,760 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:28:08,628 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:28:08,799 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:28:09,630 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:28:13,799 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:28:13,994 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:28:13,995 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:28:13,995 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:28:14,000 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:28:14,634 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:28:15,636 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:28:18,574 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:28:18,575 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:28:18,575 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:28:18,580 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:28:18,639 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:28:19,581 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:28:19,642 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:28:22,412 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:28:22,412 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:28:22,413 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:28:23,625 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:28:23,626 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:28:23,626 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:28:23,631 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:28:23,644 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:28:24,671 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:28:25,648 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:28:27,377 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:28:27,378 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:28:27,378 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:28:27,383 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:28:27,649 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:28:29,652 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:28:30,383 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:28:32,876 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 17:28:33,195 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:28:33,196 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:28:33,196 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:28:33,201 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:28:33,655 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:28:35,658 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:28:36,202 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:28:37,412 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:28:37,412 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:28:37,413 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:28:37,528 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:28:37,679 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:28:37,679 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:28:37,684 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:28:38,661 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:28:39,665 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:28:41,227 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:28:41,228 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:28:41,229 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:28:41,229 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:28:41,234 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:28:41,666 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:28:43,669 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:28:45,027 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:28:45,028 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:28:45,028 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:28:45,032 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:28:45,673 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:28:45,673 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:28:47,033 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:28:49,833 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:28:49,837 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:28:49,837 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:28:49,843 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:28:50,676 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:28:51,679 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:28:52,412 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:28:52,413 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:28:52,413 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:28:52,413 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:28:54,209 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:28:54,210 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:28:54,210 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:28:54,215 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:28:54,681 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:28:55,684 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:28:58,163 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:28:58,164 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:28:58,164 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:28:58,164 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:28:58,169 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:28:58,685 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:28:59,688 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:29:02,306 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:29:02,307 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:29:02,307 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:29:02,312 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:29:02,690 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:29:02,878 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 17:29:03,693 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:29:03,880 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:29:06,030 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:29:06,031 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:29:06,031 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:29:06,035 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:29:06,695 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:29:07,412 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:29:07,413 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:29:07,413 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:29:07,698 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:29:09,649 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:29:10,854 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:29:10,854 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:29:10,855 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:29:10,859 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:29:11,702 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:29:11,703 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:29:14,269 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:29:14,269 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:29:14,270 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:29:14,274 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:29:14,704 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:29:15,275 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:29:15,707 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:29:18,207 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:29:18,208 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:29:18,208 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:29:18,213 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:29:18,710 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:29:19,713 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:29:21,214 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:29:21,534 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:29:21,535 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:29:21,535 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:29:21,540 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:29:21,714 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:29:22,412 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:29:22,413 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:29:22,413 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:29:23,718 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:29:25,886 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:29:25,887 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:29:25,887 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:29:25,892 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:29:26,719 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:29:26,931 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:29:27,722 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:29:29,407 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:29:29,408 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:29:29,409 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:29:29,413 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:29:29,723 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:29:31,726 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:29:32,414 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:29:32,681 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:29:32,682 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:29:32,682 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:29:32,687 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:29:32,727 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:29:32,880 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 17:29:33,729 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:29:36,594 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:29:36,595 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:29:36,595 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:29:36,600 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:29:36,731 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:29:37,412 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:29:37,413 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:29:37,413 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:29:37,702 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:29:37,732 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:29:42,702 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:29:43,229 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:29:43,229 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:29:43,229 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:29:43,234 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:29:43,737 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:29:45,740 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:29:47,495 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:29:47,496 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:29:47,496 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:29:47,501 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:29:47,741 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:29:48,501 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:29:49,744 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:29:51,589 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:29:51,590 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:29:51,590 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:29:51,595 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:29:51,746 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:29:52,413 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:29:52,413 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:29:52,413 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:29:53,750 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:29:54,267 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:29:55,162 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:29:55,162 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:29:55,163 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:29:55,167 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:29:55,753 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:29:55,753 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:29:58,511 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:29:58,512 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:29:58,512 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:29:58,517 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:29:58,755 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:29:59,517 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:29:59,758 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:30:02,657 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:30:02,658 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:30:02,658 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:30:02,664 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:30:02,759 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:30:02,882 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 17:30:03,762 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:30:04,884 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:30:05,920 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:30:05,920 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:30:05,921 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:30:05,926 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:30:06,764 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:30:07,413 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:30:07,413 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:30:07,413 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:30:07,766 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:30:10,136 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:30:10,137 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:30:10,137 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:30:10,137 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:30:10,142 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:30:10,768 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:30:11,771 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:30:13,802 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:30:13,803 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:30:13,803 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:30:13,808 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:30:14,773 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:30:15,775 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:30:15,809 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:30:18,673 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:30:18,674 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:30:18,674 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:30:18,679 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:30:18,778 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:30:19,781 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:30:21,680 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:30:22,413 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:30:22,413 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:30:22,413 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:30:22,719 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:30:22,720 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:30:22,720 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:30:22,725 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:30:22,783 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:30:23,786 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:30:26,763 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:30:28,079 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:30:28,080 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:30:28,080 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:30:28,084 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:30:28,789 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:30:29,792 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:30:31,609 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:30:31,610 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:30:31,610 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:30:31,615 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:30:31,793 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:30:32,615 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:30:32,884 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 17:30:33,796 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:30:36,014 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:30:36,015 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:30:36,015 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:30:36,019 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:30:36,799 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:30:37,413 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:30:37,413 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:30:37,413 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:30:37,708 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:30:37,801 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:30:42,708 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:30:44,590 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:30:44,591 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:30:44,591 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:30:44,596 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:30:44,807 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:30:45,809 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:30:48,635 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:30:50,049 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:30:50,050 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:30:50,050 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:30:50,055 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:30:50,813 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:30:51,815 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:30:52,413 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:30:52,413 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:30:52,413 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:30:53,708 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:30:54,325 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:30:54,326 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:30:54,326 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:30:54,331 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:30:54,817 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:30:55,820 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:30:58,137 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:30:58,138 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:30:58,138 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:30:58,143 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:30:58,822 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:30:59,143 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:30:59,825 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:31:02,762 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:31:02,762 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:31:02,763 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:31:02,768 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:31:02,826 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:31:02,890 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 17:31:03,829 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:31:04,892 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:31:07,413 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:31:07,413 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:31:07,414 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:31:07,528 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:31:07,707 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:31:07,707 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:31:07,712 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:31:07,832 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:31:09,835 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:31:10,713 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:31:12,161 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:31:12,162 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:31:12,162 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:31:12,167 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:31:12,837 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:31:13,840 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:31:16,168 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:31:18,615 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:31:18,616 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:31:18,616 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:31:18,621 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:31:18,844 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:31:19,846 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:31:21,621 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:31:22,413 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:31:22,413 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:31:22,414 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:31:25,126 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:31:25,126 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:31:25,127 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:31:25,131 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:31:25,852 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:31:25,852 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:31:27,171 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:31:28,872 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:31:28,873 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:31:28,873 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:31:28,878 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:31:29,856 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:31:29,857 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:31:32,879 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:31:32,889 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 17:31:34,619 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:31:34,620 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:31:34,620 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:31:34,625 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:31:34,860 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:31:35,863 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:31:37,413 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:31:37,414 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:31:37,414 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:31:38,504 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:31:38,505 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:31:38,505 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:31:38,505 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:31:38,509 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:31:38,865 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:31:39,869 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:31:42,136 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:31:42,137 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:31:42,137 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:31:42,142 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:31:42,871 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:31:43,875 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:31:44,143 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:31:48,321 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:31:48,322 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:31:48,322 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:31:48,327 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:31:48,879 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:31:49,328 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:31:49,883 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:31:52,413 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:31:52,414 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:31:52,414 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:31:52,693 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:31:52,694 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:31:52,694 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:31:52,699 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:31:52,885 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:31:53,888 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:31:54,734 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:31:59,734 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:32:00,363 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:32:00,364 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:32:00,364 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:32:00,369 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:32:00,895 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:32:01,899 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:32:02,894 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 17:32:04,895 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:32:07,413 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:32:07,414 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:32:07,414 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:32:09,410 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:32:09,411 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:32:09,411 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:32:09,416 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:32:09,909 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:32:10,454 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:32:11,914 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:32:14,723 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:32:14,724 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:32:14,724 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:32:14,729 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:32:14,916 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:32:15,729 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:32:15,919 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:32:20,730 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:32:21,357 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:32:21,357 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:32:21,358 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:32:21,362 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:32:21,929 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:32:21,929 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:32:22,413 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:32:22,414 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:32:22,414 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:32:24,316 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:32:24,316 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:32:24,316 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:32:24,322 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:32:24,930 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:32:25,934 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:32:26,358 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:32:28,007 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:32:28,008 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:32:28,008 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:32:28,013 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:32:28,936 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:32:29,940 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:32:31,902 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:32:31,903 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:32:31,903 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:32:31,903 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:32:31,908 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:32:31,941 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:32:32,893 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 17:32:33,945 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:32:35,651 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:32:35,652 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:32:35,652 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:32:35,657 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:32:35,947 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:32:37,414 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:32:37,414 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:32:37,414 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:32:37,414 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:32:37,951 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:32:40,169 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:32:40,170 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:32:40,170 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:32:40,175 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:32:40,954 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:32:41,955 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:32:43,214 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:32:44,569 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:32:44,570 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:32:44,570 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:32:44,575 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:32:44,958 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:32:45,963 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:32:48,551 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:32:48,552 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:32:48,552 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:32:48,553 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:32:48,557 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:32:48,965 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:32:49,970 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:32:52,414 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:32:52,414 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:32:52,414 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:32:53,351 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:32:53,351 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:32:53,352 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:32:53,356 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:32:53,975 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:32:53,975 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:32:54,394 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:32:58,303 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:32:58,303 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:32:58,304 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:32:58,308 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:32:58,979 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:32:59,983 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:33:00,309 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:33:01,766 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:33:01,767 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:33:01,767 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:33:01,772 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:33:01,985 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:33:02,898 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 17:33:03,989 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:33:05,562 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:33:05,563 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:33:05,564 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:33:05,564 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:33:05,568 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:33:05,990 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:33:07,414 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:33:07,414 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:33:07,414 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:33:07,994 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:33:09,794 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:33:09,794 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:33:09,794 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:33:09,799 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:33:09,995 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:33:10,838 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:33:12,000 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:33:13,705 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:33:13,706 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:33:13,706 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:33:13,711 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:33:14,001 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:33:16,006 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:33:16,712 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:33:17,259 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:33:17,260 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:33:17,260 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:33:17,264 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:33:18,010 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:33:18,010 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:33:22,265 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:33:22,414 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:33:22,414 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:33:22,414 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:33:25,600 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:33:25,601 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:33:25,601 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:33:25,606 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:33:26,018 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:33:27,642 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:33:28,023 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:33:29,288 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:33:29,289 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:33:29,289 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:33:29,294 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:33:30,026 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:33:30,026 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:33:32,897 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 17:33:32,899 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:33:33,600 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:33:33,601 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:33:33,601 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:33:33,606 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:33:34,029 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:33:36,034 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:33:36,714 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:33:36,715 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:33:36,715 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:33:36,720 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:33:37,035 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:33:37,414 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:33:37,414 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:33:37,414 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:33:38,037 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:33:38,702 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:33:42,936 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:33:42,937 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:33:42,937 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:33:42,942 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:33:43,042 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:33:43,978 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:33:44,046 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:33:46,488 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:33:46,489 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:33:46,489 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:33:46,494 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:33:47,047 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:33:48,052 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:33:49,494 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:33:49,809 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:33:49,810 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:33:49,810 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:33:49,814 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:33:50,053 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:33:52,057 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:33:52,414 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:33:52,414 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:33:52,415 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:33:54,687 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:33:55,036 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:33:55,037 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:33:55,037 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:33:55,042 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:33:55,059 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:33:56,063 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:33:59,349 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:33:59,350 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:33:59,350 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:33:59,355 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:34:00,069 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:34:00,069 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:34:00,355 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:34:02,569 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:34:02,570 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:34:02,570 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:34:02,575 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:34:02,899 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 17:34:03,070 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:34:04,074 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:34:05,901 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:34:06,636 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:34:06,637 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:34:06,637 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:34:06,642 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:34:07,076 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:34:07,414 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:34:07,414 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:34:07,414 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:34:08,080 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:34:10,253 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:34:10,254 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:34:10,254 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:34:10,258 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:34:11,082 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:34:11,296 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:34:12,085 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:34:14,103 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:34:14,104 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:34:14,104 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:34:14,109 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:34:15,087 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:34:16,091 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:34:17,110 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:34:17,732 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:34:17,732 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:34:17,733 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:34:17,737 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:34:18,094 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:34:20,099 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:34:22,414 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:34:22,414 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:34:22,415 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:34:22,415 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:34:23,949 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:34:23,949 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:34:23,950 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:34:23,954 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:34:24,101 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:34:26,106 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:34:27,458 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:34:27,459 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:34:27,459 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:34:27,459 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:34:27,464 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:34:28,110 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:34:28,110 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:34:32,465 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:34:32,827 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:34:32,827 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:34:32,828 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:34:32,833 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:34:32,901 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 17:34:33,114 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:34:34,117 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:34:36,630 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:34:36,631 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:34:36,631 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:34:36,635 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:34:37,120 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:34:37,414 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:34:37,414 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:34:37,415 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:34:38,000 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:34:38,124 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:34:40,132 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:34:40,132 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:34:40,132 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:34:40,137 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:34:41,127 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:34:42,130 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:34:43,175 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:34:43,774 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:34:43,775 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:34:43,775 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:34:43,780 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:34:44,131 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:34:46,135 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:34:48,781 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:34:49,240 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:34:49,240 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:34:49,241 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:34:49,246 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:34:50,142 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:34:50,142 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:34:52,414 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:34:52,415 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:34:52,415 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:34:52,613 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:34:52,701 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:34:52,701 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:34:52,706 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:34:53,145 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:34:54,148 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:34:54,706 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:34:56,273 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:34:56,274 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:34:56,274 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:34:56,279 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:34:57,149 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:34:58,153 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:35:00,279 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:35:01,755 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:35:01,756 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:35:01,756 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:35:01,760 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:35:02,157 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:35:02,904 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 17:35:04,162 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:35:05,905 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:35:07,117 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:35:07,117 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:35:07,118 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:35:07,122 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:35:07,164 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:35:07,414 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:35:07,414 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:35:07,415 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:35:08,168 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:35:10,575 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:35:10,576 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:35:10,576 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:35:10,581 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:35:11,170 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:35:11,619 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:35:12,173 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:35:15,043 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:35:15,044 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:35:15,044 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:35:15,049 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:35:15,175 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:35:16,179 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:35:17,050 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:35:19,277 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:35:19,277 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:35:19,278 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:35:19,282 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:35:20,186 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:35:20,187 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:35:22,283 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:35:22,414 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:35:22,415 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:35:22,415 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:35:24,305 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:35:24,306 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:35:24,306 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:35:24,310 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:35:25,190 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:35:26,194 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:35:27,347 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:35:30,560 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:35:30,561 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:35:30,561 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:35:30,566 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:35:31,198 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:35:32,202 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:35:32,566 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:35:32,906 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 17:35:34,107 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:35:34,108 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:35:34,108 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:35:34,113 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:35:34,204 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:35:36,206 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:35:37,415 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:35:37,415 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:35:37,416 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:35:37,707 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:35:38,673 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:35:38,673 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:35:38,673 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:35:38,678 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:35:39,210 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:35:40,215 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:35:42,068 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:35:42,068 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:35:42,069 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:35:42,073 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:35:42,217 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:35:43,074 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:35:44,220 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:35:45,831 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:35:45,832 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:35:45,832 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:35:45,837 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:35:46,221 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:35:48,225 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:35:48,838 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:35:51,427 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:35:51,428 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:35:51,428 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:35:51,432 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:35:52,232 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:35:52,232 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:35:52,415 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:35:52,415 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:35:52,415 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:35:54,700 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:35:59,700 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:35:59,959 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:35:59,960 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:35:59,960 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:35:59,965 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:36:00,241 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:36:02,245 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:36:02,908 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 17:36:04,263 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:36:04,264 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:36:04,264 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:36:04,269 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:36:05,248 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:36:05,269 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:36:06,252 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:36:07,367 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:36:07,368 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:36:07,368 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:36:07,373 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:36:07,415 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:36:07,415 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:36:07,415 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:36:08,255 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:36:08,255 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:36:11,155 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:36:11,801 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:36:11,801 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:36:11,802 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:36:11,807 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:36:12,258 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:36:14,263 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:36:16,092 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:36:16,093 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:36:16,093 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:36:16,098 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:36:16,264 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:36:17,098 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:36:18,269 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:36:20,747 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:36:20,748 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:36:20,748 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:36:20,753 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:36:21,272 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:36:22,275 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:36:22,415 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:36:22,415 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:36:22,415 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:36:22,415 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:36:24,022 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:36:24,023 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:36:24,023 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:36:24,028 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:36:24,276 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:36:26,281 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:36:27,985 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:36:27,986 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:36:27,986 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:36:27,986 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:36:27,991 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:36:28,282 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:36:30,287 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:36:32,914 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 17:36:33,765 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:36:33,766 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:36:33,766 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:36:33,767 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:36:33,771 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:36:34,290 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:36:36,295 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:36:37,415 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:36:37,415 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:36:37,416 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:36:38,383 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:36:38,383 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:36:38,383 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:36:38,388 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:36:39,297 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:36:39,427 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:36:40,302 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:36:42,074 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:36:42,075 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:36:42,075 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:36:42,080 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:36:42,303 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:36:44,306 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:36:45,080 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:36:45,553 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:36:45,554 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:36:45,554 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:36:45,558 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:36:46,310 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:36:46,310 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:36:49,295 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:36:49,296 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:36:49,296 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:36:49,301 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:36:49,311 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:36:50,301 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:36:50,315 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:36:52,415 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:36:52,415 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:36:52,415 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:36:55,500 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:36:55,501 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:36:55,501 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:36:55,501 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:36:55,506 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:36:56,324 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:36:56,324 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:37:00,443 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:37:00,444 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:37:00,444 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:37:00,449 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:37:01,328 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:37:01,449 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:37:02,332 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:37:02,913 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 17:37:04,949 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:37:04,949 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:37:04,950 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:37:04,954 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:37:05,334 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:37:06,338 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:37:06,955 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:37:07,415 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:37:07,415 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:37:07,415 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:37:09,632 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:37:09,633 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:37:09,633 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:37:09,638 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:37:10,343 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:37:10,344 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:37:12,675 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:37:13,525 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:37:13,526 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:37:13,526 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:37:13,531 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:37:14,349 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:37:14,349 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:37:17,639 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:37:17,639 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:37:17,639 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:37:17,644 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:37:18,355 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:37:18,355 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:37:18,645 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:37:21,295 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:37:21,295 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:37:21,295 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:37:21,301 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:37:21,358 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:37:22,361 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:37:22,415 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:37:22,415 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:37:22,416 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:37:23,689 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:37:24,379 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:37:24,379 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:37:24,380 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:37:24,384 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:37:25,362 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:37:26,367 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:37:28,298 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:37:28,298 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:37:28,298 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:37:28,303 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:37:28,368 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:37:29,304 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:37:30,373 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:37:31,836 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:37:31,837 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:37:31,837 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:37:31,842 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:37:32,374 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:37:32,918 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 17:37:34,378 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:37:34,919 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:37:36,596 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:37:36,597 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:37:36,597 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:37:36,602 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:37:37,381 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:37:37,415 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:37:37,415 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:37:37,416 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:37:38,384 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:37:40,627 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:37:45,036 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:37:45,037 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:37:45,037 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:37:45,042 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:37:45,391 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:37:46,079 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:37:46,394 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:37:51,079 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:37:52,168 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:37:52,168 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:37:52,169 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:37:52,173 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:37:52,402 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:37:52,415 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:37:52,415 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:37:52,416 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:37:54,407 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:37:56,139 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:37:56,139 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:37:56,139 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:37:56,140 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:37:56,144 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:37:56,408 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:37:58,412 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:38:01,183 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:38:02,458 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:38:02,459 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:38:02,459 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:38:02,464 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:38:02,917 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 17:38:03,417 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:38:04,421 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:38:06,918 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:38:07,415 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:38:07,416 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:38:07,416 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:38:08,192 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:38:08,193 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:38:08,193 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:38:08,198 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:38:08,424 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:38:10,429 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:38:12,235 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:38:13,136 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:38:13,137 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:38:13,137 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:38:13,142 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:38:13,431 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:38:14,435 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:38:16,643 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:38:16,644 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:38:16,645 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:38:16,650 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:38:17,437 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:38:17,650 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:38:18,441 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:38:21,653 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:38:21,654 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:38:21,654 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:38:21,659 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:38:22,415 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:38:22,416 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:38:22,416 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:38:22,447 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:38:22,448 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:38:23,650 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:38:28,651 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:38:30,133 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:38:30,134 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:38:30,134 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:38:30,139 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:38:30,456 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:38:32,461 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:38:32,922 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 17:38:33,805 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:38:33,806 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:38:33,807 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:38:33,807 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:38:33,811 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:38:34,465 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:38:34,465 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:38:37,416 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:38:37,416 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:38:37,416 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:38:37,464 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:38:37,649 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:38:37,649 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:38:37,655 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:38:38,470 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:38:38,470 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:38:39,655 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:38:43,757 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:38:43,758 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:38:43,758 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:38:43,763 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:38:44,478 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:38:44,478 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:38:44,763 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:38:49,764 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:38:50,975 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:38:50,976 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:38:50,976 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:38:50,981 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:38:51,486 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:38:52,415 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:38:52,416 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:38:52,416 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:38:52,489 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:38:54,524 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:38:54,525 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:38:54,525 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:38:54,530 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:38:55,491 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:38:55,567 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:38:56,495 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:38:59,675 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:38:59,676 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:38:59,676 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:38:59,681 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:39:00,501 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:39:00,501 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:39:00,681 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:39:02,921 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 17:39:03,018 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:39:03,019 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:39:03,019 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:39:03,024 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:39:03,502 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:39:04,506 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:39:06,025 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:39:06,784 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:39:06,784 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:39:06,785 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:39:06,790 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:39:07,416 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:39:07,416 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:39:07,416 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:39:07,508 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:39:08,512 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:39:11,325 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:39:11,326 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:39:11,326 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:39:11,326 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:39:11,331 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:39:11,514 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:39:12,518 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:39:14,809 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:39:14,810 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:39:14,810 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:39:14,815 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:39:15,520 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:39:16,523 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:39:16,816 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:39:19,057 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:39:19,057 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:39:19,058 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:39:19,062 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:39:19,526 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:39:20,530 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:39:22,063 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:39:22,416 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:39:22,416 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:39:22,416 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:39:22,792 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:39:22,792 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:39:22,793 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:39:22,798 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:39:23,532 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:39:24,536 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:39:26,740 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:39:26,741 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:39:26,741 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:39:26,746 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:39:27,537 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:39:27,747 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:39:28,541 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:39:30,821 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:39:30,821 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:39:30,822 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:39:30,826 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:39:31,543 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:39:32,547 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:39:32,827 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:39:32,923 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 17:39:34,571 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:39:34,571 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:39:34,571 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:39:34,576 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:39:35,549 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:39:36,553 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:39:37,416 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:39:37,416 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:39:37,416 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:39:38,060 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:39:38,060 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:39:38,061 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:39:38,061 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:39:38,065 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:39:38,554 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:39:40,561 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:39:42,495 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:39:42,495 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:39:42,496 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:39:42,501 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:39:42,562 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:39:43,501 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:39:44,565 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:39:46,777 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:39:46,777 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:39:46,778 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:39:46,782 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:39:47,567 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:39:48,570 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:39:48,783 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:39:52,416 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:39:52,416 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:39:52,416 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:39:54,676 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:39:55,150 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:39:55,151 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:39:55,151 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:39:55,156 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:39:55,578 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:39:56,581 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:39:58,615 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:39:58,616 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:39:58,616 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:39:58,621 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:39:59,583 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:40:00,587 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:40:00,621 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:40:02,925 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 17:40:03,576 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:40:03,577 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:40:03,577 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:40:03,582 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:40:03,589 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:40:04,593 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:40:06,583 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:40:07,302 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:40:07,303 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:40:07,303 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:40:07,308 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:40:07,416 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:40:07,416 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:40:07,417 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:40:07,595 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:40:08,598 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:40:11,123 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:40:11,123 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:40:11,124 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:40:11,128 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:40:11,600 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:40:12,167 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:40:12,604 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:40:17,168 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:40:19,590 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:40:19,590 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:40:19,591 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:40:19,596 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:40:19,611 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:40:20,616 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:40:22,416 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:40:22,416 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:40:22,416 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:40:22,417 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:40:27,628 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:40:28,176 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:40:28,177 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:40:28,177 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:40:28,182 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:40:28,625 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:40:30,629 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:40:31,915 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:40:31,916 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:40:31,916 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:40:31,921 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:40:32,633 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:40:32,633 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:40:32,922 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:40:32,927 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 17:40:37,416 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:40:37,416 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:40:37,417 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:40:38,104 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:40:38,582 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:40:38,583 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:40:38,583 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:40:38,588 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:40:38,639 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:40:40,645 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:40:43,620 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:40:43,621 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:40:43,621 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:40:43,621 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:40:43,626 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:40:43,647 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:40:44,650 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:40:47,403 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:40:47,403 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:40:47,404 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:40:47,409 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:40:47,652 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:40:48,655 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:40:49,409 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:40:50,404 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:40:50,405 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:40:50,405 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:40:50,410 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:40:50,657 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:40:52,416 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:40:52,417 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:40:52,417 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:40:52,661 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:40:53,860 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:40:53,861 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:40:53,861 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:40:53,866 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:40:54,665 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:40:54,665 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:40:54,906 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:40:59,241 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:40:59,241 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:40:59,242 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:40:59,246 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:40:59,668 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:41:00,247 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:41:00,673 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:41:02,930 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 17:41:03,230 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:41:03,230 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:41:03,231 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:41:03,235 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:41:03,675 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:41:04,678 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:41:06,236 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:41:06,995 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:41:06,996 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:41:06,996 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:41:07,001 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:41:07,416 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:41:07,417 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:41:07,417 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:41:07,680 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:41:08,684 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:41:11,635 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:41:12,414 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:41:12,414 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:41:12,414 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:41:12,419 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:41:12,687 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:41:14,692 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:41:15,827 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:41:15,828 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:41:15,828 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:41:15,833 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:41:16,696 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:41:16,696 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:41:16,834 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:41:19,692 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:41:19,693 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:41:19,693 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:41:19,698 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:41:19,699 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:41:20,702 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:41:22,416 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:41:22,417 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:41:22,417 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:41:22,417 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:41:23,530 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:41:23,530 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:41:23,530 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:41:23,535 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:41:23,704 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:41:24,707 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:41:27,575 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:41:27,691 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:41:27,691 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:41:27,692 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:41:27,697 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:41:27,708 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:41:28,712 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:41:31,570 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:41:31,571 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:41:31,571 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:41:31,576 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:41:31,714 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:41:32,576 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:41:32,718 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:41:32,932 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 17:41:35,841 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:41:35,842 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:41:35,842 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:41:35,847 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:41:36,724 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:41:36,725 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:41:37,543 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:41:37,543 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:41:37,543 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:41:37,841 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:41:41,491 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:41:41,492 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:41:41,492 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:41:41,497 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:41:41,731 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:41:42,733 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:41:43,535 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:41:45,466 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:41:45,467 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:41:45,467 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:41:45,472 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:41:45,735 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:41:46,738 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:41:49,473 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:41:50,598 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:41:50,599 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:41:50,599 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:41:50,605 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:41:50,743 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:41:52,543 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:41:52,543 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:41:52,543 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:41:52,748 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:41:54,577 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:41:54,578 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:41:54,578 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:41:54,578 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:41:54,583 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:41:54,749 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:41:56,754 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:41:57,890 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:41:57,891 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:41:57,891 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:41:57,896 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:41:58,757 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:41:58,757 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:41:59,897 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:42:01,673 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:42:01,674 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:42:01,674 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:42:01,679 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:42:01,759 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:42:02,762 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:42:02,934 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 17:42:04,936 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:42:05,851 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:42:05,851 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:42:05,852 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:42:05,856 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:42:06,768 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:42:06,768 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:42:07,543 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:42:07,543 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:42:07,543 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:42:09,677 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:42:09,678 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:42:09,678 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:42:09,683 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:42:09,770 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:42:10,719 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:42:10,774 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:42:13,180 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:42:13,181 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:42:13,181 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:42:13,187 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:42:13,775 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:42:14,779 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:42:16,187 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:42:17,374 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:42:17,374 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:42:17,375 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:42:17,379 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:42:17,782 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:42:18,786 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:42:20,926 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:42:20,927 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:42:20,927 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:42:20,932 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:42:21,789 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:42:21,933 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:42:22,543 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:42:22,543 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:42:22,543 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:42:22,791 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:42:24,943 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:42:24,944 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:42:24,944 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:42:24,949 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:42:25,793 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:42:26,797 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:42:26,987 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:42:29,555 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:42:29,556 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:42:29,556 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:42:29,560 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:42:29,799 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:42:30,802 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:42:32,561 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:42:32,937 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 17:42:37,543 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:42:37,543 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:42:37,544 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:42:37,828 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:42:39,071 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:42:39,072 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:42:39,072 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:42:39,077 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:42:39,812 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:42:40,817 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:42:42,911 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:42:42,912 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:42:42,912 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:42:42,912 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:42:42,917 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:42:43,819 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:42:44,821 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:42:47,918 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:42:48,124 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:42:48,124 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:42:48,125 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:42:48,130 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:42:48,826 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:42:50,830 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:42:52,543 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:42:52,543 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:42:52,544 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:42:52,921 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:42:52,922 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:42:52,922 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:42:52,922 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:42:52,927 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:42:53,833 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:42:54,836 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:42:57,042 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:42:57,043 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:42:57,043 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:42:57,048 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:42:57,838 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:42:58,049 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:42:58,842 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:43:01,894 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:43:01,895 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:43:01,895 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:43:01,899 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:43:02,848 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:43:02,848 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:43:02,942 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 17:43:03,943 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:43:07,543 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:43:07,543 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:43:07,544 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:43:09,738 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:43:10,064 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:43:10,065 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:43:10,065 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:43:10,069 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:43:10,857 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:43:10,857 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:43:13,622 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:43:13,623 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:43:13,623 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:43:13,628 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:43:13,861 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:43:14,865 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:43:15,629 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:43:20,196 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:43:20,196 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:43:20,197 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:43:20,201 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:43:20,872 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:43:21,202 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:43:22,543 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:43:22,543 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:43:22,544 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:43:22,875 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:43:26,733 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:43:28,604 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:43:28,605 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:43:28,605 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:43:28,611 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:43:28,885 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:43:30,889 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:43:32,647 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:43:32,941 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 17:43:33,174 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:43:33,175 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:43:33,176 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:43:33,180 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:43:33,891 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:43:34,896 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:43:36,970 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:43:36,971 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:43:36,971 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:43:36,976 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:43:37,543 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:43:37,543 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:43:37,544 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:43:37,781 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:43:37,898 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:43:38,901 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:43:41,232 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:43:41,233 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:43:41,233 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:43:41,238 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:43:41,904 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:43:42,906 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:43:43,275 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:43:45,810 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:43:45,811 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:43:45,811 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:43:45,816 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:43:45,908 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:43:46,912 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:43:48,817 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:43:49,222 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:43:49,223 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:43:49,223 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:43:49,228 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:43:49,914 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:43:50,918 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:43:52,543 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:43:52,544 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:43:52,544 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:43:53,839 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:43:55,166 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:43:55,167 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:43:55,167 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:43:55,172 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:43:55,922 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:43:56,926 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:43:58,989 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:43:58,990 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:43:58,990 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:43:58,990 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:43:58,996 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:43:59,928 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:44:00,932 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:44:02,749 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:44:02,750 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:44:02,750 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:44:02,755 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:44:02,933 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:44:02,946 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 17:44:04,938 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:44:04,948 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:44:07,543 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:44:07,544 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:44:07,544 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:44:08,053 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:44:08,054 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:44:08,054 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:44:08,059 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:44:08,943 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:44:08,943 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:44:10,095 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:44:11,746 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:44:11,746 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:44:11,747 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:44:11,752 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:44:11,945 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:44:12,948 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:44:15,496 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:44:15,496 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:44:15,497 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:44:15,497 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:44:15,502 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:44:15,950 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:44:16,954 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:44:19,016 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:44:19,016 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:44:19,017 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:44:19,022 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:44:19,957 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:44:20,960 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:44:21,022 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:44:22,543 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:44:22,544 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:44:22,544 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:44:22,978 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:44:22,978 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:44:22,979 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:44:22,984 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:44:23,963 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:44:24,966 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:44:26,023 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:44:27,443 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:44:27,444 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:44:27,444 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:44:27,449 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:44:27,968 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:44:28,970 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:44:30,762 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:44:30,763 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:44:30,763 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:44:30,768 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:44:30,972 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:44:31,769 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:44:32,945 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 17:44:32,978 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:44:34,894 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:44:34,895 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:44:34,895 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:44:34,901 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:44:34,979 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:44:36,902 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:44:36,983 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:44:37,544 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:44:37,544 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:44:37,545 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:44:38,846 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:44:38,847 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:44:38,847 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:44:38,852 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:44:38,985 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:44:40,990 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:44:42,891 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:44:43,181 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:44:43,182 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:44:43,182 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:44:43,188 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:44:43,992 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:44:44,995 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:44:47,138 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:44:47,139 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:44:47,139 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:44:47,144 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:44:47,997 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:44:48,144 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:44:49,000 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:44:50,525 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:44:50,526 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:44:50,526 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:44:50,531 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:44:51,001 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:44:52,544 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:44:52,544 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:44:52,544 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:44:53,006 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:44:53,822 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:44:55,442 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:44:55,443 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:44:55,443 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:44:55,448 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:44:56,008 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:44:57,012 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:44:59,487 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:45:00,137 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:45:00,138 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:45:00,138 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:45:00,144 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:45:01,017 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:45:01,018 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:45:02,950 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 17:45:04,453 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:45:04,454 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:45:04,454 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:45:04,459 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:45:05,021 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:45:05,459 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:45:07,026 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:45:07,544 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:45:07,544 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:45:07,544 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:45:09,412 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:45:09,413 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:45:09,413 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:45:09,418 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:45:10,028 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:45:11,031 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:45:11,455 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:45:15,200 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:45:15,200 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:45:15,201 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:45:15,206 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:45:16,035 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:45:17,039 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:45:17,207 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:45:18,632 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:45:18,632 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:45:18,633 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:45:18,638 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:45:19,042 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:45:21,046 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:45:22,544 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:45:22,544 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:45:22,544 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:45:22,544 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:45:22,625 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:45:22,783 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:45:22,784 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:45:22,789 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:45:23,047 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:45:25,052 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:45:27,790 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:45:27,916 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:45:27,917 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:45:27,917 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:45:27,923 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:45:28,053 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:45:29,057 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:45:31,970 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:45:31,970 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:45:31,971 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:45:31,976 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:45:32,059 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:45:32,950 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 17:45:32,951 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:45:33,063 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:45:35,653 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:45:35,654 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:45:35,654 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:45:35,660 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:45:36,065 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:45:37,068 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:45:37,544 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:45:37,544 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:45:37,544 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:45:38,749 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:45:43,749 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:45:44,033 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:45:44,034 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:45:44,034 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:45:44,039 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:45:44,076 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:45:45,080 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:45:49,081 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:45:49,884 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:45:49,884 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:45:49,885 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:45:49,890 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:45:50,084 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:45:51,089 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:45:52,544 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:45:52,544 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:45:52,545 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:45:54,302 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:45:54,303 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:45:54,303 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:45:54,303 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:45:54,308 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:45:55,093 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:45:57,097 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:45:57,379 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:45:57,380 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:45:57,380 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:45:57,385 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:45:58,098 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:45:59,101 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:45:59,385 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:46:02,000 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:46:02,001 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:46:02,001 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:46:02,006 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:46:02,103 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:46:02,952 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 17:46:03,106 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:46:04,954 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:46:06,619 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:46:06,619 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:46:06,620 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:46:06,625 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:46:07,110 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:46:07,544 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:46:07,544 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:46:07,545 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:46:09,114 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:46:10,819 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:46:11,788 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:46:11,788 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:46:11,788 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:46:11,793 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:46:12,116 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:46:13,121 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:46:15,831 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:46:15,915 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:46:15,916 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:46:15,916 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:46:15,921 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:46:16,122 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:46:17,126 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:46:20,922 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:46:22,544 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:46:22,544 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:46:22,545 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:46:24,450 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:46:24,451 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:46:24,451 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:46:24,456 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:46:25,136 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:46:26,495 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:46:27,141 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:46:28,540 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:46:28,541 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:46:28,541 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:46:28,545 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:46:29,142 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:46:31,147 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:46:31,546 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:46:32,954 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 17:46:33,028 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:46:33,029 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:46:33,030 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:46:33,034 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:46:33,148 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:46:35,153 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:46:37,035 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:46:37,544 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:46:37,544 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:46:37,545 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:46:38,858 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:46:38,859 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:46:38,859 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:46:38,864 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:46:39,156 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:46:41,162 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:46:42,903 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:46:44,679 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:46:44,680 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:46:44,680 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:46:44,684 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:46:45,165 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:46:47,170 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:46:48,685 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:46:52,544 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:46:52,545 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:46:52,545 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:46:53,094 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:46:53,095 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:46:53,095 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:46:53,100 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:46:53,177 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:46:54,139 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:46:55,181 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:46:57,375 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:46:57,376 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:46:57,376 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:46:57,381 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:46:58,183 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:46:59,187 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:46:59,381 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:47:02,957 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 17:47:04,010 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:47:04,011 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:47:04,011 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:47:04,016 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:47:04,191 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:47:05,016 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:47:05,195 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:47:07,544 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:47:07,545 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:47:07,545 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:47:08,620 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:47:08,621 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:47:08,621 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:47:08,625 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:47:09,199 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:47:10,663 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:47:11,203 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:47:14,446 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:47:14,447 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:47:14,447 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:47:14,452 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:47:15,208 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:47:16,452 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:47:17,213 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:47:18,187 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:47:18,187 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:47:18,188 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:47:18,192 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:47:18,213 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:47:19,217 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:47:22,193 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:47:22,271 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:47:22,272 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:47:22,272 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:47:22,277 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:47:22,545 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:47:22,545 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:47:22,545 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:47:23,222 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:47:23,223 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:47:27,735 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:47:28,039 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:47:28,040 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:47:28,040 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:47:28,045 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:47:28,226 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:47:29,230 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:47:31,743 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:47:31,744 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:47:31,744 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:47:31,748 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:47:32,232 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:47:32,749 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:47:32,962 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 17:47:33,236 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:47:34,951 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:47:34,952 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:47:34,952 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:47:34,956 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:47:35,237 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:47:37,242 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:47:37,545 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:47:37,545 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:47:37,545 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:47:37,831 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:47:39,783 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:47:39,783 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:47:39,783 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:47:39,788 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:47:40,244 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:47:41,249 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:47:43,827 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:47:44,401 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:47:44,402 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:47:44,402 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:47:44,407 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:47:45,253 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:47:45,253 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:47:49,408 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:47:52,545 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:47:52,545 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:47:52,545 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:47:52,774 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:47:52,812 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:47:52,812 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:47:52,817 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:47:53,263 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:47:54,818 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:47:55,268 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:47:56,301 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:47:56,302 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:47:56,302 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:47:56,307 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:47:57,271 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:47:57,271 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:47:59,896 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:47:59,897 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:47:59,897 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:47:59,897 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:47:59,902 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:48:00,273 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:48:01,276 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:48:02,961 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 17:48:04,962 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:48:07,545 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:48:07,545 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:48:07,545 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:48:09,128 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:48:09,129 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:48:09,129 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:48:09,134 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:48:09,285 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:48:10,171 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:48:11,290 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:48:12,873 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:48:12,874 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:48:12,874 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:48:12,879 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:48:13,291 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:48:15,296 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:48:15,880 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:48:17,248 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:48:17,249 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:48:17,249 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:48:17,254 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:48:17,297 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:48:19,302 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:48:20,879 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:48:20,880 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:48:20,881 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:48:20,881 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:48:20,885 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:48:21,304 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:48:22,545 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:48:22,545 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:48:22,545 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:48:23,308 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:48:26,049 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:48:26,050 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:48:26,050 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:48:26,050 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:48:26,055 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:48:26,310 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:48:27,314 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:48:31,095 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:48:32,966 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 17:48:33,365 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:48:33,366 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:48:33,366 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:48:33,371 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:48:34,321 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:48:35,325 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:48:36,372 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:48:37,545 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:48:37,545 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:48:37,545 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:48:41,748 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:48:41,749 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:48:41,749 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:48:41,750 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:48:41,755 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:48:42,333 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:48:43,334 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:48:45,404 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:48:45,405 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:48:45,406 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:48:45,411 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:48:46,338 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:48:47,342 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:48:47,411 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:48:49,452 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:48:49,453 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:48:49,453 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:48:49,458 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:48:50,344 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:48:51,348 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:48:52,459 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:48:52,545 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:48:52,545 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:48:52,545 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:48:52,790 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:48:52,821 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:48:52,821 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:48:52,826 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:48:53,350 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:48:55,354 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:48:56,168 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:48:56,169 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:48:56,169 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:48:56,174 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:48:56,354 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:48:57,357 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:48:58,175 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:49:02,420 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:49:02,421 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:49:02,421 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:49:02,426 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:49:02,966 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 17:49:03,365 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:49:03,365 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:49:03,967 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:49:06,201 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:49:06,202 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:49:06,202 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:49:06,207 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:49:06,367 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:49:07,370 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:49:07,545 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:49:07,545 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:49:07,546 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:49:09,046 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:49:09,880 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:49:09,881 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:49:09,881 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:49:09,886 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:49:10,373 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:49:11,376 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:49:14,150 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:49:14,151 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:49:14,152 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:49:14,152 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:49:14,157 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:49:14,377 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:49:15,381 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:49:17,925 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:49:17,926 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:49:17,926 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:49:17,931 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:49:18,383 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:49:19,385 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:49:19,932 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:49:21,819 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:49:21,820 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:49:21,820 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:49:21,825 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:49:22,389 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:49:22,545 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:49:22,545 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:49:22,546 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:49:23,393 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:49:25,786 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:49:26,089 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:49:26,089 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:49:26,090 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:49:26,095 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:49:26,395 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:49:27,399 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:49:30,635 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:49:30,636 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:49:30,636 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:49:30,641 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:49:31,402 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:49:31,642 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:49:32,970 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 17:49:33,407 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:49:35,180 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:49:35,181 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:49:35,181 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:49:35,186 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:49:35,408 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:49:37,186 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:49:37,413 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:49:37,545 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:49:37,545 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:49:37,546 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:49:38,497 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:49:38,498 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:49:38,498 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:49:38,503 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:49:39,416 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:49:39,417 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:49:42,538 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:49:42,592 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:49:42,593 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:49:42,593 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:49:42,598 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:49:43,422 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:49:43,423 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:49:46,666 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:49:46,667 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:49:46,667 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:49:46,672 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:49:47,425 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:49:47,672 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:49:49,430 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:49:50,574 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:49:50,575 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:49:50,575 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:49:50,580 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:49:51,433 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:49:51,433 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:49:52,545 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:49:52,546 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:49:52,546 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:49:52,766 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:49:55,231 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:49:55,232 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:49:55,232 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:49:55,237 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:49:55,436 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:49:57,441 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:49:58,274 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:50:01,480 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:50:01,481 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:50:01,481 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:50:01,486 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:50:02,446 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:50:02,970 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 17:50:03,449 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:50:03,971 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:50:05,286 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:50:05,287 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:50:05,288 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:50:05,293 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:50:05,451 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:50:07,455 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:50:07,545 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:50:07,546 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:50:07,546 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:50:09,267 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:50:09,268 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:50:09,268 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:50:09,268 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:50:09,273 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:50:09,457 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:50:11,461 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:50:13,789 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:50:13,790 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:50:13,790 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:50:13,795 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:50:14,463 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:50:14,795 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:50:15,466 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:50:17,387 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:50:17,388 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:50:17,388 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:50:17,393 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:50:17,468 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:50:19,472 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:50:20,394 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:50:21,149 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:50:21,151 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:50:21,151 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:50:21,156 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:50:21,474 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:50:22,545 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:50:22,546 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:50:22,546 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:50:23,478 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:50:25,830 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:50:26,144 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:50:26,145 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:50:26,145 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:50:26,149 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:50:26,480 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:50:27,483 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:50:29,883 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:50:29,884 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:50:29,884 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:50:29,889 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:50:30,485 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:50:30,890 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:50:31,489 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:50:32,975 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 17:50:33,585 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:50:33,586 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:50:33,587 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:50:33,592 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:50:34,491 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:50:35,495 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:50:36,592 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:50:37,546 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:50:37,546 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:50:37,546 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:50:38,510 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:50:38,511 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:50:38,511 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:50:38,516 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:50:39,501 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:50:39,501 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:50:41,564 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:50:41,565 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:50:41,565 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:50:41,570 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:50:42,504 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:50:42,570 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:50:43,507 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:50:45,315 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:50:45,316 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:50:45,316 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:50:45,321 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:50:45,508 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:50:47,512 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:50:48,322 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:50:48,906 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:50:48,907 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:50:48,908 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:50:48,913 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:50:49,514 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:50:51,519 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:50:52,546 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:50:52,546 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:50:52,546 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:50:53,017 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:50:53,017 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:50:53,018 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:50:53,022 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:50:53,520 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:50:54,058 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:50:55,525 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:50:57,809 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:50:57,810 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:50:57,810 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:50:57,815 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:50:58,527 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:50:59,531 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:50:59,815 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:51:02,974 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 17:51:03,786 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:51:03,786 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:51:03,787 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:51:03,792 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:51:04,535 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:51:05,539 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:51:05,793 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:51:07,546 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:51:07,546 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:51:07,546 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:51:07,614 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:51:07,833 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:51:07,833 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:51:07,838 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:51:08,541 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:51:09,545 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:51:10,839 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:51:15,839 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:51:16,313 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:51:16,314 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:51:16,314 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:51:16,319 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:51:16,552 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:51:17,556 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:51:21,320 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:51:21,388 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:51:21,389 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:51:21,389 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:51:21,395 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:51:21,561 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:51:22,546 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:51:22,546 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:51:22,547 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:51:23,566 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:51:26,785 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:51:28,033 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:51:28,034 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:51:28,034 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:51:28,039 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:51:28,570 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:51:29,574 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:51:32,075 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:51:32,976 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 17:51:33,264 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:51:33,265 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:51:33,265 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:51:33,270 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:51:33,578 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:51:35,583 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:51:37,271 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:51:37,546 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:51:37,546 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:51:37,547 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:51:37,575 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:51:37,821 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:51:37,822 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:51:37,826 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:51:38,585 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:51:39,589 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:51:41,507 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:51:41,508 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:51:41,509 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:51:41,514 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:51:41,592 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:51:42,514 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:51:43,595 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:51:45,938 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:51:45,939 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:51:45,939 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:51:45,944 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:51:46,597 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:51:47,600 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:51:47,945 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:51:51,718 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:51:51,719 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:51:51,720 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:51:51,726 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:51:52,546 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:51:52,546 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:51:52,547 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:51:52,606 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:51:53,610 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:51:53,834 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:51:56,292 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:51:56,296 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:51:56,296 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:51:56,301 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:51:56,612 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:51:57,615 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:51:59,335 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:52:00,215 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:52:00,216 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:52:00,216 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:52:00,221 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:52:00,617 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:52:01,621 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:52:02,979 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 17:52:04,980 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:52:05,477 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:52:05,478 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:52:05,478 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:52:05,483 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:52:05,625 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:52:07,546 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:52:07,546 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:52:07,547 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:52:07,630 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:52:09,456 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:52:09,457 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:52:09,457 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:52:09,462 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:52:09,631 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:52:10,499 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:52:11,636 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:52:13,086 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:52:13,087 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:52:13,088 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:52:13,093 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:52:13,637 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:52:15,642 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:52:16,093 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:52:18,817 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:52:18,818 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:52:18,819 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:52:18,824 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:52:19,646 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:52:21,651 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:52:21,824 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:52:22,546 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:52:22,546 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:52:22,547 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:52:22,744 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:52:22,787 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:52:22,788 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:52:22,794 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:52:23,652 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:52:25,656 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:52:27,794 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:52:30,071 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:52:30,071 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:52:30,072 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:52:30,077 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:52:30,660 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:52:31,664 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:52:32,981 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 17:52:32,982 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:52:36,701 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:52:36,702 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:52:36,703 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:52:36,708 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:52:37,546 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:52:37,547 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:52:37,547 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:52:37,673 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:52:37,673 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:52:38,819 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:52:40,474 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:52:40,475 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:52:40,475 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:52:40,480 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:52:40,675 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:52:41,680 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:52:44,518 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:52:44,668 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:52:44,668 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:52:44,669 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:52:44,674 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:52:44,682 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:52:45,685 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:52:49,674 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:52:49,841 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:52:49,842 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:52:49,842 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:52:49,848 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:52:50,690 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:52:51,694 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:52:52,546 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:52:52,547 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:52:52,547 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:52:54,758 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:52:55,135 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:52:55,135 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:52:55,136 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:52:55,142 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:52:55,698 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:52:57,703 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:52:58,414 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:52:58,415 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:52:58,415 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:52:58,420 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:52:58,703 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:52:59,706 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:53:00,421 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:53:02,343 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:53:02,343 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:53:02,344 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:53:02,349 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:53:02,708 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:53:02,986 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 17:53:03,711 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:53:05,988 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:53:06,256 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:53:06,257 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:53:06,258 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:53:06,262 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:53:06,713 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:53:07,546 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:53:07,547 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:53:07,547 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:53:07,717 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:53:10,875 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:53:10,876 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:53:10,876 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:53:10,881 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:53:11,720 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:53:11,918 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:53:13,726 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:53:14,853 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:53:14,854 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:53:14,855 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:53:14,860 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:53:15,727 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:53:17,731 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:53:17,860 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:53:22,546 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:53:22,547 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:53:22,547 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:53:23,218 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:53:23,219 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:53:23,219 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:53:23,219 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:53:23,224 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:53:23,739 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:53:25,744 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:53:28,262 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:53:28,303 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:53:28,303 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:53:28,304 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:53:28,309 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:53:28,746 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:53:29,750 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:53:32,100 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:53:32,101 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:53:32,101 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:53:32,106 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:53:32,752 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:53:32,985 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 17:53:33,755 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:53:33,987 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:53:35,328 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:53:35,329 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:53:35,330 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:53:35,334 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:53:35,756 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:53:37,546 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:53:37,547 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:53:37,547 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:53:37,760 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:53:39,253 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:53:39,253 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:53:39,254 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:53:39,254 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:53:39,259 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:53:39,762 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:53:41,768 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:53:43,111 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:53:43,112 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:53:43,112 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:53:43,117 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:53:43,769 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:53:45,118 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:53:45,772 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:53:49,125 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:53:49,126 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:53:49,126 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:53:49,131 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:53:49,776 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:53:50,132 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:53:51,781 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:53:52,547 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:53:52,547 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:53:52,547 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:53:53,647 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:53:53,648 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:53:53,648 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:53:53,653 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:53:53,783 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:53:55,690 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:53:55,785 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:53:59,227 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:53:59,228 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:53:59,228 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:53:59,234 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:53:59,790 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:54:01,234 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:54:01,795 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:54:02,990 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 17:54:06,992 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:54:07,547 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:54:07,547 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:54:07,547 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:54:07,831 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:54:08,025 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:54:08,025 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:54:08,030 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:54:08,802 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:54:09,806 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:54:10,980 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:54:10,981 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:54:10,981 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:54:10,987 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:54:11,807 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:54:12,987 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:54:13,812 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:54:14,546 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:54:14,547 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:54:14,547 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:54:14,552 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:54:14,813 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:54:15,815 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:54:18,553 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:54:20,273 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:54:20,274 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:54:20,275 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:54:20,280 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:54:20,821 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:54:21,824 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:54:22,547 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:54:22,547 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:54:22,547 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:54:23,822 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:54:23,831 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:54:23,832 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:54:23,832 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:54:23,837 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:54:24,826 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:54:25,829 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:54:27,385 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:54:27,386 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:54:27,386 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:54:27,391 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:54:27,831 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:54:29,392 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:54:29,835 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:54:31,221 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:54:31,222 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:54:31,223 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:54:31,228 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:54:31,836 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:54:32,990 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 17:54:33,840 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:54:34,860 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:54:34,861 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:54:34,861 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:54:34,861 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:54:34,866 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:54:35,844 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:54:35,844 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:54:37,547 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:54:37,547 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:54:37,548 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:54:39,181 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:54:39,182 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:54:39,182 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:54:39,187 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:54:39,847 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:54:40,222 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:54:41,853 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:54:42,259 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:54:42,260 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:54:42,260 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:54:42,265 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:54:42,854 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:54:43,856 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:54:45,266 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:54:46,591 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:54:46,592 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:54:46,592 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:54:46,597 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:54:46,859 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:54:47,861 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:54:50,598 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:54:51,109 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:54:51,110 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:54:51,110 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:54:51,115 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:54:51,864 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:54:52,547 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:54:52,548 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:54:52,548 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:54:53,869 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:54:55,850 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:54:57,305 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:54:57,306 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:54:57,306 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:54:57,311 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:54:57,873 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:54:59,877 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:55:00,861 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:55:00,862 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:55:00,863 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:55:00,863 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:55:00,867 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:55:00,878 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:55:01,881 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:55:02,995 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 17:55:05,996 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:55:07,547 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:55:07,547 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:55:07,547 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:55:09,127 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:55:09,128 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:55:09,128 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:55:09,133 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:55:09,890 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:55:11,170 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:55:11,895 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:55:12,351 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:55:12,352 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:55:12,352 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:55:12,357 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:55:12,895 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:55:13,898 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:55:16,034 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:55:16,035 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:55:16,035 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:55:16,040 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:55:16,900 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:55:17,041 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:55:17,903 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:55:21,461 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:55:21,462 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:55:21,462 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:55:21,467 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:55:21,908 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:55:22,468 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:55:22,547 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:55:22,547 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:55:22,548 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:55:23,912 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:55:27,813 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:55:29,140 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:55:29,141 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:55:29,141 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:55:29,146 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:55:29,919 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:55:31,923 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:55:32,995 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 17:55:32,996 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:55:33,539 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:55:33,540 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:55:33,540 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:55:33,546 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:55:33,925 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:55:35,929 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:55:37,077 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:55:37,078 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:55:37,079 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:55:37,083 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:55:37,547 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:55:37,547 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:55:37,548 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:55:37,930 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:55:38,095 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:55:39,934 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:55:41,823 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:55:41,824 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:55:41,825 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:55:41,830 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:55:41,937 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:55:43,866 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:55:43,940 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:55:46,974 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:55:46,975 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:55:46,975 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:55:46,980 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:55:47,945 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:55:47,946 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:55:48,981 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:55:52,165 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:55:52,166 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:55:52,166 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:55:52,171 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:55:52,547 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:55:52,548 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:55:52,548 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:55:52,951 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:55:53,955 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:55:54,787 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:55:55,877 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:55:55,878 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:55:55,878 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:55:55,883 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:55:55,956 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:55:57,960 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:55:59,756 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:55:59,757 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:55:59,757 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:55:59,762 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:55:59,962 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:56:00,763 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:56:01,966 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:56:03,000 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 17:56:03,646 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:56:03,647 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:56:03,648 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:56:03,653 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:56:03,968 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:56:05,972 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:56:06,654 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:56:07,547 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:56:07,548 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:56:07,548 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:56:09,758 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:56:09,758 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:56:09,759 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:56:09,763 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:56:09,976 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:56:11,802 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:56:11,981 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:56:16,802 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:56:18,225 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:56:18,226 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:56:18,226 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:56:18,231 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:56:18,987 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:56:19,992 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:56:22,232 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:56:22,547 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:56:22,548 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:56:22,548 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:56:23,225 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:56:23,225 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:56:23,226 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:56:23,231 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:56:23,996 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:56:26,001 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:56:26,968 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:56:26,969 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:56:26,970 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:56:26,975 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:56:27,001 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:56:27,975 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:56:28,004 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:56:32,540 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:56:32,541 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:56:32,541 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:56:32,546 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:56:32,999 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 17:56:33,001 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:56:33,008 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:56:34,012 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:56:37,548 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:56:37,548 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:56:37,548 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:56:38,823 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:56:38,974 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:56:38,975 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:56:38,975 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:56:38,981 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:56:39,017 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:56:40,021 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:56:42,339 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:56:42,340 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:56:42,340 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:56:42,345 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:56:43,024 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:56:44,027 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:56:44,346 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:56:46,136 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:56:46,137 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:56:46,137 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:56:46,143 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:56:47,029 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:56:48,032 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:56:50,143 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:56:50,480 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:56:50,481 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:56:50,481 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:56:50,486 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:56:51,035 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:56:52,039 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:56:52,548 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:56:52,548 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:56:52,548 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:56:53,628 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:56:53,629 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:56:53,629 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:56:53,634 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:56:54,041 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:56:55,670 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:56:56,044 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:56:59,333 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:56:59,334 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:56:59,334 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:56:59,339 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:57:00,048 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:57:01,339 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:57:02,053 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:57:03,001 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 17:57:07,002 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:57:07,548 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:57:07,548 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:57:07,548 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:57:08,551 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:57:08,552 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:57:08,552 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:57:08,557 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:57:09,060 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:57:10,064 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:57:12,444 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:57:12,445 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:57:12,445 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:57:12,445 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:57:12,450 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:57:13,066 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:57:14,070 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:57:17,451 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:57:17,799 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:57:17,800 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:57:17,800 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:57:17,805 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:57:18,073 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:57:20,079 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:57:21,612 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:57:21,613 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:57:21,613 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:57:21,618 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:57:22,081 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:57:22,548 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:57:22,548 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:57:22,548 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:57:22,549 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:57:24,085 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:57:25,989 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:57:25,990 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:57:25,990 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:57:25,995 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:57:26,086 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:57:28,034 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:57:28,090 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:57:29,643 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:57:29,644 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:57:29,644 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:57:29,649 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:57:30,092 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:57:32,096 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:57:32,639 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:57:32,640 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:57:32,640 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:57:32,645 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:57:33,003 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 17:57:33,097 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:57:34,005 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:57:34,100 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:57:36,652 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:57:36,653 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:57:36,653 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:57:36,658 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:57:37,102 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:57:37,548 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:57:37,548 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:57:37,549 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:57:38,105 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:57:39,810 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:57:42,143 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:57:42,144 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:57:42,144 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:57:42,150 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:57:43,111 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:57:44,114 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:57:45,186 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:57:46,092 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:57:46,093 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:57:46,093 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:57:46,098 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:57:46,115 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:57:48,119 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:57:51,058 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:57:51,059 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:57:51,059 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:57:51,059 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:57:51,064 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:57:51,122 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:57:52,126 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:57:52,548 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:57:52,548 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:57:52,549 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:57:55,231 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:57:55,232 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:57:55,232 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:57:55,237 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:57:56,130 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:57:56,274 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:57:58,135 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:57:59,184 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:57:59,185 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:57:59,185 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:57:59,190 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:58:00,137 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:58:02,140 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:58:02,191 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:58:02,753 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:58:02,754 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:58:02,754 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:58:02,759 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:58:03,005 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 17:58:03,141 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:58:04,144 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:58:06,336 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:58:06,337 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:58:06,338 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:58:06,343 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:58:07,146 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:58:07,343 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:58:07,548 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:58:07,548 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:58:07,549 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:58:08,149 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:58:10,532 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:58:10,533 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:58:10,533 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:58:10,538 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:58:11,151 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:58:12,154 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:58:12,574 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:58:15,000 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:58:15,000 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:58:15,001 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:58:15,007 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:58:15,156 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:58:16,160 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:58:18,007 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:58:22,548 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:58:22,548 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:58:22,549 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:58:23,374 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:58:23,375 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:58:23,375 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:58:23,375 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:58:23,380 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:58:24,169 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:58:26,174 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:58:28,328 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:58:28,329 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:58:28,329 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:58:28,334 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:58:29,176 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:58:29,335 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:58:30,180 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:58:31,650 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:58:31,651 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:58:31,651 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:58:31,656 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:58:32,181 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:58:33,010 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 17:58:34,186 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:58:35,012 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:58:35,360 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:58:35,361 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:58:35,361 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:58:35,366 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:58:36,187 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:58:37,548 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:58:37,548 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:58:37,549 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:58:38,191 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:58:39,938 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:58:39,939 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:58:39,939 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:58:39,944 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:58:40,193 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:58:40,982 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:58:42,198 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:58:43,631 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:58:43,632 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:58:43,633 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:58:43,637 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:58:44,200 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:58:46,203 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:58:46,638 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:58:47,198 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:58:47,199 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:58:47,199 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:58:47,204 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:58:47,205 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:58:48,207 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:58:51,012 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:58:51,013 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:58:51,013 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:58:51,018 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:58:51,209 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:58:52,018 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:58:52,212 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:58:52,548 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:58:52,548 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:58:52,549 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:58:54,475 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:58:54,475 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:58:54,476 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:58:54,481 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:58:55,214 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:58:56,217 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:58:57,518 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:59:02,518 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:59:03,009 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 17:59:03,035 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:59:03,036 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:59:03,036 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:59:03,041 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:59:03,223 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:59:04,227 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:59:06,377 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:59:06,377 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:59:06,378 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:59:06,382 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:59:07,229 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:59:07,548 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:59:07,548 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:59:07,549 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:59:07,549 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:59:08,231 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:59:10,094 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:59:10,094 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:59:10,094 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:59:10,100 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:59:10,234 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:59:12,239 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:59:13,138 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:59:13,663 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:59:13,664 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:59:13,665 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:59:13,670 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:59:14,240 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:59:16,244 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:59:18,011 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:59:18,012 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:59:18,012 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:59:18,017 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:59:18,246 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:59:19,017 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:59:20,250 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:59:22,546 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:59:22,547 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:59:22,547 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:59:22,549 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:59:22,549 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:59:22,552 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:59:22,552 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:59:23,253 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:59:24,256 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:59:24,784 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:59:28,525 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:59:28,525 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:59:28,526 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:59:28,531 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:59:29,260 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:59:30,265 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:59:30,566 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:59:32,278 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:59:32,279 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:59:32,280 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:59:32,285 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:59:33,015 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 17:59:33,267 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:59:34,270 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:59:35,689 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:59:35,690 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:59:35,690 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:59:35,690 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:59:35,695 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:59:36,271 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:59:37,549 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:59:37,549 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:59:37,549 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:59:38,275 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:59:40,817 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:59:41,645 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:59:41,646 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:59:41,646 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:59:41,651 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:59:42,281 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:59:44,282 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:59:45,467 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:59:45,468 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:59:45,468 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:59:45,473 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:59:46,285 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:59:46,474 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:59:48,290 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:59:51,474 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:59:52,549 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 17:59:52,549 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 17:59:52,549 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 17:59:52,923 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:59:52,924 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:59:52,924 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:59:52,929 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:59:53,295 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 17:59:54,299 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 17:59:56,966 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 17:59:57,789 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 17:59:57,790 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 17:59:57,790 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 17:59:57,795 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 17:59:58,302 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:00:00,307 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:00:02,796 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:00:03,014 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 18:00:03,815 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:00:03,816 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:00:03,816 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:00:03,821 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:00:04,311 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:00:06,315 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:00:07,549 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:00:07,549 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:00:07,549 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:00:08,175 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:00:09,384 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:00:09,385 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:00:09,385 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:00:09,391 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:00:10,319 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:00:12,323 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:00:13,426 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:00:13,802 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:00:13,803 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:00:13,803 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:00:13,808 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:00:14,325 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:00:16,329 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:00:18,357 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:00:18,358 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:00:18,358 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:00:18,364 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:00:19,331 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:00:19,364 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:00:20,335 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:00:21,624 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:00:21,625 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:00:21,625 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:00:21,631 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:00:22,337 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:00:22,549 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:00:22,549 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:00:22,549 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:00:24,341 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:00:24,848 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:00:26,264 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:00:26,265 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:00:26,265 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:00:26,270 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:00:26,342 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:00:28,347 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:00:30,306 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:00:31,343 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:00:31,344 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:00:31,344 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:00:31,350 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:00:31,350 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:00:32,353 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:00:33,019 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 18:00:36,021 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:00:37,549 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:00:37,549 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:00:37,549 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:00:39,931 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:00:39,932 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:00:39,932 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:00:39,937 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:00:40,361 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:00:41,974 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:00:42,367 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:00:43,651 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:00:43,652 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:00:43,652 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:00:43,657 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:00:44,369 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:00:46,372 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:00:47,658 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:00:49,054 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:00:49,055 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:00:49,055 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:00:49,061 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:00:49,374 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:00:50,376 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:00:52,549 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:00:52,549 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:00:52,549 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:00:52,790 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:00:53,681 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:00:53,681 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:00:53,682 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:00:53,686 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:00:54,382 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:00:56,387 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:00:57,311 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:00:57,312 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:00:57,312 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:00:57,318 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:00:57,387 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:00:58,318 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:00:58,393 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:01:00,409 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:01:00,410 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:01:00,410 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:01:00,415 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:01:01,395 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:01:02,398 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:01:03,018 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 18:01:04,020 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:01:04,334 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:01:04,335 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:01:04,335 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:01:04,341 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:01:04,400 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:01:06,403 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:01:07,549 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:01:07,549 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:01:07,550 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:01:07,933 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:01:08,176 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:01:08,176 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:01:08,181 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:01:08,404 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:01:09,182 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:01:10,408 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:01:11,684 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:01:11,685 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:01:11,686 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:01:11,691 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:01:12,409 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:01:14,412 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:01:14,691 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:01:15,638 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:01:15,639 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:01:15,639 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:01:15,644 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:01:16,414 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:01:18,417 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:01:19,341 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:01:19,342 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:01:19,342 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:01:19,348 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:01:19,418 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:01:20,349 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:01:20,422 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:01:22,549 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:01:22,549 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:01:22,550 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:01:24,891 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:01:24,892 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:01:24,892 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:01:24,897 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:01:25,426 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:01:25,934 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:01:26,427 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:01:28,964 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:01:28,965 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:01:28,965 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:01:28,971 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:01:29,431 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:01:30,435 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:01:30,971 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:01:32,549 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:01:32,550 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:01:32,551 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:01:32,556 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:01:33,023 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 18:01:33,437 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:01:34,440 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:01:36,026 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:01:36,941 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:01:36,941 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:01:36,942 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:01:36,947 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:01:37,443 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:01:37,549 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:01:37,549 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:01:37,550 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:01:38,446 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:01:40,447 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:01:40,448 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:01:40,448 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:01:40,453 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:01:41,450 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:01:41,490 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:01:42,454 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:01:45,930 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:01:45,931 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:01:45,931 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:01:45,936 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:01:46,457 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:01:46,937 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:01:48,460 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:01:50,209 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:01:50,210 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:01:50,210 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:01:50,216 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:01:50,462 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:01:52,217 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:01:52,467 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:01:52,549 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:01:52,549 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:01:52,550 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:01:55,354 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:01:55,354 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:01:55,355 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:01:55,360 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:01:55,472 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:01:55,472 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:01:57,398 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:02:02,398 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:02:03,022 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 18:02:03,533 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:02:03,534 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:02:03,534 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:02:03,539 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:02:04,481 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:02:05,484 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:02:07,540 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:02:07,549 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:02:07,549 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:02:07,550 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:02:07,725 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:02:07,811 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:02:07,811 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:02:07,816 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:02:08,487 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:02:09,489 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:02:11,768 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:02:11,768 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:02:11,769 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:02:11,774 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:02:12,492 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:02:12,774 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:02:13,495 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:02:16,126 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:02:16,127 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:02:16,127 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:02:16,132 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:02:16,498 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:02:17,501 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:02:18,133 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:02:20,126 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:02:20,127 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:02:20,127 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:02:20,132 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:02:20,505 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:02:21,508 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:02:22,549 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:02:22,550 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:02:22,550 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:02:23,805 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:02:25,088 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:02:25,089 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:02:25,089 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:02:25,094 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:02:25,511 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:02:27,515 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:02:28,815 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:02:28,815 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:02:28,816 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:02:28,816 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:02:28,821 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:02:29,518 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:02:29,518 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:02:33,025 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 18:02:34,026 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:02:37,093 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:02:37,094 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:02:37,094 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:02:37,099 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:02:37,527 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:02:37,550 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:02:37,550 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:02:37,550 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:02:39,532 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:02:39,858 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:02:41,881 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:02:41,882 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:02:41,882 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:02:41,887 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:02:42,536 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:02:43,538 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:02:44,925 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:02:45,795 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:02:45,796 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:02:45,796 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:02:45,802 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:02:46,540 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:02:47,543 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:02:49,349 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:02:49,350 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:02:49,351 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:02:49,355 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:02:49,544 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:02:50,356 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:02:51,549 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:02:52,550 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:02:52,550 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:02:52,550 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:02:53,846 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:02:53,847 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:02:53,847 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:02:53,852 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:02:54,552 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:02:55,554 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:02:55,890 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:02:57,741 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:02:57,742 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:02:57,742 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:02:57,747 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:02:58,557 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:02:59,560 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:03:01,748 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:03:02,431 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:03:02,431 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:03:02,432 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:03:02,437 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:03:02,563 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:03:03,027 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 18:03:03,565 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:03:05,936 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:03:05,937 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:03:05,938 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:03:05,943 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:03:06,568 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:03:06,943 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:03:07,550 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:03:07,550 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:03:07,550 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:03:07,571 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:03:09,400 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:03:09,401 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:03:09,401 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:03:09,406 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:03:09,572 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:03:11,576 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:03:12,442 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:03:13,081 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:03:13,082 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:03:13,082 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:03:13,087 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:03:13,578 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:03:15,582 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:03:18,088 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:03:18,325 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:03:18,326 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:03:18,326 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:03:18,331 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:03:18,585 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:03:19,587 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:03:22,550 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:03:22,550 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:03:22,550 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:03:23,841 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:03:24,276 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:03:24,277 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:03:24,277 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:03:24,283 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:03:24,594 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:03:25,597 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:03:28,506 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:03:28,507 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:03:28,507 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:03:28,512 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:03:28,599 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:03:29,513 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:03:29,602 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:03:33,029 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 18:03:33,463 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:03:33,464 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:03:33,464 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:03:33,469 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:03:33,606 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:03:35,470 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:03:35,610 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:03:37,098 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:03:37,099 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:03:37,100 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:03:37,105 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:03:37,550 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:03:37,551 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:03:37,551 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:03:37,613 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:03:39,617 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:03:40,842 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:03:41,718 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:03:41,719 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:03:41,719 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:03:41,724 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:03:42,620 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:03:43,623 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:03:46,218 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:03:46,219 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:03:46,219 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:03:46,219 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:03:46,225 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:03:46,625 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:03:47,628 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:03:49,976 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:03:49,977 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:03:49,977 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:03:49,982 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:03:50,630 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:03:51,633 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:03:51,983 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:03:52,550 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:03:52,551 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:03:52,551 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:03:53,361 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:03:53,362 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:03:53,362 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:03:53,367 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:03:53,634 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:03:55,638 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:03:57,368 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:04:01,774 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:04:01,775 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:04:01,775 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:04:01,780 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:04:02,647 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:04:02,781 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:04:03,031 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 18:04:03,649 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:04:07,535 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:04:07,536 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:04:07,536 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:04:07,541 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:04:07,550 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:04:07,551 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:04:07,551 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:04:07,653 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:04:07,864 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:04:09,658 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:04:12,865 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:04:15,402 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:04:15,403 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:04:15,403 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:04:15,408 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:04:15,664 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:04:17,669 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:04:18,409 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:04:18,959 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:04:18,959 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:04:18,960 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:04:18,965 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:04:19,673 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:04:19,673 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:04:22,550 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:04:22,551 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:04:22,551 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:04:22,674 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:04:22,844 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:04:22,844 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:04:22,849 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:04:23,679 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:04:23,679 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:04:23,850 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:04:25,711 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:04:25,712 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:04:25,712 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:04:25,717 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:04:26,681 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:04:27,684 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:04:29,718 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:04:33,033 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 18:04:34,365 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:04:34,366 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:04:34,366 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:04:34,372 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:04:34,692 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:04:35,372 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:04:35,695 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:04:37,550 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:04:37,551 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:04:37,551 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:04:39,709 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:04:39,710 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:04:39,710 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:04:39,715 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:04:40,700 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:04:40,716 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:04:41,702 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:04:43,257 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:04:43,258 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:04:43,259 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:04:43,264 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:04:43,705 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:04:45,709 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:04:46,264 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:04:47,749 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:04:47,749 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:04:47,750 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:04:47,755 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:04:48,711 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:04:49,714 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:04:51,756 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:04:52,550 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:04:52,551 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:04:52,551 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:04:56,330 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:04:56,331 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:04:56,331 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:04:56,337 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:04:56,723 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:04:57,338 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:04:57,726 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:05:00,528 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:05:00,529 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:05:00,529 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:05:00,534 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:05:00,728 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:05:01,731 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:05:02,535 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:05:03,039 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 18:05:05,931 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:05:05,932 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:05:05,932 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:05:05,937 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:05:06,737 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:05:07,550 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:05:07,551 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:05:07,551 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:05:07,551 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:05:07,740 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:05:09,622 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:05:09,623 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:05:09,623 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:05:09,629 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:05:09,741 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:05:11,745 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:05:12,630 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:05:13,991 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:05:13,992 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:05:13,992 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:05:13,997 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:05:14,748 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:05:15,751 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:05:17,998 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:05:19,668 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:05:19,668 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:05:19,669 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:05:19,674 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:05:19,754 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:05:21,760 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:05:22,550 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:05:22,551 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:05:22,551 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:05:23,806 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:05:24,286 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:05:24,287 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:05:24,288 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:05:24,293 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:05:24,763 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:05:25,765 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:05:27,903 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:05:27,903 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:05:27,904 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:05:27,910 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:05:28,768 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:05:28,910 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:05:29,771 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:05:31,404 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:05:31,405 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:05:31,405 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:05:31,411 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:05:31,772 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:05:33,037 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 18:05:33,776 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:05:34,039 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:05:37,550 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:05:37,551 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:05:37,551 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:05:39,755 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:05:39,756 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:05:39,756 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:05:39,756 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:05:39,761 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:05:39,782 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:05:41,786 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:05:43,222 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:05:43,222 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:05:43,223 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:05:43,228 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:05:43,791 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:05:43,792 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:05:45,229 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:05:47,423 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:05:47,424 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:05:47,424 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:05:47,429 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:05:47,794 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:05:49,798 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:05:50,430 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:05:50,915 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:05:50,916 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:05:50,916 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:05:50,921 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:05:51,802 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:05:51,802 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:05:52,551 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:05:52,551 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:05:52,551 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:05:54,678 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:05:54,679 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:05:54,679 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:05:54,684 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:05:54,807 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:05:55,685 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:05:55,810 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:05:59,265 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:05:59,266 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:05:59,266 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:05:59,272 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:05:59,812 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:06:01,273 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:06:01,816 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:06:03,006 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:06:03,007 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:06:03,007 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:06:03,012 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:06:03,043 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 18:06:03,819 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:06:03,819 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:06:07,045 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:06:07,250 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:06:07,251 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:06:07,252 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:06:07,257 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:06:07,551 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:06:07,551 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:06:07,551 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:06:07,824 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:06:07,824 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:06:11,046 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:06:11,047 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:06:11,047 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:06:11,052 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:06:11,829 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:06:11,829 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:06:12,053 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:06:16,614 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:06:16,615 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:06:16,615 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:06:16,621 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:06:16,834 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:06:17,621 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:06:17,837 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:06:21,927 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:06:21,928 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:06:21,928 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:06:21,933 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:06:22,551 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:06:22,551 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:06:22,552 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:06:22,844 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:06:22,845 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:06:23,847 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:06:26,100 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:06:26,101 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:06:26,102 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:06:26,107 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:06:26,850 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:06:27,853 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:06:28,107 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:06:29,630 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:06:29,631 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:06:29,631 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:06:29,638 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:06:29,854 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:06:31,858 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:06:32,976 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:06:32,977 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:06:32,977 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:06:32,983 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:06:33,042 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 18:06:33,862 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:06:33,862 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:06:34,044 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:06:37,551 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:06:37,551 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:06:37,552 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:06:38,108 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:06:38,109 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:06:38,110 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:06:38,115 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:06:38,867 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:06:39,115 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:06:39,870 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:06:44,066 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:06:44,067 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:06:44,067 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:06:44,073 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:06:44,875 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:06:45,073 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:06:45,878 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:06:47,775 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:06:47,776 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:06:47,777 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:06:47,782 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:06:47,879 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:06:49,883 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:06:50,783 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:06:52,551 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:06:52,552 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:06:52,552 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:06:54,671 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:06:54,671 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:06:54,672 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:06:54,677 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:06:54,890 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:06:55,892 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:06:56,678 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:07:00,116 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:07:00,117 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:07:00,118 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:07:00,124 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:07:00,898 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:07:01,900 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:07:02,125 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:07:03,048 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 18:07:03,673 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:07:03,674 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:07:03,675 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:07:03,680 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:07:03,902 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:07:05,906 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:07:07,551 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:07:07,551 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:07:07,552 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:07:07,552 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:07:09,444 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:07:09,444 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:07:09,444 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:07:09,449 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:07:09,909 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:07:11,914 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:07:13,486 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:07:13,848 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:07:13,849 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:07:13,849 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:07:13,854 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:07:13,915 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:07:15,919 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:07:18,855 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:07:20,979 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:07:20,979 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:07:20,980 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:07:20,985 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:07:21,928 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:07:21,929 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:07:22,551 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:07:22,552 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:07:22,552 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:07:24,833 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:07:26,830 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:07:26,830 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:07:26,831 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:07:26,836 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:07:26,934 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:07:27,936 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:07:29,837 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:07:33,046 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 18:07:35,048 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:07:35,319 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:07:35,320 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:07:35,320 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:07:35,325 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:07:35,947 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:07:35,948 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:07:37,551 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:07:37,552 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:07:37,552 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:07:39,705 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:07:39,706 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:07:39,706 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:07:39,711 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:07:39,951 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:07:40,711 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:07:41,955 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:07:44,487 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:07:44,488 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:07:44,489 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:07:44,494 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:07:44,959 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:07:45,961 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:07:46,495 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:07:48,960 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:07:48,961 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:07:48,961 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:07:48,967 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:07:49,967 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:07:49,968 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:07:51,968 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:07:52,551 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:07:52,552 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:07:52,552 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:07:53,762 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:07:53,762 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:07:53,763 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:07:53,768 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:07:53,972 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:07:55,977 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:07:57,755 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:07:57,756 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:07:57,757 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:07:57,757 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:07:57,762 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:07:57,978 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:07:59,983 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:08:01,037 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:08:01,038 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:08:01,039 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:08:01,044 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:08:01,986 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:08:01,986 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:08:03,045 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:08:03,049 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 18:08:04,384 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:08:04,385 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:08:04,385 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:08:04,391 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:08:04,988 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:08:05,991 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:08:07,551 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:08:07,552 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:08:07,552 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:08:08,085 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:08:08,785 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:08:08,786 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:08:08,786 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:08:08,791 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:08:08,993 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:08:09,996 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:08:12,692 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:08:12,693 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:08:12,693 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:08:12,698 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:08:12,999 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:08:13,699 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:08:14,002 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:08:17,953 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:08:17,954 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:08:17,954 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:08:17,959 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:08:18,006 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:08:18,959 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:08:20,011 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:08:22,551 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:08:22,552 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:08:22,552 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:08:24,774 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:08:26,419 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:08:26,420 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:08:26,421 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:08:26,426 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:08:27,019 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:08:28,022 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:08:30,415 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:08:30,416 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:08:30,416 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:08:30,416 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:08:30,421 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:08:31,025 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:08:32,027 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:08:33,051 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 18:08:34,890 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:08:34,891 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:08:34,891 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:08:34,896 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:08:35,030 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:08:35,897 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:08:36,033 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:08:37,552 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:08:37,552 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:08:37,552 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:08:38,502 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:08:38,503 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:08:38,503 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:08:38,508 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:08:39,036 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:08:40,039 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:08:41,509 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:08:42,230 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:08:42,231 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:08:42,231 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:08:42,236 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:08:43,043 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:08:44,045 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:08:47,237 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:08:50,636 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:08:50,637 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:08:50,637 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:08:50,642 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:08:51,052 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:08:52,055 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:08:52,552 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:08:52,552 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:08:52,552 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:08:52,553 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:08:54,532 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:08:54,533 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:08:54,533 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:08:54,538 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:08:55,059 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:08:56,061 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:08:57,574 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:08:58,121 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:08:58,122 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:08:58,122 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:08:58,128 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:08:59,064 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:09:00,067 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:09:02,522 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:09:02,523 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:09:02,523 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:09:02,529 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:09:03,053 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 18:09:03,056 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:09:03,070 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:09:04,072 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:09:07,552 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:09:07,552 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:09:07,552 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:09:07,607 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:09:08,733 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:09:08,733 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:09:08,733 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:09:08,739 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:09:09,078 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:09:10,080 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:09:11,969 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:09:11,970 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:09:11,970 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:09:11,975 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:09:12,082 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:09:13,976 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:09:14,086 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:09:15,668 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:09:15,669 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:09:15,669 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:09:15,675 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:09:16,087 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:09:18,091 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:09:19,676 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:09:20,915 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:09:20,916 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:09:20,916 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:09:20,921 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:09:21,095 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:09:22,098 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:09:22,552 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:09:22,552 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:09:22,552 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:09:24,749 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:09:24,750 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:09:24,750 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:09:24,750 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:09:24,755 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:09:25,101 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:09:26,103 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:09:29,756 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:09:33,059 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 18:09:33,360 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:09:33,361 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:09:33,362 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:09:33,367 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:09:34,114 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:09:34,115 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:09:35,367 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:09:37,543 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:09:37,544 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:09:37,544 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:09:37,549 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:09:37,552 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:09:37,552 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:09:37,553 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:09:38,118 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:09:40,123 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:09:40,879 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:09:41,220 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:09:41,221 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:09:41,221 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:09:41,227 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:09:42,127 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:09:42,127 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:09:44,945 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:09:44,946 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:09:44,947 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:09:44,952 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:09:45,130 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:09:45,952 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:09:46,133 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:09:48,662 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:09:48,663 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:09:48,663 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:09:48,668 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:09:49,134 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:09:50,137 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:09:51,669 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:09:52,490 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:09:52,491 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:09:52,491 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:09:52,496 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:09:52,552 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:09:52,552 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:09:52,553 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:09:53,140 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:09:54,144 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:09:56,753 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:09:57,668 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:09:57,669 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:09:57,669 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:09:57,675 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:09:58,146 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:10:00,151 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:10:01,564 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:10:01,565 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:10:01,565 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:10:01,570 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:10:02,153 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:10:02,571 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:10:03,057 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 18:10:04,158 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:10:05,506 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:10:05,507 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:10:05,507 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:10:05,512 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:10:06,161 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:10:06,162 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:10:07,552 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:10:07,553 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:10:07,553 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:10:07,794 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:10:10,418 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:10:10,419 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:10:10,419 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:10:10,424 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:10:11,166 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:10:12,168 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:10:13,425 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:10:15,824 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:10:15,825 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:10:15,825 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:10:15,830 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:10:16,172 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:10:18,176 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:10:18,831 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:10:22,552 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:10:22,553 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:10:22,553 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:10:24,322 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:10:24,323 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:10:24,323 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:10:24,323 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:10:24,328 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:10:25,185 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:10:26,188 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:10:27,851 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:10:27,852 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:10:27,852 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:10:27,857 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:10:28,190 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:10:29,858 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:10:30,193 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:10:31,624 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:10:31,625 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:10:31,625 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:10:31,630 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:10:32,195 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:10:33,062 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 18:10:34,199 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:10:35,006 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:10:35,007 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:10:35,007 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:10:35,007 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:10:35,012 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:10:35,199 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:10:36,202 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:10:37,552 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:10:37,553 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:10:37,553 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:10:38,276 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:10:38,277 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:10:38,277 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:10:38,282 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:10:39,204 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:10:40,207 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:10:40,283 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:10:42,637 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:10:42,638 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:10:42,638 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:10:42,643 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:10:43,212 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:10:44,214 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:10:45,644 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:10:45,964 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:10:45,965 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:10:45,965 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:10:45,970 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:10:46,216 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:10:48,219 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:10:49,898 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:10:49,899 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:10:49,899 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:10:49,905 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:10:50,220 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:10:50,905 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:10:52,225 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:10:52,552 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:10:52,553 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:10:52,553 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:10:53,717 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:10:53,717 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:10:53,718 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:10:53,723 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:10:54,226 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:10:56,230 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:10:56,724 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:10:57,469 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:10:57,470 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:10:57,470 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:10:57,475 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:10:58,234 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:10:58,234 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:11:01,415 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:11:01,416 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:11:01,417 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:11:01,421 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:11:02,239 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:11:02,239 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:11:02,422 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:11:03,061 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 18:11:06,435 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:11:06,436 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:11:06,436 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:11:06,442 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:11:07,244 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:11:07,443 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:11:07,552 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:11:07,553 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:11:07,553 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:11:08,247 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:11:09,975 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:11:09,976 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:11:09,976 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:11:09,981 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:11:10,248 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:11:12,253 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:11:12,982 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:11:15,439 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:11:15,440 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:11:15,441 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:11:15,445 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:11:16,258 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:11:16,259 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:11:18,446 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:11:19,976 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:11:19,977 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:11:19,977 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:11:19,982 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:11:20,261 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:11:22,267 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:11:22,552 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:11:22,553 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:11:22,553 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:11:23,794 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:11:25,009 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:11:25,010 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:11:25,010 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:11:25,015 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:11:25,270 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:11:26,273 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:11:28,395 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:11:28,396 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:11:28,396 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:11:28,401 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:11:29,275 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:11:29,401 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:11:30,278 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:11:31,542 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:11:31,543 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:11:31,543 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:11:31,548 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:11:32,281 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:11:32,282 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:11:33,067 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 18:11:35,068 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:11:35,922 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:11:35,923 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:11:35,923 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:11:35,928 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:11:36,284 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:11:37,553 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:11:37,553 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:11:37,553 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:11:38,288 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:11:39,577 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:11:39,577 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:11:39,578 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:11:39,583 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:11:40,292 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:11:40,292 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:11:40,583 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:11:44,233 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:11:44,233 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:11:44,233 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:11:44,238 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:11:44,296 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:11:46,239 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:11:46,299 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:11:49,026 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:11:49,027 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:11:49,027 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:11:49,032 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:11:49,302 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:11:50,305 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:11:52,033 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:11:52,553 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:11:52,553 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:11:52,553 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:11:54,454 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:11:54,454 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:11:54,455 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:11:54,460 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:11:55,311 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:11:56,314 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:11:57,460 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:11:58,915 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:11:58,916 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:11:58,916 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:11:58,921 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:11:59,317 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:12:00,319 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:12:02,916 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:12:02,917 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:12:02,917 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:12:02,917 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:12:02,922 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:12:03,066 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 18:12:03,322 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:12:04,325 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:12:07,178 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:12:07,178 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:12:07,179 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:12:07,183 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:12:07,328 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:12:07,553 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:12:07,553 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:12:07,553 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:12:08,330 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:12:08,749 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:12:12,131 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:12:12,132 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:12:12,132 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:12:12,137 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:12:12,334 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:12:14,137 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:12:14,339 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:12:16,907 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:12:16,908 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:12:16,908 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:12:16,913 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:12:17,341 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:12:18,344 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:12:19,914 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:12:22,553 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:12:22,554 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:12:22,554 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:12:23,163 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:12:23,164 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:12:23,164 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:12:23,169 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:12:23,350 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:12:24,353 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:12:25,170 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:12:27,259 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:12:27,260 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:12:27,260 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:12:27,265 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:12:27,355 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:12:28,358 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:12:30,266 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:12:30,552 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:12:30,553 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:12:30,554 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:12:30,559 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:12:31,361 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:12:32,364 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:12:33,071 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 18:12:34,298 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:12:34,299 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:12:34,299 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:12:34,304 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:12:34,365 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:12:35,305 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:12:36,369 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:12:37,553 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:12:37,553 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:12:37,554 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:12:40,755 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:12:42,607 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:12:42,608 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:12:42,608 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:12:42,613 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:12:43,379 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:12:44,382 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:12:46,614 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:12:47,244 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:12:47,244 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:12:47,245 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:12:47,250 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:12:47,384 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:12:48,387 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:12:51,050 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:12:51,051 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:12:51,051 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:12:51,056 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:12:51,389 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:12:52,057 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:12:52,393 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:12:52,553 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:12:52,553 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:12:52,554 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:12:57,874 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:12:58,969 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:12:58,970 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:12:58,970 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:12:58,975 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:12:59,400 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:13:00,403 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:13:02,976 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:13:03,070 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 18:13:07,553 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:13:07,554 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:13:07,554 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:13:07,617 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:13:07,844 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:13:07,844 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:13:07,850 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:13:08,411 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:13:08,850 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:13:10,417 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:13:10,985 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:13:10,986 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:13:10,986 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:13:10,992 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:13:11,417 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:13:12,420 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:13:13,992 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:13:14,510 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:13:14,510 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:13:14,511 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:13:14,518 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:13:15,422 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:13:16,425 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:13:18,315 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:13:18,316 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:13:18,316 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:13:18,321 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:13:18,427 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:13:19,322 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:13:20,430 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:13:22,553 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:13:22,554 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:13:22,554 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:13:24,831 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:13:26,702 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:13:26,703 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:13:26,703 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:13:26,710 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:13:27,439 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:13:28,442 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:13:30,710 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:13:30,723 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:13:30,723 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:13:30,724 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:13:30,729 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:13:31,445 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:13:32,448 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:13:33,073 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 18:13:35,450 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:13:35,451 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:13:35,451 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:13:35,457 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:13:36,455 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:13:36,455 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:13:36,458 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:13:37,553 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:13:37,554 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:13:37,554 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:13:39,183 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:13:39,184 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:13:39,184 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:13:39,190 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:13:39,457 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:13:40,461 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:13:42,190 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:13:43,565 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:13:43,566 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:13:43,566 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:13:43,572 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:13:44,469 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:13:44,469 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:13:47,573 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:13:48,341 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:13:48,342 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:13:48,343 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:13:48,349 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:13:48,472 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:13:50,476 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:13:51,667 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:13:51,668 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:13:51,668 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:13:51,674 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:13:52,480 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:13:52,480 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:13:52,553 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:13:52,554 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:13:52,554 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:13:52,838 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:13:56,193 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:13:56,193 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:13:56,194 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:13:56,199 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:13:56,482 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:13:58,200 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:13:58,484 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:13:59,713 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:13:59,713 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:13:59,714 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:13:59,719 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:14:00,488 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:14:00,489 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:14:03,075 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 18:14:03,665 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:14:03,666 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:14:03,666 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:14:03,667 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:14:03,671 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:14:04,494 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:14:04,494 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:14:07,553 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:14:07,554 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:14:07,554 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:14:08,808 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:14:10,770 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:14:10,771 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:14:10,772 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:14:10,776 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:14:11,502 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:14:12,505 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:14:14,777 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:14:15,171 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:14:15,172 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:14:15,173 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:14:15,179 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:14:15,508 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:14:16,515 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:14:20,180 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:14:21,723 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:14:21,723 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:14:21,724 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:14:21,733 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:14:22,525 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:14:22,525 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:14:22,553 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:14:22,554 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:14:22,554 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:14:25,850 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:14:26,025 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:14:26,025 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:14:26,026 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:14:26,035 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:14:26,529 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:14:28,536 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:14:30,123 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:14:30,124 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:14:30,124 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:14:30,137 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:14:30,537 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:14:31,137 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:14:32,543 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:14:33,077 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 18:14:37,079 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:14:37,554 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:14:37,554 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:14:37,554 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:14:38,629 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:14:38,630 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:14:38,630 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:14:38,643 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:14:39,549 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:14:40,555 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:14:42,534 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:14:42,535 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:14:42,535 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:14:42,535 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:14:42,548 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:14:42,557 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:14:44,566 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:14:45,937 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:14:45,938 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:14:45,939 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:14:45,946 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:14:46,568 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:14:47,947 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:14:48,581 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:14:51,481 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:14:51,482 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:14:51,483 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:14:51,493 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:14:51,584 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:14:52,553 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:14:52,554 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:14:52,554 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:14:52,591 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:14:53,772 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:14:55,010 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:14:55,011 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:14:55,011 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:14:55,021 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:14:55,594 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:14:56,600 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:14:59,022 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:14:59,095 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:14:59,095 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:14:59,096 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:14:59,107 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:14:59,603 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:15:00,608 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:15:02,755 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:15:02,756 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:15:02,756 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:15:02,763 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:15:03,079 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 18:15:03,611 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:15:04,081 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:15:04,617 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:15:06,724 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:15:06,725 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:15:06,725 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:15:06,737 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:15:07,554 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:15:07,554 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:15:07,554 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:15:07,619 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:15:08,625 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:15:09,850 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:15:10,269 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:15:10,269 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:15:10,270 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:15:10,277 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:15:10,627 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:15:12,632 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:15:13,913 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:15:13,914 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:15:13,915 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:15:13,922 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:15:14,634 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:15:14,922 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:15:16,640 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:15:18,472 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:15:18,473 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:15:18,473 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:15:18,482 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:15:18,642 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:15:20,482 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:15:20,647 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:15:22,554 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:15:22,554 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:15:22,555 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:15:22,610 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:15:22,782 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:15:22,783 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:15:22,790 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:15:23,654 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:15:24,657 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:15:25,791 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:15:27,332 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:15:27,333 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:15:27,333 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:15:27,343 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:15:27,661 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:15:28,668 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:15:31,343 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:15:33,081 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 18:15:33,779 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:15:33,780 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:15:33,780 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:15:33,789 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:15:34,680 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:15:34,680 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:15:36,790 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:15:37,554 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:15:37,555 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:15:37,555 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:15:37,646 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:15:38,233 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:15:38,233 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:15:38,240 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:15:38,684 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:15:40,693 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:15:41,548 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:15:41,548 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:15:41,549 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:15:41,554 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:15:41,694 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:15:42,555 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:15:42,701 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:15:45,282 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:15:45,283 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:15:45,283 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:15:45,290 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:15:45,705 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:15:46,710 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:15:48,291 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:15:49,236 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:15:49,237 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:15:49,238 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:15:49,245 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:15:49,712 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:15:50,718 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:15:52,554 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:15:52,555 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:15:52,555 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:15:52,801 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:15:52,841 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:15:52,841 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:15:52,850 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:15:53,721 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:15:53,851 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:15:54,726 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:15:56,932 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:15:56,933 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:15:56,933 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:15:56,941 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:15:57,730 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:15:58,735 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:15:58,942 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:16:00,392 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:16:00,398 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:16:00,398 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:16:00,407 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:16:00,737 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:16:02,742 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:16:03,084 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 18:16:04,085 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:16:04,470 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:16:04,471 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:16:04,471 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:16:04,481 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:16:04,744 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:16:06,749 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:16:07,554 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:16:07,555 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:16:07,555 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:16:08,164 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:16:08,165 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:16:08,165 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:16:08,173 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:16:08,751 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:16:09,174 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:16:10,757 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:16:12,518 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:16:12,519 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:16:12,519 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:16:12,527 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:16:12,758 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:16:14,528 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:16:14,764 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:16:15,710 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:16:15,711 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:16:15,711 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:16:15,718 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:16:15,765 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:16:16,768 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:16:19,719 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:16:22,554 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:16:22,555 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:16:22,555 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:16:22,597 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:16:22,767 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:16:22,768 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:16:22,778 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:16:23,776 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:16:24,779 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:16:24,782 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:16:27,479 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:16:27,480 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:16:27,480 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:16:27,487 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:16:27,785 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:16:28,792 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:16:30,487 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:16:30,953 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:16:30,954 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:16:30,954 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:16:30,961 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:16:31,795 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:16:32,799 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:16:33,085 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 18:16:34,850 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:16:34,850 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:16:34,851 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:16:34,864 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:16:35,803 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:16:35,865 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:16:36,808 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:16:37,554 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:16:37,555 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:16:37,555 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:16:39,204 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:16:39,204 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:16:39,205 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:16:39,213 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:16:39,811 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:16:40,815 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:16:41,214 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:16:42,739 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:16:42,740 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:16:42,740 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:16:42,750 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:16:42,817 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:16:44,829 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:16:46,751 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:16:51,346 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:16:51,347 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:16:51,347 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:16:51,356 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:16:51,834 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:16:52,357 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:16:52,562 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:16:52,563 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:16:52,563 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:16:52,840 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:16:57,802 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:16:57,861 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:16:57,861 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:16:57,862 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:16:57,872 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:16:58,851 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:16:58,851 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:17:02,902 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:17:03,091 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 18:17:03,524 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:17:03,525 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:17:03,526 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:17:03,534 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:17:03,855 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:17:04,863 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:17:07,562 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:17:07,563 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:17:07,563 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:17:08,848 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:17:10,698 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:17:10,699 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:17:10,699 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:17:10,706 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:17:10,869 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:17:12,873 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:17:14,338 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:17:14,339 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:17:14,339 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:17:14,340 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:17:14,345 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:17:14,874 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:17:16,885 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:17:18,315 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:17:18,316 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:17:18,316 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:17:18,323 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:17:18,887 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:17:20,324 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:17:20,891 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:17:22,563 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:17:22,563 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:17:22,563 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:17:25,867 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:17:25,988 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:17:25,989 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:17:25,989 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:17:25,995 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:17:26,904 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:17:26,905 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:17:31,030 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:17:31,645 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:17:31,645 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:17:31,646 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:17:31,655 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:17:31,909 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:17:32,915 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:17:33,090 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 18:17:36,092 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:17:37,562 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:17:37,563 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:17:37,563 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:17:37,612 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:17:37,815 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:17:37,816 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:17:37,824 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:17:37,919 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:17:38,920 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:17:40,776 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:17:40,776 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:17:40,777 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:17:40,782 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:17:40,923 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:17:41,783 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:17:42,930 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:17:45,384 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:17:45,385 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:17:45,385 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:17:45,391 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:17:45,934 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:17:46,946 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:17:47,392 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:17:49,769 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:17:49,770 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:17:49,770 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:17:49,777 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:17:49,948 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:17:50,953 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:17:52,563 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:17:52,563 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:17:52,563 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:17:52,563 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:17:53,431 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:17:53,432 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:17:53,432 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:17:53,441 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:17:53,956 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:17:54,961 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:17:58,216 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:17:58,217 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:17:58,217 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:17:58,217 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:17:58,223 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:17:58,966 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:18:00,971 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:18:01,844 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:18:01,845 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:18:01,845 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:18:01,852 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:18:01,971 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:18:02,975 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:18:03,096 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 18:18:04,098 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:18:06,488 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:18:06,489 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:18:06,489 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:18:06,496 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:18:06,978 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:18:07,563 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:18:07,563 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:18:07,563 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:18:08,985 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:18:09,798 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:18:12,433 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:18:12,434 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:18:12,434 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:18:12,440 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:18:12,988 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:18:14,992 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:18:15,478 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:18:16,224 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:18:16,225 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:18:16,225 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:18:16,232 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:18:16,994 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:18:18,999 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:18:19,704 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:18:19,705 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:18:19,705 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:18:19,711 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:18:20,000 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:18:20,711 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:18:21,005 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:18:22,563 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:18:22,563 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:18:22,563 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:18:23,338 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:18:23,339 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:18:23,339 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:18:23,345 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:18:24,010 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:18:25,015 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:18:26,382 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:18:27,141 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:18:27,142 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:18:27,142 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:18:27,149 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:18:28,017 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:18:29,023 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:18:32,149 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:18:33,094 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 18:18:33,152 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:18:33,153 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:18:33,153 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:18:33,160 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:18:34,027 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:18:35,031 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:18:37,161 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:18:37,285 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:18:37,286 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:18:37,286 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:18:37,292 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:18:37,563 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:18:37,563 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:18:37,563 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:18:38,033 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:18:39,040 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:18:42,301 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:18:42,555 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:18:42,555 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:18:42,555 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:18:42,561 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:18:43,044 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:18:45,051 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:18:46,987 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:18:46,988 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:18:46,989 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:18:46,995 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:18:47,052 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:18:47,995 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:18:49,056 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:18:52,563 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:18:52,563 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:18:52,563 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:18:53,102 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:18:53,103 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:18:53,103 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:18:53,103 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:18:53,108 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:18:54,060 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:18:55,063 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:18:56,352 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:18:56,353 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:18:56,354 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:18:56,359 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:18:57,065 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:18:58,359 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:18:59,069 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:19:02,195 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:19:02,196 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:19:02,196 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:19:02,202 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:19:03,071 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:19:03,100 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 18:19:04,101 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:19:05,075 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:19:07,563 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:19:07,563 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:19:07,563 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:19:07,795 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:19:07,811 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:19:07,811 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:19:07,816 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:19:08,077 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:19:09,080 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:19:09,817 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:19:11,841 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:19:11,841 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:19:11,842 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:19:11,847 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:19:12,082 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:19:13,086 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:19:14,847 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:19:15,546 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:19:15,547 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:19:15,547 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:19:15,553 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:19:16,088 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:19:17,091 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:19:20,553 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:19:22,563 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:19:22,563 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:19:22,564 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:19:24,195 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:19:24,195 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:19:24,195 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:19:24,200 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:19:25,098 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:19:26,239 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:19:27,101 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:19:28,201 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:19:28,202 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:19:28,202 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:19:28,207 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:19:29,103 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:19:31,107 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:19:31,584 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:19:31,585 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:19:31,585 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:19:31,585 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:19:31,590 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:19:32,107 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:19:33,099 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 18:19:33,110 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:19:36,075 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:19:36,076 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:19:36,076 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:19:36,081 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:19:36,112 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:19:37,082 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:19:37,114 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:19:37,563 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:19:37,563 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:19:37,563 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:19:40,687 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:19:40,687 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:19:40,687 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:19:40,692 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:19:41,118 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:19:42,731 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:19:43,121 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:19:44,004 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:19:44,005 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:19:44,005 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:19:44,010 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:19:44,123 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:19:45,126 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:19:47,729 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:19:47,730 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:19:47,730 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:19:47,735 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:19:47,736 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:19:48,128 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:19:49,131 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:19:52,051 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:19:52,051 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:19:52,051 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:19:52,057 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:19:52,133 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:19:52,563 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:19:52,563 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:19:52,563 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:19:52,847 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:19:53,136 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:19:57,847 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:20:00,411 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:20:00,412 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:20:00,412 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:20:00,417 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:20:01,142 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:20:03,101 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 18:20:03,102 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:20:03,145 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:20:05,010 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:20:05,011 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:20:05,011 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:20:05,016 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:20:05,146 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:20:07,149 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:20:07,563 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:20:07,563 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:20:07,564 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:20:08,497 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:20:08,498 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:20:08,498 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:20:08,498 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:20:08,503 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:20:09,151 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:20:11,154 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:20:13,081 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:20:13,081 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:20:13,082 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:20:13,086 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:20:13,155 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:20:14,087 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:20:15,158 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:20:18,035 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:20:18,040 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:20:18,040 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:20:18,045 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:20:18,160 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:20:19,163 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:20:20,045 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:20:22,041 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:20:22,042 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:20:22,042 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:20:22,048 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:20:22,166 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:20:22,563 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:20:22,564 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:20:22,564 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:20:23,168 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:20:24,942 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:20:24,942 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:20:24,943 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:20:24,948 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:20:25,170 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:20:25,987 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:20:27,173 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:20:29,231 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:20:29,231 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:20:29,232 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:20:29,237 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:20:30,175 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:20:31,177 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:20:31,237 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:20:33,103 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 18:20:35,162 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:20:35,163 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:20:35,163 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:20:35,168 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:20:35,180 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:20:37,169 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:20:37,183 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:20:37,563 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:20:37,564 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:20:37,564 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:20:39,630 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:20:39,630 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:20:39,631 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:20:39,635 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:20:40,185 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:20:41,188 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:20:42,675 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:20:43,163 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:20:43,163 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:20:43,163 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:20:43,168 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:20:43,190 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:20:45,194 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:20:48,169 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:20:48,508 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:20:48,509 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:20:48,509 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:20:48,514 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:20:49,197 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:20:51,200 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:20:52,564 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:20:52,564 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:20:52,564 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:20:53,191 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:20:53,192 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:20:53,192 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:20:53,192 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:20:53,197 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:20:53,201 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:20:55,205 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:20:56,625 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:20:56,626 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:20:56,626 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:20:56,631 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:20:57,206 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:20:58,632 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:20:59,209 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:21:02,068 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:21:02,069 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:21:02,069 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:21:02,077 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:21:02,211 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:21:03,105 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 18:21:03,215 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:21:04,107 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:21:05,804 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:21:05,805 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:21:05,805 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:21:05,810 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:21:06,217 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:21:07,219 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:21:07,564 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:21:07,564 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:21:07,564 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:21:09,801 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:21:14,446 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:21:14,446 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:21:14,447 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:21:14,452 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:21:15,224 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:21:15,491 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:21:17,227 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:21:18,202 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:21:18,203 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:21:18,203 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:21:18,208 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:21:18,228 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:21:19,231 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:21:21,208 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:21:22,564 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:21:22,564 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:21:22,564 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:21:22,944 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:21:22,944 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:21:22,944 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:21:22,949 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:21:23,234 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:21:25,238 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:21:26,508 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:21:26,509 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:21:26,509 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:21:26,509 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:21:26,514 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:21:27,239 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:21:29,243 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:21:31,039 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:21:31,043 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:21:31,043 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:21:31,048 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:21:31,244 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:21:32,049 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:21:33,108 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 18:21:33,247 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:21:35,768 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:21:35,769 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:21:35,769 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:21:35,774 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:21:36,249 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:21:37,251 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:21:37,564 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:21:37,564 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:21:37,564 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:21:37,564 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:21:42,784 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:21:42,784 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:21:42,785 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:21:42,785 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:21:42,790 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:21:43,255 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:21:45,260 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:21:46,527 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:21:46,527 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:21:46,527 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:21:46,532 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:21:47,261 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:21:48,533 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:21:49,264 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:21:50,176 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:21:50,177 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:21:50,177 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:21:50,182 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:21:50,265 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:21:51,268 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:21:52,564 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:21:52,564 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:21:52,564 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:21:53,837 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:21:55,585 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:21:55,585 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:21:55,586 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:21:55,590 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:21:56,271 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:21:57,274 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:21:59,225 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:21:59,225 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:21:59,226 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:21:59,226 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:21:59,231 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:21:59,275 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:22:01,278 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:22:03,110 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 18:22:05,111 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:22:07,564 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:22:07,564 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:22:07,564 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:22:07,637 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:22:07,809 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:22:07,809 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:22:07,814 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:22:08,282 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:22:09,285 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:22:10,815 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:22:13,470 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:22:13,471 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:22:13,471 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:22:13,476 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:22:14,288 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:22:15,291 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:22:16,477 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:22:18,206 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:22:18,207 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:22:18,208 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:22:18,212 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:22:18,293 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:22:19,295 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:22:22,213 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:22:22,564 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:22:22,564 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:22:22,565 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:22:23,286 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:22:23,286 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:22:23,287 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:22:23,292 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:22:23,298 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:22:25,300 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:22:27,320 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:22:27,321 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:22:27,321 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:22:27,321 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:22:27,326 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:22:28,302 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:22:29,305 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:22:31,255 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:22:31,256 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:22:31,256 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:22:31,261 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:22:31,306 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:22:33,115 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 18:22:33,117 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:22:33,309 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:22:36,007 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:22:36,008 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:22:36,008 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:22:36,013 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:22:36,311 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:22:37,314 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:22:37,564 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:22:37,564 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:22:37,564 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:22:38,854 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:22:39,282 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:22:39,282 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:22:39,283 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:22:39,287 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:22:39,315 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:22:41,321 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:22:42,201 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:22:42,204 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:22:42,204 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:22:42,208 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:22:42,322 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:22:43,325 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:22:44,209 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:22:46,582 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:22:46,583 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:22:46,583 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:22:46,588 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:22:47,328 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:22:49,332 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:22:49,589 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:22:52,238 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:22:52,239 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:22:52,239 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:22:52,244 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:22:52,333 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:22:52,564 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:22:52,564 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:22:52,565 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:22:53,336 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:22:54,806 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:22:57,732 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:22:57,733 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:22:57,733 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:22:57,738 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:22:58,340 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:22:59,343 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:23:00,774 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:23:02,663 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:23:02,664 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:23:02,664 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:23:02,670 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:23:03,114 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 18:23:03,345 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:23:05,349 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:23:06,115 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:23:06,879 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:23:06,880 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:23:06,880 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:23:06,885 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:23:07,350 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:23:07,564 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:23:07,564 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:23:07,565 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:23:09,353 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:23:11,025 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:23:11,026 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:23:11,026 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:23:11,032 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:23:11,355 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:23:12,070 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:23:13,358 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:23:15,270 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:23:15,271 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:23:15,271 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:23:15,276 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:23:15,359 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:23:17,276 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:23:17,362 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:23:19,173 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:23:19,173 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:23:19,174 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:23:19,178 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:23:19,363 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:23:21,367 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:23:22,564 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:23:22,565 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:23:22,565 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:23:22,565 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:23:23,311 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:23:23,312 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:23:23,312 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:23:23,317 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:23:23,369 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:23:25,373 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:23:27,010 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:23:27,010 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:23:27,011 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:23:27,015 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:23:27,374 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:23:28,016 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:23:29,378 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:23:33,016 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:23:33,120 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 18:23:33,169 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:23:33,170 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:23:33,170 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:23:33,175 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:23:33,380 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:23:35,384 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:23:36,560 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:23:36,560 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:23:36,561 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:23:36,566 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:23:37,385 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:23:37,564 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:23:37,565 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:23:37,565 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:23:38,079 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:23:39,389 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:23:40,209 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:23:40,210 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:23:40,210 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:23:40,215 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:23:40,389 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:23:41,392 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:23:43,254 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:23:43,724 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:23:43,725 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:23:43,725 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:23:43,730 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:23:44,396 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:23:45,398 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:23:47,256 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:23:47,257 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:23:47,258 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:23:47,262 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:23:47,399 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:23:48,263 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:23:49,403 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:23:50,921 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:23:50,922 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:23:50,922 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:23:50,926 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:23:51,404 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:23:52,565 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:23:52,565 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:23:52,565 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:23:53,407 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:23:53,847 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:23:55,167 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:23:55,168 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:23:55,168 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:23:55,173 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:23:55,409 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:23:57,412 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:23:58,832 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:23:58,833 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:23:58,833 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:23:58,838 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:23:59,413 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:23:59,839 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:24:01,416 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:24:02,092 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:24:02,093 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:24:02,093 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:24:02,098 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:24:02,417 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:24:03,119 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 18:24:03,419 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:24:05,120 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:24:06,504 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:24:06,505 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:24:06,505 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:24:06,510 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:24:07,424 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:24:07,424 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:24:07,565 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:24:07,565 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:24:07,565 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:24:10,126 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:24:11,961 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:24:11,962 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:24:11,962 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:24:11,980 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:24:12,427 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:24:13,430 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:24:16,002 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:24:16,481 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:24:16,482 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:24:16,482 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:24:16,487 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:24:17,434 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:24:17,434 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:24:20,802 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:24:20,803 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:24:20,803 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:24:20,808 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:24:21,437 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:24:21,808 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:24:22,565 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:24:22,565 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:24:22,565 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:24:23,441 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:24:24,186 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:24:24,186 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:24:24,186 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:24:24,191 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:24:24,442 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:24:25,445 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:24:27,230 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:24:27,817 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:24:27,818 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:24:27,818 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:24:27,823 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:24:28,447 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:24:29,449 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:24:31,014 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:24:31,015 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:24:31,015 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:24:31,020 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:24:31,451 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:24:33,021 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:24:33,124 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 18:24:33,454 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:24:35,666 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:24:35,667 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:24:35,667 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:24:35,672 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:24:36,456 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:24:37,459 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:24:37,565 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:24:37,565 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:24:37,565 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:24:38,843 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:24:39,785 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:24:39,786 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:24:39,786 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:24:39,791 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:24:40,461 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:24:41,464 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:24:43,892 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:24:43,893 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:24:43,893 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:24:43,893 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:24:43,898 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:24:44,467 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:24:45,470 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:24:48,341 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:24:48,342 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:24:48,343 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:24:48,349 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:24:48,472 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:24:49,349 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:24:49,475 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:24:51,870 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:24:51,871 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:24:51,871 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:24:51,876 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:24:52,477 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:24:52,565 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:24:52,565 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:24:52,565 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:24:53,479 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:24:54,850 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:24:56,039 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:24:56,039 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:24:56,040 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:24:56,045 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:24:56,482 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:24:57,485 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:25:00,082 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:25:00,547 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:25:00,548 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:25:00,548 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:25:00,554 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:25:01,489 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:25:01,489 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:25:03,123 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 18:25:05,052 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:25:05,052 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:25:05,053 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:25:05,058 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:25:05,492 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:25:06,058 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:25:07,495 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:25:07,565 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:25:07,565 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:25:07,565 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:25:08,524 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:25:08,525 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:25:08,525 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:25:08,530 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:25:09,499 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:25:09,499 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:25:11,566 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:25:13,874 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:25:13,875 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:25:13,875 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:25:13,880 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:25:14,502 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:25:15,505 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:25:16,881 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:25:17,324 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:25:17,325 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:25:17,325 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:25:17,330 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:25:17,506 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:25:19,507 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:25:21,829 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:25:21,830 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:25:21,830 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:25:21,835 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:25:22,510 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:25:22,565 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:25:22,565 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:25:22,565 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:25:22,565 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:25:23,513 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:25:25,163 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:25:25,164 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:25:25,164 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:25:25,169 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:25:25,514 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:25:27,517 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:25:28,206 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:25:28,544 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:25:28,544 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:25:28,545 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:25:28,550 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:25:29,521 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:25:29,521 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:25:32,457 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:25:32,458 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:25:32,458 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:25:32,463 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:25:32,523 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:25:33,133 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 18:25:33,525 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:25:34,135 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:25:37,565 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:25:37,565 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:25:37,565 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:25:38,196 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:25:38,197 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:25:38,197 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:25:38,202 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:25:38,528 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:25:39,238 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:25:39,531 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:25:42,008 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:25:42,009 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:25:42,009 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:25:42,014 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:25:42,533 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:25:43,536 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:25:45,015 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:25:45,756 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:25:45,756 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:25:45,757 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:25:45,761 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:25:46,539 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:25:47,542 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:25:49,521 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:25:49,522 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:25:49,523 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:25:49,528 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:25:49,543 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:25:50,528 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:25:51,546 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:25:52,565 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:25:52,565 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:25:52,566 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:25:52,881 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:25:52,882 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:25:52,882 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:25:52,887 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:25:53,548 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:25:55,549 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:25:55,922 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:25:56,796 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:25:56,797 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:25:56,797 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:25:56,802 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:25:57,551 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:25:59,554 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:26:00,513 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:26:00,514 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:26:00,514 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:26:00,519 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:26:00,554 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:26:01,519 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:26:01,558 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:26:03,128 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 18:26:07,129 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:26:07,565 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:26:07,565 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:26:07,566 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:26:08,966 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:26:08,967 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:26:08,967 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:26:08,971 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:26:09,563 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:26:11,566 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:26:13,010 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:26:14,819 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:26:14,819 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:26:14,820 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:26:14,824 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:26:15,568 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:26:17,580 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:26:18,825 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:26:19,530 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:26:19,531 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:26:19,531 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:26:19,537 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:26:19,582 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:26:21,585 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:26:22,565 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:26:22,566 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:26:22,566 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:26:23,073 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:26:23,074 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:26:23,074 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:26:23,079 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:26:23,587 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:26:24,118 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:26:25,591 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:26:27,416 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:26:27,417 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:26:27,417 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:26:27,422 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:26:27,592 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:26:29,423 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:26:29,595 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:26:31,034 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:26:31,035 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:26:31,035 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:26:31,040 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:26:31,597 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:26:33,133 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 18:26:33,600 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:26:35,135 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:26:35,486 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:26:35,487 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:26:35,487 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:26:35,492 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:26:35,601 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:26:37,565 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:26:37,566 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:26:37,566 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:26:37,605 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:26:39,156 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:26:39,157 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:26:39,157 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:26:39,162 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:26:39,606 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:26:40,198 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:26:41,610 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:26:43,720 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:26:43,720 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:26:43,721 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:26:43,726 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:26:44,613 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:26:45,616 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:26:45,726 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:26:48,512 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:26:48,513 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:26:48,514 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:26:48,518 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:26:48,617 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:26:49,620 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:26:51,519 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:26:52,566 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:26:52,566 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:26:52,566 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:26:52,848 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:26:52,848 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:26:52,848 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:26:52,853 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:26:53,623 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:26:55,626 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:26:56,629 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:26:56,629 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:26:56,629 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:26:56,629 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:26:56,634 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:26:57,630 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:26:57,630 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:27:01,162 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:27:01,163 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:27:01,163 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:27:01,168 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:27:01,632 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:27:02,168 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:27:03,132 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 18:27:03,636 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:27:04,410 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:27:04,411 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:27:04,411 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:27:04,416 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:27:04,636 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:27:05,639 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:27:07,416 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:27:07,566 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:27:07,566 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:27:07,566 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:27:09,186 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:27:09,187 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:27:09,187 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:27:09,192 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:27:09,641 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:27:11,645 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:27:13,230 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:27:13,442 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:27:13,443 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:27:13,443 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:27:13,448 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:27:13,646 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:27:15,649 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:27:17,335 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:27:17,336 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:27:17,336 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:27:17,341 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:27:17,651 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:27:18,341 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:27:19,654 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:27:21,707 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:27:21,708 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:27:21,708 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:27:21,713 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:27:22,566 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:27:22,566 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:27:22,566 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:27:22,656 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:27:23,659 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:27:23,811 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:27:25,406 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:27:25,406 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:27:25,407 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:27:25,412 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:27:25,661 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:27:27,664 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:27:29,450 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:27:33,134 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 18:27:33,298 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:27:33,299 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:27:33,299 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:27:33,304 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:27:33,668 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:27:35,305 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:27:35,671 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:27:36,773 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:27:36,774 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:27:36,774 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:27:36,779 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:27:37,566 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:27:37,566 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:27:37,566 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:27:37,674 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:27:37,674 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:27:40,500 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:27:40,504 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:27:40,504 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:27:40,504 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:27:40,509 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:27:40,676 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:27:41,679 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:27:45,542 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:27:46,368 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:27:46,368 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:27:46,369 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:27:46,374 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:27:46,683 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:27:47,686 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:27:51,374 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:27:52,566 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:27:52,566 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:27:52,566 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:27:54,160 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:27:54,161 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:27:54,161 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:27:54,166 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:27:54,690 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:27:55,693 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:27:57,202 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:27:57,890 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:27:57,890 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:27:57,891 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:27:57,895 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:27:58,695 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:27:59,698 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:28:02,896 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:28:03,137 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 18:28:03,233 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:28:03,234 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:28:03,234 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:28:03,239 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:28:03,700 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:28:05,704 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:28:07,566 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:28:07,566 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:28:07,566 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:28:08,117 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:28:08,117 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:28:08,118 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:28:08,118 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:28:08,122 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:28:08,705 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:28:09,708 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:28:11,974 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:28:11,975 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:28:11,975 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:28:11,980 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:28:12,710 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:28:13,713 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:28:13,981 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:28:15,409 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:28:15,410 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:28:15,410 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:28:15,415 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:28:15,714 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:28:17,717 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:28:19,415 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:28:20,265 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:28:20,266 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:28:20,266 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:28:20,271 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:28:20,719 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:28:21,721 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:28:22,566 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:28:22,566 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:28:22,566 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:28:24,807 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:28:25,269 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:28:25,269 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:28:25,270 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:28:25,274 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:28:25,725 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:28:27,728 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:28:28,778 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:28:28,779 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:28:28,779 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:28:28,784 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:28:29,732 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:28:29,732 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:28:30,784 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:28:33,139 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 18:28:36,140 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:28:37,381 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:28:37,382 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:28:37,382 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:28:37,387 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:28:37,566 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:28:37,566 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:28:37,609 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:28:37,737 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:28:39,740 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:28:41,358 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:28:41,358 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:28:41,359 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:28:41,359 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:28:41,363 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:28:41,742 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:28:43,745 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:28:45,316 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:28:45,317 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:28:45,318 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:28:45,322 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:28:45,747 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:28:47,323 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:28:47,751 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:28:48,600 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:28:48,601 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:28:48,601 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:28:48,606 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:28:48,751 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:28:49,759 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:28:52,566 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:28:52,566 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:28:52,567 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:28:52,567 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:28:55,221 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:28:55,222 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:28:55,222 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:28:55,227 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:28:55,763 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:28:57,767 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:28:58,265 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:28:59,883 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:28:59,883 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:28:59,883 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:28:59,888 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:29:00,769 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:29:01,771 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:29:03,141 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 18:29:03,763 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:29:03,763 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:29:03,764 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:29:03,764 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:29:03,768 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:29:03,773 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:29:05,776 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:29:07,442 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:29:07,443 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:29:07,443 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:29:07,448 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:29:07,566 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:29:07,566 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:29:07,568 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:29:07,777 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:29:08,787 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:29:09,781 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:29:13,389 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:29:13,389 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:29:13,390 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:29:13,394 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:29:13,783 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:29:14,431 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:29:15,786 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:29:18,333 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:29:18,334 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:29:18,334 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:29:18,339 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:29:18,788 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:29:19,791 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:29:20,339 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:29:21,629 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:29:21,630 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:29:21,630 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:29:21,635 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:29:21,793 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:29:22,566 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:29:22,566 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:29:22,567 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:29:23,797 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:29:25,826 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:29:25,869 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:29:25,869 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:29:25,869 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:29:25,874 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:29:26,799 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:29:27,802 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:29:30,015 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:29:30,016 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:29:30,016 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:29:30,021 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:29:30,804 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:29:31,021 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:29:31,806 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:29:33,143 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 18:29:34,412 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:29:34,413 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:29:34,413 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:29:34,418 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:29:34,808 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:29:35,811 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:29:36,419 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:29:37,566 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:29:37,567 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:29:37,567 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:29:38,548 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:29:38,549 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:29:38,549 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:29:38,554 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:29:38,813 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:29:39,816 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:29:41,591 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:29:42,135 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:29:42,136 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:29:42,136 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:29:42,141 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:29:42,818 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:29:43,820 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:29:47,142 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:29:47,565 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:29:47,566 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:29:47,566 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:29:47,571 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:29:47,824 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:29:49,828 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:29:52,566 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:29:52,567 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:29:52,567 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:29:52,567 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:29:56,121 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:29:56,122 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:29:56,122 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:29:56,127 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:29:56,832 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:29:57,835 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:29:58,163 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:30:00,005 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:30:00,006 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:30:00,006 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:30:00,011 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:30:00,837 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:30:01,839 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:30:03,148 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 18:30:03,595 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:30:03,595 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:30:03,595 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:30:03,596 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:30:03,600 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:30:03,841 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:30:05,844 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:30:07,104 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:30:07,105 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:30:07,106 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:30:07,110 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:30:07,566 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:30:07,567 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:30:07,568 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:30:07,846 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:30:08,811 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:30:09,849 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:30:10,837 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:30:10,838 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:30:10,838 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:30:10,843 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:30:10,850 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:30:11,853 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:30:13,878 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:30:18,878 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:30:19,399 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:30:19,400 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:30:19,400 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:30:19,405 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:30:19,858 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:30:21,861 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:30:22,567 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:30:22,567 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:30:22,567 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:30:23,283 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:30:23,283 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:30:23,284 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:30:23,288 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:30:23,863 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:30:24,326 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:30:25,867 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:30:28,414 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:30:28,415 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:30:28,415 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:30:28,421 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:30:28,868 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:30:29,421 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:30:29,871 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:30:32,114 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:30:32,114 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:30:32,115 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:30:32,120 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:30:32,873 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:30:33,147 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 18:30:33,876 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:30:35,149 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:30:37,567 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:30:37,567 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:30:37,567 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:30:37,861 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:30:37,862 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:30:37,862 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:30:37,867 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:30:37,878 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:30:39,882 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:30:40,906 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:30:42,598 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:30:42,599 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:30:42,599 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:30:42,604 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:30:42,884 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:30:43,887 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:30:46,604 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:30:47,870 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:30:47,871 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:30:47,871 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:30:47,876 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:30:47,891 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:30:49,894 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:30:50,995 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:30:50,996 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:30:50,996 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:30:51,001 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:30:51,897 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:30:51,897 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:30:52,002 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:30:52,567 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:30:52,567 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:30:52,568 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:30:54,541 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:30:54,541 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:30:54,542 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:30:54,546 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:30:54,899 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:30:55,902 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:30:57,582 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:30:58,071 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:30:58,071 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:30:58,071 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:30:58,076 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:30:58,904 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:30:59,907 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:31:01,671 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:31:01,671 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:31:01,671 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:31:01,676 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:31:01,908 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:31:02,677 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:31:03,153 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 18:31:03,911 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:31:05,415 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:31:05,416 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:31:05,416 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:31:05,421 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:31:05,912 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:31:07,567 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:31:07,567 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:31:07,567 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:31:07,916 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:31:08,020 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:31:09,320 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:31:09,321 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:31:09,321 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:31:09,326 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:31:09,917 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:31:11,921 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:31:12,636 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:31:12,637 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:31:12,637 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:31:12,642 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:31:12,921 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:31:13,643 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:31:13,924 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:31:17,457 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:31:17,458 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:31:17,458 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:31:17,463 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:31:17,926 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:31:19,463 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:31:19,930 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:31:20,911 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:31:20,912 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:31:20,912 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:31:20,917 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:31:20,930 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:31:21,933 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:31:22,567 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:31:22,567 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:31:22,567 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:31:24,769 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:31:26,275 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:31:26,276 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:31:26,276 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:31:26,281 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:31:26,937 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:31:27,940 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:31:30,318 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:31:30,598 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:31:30,599 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:31:30,599 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:31:30,604 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:31:30,942 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:31:31,945 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:31:33,152 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 18:31:34,922 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:31:34,922 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:31:34,923 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:31:34,928 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:31:34,946 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:31:35,928 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:31:35,949 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:31:37,567 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:31:37,567 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:31:37,568 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:31:41,764 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:31:43,054 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:31:43,055 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:31:43,055 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:31:43,060 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:31:43,954 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:31:45,958 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:31:46,944 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:31:46,945 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:31:46,946 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:31:46,946 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:31:46,951 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:31:46,959 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:31:47,962 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:31:51,951 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:31:52,567 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:31:52,567 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:31:52,567 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:31:55,366 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:31:55,366 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:31:55,366 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:31:55,371 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:31:55,967 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:31:57,410 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:31:57,971 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:32:00,014 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:32:00,015 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:32:00,015 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:32:00,019 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:32:00,974 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:32:01,977 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:32:03,020 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:32:03,157 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 18:32:03,945 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:32:03,946 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:32:03,946 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:32:03,951 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:32:03,978 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:32:05,982 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:32:07,567 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:32:07,567 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:32:07,567 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:32:08,635 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:32:08,636 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:32:08,636 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:32:08,636 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:32:08,641 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:32:08,985 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:32:09,987 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:32:13,258 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:32:13,259 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:32:13,259 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:32:13,264 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:32:13,991 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:32:14,265 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:32:15,996 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:32:16,978 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:32:16,979 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:32:16,979 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:32:16,984 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:32:16,996 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:32:17,999 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:32:19,985 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:32:21,246 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:32:21,247 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:32:21,247 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:32:21,253 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:32:22,002 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:32:22,567 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:32:22,568 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:32:22,568 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:32:24,007 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:32:25,324 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:32:25,325 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:32:25,325 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:32:25,325 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:32:25,330 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:32:26,009 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:32:28,013 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:32:29,035 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:32:29,035 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:32:29,035 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:32:29,040 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:32:30,014 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:32:30,014 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:32:31,041 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:32:32,791 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:32:32,792 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:32:32,792 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:32:32,797 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:32:33,019 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:32:33,156 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 18:32:34,021 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:32:36,158 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:32:37,567 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:32:37,568 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:32:37,568 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:32:41,450 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:32:41,451 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:32:41,451 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:32:41,451 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:32:41,456 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:32:42,031 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:32:44,034 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:32:45,214 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:32:45,215 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:32:45,215 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:32:45,220 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:32:46,037 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:32:47,221 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:32:48,040 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:32:50,999 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:32:51,000 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:32:51,000 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:32:51,005 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:32:51,043 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:32:52,046 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:32:52,568 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:32:52,568 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:32:52,568 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:32:52,568 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:32:55,505 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:32:55,505 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:32:55,506 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:32:55,511 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:32:56,051 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:32:58,055 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:32:58,546 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:33:03,158 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 18:33:03,945 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:33:03,946 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:33:03,946 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:33:03,947 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:33:03,960 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:33:04,061 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:33:06,066 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:33:07,567 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:33:07,568 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:33:07,568 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:33:07,684 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:33:07,802 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:33:07,802 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:33:07,807 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:33:08,067 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:33:09,808 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:33:10,071 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:33:12,925 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:33:12,926 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:33:12,927 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:33:12,932 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:33:13,074 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:33:14,077 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:33:14,932 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:33:16,995 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:33:16,996 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:33:16,996 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:33:17,001 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:33:17,079 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:33:18,082 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:33:20,002 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:33:20,631 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:33:20,631 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:33:20,631 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:33:20,637 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:33:21,085 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:33:22,088 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:33:22,568 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:33:22,568 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:33:22,568 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:33:25,814 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:33:26,233 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:33:26,234 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:33:26,234 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:33:26,239 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:33:27,095 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:33:28,097 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:33:30,856 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:33:30,857 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:33:30,858 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:33:30,858 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:33:30,862 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:33:31,100 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:33:32,102 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:33:33,160 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 18:33:35,271 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:33:35,271 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:33:35,272 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:33:35,276 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:33:36,106 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:33:36,277 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:33:37,568 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:33:37,568 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:33:37,568 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:33:38,111 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:33:40,441 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:33:40,442 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:33:40,442 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:33:40,447 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:33:41,114 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:33:41,482 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:33:42,117 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:33:43,829 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:33:43,830 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:33:43,830 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:33:43,835 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:33:44,118 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:33:46,123 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:33:46,836 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:33:48,850 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:33:48,850 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:33:48,851 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:33:48,855 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:33:49,125 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:33:50,127 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:33:51,856 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:33:52,568 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:33:52,568 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:33:52,568 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:33:54,222 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:33:54,223 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:33:54,223 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:33:54,228 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:33:55,133 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:33:56,136 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:33:57,267 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:33:57,708 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:33:57,709 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:33:57,709 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:33:57,714 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:33:58,138 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:34:00,141 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:34:01,361 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:34:01,362 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:34:01,362 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:34:01,367 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:34:02,143 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:34:02,368 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:34:03,163 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 18:34:04,146 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:34:05,527 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:34:05,527 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:34:05,528 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:34:05,533 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:34:06,148 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:34:07,533 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:34:07,568 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:34:07,568 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:34:07,568 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:34:08,152 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:34:10,076 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:34:10,077 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:34:10,077 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:34:10,082 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:34:10,153 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:34:12,156 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:34:13,119 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:34:15,886 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:34:15,887 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:34:15,887 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:34:15,892 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:34:16,162 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:34:18,166 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:34:18,893 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:34:22,568 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:34:22,568 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:34:22,568 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:34:24,351 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:34:24,355 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:34:24,356 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:34:24,356 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:34:24,361 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:34:25,175 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:34:26,178 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:34:28,030 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:34:28,031 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:34:28,031 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:34:28,036 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:34:28,179 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:34:30,037 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:34:30,183 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:34:32,346 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:34:32,347 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:34:32,347 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:34:32,352 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:34:33,165 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 18:34:33,186 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:34:34,189 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:34:35,166 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:34:36,247 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:34:36,248 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:34:36,248 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:34:36,253 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:34:37,192 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:34:37,568 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:34:37,568 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:34:37,569 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:34:38,194 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:34:39,789 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:34:39,789 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:34:39,790 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:34:39,795 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:34:40,196 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:34:40,831 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:34:42,200 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:34:42,931 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:34:42,931 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:34:42,932 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:34:42,937 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:34:43,201 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:34:44,203 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:34:45,937 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:34:46,646 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:34:46,647 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:34:46,647 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:34:46,652 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:34:47,207 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:34:48,207 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:34:50,444 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:34:50,444 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:34:50,445 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:34:50,450 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:34:51,211 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:34:51,450 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:34:52,214 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:34:52,568 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:34:52,568 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:34:52,569 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:34:54,800 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:34:54,801 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:34:54,801 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:34:54,806 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:34:55,217 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:34:56,221 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:34:56,843 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:35:01,843 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:35:03,167 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 18:35:03,272 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:35:03,273 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:35:03,273 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:35:03,278 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:35:04,231 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:35:04,231 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:35:07,184 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:35:07,184 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:35:07,185 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:35:07,185 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:35:07,190 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:35:07,235 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:35:07,568 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:35:07,569 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:35:07,569 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:35:08,237 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:35:10,447 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:35:10,447 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:35:10,447 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:35:10,452 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:35:11,240 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:35:12,243 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:35:12,490 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:35:16,259 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:35:16,260 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:35:16,260 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:35:16,265 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:35:17,248 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:35:18,251 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:35:18,265 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:35:22,568 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:35:22,569 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:35:22,569 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:35:23,797 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:35:24,538 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:35:24,539 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:35:24,539 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:35:24,544 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:35:25,260 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:35:26,262 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:35:29,582 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:35:29,763 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:35:29,764 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:35:29,764 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:35:29,769 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:35:30,266 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:35:32,270 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:35:33,112 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:35:33,113 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:35:33,114 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:35:33,119 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:35:33,169 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 18:35:33,271 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:35:34,274 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:35:35,170 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:35:37,103 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:35:37,104 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:35:37,104 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:35:37,109 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:35:37,276 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:35:37,568 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:35:37,569 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:35:37,569 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:35:38,279 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:35:40,791 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:35:41,015 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:35:41,016 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:35:41,016 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:35:41,020 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:35:41,282 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:35:42,285 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:35:44,509 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:35:44,510 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:35:44,510 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:35:44,515 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:35:45,289 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:35:46,291 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:35:46,516 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:35:48,625 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:35:48,626 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:35:48,626 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:35:48,631 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:35:49,293 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:35:50,296 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:35:51,631 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:35:52,569 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:35:52,569 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:35:52,569 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:35:56,815 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:35:56,902 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:35:56,902 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:35:56,903 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:35:56,907 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:35:57,304 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:35:58,307 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:36:00,846 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:36:00,847 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:36:00,847 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:36:00,851 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:36:01,310 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:36:01,852 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:36:02,312 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:36:03,171 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 18:36:04,657 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:36:04,657 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:36:04,658 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:36:04,662 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:36:05,316 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:36:06,319 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:36:07,569 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:36:07,569 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:36:07,569 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:36:07,569 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:36:08,218 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:36:08,218 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:36:08,219 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:36:08,223 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:36:08,320 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:36:10,324 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:36:13,262 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:36:14,307 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:36:14,308 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:36:14,308 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:36:14,313 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:36:14,328 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:36:16,332 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:36:18,314 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:36:18,643 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:36:18,644 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:36:18,644 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:36:18,649 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:36:19,335 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:36:20,338 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:36:22,423 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:36:22,424 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:36:22,424 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:36:22,429 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:36:22,569 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:36:22,569 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:36:22,570 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:36:23,342 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:36:23,789 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:36:24,344 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:36:26,354 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:36:26,355 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:36:26,355 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:36:26,360 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:36:27,347 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:36:28,350 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:36:29,398 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:36:30,457 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:36:30,457 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:36:30,458 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:36:30,463 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:36:31,353 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:36:32,355 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:36:33,177 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 18:36:34,943 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:36:34,944 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:36:34,945 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:36:34,945 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:36:34,950 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:36:35,359 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:36:36,359 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:36:37,569 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:36:37,569 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:36:37,569 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:36:40,172 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:36:40,172 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:36:40,173 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:36:40,173 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:36:40,178 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:36:40,365 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:36:42,370 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:36:43,800 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:36:43,801 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:36:43,801 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:36:43,806 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:36:44,371 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:36:45,807 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:36:46,376 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:36:47,549 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:36:47,550 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:36:47,550 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:36:47,555 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:36:48,378 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:36:50,381 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:36:51,556 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:36:51,810 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:36:51,811 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:36:51,811 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:36:51,816 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:36:52,382 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:36:53,228 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:36:53,228 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:36:53,228 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:36:54,387 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:36:56,255 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:36:56,256 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:36:56,256 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:36:56,261 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:36:56,388 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:36:57,298 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:36:58,392 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:37:00,845 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:37:00,846 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:37:00,846 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:37:00,851 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:37:01,395 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:37:02,398 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:37:02,852 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:37:03,176 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 18:37:04,865 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:37:04,866 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:37:04,866 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:37:04,872 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:37:05,400 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:37:06,403 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:37:07,872 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:37:08,228 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:37:08,228 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:37:08,228 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:37:08,958 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:37:08,958 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:37:08,959 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:37:08,964 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:37:09,406 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:37:10,415 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:37:13,002 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:37:13,394 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:37:13,395 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:37:13,395 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:37:13,400 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:37:13,417 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:37:14,420 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:37:18,401 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:37:19,420 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:37:19,420 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:37:19,421 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:37:19,429 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:37:20,426 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:37:20,426 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:37:23,228 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:37:23,228 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:37:23,228 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:37:23,434 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:37:23,491 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:37:23,491 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:37:23,492 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:37:23,496 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:37:24,430 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:37:26,434 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:37:27,621 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:37:27,622 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:37:27,622 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:37:27,627 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:37:28,435 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:37:28,628 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:37:30,438 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:37:33,181 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 18:37:34,182 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:37:34,867 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:37:34,868 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:37:34,868 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:37:34,873 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:37:35,442 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:37:36,445 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:37:38,228 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:37:38,228 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:37:38,229 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:37:39,347 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:37:39,348 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:37:39,348 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:37:39,348 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:37:39,353 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:37:39,448 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:37:40,451 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:37:43,144 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:37:43,145 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:37:43,145 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:37:43,150 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:37:43,454 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:37:44,457 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:37:45,150 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:37:47,510 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:37:47,511 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:37:47,511 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:37:47,516 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:37:48,461 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:37:50,465 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:37:50,516 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:37:52,014 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:37:52,015 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:37:52,015 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:37:52,020 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:37:52,466 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:37:53,228 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:37:53,228 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:37:53,228 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:37:54,471 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:37:56,100 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:37:56,101 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:37:56,101 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:37:56,101 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:37:56,106 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:37:56,473 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:37:58,477 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:37:59,844 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:37:59,845 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:37:59,846 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:37:59,850 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:38:00,478 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:38:01,851 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:38:02,482 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:38:03,180 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 18:38:04,770 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:38:04,771 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:38:04,771 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:38:04,777 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:38:05,485 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:38:06,488 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:38:07,777 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:38:08,179 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:38:08,180 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:38:08,180 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:38:08,185 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:38:08,228 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:38:08,228 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:38:08,229 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:38:08,489 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:38:10,494 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:38:11,655 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:38:11,655 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:38:11,656 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:38:11,661 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:38:12,495 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:38:13,698 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:38:14,499 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:38:16,050 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:38:16,050 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:38:16,051 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:38:16,056 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:38:16,500 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:38:18,505 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:38:19,057 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:38:19,134 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:38:19,135 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:38:19,135 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:38:19,140 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:38:19,506 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:38:20,508 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:38:22,883 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:38:22,884 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:38:22,884 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:38:22,890 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:38:23,228 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:38:23,229 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:38:23,229 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:38:23,513 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:38:24,462 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:38:24,516 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:38:26,990 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:38:26,991 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:38:26,991 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:38:26,996 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:38:27,518 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:38:28,521 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:38:30,034 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:38:30,465 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:38:30,466 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:38:30,466 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:38:30,471 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:38:30,522 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:38:32,526 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:38:33,186 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 18:38:33,949 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:38:33,950 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:38:33,951 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:38:33,955 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:38:34,527 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:38:35,956 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:38:36,531 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:38:38,228 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:38:38,229 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:38:38,230 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:38:41,152 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:38:41,153 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:38:41,153 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:38:41,153 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:38:41,158 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:38:41,537 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:38:42,540 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:38:44,891 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:38:44,892 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:38:44,892 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:38:44,897 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:38:45,543 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:38:46,546 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:38:46,898 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:38:49,847 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:38:49,848 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:38:49,848 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:38:49,853 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:38:50,549 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:38:52,553 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:38:52,854 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:38:53,228 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:38:53,229 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:38:53,229 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:38:53,381 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:38:53,514 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:38:53,514 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:38:53,519 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:38:53,554 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:38:54,555 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:38:57,903 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:38:57,904 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:38:57,904 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:38:57,904 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:38:57,909 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:38:58,560 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:39:00,565 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:39:02,910 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:39:03,185 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 18:39:07,138 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:39:07,139 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:39:07,139 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:39:07,144 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:39:07,573 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:39:08,144 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:39:08,228 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:39:08,229 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:39:08,229 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:39:08,576 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:39:10,294 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:39:10,294 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:39:10,295 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:39:10,300 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:39:10,578 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:39:12,582 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:39:13,338 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:39:16,056 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:39:16,057 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:39:16,057 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:39:16,062 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:39:16,585 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:39:18,590 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:39:19,063 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:39:21,303 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:39:21,303 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:39:21,304 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:39:21,309 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:39:21,593 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:39:22,596 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:39:23,229 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:39:23,229 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:39:23,229 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:39:24,434 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:39:28,365 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:39:28,366 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:39:28,366 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:39:28,371 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:39:28,603 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:39:30,406 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:39:30,606 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:39:31,328 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:39:31,329 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:39:31,329 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:39:31,334 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:39:31,608 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:39:32,611 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:39:33,187 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 18:39:34,683 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:39:34,687 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:39:34,687 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:39:34,692 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:39:35,613 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:39:35,693 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:39:36,616 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:39:38,092 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:39:38,093 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:39:38,093 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:39:38,098 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:39:38,229 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:39:38,229 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:39:38,230 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:39:38,618 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:39:40,622 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:39:41,513 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:39:41,636 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:39:41,636 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:39:41,637 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:39:41,642 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:39:42,623 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:39:44,627 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:39:46,097 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:39:46,098 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:39:46,098 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:39:46,103 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:39:46,630 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:39:47,104 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:39:48,634 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:39:52,104 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:39:53,229 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:39:53,229 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:39:53,229 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:39:54,294 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:39:54,295 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:39:54,295 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:39:54,300 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:39:54,638 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:39:56,641 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:39:57,338 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:39:58,531 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:39:58,531 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:39:58,531 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:39:58,543 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:39:58,643 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:40:00,646 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:40:02,475 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:40:02,476 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:40:02,477 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:40:02,477 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:40:02,482 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:40:02,647 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:40:03,189 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 18:40:04,651 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:40:06,610 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:40:06,611 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:40:06,611 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:40:06,616 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:40:06,652 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:40:07,616 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:40:08,229 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:40:08,230 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:40:08,231 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:40:08,655 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:40:10,224 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:40:10,225 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:40:10,225 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:40:10,230 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:40:10,656 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:40:12,660 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:40:13,266 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:40:15,416 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:40:15,417 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:40:15,417 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:40:15,422 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:40:15,661 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:40:16,664 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:40:18,423 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:40:18,806 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:40:18,806 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:40:18,806 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:40:18,812 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:40:19,666 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:40:20,669 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:40:22,548 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:40:22,549 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:40:22,549 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:40:22,554 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:40:22,670 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:40:23,229 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:40:23,229 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:40:23,229 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:40:23,471 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:40:24,674 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:40:27,495 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:40:27,495 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:40:27,496 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:40:27,500 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:40:27,677 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:40:28,538 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:40:28,679 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:40:33,191 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 18:40:34,193 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:40:34,503 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:40:34,503 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:40:34,504 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:40:34,509 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:40:34,683 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:40:36,687 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:40:37,871 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:40:37,872 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:40:37,872 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:40:37,877 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:40:38,229 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:40:38,229 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:40:38,229 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:40:38,688 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:40:39,533 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:40:40,691 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:40:41,845 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:40:41,846 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:40:41,846 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:40:41,851 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:40:42,693 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:40:44,696 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:40:44,889 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:40:48,081 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:40:48,082 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:40:48,082 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:40:48,087 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:40:48,700 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:40:50,088 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:40:50,703 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:40:52,381 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:40:52,382 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:40:52,382 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:40:52,387 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:40:52,705 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:40:53,229 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:40:53,229 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:40:53,230 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:40:54,708 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:40:55,466 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:40:56,889 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:40:56,890 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:40:56,890 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:40:56,895 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:40:57,710 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:40:58,713 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:41:00,540 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:41:00,541 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:41:00,542 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:41:00,542 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:41:00,547 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:41:00,714 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:41:02,718 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:41:03,194 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 18:41:06,195 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:41:08,229 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:41:08,229 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:41:08,230 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:41:08,985 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:41:08,985 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:41:08,986 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:41:08,991 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:41:09,722 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:41:10,725 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:41:12,026 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:41:13,260 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:41:13,261 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:41:13,261 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:41:13,266 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:41:13,727 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:41:14,730 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:41:16,794 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:41:16,794 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:41:16,795 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:41:16,800 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:41:17,732 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:41:17,801 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:41:18,734 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:41:20,772 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:41:20,773 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:41:20,773 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:41:20,778 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:41:21,736 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:41:22,739 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:41:23,229 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:41:23,229 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:41:23,230 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:41:23,449 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:41:24,678 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:41:24,678 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:41:24,679 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:41:24,683 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:41:24,741 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:41:26,745 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:41:28,062 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:41:28,063 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:41:28,063 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:41:28,069 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:41:28,746 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:41:29,069 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:41:30,749 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:41:31,589 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:41:31,589 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:41:31,589 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:41:31,595 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:41:31,750 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:41:32,753 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:41:33,196 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 18:41:34,197 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:41:36,180 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:41:36,181 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:41:36,181 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:41:36,186 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:41:36,755 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:41:38,229 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:41:38,230 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:41:38,230 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:41:38,759 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:41:39,520 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:41:40,797 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:41:40,798 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:41:40,798 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:41:40,803 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:41:41,761 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:41:42,764 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:41:44,687 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:41:44,688 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:41:44,688 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:41:44,689 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:41:44,694 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:41:44,765 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:41:46,770 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:41:49,205 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:41:49,206 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:41:49,206 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:41:49,211 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:41:49,771 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:41:50,212 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:41:50,774 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:41:52,449 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:41:52,454 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:41:52,454 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:41:52,459 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:41:52,775 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:41:53,229 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:41:53,230 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:41:53,230 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:41:54,779 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:41:55,447 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:41:58,088 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:41:58,089 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:41:58,089 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:41:58,094 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:41:58,782 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:42:00,785 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:42:01,130 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:42:03,198 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 18:42:05,339 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:42:05,340 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:42:05,341 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:42:05,346 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:42:05,788 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:42:06,347 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:42:06,791 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:42:08,230 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:42:08,230 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:42:08,230 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:42:09,040 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:42:09,041 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:42:09,041 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:42:09,046 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:42:09,793 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:42:10,796 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:42:12,082 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:42:13,360 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:42:13,361 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:42:13,361 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:42:13,366 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:42:13,797 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:42:14,800 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:42:17,366 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:42:18,439 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:42:18,440 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:42:18,440 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:42:18,445 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:42:18,803 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:42:20,806 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:42:22,446 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:42:23,230 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:42:23,230 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:42:23,230 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:42:24,279 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:42:24,279 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:42:24,280 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:42:24,284 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:42:24,810 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:42:26,813 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:42:28,322 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:42:28,603 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:42:28,603 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:42:28,604 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:42:28,609 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:42:28,815 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:42:30,818 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:42:33,200 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 18:42:33,651 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:42:33,652 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:42:33,652 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:42:33,652 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:42:33,657 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:42:33,820 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:42:34,823 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:42:38,074 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:42:38,074 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:42:38,075 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:42:38,079 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:42:38,230 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:42:38,230 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:42:38,230 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:42:38,826 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:42:39,459 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:42:40,829 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:42:43,528 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:42:43,528 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:42:43,528 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:42:43,533 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:42:43,831 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:42:44,570 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:42:44,834 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:42:49,394 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:42:49,394 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:42:49,395 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:42:49,400 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:42:49,838 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:42:50,401 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:42:50,841 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:42:53,168 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:42:53,169 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:42:53,170 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:42:53,175 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:42:53,230 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:42:53,230 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:42:53,230 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:42:53,843 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:42:54,846 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:42:55,498 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:42:56,731 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:42:56,731 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:42:56,731 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:42:56,736 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:42:56,847 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:42:58,850 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:43:00,487 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:43:00,487 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:43:00,487 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:43:00,492 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:43:00,852 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:43:01,493 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:43:02,855 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:43:03,206 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 18:43:04,275 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:43:04,276 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:43:04,276 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:43:04,281 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:43:04,856 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:43:06,859 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:43:07,282 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:43:07,735 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:43:07,736 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:43:07,736 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:43:07,741 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:43:07,860 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:43:08,230 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:43:08,230 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:43:08,230 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:43:08,863 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:43:11,636 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:43:11,637 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:43:11,637 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:43:11,642 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:43:11,866 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:43:12,678 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:43:12,869 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:43:17,077 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:43:17,078 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:43:17,078 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:43:17,083 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:43:17,872 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:43:18,084 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:43:18,875 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:43:20,646 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:43:20,646 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:43:20,647 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:43:20,651 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:43:20,876 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:43:22,879 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:43:23,230 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:43:23,230 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:43:23,230 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:43:23,457 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:43:24,576 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:43:24,576 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:43:24,576 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:43:24,581 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:43:24,882 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:43:26,885 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:43:28,619 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:43:28,867 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:43:28,868 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:43:28,868 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:43:28,873 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:43:28,887 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:43:30,890 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:43:32,238 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:43:32,239 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:43:32,239 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:43:32,245 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:43:32,891 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:43:33,205 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 18:43:34,206 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:43:34,895 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:43:38,230 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:43:38,230 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:43:38,230 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:43:39,391 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:43:39,392 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:43:39,392 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:43:39,392 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:43:39,397 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:43:39,898 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:43:40,900 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:43:43,778 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:43:43,778 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:43:43,779 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:43:43,784 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:43:43,903 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:43:44,784 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:43:44,905 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:43:49,784 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:43:52,099 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:43:52,100 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:43:52,101 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:43:52,106 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:43:52,912 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:43:53,230 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:43:53,230 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:43:53,230 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:43:54,915 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:43:55,416 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:43:56,506 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:43:56,507 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:43:56,507 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:43:56,512 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:43:56,917 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:43:58,921 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:44:00,551 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:44:00,980 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:44:00,981 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:44:00,981 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:44:00,986 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:44:01,923 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:44:02,925 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:44:03,210 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 18:44:04,477 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:44:04,478 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:44:04,478 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:44:04,483 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:44:04,927 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:44:06,484 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:44:06,930 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:44:08,230 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:44:08,230 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:44:08,231 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:44:08,601 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:44:08,601 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:44:08,602 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:44:08,607 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:44:08,932 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:44:10,935 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:44:11,643 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:44:13,793 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:44:13,794 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:44:13,794 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:44:13,799 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:44:13,937 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:44:14,940 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:44:16,800 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:44:17,547 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:44:17,548 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:44:17,548 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:44:17,553 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:44:17,942 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:44:18,945 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:44:21,974 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:44:21,975 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:44:21,975 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:44:21,975 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:44:21,980 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:44:22,950 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:44:22,950 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:44:23,230 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:44:23,230 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:44:23,231 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:44:25,307 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:44:25,308 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:44:25,308 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:44:25,314 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:44:25,953 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:44:26,956 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:44:27,351 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:44:32,351 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:44:33,209 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 18:44:33,840 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:44:33,841 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:44:33,841 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:44:33,846 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:44:33,960 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:44:34,963 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:44:37,847 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:44:38,037 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:44:38,040 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:44:38,040 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:44:38,045 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:44:38,230 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:44:38,231 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:44:38,232 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:44:38,966 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:44:40,969 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:44:41,501 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:44:41,502 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:44:41,502 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:44:41,507 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:44:41,970 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:44:42,973 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:44:43,543 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:44:45,885 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:44:45,886 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:44:45,886 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:44:45,892 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:44:45,976 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:44:46,979 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:44:48,892 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:44:50,718 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:44:50,719 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:44:50,720 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:44:50,725 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:44:50,982 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:44:52,985 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:44:53,230 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:44:53,231 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:44:53,231 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:44:54,448 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:44:54,639 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:44:54,639 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:44:54,640 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:44:54,645 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:44:54,987 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:44:56,990 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:44:58,099 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:44:58,100 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:44:58,100 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:44:58,105 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:44:58,992 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:45:00,105 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:45:00,995 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:45:01,607 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:45:01,608 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:45:01,608 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:45:01,613 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:45:01,996 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:45:02,998 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:45:03,215 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 18:45:05,216 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:45:06,327 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:45:06,328 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:45:06,329 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:45:06,334 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:45:07,001 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:45:08,230 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:45:08,231 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:45:08,231 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:45:09,004 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:45:09,676 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:45:09,677 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:45:09,677 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:45:09,682 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:45:10,005 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:45:10,683 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:45:11,008 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:45:13,195 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:45:13,196 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:45:13,197 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:45:13,202 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:45:14,009 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:45:15,012 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:45:16,203 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:45:20,450 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:45:20,451 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:45:20,451 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:45:20,457 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:45:21,016 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:45:21,457 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:45:23,019 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:45:23,231 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:45:23,231 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:45:23,231 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:45:24,261 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:45:24,262 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:45:24,262 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:45:24,267 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:45:25,022 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:45:27,025 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:45:27,303 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:45:28,528 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:45:28,529 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:45:28,530 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:45:28,535 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:45:29,026 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:45:31,030 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:45:32,536 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:45:33,214 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 18:45:34,565 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:45:34,566 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:45:34,566 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:45:34,571 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:45:35,033 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:45:37,036 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:45:37,572 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:45:38,231 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:45:38,231 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:45:38,231 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:45:38,232 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:45:38,477 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:45:38,477 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:45:38,482 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:45:39,037 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:45:41,041 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:45:43,482 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:45:46,666 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:45:46,667 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:45:46,667 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:45:46,672 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:45:47,046 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:45:48,673 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:45:49,049 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:45:50,526 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:45:50,527 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:45:50,527 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:45:50,532 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:45:51,051 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:45:53,054 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:45:53,231 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:45:53,231 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:45:53,231 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:45:54,492 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:45:55,250 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:45:55,251 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:45:55,251 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:45:55,256 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:45:56,056 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:45:57,059 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:45:59,243 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:45:59,244 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:45:59,244 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:45:59,249 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:46:00,061 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:46:00,250 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:46:01,064 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:46:02,790 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:46:02,791 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:46:02,791 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:46:02,796 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:46:03,066 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:46:03,219 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 18:46:05,069 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:46:06,221 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:46:07,955 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:46:07,956 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:46:07,956 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:46:07,961 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:46:08,071 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:46:08,231 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:46:08,231 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:46:08,231 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:46:09,074 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:46:11,579 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:46:11,579 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:46:11,580 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:46:11,580 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:46:11,585 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:46:12,076 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:46:13,078 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:46:14,976 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:46:14,977 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:46:14,977 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:46:14,982 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:46:15,080 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:46:16,983 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:46:17,084 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:46:18,907 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:46:18,908 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:46:18,908 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:46:18,913 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:46:19,085 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:46:21,089 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:46:22,914 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:46:23,231 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:46:23,231 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:46:23,231 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:46:23,293 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:46:23,519 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:46:23,519 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:46:23,524 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:46:24,092 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:46:25,095 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:46:28,332 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:46:28,333 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:46:28,333 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:46:28,333 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:46:28,339 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:46:29,098 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:46:31,101 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:46:32,217 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:46:32,218 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:46:32,218 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:46:32,224 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:46:33,103 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:46:33,218 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 18:46:34,220 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:46:35,104 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:46:36,987 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:46:36,988 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:46:36,988 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:46:36,993 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:46:37,105 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:46:38,231 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:46:38,231 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:46:38,231 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:46:39,109 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:46:39,538 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:46:41,509 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:46:41,510 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:46:41,510 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:46:41,515 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:46:42,111 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:46:43,114 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:46:44,551 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:46:49,551 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:46:49,630 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:46:49,630 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:46:49,631 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:46:49,636 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:46:50,120 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:46:51,122 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:46:53,231 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:46:53,231 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:46:53,231 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:46:53,926 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:46:53,927 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:46:53,927 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:46:53,932 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:46:54,124 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:46:54,933 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:46:55,127 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:46:57,594 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:46:57,595 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:46:57,595 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:46:57,601 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:46:58,130 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:46:59,132 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:47:00,602 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:47:03,221 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 18:47:03,345 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:47:03,346 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:47:03,346 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:47:03,351 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:47:04,135 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:47:05,138 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:47:06,351 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:47:06,760 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:47:06,761 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:47:06,761 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:47:06,766 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:47:07,139 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:47:08,231 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:47:08,231 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:47:08,232 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:47:09,143 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:47:11,515 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:47:15,061 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:47:15,061 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:47:15,062 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:47:15,067 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:47:15,146 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:47:17,068 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:47:17,150 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:47:19,338 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:47:19,338 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:47:19,339 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:47:19,344 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:47:20,152 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:47:21,154 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:47:22,345 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:47:22,767 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:47:22,768 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:47:22,768 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:47:22,773 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:47:23,156 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:47:23,231 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:47:23,231 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:47:23,231 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:47:25,160 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:47:26,580 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:47:26,580 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:47:26,581 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:47:26,586 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:47:27,162 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:47:27,623 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:47:29,165 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:47:29,948 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:47:29,949 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:47:29,949 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:47:29,955 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:47:30,166 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:47:31,169 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:47:32,955 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:47:33,223 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 18:47:35,359 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:47:35,360 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:47:35,360 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:47:35,365 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:47:36,172 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:47:37,174 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:47:38,231 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:47:38,231 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:47:38,232 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:47:38,502 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:47:41,031 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:47:41,032 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:47:41,033 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:47:41,037 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:47:41,177 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:47:43,180 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:47:44,038 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:47:44,808 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:47:44,809 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:47:44,809 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:47:44,814 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:47:45,182 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:47:47,185 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:47:49,331 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:47:49,332 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:47:49,332 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:47:49,333 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:47:49,338 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:47:50,187 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:47:51,190 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:47:53,231 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:47:53,232 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:47:53,232 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:47:54,349 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:47:54,350 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:47:54,351 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:47:54,351 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:47:54,356 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:47:55,192 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:47:57,196 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:47:58,283 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:47:58,284 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:47:58,284 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:47:58,289 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:47:59,198 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:48:00,290 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:48:01,201 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:48:03,079 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:48:03,080 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:48:03,080 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:48:03,085 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:48:03,203 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:48:03,225 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 18:48:05,206 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:48:06,226 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:48:07,885 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:48:07,886 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:48:07,886 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:48:07,892 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:48:08,208 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:48:08,232 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:48:08,232 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:48:08,232 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:48:11,212 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:48:11,458 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:48:13,337 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:48:13,338 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:48:13,338 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:48:13,343 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:48:14,214 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:48:15,216 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:48:17,344 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:48:17,361 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:48:17,362 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:48:17,362 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:48:17,367 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:48:18,218 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:48:19,221 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:48:20,862 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:48:20,863 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:48:20,863 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:48:20,868 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:48:21,222 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:48:22,869 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:48:23,224 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:48:23,232 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:48:23,232 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:48:23,232 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:48:24,389 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:48:24,390 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:48:24,390 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:48:24,395 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:48:25,226 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:48:27,230 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:48:28,161 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:48:28,162 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:48:28,162 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:48:28,162 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:48:28,167 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:48:28,230 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:48:31,234 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:48:31,789 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:48:31,790 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:48:31,791 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:48:31,796 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:48:32,235 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:48:33,227 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 18:48:33,229 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:48:35,239 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:48:36,468 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:48:36,469 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:48:36,469 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:48:36,474 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:48:37,240 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:48:38,232 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:48:38,232 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:48:38,232 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:48:38,520 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:48:39,244 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:48:40,440 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:48:40,441 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:48:40,441 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:48:40,446 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:48:41,246 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:48:43,249 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:48:43,987 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:48:43,988 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:48:43,988 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:48:43,989 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:48:43,993 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:48:44,250 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:48:47,255 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:48:48,122 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:48:48,123 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:48:48,123 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:48:48,128 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:48:48,256 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:48:49,129 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:48:51,260 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:48:52,769 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:48:52,770 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:48:52,770 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:48:52,775 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:48:53,232 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:48:53,232 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:48:53,232 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:48:53,261 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:48:54,531 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:48:55,264 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:48:56,760 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:48:56,761 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:48:56,761 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:48:56,766 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:48:57,266 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:48:59,269 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:48:59,767 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:49:00,463 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:49:00,464 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:49:00,464 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:49:00,469 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:49:01,271 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:49:03,229 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 18:49:03,274 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:49:05,003 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:49:05,004 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:49:05,004 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:49:05,004 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:49:05,009 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:49:05,275 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:49:07,278 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:49:08,232 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:49:08,232 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:49:08,232 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:49:10,537 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:49:13,362 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:49:13,363 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:49:13,363 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:49:13,368 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:49:14,283 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:49:15,286 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:49:16,369 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:49:16,614 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:49:16,615 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:49:16,615 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:49:16,620 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:49:17,287 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:49:19,290 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:49:20,169 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:49:20,170 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:49:20,170 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:49:20,175 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:49:20,291 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:49:22,176 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:49:23,232 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:49:23,232 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:49:23,232 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:49:23,295 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:49:24,327 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:49:24,328 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:49:24,329 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:49:24,334 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:49:25,297 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:49:27,301 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:49:27,334 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:49:28,302 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:49:28,302 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:49:28,302 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:49:28,307 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:49:29,302 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:49:31,306 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:49:32,618 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:49:32,619 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:49:32,619 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:49:32,619 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:49:32,625 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:49:33,235 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 18:49:33,307 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:49:35,310 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:49:37,630 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:49:37,631 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:49:37,631 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:49:37,631 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:49:37,636 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:49:38,232 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:49:38,232 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:49:38,233 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:49:38,312 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:49:39,315 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:49:42,127 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:49:42,127 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:49:42,128 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:49:42,133 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:49:42,317 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:49:43,170 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:49:45,321 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:49:48,170 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:49:48,212 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:49:48,212 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:49:48,213 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:49:48,218 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:49:48,324 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:49:51,328 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:49:51,685 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:49:51,686 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:49:51,686 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:49:51,691 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:49:52,329 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:49:53,232 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:49:53,232 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:49:53,233 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:49:53,331 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:49:53,542 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:49:55,914 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:49:55,915 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:49:55,915 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:49:55,920 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:49:56,333 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:49:58,921 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:49:59,338 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:49:59,518 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:49:59,518 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:49:59,519 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:49:59,523 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:50:00,339 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:50:01,341 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:50:03,234 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 18:50:03,309 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:50:03,310 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:50:03,310 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:50:03,315 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:50:03,343 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:50:04,315 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:50:05,344 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:50:07,546 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:50:07,547 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:50:07,547 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:50:07,552 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:50:08,232 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:50:08,232 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:50:08,233 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:50:08,346 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:50:09,349 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:50:09,514 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:50:11,088 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:50:11,089 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:50:11,089 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:50:11,094 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:50:11,350 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:50:13,354 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:50:14,594 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:50:14,595 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:50:14,595 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:50:14,595 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:50:14,600 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:50:15,355 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:50:17,359 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:50:19,601 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:50:20,229 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:50:20,230 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:50:20,230 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:50:20,235 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:50:20,361 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:50:23,232 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:50:23,233 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:50:23,233 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:50:23,365 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:50:24,322 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:50:24,323 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:50:24,323 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:50:24,328 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:50:24,365 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:50:25,329 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:50:27,370 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:50:28,501 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:50:28,502 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:50:28,502 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:50:28,507 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:50:29,371 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:50:30,507 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:50:31,374 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:50:32,073 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:50:32,074 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:50:32,074 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:50:32,079 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:50:32,375 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:50:33,240 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 18:50:35,379 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:50:36,241 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:50:37,505 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:50:37,506 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:50:37,506 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:50:37,511 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:50:38,233 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:50:38,233 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:50:38,233 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:50:38,381 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:50:39,384 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:50:41,483 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:50:42,528 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:50:42,528 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:50:42,528 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:50:42,533 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:50:43,387 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:50:45,390 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:50:46,384 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:50:46,385 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:50:46,385 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:50:46,390 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:50:46,392 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:50:47,391 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:50:49,396 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:50:52,391 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:50:53,233 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:50:53,233 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:50:53,233 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:50:54,707 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:50:54,708 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:50:54,708 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:50:54,714 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:50:55,400 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:50:57,404 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:50:57,715 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:50:58,453 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:50:58,454 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:50:58,454 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:50:58,459 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:50:59,405 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:51:01,409 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:51:01,998 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:51:01,999 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:51:01,999 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:51:02,004 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:51:02,409 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:51:03,004 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:51:03,239 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 18:51:05,413 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:51:08,233 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:51:08,233 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:51:08,233 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:51:08,277 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:51:08,538 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:51:08,538 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:51:08,538 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:51:08,544 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:51:09,416 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:51:11,419 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:51:13,247 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:51:13,248 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:51:13,248 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:51:13,253 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:51:13,421 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:51:14,254 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:51:15,424 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:51:18,467 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:51:18,468 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:51:18,468 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:51:18,473 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:51:19,427 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:51:19,474 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:51:21,430 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:51:22,721 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:51:22,722 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:51:22,722 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:51:22,727 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:51:23,233 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:51:23,233 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:51:23,233 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:51:23,431 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:51:24,527 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:51:25,435 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:51:26,278 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:51:26,279 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:51:26,279 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:51:26,284 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:51:26,436 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:51:29,440 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:51:29,791 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:51:29,792 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:51:29,792 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:51:29,792 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:51:29,797 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:51:30,441 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:51:31,444 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:51:33,244 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 18:51:34,419 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:51:34,420 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:51:34,420 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:51:34,425 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:51:34,446 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:51:35,425 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:51:37,450 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:51:38,233 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:51:38,233 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:51:38,233 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:51:39,485 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:51:39,486 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:51:39,486 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:51:39,491 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:51:40,451 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:51:40,492 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:51:41,454 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:51:44,210 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:51:44,211 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:51:44,211 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:51:44,216 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:51:44,456 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:51:46,217 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:51:47,462 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:51:47,495 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:51:47,495 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:51:47,495 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:51:47,500 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:51:48,462 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:51:49,465 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:51:51,501 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:51:51,939 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:51:51,940 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:51:51,940 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:51:51,945 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:51:52,467 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:51:53,233 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:51:53,233 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:51:53,233 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:51:53,470 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:51:56,559 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:51:56,560 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:51:56,560 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:51:56,560 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:51:56,565 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:51:57,473 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:51:59,476 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:52:00,101 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:52:00,102 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:52:00,102 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:52:00,107 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:52:00,477 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:52:02,107 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:52:03,243 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 18:52:03,481 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:52:04,541 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:52:04,542 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:52:04,542 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:52:04,547 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:52:05,483 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:52:07,490 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:52:07,547 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:52:08,233 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:52:08,233 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:52:08,233 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:52:08,987 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:52:08,988 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:52:08,988 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:52:08,993 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:52:09,492 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:52:11,495 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:52:12,265 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:52:12,266 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:52:12,266 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:52:12,271 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:52:12,495 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:52:13,271 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:52:15,500 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:52:15,624 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:52:15,624 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:52:15,624 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:52:15,629 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:52:16,500 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:52:17,503 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:52:18,630 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:52:19,243 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:52:19,244 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:52:19,244 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:52:19,249 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:52:19,504 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:52:20,508 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:52:23,233 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:52:23,233 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:52:23,233 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:52:23,681 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:52:23,682 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:52:23,682 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:52:23,682 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:52:23,687 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:52:24,514 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:52:24,514 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:52:27,793 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:52:27,794 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:52:27,794 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:52:27,799 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:52:28,519 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:52:28,519 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:52:28,799 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:52:31,077 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:52:31,078 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:52:31,078 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:52:31,083 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:52:31,521 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:52:32,524 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:52:33,249 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 18:52:34,250 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:52:34,845 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:52:34,846 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:52:34,846 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:52:34,851 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:52:35,526 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:52:36,528 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:52:37,911 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:52:37,912 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:52:37,912 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:52:37,917 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:52:38,233 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:52:38,233 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:52:38,233 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:52:38,532 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:52:38,532 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:52:39,529 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:52:44,378 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:52:44,378 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:52:44,379 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:52:44,383 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:52:44,536 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:52:45,384 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:52:46,540 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:52:47,703 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:52:47,704 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:52:47,704 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:52:47,709 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:52:48,544 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:52:48,544 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:52:50,710 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:52:51,255 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:52:51,255 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:52:51,256 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:52:51,261 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:52:51,546 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:52:52,549 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:52:53,233 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:52:53,233 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:52:53,234 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:52:54,872 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:52:54,872 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:52:54,872 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:52:54,878 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:52:55,551 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:52:55,878 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:52:56,554 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:52:58,374 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:52:58,374 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:52:58,375 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:52:58,380 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:52:58,555 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:53:00,559 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:53:01,380 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:53:03,248 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 18:53:03,668 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:53:03,669 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:53:03,669 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:53:03,674 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:53:04,563 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:53:04,564 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:53:06,675 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:53:07,349 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:53:07,349 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:53:07,350 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:53:07,355 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:53:07,565 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:53:08,233 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:53:08,233 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:53:08,234 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:53:08,568 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:53:10,714 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:53:10,714 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:53:10,715 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:53:10,720 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:53:11,570 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:53:11,758 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:53:12,573 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:53:14,901 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:53:14,902 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:53:14,902 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:53:14,907 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:53:15,575 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:53:16,577 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:53:16,908 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:53:18,589 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:53:18,590 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:53:18,590 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:53:18,595 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:53:19,581 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:53:20,583 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:53:22,068 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:53:22,069 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:53:22,070 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:53:22,070 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:53:22,075 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:53:22,586 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:53:22,587 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:53:23,233 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:53:23,233 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:53:23,233 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:53:27,429 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:53:32,429 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:53:33,254 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 18:53:34,236 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:53:34,237 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:53:34,237 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:53:34,242 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:53:34,595 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:53:36,598 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:53:37,945 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:53:37,946 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:53:37,946 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:53:37,947 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:53:37,952 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:53:38,233 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:53:38,234 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:53:38,234 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:53:38,602 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:53:38,602 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:53:41,637 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:53:41,638 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:53:41,638 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:53:41,643 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:53:42,606 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:53:42,607 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:53:43,678 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:53:45,434 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:53:45,434 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:53:45,435 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:53:45,440 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:53:45,609 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:53:46,613 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:53:48,954 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:53:48,955 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:53:48,955 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:53:48,955 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:53:48,960 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:53:49,615 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:53:50,617 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:53:53,233 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:53:53,234 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:53:53,234 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:53:53,414 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:53:53,481 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:53:53,481 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:53:53,486 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:53:53,619 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:53:54,487 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:53:54,622 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:53:56,973 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:53:56,974 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:53:56,974 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:53:56,979 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:53:57,625 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:53:58,627 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:53:59,980 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:54:00,727 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:54:00,728 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:54:00,728 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:54:00,733 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:54:01,629 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:54:02,632 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:54:03,252 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 18:54:05,134 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:54:05,135 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:54:05,135 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:54:05,135 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:54:05,144 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:54:05,634 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:54:06,637 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:54:08,234 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:54:08,234 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:54:08,234 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:54:09,231 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:54:09,232 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:54:09,232 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:54:09,238 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:54:09,639 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:54:10,239 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:54:10,641 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:54:13,666 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:54:13,666 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:54:13,667 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:54:13,672 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:54:14,646 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:54:14,646 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:54:15,672 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:54:20,673 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:54:22,064 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:54:22,065 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:54:22,065 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:54:22,070 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:54:22,653 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:54:22,653 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:54:23,234 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:54:23,234 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:54:23,234 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:54:26,351 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:54:26,352 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:54:26,352 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:54:26,352 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:54:26,358 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:54:26,657 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:54:28,660 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:54:31,316 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:54:31,317 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:54:31,317 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:54:31,322 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:54:31,662 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:54:32,322 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:54:32,664 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:54:33,254 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 18:54:35,040 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:54:35,041 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:54:35,042 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:54:35,047 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:54:35,667 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:54:36,669 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:54:38,048 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:54:38,234 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:54:38,234 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:54:38,234 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:54:41,673 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:54:41,674 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:54:41,674 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:54:41,679 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:54:42,675 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:54:42,675 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:54:43,718 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:54:45,499 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:54:45,499 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:54:45,500 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:54:45,505 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:54:45,677 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:54:46,681 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:54:49,505 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:54:51,905 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:54:51,906 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:54:51,906 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:54:51,911 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:54:52,687 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:54:52,687 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:54:53,234 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:54:53,234 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:54:53,234 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:54:55,481 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:54:55,572 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:54:55,573 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:54:55,573 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:54:55,579 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:54:55,689 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:54:56,691 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:55:00,614 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:55:03,256 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 18:55:03,979 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:55:03,979 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:55:03,980 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:55:03,985 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:55:04,699 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:55:04,699 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:55:05,985 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:55:08,234 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:55:08,234 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:55:08,234 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:55:08,349 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:55:08,569 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:55:08,569 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:55:08,574 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:55:08,702 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:55:10,705 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:55:11,575 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:55:12,151 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:55:12,152 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:55:12,152 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:55:12,159 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:55:12,708 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:55:12,709 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:55:16,402 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:55:16,403 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:55:16,403 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:55:16,408 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:55:16,711 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:55:17,408 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:55:18,714 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:55:21,201 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:55:21,201 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:55:21,202 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:55:21,207 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:55:21,716 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:55:22,719 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:55:23,207 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:55:23,234 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:55:23,234 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:55:23,234 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:55:25,508 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:55:25,509 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:55:25,509 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:55:25,514 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:55:25,722 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:55:26,725 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:55:28,515 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:55:28,790 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:55:28,791 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:55:28,791 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:55:28,796 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:55:29,727 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:55:30,730 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:55:32,998 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:55:32,999 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:55:32,999 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:55:33,004 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:55:33,258 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 18:55:33,732 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:55:34,260 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:55:34,735 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:55:38,234 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:55:38,234 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:55:38,234 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:55:38,362 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:55:38,448 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:55:38,448 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:55:38,453 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:55:38,738 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:55:39,454 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:55:40,741 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:55:44,213 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:55:44,214 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:55:44,214 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:55:44,219 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:55:44,746 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:55:44,746 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:55:45,220 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:55:50,220 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:55:50,827 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:55:50,828 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:55:50,828 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:55:50,833 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:55:51,753 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:55:52,755 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:55:53,234 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:55:53,234 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:55:53,235 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:55:55,453 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:55:56,434 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:55:56,434 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:55:56,435 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:55:56,440 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:55:56,758 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:55:58,762 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:56:00,079 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:56:00,080 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:56:00,080 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:56:00,087 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:56:00,766 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:56:00,766 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:56:01,088 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:56:03,261 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 18:56:03,650 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:56:03,650 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:56:03,651 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:56:03,656 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:56:03,768 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:56:04,770 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:56:06,656 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:56:07,342 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:56:07,343 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:56:07,343 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:56:07,348 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:56:07,772 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:56:08,234 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:56:08,234 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:56:08,235 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:56:08,775 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:56:11,764 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:56:12,746 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:56:12,747 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:56:12,747 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:56:12,752 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:56:12,778 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:56:14,782 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:56:17,558 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:56:17,559 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:56:17,559 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:56:17,559 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:56:17,564 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:56:17,783 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:56:18,786 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:56:22,565 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:56:23,234 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:56:23,235 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:56:23,235 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:56:24,381 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:56:24,381 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:56:24,381 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:56:24,387 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:56:24,791 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:56:26,795 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:56:28,422 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:56:28,994 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:56:28,995 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:56:28,995 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:56:29,001 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:56:29,797 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:56:30,800 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:56:33,263 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 18:56:33,889 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:56:33,890 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:56:33,890 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:56:33,890 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:56:33,895 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:56:34,804 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:56:34,805 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:56:37,433 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:56:37,433 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:56:37,434 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:56:37,439 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:56:37,806 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:56:38,234 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:56:38,234 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:56:38,234 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:56:38,809 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:56:39,649 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:56:43,172 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:56:43,173 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:56:43,173 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:56:43,179 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:56:43,813 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:56:44,815 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:56:45,214 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:56:47,798 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:56:47,799 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:56:47,799 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:56:47,805 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:56:47,819 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:56:48,822 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:56:50,805 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:56:52,113 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:56:52,114 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:56:52,114 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:56:52,119 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:56:52,827 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:56:52,827 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:56:53,234 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:56:53,235 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:56:53,235 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:56:56,205 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:56:56,206 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:56:56,206 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:56:56,206 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:56:56,211 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:56:56,832 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:56:56,832 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:56:59,507 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:56:59,508 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:56:59,509 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:56:59,513 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:56:59,834 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:57:00,837 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:57:01,514 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:57:03,269 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 18:57:05,496 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:57:05,497 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:57:05,497 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:57:05,502 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:57:05,840 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:57:06,843 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:57:07,503 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:57:08,234 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:57:08,235 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:57:08,235 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:57:09,468 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:57:09,469 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:57:09,469 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:57:09,474 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:57:09,845 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:57:10,848 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:57:12,510 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:57:13,888 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:57:13,889 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:57:13,889 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:57:13,894 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:57:14,853 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:57:14,853 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:57:17,423 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:57:17,424 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:57:17,424 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:57:17,430 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:57:17,855 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:57:18,430 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:57:18,858 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:57:22,295 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:57:22,296 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:57:22,297 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:57:22,302 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:57:22,862 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:57:22,863 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:57:23,235 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:57:23,235 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:57:23,235 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:57:23,468 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:57:28,469 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:57:31,028 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:57:31,029 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:57:31,029 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:57:31,034 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:57:31,870 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:57:32,873 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:57:33,268 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 18:57:34,269 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:57:34,441 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:57:34,442 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:57:34,442 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:57:34,447 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:57:34,874 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:57:36,878 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:57:38,000 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:57:38,000 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:57:38,001 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:57:38,006 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:57:38,235 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:57:38,235 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:57:38,235 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:57:38,881 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:57:38,881 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:57:39,568 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:57:43,858 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:57:43,859 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:57:43,859 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:57:43,864 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:57:43,884 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:57:44,887 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:57:44,902 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:57:48,037 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:57:48,038 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:57:48,038 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:57:48,043 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:57:48,893 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:57:48,893 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:57:50,044 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:57:51,555 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:57:51,556 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:57:51,556 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:57:51,561 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:57:51,895 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:57:52,898 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:57:53,235 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:57:53,235 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:57:53,235 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:57:55,504 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:57:59,843 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:57:59,844 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:57:59,844 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:57:59,849 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:57:59,903 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:58:00,849 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:58:00,906 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:58:02,898 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:58:02,899 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:58:02,899 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:58:02,904 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:58:02,907 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:58:03,273 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 18:58:04,910 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:58:06,275 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:58:07,769 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:58:07,770 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:58:07,770 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:58:07,775 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:58:07,912 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:58:08,235 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:58:08,235 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:58:08,235 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:58:08,915 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:58:11,407 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:58:11,408 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:58:11,408 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:58:11,408 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:58:11,413 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:58:11,917 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:58:12,920 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:58:15,498 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:58:15,498 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:58:15,499 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:58:15,504 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:58:15,922 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:58:16,504 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:58:16,924 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:58:19,264 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:58:19,265 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:58:19,265 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:58:19,270 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:58:19,926 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:58:20,929 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:58:22,271 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:58:22,623 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:58:22,624 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:58:22,624 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:58:22,629 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:58:22,931 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:58:23,235 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:58:23,235 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:58:23,235 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:58:24,935 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:58:27,529 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:58:27,826 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:58:27,827 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:58:27,828 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:58:27,832 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:58:27,937 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:58:28,940 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:58:32,833 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:58:32,867 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:58:32,868 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:58:32,868 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:58:32,873 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:58:32,943 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:58:33,272 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 18:58:34,946 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:58:37,149 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:58:37,150 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:58:37,150 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:58:37,154 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:58:37,948 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:58:38,155 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:58:38,235 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:58:38,235 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:58:38,235 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:58:38,950 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:58:41,540 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:58:41,541 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:58:41,541 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:58:41,546 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:58:41,952 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:58:42,955 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:58:43,583 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:58:46,850 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:58:46,851 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:58:46,851 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:58:46,857 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:58:46,960 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:58:48,858 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:58:48,963 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:58:53,235 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:58:53,235 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:58:53,235 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:58:53,461 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:58:53,462 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:58:53,462 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:58:53,467 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:58:53,966 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:58:54,507 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:58:54,969 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:58:57,377 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:58:57,378 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:58:57,378 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:58:57,383 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:58:57,971 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:58:58,974 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:59:00,383 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:59:01,814 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:59:01,815 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:59:01,815 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:59:01,820 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:59:01,976 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:59:02,979 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:59:03,277 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 18:59:06,279 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:59:06,548 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:59:06,549 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:59:06,549 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:59:06,554 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:59:06,981 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:59:08,235 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:59:08,235 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:59:08,235 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:59:08,984 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:59:11,532 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:59:12,778 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:59:12,779 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:59:12,779 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:59:12,784 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:59:12,987 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:59:14,990 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:59:16,112 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:59:16,113 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:59:16,113 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:59:16,119 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:59:16,994 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:59:16,994 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:59:17,119 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:59:19,484 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:59:19,484 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:59:19,484 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:59:19,489 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:59:19,996 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:59:20,998 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:59:22,490 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:59:22,806 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:59:22,807 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:59:22,807 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:59:22,812 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:59:23,000 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:59:23,235 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:59:23,235 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:59:23,235 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:59:25,004 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:59:26,049 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:59:26,049 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:59:26,050 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:59:26,054 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:59:27,008 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:59:27,008 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:59:28,091 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:59:29,496 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:59:29,497 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:59:29,497 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:59:29,502 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:59:30,010 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:59:31,012 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:59:33,277 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 18:59:33,278 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:59:38,235 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:59:38,236 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:59:38,236 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:59:38,524 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:59:38,593 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:59:38,593 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:59:38,593 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:59:38,598 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:59:39,018 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:59:41,021 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:59:43,635 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:59:44,376 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:59:44,376 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:59:44,377 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:59:44,382 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:59:45,025 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:59:45,026 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:59:47,861 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:59:47,862 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:59:47,862 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:59:47,866 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:59:48,029 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:59:48,867 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:59:49,032 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:59:51,218 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:59:51,219 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:59:51,219 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:59:51,224 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:59:52,034 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:59:53,037 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:59:53,235 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 18:59:53,236 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 18:59:53,236 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 18:59:54,436 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 18:59:54,614 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:59:54,614 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:59:54,615 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:59:54,619 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:59:55,038 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:59:57,041 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 18:59:58,083 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 18:59:58,084 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 18:59:58,084 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 18:59:58,089 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 18:59:59,045 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 18:59:59,045 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:00:00,089 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:00:02,165 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:00:02,166 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:00:02,166 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:00:02,171 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:00:03,049 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:00:03,050 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:00:03,282 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 19:00:05,283 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:00:05,948 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:00:05,949 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:00:05,949 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:00:05,954 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:00:06,051 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:00:07,054 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:00:08,235 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:00:08,236 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:00:08,236 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:00:09,522 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:00:09,522 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:00:09,522 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:00:09,528 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:00:10,056 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:00:10,567 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:00:11,059 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:00:14,411 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:00:14,412 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:00:14,412 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:00:14,417 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:00:15,063 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:00:15,064 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:00:16,418 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:00:18,818 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:00:18,819 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:00:18,819 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:00:18,824 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:00:19,066 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:00:21,070 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:00:21,825 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:00:22,292 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:00:22,293 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:00:22,293 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:00:22,298 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:00:23,073 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:00:23,073 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:00:23,236 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:00:23,236 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:00:23,236 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:00:26,027 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:00:26,027 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:00:26,027 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:00:26,032 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:00:26,076 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:00:27,033 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:00:27,079 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:00:30,527 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:00:30,528 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:00:30,528 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:00:30,533 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:00:31,083 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:00:31,084 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:00:32,533 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:00:33,281 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 19:00:35,470 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:00:35,471 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:00:35,471 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:00:35,476 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:00:36,087 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:00:37,089 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:00:38,236 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:00:38,236 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:00:38,236 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:00:38,446 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:00:39,549 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:00:39,549 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:00:39,549 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:00:39,555 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:00:40,091 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:00:41,094 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:00:43,591 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:00:44,027 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:00:44,028 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:00:44,028 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:00:44,033 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:00:44,096 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:00:45,099 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:00:47,699 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:00:47,699 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:00:47,699 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:00:47,704 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:00:48,102 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:00:48,705 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:00:49,104 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:00:51,174 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:00:51,175 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:00:51,175 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:00:51,180 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:00:52,106 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:00:53,109 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:00:53,236 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:00:53,236 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:00:53,236 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:00:54,438 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:00:54,892 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:00:54,893 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:00:54,893 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:00:54,898 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:00:55,110 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:00:57,113 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:00:58,437 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:00:58,438 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:00:58,438 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:00:58,443 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:00:59,117 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:00:59,117 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:00:59,443 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:01:02,914 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:01:02,914 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:01:02,915 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:01:02,919 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:01:03,120 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:01:03,288 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 19:01:05,123 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:01:05,289 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:01:07,761 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:01:07,762 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:01:07,762 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:01:07,768 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:01:08,125 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:01:08,236 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:01:08,236 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:01:08,236 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:01:09,128 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:01:10,522 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:01:12,433 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:01:12,434 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:01:12,434 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:01:12,439 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:01:13,132 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:01:13,132 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:01:16,440 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:01:18,287 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:01:18,287 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:01:18,287 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:01:18,292 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:01:19,138 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:01:19,138 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:01:21,663 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:01:21,664 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:01:21,664 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:01:21,664 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:01:21,669 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:01:22,140 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:01:23,143 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:01:23,236 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:01:23,236 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:01:23,236 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:01:27,059 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:01:27,060 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:01:27,060 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:01:27,060 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:01:27,065 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:01:27,147 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:01:29,150 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:01:30,798 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:01:30,798 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:01:30,799 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:01:30,804 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:01:31,151 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:01:32,804 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:01:33,155 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:01:33,285 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 19:01:35,310 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:01:35,310 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:01:35,311 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:01:35,315 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:01:36,157 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:01:37,160 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:01:38,236 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:01:38,236 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:01:38,236 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:01:38,526 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:01:38,832 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:01:38,832 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:01:38,832 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:01:38,837 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:01:39,161 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:01:41,164 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:01:43,219 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:01:43,220 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:01:43,220 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:01:43,225 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:01:44,167 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:01:44,225 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:01:45,170 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:01:47,150 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:01:47,150 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:01:47,151 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:01:47,156 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:01:47,172 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:01:49,176 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:01:50,156 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:01:51,433 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:01:51,434 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:01:51,434 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:01:51,440 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:01:52,178 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:01:53,180 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:01:53,236 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:01:53,236 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:01:53,237 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:01:55,470 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:01:55,841 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:01:55,842 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:01:55,842 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:01:55,848 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:01:56,182 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:01:57,185 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:01:59,914 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:01:59,914 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:01:59,915 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:01:59,920 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:02:00,188 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:02:00,920 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:02:01,190 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:02:03,288 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 19:02:06,289 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:02:07,289 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:02:07,290 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:02:07,290 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:02:07,295 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:02:08,195 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:02:08,236 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:02:08,236 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:02:08,237 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:02:09,198 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:02:11,528 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:02:14,159 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:02:14,160 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:02:14,160 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:02:14,165 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:02:14,201 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:02:15,204 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:02:17,203 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:02:18,115 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:02:18,116 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:02:18,116 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:02:18,121 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:02:18,206 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:02:19,208 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:02:21,214 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:02:21,215 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:02:21,215 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:02:21,221 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:02:22,210 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:02:22,221 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:02:23,213 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:02:23,236 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:02:23,236 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:02:23,237 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:02:27,182 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:02:27,182 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:02:27,183 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:02:27,188 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:02:27,217 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:02:27,227 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:02:29,220 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:02:32,227 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:02:33,290 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 19:02:33,752 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:02:33,753 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:02:33,753 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:02:33,758 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:02:34,223 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:02:35,226 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:02:37,759 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:02:38,236 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:02:38,237 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:02:38,237 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:02:38,505 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:02:38,505 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:02:38,506 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:02:38,511 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:02:39,230 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:02:39,230 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:02:43,269 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:02:43,270 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:02:43,270 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:02:43,271 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:02:43,276 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:02:44,234 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:02:45,237 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:02:47,811 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:02:47,811 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:02:47,811 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:02:47,817 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:02:48,240 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:02:48,817 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:02:49,243 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:02:50,780 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:02:50,781 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:02:50,781 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:02:50,786 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:02:51,246 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:02:51,246 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:02:53,236 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:02:53,237 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:02:53,237 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:02:54,421 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:02:54,422 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:02:54,422 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:02:54,422 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:02:54,428 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:02:55,251 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:02:55,251 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:02:59,090 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:02:59,091 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:02:59,091 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:02:59,096 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:02:59,254 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:03:00,096 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:03:01,257 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:03:02,836 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:03:02,837 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:03:02,837 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:03:02,842 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:03:03,258 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:03:03,292 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 19:03:05,262 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:03:05,294 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:03:06,412 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:03:06,413 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:03:06,413 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:03:06,418 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:03:07,265 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:03:07,265 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:03:08,236 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:03:08,237 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:03:08,237 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:03:10,448 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:03:11,455 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:03:11,456 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:03:11,456 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:03:11,461 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:03:12,269 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:03:13,271 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:03:15,499 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:03:19,974 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:03:19,975 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:03:19,975 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:03:19,981 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:03:20,276 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:03:20,981 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:03:21,279 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:03:23,237 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:03:23,237 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:03:23,237 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:03:24,719 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:03:24,720 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:03:24,720 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:03:24,725 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:03:25,285 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:03:25,285 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:03:26,763 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:03:31,764 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:03:33,295 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 19:03:33,873 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:03:33,874 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:03:33,875 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:03:33,880 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:03:34,291 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:03:35,294 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:03:36,880 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:03:37,332 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:03:37,333 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:03:37,333 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:03:37,339 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:03:38,237 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:03:38,237 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:03:38,237 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:03:38,296 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:03:39,298 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:03:42,523 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:03:42,572 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:03:42,572 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:03:42,573 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:03:42,578 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:03:43,303 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:03:43,303 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:03:46,660 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:03:46,660 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:03:46,661 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:03:46,666 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:03:47,310 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:03:47,310 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:03:47,667 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:03:50,760 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:03:50,761 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:03:50,761 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:03:50,766 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:03:51,314 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:03:51,315 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:03:52,766 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:03:53,237 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:03:53,237 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:03:53,237 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:03:55,027 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:03:55,028 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:03:55,028 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:03:55,033 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:03:55,317 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:03:57,320 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:03:58,071 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:03:58,190 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:03:58,191 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:03:58,191 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:03:58,196 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:03:58,322 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:03:59,324 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:04:03,197 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:04:03,297 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 19:04:03,612 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:04:03,613 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:04:03,613 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:04:03,618 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:04:04,328 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:04:05,330 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:04:08,237 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:04:08,237 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:04:08,237 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:04:08,537 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:04:09,680 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:04:09,681 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:04:09,681 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:04:09,686 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:04:10,334 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:04:11,336 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:04:13,203 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:04:13,204 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:04:13,204 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:04:13,209 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:04:13,338 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:04:14,210 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:04:15,341 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:04:19,211 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:04:21,685 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:04:21,686 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:04:21,686 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:04:21,692 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:04:22,346 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:04:23,238 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:04:23,238 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:04:23,238 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:04:23,349 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:04:24,506 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:04:26,177 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:04:26,177 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:04:26,178 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:04:26,183 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:04:26,352 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:04:27,355 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:04:29,473 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:04:29,474 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:04:29,474 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:04:29,479 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:04:30,357 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:04:30,480 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:04:31,360 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:04:33,303 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 19:04:34,590 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:04:34,591 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:04:34,592 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:04:34,597 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:04:35,365 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:04:35,365 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:04:35,598 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:04:38,237 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:04:38,237 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:04:38,238 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:04:38,239 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:04:38,528 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:04:38,528 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:04:38,533 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:04:39,370 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:04:39,370 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:04:41,534 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:04:46,534 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:04:46,806 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:04:46,807 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:04:46,807 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:04:46,812 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:04:47,378 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:04:47,379 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:04:51,813 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:04:52,799 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:04:52,800 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:04:52,801 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:04:52,806 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:04:53,237 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:04:53,237 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:04:53,238 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:04:53,384 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:04:53,385 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:04:55,951 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:04:55,952 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:04:55,953 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:04:55,958 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:04:56,387 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:04:56,995 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:04:57,389 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:05:01,360 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:05:01,361 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:05:01,361 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:05:01,366 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:05:01,393 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:05:02,367 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:05:03,301 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 19:05:03,396 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:05:05,592 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:05:05,593 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:05:05,593 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:05:05,598 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:05:06,398 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:05:07,401 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:05:07,599 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:05:08,237 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:05:08,237 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:05:08,238 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:05:09,543 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:05:09,544 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:05:09,544 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:05:09,549 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:05:10,403 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:05:11,405 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:05:13,587 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:05:18,082 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:05:18,083 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:05:18,083 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:05:18,088 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:05:18,410 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:05:19,089 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:05:19,413 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:05:22,197 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:05:22,198 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:05:22,198 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:05:22,203 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:05:22,415 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:05:23,237 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:05:23,237 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:05:23,238 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:05:23,418 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:05:24,428 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:05:25,650 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:05:25,650 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:05:25,651 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:05:25,656 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:05:26,421 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:05:27,424 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:05:29,695 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:05:29,697 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:05:29,697 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:05:29,698 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:05:29,703 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:05:30,426 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:05:31,429 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:05:32,986 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:05:32,987 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:05:32,987 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:05:32,992 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:05:33,307 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 19:05:33,430 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:05:35,308 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:05:35,434 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:05:36,734 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:05:36,734 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:05:36,735 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:05:36,740 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:05:37,437 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:05:37,438 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:05:38,237 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:05:38,238 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:05:38,238 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:05:40,193 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:05:40,194 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:05:40,194 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:05:40,199 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:05:40,440 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:05:41,239 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:05:41,442 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:05:44,008 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:05:44,009 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:05:44,009 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:05:44,015 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:05:44,445 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:05:45,447 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:05:47,015 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:05:47,966 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:05:47,967 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:05:47,967 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:05:47,972 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:05:48,451 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:05:49,454 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:05:51,969 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:05:51,970 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:05:51,970 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:05:51,975 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:05:52,456 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:05:52,975 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:05:53,237 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:05:53,238 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:05:53,238 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:05:53,458 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:05:55,489 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:05:55,489 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:05:55,490 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:05:55,494 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:05:56,460 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:05:57,463 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:05:58,531 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:06:01,457 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:06:01,458 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:06:01,458 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:06:01,464 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:06:01,466 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:06:03,306 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 19:06:03,469 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:06:04,307 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:06:06,193 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:06:06,194 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:06:06,195 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:06:06,200 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:06:06,471 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:06:07,474 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:06:08,238 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:06:08,238 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:06:08,238 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:06:09,464 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:06:10,819 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:06:10,820 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:06:10,820 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:06:10,825 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:06:11,478 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:06:11,478 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:06:14,608 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:06:14,609 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:06:14,609 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:06:14,609 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:06:14,614 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:06:15,483 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:06:15,483 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:06:19,615 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:06:19,995 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:06:19,996 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:06:19,997 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:06:20,002 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:06:20,486 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:06:21,489 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:06:23,238 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:06:23,238 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:06:23,238 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:06:23,739 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:06:23,739 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:06:23,740 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:06:23,745 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:06:24,490 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:06:24,783 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:06:25,494 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:06:27,121 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:06:27,122 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:06:27,122 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:06:27,128 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:06:27,496 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:06:29,499 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:06:30,128 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:06:31,511 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:06:31,512 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:06:31,512 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:06:31,517 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:06:32,501 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:06:33,311 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 19:06:33,504 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:06:35,313 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:06:35,364 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:06:35,365 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:06:35,366 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:06:35,371 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:06:35,505 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:06:37,508 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:06:38,238 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:06:38,238 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:06:38,238 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:06:39,132 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:06:39,132 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:06:39,133 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:06:39,144 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:06:39,510 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:06:41,175 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:06:41,513 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:06:44,759 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:06:44,760 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:06:44,760 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:06:44,765 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:06:45,518 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:06:45,518 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:06:46,766 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:06:48,748 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:06:48,749 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:06:48,749 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:06:48,754 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:06:49,524 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:06:49,524 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:06:52,083 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:06:52,084 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:06:52,084 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:06:52,084 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:06:52,089 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:06:52,526 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:06:53,238 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:06:53,238 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:06:53,238 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:06:53,528 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:06:57,408 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:06:57,408 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:06:57,409 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:06:57,410 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:06:57,415 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:06:57,531 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:06:59,534 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:07:02,451 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:07:03,030 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:07:03,031 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:07:03,031 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:07:03,036 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:07:03,310 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 19:07:03,539 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:07:03,539 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:07:08,065 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:07:08,066 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:07:08,066 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:07:08,066 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:07:08,071 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:07:08,238 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:07:08,238 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:07:08,238 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:07:08,542 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:07:09,545 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:07:11,850 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:07:11,850 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:07:11,851 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:07:11,856 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:07:12,547 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:07:13,549 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:07:13,895 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:07:15,596 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:07:15,596 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:07:15,596 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:07:15,602 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:07:16,551 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:07:17,554 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:07:19,250 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:07:19,251 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:07:19,252 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:07:19,252 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:07:19,256 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:07:19,555 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:07:21,558 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:07:23,238 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:07:23,238 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:07:23,239 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:07:24,487 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:07:24,811 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:07:24,812 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:07:24,812 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:07:24,817 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:07:25,564 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:07:25,564 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:07:28,894 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:07:28,895 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:07:28,896 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:07:28,901 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:07:29,568 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:07:29,569 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:07:29,902 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:07:32,583 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:07:32,583 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:07:32,583 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:07:32,589 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:07:33,315 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 19:07:33,573 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:07:33,573 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:07:35,316 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:07:37,135 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:07:37,136 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:07:37,136 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:07:37,141 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:07:37,576 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:07:38,238 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:07:38,238 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:07:38,238 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:07:39,579 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:07:40,451 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:07:40,658 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:07:40,658 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:07:40,659 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:07:40,664 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:07:41,582 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:07:41,582 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:07:44,571 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:07:44,572 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:07:44,573 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:07:44,577 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:07:44,584 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:07:45,578 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:07:45,587 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:07:49,070 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:07:49,071 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:07:49,071 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:07:49,076 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:07:49,592 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:07:49,592 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:07:51,076 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:07:52,609 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:07:52,610 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:07:52,610 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:07:52,615 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:07:53,238 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:07:53,238 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:07:53,239 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:07:53,597 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:07:53,597 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:07:56,506 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:07:57,921 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:07:57,921 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:07:57,921 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:07:57,926 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:07:58,600 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:07:59,603 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:08:01,963 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:08:02,798 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:08:02,798 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:08:02,799 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:08:02,803 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:08:03,315 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 19:08:03,607 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:08:03,608 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:08:06,154 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:08:06,155 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:08:06,155 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:08:06,160 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:08:06,609 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:08:07,160 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:08:07,612 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:08:08,238 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:08:08,238 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:08:08,239 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:08:12,465 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:08:15,274 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:08:15,275 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:08:15,275 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:08:15,280 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:08:15,617 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:08:17,620 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:08:18,319 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:08:21,214 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:08:21,215 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:08:21,215 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:08:21,220 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:08:21,622 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:08:23,238 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:08:23,238 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:08:23,239 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:08:23,550 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:08:23,626 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:08:24,553 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:08:24,553 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:08:24,553 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:08:24,558 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:08:24,626 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:08:25,630 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:08:28,595 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:08:33,000 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:08:33,001 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:08:33,001 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:08:33,006 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:08:33,317 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 19:08:33,637 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:08:33,637 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:08:34,318 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:08:36,784 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:08:36,785 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:08:36,785 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:08:36,790 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:08:37,641 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:08:37,642 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:08:38,238 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:08:38,238 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:08:38,239 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:08:39,711 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:08:40,061 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:08:40,061 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:08:40,062 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:08:40,066 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:08:40,644 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:08:41,646 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:08:44,368 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:08:44,369 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:08:44,369 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:08:44,374 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:08:44,648 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:08:45,374 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:08:45,651 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:08:48,671 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:08:48,672 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:08:48,672 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:08:48,677 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:08:49,656 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:08:49,657 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:08:50,678 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:08:53,224 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:08:53,224 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:08:53,225 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:08:53,230 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:08:53,238 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:08:53,239 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:08:53,239 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:08:53,659 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:08:55,662 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:08:56,476 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:08:58,659 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:08:58,660 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:08:58,660 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:08:58,665 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:08:58,666 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:08:59,668 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:09:01,703 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:09:02,527 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:09:02,528 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:09:02,528 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:09:02,533 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:09:02,670 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:09:03,319 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 19:09:03,673 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:09:07,321 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:09:08,238 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:09:08,239 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:09:08,239 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:09:09,385 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:09:09,386 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:09:09,386 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:09:09,391 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:09:09,677 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:09:11,680 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:09:12,431 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:09:12,713 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:09:12,713 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:09:12,714 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:09:12,719 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:09:13,683 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:09:13,684 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:09:16,069 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:09:16,070 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:09:16,071 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:09:16,076 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:09:16,685 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:09:17,688 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:09:18,076 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:09:23,077 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:09:23,238 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:09:23,239 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:09:23,239 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:09:24,505 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:09:24,505 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:09:24,506 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:09:24,511 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:09:24,692 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:09:25,696 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:09:28,547 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:09:29,934 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:09:29,935 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:09:29,935 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:09:29,941 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:09:30,699 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:09:31,702 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:09:33,321 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 19:09:33,507 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:09:33,507 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:09:33,508 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:09:33,513 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:09:33,703 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:09:34,513 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:09:35,706 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:09:37,386 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:09:37,387 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:09:37,387 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:09:37,392 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:09:37,708 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:09:38,239 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:09:38,239 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:09:38,239 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:09:39,711 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:09:40,507 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:09:41,525 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:09:41,525 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:09:41,526 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:09:41,531 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:09:41,712 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:09:43,715 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:09:45,440 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:09:45,441 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:09:45,441 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:09:45,446 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:09:45,717 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:09:46,446 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:09:47,722 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:09:50,484 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:09:50,488 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:09:50,488 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:09:50,494 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:09:50,724 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:09:51,494 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:09:51,726 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:09:53,239 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:09:53,239 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:09:53,240 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:09:55,281 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:09:55,281 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:09:55,282 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:09:55,286 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:09:55,729 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:09:57,323 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:09:57,732 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:09:58,618 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:09:58,619 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:09:58,619 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:09:58,624 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:09:58,733 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:09:59,736 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:10:02,172 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:10:02,173 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:10:02,174 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:10:02,178 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:10:02,738 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:10:03,179 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:10:03,323 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 19:10:03,740 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:10:05,868 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:10:05,868 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:10:05,869 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:10:05,874 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:10:06,742 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:10:07,745 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:10:08,239 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:10:08,239 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:10:08,239 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:10:08,457 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:10:09,128 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:10:09,129 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:10:09,129 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:10:09,134 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:10:09,748 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:10:09,749 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:10:13,069 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:10:13,070 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:10:13,071 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:10:13,076 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:10:13,753 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:10:13,754 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:10:14,076 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:10:16,702 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:10:16,703 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:10:16,703 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:10:16,708 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:10:16,755 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:10:17,758 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:10:19,709 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:10:20,468 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:10:20,469 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:10:20,469 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:10:20,474 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:10:20,760 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:10:21,763 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:10:23,239 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:10:23,239 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:10:23,240 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:10:24,009 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:10:24,010 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:10:24,010 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:10:24,015 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:10:24,765 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:10:25,051 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:10:25,768 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:10:30,051 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:10:30,266 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:10:30,267 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:10:30,267 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:10:30,272 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:10:30,772 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:10:31,774 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:10:33,325 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 19:10:34,750 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:10:34,751 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:10:34,751 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:10:34,756 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:10:34,776 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:10:35,756 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:10:35,779 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:10:38,239 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:10:38,239 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:10:38,239 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:10:39,071 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:10:39,071 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:10:39,071 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:10:39,076 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:10:39,783 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:10:39,784 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:10:41,115 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:10:42,795 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:10:42,796 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:10:42,796 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:10:42,801 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:10:43,788 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:10:43,788 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:10:46,802 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:10:48,166 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:10:48,167 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:10:48,167 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:10:48,172 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:10:48,793 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:10:49,796 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:10:52,173 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:10:52,269 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:10:52,270 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:10:52,270 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:10:52,275 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:10:52,798 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:10:53,239 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:10:53,239 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:10:53,240 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:10:53,801 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:10:55,701 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:10:55,702 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:10:55,702 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:10:55,707 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:10:55,802 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:10:57,743 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:10:57,805 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:10:59,435 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:10:59,435 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:10:59,436 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:10:59,440 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:10:59,807 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:11:01,810 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:11:03,328 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 19:11:03,330 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:11:03,918 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:11:03,919 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:11:03,919 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:11:03,924 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:11:04,812 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:11:05,815 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:11:07,195 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:11:07,195 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:11:07,196 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:11:07,201 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:11:07,818 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:11:07,819 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:11:08,239 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:11:08,239 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:11:08,240 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:11:08,505 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:11:10,866 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:11:10,867 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:11:10,867 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:11:10,872 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:11:11,823 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:11:11,823 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:11:13,911 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:11:14,248 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:11:14,248 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:11:14,249 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:11:14,253 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:11:14,825 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:11:15,828 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:11:18,092 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:11:18,093 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:11:18,094 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:11:18,099 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:11:18,830 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:11:19,100 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:11:19,833 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:11:23,239 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:11:23,240 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:11:23,240 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:11:24,528 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:11:26,540 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:11:26,541 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:11:26,541 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:11:26,546 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:11:26,838 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:11:27,841 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:11:29,583 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:11:31,213 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:11:31,214 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:11:31,215 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:11:31,220 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:11:31,846 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:11:31,846 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:11:33,330 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 19:11:35,332 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:11:35,544 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:11:35,545 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:11:35,545 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:11:35,550 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:11:35,848 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:11:37,852 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:11:38,239 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:11:38,240 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:11:38,240 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:11:38,890 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:11:38,891 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:11:38,891 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:11:38,896 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:11:39,855 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:11:39,855 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:11:40,935 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:11:42,331 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:11:42,332 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:11:42,332 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:11:42,337 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:11:42,857 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:11:43,860 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:11:46,230 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:11:46,231 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:11:46,231 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:11:46,231 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:11:46,236 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:11:46,862 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:11:47,866 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:11:51,237 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:11:51,639 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:11:51,640 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:11:51,640 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:11:51,645 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:11:51,868 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:11:53,240 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:11:53,240 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:11:53,240 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:11:53,871 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:11:55,870 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:11:55,871 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:11:55,871 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:11:55,876 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:11:56,873 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:11:56,918 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:11:57,876 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:12:00,046 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:12:00,047 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:12:00,047 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:12:00,052 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:12:00,878 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:12:01,881 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:12:02,053 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:12:03,332 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 19:12:03,741 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:12:03,742 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:12:03,742 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:12:03,747 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:12:03,882 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:12:05,885 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:12:07,748 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:12:07,843 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:12:07,843 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:12:07,844 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:12:07,849 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:12:07,887 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:12:08,240 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:12:08,240 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:12:08,240 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:12:09,890 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:12:12,956 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:12:13,059 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:12:13,060 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:12:13,060 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:12:13,065 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:12:13,895 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:12:13,895 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:12:16,890 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:12:16,891 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:12:16,891 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:12:16,896 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:12:16,897 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:12:17,899 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:12:18,897 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:12:20,606 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:12:20,607 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:12:20,607 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:12:20,612 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:12:20,901 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:12:21,904 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:12:23,240 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:12:23,240 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:12:23,240 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:12:24,429 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:12:26,363 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:12:26,364 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:12:26,364 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:12:26,369 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:12:26,908 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:12:27,911 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:12:29,918 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:12:29,919 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:12:29,919 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:12:29,919 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:12:29,924 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:12:30,913 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:12:31,915 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:12:33,335 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 19:12:34,090 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:12:34,091 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:12:34,091 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:12:34,096 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:12:34,917 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:12:35,097 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:12:35,920 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:12:38,046 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:12:38,046 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:12:38,047 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:12:38,052 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:12:38,240 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:12:38,240 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:12:38,240 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:12:38,922 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:12:39,925 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:12:40,584 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:12:43,705 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:12:43,705 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:12:43,706 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:12:43,711 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:12:43,927 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:12:45,747 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:12:45,931 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:12:47,791 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:12:47,792 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:12:47,792 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:12:47,797 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:12:47,933 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:12:49,936 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:12:50,798 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:12:52,601 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:12:52,602 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:12:52,602 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:12:52,607 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:12:52,939 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:12:53,240 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:12:53,240 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:12:53,240 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:12:53,941 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:12:56,222 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:12:56,223 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:12:56,223 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:12:56,224 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:12:56,229 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:12:56,943 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:12:57,946 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:12:59,711 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:12:59,711 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:12:59,712 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:12:59,717 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:12:59,948 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:13:01,718 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:13:01,951 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:13:03,144 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:13:03,159 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:13:03,159 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:13:03,164 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:13:03,337 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 19:13:03,954 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:13:03,954 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:13:07,338 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:13:08,184 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:13:08,185 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:13:08,185 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:13:08,190 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:13:08,240 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:13:08,240 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:13:08,241 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:13:08,957 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:13:09,960 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:13:11,644 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:13:11,645 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:13:11,645 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:13:11,650 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:13:11,962 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:13:12,687 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:13:13,965 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:13:15,101 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:13:15,101 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:13:15,102 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:13:15,106 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:13:15,968 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:13:15,968 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:13:18,107 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:13:23,108 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:13:23,240 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:13:23,240 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:13:23,240 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:13:23,591 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:13:23,592 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:13:23,592 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:13:23,597 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:13:23,973 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:13:25,978 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:13:26,555 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:13:26,556 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:13:26,557 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:13:26,562 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:13:26,978 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:13:27,982 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:13:28,563 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:13:30,315 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:13:30,316 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:13:30,316 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:13:30,321 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:13:30,984 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:13:31,986 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:13:33,339 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 19:13:33,787 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:13:33,787 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:13:33,788 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:13:33,788 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:13:33,793 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:13:33,988 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:13:35,991 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:13:37,517 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:13:37,518 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:13:37,518 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:13:37,523 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:13:37,995 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:13:37,995 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:13:38,240 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:13:38,240 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:13:38,240 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:13:39,442 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:13:42,055 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:13:42,056 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:13:42,056 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:13:42,061 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:13:42,998 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:13:44,001 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:13:45,099 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:13:46,184 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:13:46,185 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:13:46,185 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:13:46,190 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:13:47,003 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:13:48,007 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:13:50,191 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:13:50,256 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:13:50,257 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:13:50,257 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:13:50,263 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:13:51,009 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:13:52,012 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:13:53,240 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:13:53,241 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:13:53,241 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:13:55,471 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:13:58,914 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:13:58,915 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:13:58,915 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:13:58,920 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:13:59,016 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:14:00,019 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:14:00,959 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:14:03,344 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 19:14:04,389 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:14:04,389 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:14:04,390 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:14:04,394 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:14:05,022 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:14:06,025 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:14:06,395 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:14:08,062 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:14:08,063 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:14:08,063 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:14:08,068 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:14:08,240 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:14:08,240 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:14:08,241 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:14:09,027 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:14:10,030 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:14:11,505 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:14:11,941 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:14:11,942 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:14:11,942 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:14:11,947 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:14:12,031 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:14:14,034 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:14:16,310 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:14:16,311 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:14:16,311 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:14:16,316 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:14:17,036 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:14:17,317 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:14:18,039 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:14:20,218 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:14:20,218 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:14:20,219 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:14:20,224 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:14:21,041 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:14:22,044 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:14:23,205 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:14:23,206 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:14:23,206 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:14:23,206 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:14:23,211 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:14:23,240 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:14:23,241 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:14:23,241 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:14:24,047 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:14:24,047 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:14:27,085 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:14:27,085 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:14:27,085 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:14:27,090 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:14:28,053 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:14:28,054 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:14:29,127 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:14:31,012 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:14:31,013 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:14:31,013 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:14:31,019 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:14:31,055 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:14:32,058 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:14:33,343 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 19:14:34,344 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:14:37,198 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:14:37,199 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:14:37,199 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:14:37,204 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:14:38,064 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:14:38,064 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:14:38,240 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:14:38,241 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:14:38,241 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:14:39,645 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:14:40,806 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:14:40,807 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:14:40,807 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:14:40,812 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:14:41,066 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:14:42,069 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:14:44,195 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:14:44,196 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:14:44,196 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:14:44,201 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:14:45,071 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:14:45,201 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:14:46,074 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:14:47,924 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:14:47,925 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:14:47,925 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:14:47,930 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:14:48,077 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:14:50,080 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:14:50,931 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:14:52,347 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:14:52,348 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:14:52,348 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:14:52,353 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:14:53,082 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:14:53,240 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:14:53,241 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:14:53,241 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:14:54,085 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:14:56,079 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:14:56,079 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:14:56,080 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:14:56,080 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:14:56,085 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:14:56,086 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:14:58,090 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:14:59,945 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:14:59,946 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:14:59,946 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:14:59,951 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:15:00,091 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:15:01,952 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:15:02,096 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:15:03,348 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 19:15:07,350 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:15:08,241 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:15:08,241 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:15:08,241 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:15:08,456 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:15:08,533 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:15:08,533 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:15:08,538 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:15:09,100 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:15:10,103 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:15:11,607 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:15:11,607 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:15:11,607 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:15:11,612 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:15:12,106 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:15:12,106 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:15:12,651 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:15:15,523 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:15:15,524 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:15:15,524 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:15:15,529 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:15:16,111 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:15:16,111 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:15:18,530 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:15:22,063 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:15:22,064 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:15:22,064 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:15:22,069 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:15:22,115 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:15:23,241 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:15:23,241 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:15:23,241 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:15:24,118 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:15:24,456 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:15:25,330 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:15:25,331 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:15:25,331 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:15:25,336 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:15:26,122 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:15:26,122 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:15:29,158 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:15:29,159 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:15:29,160 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:15:29,164 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:15:30,127 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:15:30,128 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:15:30,165 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:15:33,347 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 19:15:35,349 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:15:35,707 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:15:35,708 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:15:35,708 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:15:35,713 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:15:36,131 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:15:38,134 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:15:38,241 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:15:38,241 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:15:38,241 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:15:40,538 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:15:40,995 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:15:40,996 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:15:40,996 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:15:41,001 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:15:41,137 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:15:42,140 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:15:44,699 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:15:44,700 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:15:44,700 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:15:44,705 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:15:45,142 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:15:45,706 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:15:46,145 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:15:48,459 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:15:48,459 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:15:48,460 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:15:48,465 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:15:49,148 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:15:50,150 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:15:51,465 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:15:52,840 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:15:52,841 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:15:52,841 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:15:52,846 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:15:53,152 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:15:53,241 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:15:53,241 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:15:53,241 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:15:54,155 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:15:55,881 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:15:55,882 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:15:55,882 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:15:55,887 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:15:56,156 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:15:56,923 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:15:58,159 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:15:59,328 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:15:59,329 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:15:59,329 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:15:59,334 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:16:00,163 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:16:00,163 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:16:02,335 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:16:03,352 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 19:16:03,814 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:16:03,815 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:16:03,815 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:16:03,820 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:16:04,166 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:16:06,169 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:16:07,821 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:16:07,857 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:16:07,858 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:16:07,858 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:16:07,863 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:16:08,170 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:16:08,241 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:16:08,241 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:16:08,241 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:16:10,174 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:16:12,179 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:16:12,179 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:16:12,179 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:16:12,184 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:16:13,175 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:16:13,223 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:16:14,178 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:16:18,223 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:16:18,782 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:16:18,782 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:16:18,783 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:16:18,788 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:16:19,181 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:16:20,184 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:16:23,241 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:16:23,241 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:16:23,242 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:16:23,470 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:16:24,865 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:16:24,865 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:16:24,866 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:16:24,871 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:16:25,187 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:16:26,190 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:16:28,488 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:16:28,489 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:16:28,489 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:16:28,489 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:16:28,494 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:16:29,192 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:16:30,195 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:16:31,900 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:16:31,901 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:16:31,901 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:16:31,906 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:16:32,196 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:16:33,352 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 19:16:34,200 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:16:34,353 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:16:37,174 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:16:37,175 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:16:37,175 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:16:37,180 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:16:37,201 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:16:38,204 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:16:38,241 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:16:38,241 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:16:38,242 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:16:39,478 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:16:40,811 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:16:40,812 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:16:40,812 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:16:40,820 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:16:41,207 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:16:42,209 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:16:44,855 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:16:49,768 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:16:49,769 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:16:49,769 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:16:49,775 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:16:50,218 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:16:50,218 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:16:50,775 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:16:53,241 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:16:53,241 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:16:53,242 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:16:54,564 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:16:54,565 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:16:54,565 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:16:54,570 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:16:55,222 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:16:56,224 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:16:56,607 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:16:58,381 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:16:58,382 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:16:58,382 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:16:58,387 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:16:59,227 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:17:00,229 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:17:02,388 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:17:03,357 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 19:17:04,963 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:17:04,964 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:17:04,964 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:17:04,969 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:17:05,232 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:17:06,235 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:17:07,970 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:17:08,241 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:17:08,241 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:17:08,242 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:17:11,315 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:17:11,316 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:17:11,316 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:17:11,320 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:17:12,241 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:17:12,241 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:17:13,359 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:17:15,406 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:17:15,407 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:17:15,407 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:17:15,412 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:17:16,245 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:17:16,245 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:17:18,413 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:17:21,323 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:17:21,324 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:17:21,324 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:17:21,329 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:17:22,251 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:17:22,251 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:17:23,241 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:17:23,241 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:17:23,242 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:17:23,438 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:17:26,232 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:17:26,233 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:17:26,233 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:17:26,239 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:17:26,254 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:17:28,257 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:17:29,275 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:17:29,867 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:17:29,867 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:17:29,868 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:17:29,873 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:17:30,259 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:17:32,262 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:17:33,356 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 19:17:34,179 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:17:34,180 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:17:34,180 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:17:34,185 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:17:34,263 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:17:35,185 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:17:36,266 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:17:38,241 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:17:38,242 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:17:38,242 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:17:38,678 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:17:38,678 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:17:38,679 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:17:38,684 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:17:39,268 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:17:40,271 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:17:40,723 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:17:42,149 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:17:42,150 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:17:42,151 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:17:42,155 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:17:42,272 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:17:44,275 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:17:46,156 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:17:46,976 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:17:46,977 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:17:46,977 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:17:46,982 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:17:47,277 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:17:48,281 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:17:51,983 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:17:53,241 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:17:53,242 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:17:53,242 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:17:55,336 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:17:55,337 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:17:55,337 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:17:55,342 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:17:56,290 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:17:56,291 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:17:57,379 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:17:59,269 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:17:59,270 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:17:59,270 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:17:59,275 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:17:59,292 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:18:00,295 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:18:03,276 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:18:03,358 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 19:18:03,857 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:18:03,858 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:18:03,859 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:18:03,863 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:18:04,299 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:18:04,300 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:18:08,242 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:18:08,242 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:18:08,242 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:18:08,673 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:18:08,787 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:18:08,788 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:18:08,788 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:18:08,793 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:18:09,303 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:18:10,305 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:18:13,831 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:18:16,373 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:18:16,374 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:18:16,374 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:18:16,379 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:18:17,310 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:18:18,313 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:18:19,380 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:18:20,642 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:18:20,643 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:18:20,643 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:18:20,648 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:18:21,315 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:18:22,318 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:18:23,242 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:18:23,242 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:18:23,242 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:18:24,489 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:18:24,731 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:18:24,732 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:18:24,732 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:18:24,737 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:18:25,320 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:18:26,323 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:18:29,775 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:18:30,139 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:18:30,140 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:18:30,140 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:18:30,145 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:18:30,327 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:18:32,330 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:18:33,360 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 19:18:34,075 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:18:34,076 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:18:34,076 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:18:34,081 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:18:34,331 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:18:35,082 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:18:36,335 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:18:38,040 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:18:38,041 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:18:38,041 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:18:38,046 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:18:38,242 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:18:38,242 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:18:38,242 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:18:38,336 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:18:40,339 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:18:40,466 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:18:41,107 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:18:41,108 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:18:41,108 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:18:41,114 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:18:41,340 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:18:42,343 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:18:46,151 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:18:46,767 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:18:46,768 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:18:46,768 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:18:46,773 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:18:47,347 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:18:48,351 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:18:51,315 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:18:51,316 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:18:51,316 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:18:51,316 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:18:51,321 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:18:51,353 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:18:52,355 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:18:53,242 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:18:53,242 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:18:53,242 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:18:54,921 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:18:54,922 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:18:54,922 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:18:54,928 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:18:55,357 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:18:56,360 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:18:56,963 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:18:58,215 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:18:58,216 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:18:58,216 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:18:58,221 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:18:58,361 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:19:00,365 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:19:01,732 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:19:01,733 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:19:01,734 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:19:01,739 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:19:02,368 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:19:02,368 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:19:02,739 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:19:03,363 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 19:19:05,084 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:19:05,085 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:19:05,085 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:19:05,090 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:19:05,370 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:19:06,377 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:19:08,091 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:19:08,242 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:19:08,242 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:19:08,243 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:19:13,342 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:19:13,343 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:19:13,343 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:19:13,343 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:19:13,348 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:19:13,381 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:19:14,384 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:19:18,387 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:19:22,312 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:19:22,313 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:19:22,313 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:19:22,318 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:19:22,389 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:19:23,242 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:19:23,242 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:19:23,242 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:19:23,542 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:19:24,392 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:19:28,542 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:19:30,611 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:19:30,611 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:19:30,612 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:19:30,616 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:19:31,398 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:19:32,400 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:19:33,369 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 19:19:34,248 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:19:34,249 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:19:34,249 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:19:34,250 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:19:34,254 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:19:34,402 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:19:36,405 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:19:38,242 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:19:38,242 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:19:38,243 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:19:39,441 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:19:40,197 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:19:40,198 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:19:40,198 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:19:40,203 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:19:40,407 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:19:42,411 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:19:45,130 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:19:45,131 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:19:45,131 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:19:45,131 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:19:45,136 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:19:45,413 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:19:46,415 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:19:49,672 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:19:49,673 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:19:49,673 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:19:49,680 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:19:50,421 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:19:50,422 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:19:50,680 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:19:53,242 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:19:53,242 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:19:53,243 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:19:53,557 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:19:53,558 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:19:53,558 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:19:53,563 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:19:54,426 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:19:54,426 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:19:56,599 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:19:58,299 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:19:58,300 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:19:58,301 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:19:58,305 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:19:58,428 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:20:00,432 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:20:02,306 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:20:02,370 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:20:02,371 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:20:02,371 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:20:02,376 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:20:02,433 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:20:03,367 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 19:20:04,436 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:20:07,368 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:20:07,972 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:20:07,973 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:20:07,973 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:20:07,978 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:20:08,242 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:20:08,242 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:20:08,243 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:20:08,441 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:20:08,441 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:20:11,941 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:20:11,942 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:20:11,942 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:20:11,947 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:20:12,445 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:20:12,446 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:20:12,987 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:20:16,605 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:20:16,606 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:20:16,606 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:20:16,611 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:20:17,449 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:20:18,451 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:20:18,612 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:20:21,227 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:20:21,228 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:20:21,228 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:20:21,233 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:20:21,453 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:20:22,456 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:20:23,242 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:20:23,242 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:20:23,243 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:20:24,536 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:20:25,954 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:20:25,955 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:20:25,955 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:20:25,960 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:20:26,461 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:20:26,461 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:20:29,999 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:20:31,418 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:20:31,418 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:20:31,419 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:20:31,423 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:20:31,464 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:20:32,467 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:20:33,372 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 19:20:34,867 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:20:34,868 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:20:34,868 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:20:34,873 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:20:35,469 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:20:35,873 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:20:36,471 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:20:38,242 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:20:38,243 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:20:38,243 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:20:39,620 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:20:39,620 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:20:39,620 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:20:39,625 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:20:40,476 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:20:40,476 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:20:41,663 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:20:44,513 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:20:44,514 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:20:44,514 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:20:44,519 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:20:45,479 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:20:46,482 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:20:47,519 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:20:49,064 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:20:49,065 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:20:49,065 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:20:49,070 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:20:49,485 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:20:50,488 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:20:52,950 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:20:52,951 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:20:52,951 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:20:52,951 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:20:52,956 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:20:53,242 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:20:53,243 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:20:53,243 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:20:53,490 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:20:54,495 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:20:56,985 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:20:56,986 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:20:56,986 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:20:56,991 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:20:57,497 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:20:58,027 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:20:58,499 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:21:01,726 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:21:01,727 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:21:01,727 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:21:01,732 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:21:02,504 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:21:02,504 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:21:03,371 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 19:21:03,373 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:21:06,365 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:21:06,366 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:21:06,366 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:21:06,372 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:21:06,506 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:21:08,243 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:21:08,243 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:21:08,243 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:21:08,514 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:21:08,532 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:21:09,889 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:21:09,890 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:21:09,890 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:21:09,895 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:21:10,517 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:21:10,518 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:21:13,935 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:21:14,642 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:21:14,643 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:21:14,644 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:21:14,648 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:21:15,521 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:21:16,523 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:21:19,649 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:21:23,171 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:21:23,172 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:21:23,172 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:21:23,177 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:21:23,243 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:21:23,243 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:21:23,243 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:21:23,527 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:21:24,530 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:21:25,523 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:21:26,411 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:21:26,412 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:21:26,412 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:21:26,417 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:21:26,532 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:21:28,535 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:21:30,009 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:21:30,010 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:21:30,011 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:21:30,015 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:21:30,538 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:21:30,539 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:21:31,016 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:21:33,377 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 19:21:34,203 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:21:34,204 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:21:34,205 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:21:34,210 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:21:34,541 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:21:36,210 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:21:36,545 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:21:38,243 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:21:38,243 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:21:38,243 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:21:41,484 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:21:42,680 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:21:42,681 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:21:42,681 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:21:42,686 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:21:43,549 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:21:44,551 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:21:46,724 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:21:48,319 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:21:48,320 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:21:48,321 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:21:48,325 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:21:48,555 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:21:50,559 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:21:52,326 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:21:53,190 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:21:53,190 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:21:53,191 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:21:53,196 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:21:53,243 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:21:53,243 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:21:53,244 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:21:53,560 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:21:54,563 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:21:57,491 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:21:57,732 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:21:57,732 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:21:57,732 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:21:57,737 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:21:58,568 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:21:58,568 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:22:02,775 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:22:03,376 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 19:22:06,326 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:22:06,327 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:22:06,327 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:22:06,332 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:22:06,573 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:22:08,243 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:22:08,243 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:22:08,243 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:22:08,525 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:22:08,576 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:22:10,699 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:22:10,699 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:22:10,700 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:22:10,704 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:22:11,578 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:22:12,581 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:22:13,743 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:22:15,786 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:22:15,787 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:22:15,787 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:22:15,792 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:22:16,585 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:22:16,586 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:22:18,793 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:22:19,755 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:22:19,756 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:22:19,756 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:22:19,761 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:22:20,588 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:22:20,588 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:22:23,032 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:22:23,032 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:22:23,033 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:22:23,038 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:22:23,243 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:22:23,243 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:22:23,244 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:22:23,590 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:22:24,479 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:22:24,592 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:22:26,607 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:22:26,608 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:22:26,608 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:22:26,613 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:22:27,595 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:22:28,598 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:22:29,651 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:22:31,136 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:22:31,137 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:22:31,137 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:22:31,142 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:22:31,600 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:22:32,602 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:22:33,382 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 19:22:35,383 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:22:36,756 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:22:36,757 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:22:36,758 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:22:36,762 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:22:37,606 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:22:38,243 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:22:38,243 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:22:38,244 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:22:38,608 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:22:40,496 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:22:40,497 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:22:40,498 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:22:40,498 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:22:40,502 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:22:40,609 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:22:42,612 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:22:44,075 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:22:44,075 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:22:44,076 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:22:44,080 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:22:44,616 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:22:44,616 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:22:46,081 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:22:47,837 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:22:47,838 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:22:47,838 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:22:47,843 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:22:48,623 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:22:48,623 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:22:51,844 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:22:51,945 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:22:51,946 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:22:51,946 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:22:51,951 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:22:52,627 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:22:52,627 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:22:53,243 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:22:53,243 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:22:53,245 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:22:57,493 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:22:57,909 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:22:57,909 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:22:57,910 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:22:57,915 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:22:58,631 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:22:58,631 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:23:02,382 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:23:02,383 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:23:02,383 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:23:02,388 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:23:02,634 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:23:03,380 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 19:23:03,382 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:23:04,637 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:23:05,704 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:23:05,705 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:23:05,706 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:23:05,711 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:23:06,641 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:23:06,641 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:23:08,243 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:23:08,244 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:23:08,245 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:23:08,531 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:23:13,532 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:23:14,118 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:23:14,119 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:23:14,119 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:23:14,124 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:23:14,647 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:23:14,648 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:23:19,163 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:23:19,947 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:23:19,948 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:23:19,949 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:23:19,953 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:23:20,653 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:23:20,653 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:23:23,243 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:23:23,244 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:23:23,245 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:23:23,356 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:23:23,482 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:23:23,482 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:23:23,487 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:23:23,655 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:23:24,488 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:23:24,657 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:23:28,744 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:23:28,745 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:23:28,745 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:23:28,750 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:23:29,662 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:23:29,751 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:23:30,664 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:23:33,125 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:23:33,126 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:23:33,127 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:23:33,132 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:23:33,382 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 19:23:33,666 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:23:34,669 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:23:35,384 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:23:37,392 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:23:37,393 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:23:37,393 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:23:37,398 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:23:37,670 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:23:38,243 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:23:38,244 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:23:38,245 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:23:38,674 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:23:40,499 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:23:40,499 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:23:40,500 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:23:40,500 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:23:40,504 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:23:40,676 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:23:42,679 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:23:45,483 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:23:45,484 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:23:45,485 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:23:45,489 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:23:45,681 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:23:46,490 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:23:46,684 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:23:49,560 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:23:49,561 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:23:49,561 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:23:49,567 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:23:49,687 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:23:50,690 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:23:51,567 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:23:53,243 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:23:53,244 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:23:53,244 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:23:53,690 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:23:53,691 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:23:53,691 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:23:53,696 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:23:54,695 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:23:54,696 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:23:56,735 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:23:57,245 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:23:57,246 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:23:57,246 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:23:57,251 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:23:57,697 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:23:58,700 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:24:00,588 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:24:00,589 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:24:00,589 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:24:00,594 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:24:00,702 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:24:02,595 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:24:02,703 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:24:03,385 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 19:24:04,074 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:24:04,075 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:24:04,075 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:24:04,080 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:24:04,706 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:24:04,707 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:24:08,081 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:24:08,244 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:24:08,244 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:24:08,244 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:24:09,856 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:24:09,857 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:24:09,857 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:24:09,862 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:24:10,712 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:24:10,712 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:24:13,899 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:24:15,271 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:24:15,272 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:24:15,272 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:24:15,277 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:24:15,715 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:24:16,718 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:24:19,278 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:24:20,682 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:24:20,683 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:24:20,683 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:24:20,688 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:24:20,720 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:24:22,724 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:24:23,244 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:24:23,244 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:24:23,244 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:24:24,490 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:24:25,454 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:24:25,455 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:24:25,455 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:24:25,460 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:24:25,726 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:24:26,729 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:24:29,499 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:24:30,276 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:24:30,277 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:24:30,277 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:24:30,282 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:24:30,734 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:24:30,734 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:24:33,387 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 19:24:33,776 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:24:33,777 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:24:33,778 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:24:33,782 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:24:34,739 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:24:34,739 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:24:34,783 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:24:38,244 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:24:38,244 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:24:38,244 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:24:38,499 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:24:38,517 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:24:38,518 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:24:38,523 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:24:38,741 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:24:40,524 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:24:40,745 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:24:43,924 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:24:43,925 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:24:43,925 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:24:43,930 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:24:44,749 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:24:44,749 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:24:45,967 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:24:47,222 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:24:47,223 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:24:47,223 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:24:47,228 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:24:47,752 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:24:48,755 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:24:50,383 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:24:50,383 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:24:50,384 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:24:50,389 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:24:50,757 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:24:51,389 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:24:52,760 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:24:53,244 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:24:53,244 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:24:53,244 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:24:56,441 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:24:56,442 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:24:56,442 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:24:56,443 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:24:56,447 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:24:56,762 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:24:58,766 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:25:00,335 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:25:00,336 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:25:00,336 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:25:00,341 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:25:00,769 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:25:00,770 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:25:02,342 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:25:03,389 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 19:25:03,641 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:25:03,642 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:25:03,642 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:25:03,647 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:25:03,771 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:25:04,774 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:25:06,604 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:25:06,605 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:25:06,605 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:25:06,610 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:25:06,775 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:25:07,611 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:25:08,244 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:25:08,244 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:25:08,246 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:25:08,778 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:25:10,664 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:25:10,664 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:25:10,665 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:25:10,670 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:25:10,779 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:25:12,707 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:25:12,783 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:25:13,958 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:25:13,959 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:25:13,959 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:25:13,972 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:25:14,786 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:25:14,786 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:25:17,973 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:25:18,545 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:25:18,546 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:25:18,546 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:25:18,551 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:25:18,788 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:25:20,791 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:25:22,436 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:25:22,437 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:25:22,437 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:25:22,442 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:25:22,793 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:25:23,244 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:25:23,244 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:25:23,245 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:25:23,495 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:25:24,796 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:25:26,024 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:25:26,025 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:25:26,025 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:25:26,030 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:25:26,800 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:25:26,800 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:25:29,067 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:25:29,727 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:25:29,728 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:25:29,728 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:25:29,733 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:25:29,802 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:25:30,805 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:25:33,391 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 19:25:33,613 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:25:33,614 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:25:33,614 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:25:33,619 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:25:33,807 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:25:34,620 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:25:34,809 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:25:37,779 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:25:37,780 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:25:37,780 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:25:37,785 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:25:37,811 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:25:38,244 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:25:38,244 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:25:38,245 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:25:38,814 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:25:40,532 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:25:44,930 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:25:44,931 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:25:44,931 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:25:44,936 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:25:45,819 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:25:45,975 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:25:46,821 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:25:49,249 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:25:49,250 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:25:49,250 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:25:49,255 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:25:49,825 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:25:50,827 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:25:51,256 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:25:53,244 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:25:53,244 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:25:53,245 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:25:56,537 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:25:57,698 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:25:57,699 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:25:57,699 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:25:57,704 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:25:57,831 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:25:58,834 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:26:01,743 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:26:01,865 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:26:01,866 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:26:01,867 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:26:01,872 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:26:02,839 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:26:02,840 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:26:03,394 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 19:26:06,221 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:26:06,222 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:26:06,222 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:26:06,228 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:26:06,844 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:26:06,844 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:26:07,228 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:26:08,244 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:26:08,245 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:26:08,245 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:26:11,047 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:26:11,048 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:26:11,048 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:26:11,053 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:26:11,847 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:26:12,850 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:26:13,091 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:26:17,267 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:26:17,268 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:26:17,269 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:26:17,273 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:26:17,853 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:26:18,274 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:26:18,856 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:26:20,803 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:26:20,803 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:26:20,804 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:26:20,809 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:26:20,857 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:26:22,860 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:26:23,244 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:26:23,245 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:26:23,245 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:26:23,528 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:26:25,744 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:26:25,744 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:26:25,745 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:26:25,750 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:26:25,862 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:26:26,866 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:26:28,787 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:26:29,039 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:26:29,039 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:26:29,040 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:26:29,045 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:26:29,868 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:26:30,871 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:26:33,363 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:26:33,364 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:26:33,364 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:26:33,369 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:26:33,396 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 19:26:33,873 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:26:34,397 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:26:34,875 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:26:36,507 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:26:36,509 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:26:36,509 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:26:36,514 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:26:36,877 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:26:38,244 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:26:38,245 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:26:38,245 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:26:38,880 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:26:39,537 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:26:44,251 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:26:44,252 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:26:44,252 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:26:44,257 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:26:44,886 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:26:44,886 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:26:45,295 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:26:47,951 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:26:47,952 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:26:47,952 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:26:47,957 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:26:48,892 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:26:48,892 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:26:50,958 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:26:51,457 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:26:51,458 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:26:51,458 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:26:51,464 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:26:51,894 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:26:52,897 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:26:53,244 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:26:53,245 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:26:53,245 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:26:55,119 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:26:55,119 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:26:55,120 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:26:55,125 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:26:55,898 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:26:56,163 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:26:56,901 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:26:59,960 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:26:59,961 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:26:59,961 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:26:59,966 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:27:00,906 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:27:00,906 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:27:01,967 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:27:03,402 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 19:27:04,275 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:27:04,276 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:27:04,276 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:27:04,281 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:27:04,911 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:27:04,911 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:27:07,282 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:27:08,245 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:27:08,245 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:27:08,245 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:27:08,541 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:27:08,542 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:27:08,542 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:27:08,547 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:27:08,913 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:27:10,917 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:27:12,047 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:27:12,048 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:27:12,048 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:27:12,054 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:27:12,920 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:27:12,920 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:27:13,054 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:27:15,757 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:27:15,758 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:27:15,758 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:27:15,763 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:27:15,922 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:27:16,925 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:27:18,764 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:27:23,245 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:27:23,245 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:27:23,245 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:27:24,169 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:27:24,169 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:27:24,170 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:27:24,170 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:27:24,175 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:27:24,932 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:27:24,932 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:27:29,211 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:27:29,717 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:27:29,718 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:27:29,719 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:27:29,724 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:27:29,936 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:27:30,939 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:27:33,400 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 19:27:34,402 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:27:35,598 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:27:35,599 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:27:35,599 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:27:35,604 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:27:35,942 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:27:36,945 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:27:38,245 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:27:38,245 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:27:38,245 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:27:39,105 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:27:39,105 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:27:39,106 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:27:39,111 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:27:39,947 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:27:40,147 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:27:40,948 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:27:42,647 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:27:42,648 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:27:42,648 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:27:42,654 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:27:42,949 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:27:44,952 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:27:45,654 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:27:50,655 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:27:51,608 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:27:51,608 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:27:51,609 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:27:51,614 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:27:51,958 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:27:52,961 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:27:53,245 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:27:53,245 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:27:53,247 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:27:55,882 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:27:55,883 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:27:55,883 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:27:55,883 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:27:55,888 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:27:55,963 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:27:56,966 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:28:00,927 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:28:01,115 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:28:01,115 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:28:01,116 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:28:01,121 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:28:01,969 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:28:02,972 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:28:03,406 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 19:28:04,825 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:28:04,826 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:28:04,826 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:28:04,831 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:28:04,973 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:28:06,832 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:28:06,976 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:28:08,245 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:28:08,245 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:28:08,246 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:28:12,482 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:28:13,127 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:28:13,128 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:28:13,128 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:28:13,133 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:28:13,981 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:28:14,984 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:28:18,171 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:28:22,285 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:28:22,286 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:28:22,287 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:28:22,292 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:28:22,990 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:28:22,991 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:28:23,245 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:28:23,245 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:28:23,246 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:28:23,468 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:28:25,527 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:28:25,528 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:28:25,528 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:28:25,533 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:28:25,992 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:28:26,996 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:28:28,571 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:28:28,814 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:28:28,815 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:28:28,815 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:28:28,820 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:28:28,997 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:28:31,000 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:28:32,350 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:28:32,351 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:28:32,351 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:28:32,357 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:28:33,004 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:28:33,004 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:28:33,404 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 19:28:34,406 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:28:36,213 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:28:36,214 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:28:36,214 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:28:36,219 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:28:37,008 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:28:37,009 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:28:38,245 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:28:38,245 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:28:38,246 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:28:39,490 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:28:41,388 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:28:41,389 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:28:41,389 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:28:41,395 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:28:42,011 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:28:43,014 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:28:44,867 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:28:44,868 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:28:44,868 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:28:44,868 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:28:44,873 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:28:45,015 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:28:47,019 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:28:49,874 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:28:52,790 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:28:52,791 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:28:52,791 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:28:52,796 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:28:53,024 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:28:53,245 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:28:53,245 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:28:53,246 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:28:55,027 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:28:55,467 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:28:56,251 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:28:56,252 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:28:56,252 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:28:56,257 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:28:57,030 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:28:57,031 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:28:59,906 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:28:59,907 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:28:59,907 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:28:59,912 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:29:00,032 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:29:00,913 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:29:01,035 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:29:03,410 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 19:29:03,638 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:29:03,639 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:29:03,639 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:29:03,644 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:29:04,037 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:29:05,037 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:29:06,644 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:29:08,039 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:29:08,039 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:29:08,040 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:29:08,045 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:29:08,245 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:29:08,246 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:29:08,246 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:29:09,043 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:29:09,043 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:29:12,456 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:29:14,497 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:29:14,498 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:29:14,498 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:29:14,503 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:29:15,048 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:29:15,049 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:29:17,543 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:29:18,066 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:29:18,067 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:29:18,067 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:29:18,072 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:29:19,053 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:29:19,053 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:29:23,073 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:29:23,245 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:29:23,246 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:29:23,246 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:29:25,990 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:29:25,990 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:29:25,991 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:29:25,996 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:29:26,057 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:29:27,061 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:29:29,035 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:29:30,681 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:29:30,682 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:29:30,683 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:29:30,688 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:29:31,066 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:29:31,066 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:29:33,409 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 19:29:34,302 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:29:34,303 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:29:34,303 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:29:34,304 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:29:34,310 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:29:35,071 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:29:35,071 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:29:37,947 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:29:37,948 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:29:37,948 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:29:37,953 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:29:38,072 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:29:38,245 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:29:38,246 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:29:38,246 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:29:39,075 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:29:39,468 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:29:41,186 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:29:41,186 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:29:41,187 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:29:41,192 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:29:42,077 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:29:43,086 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:29:45,231 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:29:46,633 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:29:46,634 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:29:46,634 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:29:46,639 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:29:47,091 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:29:47,091 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:29:50,640 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:29:53,246 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:29:53,246 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:29:53,246 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:29:55,122 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:29:55,123 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:29:55,123 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:29:55,128 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:29:56,098 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:29:56,167 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:29:57,101 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:30:01,167 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:30:03,150 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:30:03,151 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:30:03,151 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:30:03,156 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:30:03,411 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 19:30:04,106 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:30:05,108 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:30:06,413 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:30:06,816 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:30:06,817 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:30:06,817 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:30:06,822 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:30:07,110 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:30:08,246 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:30:08,246 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:30:08,246 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:30:09,113 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:30:11,426 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:30:11,430 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:30:11,431 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:30:11,431 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:30:11,435 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:30:12,115 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:30:13,117 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:30:15,955 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:30:15,956 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:30:15,956 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:30:15,961 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:30:16,119 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:30:16,961 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:30:17,122 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:30:19,740 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:30:19,741 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:30:19,742 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:30:19,747 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:30:20,124 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:30:21,124 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:30:22,748 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:30:23,246 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:30:23,246 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:30:23,246 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:30:23,536 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:30:23,537 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:30:23,537 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:30:23,543 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:30:24,126 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:30:25,129 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:30:28,579 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:30:28,962 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:30:28,963 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:30:28,963 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:30:28,969 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:30:29,133 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:30:31,136 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:30:32,869 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:30:32,870 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:30:32,871 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:30:32,875 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:30:33,137 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:30:33,414 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 19:30:34,415 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:30:35,140 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:30:36,427 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:30:36,428 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:30:36,429 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:30:36,434 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:30:37,143 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:30:37,144 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:30:38,246 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:30:38,246 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:30:38,246 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:30:39,979 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:30:40,195 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:30:40,196 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:30:40,196 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:30:40,201 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:30:41,148 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:30:41,148 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:30:43,940 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:30:43,940 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:30:43,941 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:30:43,946 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:30:44,150 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:30:45,151 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:30:45,947 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:30:47,476 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:30:47,477 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:30:47,477 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:30:47,482 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:30:48,153 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:30:49,157 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:30:51,483 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:30:53,246 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:30:53,246 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:30:53,246 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:30:53,336 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:30:53,496 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:30:53,496 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:30:53,501 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:30:54,160 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:30:55,163 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:30:56,502 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:30:57,074 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:30:57,075 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:30:57,075 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:30:57,080 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:30:57,164 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:30:59,168 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:31:02,081 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:31:03,416 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 19:31:05,425 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:31:05,426 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:31:05,426 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:31:05,435 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:31:06,173 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:31:07,175 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:31:07,435 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:31:08,246 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:31:08,246 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:31:08,247 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:31:09,142 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:31:09,143 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:31:09,143 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:31:09,148 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:31:09,177 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:31:11,180 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:31:12,446 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:31:12,447 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:31:12,447 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:31:12,447 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:31:12,452 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:31:13,183 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:31:13,184 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:31:15,716 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:31:15,724 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:31:15,724 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:31:15,729 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:31:16,185 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:31:17,188 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:31:17,730 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:31:19,410 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:31:19,410 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:31:19,411 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:31:19,416 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:31:20,190 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:31:21,191 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:31:23,246 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:31:23,246 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:31:23,247 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:31:23,462 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:31:24,738 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:31:24,739 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:31:24,739 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:31:24,744 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:31:25,195 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:31:25,195 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:31:28,783 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:31:28,848 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:31:28,849 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:31:28,849 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:31:28,854 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:31:29,199 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:31:31,203 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:31:33,418 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 19:31:34,420 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:31:34,909 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:31:34,910 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:31:34,910 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:31:34,916 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:31:35,205 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:31:37,209 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:31:38,246 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:31:38,246 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:31:38,247 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:31:38,547 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:31:38,548 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:31:38,548 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:31:38,553 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:31:39,212 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:31:39,212 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:31:39,591 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:31:42,292 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:31:42,292 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:31:42,293 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:31:42,298 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:31:43,217 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:31:43,217 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:31:45,298 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:31:45,737 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:31:45,738 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:31:45,738 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:31:45,744 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:31:46,219 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:31:47,222 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:31:50,744 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:31:51,511 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:31:51,512 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:31:51,512 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:31:51,517 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:31:52,226 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:31:53,229 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:31:53,246 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:31:53,246 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:31:53,247 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:31:56,468 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:31:56,894 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:31:56,894 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:31:56,894 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:31:56,900 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:31:57,231 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:31:59,234 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:32:01,939 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:32:03,421 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 19:32:05,620 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:32:05,621 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:32:05,621 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:32:05,626 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:32:06,239 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:32:07,240 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:32:07,627 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:32:08,246 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:32:08,246 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:32:08,247 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:32:08,906 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:32:08,906 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:32:08,907 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:32:08,912 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:32:09,241 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:32:11,245 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:32:12,951 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:32:14,377 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:32:14,378 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:32:14,378 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:32:14,384 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:32:15,249 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:32:15,250 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:32:18,385 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:32:19,209 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:32:19,210 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:32:19,210 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:32:19,215 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:32:19,252 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:32:21,255 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:32:22,845 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:32:22,846 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:32:22,846 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:32:22,851 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:32:23,246 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:32:23,247 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:32:23,247 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:32:23,258 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:32:23,259 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:32:23,541 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:32:26,220 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:32:26,221 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:32:26,221 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:32:26,227 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:32:26,260 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:32:27,264 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:32:29,263 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:32:31,566 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:32:31,566 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:32:31,567 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:32:31,572 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:32:32,268 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:32:33,270 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:32:33,423 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 19:32:34,424 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:32:36,566 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:32:36,566 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:32:36,567 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:32:36,572 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:32:37,275 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:32:37,275 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:32:38,246 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:32:38,247 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:32:38,247 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:32:39,453 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:32:40,478 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:32:40,479 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:32:40,479 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:32:40,484 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:32:41,280 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:32:41,280 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:32:43,827 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:32:43,827 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:32:43,828 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:32:43,833 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:32:44,282 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:32:44,833 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:32:45,284 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:32:48,044 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:32:48,044 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:32:48,045 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:32:48,050 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:32:48,287 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:32:49,289 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:32:50,051 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:32:53,246 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:32:53,247 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:32:53,247 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:32:54,073 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:32:54,073 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:32:54,074 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:32:54,079 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:32:54,292 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:32:55,115 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:32:55,295 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:32:57,701 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:32:57,702 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:32:57,702 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:32:57,708 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:32:58,297 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:32:59,300 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:33:00,708 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:33:03,199 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:33:03,200 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:33:03,201 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:33:03,206 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:33:03,303 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:33:03,425 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 19:33:05,307 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:33:06,426 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:33:08,247 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:33:08,247 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:33:08,247 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:33:11,225 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:33:11,225 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:33:11,225 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:33:11,231 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:33:11,311 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:33:12,267 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:33:13,314 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:33:17,267 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:33:18,272 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:33:18,273 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:33:18,274 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:33:18,279 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:33:18,317 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:33:19,320 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:33:22,022 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:33:22,023 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:33:22,024 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:33:22,029 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:33:22,322 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:33:23,029 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:33:23,247 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:33:23,247 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:33:23,247 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:33:23,324 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:33:27,396 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:33:27,397 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:33:27,397 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:33:27,402 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:33:28,329 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:33:28,439 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:33:29,332 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:33:32,364 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:33:32,365 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:33:32,365 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:33:32,371 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:33:33,336 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:33:33,336 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:33:33,431 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 19:33:34,433 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:33:36,077 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:33:36,077 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:33:36,078 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:33:36,083 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:33:36,338 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:33:37,341 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:33:38,247 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:33:38,247 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:33:38,247 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:33:39,496 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:33:39,802 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:33:39,803 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:33:39,803 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:33:39,808 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:33:40,343 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:33:41,346 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:33:44,847 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:33:45,681 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:33:45,682 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:33:45,682 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:33:45,687 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:33:46,349 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:33:47,352 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:33:49,778 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:33:49,779 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:33:49,779 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:33:49,784 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:33:50,355 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:33:50,784 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:33:51,358 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:33:53,247 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:33:53,247 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:33:53,247 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:33:53,519 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:33:53,525 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:33:53,526 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:33:53,531 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:33:54,360 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:33:55,362 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:33:56,531 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:33:57,816 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:33:57,816 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:33:57,817 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:33:57,822 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:33:58,364 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:33:59,367 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:34:01,315 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:34:01,316 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:34:01,316 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:34:01,321 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:34:01,369 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:34:02,322 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:34:03,372 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:34:03,430 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 19:34:06,682 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:34:06,683 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:34:06,683 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:34:06,688 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:34:07,374 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:34:07,689 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:34:08,247 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:34:08,247 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:34:08,247 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:34:09,378 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:34:13,460 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:34:13,521 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:34:13,522 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:34:13,522 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:34:13,527 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:34:14,381 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:34:15,383 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:34:17,261 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:34:17,262 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:34:17,263 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:34:17,268 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:34:17,385 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:34:19,268 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:34:19,388 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:34:22,294 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:34:22,294 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:34:22,295 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:34:22,300 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:34:22,390 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:34:23,247 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:34:23,247 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:34:23,248 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:34:23,392 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:34:24,451 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:34:26,030 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:34:26,031 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:34:26,031 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:34:26,036 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:34:26,394 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:34:27,398 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:34:30,075 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:34:30,173 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:34:30,174 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:34:30,174 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:34:30,179 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:34:30,400 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:34:31,403 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:34:33,435 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 19:34:34,276 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:34:34,277 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:34:34,277 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:34:34,282 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:34:34,404 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:34:35,283 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:34:35,405 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:34:38,247 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:34:38,247 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:34:38,248 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:34:39,135 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:34:39,136 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:34:39,136 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:34:39,141 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:34:39,408 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:34:41,179 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:34:41,411 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:34:43,012 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:34:43,012 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:34:43,013 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:34:43,018 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:34:43,412 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:34:45,415 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:34:46,766 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:34:46,767 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:34:46,767 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:34:46,767 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:34:46,772 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:34:47,417 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:34:49,422 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:34:50,710 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:34:50,711 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:34:50,711 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:34:50,716 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:34:51,423 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:34:52,717 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:34:53,247 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:34:53,247 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:34:53,248 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:34:53,426 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:34:57,555 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:34:57,556 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:34:57,557 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:34:57,561 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:34:58,430 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:34:58,599 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:34:59,432 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:35:01,439 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:35:01,440 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:35:01,440 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:35:01,445 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:35:02,435 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:35:03,434 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 19:35:03,436 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:35:04,436 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:35:07,605 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:35:07,606 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:35:07,606 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:35:07,611 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:35:08,247 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:35:08,247 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:35:08,248 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:35:08,440 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:35:09,442 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:35:09,527 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:35:12,421 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:35:12,422 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:35:12,422 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:35:12,427 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:35:12,444 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:35:13,447 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:35:15,463 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:35:17,304 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:35:17,304 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:35:17,305 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:35:17,310 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:35:17,449 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:35:19,453 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:35:21,310 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:35:23,247 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:35:23,247 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:35:23,248 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:35:25,577 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:35:25,578 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:35:25,578 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:35:25,584 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:35:26,457 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:35:26,623 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:35:27,460 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:35:29,806 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:35:29,807 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:35:29,807 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:35:29,813 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:35:30,462 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:35:31,465 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:35:31,814 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:35:33,284 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:35:33,285 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:35:33,285 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:35:33,291 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:35:33,439 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 19:35:33,466 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:35:35,470 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:35:37,441 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:35:37,463 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:35:37,464 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:35:37,464 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:35:37,470 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:35:37,471 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:35:38,247 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:35:38,248 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:35:38,248 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:35:39,474 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:35:42,234 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:35:42,235 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:35:42,235 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:35:42,241 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:35:42,476 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:35:43,279 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:35:43,478 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:35:45,789 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:35:45,790 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:35:45,790 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:35:45,795 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:35:46,481 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:35:47,483 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:35:48,796 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:35:49,156 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:35:49,157 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:35:49,157 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:35:49,162 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:35:49,486 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:35:51,489 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:35:52,176 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:35:52,177 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:35:52,177 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:35:52,183 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:35:52,490 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:35:53,248 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:35:53,248 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:35:53,249 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:35:53,492 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:35:54,539 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:35:56,474 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:35:56,475 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:35:56,475 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:35:56,480 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:35:56,494 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:35:57,497 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:36:00,112 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:36:00,113 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:36:00,113 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:36:00,113 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:36:00,118 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:36:00,499 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:36:01,501 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:36:03,439 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 19:36:03,896 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:36:03,897 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:36:03,897 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:36:03,902 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:36:04,503 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:36:05,506 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:36:05,903 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:36:08,248 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:36:08,248 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:36:08,248 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:36:09,860 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:36:09,860 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:36:09,861 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:36:09,865 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:36:10,509 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:36:11,511 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:36:11,903 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:36:14,420 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:36:14,421 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:36:14,421 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:36:14,427 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:36:14,513 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:36:15,516 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:36:17,427 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:36:18,419 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:36:18,420 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:36:18,421 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:36:18,426 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:36:18,517 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:36:19,520 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:36:23,248 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:36:23,248 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:36:23,248 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:36:23,532 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:36:23,683 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:36:23,684 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:36:23,684 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:36:23,689 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:36:24,523 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:36:25,526 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:36:28,503 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:36:28,504 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:36:28,504 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:36:28,509 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:36:28,529 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:36:29,510 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:36:29,531 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:36:32,901 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:36:32,902 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:36:32,902 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:36:32,907 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:36:33,444 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 19:36:33,534 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:36:35,446 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:36:35,537 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:36:38,248 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:36:38,248 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:36:38,248 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:36:38,684 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:36:38,685 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:36:38,685 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:36:38,690 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:36:39,540 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:36:40,727 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:36:41,543 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:36:45,727 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:36:47,916 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:36:47,917 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:36:47,918 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:36:47,923 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:36:48,548 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:36:49,552 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:36:50,923 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:36:51,596 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:36:51,597 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:36:51,597 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:36:51,602 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:36:52,554 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:36:53,248 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:36:53,248 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:36:53,248 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:36:53,556 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:36:55,067 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:36:55,068 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:36:55,068 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:36:55,074 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:36:55,557 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:36:56,111 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:36:57,561 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:37:01,111 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:37:02,346 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:37:02,347 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:37:02,347 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:37:02,352 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:37:02,564 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:37:03,443 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 19:37:03,567 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:37:06,114 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:37:06,115 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:37:06,115 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:37:06,115 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:37:06,120 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:37:06,569 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:37:07,571 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:37:08,248 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:37:08,248 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:37:08,248 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:37:09,883 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:37:09,884 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:37:09,884 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:37:09,889 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:37:10,573 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:37:11,576 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:37:11,927 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:37:13,711 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:37:13,712 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:37:13,712 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:37:13,717 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:37:14,578 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:37:15,580 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:37:16,946 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:37:16,947 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:37:16,947 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:37:16,947 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:37:16,952 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:37:17,582 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:37:19,585 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:37:20,863 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:37:20,863 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:37:20,864 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:37:20,869 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:37:21,586 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:37:22,869 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:37:23,248 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:37:23,248 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:37:23,250 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:37:23,589 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:37:25,079 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:37:25,080 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:37:25,080 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:37:25,085 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:37:25,591 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:37:27,595 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:37:28,123 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:37:30,877 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:37:30,878 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:37:30,878 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:37:30,883 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:37:31,598 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:37:33,445 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 19:37:33,446 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:37:33,601 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:37:35,792 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:37:35,793 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:37:35,793 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:37:35,798 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:37:36,603 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:37:37,606 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:37:38,248 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:37:38,248 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:37:38,249 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:37:38,461 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:37:39,908 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:37:39,913 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:37:39,913 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:37:39,918 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:37:40,608 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:37:41,610 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:37:43,850 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:37:43,851 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:37:43,851 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:37:43,852 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:37:43,856 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:37:44,612 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:37:45,615 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:37:47,747 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:37:47,747 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:37:47,748 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:37:47,753 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:37:48,617 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:37:49,621 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:37:49,754 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:37:51,282 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:37:51,283 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:37:51,283 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:37:51,289 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:37:51,622 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:37:53,248 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:37:53,248 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:37:53,249 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:37:53,625 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:37:55,564 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:37:56,146 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:37:56,147 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:37:56,147 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:37:56,152 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:37:56,627 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:37:57,630 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:37:59,290 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:37:59,291 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:37:59,291 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:37:59,296 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:37:59,631 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:38:01,297 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:38:01,635 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:38:03,447 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 19:38:06,448 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:38:07,833 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:38:07,834 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:38:07,835 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:38:07,840 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:38:08,248 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:38:08,248 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:38:08,249 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:38:08,639 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:38:09,642 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:38:11,461 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:38:11,690 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:38:11,691 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:38:11,691 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:38:11,697 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:38:12,644 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:38:13,646 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:38:15,061 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:38:15,062 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:38:15,062 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:38:15,067 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:38:15,647 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:38:17,068 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:38:17,651 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:38:19,274 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:38:19,275 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:38:19,275 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:38:19,281 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:38:19,652 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:38:21,655 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:38:22,282 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:38:23,010 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:38:23,011 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:38:23,011 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:38:23,016 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:38:23,249 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:38:23,249 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:38:23,249 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:38:23,656 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:38:25,660 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:38:26,463 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:38:26,464 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:38:26,464 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:38:26,469 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:38:26,660 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:38:27,507 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:38:27,664 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:38:30,846 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:38:30,847 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:38:30,848 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:38:30,853 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:38:31,667 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:38:32,854 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:38:33,449 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 19:38:33,670 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:38:35,012 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:38:35,012 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:38:35,012 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:38:35,018 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:38:35,672 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:38:37,675 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:38:38,018 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:38:38,248 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:38:38,249 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:38:38,249 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:38:41,302 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:38:41,302 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:38:41,302 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:38:41,307 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:38:41,677 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:38:43,347 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:38:43,680 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:38:44,935 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:38:44,935 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:38:44,936 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:38:44,941 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:38:45,682 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:38:47,685 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:38:48,942 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:38:49,976 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:38:49,977 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:38:49,977 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:38:49,982 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:38:50,688 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:38:51,691 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:38:53,248 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:38:53,249 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:38:53,249 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:38:53,654 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:38:53,654 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:38:53,655 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:38:53,660 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:38:53,692 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:38:54,699 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:38:55,695 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:38:57,193 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:38:57,194 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:38:57,194 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:38:57,199 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:38:57,697 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:38:59,700 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:39:00,200 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:39:00,699 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:39:00,700 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:39:00,700 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:39:00,705 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:39:01,704 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:39:01,704 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:39:03,451 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 19:39:05,147 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:39:05,148 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:39:05,148 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:39:05,153 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:39:05,706 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:39:06,154 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:39:07,710 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:39:08,249 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:39:08,249 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:39:08,249 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:39:10,709 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:39:10,710 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:39:10,710 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:39:10,715 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:39:11,714 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:39:11,715 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:39:11,751 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:39:14,849 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:39:14,850 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:39:14,850 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:39:14,856 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:39:15,719 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:39:15,719 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:39:16,856 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:39:20,920 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:39:20,921 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:39:20,921 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:39:20,927 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:39:21,723 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:39:21,927 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:39:23,249 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:39:23,249 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:39:23,249 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:39:23,726 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:39:25,874 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:39:25,874 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:39:25,875 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:39:25,879 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:39:26,728 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:39:27,731 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:39:27,919 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:39:31,073 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:39:31,074 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:39:31,075 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:39:31,079 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:39:31,734 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:39:33,080 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:39:33,453 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 19:39:33,738 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:39:35,541 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:39:35,542 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:39:35,542 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:39:35,547 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:39:35,740 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:39:37,743 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:39:38,249 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:39:38,249 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:39:38,249 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:39:38,515 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:39:39,067 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:39:39,068 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:39:39,068 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:39:39,073 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:39:39,744 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:39:41,748 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:39:42,385 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:39:42,386 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:39:42,386 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:39:42,391 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:39:42,748 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:39:43,751 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:39:44,392 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:39:49,393 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:39:50,734 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:39:50,735 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:39:50,735 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:39:50,740 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:39:50,757 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:39:51,759 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:39:53,249 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:39:53,249 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:39:53,249 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:39:54,115 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:39:54,115 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:39:54,116 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:39:54,121 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:39:54,761 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:39:55,159 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:39:55,764 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:39:58,182 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:39:58,183 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:39:58,183 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:39:58,188 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:39:58,766 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:39:59,768 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:40:00,189 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:40:03,459 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 19:40:03,665 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:40:03,666 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:40:03,666 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:40:03,671 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:40:03,771 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:40:05,672 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:40:05,774 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:40:08,221 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:40:08,221 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:40:08,222 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:40:08,227 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:40:08,249 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:40:08,249 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:40:08,251 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:40:08,776 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:40:09,779 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:40:11,545 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:40:14,235 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:40:14,236 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:40:14,236 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:40:14,241 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:40:14,782 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:40:15,784 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:40:17,279 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:40:18,961 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:40:18,962 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:40:18,962 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:40:18,967 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:40:19,787 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:40:21,790 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:40:22,968 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:40:23,249 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:40:23,249 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:40:23,250 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:40:26,044 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:40:26,045 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:40:26,045 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:40:26,050 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:40:26,793 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:40:27,796 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:40:28,087 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:40:29,796 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:40:29,797 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:40:29,798 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:40:29,803 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:40:30,799 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:40:31,802 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:40:33,458 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 19:40:33,459 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:40:33,743 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:40:33,744 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:40:33,744 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:40:33,749 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:40:33,803 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:40:35,806 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:40:37,775 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:40:37,776 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:40:37,776 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:40:37,781 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:40:37,807 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:40:38,249 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:40:38,250 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:40:38,251 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:40:38,467 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:40:39,811 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:40:43,468 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:40:46,309 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:40:46,310 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:40:46,311 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:40:46,316 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:40:46,816 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:40:47,818 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:40:49,355 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:40:49,625 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:40:49,626 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:40:49,626 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:40:49,631 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:40:49,821 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:40:51,824 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:40:53,184 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:40:53,185 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:40:53,186 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:40:53,191 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:40:53,249 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:40:53,249 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:40:53,250 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:40:53,825 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:40:54,496 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:40:55,828 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:40:57,997 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:40:57,998 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:40:57,998 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:40:58,003 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:40:58,830 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:40:59,833 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:41:00,043 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:41:02,092 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:41:02,093 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:41:02,093 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:41:02,098 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:41:02,836 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:41:03,463 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 19:41:03,838 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:41:05,465 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:41:05,784 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:41:05,784 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:41:05,785 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:41:05,790 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:41:05,839 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:41:07,843 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:41:08,249 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:41:08,250 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:41:08,250 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:41:09,728 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:41:09,729 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:41:09,730 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:41:09,735 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:41:09,844 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:41:10,771 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:41:11,847 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:41:13,427 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:41:13,428 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:41:13,428 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:41:13,433 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:41:13,849 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:41:15,852 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:41:16,434 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:41:17,116 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:41:17,117 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:41:17,118 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:41:17,123 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:41:17,853 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:41:19,857 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:41:20,638 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:41:20,639 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:41:20,639 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:41:20,645 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:41:20,857 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:41:21,645 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:41:21,860 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:41:23,249 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:41:23,250 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:41:23,250 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:41:24,293 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:41:24,293 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:41:24,294 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:41:24,299 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:41:24,862 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:41:25,863 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:41:27,335 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:41:27,911 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:41:27,911 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:41:27,912 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:41:27,917 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:41:28,866 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:41:29,869 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:41:32,918 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:41:33,462 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 19:41:36,065 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:41:36,066 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:41:36,066 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:41:36,082 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:41:36,873 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:41:37,875 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:41:38,083 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:41:38,249 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:41:38,250 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:41:38,250 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:41:40,227 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:41:40,227 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:41:40,228 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:41:40,233 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:41:40,877 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:41:41,880 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:41:43,271 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:41:44,439 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:41:44,440 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:41:44,441 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:41:44,446 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:41:44,882 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:41:45,885 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:41:48,326 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:41:48,327 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:41:48,327 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:41:48,328 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:41:48,332 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:41:48,887 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:41:49,891 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:41:52,621 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:41:52,622 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:41:52,622 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:41:52,628 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:41:52,893 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:41:53,250 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:41:53,250 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:41:53,250 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:41:53,451 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:41:53,896 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:41:57,488 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:41:57,489 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:41:57,490 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:41:57,495 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:41:57,898 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:41:58,531 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:41:59,901 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:42:00,626 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:42:00,627 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:42:00,627 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:42:00,632 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:42:00,902 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:42:01,905 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:42:03,468 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 19:42:04,469 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:42:05,543 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:42:05,544 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:42:05,544 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:42:05,549 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:42:05,907 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:42:07,911 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:42:08,250 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:42:08,250 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:42:08,250 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:42:09,509 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:42:13,939 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:42:13,940 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:42:13,940 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:42:13,945 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:42:14,915 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:42:14,983 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:42:15,917 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:42:19,793 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:42:19,794 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:42:19,795 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:42:19,800 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:42:19,920 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:42:20,800 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:42:21,923 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:42:23,250 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:42:23,250 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:42:23,250 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:42:23,739 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:42:23,740 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:42:23,740 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:42:23,745 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:42:23,924 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:42:25,926 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:42:26,783 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:42:27,249 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:42:27,250 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:42:27,250 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:42:27,255 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:42:27,928 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:42:29,931 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:42:32,256 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:42:32,873 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:42:32,874 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:42:32,874 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:42:32,879 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:42:32,933 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:42:33,467 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 19:42:33,936 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:42:36,585 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:42:36,586 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:42:36,587 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:42:36,592 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:42:36,938 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:42:37,592 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:42:37,940 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:42:38,250 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:42:38,250 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:42:38,250 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:42:40,420 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:42:40,420 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:42:40,421 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:42:40,426 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:42:40,942 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:42:41,945 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:42:43,463 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:42:48,464 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:42:49,450 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:42:49,450 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:42:49,451 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:42:49,455 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:42:49,951 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:42:51,955 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:42:52,620 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:42:52,621 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:42:52,621 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:42:52,626 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:42:52,956 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:42:53,250 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:42:53,250 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:42:53,251 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:42:53,500 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:42:53,958 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:42:57,663 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:42:57,663 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:42:57,663 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:42:57,668 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:42:57,961 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:42:58,707 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:42:59,965 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:43:03,472 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 19:43:04,474 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:43:06,012 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:43:06,013 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:43:06,013 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:43:06,018 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:43:06,973 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:43:07,976 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:43:08,250 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:43:08,250 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:43:08,250 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:43:09,547 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:43:11,617 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:43:11,618 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:43:11,618 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:43:11,623 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:43:11,979 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:43:13,984 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:43:14,659 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:43:15,198 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:43:15,199 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:43:15,199 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:43:15,204 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:43:15,985 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:43:17,990 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:43:19,438 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:43:19,439 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:43:19,439 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:43:19,444 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:43:19,991 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:43:20,445 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:43:21,996 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:43:23,250 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:43:23,250 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:43:23,251 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:43:25,481 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:43:26,239 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:43:26,240 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:43:26,240 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:43:26,245 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:43:27,002 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:43:28,006 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:43:31,283 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:43:33,471 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 19:43:34,473 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:43:34,474 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:43:34,474 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:43:34,479 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:43:35,014 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:43:36,017 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:43:36,480 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:43:38,250 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:43:38,250 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:43:38,251 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:43:39,380 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:43:39,380 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:43:39,381 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:43:39,385 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:43:40,020 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:43:42,025 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:43:42,423 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:43:42,948 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:43:42,949 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:43:42,949 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:43:42,954 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:43:43,026 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:43:44,028 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:43:47,955 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:43:49,033 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:43:49,034 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:43:49,035 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:43:49,040 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:43:50,038 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:43:50,038 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:43:53,041 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:43:53,056 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:43:53,057 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:43:53,057 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:43:53,062 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:43:53,250 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:43:53,251 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:43:53,251 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:43:54,042 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:43:54,043 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:43:56,290 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:43:56,291 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:43:56,291 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:43:56,296 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:43:57,045 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:43:58,047 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:43:58,335 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:44:01,613 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:44:01,614 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:44:01,614 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:44:01,619 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:44:02,052 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:44:03,473 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 19:44:03,475 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:44:04,057 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:44:07,250 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:44:07,254 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:44:07,255 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:44:07,260 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:44:08,061 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:44:08,250 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:44:08,251 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:44:08,251 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:44:08,504 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:44:10,065 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:44:13,108 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:44:13,108 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:44:13,109 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:44:13,114 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:44:14,071 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:44:14,072 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:44:14,151 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:44:17,234 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:44:17,235 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:44:17,235 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:44:17,240 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:44:18,076 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:44:18,077 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:44:19,241 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:44:23,250 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:44:23,251 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:44:23,251 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:44:24,454 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:44:25,658 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:44:25,659 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:44:25,659 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:44:25,664 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:44:26,086 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:44:28,091 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:44:29,703 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:44:33,476 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 19:44:34,372 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:44:34,372 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:44:34,373 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:44:34,378 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:44:35,100 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:44:35,378 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:44:36,102 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:44:38,250 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:44:38,251 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:44:38,251 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:44:39,193 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:44:39,193 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:44:39,194 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:44:39,199 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:44:40,107 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:44:40,108 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:44:41,235 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:44:43,196 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:44:43,197 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:44:43,198 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:44:43,203 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:44:44,113 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:44:44,114 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:44:47,203 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:44:48,581 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:44:48,582 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:44:48,582 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:44:48,587 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:44:49,119 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:44:50,123 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:44:52,588 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:44:52,704 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:44:52,705 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:44:52,705 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:44:52,710 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:44:53,125 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:44:53,251 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:44:53,251 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:44:53,251 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:44:54,128 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:44:58,122 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:44:58,122 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:44:58,123 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:44:58,123 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:44:58,128 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:44:58,131 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:45:00,135 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:45:01,424 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:45:01,425 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:45:01,426 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:45:01,430 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:45:02,137 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:45:03,431 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:45:03,478 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 19:45:04,142 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:45:08,251 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:45:08,251 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:45:08,251 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:45:08,467 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:45:09,107 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:45:09,108 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:45:09,109 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:45:09,113 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:45:09,148 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:45:10,150 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:45:14,114 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:45:14,168 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:45:14,169 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:45:14,169 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:45:14,174 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:45:15,157 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:45:16,159 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:45:18,132 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:45:18,133 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:45:18,134 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:45:18,139 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:45:18,161 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:45:19,140 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:45:20,164 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:45:21,343 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:45:21,344 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:45:21,344 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:45:21,349 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:45:22,166 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:45:23,251 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:45:23,251 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:45:23,251 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:45:24,170 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:45:24,464 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:45:24,739 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:45:24,744 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:45:24,744 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:45:24,749 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:45:25,171 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:45:26,173 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:45:29,783 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:45:30,686 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:45:30,687 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:45:30,687 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:45:30,692 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:45:31,180 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:45:32,182 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:45:33,480 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 19:45:35,481 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:45:36,546 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:45:36,547 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:45:36,547 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:45:36,552 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:45:37,188 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:45:38,191 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:45:38,251 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:45:38,251 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:45:38,251 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:45:40,448 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:45:40,449 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:45:40,449 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:45:40,454 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:45:40,491 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:45:41,194 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:45:42,197 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:45:45,491 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:45:46,026 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:45:46,027 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:45:46,028 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:45:46,033 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:45:46,201 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:45:48,205 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:45:50,898 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:45:50,899 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:45:50,899 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:45:50,899 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:45:50,904 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:45:51,209 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:45:52,211 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:45:53,251 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:45:53,251 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:45:53,252 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:45:56,460 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:45:57,154 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:45:57,155 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:45:57,155 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:45:57,160 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:45:57,216 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:45:58,219 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:46:01,526 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:46:01,527 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:46:01,527 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:46:01,527 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:46:01,532 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:46:02,223 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:46:03,482 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 19:46:04,228 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:46:04,854 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:46:04,855 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:46:04,855 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:46:04,860 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:46:05,229 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:46:06,231 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:46:06,861 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:46:08,251 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:46:08,251 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:46:08,251 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:46:12,696 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:46:13,313 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:46:13,313 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:46:13,314 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:46:13,319 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:46:14,242 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:46:14,243 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:46:18,354 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:46:18,355 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:46:18,356 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:46:18,356 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:46:18,361 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:46:19,248 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:46:20,250 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:46:23,058 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:46:23,059 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:46:23,059 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:46:23,064 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:46:23,253 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:46:23,253 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:46:23,253 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:46:23,255 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:46:23,459 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:46:24,257 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:46:27,437 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:46:27,438 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:46:27,438 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:46:27,443 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:46:28,261 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:46:28,478 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:46:30,266 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:46:31,474 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:46:31,475 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:46:31,476 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:46:31,481 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:46:32,267 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:46:33,481 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:46:33,488 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 19:46:34,271 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:46:35,702 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:46:35,703 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:46:35,703 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:46:35,708 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:46:36,272 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:46:38,251 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:46:38,251 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:46:38,252 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:46:38,276 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:46:38,537 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:46:39,199 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:46:39,200 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:46:39,200 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:46:39,205 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:46:39,277 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:46:40,281 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:46:44,242 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:46:45,617 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:46:45,618 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:46:45,619 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:46:45,624 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:46:46,286 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:46:48,290 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:46:49,624 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:46:50,235 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:46:50,236 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:46:50,236 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:46:50,241 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:46:50,293 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:46:52,296 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:46:53,251 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:46:53,252 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:46:53,252 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:46:54,139 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:46:54,139 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:46:54,140 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:46:54,145 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:46:54,298 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:46:55,182 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:46:56,302 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:46:57,945 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:46:57,945 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:46:57,946 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:46:57,950 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:46:58,303 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:47:00,306 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:47:00,951 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:47:02,560 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:47:02,561 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:47:02,561 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:47:02,566 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:47:03,311 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:47:03,486 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 19:47:04,313 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:47:06,488 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:47:06,580 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:47:06,581 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:47:06,581 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:47:06,586 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:47:07,316 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:47:08,251 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:47:08,252 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:47:08,252 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:47:08,319 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:47:12,062 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:47:12,063 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:47:12,063 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:47:12,063 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:47:12,068 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:47:12,323 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:47:14,327 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:47:16,940 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:47:16,941 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:47:16,941 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:47:16,947 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:47:17,331 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:47:17,947 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:47:18,333 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:47:21,117 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:47:21,118 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:47:21,118 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:47:21,123 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:47:21,336 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:47:22,339 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:47:23,124 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:47:23,251 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:47:23,252 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:47:23,252 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:47:25,675 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:47:25,676 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:47:25,676 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:47:25,681 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:47:26,342 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:47:28,346 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:47:28,718 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:47:32,276 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:47:32,276 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:47:32,276 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:47:32,281 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:47:32,351 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:47:33,492 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 19:47:34,356 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:47:34,493 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:47:35,763 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:47:35,764 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:47:35,764 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:47:35,769 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:47:36,357 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:47:38,252 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:47:38,252 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:47:38,252 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:47:38,361 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:47:39,534 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:47:39,534 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:47:39,535 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:47:39,535 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:47:39,540 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:47:40,363 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:47:42,367 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:47:43,056 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:47:43,057 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:47:43,057 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:47:43,062 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:47:43,368 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:47:44,370 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:47:45,062 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:47:47,197 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:47:47,198 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:47:47,198 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:47:47,203 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:47:47,373 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:47:48,376 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:47:50,203 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:47:51,107 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:47:51,108 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:47:51,108 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:47:51,113 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:47:51,379 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:47:52,382 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:47:53,252 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:47:53,252 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:47:53,252 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:47:55,306 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:47:55,307 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:47:55,307 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:47:55,307 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:47:55,312 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:47:55,385 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:47:56,387 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:47:59,015 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:47:59,016 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:47:59,016 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:47:59,021 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:47:59,391 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:48:00,393 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:48:01,022 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:48:03,491 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 19:48:03,597 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:48:03,597 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:48:03,598 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:48:03,603 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:48:04,397 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:48:06,402 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:48:06,603 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:48:08,252 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:48:08,252 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:48:08,252 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:48:09,917 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:48:09,918 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:48:09,918 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:48:09,923 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:48:10,406 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:48:11,958 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:48:12,410 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:48:13,915 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:48:13,915 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:48:13,916 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:48:13,921 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:48:14,412 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:48:16,416 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:48:17,922 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:48:19,210 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:48:19,211 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:48:19,211 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:48:19,216 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:48:19,419 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:48:20,422 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:48:22,771 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:48:22,772 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:48:22,772 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:48:22,777 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:48:23,252 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:48:23,252 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:48:23,252 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:48:23,424 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:48:23,466 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:48:24,427 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:48:26,487 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:48:26,487 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:48:26,488 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:48:26,492 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:48:27,430 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:48:28,432 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:48:28,530 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:48:30,135 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:48:30,136 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:48:30,137 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:48:30,141 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:48:30,435 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:48:32,439 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:48:33,496 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 19:48:34,498 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:48:35,777 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:48:35,778 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:48:35,778 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:48:35,783 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:48:36,443 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:48:38,252 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:48:38,252 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:48:38,252 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:48:38,448 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:48:40,345 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:48:40,346 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:48:40,346 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:48:40,346 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:48:40,351 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:48:40,449 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:48:42,453 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:48:44,436 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:48:44,437 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:48:44,438 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:48:44,442 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:48:44,454 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:48:45,443 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:48:46,458 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:48:47,961 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:48:47,961 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:48:47,962 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:48:47,967 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:48:48,460 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:48:50,465 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:48:50,967 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:48:53,014 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:48:53,015 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:48:53,015 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:48:53,020 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:48:53,252 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:48:53,252 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:48:53,252 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:48:53,467 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:48:54,470 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:48:56,502 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:48:57,369 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:48:57,370 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:48:57,370 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:48:57,375 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:48:57,472 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:48:58,473 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:49:02,410 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:49:03,018 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:49:03,019 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:49:03,019 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:49:03,024 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:49:03,477 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:49:03,496 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 19:49:04,479 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:49:06,821 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:49:06,822 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:49:06,822 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:49:06,827 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:49:07,481 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:49:07,827 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:49:08,252 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:49:08,252 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:49:08,253 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:49:08,484 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:49:10,620 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:49:10,620 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:49:10,621 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:49:10,626 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:49:11,486 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:49:12,489 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:49:13,662 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:49:14,580 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:49:14,580 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:49:14,581 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:49:14,586 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:49:15,490 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:49:16,493 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:49:18,235 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:49:18,235 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:49:18,236 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:49:18,240 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:49:18,494 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:49:19,241 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:49:20,498 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:49:21,319 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:49:21,319 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:49:21,320 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:49:21,325 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:49:21,498 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:49:22,501 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:49:23,252 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:49:23,252 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:49:23,253 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:49:24,481 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:49:24,860 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:49:24,861 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:49:24,861 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:49:24,866 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:49:25,503 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:49:26,503 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:49:28,503 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:49:28,503 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:49:28,503 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:49:28,510 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:49:29,506 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:49:29,510 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:49:30,509 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:49:32,242 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:49:32,243 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:49:32,243 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:49:32,248 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:49:32,510 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:49:33,501 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 19:49:34,513 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:49:35,503 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:49:36,572 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:49:36,573 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:49:36,573 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:49:36,578 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:49:37,515 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:49:38,252 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:49:38,252 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:49:38,253 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:49:38,518 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:49:40,506 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:49:40,525 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:49:40,525 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:49:40,526 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:49:40,531 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:49:41,520 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:49:42,523 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:49:45,086 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:49:45,087 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:49:45,087 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:49:45,092 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:49:45,525 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:49:46,092 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:49:46,527 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:49:51,093 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:49:51,138 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:49:51,139 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:49:51,139 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:49:51,144 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:49:51,532 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:49:52,535 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:49:53,252 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:49:53,252 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:49:53,253 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:49:54,721 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:49:54,722 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:49:54,722 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:49:54,727 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:49:55,537 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:49:56,537 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:49:56,766 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:49:58,114 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:49:58,115 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:49:58,115 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:49:58,120 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:49:58,539 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:50:00,542 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:50:01,884 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:50:01,885 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:50:01,886 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:50:01,886 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:50:01,891 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:50:02,544 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:50:03,500 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 19:50:04,547 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:50:05,604 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:50:05,605 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:50:05,605 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:50:05,611 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:50:06,551 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:50:06,551 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:50:07,611 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:50:08,252 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:50:08,253 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:50:08,253 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:50:08,879 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:50:08,880 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:50:08,880 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:50:08,885 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:50:09,552 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:50:10,555 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:50:12,922 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:50:13,630 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:50:13,631 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:50:13,631 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:50:13,636 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:50:14,560 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:50:14,560 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:50:18,637 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:50:21,896 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:50:21,897 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:50:21,897 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:50:21,903 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:50:22,565 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:50:23,253 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:50:23,253 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:50:23,253 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:50:24,548 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:50:24,569 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:50:25,971 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:50:25,971 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:50:25,971 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:50:25,977 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:50:26,570 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:50:28,574 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:50:29,251 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:50:29,252 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:50:29,252 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:50:29,257 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:50:29,575 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:50:30,257 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:50:30,578 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:50:33,506 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 19:50:35,507 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:50:37,634 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:50:37,634 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:50:37,635 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:50:37,640 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:50:38,253 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:50:38,253 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:50:38,253 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:50:38,585 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:50:38,585 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:50:40,547 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:50:42,587 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:50:42,588 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:50:42,588 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:50:42,593 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:50:43,589 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:50:44,591 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:50:45,630 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:50:45,659 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:50:45,660 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:50:45,660 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:50:45,665 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:50:46,594 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:50:46,595 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:50:49,751 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:50:49,752 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:50:49,752 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:50:49,757 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:50:50,599 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:50:50,758 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:50:52,602 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:50:53,253 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:50:53,253 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:50:53,253 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:50:53,297 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:50:53,452 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:50:53,452 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:50:53,457 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:50:53,602 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:50:54,605 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:50:56,458 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:50:56,765 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:50:56,766 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:50:56,766 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:50:56,771 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:50:57,607 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:50:58,609 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:51:01,772 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:51:03,504 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 19:51:04,016 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:51:04,016 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:51:04,017 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:51:04,022 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:51:04,614 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:51:06,617 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:51:07,022 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:51:08,253 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:51:08,253 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:51:08,253 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:51:08,348 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:51:08,500 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:51:08,500 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:51:08,506 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:51:08,618 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:51:10,622 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:51:12,506 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:51:12,891 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:51:12,892 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:51:12,892 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:51:12,897 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:51:13,623 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:51:14,626 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:51:16,181 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:51:16,182 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:51:16,182 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:51:16,187 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:51:16,627 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:51:18,188 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:51:18,630 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:51:20,429 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:51:20,430 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:51:20,430 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:51:20,435 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:51:20,632 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:51:22,635 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:51:23,253 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:51:23,253 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:51:23,253 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:51:23,541 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:51:24,216 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:51:24,216 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:51:24,216 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:51:24,221 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:51:24,636 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:51:26,640 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:51:27,689 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:51:27,690 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:51:27,691 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:51:27,696 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:51:28,642 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:51:28,642 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:51:28,696 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:51:31,476 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:51:31,477 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:51:31,477 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:51:31,482 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:51:31,645 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:51:32,647 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:51:33,510 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 19:51:34,512 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:51:36,016 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:51:36,017 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:51:36,017 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:51:36,022 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:51:36,650 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:51:38,253 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:51:38,253 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:51:38,254 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:51:38,654 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:51:39,469 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:51:39,470 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:51:39,470 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:51:39,475 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:51:39,514 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:51:39,654 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:51:40,657 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:51:42,790 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:51:42,791 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:51:42,791 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:51:42,796 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:51:43,659 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:51:44,661 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:51:44,796 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:51:46,410 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:51:46,411 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:51:46,412 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:51:46,417 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:51:46,663 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:51:48,666 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:51:50,162 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:51:50,162 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:51:50,163 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:51:50,163 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:51:50,167 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:51:50,669 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:51:52,672 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:51:53,253 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:51:53,253 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:51:53,254 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:51:55,487 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:51:56,372 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:51:56,373 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:51:56,373 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:51:56,378 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:51:56,675 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:51:58,678 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:52:01,088 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:52:01,089 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:52:01,089 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:52:01,089 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:52:01,094 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:52:01,680 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:52:02,683 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:52:03,508 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 19:52:05,159 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:52:05,159 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:52:05,160 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:52:05,164 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:52:05,685 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:52:06,165 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:52:06,692 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:52:08,253 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:52:08,253 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:52:08,254 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:52:09,208 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:52:09,209 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:52:09,209 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:52:09,214 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:52:09,694 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:52:10,697 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:52:11,250 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:52:13,939 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:52:13,940 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:52:13,941 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:52:13,946 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:52:14,699 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:52:16,702 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:52:16,946 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:52:17,649 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:52:17,650 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:52:17,650 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:52:17,655 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:52:17,703 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:52:18,706 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:52:22,655 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:52:23,253 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:52:23,253 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:52:23,254 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:52:26,064 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:52:26,065 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:52:26,065 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:52:26,070 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:52:26,710 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:52:28,106 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:52:28,714 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:52:30,616 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:52:30,617 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:52:30,617 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:52:30,622 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:52:30,716 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:52:32,719 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:52:33,510 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 19:52:33,512 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:52:34,677 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:52:34,678 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:52:34,678 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:52:34,683 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:52:34,721 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:52:36,724 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:52:38,253 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:52:38,254 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:52:38,254 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:52:38,558 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:52:40,339 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:52:40,339 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:52:40,340 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:52:40,344 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:52:40,726 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:52:42,729 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:52:44,382 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:52:45,313 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:52:45,314 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:52:45,314 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:52:45,319 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:52:45,731 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:52:46,734 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:52:50,320 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:52:53,253 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:52:53,254 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:52:53,254 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:52:53,412 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:52:53,534 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:52:53,534 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:52:53,539 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:52:53,740 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:52:54,743 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:52:55,540 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:52:57,090 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:52:57,090 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:52:57,091 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:52:57,096 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:52:57,745 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:52:58,745 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:53:01,096 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:53:02,429 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:53:02,430 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:53:02,431 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:53:02,435 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:53:02,748 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:53:03,513 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 19:53:04,751 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:53:06,514 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:53:08,253 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:53:08,254 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:53:08,254 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:53:08,507 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:53:08,564 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:53:08,564 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:53:08,569 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:53:08,754 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:53:10,757 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:53:11,570 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:53:13,084 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:53:13,085 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:53:13,085 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:53:13,090 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:53:13,759 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:53:14,761 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:53:17,126 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:53:19,029 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:53:19,030 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:53:19,030 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:53:19,036 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:53:19,765 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:53:20,767 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:53:23,037 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:53:23,254 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:53:23,254 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:53:23,254 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:53:23,301 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:53:23,505 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:53:23,505 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:53:23,510 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:53:23,769 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:53:24,772 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:53:27,256 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:53:27,257 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:53:27,257 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:53:27,262 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:53:27,773 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:53:28,263 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:53:28,777 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:53:32,141 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:53:32,142 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:53:32,143 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:53:32,147 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:53:32,780 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:53:33,515 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 19:53:33,516 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:53:34,783 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:53:36,346 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:53:36,347 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:53:36,348 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:53:36,352 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:53:36,784 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:53:38,254 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:53:38,254 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:53:38,254 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:53:38,551 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:53:38,786 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:53:40,321 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:53:40,322 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:53:40,322 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:53:40,327 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:53:40,787 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:53:42,790 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:53:44,015 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:53:44,016 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:53:44,017 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:53:44,017 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:53:44,021 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:53:44,792 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:53:46,795 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:53:49,022 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:53:49,026 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:53:49,026 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:53:49,027 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:53:49,031 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:53:49,797 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:53:50,801 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:53:52,713 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:53:52,714 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:53:52,714 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:53:52,719 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:53:52,802 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:53:53,254 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:53:53,254 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:53:53,254 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:53:54,545 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:53:54,805 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:53:57,120 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:53:57,120 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:53:57,121 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:53:57,125 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:53:57,807 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:53:58,810 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:54:00,162 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:54:00,798 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:54:00,802 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:54:00,803 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:54:00,808 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:54:00,811 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:54:02,815 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:54:03,517 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 19:54:04,919 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:54:04,920 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:54:04,920 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:54:04,925 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:54:05,817 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:54:05,926 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:54:06,819 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:54:08,254 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:54:08,254 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:54:08,254 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:54:09,178 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:54:09,179 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:54:09,179 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:54:09,184 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:54:09,821 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:54:10,824 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:54:11,222 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:54:12,698 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:54:12,699 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:54:12,699 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:54:12,704 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:54:12,825 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:54:14,829 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:54:16,705 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:54:21,026 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:54:21,027 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:54:21,027 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:54:21,033 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:54:21,833 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:54:22,033 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:54:22,836 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:54:23,254 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:54:23,254 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:54:23,254 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:54:24,968 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:54:24,968 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:54:24,969 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:54:24,974 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:54:25,838 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:54:26,840 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:54:28,010 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:54:29,893 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:54:29,894 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:54:29,894 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:54:29,899 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:54:30,846 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:54:30,846 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:54:33,519 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 19:54:33,521 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:54:33,522 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:54:33,523 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:54:33,523 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:54:33,527 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:54:33,848 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:54:34,851 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:54:38,254 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:54:38,254 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:54:38,255 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:54:39,476 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:54:40,076 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:54:40,077 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:54:40,077 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:54:40,082 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:54:40,854 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:54:42,858 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:54:43,322 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:54:43,322 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:54:43,323 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:54:43,328 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:54:43,858 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:54:44,859 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:54:45,329 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:54:47,121 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:54:47,122 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:54:47,122 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:54:47,127 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:54:47,861 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:54:48,864 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:54:51,128 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:54:51,682 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:54:51,683 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:54:51,683 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:54:51,688 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:54:51,867 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:54:52,870 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:54:53,254 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:54:53,254 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:54:53,255 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:54:56,555 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:54:57,634 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:54:57,635 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:54:57,635 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:54:57,640 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:54:57,873 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:54:58,876 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:55:01,184 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:55:01,185 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:55:01,185 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:55:01,190 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:55:01,879 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:55:02,191 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:55:02,882 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:55:03,525 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 19:55:04,746 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:55:04,747 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:55:04,747 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:55:04,752 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:55:04,883 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:55:06,886 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:55:07,752 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:55:08,178 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:55:08,179 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:55:08,179 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:55:08,184 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:55:08,254 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:55:08,254 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:55:08,254 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:55:08,887 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:55:10,891 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:55:11,693 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:55:11,693 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:55:11,693 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:55:11,698 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:55:11,891 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:55:12,892 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:55:13,734 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:55:15,412 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:55:15,413 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:55:15,413 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:55:15,418 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:55:15,894 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:55:16,896 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:55:19,419 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:55:20,109 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:55:20,110 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:55:20,110 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:55:20,116 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:55:20,899 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:55:22,902 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:55:23,254 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:55:23,255 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:55:23,255 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:55:23,495 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:55:23,496 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:55:23,496 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:55:23,501 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:55:23,903 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:55:24,538 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:55:24,904 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:55:27,083 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:55:27,083 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:55:27,083 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:55:27,088 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:55:27,905 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:55:28,909 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:55:30,089 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:55:30,830 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:55:30,830 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:55:30,831 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:55:30,836 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:55:30,911 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:55:32,914 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:55:33,524 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 19:55:34,856 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:55:34,857 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:55:34,857 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:55:34,862 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:55:34,915 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:55:35,863 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:55:36,918 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:55:38,254 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:55:38,255 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:55:38,255 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:55:38,333 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:55:38,551 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:55:38,551 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:55:38,556 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:55:38,920 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:55:40,923 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:55:41,556 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:55:43,248 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:55:43,249 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:55:43,249 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:55:43,254 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:55:43,925 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:55:44,927 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:55:46,659 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:55:46,660 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:55:46,660 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:55:46,660 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:55:46,665 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:55:46,929 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:55:48,932 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:55:51,666 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:55:53,254 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:55:53,255 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:55:53,255 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:55:54,393 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:55:54,394 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:55:54,394 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:55:54,399 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:55:54,937 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:55:56,941 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:55:57,434 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:55:57,704 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:55:57,705 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:55:57,705 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:55:57,714 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:55:57,941 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:55:58,942 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:56:01,474 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:56:01,475 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:56:01,475 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:56:01,480 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:56:01,944 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:56:02,481 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:56:02,947 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:56:03,530 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 19:56:05,019 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:56:05,020 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:56:05,020 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:56:05,025 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:56:05,949 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:56:06,951 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:56:08,026 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:56:08,255 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:56:08,255 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:56:08,255 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:56:09,167 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:56:09,168 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:56:09,168 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:56:09,173 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:56:09,953 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:56:10,956 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:56:12,437 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:56:12,437 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:56:12,438 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:56:12,445 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:56:12,957 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:56:13,445 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:56:14,960 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:56:16,533 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:56:16,534 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:56:16,534 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:56:16,539 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:56:16,962 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:56:18,540 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:56:18,965 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:56:20,972 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:56:20,973 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:56:20,973 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:56:20,978 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:56:21,967 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:56:22,969 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:56:23,255 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:56:23,255 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:56:23,255 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:56:24,449 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:56:24,449 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:56:24,450 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:56:24,451 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:56:24,455 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:56:24,971 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:56:26,974 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:56:29,140 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:56:29,141 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:56:29,141 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:56:29,146 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:56:29,977 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:56:30,147 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:56:30,980 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:56:33,022 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:56:33,023 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:56:33,023 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:56:33,028 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:56:33,528 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 19:56:33,981 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:56:34,984 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:56:35,530 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:56:37,040 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:56:37,041 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:56:37,041 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:56:37,046 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:56:37,986 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:56:38,255 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:56:38,255 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:56:38,255 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:56:38,988 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:56:40,552 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:56:41,185 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:56:41,185 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:56:41,186 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:56:41,190 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:56:41,990 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:56:42,993 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:56:45,985 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:56:45,986 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:56:45,986 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:56:45,987 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:56:45,991 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:56:45,994 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:56:46,997 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:56:49,703 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:56:49,704 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:56:49,704 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:56:49,709 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:56:49,999 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:56:51,003 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:56:51,710 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:56:53,255 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:56:53,255 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:56:53,255 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:56:57,346 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:56:57,347 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:56:57,347 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:56:57,348 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:56:57,352 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:56:58,007 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:56:59,010 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:57:02,390 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:57:03,534 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 19:57:06,941 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:57:06,942 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:57:06,942 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:57:06,947 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:57:07,015 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:57:07,948 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:57:08,255 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:57:08,255 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:57:08,255 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:57:09,018 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:57:11,556 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:57:11,556 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:57:11,557 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:57:11,561 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:57:12,020 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:57:13,023 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:57:13,598 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:57:16,504 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:57:16,504 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:57:16,505 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:57:16,510 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:57:17,025 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:57:19,027 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:57:19,511 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:57:21,486 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:57:21,487 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:57:21,487 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:57:21,492 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:57:22,029 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:57:23,031 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:57:23,255 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:57:23,255 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:57:23,255 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:57:25,479 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:57:25,566 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:57:25,567 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:57:25,567 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:57:25,572 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:57:26,033 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:57:27,036 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:57:30,385 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:57:30,386 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:57:30,386 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:57:30,391 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:57:31,040 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:57:31,391 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:57:33,043 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:57:33,533 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 19:57:34,732 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:57:34,733 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:57:34,733 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:57:34,738 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:57:35,044 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:57:36,738 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:57:37,047 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:57:38,255 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:57:38,255 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:57:38,255 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:57:38,279 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:57:38,495 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:57:38,495 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:57:38,500 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:57:39,049 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:57:41,052 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:57:42,359 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:57:42,360 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:57:42,360 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:57:42,361 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:57:42,365 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:57:43,053 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:57:45,056 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:57:46,280 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:57:46,281 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:57:46,281 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:57:46,286 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:57:47,057 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:57:48,287 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:57:49,061 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:57:50,016 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:57:50,017 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:57:50,017 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:57:50,022 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:57:50,061 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:57:51,065 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:57:53,255 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:57:53,255 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:57:53,255 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:57:53,458 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:57:53,463 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:57:53,463 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:57:53,464 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:57:53,469 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:57:54,067 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:57:55,070 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:57:58,506 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:57:59,745 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:57:59,746 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:57:59,746 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:57:59,751 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:58:00,073 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:58:01,075 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:58:03,296 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:58:03,297 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:58:03,297 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:58:03,302 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:58:03,538 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 19:58:03,540 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:58:04,078 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:58:05,079 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:58:08,255 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:58:08,255 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:58:08,255 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:58:08,942 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:58:08,943 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:58:08,943 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:58:08,943 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:58:08,948 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:58:09,081 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:58:11,085 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:58:13,986 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:58:16,903 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:58:16,904 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:58:16,904 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:58:16,909 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:58:17,088 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:58:19,091 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:58:19,910 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:58:21,338 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:58:21,339 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:58:21,339 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:58:21,344 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:58:22,093 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:58:23,096 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:58:23,255 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:58:23,256 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:58:23,256 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:58:25,477 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:58:29,681 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:58:29,682 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:58:29,682 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:58:29,687 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:58:30,101 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:58:30,722 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:58:31,104 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:58:33,109 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:58:33,110 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:58:33,110 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:58:33,115 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:58:33,537 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 19:58:34,106 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:58:35,107 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:58:36,538 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:58:38,194 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:58:38,195 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:58:38,195 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:58:38,200 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:58:38,255 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:58:38,255 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:58:38,255 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:58:39,111 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:58:39,111 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:58:41,953 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:58:41,954 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:58:41,954 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:58:41,954 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:58:41,959 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:58:42,113 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:58:43,115 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:58:45,950 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:58:45,951 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:58:45,951 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:58:45,956 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:58:46,117 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:58:46,957 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:58:47,118 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:58:51,957 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:58:52,547 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:58:52,548 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:58:52,548 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:58:52,553 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:58:53,123 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:58:53,255 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:58:53,256 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:58:53,256 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:58:55,127 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:58:57,552 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:58:57,553 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:58:57,553 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:58:57,553 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:58:57,558 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:58:58,129 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:58:59,131 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:59:01,431 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:59:01,432 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:59:01,432 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:59:01,436 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:59:02,134 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:59:03,135 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:59:03,437 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:59:03,542 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 19:59:07,002 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:59:07,003 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:59:07,003 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:59:07,008 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:59:07,137 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:59:08,255 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:59:08,256 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:59:08,256 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:59:08,535 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:59:09,141 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:59:11,283 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:59:11,284 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:59:11,284 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:59:11,289 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:59:12,142 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:59:13,145 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:59:14,326 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:59:15,036 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:59:15,037 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:59:15,037 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:59:15,042 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:59:15,146 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:59:17,150 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:59:19,931 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:59:19,931 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:59:19,932 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:59:19,932 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:59:19,937 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:59:20,151 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:59:21,154 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:59:23,256 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:59:23,256 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:59:23,256 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:59:23,445 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:59:23,536 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:59:23,537 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:59:23,542 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:59:24,156 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:59:25,157 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:59:25,542 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:59:27,203 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:59:27,204 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:59:27,204 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:59:27,209 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:59:28,159 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:59:29,163 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:59:31,210 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:59:32,099 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:59:32,099 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:59:32,100 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:59:32,105 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:59:32,165 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:59:33,168 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:59:33,541 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 19:59:35,396 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:59:35,397 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:59:35,397 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:59:35,401 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:59:36,170 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:59:36,402 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:59:37,172 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:59:38,256 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:59:38,256 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:59:38,256 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:59:40,323 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:59:40,324 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:59:40,324 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:59:40,329 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:59:41,175 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:59:42,366 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:59:43,179 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:59:45,059 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:59:45,060 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:59:45,061 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:59:45,066 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:59:45,180 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:59:47,183 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:59:48,066 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:59:48,647 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:59:48,647 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:59:48,648 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:59:48,652 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:59:49,185 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:59:51,190 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:59:53,256 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 19:59:53,256 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 19:59:53,256 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 19:59:53,597 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 19:59:53,687 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:59:53,687 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:59:53,687 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:59:53,692 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:59:54,192 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:59:55,194 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:59:57,226 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 19:59:57,227 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 19:59:57,227 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 19:59:57,232 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 19:59:58,196 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 19:59:59,199 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 19:59:59,232 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:00:03,543 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 20:00:04,545 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:00:05,286 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:00:05,287 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:00:05,287 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:00:05,292 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:00:06,204 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:00:07,206 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:00:08,256 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:00:08,256 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:00:08,256 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:00:09,540 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:00:09,541 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:00:09,541 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:00:09,546 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:00:09,546 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:00:10,208 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:00:11,211 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:00:14,586 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:00:17,755 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:00:17,756 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:00:17,756 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:00:17,761 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:00:18,215 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:00:19,218 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:00:19,762 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:00:21,449 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:00:21,450 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:00:21,450 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:00:21,455 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:00:22,220 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:00:23,220 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:00:23,256 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:00:23,256 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:00:23,256 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:00:24,980 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:00:24,980 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:00:24,981 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:00:24,981 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:00:24,986 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:00:25,222 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:00:27,227 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:00:28,760 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:00:28,761 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:00:28,761 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:00:28,766 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:00:29,229 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:00:30,767 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:00:31,233 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:00:33,545 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 20:00:34,193 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:00:34,194 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:00:34,194 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:00:34,199 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:00:34,235 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:00:35,238 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:00:36,199 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:00:37,937 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:00:37,938 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:00:37,938 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:00:37,943 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:00:38,240 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:00:38,256 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:00:38,256 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:00:38,256 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:00:39,242 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:00:41,497 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:00:44,201 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:00:44,201 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:00:44,201 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:00:44,206 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:00:44,246 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:00:45,249 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:00:47,242 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:00:48,160 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:00:48,161 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:00:48,161 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:00:48,166 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:00:48,251 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:00:49,253 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:00:52,106 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:00:52,107 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:00:52,107 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:00:52,112 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:00:52,256 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:00:53,113 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:00:53,256 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:00:53,256 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:00:53,256 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:00:53,260 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:00:55,901 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:00:55,901 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:00:55,902 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:00:55,907 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:00:56,262 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:00:57,265 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:00:58,942 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:00:59,982 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:00:59,983 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:00:59,983 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:00:59,988 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:01:00,267 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:01:01,269 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:01:03,547 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 20:01:03,920 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:01:03,921 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:01:03,922 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:01:03,926 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:01:04,272 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:01:04,927 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:01:05,274 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:01:08,256 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:01:08,256 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:01:08,258 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:01:08,332 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:01:08,497 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:01:08,497 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:01:08,503 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:01:09,277 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:01:10,504 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:01:11,280 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:01:11,603 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:01:11,604 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:01:11,604 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:01:11,609 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:01:12,281 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:01:13,284 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:01:15,610 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:01:19,770 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:01:19,774 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:01:19,774 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:01:19,779 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:01:20,288 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:01:20,780 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:01:21,290 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:01:23,256 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:01:23,256 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:01:23,258 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:01:23,317 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:01:23,570 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:01:23,570 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:01:23,575 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:01:24,292 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:01:25,295 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:01:26,576 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:01:27,071 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:01:27,072 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:01:27,072 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:01:27,077 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:01:27,296 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:01:29,300 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:01:32,077 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:01:32,255 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:01:32,255 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:01:32,255 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:01:32,260 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:01:32,302 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:01:33,305 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:01:33,550 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 20:01:36,007 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:01:36,008 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:01:36,008 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:01:36,013 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:01:36,307 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:01:37,308 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:01:38,013 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:01:38,256 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:01:38,256 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:01:38,258 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:01:40,042 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:01:40,042 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:01:40,042 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:01:40,048 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:01:40,309 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:01:41,312 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:01:43,086 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:01:44,230 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:01:44,231 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:01:44,231 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:01:44,236 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:01:44,314 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:01:45,317 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:01:48,237 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:01:48,351 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:01:48,352 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:01:48,352 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:01:48,358 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:01:49,322 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:01:49,322 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:01:53,256 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:01:53,257 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:01:53,257 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:01:53,358 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:01:53,742 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:01:53,742 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:01:53,742 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:01:53,748 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:01:54,326 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:01:55,329 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:01:58,748 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:01:59,573 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:01:59,574 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:01:59,574 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:01:59,579 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:02:00,332 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:02:01,334 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:02:03,552 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 20:02:04,554 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:02:08,037 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:02:08,038 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:02:08,038 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:02:08,044 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:02:08,256 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:02:08,257 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:02:08,258 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:02:08,339 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:02:09,342 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:02:09,556 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:02:11,356 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:02:11,357 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:02:11,357 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:02:11,362 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:02:12,344 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:02:13,344 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:02:15,260 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:02:15,261 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:02:15,261 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:02:15,261 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:02:15,266 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:02:15,346 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:02:17,349 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:02:19,894 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:02:19,895 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:02:19,895 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:02:19,899 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:02:20,351 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:02:20,900 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:02:21,353 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:02:23,257 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:02:23,257 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:02:23,258 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:02:24,721 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:02:24,722 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:02:24,722 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:02:24,727 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:02:25,356 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:02:26,766 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:02:27,359 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:02:28,519 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:02:28,524 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:02:28,524 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:02:28,529 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:02:29,363 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:02:29,364 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:02:32,530 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:02:33,555 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 20:02:36,977 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:02:36,978 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:02:36,979 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:02:36,984 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:02:37,369 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:02:37,985 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:02:38,257 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:02:38,257 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:02:38,257 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:02:39,372 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:02:40,764 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:02:40,765 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:02:40,765 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:02:40,770 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:02:41,373 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:02:43,375 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:02:43,806 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:02:44,432 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:02:44,433 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:02:44,433 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:02:44,438 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:02:45,378 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:02:45,378 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:02:49,439 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:02:53,257 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:02:53,257 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:02:53,257 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:02:54,206 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:02:54,207 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:02:54,207 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:02:54,212 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:02:54,385 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:02:55,250 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:02:55,388 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:02:59,892 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:02:59,893 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:02:59,893 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:02:59,899 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:03:00,391 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:03:00,899 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:03:01,394 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:03:03,557 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 20:03:04,482 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:03:04,483 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:03:04,483 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:03:04,488 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:03:05,399 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:03:05,400 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:03:06,488 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:03:08,257 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:03:08,257 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:03:08,257 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:03:09,035 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:03:09,036 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:03:09,036 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:03:09,042 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:03:09,402 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:03:11,406 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:03:12,078 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:03:13,217 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:03:13,218 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:03:13,218 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:03:13,223 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:03:13,407 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:03:15,410 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:03:17,224 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:03:17,767 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:03:17,768 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:03:17,769 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:03:17,774 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:03:18,412 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:03:19,415 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:03:22,774 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:03:23,257 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:03:23,257 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:03:23,258 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:03:26,909 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:03:26,910 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:03:26,910 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:03:26,915 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:03:27,420 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:03:27,950 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:03:29,425 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:03:32,017 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:03:32,018 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:03:32,018 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:03:32,023 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:03:32,427 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:03:33,024 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:03:33,428 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:03:33,563 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 20:03:36,226 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:03:36,227 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:03:36,227 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:03:36,233 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:03:36,430 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:03:37,432 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:03:38,233 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:03:38,257 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:03:38,257 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:03:38,257 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:03:41,183 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:03:41,184 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:03:41,184 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:03:41,189 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:03:41,435 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:03:43,436 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:03:44,226 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:03:44,824 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:03:44,825 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:03:44,825 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:03:44,830 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:03:45,438 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:03:47,441 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:03:48,158 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:03:48,159 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:03:48,159 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:03:48,164 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:03:48,442 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:03:49,444 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:03:50,164 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:03:53,045 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:03:53,046 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:03:53,046 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:03:53,051 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:03:53,257 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:03:53,257 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:03:53,258 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:03:53,448 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:03:55,452 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:03:55,563 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:03:56,380 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:03:56,381 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:03:56,381 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:03:56,387 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:03:56,452 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:03:57,455 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:04:01,003 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:04:01,004 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:04:01,004 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:04:01,004 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:04:01,009 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:04:01,457 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:04:03,461 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:04:03,561 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 20:04:05,934 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:04:05,935 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:04:05,935 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:04:05,940 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:04:06,463 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:04:06,941 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:04:07,466 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:04:08,257 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:04:08,257 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:04:08,258 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:04:10,043 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:04:10,043 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:04:10,044 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:04:10,049 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:04:10,468 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:04:11,471 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:04:12,086 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:04:15,237 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:04:15,238 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:04:15,238 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:04:15,243 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:04:15,473 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:04:17,244 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:04:17,474 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:04:19,601 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:04:19,602 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:04:19,602 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:04:19,607 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:04:20,476 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:04:21,479 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:04:22,608 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:04:23,257 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:04:23,258 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:04:23,258 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:04:23,984 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:04:23,985 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:04:23,985 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:04:23,990 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:04:24,481 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:04:25,483 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:04:27,646 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:04:27,647 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:04:27,647 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:04:27,647 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:04:27,652 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:04:28,485 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:04:29,489 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:04:32,653 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:04:32,869 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:04:32,870 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:04:32,870 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:04:32,875 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:04:33,492 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:04:33,567 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 20:04:35,495 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:04:37,155 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:04:37,156 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:04:37,156 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:04:37,161 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:04:37,497 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:04:38,161 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:04:38,257 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:04:38,258 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:04:38,258 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:04:39,500 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:04:43,100 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:04:43,101 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:04:43,101 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:04:43,106 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:04:43,503 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:04:44,142 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:04:45,506 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:04:47,589 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:04:47,590 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:04:47,590 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:04:47,595 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:04:48,508 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:04:49,511 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:04:49,596 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:04:51,533 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:04:51,534 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:04:51,535 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:04:51,540 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:04:52,514 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:04:53,257 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:04:53,258 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:04:53,258 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:04:53,517 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:04:55,059 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:04:55,059 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:04:55,059 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:04:55,060 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:04:55,065 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:04:55,519 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:04:57,522 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:04:58,694 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:04:58,695 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:04:58,695 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:04:58,701 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:04:59,525 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:04:59,525 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:05:00,701 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:05:02,448 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:05:02,449 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:05:02,449 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:05:02,454 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:05:02,527 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:05:03,530 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:05:03,565 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 20:05:06,166 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:05:06,167 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:05:06,167 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:05:06,167 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:05:06,172 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:05:06,532 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:05:07,534 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:05:08,257 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:05:08,258 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:05:08,258 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:05:09,524 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:05:09,525 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:05:09,525 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:05:09,530 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:05:09,536 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:05:11,537 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:05:11,566 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:05:13,692 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:05:13,693 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:05:13,693 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:05:13,698 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:05:14,539 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:05:15,541 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:05:16,699 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:05:19,098 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:05:19,098 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:05:19,099 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:05:19,104 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:05:19,544 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:05:21,547 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:05:22,105 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:05:23,258 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:05:23,258 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:05:23,258 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:05:25,561 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:05:25,562 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:05:25,562 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:05:25,567 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:05:26,550 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:05:27,551 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:05:27,602 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:05:29,292 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:05:29,293 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:05:29,294 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:05:29,299 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:05:29,553 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:05:31,557 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:05:33,296 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:05:33,297 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:05:33,297 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:05:33,297 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:05:33,303 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:05:33,558 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:05:33,571 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 20:05:35,561 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:05:38,258 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:05:38,258 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:05:38,258 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:05:38,543 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:05:38,671 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:05:38,671 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:05:38,672 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:05:38,677 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:05:39,566 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:05:39,566 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:05:42,460 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:05:42,461 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:05:42,461 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:05:42,467 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:05:42,569 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:05:43,570 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:05:44,468 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:05:49,468 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:05:51,797 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:05:51,798 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:05:51,799 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:05:51,804 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:05:52,577 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:05:53,258 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:05:53,258 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:05:53,258 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:05:53,580 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:05:54,478 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:05:54,810 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:05:54,810 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:05:54,811 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:05:54,817 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:05:55,581 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:05:57,585 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:05:59,420 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:05:59,421 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:05:59,422 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:05:59,428 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:05:59,586 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:06:00,428 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:06:01,589 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:06:03,094 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:06:03,094 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:06:03,095 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:06:03,100 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:06:03,570 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 20:06:03,591 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:06:05,571 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:06:05,593 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:06:07,005 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:06:07,006 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:06:07,006 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:06:07,012 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:06:07,594 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:06:08,258 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:06:08,258 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:06:08,258 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:06:09,597 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:06:10,258 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:06:10,259 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:06:10,259 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:06:10,264 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:06:10,598 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:06:11,302 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:06:11,601 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:06:14,268 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:06:14,269 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:06:14,269 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:06:14,275 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:06:14,603 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:06:15,605 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:06:17,275 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:06:17,997 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:06:17,997 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:06:17,998 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:06:18,003 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:06:18,607 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:06:19,610 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:06:23,004 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:06:23,258 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:06:23,258 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:06:23,260 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:06:23,829 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:06:23,830 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:06:23,830 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:06:23,835 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:06:24,613 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:06:25,616 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:06:27,333 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:06:27,334 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:06:27,334 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:06:27,339 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:06:27,617 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:06:28,340 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:06:29,621 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:06:31,018 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:06:31,019 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:06:31,019 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:06:31,025 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:06:31,625 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:06:33,576 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 20:06:33,577 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:06:33,626 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:06:34,610 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:06:34,611 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:06:34,611 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:06:34,617 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:06:34,627 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:06:35,629 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:06:38,258 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:06:38,258 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:06:38,258 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:06:38,427 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:06:38,535 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:06:38,535 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:06:38,540 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:06:38,631 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:06:39,541 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:06:39,634 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:06:42,108 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:06:42,109 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:06:42,109 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:06:42,114 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:06:42,636 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:06:43,638 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:06:45,115 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:06:45,925 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:06:45,926 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:06:45,927 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:06:45,932 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:06:46,640 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:06:47,643 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:06:49,629 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:06:49,630 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:06:49,630 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:06:49,636 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:06:49,644 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:06:50,636 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:06:51,649 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:06:53,258 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:06:53,259 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:06:53,259 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:06:53,754 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:06:53,754 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:06:53,755 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:06:53,760 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:06:54,651 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:06:55,654 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:06:55,798 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:06:58,125 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:06:58,126 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:06:58,126 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:06:58,131 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:06:58,656 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:06:59,658 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:07:01,132 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:07:01,761 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:07:01,762 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:07:01,762 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:07:01,768 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:07:02,661 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:07:03,574 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 20:07:03,664 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:07:06,576 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:07:07,770 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:07:07,770 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:07:07,771 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:07:07,776 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:07:08,258 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:07:08,259 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:07:08,260 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:07:08,667 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:07:09,669 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:07:12,446 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:07:12,447 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:07:12,447 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:07:12,447 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:07:12,452 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:07:12,671 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:07:13,674 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:07:16,108 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:07:16,109 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:07:16,110 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:07:16,115 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:07:16,676 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:07:17,677 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:07:18,116 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:07:23,065 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:07:23,066 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:07:23,066 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:07:23,072 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:07:23,258 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:07:23,259 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:07:23,259 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:07:23,505 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:07:23,681 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:07:25,684 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:07:28,505 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:07:29,289 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:07:29,290 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:07:29,290 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:07:29,295 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:07:29,688 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:07:31,691 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:07:32,772 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:07:32,773 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:07:32,773 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:07:32,779 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:07:33,580 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 20:07:33,582 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:07:33,692 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:07:33,693 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:07:37,013 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:07:37,014 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:07:37,014 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:07:37,020 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:07:37,695 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:07:38,258 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:07:38,259 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:07:38,259 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:07:39,530 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:07:39,698 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:07:41,086 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:07:41,087 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:07:41,087 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:07:41,092 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:07:41,700 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:07:43,703 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:07:45,130 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:07:45,700 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:07:45,700 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:07:45,701 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:07:45,705 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:07:45,706 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:07:47,710 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:07:49,457 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:07:49,457 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:07:49,458 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:07:49,463 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:07:49,711 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:07:50,463 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:07:51,716 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:07:52,962 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:07:52,962 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:07:52,963 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:07:52,968 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:07:53,258 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:07:53,259 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:07:53,259 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:07:53,717 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:07:55,519 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:07:55,719 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:08:00,520 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:08:01,526 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:08:01,527 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:08:01,527 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:08:01,532 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:08:01,723 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:08:03,579 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 20:08:03,726 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:08:05,580 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:08:07,496 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:08:07,496 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:08:07,497 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:08:07,501 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:08:07,729 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:08:08,259 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:08:08,259 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:08:08,259 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:08:09,733 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:08:11,031 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:08:11,032 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:08:11,032 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:08:11,032 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:08:11,037 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:08:11,734 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:08:13,737 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:08:15,841 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:08:15,842 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:08:15,842 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:08:15,848 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:08:16,740 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:08:16,848 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:08:17,742 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:08:21,686 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:08:21,687 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:08:21,687 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:08:21,692 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:08:21,745 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:08:22,692 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:08:23,259 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:08:23,259 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:08:23,259 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:08:23,748 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:08:25,264 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:08:25,265 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:08:25,265 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:08:25,270 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:08:25,750 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:08:27,753 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:08:28,307 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:08:30,918 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:08:30,919 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:08:30,919 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:08:30,924 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:08:31,759 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:08:31,759 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:08:33,581 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 20:08:33,582 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:08:36,109 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:08:36,110 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:08:36,110 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:08:36,116 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:08:36,762 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:08:37,765 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:08:38,259 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:08:38,259 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:08:38,259 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:08:39,517 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:08:40,524 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:08:40,524 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:08:40,524 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:08:40,529 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:08:40,767 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:08:41,769 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:08:44,567 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:08:45,418 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:08:45,422 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:08:45,422 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:08:45,427 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:08:45,772 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:08:47,775 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:08:49,652 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:08:49,652 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:08:49,653 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:08:49,653 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:08:49,658 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:08:49,777 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:08:51,782 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:08:53,259 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:08:53,259 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:08:53,259 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:08:55,332 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:08:55,332 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:08:55,333 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:08:55,333 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:08:55,338 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:08:55,784 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:08:57,788 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:08:59,127 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:08:59,128 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:08:59,128 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:08:59,133 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:08:59,789 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:09:01,134 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:09:01,793 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:09:02,390 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:09:02,391 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:09:02,391 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:09:02,397 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:09:02,794 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:09:03,583 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 20:09:03,797 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:09:06,585 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:09:08,259 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:09:08,259 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:09:08,259 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:09:08,459 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:09:08,489 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:09:08,490 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:09:08,495 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:09:08,800 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:09:09,802 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:09:11,728 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:09:11,729 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:09:11,729 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:09:11,729 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:09:11,734 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:09:11,804 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:09:13,807 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:09:15,990 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:09:15,991 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:09:15,991 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:09:15,997 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:09:16,809 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:09:16,997 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:09:17,812 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:09:19,546 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:09:19,547 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:09:19,547 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:09:19,552 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:09:19,813 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:09:21,817 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:09:22,553 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:09:23,259 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:09:23,259 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:09:23,259 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:09:23,642 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:09:23,642 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:09:23,643 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:09:23,648 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:09:23,818 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:09:25,821 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:09:27,687 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:09:28,015 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:09:28,016 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:09:28,016 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:09:28,021 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:09:28,823 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:09:29,827 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:09:33,022 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:09:33,586 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 20:09:36,658 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:09:36,658 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:09:36,659 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:09:36,664 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:09:36,832 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:09:37,834 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:09:38,259 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:09:38,259 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:09:38,259 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:09:38,553 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:09:42,426 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:09:42,426 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:09:42,426 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:09:42,431 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:09:42,838 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:09:43,838 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:09:44,471 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:09:47,286 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:09:47,287 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:09:47,287 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:09:47,292 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:09:47,841 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:09:49,845 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:09:50,292 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:09:51,043 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:09:51,043 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:09:51,044 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:09:51,049 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:09:51,850 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:09:51,850 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:09:53,259 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:09:53,259 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:09:53,259 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:09:55,533 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:09:55,643 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:09:55,643 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:09:55,644 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:09:55,649 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:09:55,852 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:09:57,856 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:10:00,687 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:10:03,588 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 20:10:04,225 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:10:04,226 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:10:04,226 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:10:04,231 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:10:04,861 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:10:05,861 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:10:06,232 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:10:08,259 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:10:08,260 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:10:08,260 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:10:08,718 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:10:08,718 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:10:08,719 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:10:08,724 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:10:08,863 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:10:09,866 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:10:11,763 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:10:12,424 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:10:12,425 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:10:12,425 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:10:12,430 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:10:12,868 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:10:13,870 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:10:17,018 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:10:17,019 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:10:17,019 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:10:17,019 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:10:17,024 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:10:17,874 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:10:17,875 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:10:22,025 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:10:23,259 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:10:23,260 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:10:23,260 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:10:25,634 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:10:25,635 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:10:25,635 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:10:25,640 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:10:25,879 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:10:27,679 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:10:27,883 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:10:32,345 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:10:32,346 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:10:32,346 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:10:32,351 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:10:32,887 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:10:33,352 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:10:33,590 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 20:10:33,890 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:10:35,825 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:10:35,825 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:10:35,826 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:10:35,830 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:10:35,891 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:10:37,892 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:10:38,259 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:10:38,260 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:10:38,260 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:10:38,484 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:10:39,751 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:10:39,751 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:10:39,751 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:10:39,757 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:10:39,894 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:10:41,897 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:10:43,795 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:10:44,740 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:10:44,741 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:10:44,741 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:10:44,746 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:10:44,899 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:10:45,902 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:10:48,310 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:10:48,310 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:10:48,311 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:10:48,315 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:10:48,904 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:10:49,316 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:10:49,907 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:10:52,077 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:10:52,078 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:10:52,079 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:10:52,083 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:10:52,910 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:10:53,259 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:10:53,260 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:10:53,260 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:10:53,913 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:10:54,463 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:10:55,800 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:10:55,800 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:10:55,800 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:10:55,806 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:10:55,914 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:10:57,917 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:10:59,700 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:10:59,701 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:10:59,701 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:10:59,701 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:10:59,706 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:10:59,918 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:11:01,922 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:11:03,289 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:11:03,290 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:11:03,290 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:11:03,295 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:11:03,592 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 20:11:03,924 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:11:05,593 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:11:05,927 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:11:06,852 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:11:06,853 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:11:06,853 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:11:06,858 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:11:06,928 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:11:07,930 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:11:08,260 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:11:08,260 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:11:08,260 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:11:10,363 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:11:10,364 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:11:10,364 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:11:10,369 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:11:10,932 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:11:11,407 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:11:11,935 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:11:13,843 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:11:13,843 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:11:13,843 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:11:13,848 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:11:13,936 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:11:15,939 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:11:16,849 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:11:17,182 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:11:17,183 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:11:17,183 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:11:17,188 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:11:17,940 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:11:19,944 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:11:21,688 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:11:21,689 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:11:21,690 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:11:21,695 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:11:21,945 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:11:22,695 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:11:23,260 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:11:23,260 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:11:23,260 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:11:23,948 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:11:27,025 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:11:27,025 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:11:27,026 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:11:27,031 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:11:27,951 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:11:27,951 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:11:28,067 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:11:30,927 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:11:30,928 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:11:30,928 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:11:30,933 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:11:30,954 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:11:31,957 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:11:33,594 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 20:11:33,595 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:11:34,838 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:11:34,839 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:11:34,839 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:11:34,850 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:11:34,958 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:11:35,961 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:11:38,260 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:11:38,260 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:11:38,260 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:11:39,092 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:11:39,092 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:11:39,092 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:11:39,093 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:11:39,098 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:11:39,966 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:11:39,966 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:11:44,135 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:11:44,686 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:11:44,687 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:11:44,687 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:11:44,692 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:11:44,969 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:11:45,972 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:11:49,693 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:11:50,082 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:11:50,083 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:11:50,083 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:11:50,089 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:11:50,975 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:11:51,979 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:11:53,260 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:11:53,260 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:11:53,260 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:11:54,141 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:11:54,141 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:11:54,142 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:11:54,147 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:11:54,981 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:11:55,183 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:11:55,984 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:11:58,355 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:11:58,356 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:11:58,356 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:11:58,362 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:11:58,985 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:11:59,988 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:12:00,363 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:12:03,599 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 20:12:05,601 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:12:06,969 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:12:06,970 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:12:06,970 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:12:06,975 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:12:06,992 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:12:07,995 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:12:08,260 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:12:08,260 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:12:08,260 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:12:11,454 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:12:12,963 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:12:12,964 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:12:12,964 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:12:12,969 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:12:12,998 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:12:14,001 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:12:16,297 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:12:16,298 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:12:16,298 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:12:16,303 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:12:17,003 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:12:17,303 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:12:18,005 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:12:22,294 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:12:22,295 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:12:22,295 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:12:22,300 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:12:23,008 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:12:23,260 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:12:23,260 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:12:23,261 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:12:23,513 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:12:24,011 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:12:26,791 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:12:26,791 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:12:26,791 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:12:26,796 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:12:27,013 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:12:28,013 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:12:28,835 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:12:31,172 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:12:31,173 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:12:31,173 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:12:31,179 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:12:32,019 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:12:32,019 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:12:33,598 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 20:12:34,600 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:12:38,260 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:12:38,260 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:12:38,261 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:12:39,974 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:12:40,187 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:12:40,188 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:12:40,188 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:12:40,193 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:12:41,025 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:12:42,027 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:12:45,231 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:12:45,506 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:12:45,507 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:12:45,507 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:12:45,512 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:12:46,030 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:12:48,033 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:12:49,955 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:12:49,956 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:12:49,956 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:12:49,961 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:12:50,035 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:12:50,961 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:12:52,039 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:12:53,260 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:12:53,260 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:12:53,261 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:12:56,123 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:12:56,124 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:12:56,124 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:12:56,124 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:12:56,129 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:12:57,043 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:12:58,045 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:13:00,091 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:13:00,092 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:13:00,092 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:13:00,097 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:13:01,047 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:13:02,050 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:13:02,098 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:13:03,604 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 20:13:03,923 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:13:03,924 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:13:03,924 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:13:03,929 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:13:04,052 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:13:06,055 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:13:07,930 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:13:08,260 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:13:08,261 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:13:08,261 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:13:10,310 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:13:10,310 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:13:10,310 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:13:10,315 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:13:11,058 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:13:12,061 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:13:13,355 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:13:14,865 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:13:14,866 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:13:14,866 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:13:14,871 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:13:15,063 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:13:16,066 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:13:18,871 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:13:19,329 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:13:19,329 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:13:19,330 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:13:19,334 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:13:20,068 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:13:22,071 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:13:23,122 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:13:23,123 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:13:23,123 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:13:23,128 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:13:23,260 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:13:23,261 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:13:23,262 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:13:24,075 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:13:24,075 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:13:24,580 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:13:27,211 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:13:27,211 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:13:27,212 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:13:27,217 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:13:28,080 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:13:28,080 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:13:30,255 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:13:31,462 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:13:31,463 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:13:31,463 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:13:31,468 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:13:32,084 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:13:33,603 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 20:13:34,087 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:13:35,604 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:13:36,903 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:13:36,904 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:13:36,904 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:13:36,909 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:13:37,089 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:13:38,092 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:13:38,260 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:13:38,261 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:13:38,261 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:13:40,272 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:13:40,273 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:13:40,273 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:13:40,278 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:13:41,094 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:13:41,315 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:13:42,096 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:13:44,621 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:13:44,621 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:13:44,622 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:13:44,626 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:13:45,098 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:13:46,101 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:13:46,627 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:13:48,342 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:13:48,343 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:13:48,343 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:13:48,348 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:13:49,103 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:13:50,106 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:13:51,884 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:13:51,885 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:13:51,885 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:13:51,885 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:13:51,890 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:13:52,109 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:13:53,261 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:13:53,261 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:13:53,261 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:13:54,112 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:13:55,510 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:13:55,511 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:13:55,511 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:13:55,516 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:13:56,113 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:13:57,517 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:13:58,116 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:13:59,409 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:13:59,409 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:13:59,409 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:13:59,414 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:14:00,118 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:14:02,121 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:14:02,934 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:14:02,935 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:14:02,935 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:14:02,935 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:14:02,940 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:14:03,122 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:14:03,608 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 20:14:04,124 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:14:06,464 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:14:06,465 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:14:06,465 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:14:06,470 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:14:07,126 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:14:08,129 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:14:08,261 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:14:08,261 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:14:08,261 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:14:08,532 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:14:11,522 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:14:11,522 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:14:11,523 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:14:11,528 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:14:12,132 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:14:13,567 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:14:14,133 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:14:18,567 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:14:19,984 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:14:19,984 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:14:19,985 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:14:19,990 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:14:20,137 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:14:22,140 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:14:23,261 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:14:23,261 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:14:23,261 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:14:24,152 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:14:24,152 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:14:24,153 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:14:24,153 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:14:24,158 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:14:25,142 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:14:26,144 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:14:28,126 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:14:28,127 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:14:28,127 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:14:28,132 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:14:28,146 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:14:30,132 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:14:30,150 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:14:33,607 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 20:14:33,681 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:14:33,682 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:14:33,682 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:14:33,687 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:14:34,153 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:14:35,688 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:14:36,156 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:14:37,656 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:14:37,657 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:14:37,657 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:14:37,662 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:14:38,157 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:14:38,261 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:14:38,261 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:14:38,261 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:14:40,161 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:14:41,544 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:14:43,046 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:14:43,047 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:14:43,047 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:14:43,052 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:14:43,163 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:14:44,165 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:14:47,091 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:14:48,329 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:14:48,330 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:14:48,330 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:14:48,335 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:14:49,169 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:14:50,172 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:14:52,336 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:14:52,711 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:14:52,712 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:14:52,712 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:14:52,717 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:14:53,175 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:14:53,261 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:14:53,261 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:14:53,261 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:14:54,177 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:14:56,412 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:14:56,413 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:14:56,413 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:14:56,418 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:14:57,179 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:14:57,455 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:14:58,180 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:14:59,997 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:14:59,998 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:14:59,998 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:15:00,003 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:15:00,181 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:15:02,185 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:15:03,004 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:15:03,613 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 20:15:05,043 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:15:05,043 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:15:05,044 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:15:05,048 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:15:05,187 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:15:06,190 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:15:08,049 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:15:08,261 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:15:08,261 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:15:08,261 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:15:10,380 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:15:10,381 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:15:10,381 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:15:10,387 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:15:11,193 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:15:12,195 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:15:13,423 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:15:16,209 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:15:16,210 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:15:16,210 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:15:16,215 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:15:17,199 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:15:18,201 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:15:19,215 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:15:19,748 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:15:19,749 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:15:19,749 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:15:19,754 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:15:20,203 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:15:22,206 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:15:23,261 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:15:23,261 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:15:23,262 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:15:23,296 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:15:23,473 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:15:23,473 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:15:23,478 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:15:24,208 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:15:24,479 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:15:26,211 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:15:26,599 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:15:26,600 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:15:26,600 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:15:26,605 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:15:27,212 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:15:28,214 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:15:29,606 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:15:30,322 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:15:30,322 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:15:30,323 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:15:30,328 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:15:31,217 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:15:32,220 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:15:33,612 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 20:15:34,613 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:15:34,855 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:15:34,856 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:15:34,856 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:15:34,861 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:15:35,222 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:15:36,225 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:15:38,261 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:15:38,261 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:15:38,261 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:15:38,574 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:15:38,574 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:15:38,574 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:15:38,579 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:15:39,227 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:15:40,229 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:15:40,580 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:15:43,297 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:15:43,298 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:15:43,298 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:15:43,303 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:15:44,234 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:15:44,234 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:15:46,304 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:15:46,649 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:15:46,650 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:15:46,650 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:15:46,656 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:15:47,236 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:15:48,239 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:15:50,110 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:15:50,111 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:15:50,111 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:15:50,116 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:15:50,240 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:15:52,116 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:15:52,252 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:15:53,261 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:15:53,261 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:15:53,262 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:15:53,821 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:15:53,821 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:15:53,822 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:15:53,826 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:15:54,253 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:15:56,257 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:15:57,863 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:15:59,800 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:15:59,801 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:15:59,801 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:15:59,806 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:16:00,259 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:16:02,262 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:16:03,614 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 20:16:03,615 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:16:04,070 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:16:04,074 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:16:04,074 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:16:04,080 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:16:04,264 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:16:06,267 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:16:07,786 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:16:07,787 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:16:07,787 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:16:07,792 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:16:08,261 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:16:08,261 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:16:08,262 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:16:08,268 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:16:09,483 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:16:10,272 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:16:11,338 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:16:11,339 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:16:11,339 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:16:11,344 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:16:12,275 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:16:12,275 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:16:15,344 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:16:16,006 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:16:16,007 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:16:16,007 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:16:16,012 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:16:16,277 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:16:18,281 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:16:20,211 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:16:20,212 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:16:20,212 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:16:20,220 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:16:20,282 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:16:21,220 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:16:22,285 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:16:23,261 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:16:23,262 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:16:23,262 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:16:24,051 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:16:24,052 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:16:24,052 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:16:24,057 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:16:24,287 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:16:26,290 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:16:27,058 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:16:29,666 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:16:29,667 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:16:29,667 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:16:29,672 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:16:30,294 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:16:32,295 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:16:32,673 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:16:33,616 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 20:16:33,652 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:16:33,652 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:16:33,652 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:16:33,658 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:16:34,297 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:16:36,300 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:16:38,261 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:16:38,262 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:16:38,262 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:16:38,494 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:16:39,562 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:16:39,562 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:16:39,563 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:16:39,567 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:16:40,303 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:16:42,306 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:16:43,283 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:16:43,283 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:16:43,284 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:16:43,288 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:16:43,306 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:16:44,289 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:16:44,309 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:16:46,869 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:16:46,870 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:16:46,870 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:16:46,875 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:16:47,311 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:16:48,313 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:16:49,876 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:16:53,262 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:16:53,262 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:16:53,262 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:16:55,453 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:16:55,454 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:16:55,454 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:16:55,454 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:16:55,459 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:16:56,322 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:16:56,322 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:16:59,799 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:16:59,800 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:16:59,800 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:16:59,806 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:17:00,325 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:17:00,806 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:17:02,328 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:17:03,619 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 20:17:04,500 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:17:04,501 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:17:04,501 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:17:04,506 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:17:05,330 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:17:06,331 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:17:06,507 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:17:08,262 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:17:08,262 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:17:08,262 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:17:09,609 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:17:09,610 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:17:09,610 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:17:09,615 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:17:10,333 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:17:11,658 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:17:12,337 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:17:13,291 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:17:13,292 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:17:13,292 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:17:13,296 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:17:13,337 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:17:14,340 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:17:17,297 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:17:17,432 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:17:17,433 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:17:17,433 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:17:17,438 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:17:18,344 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:17:18,345 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:17:22,277 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:17:22,278 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:17:22,278 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:17:22,283 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:17:22,347 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:17:23,262 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:17:23,262 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:17:23,262 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:17:23,581 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:17:24,350 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:17:26,390 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:17:26,391 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:17:26,391 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:17:26,396 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:17:27,352 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:17:28,353 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:17:29,396 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:17:31,326 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:17:31,327 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:17:31,327 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:17:31,332 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:17:31,356 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:17:32,359 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:17:33,620 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 20:17:34,622 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:17:35,775 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:17:35,775 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:17:35,776 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:17:35,780 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:17:36,362 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:17:38,262 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:17:38,262 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:17:38,262 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:17:38,365 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:17:40,511 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:17:40,708 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:17:40,709 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:17:40,709 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:17:40,714 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:17:41,367 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:17:42,370 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:17:45,751 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:17:45,764 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:17:45,765 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:17:45,765 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:17:45,770 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:17:46,372 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:17:48,375 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:17:49,251 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:17:49,252 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:17:49,252 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:17:49,257 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:17:49,377 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:17:50,379 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:17:51,258 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:17:53,262 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:17:53,262 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:17:53,262 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:17:54,534 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:17:54,534 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:17:54,535 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:17:54,540 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:17:55,384 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:17:56,386 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:17:56,579 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:17:59,609 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:17:59,610 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:17:59,610 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:17:59,616 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:18:00,391 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:18:00,391 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:18:01,616 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:18:03,623 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 20:18:03,753 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:18:03,754 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:18:03,754 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:18:03,760 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:18:04,394 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:18:06,397 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:18:06,761 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:18:07,224 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:18:07,224 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:18:07,225 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:18:07,230 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:18:07,398 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:18:08,262 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:18:08,262 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:18:08,262 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:18:08,400 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:18:11,167 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:18:11,168 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:18:11,169 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:18:11,174 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:18:11,402 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:18:12,211 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:18:12,403 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:18:14,689 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:18:14,690 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:18:14,690 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:18:14,696 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:18:15,405 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:18:16,408 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:18:17,696 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:18:18,489 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:18:18,489 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:18:18,490 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:18:18,494 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:18:19,410 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:18:20,412 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:18:23,262 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:18:23,262 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:18:23,263 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:18:23,511 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:18:27,383 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:18:27,384 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:18:27,384 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:18:27,389 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:18:27,417 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:18:28,419 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:18:29,430 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:18:30,870 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:18:30,871 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:18:30,871 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:18:30,876 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:18:31,422 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:18:32,426 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:18:33,625 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 20:18:34,626 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:18:35,618 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:18:35,618 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:18:35,619 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:18:35,623 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:18:36,430 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:18:36,430 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:18:38,262 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:18:38,262 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:18:38,263 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:18:39,312 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:18:39,313 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:18:39,314 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:18:39,319 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:18:39,432 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:18:40,319 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:18:40,435 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:18:43,320 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:18:43,321 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:18:43,321 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:18:43,326 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:18:43,437 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:18:44,440 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:18:45,326 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:18:50,327 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:18:51,335 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:18:51,336 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:18:51,336 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:18:51,341 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:18:51,445 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:18:52,448 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:18:53,262 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:18:53,262 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:18:53,262 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:18:55,325 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:18:55,325 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:18:55,325 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:18:55,330 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:18:55,331 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:18:55,450 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:18:56,453 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:18:58,961 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:18:58,962 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:18:58,962 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:18:58,967 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:18:59,455 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:19:00,458 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:19:00,968 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:19:02,641 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:19:02,642 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:19:02,642 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:19:02,647 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:19:03,461 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:19:03,627 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 20:19:04,463 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:19:06,114 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:19:06,115 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:19:06,115 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:19:06,115 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:19:06,120 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:19:06,465 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:19:08,262 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:19:08,262 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:19:08,263 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:19:08,468 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:19:09,975 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:19:09,975 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:19:09,975 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:19:09,981 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:19:10,469 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:19:11,982 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:19:12,473 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:19:13,440 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:19:13,441 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:19:13,441 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:19:13,446 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:19:13,473 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:19:14,476 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:19:16,955 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:19:16,956 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:19:16,956 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:19:16,961 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:19:17,478 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:19:17,962 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:19:18,481 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:19:20,661 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:19:20,662 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:19:20,662 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:19:20,666 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:19:21,483 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:19:22,483 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:19:23,262 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:19:23,262 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:19:23,263 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:19:23,547 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:19:24,760 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:19:24,761 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:19:24,761 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:19:24,766 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:19:25,485 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:19:26,488 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:19:28,803 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:19:29,242 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:19:29,242 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:19:29,243 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:19:29,247 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:19:29,490 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:19:30,493 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:19:33,547 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:19:33,548 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:19:33,548 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:19:33,553 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:19:33,630 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 20:19:34,496 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:19:34,497 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:19:34,631 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:19:37,446 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:19:37,447 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:19:37,447 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:19:37,452 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:19:37,498 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:19:38,262 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:19:38,263 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:19:38,263 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:19:38,501 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:19:40,515 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:19:41,422 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:19:41,423 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:19:41,423 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:19:41,428 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:19:41,503 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:19:42,505 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:19:45,333 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:19:45,334 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:19:45,334 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:19:45,339 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:19:45,508 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:19:46,340 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:19:46,510 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:19:51,340 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:19:51,439 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:19:51,440 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:19:51,440 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:19:51,445 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:19:51,514 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:19:52,517 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:19:53,262 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:19:53,263 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:19:53,263 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:19:54,884 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:19:54,884 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:19:54,885 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:19:54,890 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:19:55,519 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:19:56,522 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:19:56,927 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:19:58,969 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:19:58,970 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:19:58,970 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:19:58,975 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:19:59,524 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:20:00,527 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:20:01,976 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:20:03,060 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:20:03,061 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:20:03,061 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:20:03,066 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:20:03,529 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:20:03,632 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 20:20:04,531 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:20:07,352 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:20:07,353 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:20:07,353 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:20:07,353 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:20:07,358 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:20:07,533 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:20:08,263 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:20:08,263 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:20:08,263 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:20:08,536 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:20:12,522 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:20:12,523 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:20:12,523 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:20:12,523 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:20:12,528 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:20:12,539 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:20:14,542 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:20:16,637 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:20:16,638 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:20:16,638 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:20:16,644 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:20:17,544 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:20:17,644 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:20:18,547 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:20:22,644 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:20:23,263 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:20:23,263 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:20:23,263 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:20:25,142 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:20:25,143 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:20:25,143 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:20:25,148 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:20:25,551 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:20:26,554 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:20:28,187 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:20:29,950 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:20:29,951 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:20:29,951 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:20:29,956 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:20:30,557 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:20:32,561 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:20:33,638 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 20:20:33,639 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:20:34,933 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:20:34,934 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:20:34,934 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:20:34,939 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:20:35,563 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:20:36,565 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:20:38,263 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:20:38,263 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:20:38,263 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:20:39,215 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:20:39,216 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:20:39,216 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:20:39,216 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:20:39,221 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:20:39,567 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:20:40,570 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:20:42,723 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:20:42,723 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:20:42,723 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:20:42,729 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:20:43,572 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:20:44,575 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:20:44,730 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:20:47,255 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:20:47,256 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:20:47,256 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:20:47,261 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:20:47,576 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:20:48,579 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:20:50,262 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:20:53,263 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:20:53,263 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:20:53,263 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:20:55,489 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:20:55,687 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:20:55,687 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:20:55,687 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:20:55,692 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:20:56,586 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:20:58,587 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:21:00,731 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:21:01,945 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:21:01,946 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:21:01,946 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:21:01,951 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:21:02,590 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:21:03,636 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 20:21:04,594 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:21:05,526 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:21:05,527 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:21:05,527 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:21:05,532 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:21:05,594 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:21:06,532 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:21:08,263 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:21:08,263 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:21:08,263 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:21:08,598 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:21:09,277 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:21:09,278 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:21:09,279 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:21:09,284 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:21:09,599 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:21:10,602 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:21:12,284 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:21:13,492 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:21:13,492 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:21:13,493 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:21:13,499 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:21:13,604 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:21:16,607 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:21:17,499 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:21:18,838 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:21:18,839 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:21:18,839 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:21:18,844 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:21:19,609 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:21:20,612 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:21:22,845 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:21:23,263 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:21:23,263 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:21:23,263 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:21:23,414 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:21:23,493 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:21:23,493 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:21:23,498 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:21:23,614 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:21:26,618 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:21:26,731 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:21:26,731 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:21:26,731 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:21:26,736 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:21:27,618 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:21:28,619 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:21:28,775 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:21:33,645 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 20:21:34,646 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:21:38,236 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:21:38,237 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:21:38,237 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:21:38,243 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:21:38,263 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:21:38,263 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:21:38,263 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:21:38,626 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:21:40,531 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:21:40,630 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:21:43,031 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:21:43,032 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:21:43,032 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:21:43,037 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:21:43,631 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:21:44,634 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:21:46,037 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:21:46,690 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:21:46,690 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:21:46,690 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:21:46,695 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:21:47,636 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:21:48,639 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:21:50,659 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:21:50,660 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:21:50,660 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:21:50,665 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:21:51,641 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:21:51,666 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:21:52,645 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:21:53,263 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:21:53,263 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:21:53,264 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:21:55,824 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:21:55,825 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:21:55,825 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:21:55,830 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:21:56,647 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:21:56,867 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:21:58,650 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:21:59,329 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:21:59,329 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:21:59,330 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:21:59,334 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:21:59,651 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:22:00,654 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:22:02,335 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:22:02,521 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:22:02,522 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:22:02,522 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:22:02,528 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:22:02,655 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:22:03,641 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 20:22:04,659 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:22:07,643 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:22:08,263 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:22:08,263 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:22:08,263 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:22:08,601 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:22:08,770 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:22:08,770 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:22:08,775 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:22:09,662 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:22:10,664 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:22:12,772 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:22:12,773 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:22:12,774 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:22:12,774 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:22:12,779 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:22:13,666 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:22:14,667 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:22:16,726 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:22:16,726 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:22:16,727 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:22:16,732 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:22:17,669 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:22:18,672 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:22:18,732 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:22:22,296 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:22:22,297 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:22:22,297 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:22:22,302 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:22:22,674 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:22:23,263 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:22:23,264 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:22:23,264 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:22:24,558 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:22:24,677 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:22:27,813 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:22:27,814 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:22:27,814 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:22:27,819 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:22:28,680 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:22:29,859 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:22:30,684 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:22:31,545 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:22:31,546 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:22:31,546 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:22:31,551 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:22:31,685 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:22:33,646 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 20:22:34,689 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:22:35,648 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:22:36,549 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:22:36,550 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:22:36,551 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:22:36,555 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:22:36,690 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:22:38,263 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:22:38,264 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:22:38,264 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:22:38,694 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:22:40,440 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:22:40,441 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:22:40,441 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:22:40,447 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:22:40,695 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:22:41,447 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:22:42,698 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:22:46,448 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:22:48,875 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:22:48,876 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:22:48,877 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:22:48,882 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:22:49,703 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:22:50,706 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:22:51,882 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:22:52,606 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:22:52,607 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:22:52,607 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:22:52,612 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:22:52,708 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:22:53,264 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:22:53,264 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:22:53,264 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:22:54,712 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:22:56,572 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:22:56,572 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:22:56,573 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:22:56,578 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:22:56,713 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:22:57,615 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:22:58,717 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:23:01,308 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:23:01,309 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:23:01,309 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:23:01,314 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:23:01,718 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:23:02,719 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:23:03,314 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:23:03,646 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 20:23:04,887 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:23:04,888 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:23:04,888 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:23:04,893 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:23:05,721 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:23:06,724 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:23:08,264 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:23:08,264 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:23:08,264 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:23:08,502 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:23:08,540 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:23:08,540 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:23:08,540 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:23:08,546 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:23:08,725 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:23:10,729 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:23:12,498 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:23:12,498 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:23:12,499 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:23:12,503 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:23:12,730 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:23:13,504 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:23:14,733 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:23:17,008 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:23:17,009 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:23:17,009 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:23:17,014 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:23:17,735 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:23:18,736 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:23:19,015 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:23:22,633 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:23:22,634 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:23:22,634 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:23:22,640 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:23:22,738 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:23:23,264 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:23:23,264 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:23:23,264 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:23:24,533 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:23:24,741 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:23:27,030 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:23:27,031 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:23:27,031 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:23:27,036 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:23:27,743 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:23:28,746 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:23:30,075 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:23:33,281 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:23:33,282 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:23:33,282 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:23:33,287 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:23:33,651 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 20:23:33,750 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:23:34,753 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:23:35,652 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:23:36,681 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:23:36,681 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:23:36,682 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:23:36,686 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:23:36,754 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:23:38,264 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:23:38,264 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:23:38,264 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:23:38,757 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:23:41,548 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:23:42,771 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:23:42,772 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:23:42,772 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:23:42,777 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:23:43,760 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:23:44,763 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:23:46,815 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:23:47,587 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:23:47,587 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:23:47,588 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:23:47,593 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:23:47,765 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:23:50,769 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:23:51,479 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:23:51,479 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:23:51,480 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:23:51,485 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:23:51,770 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:23:52,485 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:23:52,774 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:23:53,264 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:23:53,264 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:23:53,264 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:23:55,529 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:23:55,530 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:23:55,530 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:23:55,535 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:23:55,776 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:23:57,575 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:23:58,780 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:24:02,575 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:24:03,650 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 20:24:03,957 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:24:03,958 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:24:03,958 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:24:03,963 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:24:04,784 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:24:06,788 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:24:07,964 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:24:08,264 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:24:08,264 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:24:08,264 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:24:12,415 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:24:12,416 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:24:12,416 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:24:12,421 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:24:12,791 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:24:13,459 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:24:14,796 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:24:17,653 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:24:17,654 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:24:17,654 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:24:17,660 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:24:17,798 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:24:18,660 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:24:20,802 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:24:22,175 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:24:22,176 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:24:22,176 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:24:22,182 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:24:22,804 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:24:23,264 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:24:23,264 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:24:23,264 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:24:24,512 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:24:24,807 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:24:25,949 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:24:25,950 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:24:25,950 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:24:25,956 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:24:26,808 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:24:28,812 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:24:29,995 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:24:30,050 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:24:30,051 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:24:30,052 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:24:30,057 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:24:30,814 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:24:32,818 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:24:33,652 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 20:24:33,962 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:24:33,963 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:24:33,963 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:24:33,969 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:24:34,820 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:24:35,969 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:24:36,823 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:24:38,200 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:24:38,200 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:24:38,201 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:24:38,206 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:24:38,264 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:24:38,264 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:24:38,265 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:24:38,825 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:24:40,828 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:24:41,499 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:24:41,948 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:24:41,949 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:24:41,949 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:24:41,955 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:24:42,830 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:24:44,833 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:24:46,995 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:24:48,649 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:24:48,650 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:24:48,651 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:24:48,656 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:24:48,836 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:24:50,840 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:24:52,324 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:24:52,325 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:24:52,325 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:24:52,325 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:24:52,331 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:24:52,843 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:24:53,264 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:24:53,264 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:24:53,265 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:24:54,847 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:24:56,058 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:24:56,058 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:24:56,059 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:24:56,065 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:24:56,848 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:24:58,103 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:24:58,852 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:24:59,335 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:24:59,336 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:24:59,336 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:24:59,342 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:24:59,853 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:25:00,856 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:25:03,343 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:25:03,654 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 20:25:07,606 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:25:07,607 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:25:07,608 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:25:07,613 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:25:07,860 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:25:08,264 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:25:08,265 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:25:08,265 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:25:08,467 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:25:08,861 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:25:11,876 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:25:11,877 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:25:11,877 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:25:11,882 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:25:12,864 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:25:13,919 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:25:14,867 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:25:18,919 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:25:20,096 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:25:20,097 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:25:20,097 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:25:20,102 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:25:20,870 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:25:22,874 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:25:23,264 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:25:23,265 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:25:23,265 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:25:23,710 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:25:23,710 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:25:23,711 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:25:23,716 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:25:23,874 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:25:24,755 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:25:26,878 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:25:28,282 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:25:28,283 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:25:28,283 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:25:28,289 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:25:28,879 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:25:30,289 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:25:30,883 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:25:32,403 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:25:32,404 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:25:32,404 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:25:32,409 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:25:32,885 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:25:33,656 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 20:25:34,888 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:25:35,658 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:25:36,493 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:25:36,494 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:25:36,494 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:25:36,499 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:25:36,890 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:25:38,265 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:25:38,265 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:25:38,265 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:25:38,893 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:25:40,399 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:25:40,400 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:25:40,400 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:25:40,405 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:25:40,894 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:25:41,443 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:25:42,898 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:25:44,247 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:25:44,248 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:25:44,248 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:25:44,253 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:25:44,899 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:25:46,902 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:25:47,254 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:25:48,368 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:25:48,369 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:25:48,370 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:25:48,375 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:25:48,904 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:25:50,907 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:25:52,252 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:25:52,257 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:25:52,257 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:25:52,257 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:25:52,262 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:25:52,910 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:25:53,265 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:25:53,265 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:25:53,265 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:25:54,913 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:25:57,259 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:25:57,259 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:25:57,260 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:25:57,260 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:25:57,265 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:25:57,915 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:25:58,918 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:26:00,641 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:26:00,642 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:26:00,642 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:26:00,647 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:26:00,919 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:26:02,647 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:26:02,920 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:26:03,659 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 20:26:07,661 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:26:08,265 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:26:08,265 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:26:08,265 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:26:09,171 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:26:09,171 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:26:09,172 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:26:09,176 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:26:09,925 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:26:10,928 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:26:13,215 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:26:13,713 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:26:13,714 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:26:13,714 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:26:13,719 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:26:13,930 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:26:14,933 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:26:17,257 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:26:17,258 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:26:17,259 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:26:17,263 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:26:17,935 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:26:18,264 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:26:18,935 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:26:20,969 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:26:20,970 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:26:20,970 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:26:20,975 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:26:21,937 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:26:22,940 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:26:23,265 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:26:23,265 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:26:23,265 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:26:23,479 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:26:24,634 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:26:24,634 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:26:24,635 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:26:24,639 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:26:24,941 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:26:26,945 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:26:28,073 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:26:28,074 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:26:28,074 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:26:28,079 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:26:28,946 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:26:29,080 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:26:30,950 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:26:33,661 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 20:26:34,663 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:26:36,137 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:26:36,138 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:26:36,138 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:26:36,143 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:26:36,954 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:26:38,265 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:26:38,265 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:26:38,266 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:26:38,957 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:26:39,609 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:26:39,609 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:26:39,610 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:26:39,614 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:26:39,958 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:26:40,651 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:26:40,961 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:26:45,651 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:26:46,478 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:26:46,479 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:26:46,479 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:26:46,484 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:26:46,964 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:26:48,968 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:26:51,024 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:26:51,025 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:26:51,025 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:26:51,025 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:26:51,030 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:26:51,970 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:26:52,974 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:26:53,265 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:26:53,265 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:26:53,266 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:26:54,674 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:26:54,675 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:26:54,675 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:26:54,680 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:26:54,975 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:26:56,719 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:26:56,978 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:26:58,725 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:26:58,725 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:26:58,726 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:26:58,731 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:26:58,979 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:27:00,983 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:27:01,731 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:27:02,409 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:27:02,410 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:27:02,411 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:27:02,416 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:27:02,984 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:27:03,663 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 20:27:04,988 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:27:06,410 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:27:06,411 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:27:06,411 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:27:06,416 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:27:06,989 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:27:07,417 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:27:08,265 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:27:08,265 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:27:08,266 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:27:08,992 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:27:10,347 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:27:10,348 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:27:10,348 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:27:10,353 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:27:10,993 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:27:12,997 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:27:13,391 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:27:14,919 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:27:14,920 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:27:14,920 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:27:14,926 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:27:14,998 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:27:17,001 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:27:18,552 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:27:18,552 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:27:18,553 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:27:18,553 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:27:18,558 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:27:19,002 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:27:21,006 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:27:22,281 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:27:22,282 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:27:22,282 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:27:22,287 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:27:23,007 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:27:23,265 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:27:23,265 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:27:23,266 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:27:24,544 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:27:25,010 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:27:26,166 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:27:26,167 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:27:26,167 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:27:26,172 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:27:27,011 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:27:29,014 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:27:29,526 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:27:29,527 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:27:29,527 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:27:29,532 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:27:30,015 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:27:30,533 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:27:31,019 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:27:33,666 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 20:27:35,315 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:27:35,315 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:27:35,316 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:27:35,321 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:27:36,022 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:27:36,321 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:27:37,025 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:27:38,265 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:27:38,265 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:27:38,266 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:27:41,498 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:27:43,833 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:27:43,834 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:27:43,834 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:27:43,839 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:27:44,029 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:27:46,875 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:27:47,031 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:27:48,560 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:27:48,561 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:27:48,561 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:27:48,566 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:27:49,032 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:27:51,036 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:27:52,567 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:27:53,265 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:27:53,266 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:27:53,266 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:27:54,350 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:27:54,350 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:27:54,351 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:27:54,356 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:27:55,039 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:27:57,043 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:27:58,395 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:27:58,439 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:27:58,440 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:27:58,440 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:27:58,445 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:27:59,044 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:28:01,047 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:28:03,446 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:28:03,672 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 20:28:03,687 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:28:03,688 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:28:03,688 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:28:03,693 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:28:04,050 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:28:05,052 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:28:08,265 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:28:08,266 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:28:08,266 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:28:08,507 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:28:09,561 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:28:09,562 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:28:09,562 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:28:09,567 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:28:10,055 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:28:11,058 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:28:13,607 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:28:17,636 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:28:17,637 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:28:17,637 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:28:17,642 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:28:18,062 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:28:18,643 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:28:19,065 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:28:22,785 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:28:22,786 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:28:22,786 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:28:22,791 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:28:23,067 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:28:23,265 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:28:23,266 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:28:23,266 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:28:24,560 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:28:25,071 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:28:26,735 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:28:26,736 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:28:26,736 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:28:26,741 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:28:27,073 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:28:29,076 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:28:29,779 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:28:33,670 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 20:28:33,990 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:28:33,991 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:28:33,991 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:28:33,996 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:28:34,080 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:28:34,996 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:28:37,084 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:28:37,795 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:28:37,796 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:28:37,796 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:28:37,801 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:28:38,084 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:28:38,266 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:28:38,266 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:28:38,266 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:28:39,085 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:28:40,549 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:28:41,744 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:28:41,744 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:28:41,745 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:28:41,749 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:28:42,087 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:28:43,090 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:28:45,787 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:28:46,024 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:28:46,025 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:28:46,025 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:28:46,030 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:28:46,091 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:28:49,095 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:28:51,031 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:28:53,266 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:28:53,266 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:28:53,266 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:28:53,815 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:28:53,815 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:28:53,816 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:28:53,820 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:28:54,100 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:28:55,103 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:28:56,859 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:28:57,814 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:28:57,815 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:28:57,815 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:28:57,820 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:28:58,105 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:28:59,107 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:29:01,497 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:29:01,498 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:29:01,498 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:29:01,503 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:29:02,109 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:29:02,504 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:29:03,112 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:29:03,675 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 20:29:05,402 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:29:05,403 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:29:05,403 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:29:05,408 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:29:06,114 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:29:07,117 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:29:08,266 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:29:08,266 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:29:08,266 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:29:08,521 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:29:09,380 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:29:09,381 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:29:09,381 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:29:09,386 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:29:10,119 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:29:11,122 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:29:14,423 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:29:17,929 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:29:17,930 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:29:17,930 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:29:17,935 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:29:18,126 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:29:19,129 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:29:19,936 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:29:21,467 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:29:21,468 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:29:21,468 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:29:21,473 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:29:22,130 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:29:23,133 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:29:23,266 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:29:23,266 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:29:23,266 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:29:25,494 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:29:26,628 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:29:26,629 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:29:26,629 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:29:26,634 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:29:27,136 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:29:29,139 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:29:30,671 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:29:30,756 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:29:30,757 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:29:30,757 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:29:30,762 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:29:31,141 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:29:33,144 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:29:33,675 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 20:29:34,939 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:29:34,939 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:29:34,940 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:29:34,944 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:29:35,146 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:29:35,945 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:29:37,149 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:29:38,266 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:29:38,266 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:29:38,266 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:29:38,642 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:29:38,642 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:29:38,642 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:29:38,647 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:29:39,150 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:29:41,152 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:29:41,687 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:29:44,098 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:29:44,099 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:29:44,099 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:29:44,104 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:29:44,153 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:29:47,105 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:29:47,157 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:29:47,558 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:29:47,559 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:29:47,559 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:29:47,565 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:29:48,158 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:29:49,161 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:29:51,184 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:29:51,184 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:29:51,185 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:29:51,189 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:29:52,163 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:29:52,190 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:29:53,167 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:29:53,266 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:29:53,266 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:29:53,266 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:29:57,528 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:29:59,754 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:29:59,754 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:29:59,755 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:29:59,759 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:30:00,171 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:30:01,174 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:30:02,799 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:30:03,680 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 20:30:04,278 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:30:04,278 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:30:04,279 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:30:04,283 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:30:05,177 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:30:07,180 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:30:08,266 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:30:08,266 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:30:08,267 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:30:08,391 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:30:08,496 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:30:08,497 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:30:08,497 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:30:08,502 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:30:09,181 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:30:11,185 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:30:13,502 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:30:16,960 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:30:16,960 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:30:16,960 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:30:16,965 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:30:17,188 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:30:18,966 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:30:19,191 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:30:20,681 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:30:20,682 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:30:20,682 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:30:20,687 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:30:21,193 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:30:23,196 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:30:23,266 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:30:23,266 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:30:23,267 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:30:24,165 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:30:24,166 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:30:24,166 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:30:24,166 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:30:24,171 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:30:24,196 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:30:27,200 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:30:28,112 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:30:28,112 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:30:28,113 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:30:28,118 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:30:28,201 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:30:30,118 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:30:31,205 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:30:33,679 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 20:30:34,126 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:30:34,127 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:30:34,127 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:30:34,131 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:30:34,208 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:30:35,132 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:30:37,212 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:30:38,266 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:30:38,266 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:30:38,267 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:30:38,491 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:30:38,569 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:30:38,569 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:30:38,574 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:30:39,213 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:30:40,575 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:30:41,216 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:30:42,018 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:30:42,019 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:30:42,019 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:30:42,024 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:30:42,217 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:30:45,221 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:30:45,450 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:30:45,451 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:30:45,451 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:30:45,456 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:30:46,221 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:30:46,457 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:30:47,224 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:30:48,845 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:30:48,846 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:30:48,847 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:30:48,851 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:30:49,225 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:30:51,229 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:30:51,852 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:30:52,509 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:30:52,510 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:30:52,510 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:30:52,515 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:30:53,232 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:30:53,266 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:30:53,266 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:30:53,267 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:30:55,235 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:30:57,496 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:30:58,423 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:30:58,423 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:30:58,424 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:30:58,428 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:30:59,237 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:31:01,241 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:31:01,330 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:31:01,331 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:31:01,331 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:31:01,336 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:31:02,241 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:31:03,242 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:31:03,336 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:31:03,685 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 20:31:06,953 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:31:06,954 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:31:06,954 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:31:06,959 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:31:07,245 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:31:08,266 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:31:08,267 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:31:08,267 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:31:08,985 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:31:09,248 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:31:13,986 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:31:15,529 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:31:15,530 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:31:15,530 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:31:15,535 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:31:16,253 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:31:17,255 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:31:18,990 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:31:18,991 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:31:18,991 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:31:18,992 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:31:18,997 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:31:19,257 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:31:21,260 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:31:22,122 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:31:22,123 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:31:22,123 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:31:22,128 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:31:22,261 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:31:23,266 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:31:23,266 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:31:23,267 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:31:24,528 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:31:25,265 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:31:28,149 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:31:28,150 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:31:28,150 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:31:28,155 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:31:28,266 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:31:30,195 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:31:31,271 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:31:32,085 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:31:32,086 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:31:32,086 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:31:32,091 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:31:32,272 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:31:33,274 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:31:33,683 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 20:31:35,685 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:31:38,267 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:31:38,267 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:31:38,267 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:31:38,950 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:31:38,951 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:31:38,951 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:31:38,956 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:31:39,278 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:31:40,995 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:31:41,280 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:31:42,596 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:31:42,597 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:31:42,597 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:31:42,602 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:31:43,281 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:31:45,284 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:31:46,111 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:31:46,112 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:31:46,112 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:31:46,113 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:31:46,117 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:31:46,285 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:31:49,289 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:31:49,628 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:31:49,629 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:31:49,629 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:31:49,634 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:31:50,290 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:31:51,292 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:31:51,635 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:31:53,266 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:31:53,267 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:31:53,267 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:31:56,474 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:31:56,474 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:31:56,475 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:31:56,479 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:31:57,297 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:31:57,519 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:31:59,300 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:32:00,393 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:32:00,394 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:32:00,394 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:32:00,399 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:32:01,302 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:32:03,305 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:32:03,400 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:32:03,685 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 20:32:04,497 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:32:04,498 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:32:04,498 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:32:04,503 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:32:05,307 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:32:07,310 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:32:08,267 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:32:08,267 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:32:08,267 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:32:08,560 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:32:10,123 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:32:10,124 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:32:10,124 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:32:10,129 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:32:10,312 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:32:13,316 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:32:14,171 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:32:14,717 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:32:14,718 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:32:14,718 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:32:14,723 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:32:15,317 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:32:17,320 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:32:18,840 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:32:18,841 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:32:18,842 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:32:18,846 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:32:19,321 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:32:19,847 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:32:21,325 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:32:23,267 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:32:23,267 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:32:23,267 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:32:23,439 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:32:23,515 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:32:23,515 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:32:23,520 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:32:24,327 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:32:25,329 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:32:25,521 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:32:27,783 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:32:27,784 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:32:27,784 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:32:27,789 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:32:28,331 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:32:29,334 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:32:30,827 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:32:33,030 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:32:33,031 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:32:33,031 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:32:33,036 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:32:33,338 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:32:33,688 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 20:32:35,341 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:32:36,685 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:32:36,686 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:32:36,686 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:32:36,686 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:32:36,691 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:32:37,342 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:32:38,267 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:32:38,267 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:32:38,267 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:32:39,345 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:32:40,159 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:32:40,160 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:32:40,160 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:32:40,165 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:32:40,346 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:32:42,203 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:32:43,350 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:32:44,039 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:32:44,040 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:32:44,041 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:32:44,046 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:32:44,350 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:32:45,353 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:32:48,047 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:32:49,702 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:32:49,703 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:32:49,703 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:32:49,708 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:32:50,357 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:32:51,359 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:32:53,267 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:32:53,267 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:32:53,267 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:32:53,523 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:32:53,594 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:32:53,594 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:32:53,594 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:32:53,599 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:32:54,362 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:32:55,365 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:32:57,444 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:32:57,445 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:32:57,445 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:32:57,450 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:32:58,367 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:32:59,370 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:32:59,450 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:33:02,748 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:33:02,749 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:33:02,749 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:33:02,754 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:33:03,372 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:33:03,690 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 20:33:04,692 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:33:05,376 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:33:06,078 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:33:06,079 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:33:06,079 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:33:06,084 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:33:06,377 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:33:07,379 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:33:08,267 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:33:08,267 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:33:08,268 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:33:10,371 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:33:10,371 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:33:10,372 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:33:10,372 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:33:10,376 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:33:10,381 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:33:13,385 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:33:15,415 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:33:16,824 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:33:16,828 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:33:16,828 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:33:16,833 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:33:17,388 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:33:19,391 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:33:20,834 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:33:20,881 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:33:20,881 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:33:20,882 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:33:20,886 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:33:21,392 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:33:23,267 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:33:23,267 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:33:23,267 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:33:23,395 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:33:25,247 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:33:25,248 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:33:25,248 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:33:25,253 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:33:25,396 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:33:26,291 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:33:27,400 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:33:30,242 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:33:30,243 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:33:30,243 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:33:30,248 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:33:30,402 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:33:32,248 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:33:33,407 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:33:33,692 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 20:33:34,052 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:33:34,052 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:33:34,053 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:33:34,057 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:33:34,407 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:33:35,408 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:33:38,058 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:33:38,173 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:33:38,174 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:33:38,174 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:33:38,179 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:33:38,267 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:33:38,267 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:33:38,268 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:33:38,410 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:33:39,413 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:33:42,572 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:33:42,573 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:33:42,573 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:33:42,579 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:33:43,415 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:33:43,615 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:33:45,418 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:33:45,827 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:33:45,827 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:33:45,828 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:33:45,833 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:33:46,419 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:33:47,422 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:33:48,833 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:33:49,957 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:33:49,958 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:33:49,958 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:33:49,963 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:33:50,424 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:33:51,427 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:33:53,267 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:33:53,267 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:33:53,268 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:33:53,941 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:33:53,942 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:33:53,942 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:33:53,942 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:33:53,947 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:33:54,430 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:33:55,433 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:33:57,074 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:33:57,075 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:33:57,075 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:33:57,080 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:33:57,434 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:33:59,080 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:33:59,435 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:34:00,960 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:34:00,961 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:34:00,961 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:34:00,966 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:34:01,437 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:34:03,440 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:34:03,695 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 20:34:04,621 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:34:04,622 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:34:04,622 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:34:04,622 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:34:04,627 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:34:05,442 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:34:07,445 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:34:08,267 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:34:08,268 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:34:08,268 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:34:10,485 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:34:10,486 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:34:10,486 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:34:10,486 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:34:10,491 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:34:11,447 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:34:13,450 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:34:13,751 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:34:13,752 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:34:13,752 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:34:13,757 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:34:14,451 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:34:15,454 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:34:15,758 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:34:18,344 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:34:18,345 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:34:18,346 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:34:18,351 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:34:18,455 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:34:21,351 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:34:21,457 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:34:23,267 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:34:23,268 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:34:23,268 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:34:24,086 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:34:24,086 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:34:24,087 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:34:24,092 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:34:24,459 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:34:25,462 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:34:27,131 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:34:28,368 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:34:28,368 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:34:28,369 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:34:28,374 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:34:28,464 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:34:31,469 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:34:32,375 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:34:32,971 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:34:32,972 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:34:32,972 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:34:32,977 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:34:33,471 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:34:33,697 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 20:34:35,474 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:34:37,084 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:34:37,085 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:34:37,085 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:34:37,090 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:34:37,475 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:34:38,090 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:34:38,267 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:34:38,268 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:34:38,268 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:34:39,478 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:34:40,800 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:34:40,800 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:34:40,801 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:34:40,805 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:34:41,479 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:34:43,482 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:34:43,843 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:34:44,104 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:34:44,105 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:34:44,105 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:34:44,110 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:34:44,483 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:34:45,486 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:34:49,111 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:34:49,584 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:34:49,585 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:34:49,585 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:34:49,590 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:34:50,489 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:34:51,492 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:34:52,850 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:34:52,850 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:34:52,851 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:34:52,855 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:34:53,268 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:34:53,269 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:34:53,269 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:34:53,494 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:34:54,511 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:34:55,497 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:34:59,511 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:35:00,921 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:35:00,922 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:35:00,922 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:35:00,927 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:35:01,501 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:35:03,504 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:35:03,699 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 20:35:04,701 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:35:05,639 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:35:05,640 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:35:05,640 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:35:05,645 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:35:06,507 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:35:07,509 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:35:08,268 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:35:08,268 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:35:08,268 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:35:09,393 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:35:09,394 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:35:09,394 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:35:09,399 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:35:09,510 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:35:10,435 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:35:11,514 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:35:12,754 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:35:12,755 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:35:12,755 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:35:12,760 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:35:13,515 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:35:15,518 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:35:15,761 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:35:20,101 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:35:20,102 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:35:20,103 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:35:20,107 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:35:20,521 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:35:21,108 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:35:21,523 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:35:23,268 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:35:23,268 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:35:23,270 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:35:24,460 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:35:24,460 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:35:24,461 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:35:24,465 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:35:24,525 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:35:26,506 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:35:27,529 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:35:28,197 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:35:28,198 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:35:28,198 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:35:28,203 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:35:28,530 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:35:29,533 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:35:31,689 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:35:31,690 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:35:31,691 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:35:31,691 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:35:31,696 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:35:32,536 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:35:33,539 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:35:33,702 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 20:35:35,841 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:35:35,842 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:35:35,842 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:35:35,847 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:35:36,541 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:35:36,848 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:35:37,543 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:35:38,268 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:35:38,268 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:35:38,270 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:35:39,530 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:35:39,531 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:35:39,531 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:35:39,536 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:35:39,544 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:35:41,548 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:35:42,575 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:35:42,831 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:35:42,832 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:35:42,832 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:35:42,837 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:35:43,549 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:35:45,552 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:35:47,838 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:35:48,316 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:35:48,317 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:35:48,317 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:35:48,322 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:35:48,554 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:35:49,557 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:35:51,744 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:35:51,745 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:35:51,745 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:35:51,750 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:35:52,559 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:35:53,268 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:35:53,268 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:35:53,269 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:35:53,484 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:35:53,562 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:35:54,769 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:35:54,769 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:35:54,770 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:35:54,774 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:35:55,564 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:35:57,567 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:35:58,811 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:36:00,171 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:36:00,172 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:36:00,172 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:36:00,177 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:36:00,570 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:36:01,572 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:36:03,704 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 20:36:03,879 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:36:03,880 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:36:03,880 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:36:03,880 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:36:03,885 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:36:04,575 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:36:05,575 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:36:08,268 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:36:08,268 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:36:08,269 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:36:08,848 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:36:08,849 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:36:08,849 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:36:08,854 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:36:08,891 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:36:09,578 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:36:11,581 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:36:13,682 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:36:13,683 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:36:13,684 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:36:13,688 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:36:14,583 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:36:14,689 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:36:15,586 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:36:18,271 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:36:18,272 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:36:18,273 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:36:18,278 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:36:18,588 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:36:19,590 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:36:20,278 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:36:22,016 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:36:22,017 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:36:22,017 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:36:22,023 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:36:22,592 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:36:23,268 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:36:23,268 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:36:23,269 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:36:23,593 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:36:25,570 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:36:25,838 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:36:25,843 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:36:25,843 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:36:25,847 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:36:26,594 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:36:27,597 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:36:30,461 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:36:30,462 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:36:30,462 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:36:30,466 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:36:30,599 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:36:31,467 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:36:31,602 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:36:33,706 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 20:36:36,303 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:36:36,304 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:36:36,305 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:36:36,309 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:36:36,606 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:36:37,310 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:36:37,608 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:36:38,268 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:36:38,268 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:36:38,269 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:36:41,971 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:36:41,971 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:36:41,971 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:36:41,976 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:36:42,611 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:36:43,015 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:36:43,614 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:36:45,516 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:36:45,516 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:36:45,517 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:36:45,522 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:36:45,615 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:36:47,616 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:36:48,522 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:36:51,664 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:36:51,664 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:36:51,665 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:36:51,669 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:36:52,620 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:36:53,268 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:36:53,268 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:36:53,269 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:36:53,544 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:36:53,623 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:36:58,545 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:36:58,728 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:36:58,728 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:36:58,728 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:36:58,733 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:36:59,627 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:37:01,630 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:37:02,605 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:37:02,606 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:37:02,606 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:37:02,611 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:37:02,631 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:37:03,612 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:37:03,708 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 20:37:05,635 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:37:06,571 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:37:06,572 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:37:06,572 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:37:06,577 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:37:06,636 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:37:08,268 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:37:08,269 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:37:08,269 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:37:09,486 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:37:09,640 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:37:09,777 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:37:09,778 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:37:09,778 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:37:09,783 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:37:10,641 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:37:11,643 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:37:14,568 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:37:14,569 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:37:14,569 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:37:14,570 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:37:14,574 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:37:14,645 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:37:17,649 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:37:19,575 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:37:20,402 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:37:20,403 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:37:20,403 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:37:20,408 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:37:20,651 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:37:21,653 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:37:23,269 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:37:23,269 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:37:23,269 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:37:24,323 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:37:24,324 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:37:24,324 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:37:24,329 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:37:24,655 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:37:25,366 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:37:25,656 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:37:28,283 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:37:28,284 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:37:28,284 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:37:28,289 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:37:28,658 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:37:29,660 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:37:31,290 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:37:32,673 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:37:32,674 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:37:32,674 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:37:32,679 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:37:33,664 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:37:33,714 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 20:37:35,667 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:37:36,715 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:37:37,417 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:37:37,418 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:37:37,418 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:37:37,423 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:37:37,668 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:37:38,290 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:37:38,290 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:37:38,291 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:37:39,671 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:37:42,417 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:37:42,418 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:37:42,418 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:37:42,418 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:37:42,423 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:37:42,673 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:37:43,676 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:37:46,197 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:37:46,198 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:37:46,199 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:37:46,204 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:37:46,678 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:37:47,678 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:37:48,204 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:37:50,307 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:37:50,307 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:37:50,308 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:37:50,313 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:37:50,681 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:37:51,683 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:37:53,290 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:37:53,290 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:37:53,290 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:37:53,539 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:37:54,254 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:37:54,254 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:37:54,254 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:37:54,259 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:37:54,686 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:37:55,689 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:37:57,879 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:37:57,880 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:37:57,880 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:37:57,885 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:37:58,691 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:37:58,885 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:37:59,693 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:38:03,713 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 20:38:04,018 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:38:04,019 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:38:04,019 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:38:04,019 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:38:04,024 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:38:04,697 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:38:05,699 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:38:07,840 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:38:07,841 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:38:07,842 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:38:07,846 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:38:08,290 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:38:08,290 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:38:08,290 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:38:08,701 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:38:09,508 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:38:09,702 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:38:12,443 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:38:12,444 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:38:12,444 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:38:12,449 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:38:12,704 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:38:13,706 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:38:15,486 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:38:17,436 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:38:17,437 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:38:17,437 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:38:17,443 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:38:17,709 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:38:19,712 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:38:21,369 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:38:21,370 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:38:21,370 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:38:21,370 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:38:21,375 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:38:21,713 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:38:23,290 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:38:23,290 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:38:23,290 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:38:23,717 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:38:25,893 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:38:25,894 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:38:25,894 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:38:25,898 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:38:26,719 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:38:26,934 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:38:27,721 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:38:29,931 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:38:29,931 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:38:29,931 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:38:29,936 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:38:30,723 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:38:31,727 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:38:31,937 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:38:33,718 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 20:38:34,570 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:38:34,571 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:38:34,571 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:38:34,576 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:38:34,729 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:38:35,732 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:38:37,576 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:38:38,069 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:38:38,070 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:38:38,070 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:38:38,075 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:38:38,290 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:38:38,290 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:38:38,291 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:38:38,733 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:38:39,736 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:38:42,437 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:38:42,438 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:38:42,438 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:38:42,443 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:38:42,738 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:38:43,478 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:38:43,740 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:38:46,154 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:38:46,155 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:38:46,155 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:38:46,160 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:38:46,742 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:38:47,745 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:38:49,160 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:38:51,374 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:38:51,375 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:38:51,375 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:38:51,380 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:38:51,748 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:38:53,290 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:38:53,290 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:38:53,290 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:38:53,752 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:38:54,592 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:38:54,956 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:38:54,957 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:38:54,957 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:38:54,962 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:38:55,753 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:38:57,757 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:38:58,994 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:38:58,995 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:38:58,995 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:38:58,999 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:38:59,758 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:39:00,000 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:39:01,761 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:39:02,559 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:39:02,560 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:39:02,560 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:39:02,565 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:39:02,762 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:39:03,717 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 20:39:03,764 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:39:05,719 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:39:08,290 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:39:08,290 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:39:08,290 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:39:08,413 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:39:08,585 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:39:08,585 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:39:08,590 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:39:08,768 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:39:09,770 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:39:11,591 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:39:12,202 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:39:12,203 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:39:12,203 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:39:12,207 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:39:12,772 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:39:13,773 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:39:17,208 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:39:21,164 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:39:21,165 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:39:21,165 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:39:21,170 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:39:21,778 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:39:23,170 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:39:23,290 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:39:23,290 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:39:23,292 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:39:23,781 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:39:24,804 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:39:24,804 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:39:24,804 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:39:24,809 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:39:25,782 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:39:27,785 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:39:28,846 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:39:29,059 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:39:29,059 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:39:29,060 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:39:29,065 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:39:29,786 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:39:31,790 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:39:33,718 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 20:39:33,825 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:39:33,825 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:39:33,826 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:39:33,831 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:39:34,793 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:39:34,831 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:39:35,795 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:39:38,290 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:39:38,290 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:39:38,290 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:39:38,574 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:39:38,575 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:39:38,575 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:39:38,580 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:39:38,797 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:39:39,800 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:39:40,618 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:39:45,618 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:39:47,012 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:39:47,012 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:39:47,013 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:39:47,017 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:39:47,805 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:39:49,808 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:39:50,547 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:39:50,548 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:39:50,548 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:39:50,553 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:39:50,809 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:39:51,553 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:39:51,812 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:39:53,290 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:39:53,291 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:39:53,291 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:39:54,453 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:39:54,454 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:39:54,454 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:39:54,459 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:39:54,815 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:39:55,817 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:39:57,498 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:39:58,203 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:39:58,204 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:39:58,204 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:39:58,209 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:39:58,819 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:39:59,822 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:40:01,574 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:40:01,575 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:40:01,575 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:40:01,580 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:40:01,823 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:40:02,580 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:40:03,720 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 20:40:03,826 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:40:07,722 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:40:08,290 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:40:08,290 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:40:08,291 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:40:08,727 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:40:08,728 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:40:08,728 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:40:08,733 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:40:08,830 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:40:09,832 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:40:12,770 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:40:12,966 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:40:12,967 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:40:12,967 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:40:12,972 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:40:13,835 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:40:15,838 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:40:17,298 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:40:17,299 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:40:17,299 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:40:17,304 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:40:17,839 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:40:18,304 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:40:19,843 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:40:23,290 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:40:23,291 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:40:23,291 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:40:23,502 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:40:25,776 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:40:25,777 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:40:25,777 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:40:25,782 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:40:25,846 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:40:27,849 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:40:28,818 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:40:29,645 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:40:29,645 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:40:29,646 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:40:29,650 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:40:29,851 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:40:31,855 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:40:33,709 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:40:33,709 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:40:33,710 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:40:33,715 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:40:33,722 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 20:40:33,857 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:40:34,724 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:40:35,860 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:40:37,257 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:40:37,258 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:40:37,258 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:40:37,263 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:40:37,861 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:40:38,290 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:40:38,291 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:40:38,291 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:40:39,864 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:40:40,507 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:40:44,262 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:40:44,262 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:40:44,262 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:40:44,267 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:40:44,867 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:40:45,868 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:40:46,306 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:40:48,407 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:40:48,407 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:40:48,407 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:40:48,412 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:40:48,870 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:40:49,872 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:40:51,413 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:40:52,146 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:40:52,147 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:40:52,147 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:40:52,152 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:40:52,875 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:40:53,291 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:40:53,291 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:40:53,292 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:40:53,879 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:40:56,549 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:40:57,711 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:40:57,715 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:40:57,715 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:40:57,720 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:40:57,881 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:40:59,885 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:41:01,551 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:41:01,551 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:41:01,551 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:41:01,552 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:41:01,556 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:41:01,886 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:41:03,725 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 20:41:03,889 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:41:05,466 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:41:05,466 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:41:05,467 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:41:05,472 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:41:05,891 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:41:07,472 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:41:07,894 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:41:08,291 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:41:08,291 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:41:08,292 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:41:08,460 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:41:08,521 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:41:08,521 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:41:08,526 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:41:08,895 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:41:09,897 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:41:12,047 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:41:12,047 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:41:12,047 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:41:12,052 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:41:12,899 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:41:13,090 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:41:13,900 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:41:16,353 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:41:16,354 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:41:16,354 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:41:16,359 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:41:16,902 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:41:17,904 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:41:18,359 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:41:20,238 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:41:20,238 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:41:20,239 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:41:20,244 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:41:20,906 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:41:21,909 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:41:23,291 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:41:23,291 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:41:23,292 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:41:23,537 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:41:24,579 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:41:24,580 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:41:24,580 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:41:24,585 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:41:24,911 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:41:25,912 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:41:28,622 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:41:28,933 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:41:28,933 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:41:28,934 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:41:28,938 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:41:29,914 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:41:31,916 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:41:33,090 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:41:33,091 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:41:33,091 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:41:33,096 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:41:33,727 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 20:41:33,728 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:41:33,918 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:41:35,921 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:41:36,737 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:41:36,738 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:41:36,738 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:41:36,743 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:41:36,922 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:41:37,924 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:41:38,291 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:41:38,291 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:41:38,292 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:41:39,521 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:41:40,897 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:41:40,897 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:41:40,898 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:41:40,903 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:41:40,926 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:41:43,930 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:41:44,295 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:41:44,296 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:41:44,296 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:41:44,301 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:41:44,931 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:41:45,302 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:41:45,933 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:41:50,302 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:41:50,478 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:41:50,479 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:41:50,479 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:41:50,484 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:41:50,937 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:41:51,939 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:41:53,291 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:41:53,291 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:41:53,292 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:41:54,280 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:41:54,281 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:41:54,281 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:41:54,286 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:41:54,942 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:41:55,325 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:41:55,943 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:42:00,326 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:42:02,832 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:42:02,833 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:42:02,833 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:42:02,838 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:42:02,947 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:42:03,729 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 20:42:03,950 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:42:05,731 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:42:06,942 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:42:06,943 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:42:06,943 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:42:06,948 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:42:06,952 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:42:08,291 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:42:08,291 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:42:08,293 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:42:09,956 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:42:10,225 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:42:10,226 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:42:10,226 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:42:10,231 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:42:10,957 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:42:11,267 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:42:11,959 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:42:16,267 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:42:16,383 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:42:16,384 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:42:16,384 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:42:16,389 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:42:16,963 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:42:17,966 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:42:20,131 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:42:20,131 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:42:20,132 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:42:20,136 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:42:20,968 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:42:21,968 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:42:22,137 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:42:23,291 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:42:23,291 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:42:23,293 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:42:24,684 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:42:24,684 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:42:24,684 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:42:24,689 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:42:24,970 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:42:25,973 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:42:27,727 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:42:28,438 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:42:28,438 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:42:28,439 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:42:28,443 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:42:28,975 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:42:29,978 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:42:32,534 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:42:32,535 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:42:32,535 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:42:32,540 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:42:32,981 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:42:33,540 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:42:33,731 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 20:42:33,984 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:42:36,228 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:42:36,229 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:42:36,229 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:42:36,234 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:42:36,986 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:42:37,989 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:42:38,291 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:42:38,291 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:42:38,293 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:42:38,593 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:42:40,733 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:42:40,733 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:42:40,734 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:42:40,739 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:42:40,991 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:42:41,993 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:42:43,775 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:42:44,651 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:42:44,652 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:42:44,652 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:42:44,657 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:42:44,995 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:42:45,998 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:42:49,658 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:42:53,291 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:42:53,291 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:42:53,293 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:42:53,375 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:42:53,556 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:42:53,556 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:42:53,561 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:42:54,009 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:42:55,562 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:42:56,012 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:42:57,963 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:42:57,963 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:42:57,963 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:42:57,968 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:42:58,013 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:43:00,017 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:43:00,969 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:43:02,126 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:43:02,126 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:43:02,127 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:43:02,131 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:43:03,020 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:43:03,737 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 20:43:04,023 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:43:06,544 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:43:06,544 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:43:06,545 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:43:06,545 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:43:06,550 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:43:07,026 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:43:08,029 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:43:08,291 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:43:08,291 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:43:08,293 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:43:11,197 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:43:11,198 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:43:11,198 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:43:11,204 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:43:12,033 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:43:12,243 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:43:14,038 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:43:16,188 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:43:16,189 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:43:16,189 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:43:16,194 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:43:17,040 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:43:18,043 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:43:18,195 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:43:20,388 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:43:20,388 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:43:20,388 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:43:20,393 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:43:21,045 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:43:22,049 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:43:23,291 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:43:23,292 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:43:23,293 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:43:23,521 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:43:28,522 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:43:28,894 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:43:28,895 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:43:28,895 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:43:28,900 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:43:29,055 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:43:30,060 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:43:33,735 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 20:43:33,737 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:43:35,882 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:43:35,882 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:43:35,883 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:43:35,888 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:43:36,067 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:43:38,072 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:43:38,291 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:43:38,292 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:43:38,293 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:43:39,542 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:43:40,365 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:43:40,366 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:43:40,366 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:43:40,371 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:43:41,074 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:43:42,077 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:43:44,046 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:43:44,047 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:43:44,047 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:43:44,052 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:43:44,078 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:43:45,053 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:43:46,083 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:43:48,249 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:43:48,250 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:43:48,250 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:43:48,255 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:43:49,084 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:43:50,088 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:43:50,255 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:43:51,735 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:43:51,736 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:43:51,736 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:43:51,741 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:43:52,090 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:43:53,291 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:43:53,292 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:43:53,292 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:43:54,093 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:43:55,241 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:43:55,241 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:43:55,242 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:43:55,247 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:43:55,283 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:43:56,095 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:43:58,099 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:43:58,631 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:43:58,631 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:43:58,632 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:43:58,636 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:43:59,100 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:44:00,102 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:44:00,637 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:44:02,718 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:44:02,719 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:44:02,719 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:44:02,724 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:44:03,104 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:44:03,741 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 20:44:04,108 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:44:05,742 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:44:07,221 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:44:07,221 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:44:07,222 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:44:07,227 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:44:08,112 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:44:08,292 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:44:08,292 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:44:08,292 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:44:10,117 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:44:11,513 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:44:12,174 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:44:12,175 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:44:12,175 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:44:12,180 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:44:13,119 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:44:14,123 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:44:17,219 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:44:18,077 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:44:18,078 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:44:18,078 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:44:18,083 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:44:18,126 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:44:20,131 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:44:21,821 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:44:21,822 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:44:21,822 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:44:21,827 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:44:22,132 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:44:22,827 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:44:23,292 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:44:23,292 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:44:23,293 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:44:24,137 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:44:28,545 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:44:28,707 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:44:28,708 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:44:28,708 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:44:28,713 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:44:29,141 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:44:30,145 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:44:32,223 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:44:32,223 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:44:32,224 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:44:32,228 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:44:33,148 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:44:33,740 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 20:44:33,741 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:44:34,150 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:44:38,292 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:44:38,292 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:44:38,292 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:44:39,184 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:44:39,185 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:44:39,185 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:44:39,186 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:44:39,191 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:44:40,158 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:44:42,162 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:44:44,227 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:44:47,419 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:44:47,420 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:44:47,420 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:44:47,425 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:44:48,169 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:44:49,426 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:44:50,174 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:44:50,727 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:44:50,728 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:44:50,728 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:44:50,733 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:44:51,175 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:44:52,177 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:44:53,292 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:44:53,292 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:44:53,294 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:44:54,500 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:44:57,363 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:44:57,363 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:44:57,364 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:44:57,369 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:44:58,183 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:45:00,187 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:45:00,407 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:45:02,172 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:45:02,173 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:45:02,173 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:45:02,178 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:45:02,189 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:45:03,745 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 20:45:04,194 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:45:05,746 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:45:06,358 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:45:06,358 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:45:06,359 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:45:06,364 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:45:07,196 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:45:08,200 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:45:08,292 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:45:08,292 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:45:08,292 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:45:11,529 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:45:12,662 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:45:12,662 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:45:12,663 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:45:12,668 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:45:13,204 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:45:14,208 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:45:16,707 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:45:17,878 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:45:17,879 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:45:17,879 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:45:17,884 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:45:18,212 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:45:20,216 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:45:21,885 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:45:21,986 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:45:21,987 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:45:21,987 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:45:21,992 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:45:22,218 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:45:23,292 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:45:23,293 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:45:23,293 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:45:24,223 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:45:25,653 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:45:25,653 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:45:25,654 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:45:25,659 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:45:26,224 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:45:27,695 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:45:28,229 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:45:29,879 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:45:29,880 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:45:29,881 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:45:29,886 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:45:30,230 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:45:32,235 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:45:32,886 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:45:33,388 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:45:33,389 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:45:33,389 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:45:33,394 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:45:33,744 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 20:45:34,237 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:45:36,240 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:45:37,928 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:45:37,929 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:45:37,929 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:45:37,929 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:45:37,934 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:45:38,242 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:45:38,292 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:45:38,292 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:45:38,294 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:45:40,246 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:45:41,009 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:45:41,010 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:45:41,010 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:45:41,015 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:45:41,247 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:45:42,250 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:45:43,055 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:45:44,250 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:45:44,251 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:45:44,251 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:45:44,256 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:45:45,253 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:45:46,256 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:45:48,257 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:45:49,193 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:45:49,194 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:45:49,194 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:45:49,198 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:45:49,258 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:45:52,263 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:45:52,665 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:45:52,666 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:45:52,666 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:45:52,671 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:45:53,264 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:45:53,292 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:45:53,292 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:45:53,294 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:45:53,546 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:45:54,268 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:45:57,014 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:45:57,015 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:45:57,015 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:45:57,020 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:45:57,270 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:45:58,272 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:45:59,059 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:46:02,571 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:46:02,572 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:46:02,572 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:46:02,577 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:46:03,277 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:46:03,750 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 20:46:04,281 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:46:04,751 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:46:06,702 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:46:06,703 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:46:06,703 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:46:06,708 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:46:07,283 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:46:08,286 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:46:08,292 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:46:08,293 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:46:08,293 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:46:10,255 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:46:10,255 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:46:10,255 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:46:10,256 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:46:10,260 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:46:10,288 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:46:12,292 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:46:15,299 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:46:18,848 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:46:18,848 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:46:18,849 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:46:18,854 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:46:19,299 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:46:20,303 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:46:20,854 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:46:22,616 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:46:22,616 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:46:22,617 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:46:22,622 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:46:23,292 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:46:23,293 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:46:23,293 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:46:23,305 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:46:24,307 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:46:26,587 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:46:27,808 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:46:27,808 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:46:27,808 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:46:27,814 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:46:28,313 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:46:30,318 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:46:31,851 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:46:31,863 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:46:31,864 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:46:31,864 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:46:31,869 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:46:32,320 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:46:33,749 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 20:46:34,324 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:46:36,804 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:46:36,805 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:46:36,805 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:46:36,810 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:46:37,326 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:46:37,811 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:46:38,292 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:46:38,293 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:46:38,293 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:46:38,330 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:46:40,360 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:46:40,361 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:46:40,361 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:46:40,366 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:46:41,332 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:46:42,333 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:46:43,403 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:46:46,132 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:46:46,133 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:46:46,133 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:46:46,138 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:46:46,339 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:46:48,341 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:46:49,139 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:46:51,210 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:46:51,211 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:46:51,211 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:46:51,216 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:46:51,346 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:46:52,349 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:46:53,293 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:46:53,293 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:46:53,293 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:46:54,557 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:46:56,331 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:46:56,331 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:46:56,331 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:46:56,336 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:46:56,353 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:46:58,358 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:46:59,301 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:46:59,302 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:46:59,302 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:46:59,307 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:46:59,359 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:47:00,308 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:47:02,364 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:47:03,417 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:47:03,418 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:47:03,418 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:47:03,423 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:47:03,754 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 20:47:04,365 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:47:05,755 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:47:06,368 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:47:07,613 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:47:07,614 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:47:07,614 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:47:07,619 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:47:08,293 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:47:08,293 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:47:08,293 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:47:08,371 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:47:10,376 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:47:11,096 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:47:11,096 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:47:11,097 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:47:11,097 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:47:11,102 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:47:11,377 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:47:12,379 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:47:14,477 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:47:14,478 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:47:14,478 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:47:14,483 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:47:15,381 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:47:16,382 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:47:16,484 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:47:17,769 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:47:17,769 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:47:17,770 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:47:17,774 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:47:18,385 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:47:20,390 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:47:21,775 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:47:22,458 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:47:22,458 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:47:22,459 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:47:22,464 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:47:23,293 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:47:23,293 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:47:23,295 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:47:23,392 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:47:24,395 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:47:26,014 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:47:26,015 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:47:26,015 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:47:26,020 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:47:26,397 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:47:27,058 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:47:28,401 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:47:30,554 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:47:30,554 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:47:30,555 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:47:30,560 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:47:31,403 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:47:32,405 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:47:32,561 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:47:33,753 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 20:47:34,895 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:47:34,896 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:47:34,896 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:47:34,901 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:47:35,409 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:47:36,413 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:47:37,902 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:47:38,293 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:47:38,293 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:47:38,293 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:47:40,348 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:47:40,348 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:47:40,348 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:47:40,353 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:47:40,417 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:47:42,422 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:47:43,390 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:47:45,203 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:47:45,204 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:47:45,204 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:47:45,210 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:47:45,424 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:47:46,427 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:47:49,210 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:47:50,100 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:47:50,101 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:47:50,101 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:47:50,106 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:47:50,431 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:47:52,435 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:47:53,293 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:47:53,294 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:47:53,294 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:47:54,245 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:47:54,246 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:47:54,246 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:47:54,246 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:47:54,251 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:47:54,455 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:47:56,459 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:47:59,286 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:48:00,403 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:48:00,403 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:48:00,404 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:48:00,409 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:48:00,462 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:48:02,467 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:48:03,755 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 20:48:03,982 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:48:03,982 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:48:03,982 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:48:03,988 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:48:04,469 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:48:04,988 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:48:06,471 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:48:08,293 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:48:08,294 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:48:08,294 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:48:08,475 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:48:08,588 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:48:08,588 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:48:08,593 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:48:09,475 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:48:10,479 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:48:10,594 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:48:13,119 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:48:13,119 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:48:13,119 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:48:13,124 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:48:13,481 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:48:14,484 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:48:16,125 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:48:18,331 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:48:18,332 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:48:18,332 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:48:18,337 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:48:18,487 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:48:20,492 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:48:21,338 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:48:21,954 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:48:21,954 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:48:21,955 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:48:21,960 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:48:22,493 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:48:23,293 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:48:23,294 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:48:23,294 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:48:24,496 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:48:26,251 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:48:26,251 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:48:26,251 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:48:26,256 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:48:26,499 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:48:27,295 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:48:28,503 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:48:31,301 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:48:31,302 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:48:31,302 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:48:31,307 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:48:31,505 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:48:32,307 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:48:32,511 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:48:33,758 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 20:48:35,858 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:48:35,858 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:48:35,858 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:48:35,863 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:48:36,514 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:48:37,864 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:48:38,293 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:48:38,294 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:48:38,294 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:48:38,517 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:48:39,229 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:48:39,230 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:48:39,230 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:48:39,235 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:48:39,518 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:48:40,520 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:48:42,780 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:48:42,781 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:48:42,781 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:48:42,786 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:48:43,522 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:48:43,787 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:48:44,526 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:48:46,150 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:48:46,151 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:48:46,151 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:48:46,156 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:48:46,527 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:48:48,529 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:48:49,157 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:48:50,075 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:48:50,075 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:48:50,075 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:48:50,080 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:48:50,532 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:48:52,537 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:48:53,293 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:48:53,294 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:48:53,294 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:48:53,657 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:48:53,657 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:48:53,658 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:48:53,662 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:48:54,540 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:48:54,699 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:48:55,542 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:48:57,604 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:48:57,604 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:48:57,605 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:48:57,610 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:48:58,544 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:48:59,547 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:49:00,611 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:49:01,921 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:49:01,921 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:49:01,922 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:49:01,927 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:49:02,549 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:49:03,552 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:49:03,760 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 20:49:05,761 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:49:05,816 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:49:05,820 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:49:05,820 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:49:05,825 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:49:06,555 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:49:07,558 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:49:08,293 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:49:08,294 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:49:08,294 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:49:10,338 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:49:10,339 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:49:10,339 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:49:10,344 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:49:10,561 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:49:11,382 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:49:11,563 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:49:16,149 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:49:16,150 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:49:16,150 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:49:16,155 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:49:16,569 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:49:17,156 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:49:17,571 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:49:21,091 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:49:21,092 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:49:21,092 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:49:21,097 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:49:21,577 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:49:21,578 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:49:23,098 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:49:23,294 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:49:23,294 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:49:23,294 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:49:24,514 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:49:24,515 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:49:24,515 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:49:24,520 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:49:24,580 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:49:25,582 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:49:27,901 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:49:27,902 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:49:27,902 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:49:27,907 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:49:28,585 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:49:28,908 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:49:29,588 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:49:32,308 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:49:32,309 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:49:32,309 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:49:32,314 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:49:32,591 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:49:33,593 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:49:33,761 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 20:49:34,762 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:49:37,374 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:49:37,375 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:49:37,375 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:49:37,380 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:49:37,600 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:49:37,600 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:49:38,294 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:49:38,294 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:49:38,294 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:49:40,589 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:49:45,590 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:49:45,684 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:49:45,684 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:49:45,684 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:49:45,689 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:49:46,611 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:49:47,613 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:49:49,375 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:49:49,375 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:49:49,376 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:49:49,381 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:49:49,616 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:49:49,616 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:49:51,382 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:49:53,292 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:49:53,293 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:49:53,294 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:49:53,294 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:49:53,294 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:49:53,299 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:49:53,299 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:49:53,621 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:49:53,622 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:49:56,586 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:50:01,586 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:50:01,827 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:50:01,828 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:50:01,828 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:50:01,833 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:50:02,632 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:50:03,635 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:50:03,763 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 20:50:05,410 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:50:05,411 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:50:05,411 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:50:05,416 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:50:05,638 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:50:05,638 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:50:07,417 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:50:08,294 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:50:08,294 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:50:08,295 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:50:09,045 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:50:09,046 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:50:09,046 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:50:09,051 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:50:09,644 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:50:09,644 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:50:12,499 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:50:12,500 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:50:12,500 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:50:12,500 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:50:12,505 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:50:12,647 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:50:13,649 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:50:17,506 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:50:17,543 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:50:17,543 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:50:17,543 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:50:17,549 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:50:17,653 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:50:19,657 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:50:21,933 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:50:21,934 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:50:21,934 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:50:21,939 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:50:22,661 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:50:22,939 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:50:23,295 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:50:23,295 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:50:23,295 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:50:23,663 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:50:25,652 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:50:25,653 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:50:25,653 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:50:25,658 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:50:25,664 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:50:27,669 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:50:28,695 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:50:29,971 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:50:29,972 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:50:29,972 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:50:29,977 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:50:30,671 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:50:31,674 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:50:33,746 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:50:33,747 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:50:33,747 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:50:33,747 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:50:33,752 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:50:33,766 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 20:50:34,678 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:50:35,681 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:50:37,208 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:50:37,209 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:50:37,209 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:50:37,214 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:50:37,684 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:50:37,684 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:50:38,294 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:50:38,295 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:50:38,295 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:50:39,499 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:50:40,738 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:50:40,738 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:50:40,738 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:50:40,744 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:50:41,689 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:50:41,690 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:50:44,783 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:50:47,608 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:50:47,609 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:50:47,609 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:50:47,614 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:50:47,695 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:50:49,699 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:50:50,615 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:50:51,276 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:50:51,277 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:50:51,277 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:50:51,283 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:50:51,703 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:50:51,703 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:50:53,295 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:50:53,295 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:50:53,295 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:50:54,688 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:50:54,688 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:50:54,688 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:50:54,693 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:50:54,707 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:50:55,709 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:50:55,731 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:50:58,644 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:50:58,645 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:50:58,645 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:50:58,650 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:50:58,711 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:50:59,714 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:51:01,651 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:51:02,196 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:51:02,197 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:51:02,197 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:51:02,202 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:51:02,716 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:51:03,719 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:51:03,769 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 20:51:06,770 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:51:07,843 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:51:07,843 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:51:07,844 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:51:07,848 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:51:08,294 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:51:08,295 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:51:08,295 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:51:08,724 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:51:09,727 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:51:11,731 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:51:11,732 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:51:11,732 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:51:11,737 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:51:11,775 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:51:12,731 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:51:13,733 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:51:15,695 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:51:15,696 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:51:15,696 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:51:15,701 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:51:15,734 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:51:17,702 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:51:17,739 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:51:20,794 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:51:20,795 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:51:20,795 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:51:20,800 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:51:21,744 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:51:21,745 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:51:22,800 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:51:23,295 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:51:23,295 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:51:23,295 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:51:24,026 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:51:24,027 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:51:24,027 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:51:24,032 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:51:24,747 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:51:25,750 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:51:28,071 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:51:28,952 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:51:28,952 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:51:28,953 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:51:28,958 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:51:29,755 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:51:29,755 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:51:33,771 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 20:51:33,773 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:51:37,340 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:51:37,340 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:51:37,341 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:51:37,346 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:51:37,767 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:51:37,767 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:51:38,295 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:51:38,295 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:51:38,295 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:51:39,586 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:51:41,069 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:51:41,070 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:51:41,070 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:51:41,075 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:51:41,772 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:51:41,772 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:51:44,902 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:51:44,902 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:51:44,903 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:51:44,903 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:51:44,908 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:51:45,778 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:51:45,778 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:51:49,842 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:51:49,843 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:51:49,843 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:51:49,848 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:51:50,783 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:51:50,848 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:51:51,786 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:51:53,295 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:51:53,295 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:51:53,295 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:51:54,621 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:51:54,621 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:51:54,622 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:51:54,626 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:51:54,790 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:51:55,791 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:51:56,663 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:51:58,373 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:51:58,374 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:51:58,374 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:51:58,379 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:51:58,794 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:51:59,797 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:52:02,122 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:52:02,123 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:52:02,123 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:52:02,123 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:52:02,128 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:52:02,800 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:52:03,773 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 20:52:03,803 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:52:06,662 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:52:06,663 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:52:06,663 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:52:06,668 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:52:06,806 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:52:07,669 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:52:07,809 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:52:08,295 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:52:08,295 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:52:08,295 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:52:11,439 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:52:11,439 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:52:11,439 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:52:11,444 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:52:11,814 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:52:11,814 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:52:13,483 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:52:15,164 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:52:15,165 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:52:15,165 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:52:15,170 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:52:15,820 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:52:15,821 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:52:18,672 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:52:18,673 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:52:18,673 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:52:18,673 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:52:18,678 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:52:18,823 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:52:19,826 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:52:23,295 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:52:23,295 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:52:23,296 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:52:24,589 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:52:24,590 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:52:24,590 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:52:24,590 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:52:24,595 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:52:24,831 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:52:25,834 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:52:28,387 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:52:28,388 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:52:28,388 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:52:28,393 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:52:28,837 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:52:29,838 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:52:30,393 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:52:32,801 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:52:32,802 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:52:32,802 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:52:32,808 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:52:32,842 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:52:33,775 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 20:52:33,845 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:52:35,777 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:52:36,794 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:52:36,794 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:52:36,794 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:52:36,800 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:52:36,848 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:52:37,851 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:52:38,295 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:52:38,295 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:52:38,295 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:52:41,515 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:52:46,515 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:52:47,515 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:52:47,515 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:52:47,515 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:52:47,520 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:52:47,864 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:52:47,865 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:52:51,558 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:52:51,628 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:52:51,628 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:52:51,629 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:52:51,633 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:52:51,871 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:52:51,871 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:52:53,295 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:52:53,295 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:52:53,296 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:52:55,430 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:52:55,430 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:52:55,430 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:52:55,436 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:52:55,878 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:52:55,878 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:52:57,474 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:52:59,145 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:52:59,146 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:52:59,146 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:52:59,151 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:52:59,882 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:52:59,882 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:53:03,152 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:53:03,778 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 20:53:04,712 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:53:04,713 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:53:04,713 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:53:04,718 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:53:04,887 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:53:05,891 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:53:08,295 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:53:08,295 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:53:08,295 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:53:08,296 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:53:13,232 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:53:13,232 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:53:13,232 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:53:13,238 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:53:13,902 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:53:13,902 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:53:14,275 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:53:17,656 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:53:17,656 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:53:17,657 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:53:17,661 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:53:17,907 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:53:17,908 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:53:19,662 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:53:21,880 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:53:21,881 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:53:21,881 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:53:21,886 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:53:21,911 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:53:23,296 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:53:23,296 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:53:23,296 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:53:23,916 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:53:25,512 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:53:26,148 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:53:26,149 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:53:26,149 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:53:26,154 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:53:26,919 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:53:27,922 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:53:30,242 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:53:30,243 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:53:30,243 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:53:30,248 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:53:30,925 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:53:31,249 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:53:31,927 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:53:33,783 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 20:53:34,027 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:53:34,028 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:53:34,028 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:53:34,033 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:53:34,931 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:53:35,934 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:53:37,034 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:53:38,296 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:53:38,296 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:53:38,296 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:53:38,408 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:53:38,520 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:53:38,521 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:53:38,526 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:53:38,936 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:53:39,939 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:53:42,204 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:53:42,205 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:53:42,205 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:53:42,205 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:53:42,211 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:53:42,942 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:53:43,944 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:53:45,952 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:53:45,952 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:53:45,953 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:53:45,958 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:53:46,947 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:53:47,950 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:53:47,958 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:53:49,882 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:53:49,883 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:53:49,883 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:53:49,888 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:53:49,951 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:53:51,956 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:53:53,296 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:53:53,296 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:53:53,296 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:53:53,296 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:53:55,268 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:53:55,268 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:53:55,268 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:53:55,273 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:53:55,963 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:53:55,963 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:53:58,311 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:53:58,940 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:53:58,941 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:53:58,941 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:53:58,946 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:53:58,965 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:53:59,967 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:54:03,782 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 20:54:03,783 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:54:04,107 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:54:04,107 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:54:04,107 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:54:04,112 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:54:04,972 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:54:05,975 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:54:08,296 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:54:08,296 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:54:08,296 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:54:09,568 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:54:09,569 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:54:09,569 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:54:09,569 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:54:09,574 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:54:09,981 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:54:09,981 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:54:13,843 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:54:13,844 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:54:13,844 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:54:13,849 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:54:13,985 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:54:14,850 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:54:15,989 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:54:19,850 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:54:22,132 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:54:22,133 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:54:22,133 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:54:22,138 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:54:22,998 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:54:23,296 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:54:23,296 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:54:23,296 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:54:24,000 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:54:25,567 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:54:26,061 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:54:26,062 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:54:26,062 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:54:26,067 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:54:27,004 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:54:28,006 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:54:29,807 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:54:29,808 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:54:29,808 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:54:29,813 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:54:30,007 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:54:30,814 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:54:32,012 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:54:33,787 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 20:54:36,337 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:54:36,337 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:54:36,338 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:54:36,338 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:54:36,342 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:54:37,018 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:54:38,021 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:54:38,296 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:54:38,296 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:54:38,296 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:54:39,906 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:54:39,907 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:54:39,907 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:54:39,912 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:54:40,022 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:54:41,950 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:54:42,026 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:54:44,297 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:54:44,298 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:54:44,298 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:54:44,303 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:54:45,029 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:54:46,032 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:54:47,303 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:54:50,524 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:54:50,525 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:54:50,526 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:54:50,530 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:54:51,038 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:54:52,040 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:54:52,531 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:54:53,296 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:54:53,296 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:54:53,296 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:54:54,674 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:54:54,675 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:54:54,675 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:54:54,680 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:54:55,044 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:54:56,047 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:54:57,718 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:54:58,042 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:54:58,043 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:54:58,043 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:54:58,048 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:54:58,048 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:55:00,052 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:55:03,048 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:55:03,787 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 20:55:06,372 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:55:06,373 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:55:06,373 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:55:06,378 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:55:07,058 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:55:08,061 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:55:08,296 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:55:08,296 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:55:08,296 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:55:08,297 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:55:11,169 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:55:11,169 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:55:11,170 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:55:11,175 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:55:12,069 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:55:12,069 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:55:14,210 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:55:14,258 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:55:14,259 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:55:14,259 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:55:14,264 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:55:15,072 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:55:16,074 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:55:17,793 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:55:17,794 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:55:17,794 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:55:17,799 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:55:18,077 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:55:18,078 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:55:19,799 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:55:21,861 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:55:21,861 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:55:21,862 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:55:21,867 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:55:22,081 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:55:23,296 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:55:23,297 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:55:23,297 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:55:24,086 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:55:25,579 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:55:26,207 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:55:26,208 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:55:26,208 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:55:26,213 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:55:27,089 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:55:28,091 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:55:29,991 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:55:29,992 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:55:29,992 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:55:29,997 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:55:30,093 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:55:30,998 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:55:32,097 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:55:33,792 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 20:55:33,819 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:55:33,820 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:55:33,820 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:55:33,825 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:55:34,101 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:55:34,102 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:55:36,825 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:55:37,305 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:55:37,306 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:55:37,306 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:55:37,311 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:55:38,106 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:55:38,106 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:55:38,296 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:55:38,296 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:55:38,297 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:55:42,635 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:55:42,928 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:55:42,929 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:55:42,929 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:55:42,934 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:55:43,111 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:55:44,114 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:55:47,971 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:55:48,776 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:55:48,776 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:55:48,777 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:55:48,781 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:55:49,120 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:55:50,122 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:55:53,296 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:55:53,297 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:55:53,297 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:55:53,297 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:55:53,317 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:55:53,589 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:55:53,589 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:55:53,594 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:55:54,127 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:55:54,127 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:55:57,331 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:55:57,332 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:55:57,332 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:55:57,337 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:55:58,133 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:55:58,133 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:55:58,337 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:56:01,291 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:56:01,292 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:56:01,292 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:56:01,297 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:56:02,138 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:56:02,138 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:56:03,790 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 20:56:03,791 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:56:04,548 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:56:04,548 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:56:04,549 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:56:04,554 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:56:05,142 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:56:06,145 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:56:08,262 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:56:08,263 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:56:08,263 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:56:08,268 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:56:08,297 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:56:08,297 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:56:08,297 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:56:09,147 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:56:09,501 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:56:10,150 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:56:12,033 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:56:12,033 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:56:12,033 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:56:12,038 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:56:12,151 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:56:14,155 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:56:15,075 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:56:20,075 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:56:20,616 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:56:20,617 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:56:20,617 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:56:20,622 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:56:21,163 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:56:22,166 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:56:23,297 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:56:23,297 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:56:23,297 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:56:24,084 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:56:24,085 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:56:24,085 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:56:24,090 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:56:24,167 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:56:25,127 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:56:26,171 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:56:29,330 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:56:29,331 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:56:29,331 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:56:29,336 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:56:30,176 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:56:30,177 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:56:30,336 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:56:32,883 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:56:32,884 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:56:32,884 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:56:32,889 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:56:33,180 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:56:33,796 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 20:56:34,182 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:56:35,798 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:56:37,685 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:56:37,686 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:56:37,686 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:56:37,691 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:56:38,189 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:56:38,190 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:56:38,297 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:56:38,297 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:56:38,297 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:56:41,312 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:56:41,313 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:56:41,313 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:56:41,314 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:56:41,319 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:56:42,195 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:56:42,195 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:56:44,528 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:56:44,529 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:56:44,530 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:56:44,534 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:56:45,198 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:56:46,201 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:56:46,535 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:56:49,278 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:56:49,279 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:56:49,279 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:56:49,284 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:56:50,207 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:56:50,207 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:56:52,284 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:56:53,297 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:56:53,297 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:56:53,297 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:56:53,643 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:56:53,643 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:56:53,644 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:56:53,648 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:56:54,212 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:56:54,212 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:56:57,166 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:56:57,167 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:56:57,167 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:56:57,172 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:56:57,215 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:56:58,172 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:56:58,218 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:57:00,693 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:57:00,694 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:57:00,694 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:57:00,699 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:57:01,220 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:57:02,222 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:57:03,700 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:57:03,795 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 20:57:04,442 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:57:04,443 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:57:04,443 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:57:04,448 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:57:05,226 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:57:06,229 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:57:08,297 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:57:08,297 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:57:08,297 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:57:09,301 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:57:09,302 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:57:09,302 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:57:09,302 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:57:09,307 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:57:10,235 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:57:10,235 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:57:13,492 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:57:13,493 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:57:13,494 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:57:13,499 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:57:14,240 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:57:14,240 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:57:14,499 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:57:17,034 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:57:17,035 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:57:17,035 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:57:17,040 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:57:17,243 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:57:18,245 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:57:20,040 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:57:20,501 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:57:20,502 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:57:20,502 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:57:20,507 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:57:21,248 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:57:22,249 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:57:23,297 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:57:23,297 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:57:23,298 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:57:25,604 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:57:27,331 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:57:27,332 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:57:27,332 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:57:27,337 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:57:28,259 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:57:28,259 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:57:31,375 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:57:32,146 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:57:32,147 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:57:32,147 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:57:32,152 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:57:32,263 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:57:33,801 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 20:57:34,268 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:57:36,802 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:57:36,878 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:57:36,878 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:57:36,879 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:57:36,884 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:57:37,272 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:57:38,274 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:57:38,297 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:57:38,297 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:57:38,297 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:57:40,296 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:57:40,297 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:57:40,297 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:57:40,302 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:57:41,278 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:57:42,280 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:57:42,339 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:57:47,339 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:57:48,751 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:57:48,752 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:57:48,753 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:57:48,758 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:57:49,289 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:57:50,291 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:57:52,759 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:57:53,170 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:57:53,170 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:57:53,170 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:57:53,176 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:57:53,295 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:57:53,297 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:57:53,297 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:57:53,297 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:57:54,297 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:57:56,206 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:57:56,206 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:57:56,206 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:57:56,211 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:57:56,300 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:57:58,251 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:57:58,304 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:58:00,626 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:58:00,626 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:58:00,627 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:58:00,632 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:58:01,306 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:58:02,309 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:58:03,632 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:58:03,800 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 20:58:05,434 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:58:05,434 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:58:05,435 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:58:05,439 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:58:06,315 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:58:06,315 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:58:08,297 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:58:08,298 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:58:08,298 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:58:09,603 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:58:10,383 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:58:10,383 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:58:10,383 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:58:10,388 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:58:11,321 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:58:12,323 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:58:14,571 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:58:14,572 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:58:14,572 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:58:14,577 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:58:15,327 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:58:15,577 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:58:16,329 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:58:18,411 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:58:18,412 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:58:18,412 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:58:18,417 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:58:19,332 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:58:20,335 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:58:21,418 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:58:22,224 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:58:22,225 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:58:22,225 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:58:22,230 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:58:22,336 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:58:23,297 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:58:23,298 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:58:23,298 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:58:24,340 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:58:25,784 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:58:25,785 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:58:25,785 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:58:25,790 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:58:26,344 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:58:26,344 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:58:26,827 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:58:30,153 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:58:30,154 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:58:30,154 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:58:30,159 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:58:30,347 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:58:32,160 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:58:32,351 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:58:33,805 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 20:58:34,643 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:58:34,643 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:58:34,643 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:58:34,648 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:58:35,356 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:58:36,359 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:58:37,649 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:58:38,297 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:58:38,298 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:58:38,298 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:58:38,309 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:58:38,518 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:58:38,518 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:58:38,523 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:58:39,361 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:58:40,363 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:58:42,639 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:58:42,639 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:58:42,639 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:58:42,645 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:58:43,367 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:58:43,645 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:58:44,369 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:58:46,017 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:58:46,018 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:58:46,019 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:58:46,024 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:58:46,373 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:58:46,373 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:58:49,024 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:58:50,292 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:58:50,292 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:58:50,293 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:58:50,298 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:58:50,376 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:58:52,381 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:58:53,298 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:58:53,298 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:58:53,298 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:58:53,525 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:58:53,531 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:58:53,531 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:58:53,536 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:58:54,384 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:58:54,385 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:58:54,537 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:58:57,157 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:58:57,158 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:58:57,158 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:58:57,163 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:58:57,388 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:58:58,390 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:59:00,163 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:59:01,678 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:59:01,679 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:59:01,679 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:59:01,684 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:59:02,395 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:59:02,395 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:59:03,804 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 20:59:05,749 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:59:05,750 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:59:05,750 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:59:05,750 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:59:05,755 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:59:06,401 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:59:06,401 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:59:08,298 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:59:08,298 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:59:08,298 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:59:11,303 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:59:11,304 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:59:11,304 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:59:11,304 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:59:11,309 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:59:11,406 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:59:12,409 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:59:14,859 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:59:14,860 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:59:14,860 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:59:14,865 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:59:15,412 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:59:16,413 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:59:16,865 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:59:20,015 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:59:20,016 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:59:20,016 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:59:20,021 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:59:20,420 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:59:20,421 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:59:22,022 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:59:23,298 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:59:23,298 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:59:23,298 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:59:24,055 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:59:24,055 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:59:24,055 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:59:24,060 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:59:24,426 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:59:24,426 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:59:27,099 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:59:28,406 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:59:28,407 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:59:28,407 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:59:28,412 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:59:28,430 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:59:30,434 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:59:32,412 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:59:33,810 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 20:59:36,707 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:59:36,707 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:59:36,707 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:59:36,712 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:59:37,443 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:59:37,713 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:59:38,298 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:59:38,298 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:59:38,298 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:59:38,446 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:59:43,567 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:59:45,602 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:59:45,603 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:59:45,603 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:59:45,608 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:59:46,456 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:59:46,457 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:59:48,647 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:59:49,298 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:59:49,299 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:59:49,299 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:59:49,304 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:59:49,460 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:59:50,462 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:59:52,832 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:59:52,832 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:59:52,833 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:59:52,838 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:59:53,298 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 20:59:53,298 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 20:59:53,298 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 20:59:53,465 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:59:54,468 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:59:54,502 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 20:59:56,923 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 20:59:56,923 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 20:59:56,924 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 20:59:56,928 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 20:59:57,471 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 20:59:58,474 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 20:59:59,967 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:00:02,079 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:00:02,080 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 21:00:02,080 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:00:02,085 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:00:02,479 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 21:00:02,479 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 21:00:03,809 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 21:00:05,810 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:00:07,685 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:00:07,686 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 21:00:07,686 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:00:07,691 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:00:08,298 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:00:08,298 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:00:08,341 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:00:08,488 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 21:00:08,488 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 21:00:11,642 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:00:16,070 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:00:16,071 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 21:00:16,071 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:00:16,076 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:00:16,499 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 21:00:16,499 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 21:00:17,115 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:00:20,863 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:00:20,864 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 21:00:20,864 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:00:20,869 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:00:21,505 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 21:00:22,507 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 21:00:22,870 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:00:23,298 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:00:23,298 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:00:23,299 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:00:25,299 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:00:25,300 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 21:00:25,300 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:00:25,305 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:00:25,510 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 21:00:26,513 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 21:00:28,343 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:00:28,774 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:00:28,774 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 21:00:28,775 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:00:28,780 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:00:29,516 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 21:00:30,519 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 21:00:32,442 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:00:32,443 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 21:00:32,443 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:00:32,448 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:00:32,520 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 21:00:33,448 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:00:33,811 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 21:00:34,525 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 21:00:36,563 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:00:36,564 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 21:00:36,564 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:00:36,569 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:00:37,527 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 21:00:38,298 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:00:38,298 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:00:38,299 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:00:38,530 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 21:00:38,533 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:00:40,500 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:00:40,501 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 21:00:40,501 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:00:40,506 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:00:40,532 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 21:00:42,536 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 21:00:43,546 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:00:44,072 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:00:44,073 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 21:00:44,073 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:00:44,078 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:00:44,539 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 21:00:44,539 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 21:00:47,753 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:00:47,754 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 21:00:47,754 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:00:47,760 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:00:48,544 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 21:00:48,544 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 21:00:48,760 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:00:51,109 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:00:51,110 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 21:00:51,110 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:00:51,115 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:00:51,547 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 21:00:52,550 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 21:00:53,298 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:00:53,299 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:00:53,299 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:00:54,527 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:00:55,493 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:00:55,493 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 21:00:55,494 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:00:55,498 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:00:55,554 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 21:00:56,556 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 21:00:59,067 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:00:59,068 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 21:00:59,068 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:00:59,073 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:00:59,559 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 21:01:00,073 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:01:00,561 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 21:01:03,486 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:01:03,487 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 21:01:03,487 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:01:03,492 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:01:03,564 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 21:01:03,813 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 21:01:04,567 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 21:01:05,814 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:01:07,665 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:01:07,666 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 21:01:07,666 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:01:07,671 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:01:08,298 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:01:08,299 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:01:08,299 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:01:08,573 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 21:01:08,573 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 21:01:11,474 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:01:11,475 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 21:01:11,475 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:01:11,475 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:01:11,480 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:01:11,576 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 21:01:12,579 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 21:01:15,288 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:01:15,289 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 21:01:15,289 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:01:15,294 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:01:15,582 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 21:01:16,585 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 21:01:17,295 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:01:18,216 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:01:18,217 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 21:01:18,217 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:01:18,222 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:01:18,588 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 21:01:18,588 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 21:01:22,324 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:01:22,324 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 21:01:22,325 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:01:22,325 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:01:22,330 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:01:22,591 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 21:01:23,298 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:01:23,299 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:01:23,299 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:01:24,595 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 21:01:26,897 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:01:26,897 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 21:01:26,897 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:01:26,903 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:01:27,598 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 21:01:27,939 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:01:28,601 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 21:01:32,516 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:01:32,517 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 21:01:32,517 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:01:32,523 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:01:32,605 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 21:01:33,524 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:01:33,815 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 21:01:34,611 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 21:01:37,309 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:01:37,310 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 21:01:37,311 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:01:37,316 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:01:37,614 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 21:01:38,299 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:01:38,299 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:01:38,299 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:01:38,617 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 21:01:39,520 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:01:41,648 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:01:41,652 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 21:01:41,652 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:01:41,658 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:01:42,621 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 21:01:42,622 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 21:01:44,691 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:01:45,617 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:01:45,617 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 21:01:45,618 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:01:45,623 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:01:45,624 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 21:01:46,628 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 21:01:50,624 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:01:53,299 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:01:53,299 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:01:53,299 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:01:54,258 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:01:54,259 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 21:01:54,259 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:01:54,264 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:01:54,638 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 21:01:54,638 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 21:01:56,303 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:01:57,890 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:01:57,891 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 21:01:57,891 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:01:57,898 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:01:58,644 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 21:01:58,644 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 21:02:01,898 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:02:03,818 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 21:02:04,115 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:02:04,115 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 21:02:04,115 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:02:04,120 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:02:04,653 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 21:02:04,653 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 21:02:07,121 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:02:07,257 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:02:07,258 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 21:02:07,258 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:02:07,264 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:02:07,656 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 21:02:08,299 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:02:08,299 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:02:08,299 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:02:08,659 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 21:02:11,130 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:02:11,131 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 21:02:11,131 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:02:11,137 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:02:11,662 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 21:02:12,175 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:02:12,664 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 21:02:14,577 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:02:14,578 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 21:02:14,578 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:02:14,583 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:02:14,666 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 21:02:16,670 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 21:02:17,584 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:02:18,137 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:02:18,137 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 21:02:18,138 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:02:18,143 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:02:18,672 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 21:02:20,675 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 21:02:23,144 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:02:23,299 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:02:23,299 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:02:23,341 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:02:26,532 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:02:26,533 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 21:02:26,533 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:02:26,538 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:02:26,681 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 21:02:28,575 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:02:28,684 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 21:02:31,271 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:02:31,271 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 21:02:31,272 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:02:31,277 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:02:31,689 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 21:02:32,692 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 21:02:33,819 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 21:02:33,820 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:02:35,656 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:02:35,657 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 21:02:35,658 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:02:35,663 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:02:35,696 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 21:02:36,698 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 21:02:38,299 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:02:38,299 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:02:38,299 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:02:39,595 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:02:39,762 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:02:39,763 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 21:02:39,763 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:02:39,768 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:02:40,704 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 21:02:40,704 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 21:02:44,442 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:02:44,443 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 21:02:44,443 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:02:44,448 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:02:44,707 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 21:02:45,449 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:02:46,712 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 21:02:50,449 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:02:50,992 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:02:50,993 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 21:02:50,993 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:02:50,998 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:02:51,718 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 21:02:52,721 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 21:02:53,299 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:02:53,299 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:02:53,299 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:02:54,611 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:02:54,612 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 21:02:54,612 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:02:54,617 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:02:54,722 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 21:02:55,655 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:02:56,728 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 21:02:59,308 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:02:59,309 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 21:02:59,309 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:02:59,314 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:02:59,730 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 21:03:00,732 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 21:03:01,315 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:03:03,091 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:03:03,092 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 21:03:03,092 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:03:03,097 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:03:03,735 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 21:03:03,822 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 21:03:04,737 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 21:03:06,823 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:03:08,119 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:03:08,119 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 21:03:08,119 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:03:08,125 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:03:08,299 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:03:08,301 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:03:08,301 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:03:08,742 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 21:03:10,747 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 21:03:12,591 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:03:16,304 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:03:16,305 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 21:03:16,305 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:03:16,310 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:03:16,752 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 21:03:18,347 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:03:18,757 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 21:03:20,889 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:03:20,890 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 21:03:20,890 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:03:20,895 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:03:21,759 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 21:03:22,762 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 21:03:23,299 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:03:23,299 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:03:23,299 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:03:23,823 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:03:26,575 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:03:26,575 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 21:03:26,576 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:03:26,581 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:03:26,766 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 21:03:28,770 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 21:03:29,619 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:03:31,212 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:03:31,213 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 21:03:31,213 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:03:31,219 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:03:31,773 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 21:03:32,776 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 21:03:33,824 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 21:03:34,825 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:03:38,299 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:03:38,299 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:03:38,300 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:03:38,737 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:03:38,737 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 21:03:38,737 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:03:38,743 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:03:38,783 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 21:03:40,779 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:03:40,788 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 21:03:43,594 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:03:43,595 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 21:03:43,596 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:03:43,601 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:03:43,791 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 21:03:44,794 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 21:03:46,602 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:03:47,465 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:03:47,465 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 21:03:47,466 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:03:47,471 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:03:47,797 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 21:03:48,799 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 21:03:51,745 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:03:51,745 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 21:03:51,746 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:03:51,746 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:03:51,751 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:03:51,802 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 21:03:52,805 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 21:03:53,299 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:03:53,299 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:03:53,299 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:03:55,490 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:03:55,491 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 21:03:55,491 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:03:55,496 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:03:55,809 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 21:03:56,812 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 21:03:57,535 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:03:59,255 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:03:59,256 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 21:03:59,256 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:03:59,261 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:03:59,814 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 21:04:00,816 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 21:04:03,019 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:04:03,019 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 21:04:03,019 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:04:03,020 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:04:03,025 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:04:03,819 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 21:04:03,830 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 21:04:04,821 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 21:04:06,283 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:04:06,283 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 21:04:06,283 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:04:06,288 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:04:06,823 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 21:04:08,289 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:04:08,299 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:04:08,299 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:04:08,341 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:04:08,827 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 21:04:10,249 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:04:10,249 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 21:04:10,249 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:04:10,255 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:04:10,829 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 21:04:12,833 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 21:04:13,291 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:04:14,143 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:04:14,144 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 21:04:14,144 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:04:14,149 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:04:14,835 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 21:04:16,839 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 21:04:18,640 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:04:18,640 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 21:04:18,641 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:04:18,641 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:04:18,646 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:04:18,840 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 21:04:20,844 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 21:04:22,806 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:04:22,807 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 21:04:22,807 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:04:22,812 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:04:22,846 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 21:04:23,299 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:04:23,300 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:04:23,300 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:04:24,594 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:04:24,850 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 21:04:26,302 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:04:26,302 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 21:04:26,302 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:04:26,308 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:04:26,851 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 21:04:28,855 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 21:04:30,347 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:04:31,492 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:04:31,493 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 21:04:31,493 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:04:31,498 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:04:31,858 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 21:04:32,860 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 21:04:33,828 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 21:04:35,830 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:04:36,412 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:04:36,413 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 21:04:36,413 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:04:36,418 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:04:36,865 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 21:04:38,299 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:04:38,300 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:04:38,300 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:04:38,869 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 21:04:41,606 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:04:43,120 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:04:43,120 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 21:04:43,121 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:04:43,126 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:04:43,875 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 21:04:44,878 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 21:04:47,163 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:04:51,434 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:04:51,435 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 21:04:51,435 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:04:51,440 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:04:51,886 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 21:04:52,441 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:04:52,890 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 21:04:53,300 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:04:53,300 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:04:53,300 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:04:57,560 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:05:00,006 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:05:00,007 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 21:05:00,007 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:05:00,012 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:05:00,898 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 21:05:02,900 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 21:05:03,051 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:05:03,730 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:05:03,731 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 21:05:03,731 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:05:03,736 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:05:03,834 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 21:05:03,902 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 21:05:04,905 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 21:05:06,716 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:05:06,717 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 21:05:06,717 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:05:06,722 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:05:06,907 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 21:05:08,300 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:05:08,300 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:05:08,300 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:05:08,300 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:05:08,912 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 21:05:13,593 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:05:14,985 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:05:14,986 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 21:05:14,986 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:05:14,991 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:05:15,920 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 21:05:16,923 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 21:05:19,027 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:05:19,046 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:05:19,047 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 21:05:19,047 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:05:19,052 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:05:19,926 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 21:05:20,928 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 21:05:23,168 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:05:23,169 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 21:05:23,169 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:05:23,174 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:05:23,300 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:05:23,300 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:05:23,300 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:05:23,932 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 21:05:24,559 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:05:24,934 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 21:05:26,719 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:05:26,719 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 21:05:26,720 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:05:26,725 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:05:26,935 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 21:05:28,940 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 21:05:29,763 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:05:33,833 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 21:05:34,834 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:05:35,099 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:05:35,099 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 21:05:35,099 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:05:35,104 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:05:35,948 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 21:05:36,951 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 21:05:38,300 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:05:38,300 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:05:38,300 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:05:39,203 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:05:39,204 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 21:05:39,204 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:05:39,209 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:05:39,954 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 21:05:40,247 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:05:40,957 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 21:05:42,562 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:05:42,562 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 21:05:42,563 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:05:42,568 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:05:42,958 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 21:05:44,963 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 21:05:45,568 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:05:46,963 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:05:46,964 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 21:05:46,964 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:05:46,969 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:05:47,966 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 21:05:48,968 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 21:05:50,970 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:05:51,148 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:05:51,149 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 21:05:51,149 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:05:51,154 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:05:51,971 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 21:05:52,974 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 21:05:53,300 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:05:53,300 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:05:53,300 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:05:54,940 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:05:54,941 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 21:05:54,941 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:05:54,946 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:05:54,976 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 21:05:55,983 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:05:56,980 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 21:06:00,983 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:06:03,234 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:06:03,235 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 21:06:03,235 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:06:03,240 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:06:03,838 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 21:06:03,988 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 21:06:04,990 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 21:06:06,840 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:06:08,300 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:06:08,300 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:06:08,300 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:06:08,703 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:06:08,703 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 21:06:08,703 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:06:08,708 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:06:08,994 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 21:06:11,000 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 21:06:12,747 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:06:12,928 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:06:12,929 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 21:06:12,930 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:06:12,935 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:06:13,001 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 21:06:15,005 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 21:06:17,935 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:06:21,397 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:06:21,398 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 21:06:21,398 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:06:21,403 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:06:22,013 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 21:06:23,015 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 21:06:23,300 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:06:23,300 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:06:23,300 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:06:23,301 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:06:26,057 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:06:26,058 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 21:06:26,058 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:06:26,063 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:06:27,021 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 21:06:27,021 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 21:06:29,099 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:06:30,201 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:06:30,202 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 21:06:30,202 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:06:30,207 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:06:31,025 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 21:06:33,030 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 21:06:33,837 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 21:06:34,325 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:06:34,326 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 21:06:34,326 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:06:34,326 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:06:34,331 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:06:35,032 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 21:06:37,037 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 21:06:38,046 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:06:38,047 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 21:06:38,048 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:06:38,053 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:06:38,300 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:06:38,300 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:06:38,301 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:06:39,040 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 21:06:39,040 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 21:06:39,522 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:06:41,693 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:06:41,694 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 21:06:41,694 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:06:41,699 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:06:42,042 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 21:06:43,045 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 21:06:44,735 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:06:45,648 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:06:45,648 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 21:06:45,649 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:06:45,654 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:06:46,048 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 21:06:47,050 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 21:06:50,636 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:06:50,637 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 21:06:50,637 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:06:50,637 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:06:50,642 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:06:51,054 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 21:06:53,059 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 21:06:53,300 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:06:53,300 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:06:53,300 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:06:54,618 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:06:54,618 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 21:06:54,619 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:06:54,624 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:06:55,061 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 21:06:55,663 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:06:57,066 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 21:06:59,844 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:06:59,845 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 21:06:59,845 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:06:59,850 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:07:00,068 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 21:07:00,851 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:07:01,070 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 21:07:03,843 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 21:07:06,844 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:07:08,182 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:07:08,183 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 21:07:08,183 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:07:08,188 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:07:08,300 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:07:08,300 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:07:08,301 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:07:09,079 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 21:07:11,084 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 21:07:12,413 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:07:12,414 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 21:07:12,414 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:07:12,415 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:07:12,420 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:07:13,085 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 21:07:15,090 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 21:07:16,273 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:07:16,274 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 21:07:16,274 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:07:16,279 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:07:17,091 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 21:07:18,280 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:07:19,095 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 21:07:20,372 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:07:20,373 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 21:07:20,373 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:07:20,378 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:07:21,096 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 21:07:23,101 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 21:07:23,300 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:07:23,300 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:07:23,300 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:07:23,301 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:07:24,224 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:07:24,225 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 21:07:24,225 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:07:24,230 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:07:25,102 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 21:07:27,106 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 21:07:29,267 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:07:31,765 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:07:31,765 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 21:07:31,766 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:07:31,771 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:07:32,113 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 21:07:33,115 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 21:07:33,842 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 21:07:34,843 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:07:35,933 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:07:35,934 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 21:07:35,934 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:07:35,939 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:07:36,119 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 21:07:37,119 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 21:07:38,300 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:07:38,301 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:07:38,301 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:07:40,039 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:07:40,039 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 21:07:40,039 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:07:40,039 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:07:40,045 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:07:40,124 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 21:07:41,126 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 21:07:43,768 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:07:43,769 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 21:07:43,770 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:07:43,775 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:07:44,130 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 21:07:45,132 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 21:07:45,776 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:07:48,065 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:07:48,066 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 21:07:48,066 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:07:48,071 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:07:48,135 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 21:07:49,136 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 21:07:51,072 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:07:52,646 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:07:52,647 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 21:07:52,647 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:07:52,652 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:07:53,142 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 21:07:53,300 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:07:53,301 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:07:53,301 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:07:55,146 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 21:07:56,530 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:07:56,530 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 21:07:56,530 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:07:56,530 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:07:56,535 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:07:57,149 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 21:07:59,152 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 21:08:00,629 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:08:00,630 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 21:08:00,630 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:08:00,635 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:08:01,153 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 21:08:01,635 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:08:03,157 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 21:08:03,848 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 21:08:05,598 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:08:05,599 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 21:08:05,599 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:08:05,604 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:08:06,161 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 21:08:07,164 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 21:08:07,605 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:08:08,300 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:08:08,301 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:08:08,301 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:08:13,603 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:08:13,804 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:08:13,804 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 21:08:13,805 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:08:13,810 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:08:14,171 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 21:08:15,174 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 21:08:18,847 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:08:22,207 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:08:22,208 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 21:08:22,208 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:08:22,213 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:08:23,185 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 21:08:23,185 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 21:08:23,301 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:08:23,301 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:08:23,301 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:08:24,512 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:08:26,050 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:08:26,051 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 21:08:26,051 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:08:26,055 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:08:26,188 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 21:08:27,190 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 21:08:30,095 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:08:31,506 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:08:31,507 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 21:08:31,507 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:08:31,512 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:08:32,196 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 21:08:33,199 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 21:08:33,846 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 21:08:35,775 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:08:35,776 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 21:08:35,776 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:08:35,776 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:08:35,781 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:08:36,203 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 21:08:37,206 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 21:08:38,301 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:08:38,301 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:08:38,301 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:08:39,692 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:08:39,692 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 21:08:39,693 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:08:39,698 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:08:40,208 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 21:08:41,209 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 21:08:41,735 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:08:44,615 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:08:44,616 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 21:08:44,616 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:08:44,621 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:08:45,214 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 21:08:47,219 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 21:08:47,622 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:08:48,162 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:08:48,162 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 21:08:48,162 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:08:48,167 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:08:48,220 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 21:08:49,222 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 21:08:53,168 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:08:53,301 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:08:53,301 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:08:53,301 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:08:55,108 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:08:55,109 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 21:08:55,109 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:08:55,114 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:08:55,228 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 21:08:57,233 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 21:08:59,151 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:08:59,292 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:08:59,293 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 21:08:59,293 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:08:59,298 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:09:00,235 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 21:09:01,238 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 21:09:02,987 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:09:02,987 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 21:09:02,988 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:09:02,992 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:09:03,239 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 21:09:03,848 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 21:09:04,849 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:09:05,244 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 21:09:07,982 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:09:07,983 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 21:09:07,983 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:09:07,988 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:09:08,247 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 21:09:08,301 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:09:08,302 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:09:08,302 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:09:09,250 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 21:09:10,658 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:09:12,924 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:09:12,925 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 21:09:12,925 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:09:12,930 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:09:13,253 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 21:09:15,258 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 21:09:15,930 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:09:18,935 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:09:18,936 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 21:09:18,936 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:09:18,941 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:09:19,261 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 21:09:20,942 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:09:21,264 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 21:09:22,072 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:09:22,072 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 21:09:22,072 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:09:22,078 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:09:22,267 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 21:09:23,269 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 21:09:23,301 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:09:23,301 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:09:23,301 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:09:25,822 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:09:25,822 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 21:09:25,822 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:09:25,827 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:09:26,272 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 21:09:26,867 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:09:27,275 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 21:09:29,513 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:09:29,514 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 21:09:29,514 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:09:29,519 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:09:30,277 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 21:09:31,280 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 21:09:32,519 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:09:33,850 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 21:09:34,268 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:09:34,269 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 21:09:34,269 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:09:34,274 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:09:34,283 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 21:09:35,285 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 21:09:38,275 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:09:38,301 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:09:38,301 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:09:38,301 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:09:38,448 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:09:38,590 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 21:09:38,591 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:09:38,595 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:09:39,290 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 21:09:41,295 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 21:09:42,233 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:09:42,233 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 21:09:42,234 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:09:42,238 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:09:42,295 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 21:09:43,298 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 21:09:44,239 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:09:45,840 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:09:45,841 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 21:09:45,841 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:09:45,846 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:09:46,300 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 21:09:47,303 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 21:09:49,366 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:09:49,366 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 21:09:49,367 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:09:49,367 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:09:49,371 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:09:50,306 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 21:09:51,308 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 21:09:53,301 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:09:53,301 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:09:53,302 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:09:53,865 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:09:53,866 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 21:09:53,866 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:09:53,871 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:09:54,312 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 21:09:54,907 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:09:55,314 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 21:09:59,907 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:10:02,012 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:10:02,013 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 21:10:02,013 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:10:02,018 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:10:02,322 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 21:10:03,325 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 21:10:03,852 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 21:10:05,284 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:10:05,285 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 21:10:05,285 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:10:05,285 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:10:05,290 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:10:05,326 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 21:10:07,331 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 21:10:08,301 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:10:08,302 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:10:08,302 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:10:08,447 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:10:08,622 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 21:10:08,622 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:10:08,627 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:10:09,332 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 21:10:10,627 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:10:11,337 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 21:10:13,431 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:10:13,432 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 21:10:13,432 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:10:13,438 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:10:14,339 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 21:10:15,342 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 21:10:16,438 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:10:18,142 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:10:18,143 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 21:10:18,143 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:10:18,148 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:10:18,345 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 21:10:19,347 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 21:10:22,149 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:10:22,303 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:10:22,304 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 21:10:22,304 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:10:22,309 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:10:22,350 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 21:10:23,301 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:10:23,301 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:10:23,302 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:10:23,353 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 21:10:27,048 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:10:27,048 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 21:10:27,049 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:10:27,053 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:10:27,356 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 21:10:28,091 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:10:29,361 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 21:10:31,170 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:10:31,171 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 21:10:31,171 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:10:31,176 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:10:31,363 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 21:10:33,177 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:10:33,367 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 21:10:33,854 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 21:10:36,114 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:10:36,115 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 21:10:36,115 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:10:36,120 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:10:36,371 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 21:10:37,373 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 21:10:38,302 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:10:38,302 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:10:38,302 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:10:38,302 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:10:41,610 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:10:41,611 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 21:10:41,611 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:10:41,616 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:10:42,379 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 21:10:43,381 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 21:10:43,655 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:10:48,655 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:10:50,020 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:10:50,021 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 21:10:50,021 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:10:50,026 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:10:50,390 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 21:10:51,392 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 21:10:53,302 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:10:53,302 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:10:53,302 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:10:54,503 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:10:56,030 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:10:56,031 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 21:10:56,031 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:10:56,036 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:10:56,400 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 21:10:57,402 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 21:10:59,824 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:10:59,825 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 21:10:59,826 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:10:59,826 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:10:59,830 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:11:00,404 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 21:11:01,405 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 21:11:03,857 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 21:11:04,821 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:11:04,822 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 21:11:04,822 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:11:04,827 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:11:04,827 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:11:05,410 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 21:11:07,415 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 21:11:08,302 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:11:08,302 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:11:08,302 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:11:08,986 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:11:08,987 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 21:11:08,987 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:11:08,993 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:11:09,417 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 21:11:10,031 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:11:11,421 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 21:11:14,177 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:11:14,178 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 21:11:14,179 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:11:14,183 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:11:14,423 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 21:11:15,184 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:11:15,426 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 21:11:18,090 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:11:18,091 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 21:11:18,092 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:11:18,096 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:11:18,429 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 21:11:19,430 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 21:11:21,097 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:11:23,302 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:11:23,302 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:11:23,302 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:11:23,724 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:11:23,725 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 21:11:23,725 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:11:23,730 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:11:24,437 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 21:11:25,440 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 21:11:26,767 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:11:31,767 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:11:32,197 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:11:32,198 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 21:11:32,198 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:11:32,203 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:11:32,448 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 21:11:33,451 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 21:11:33,862 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 21:11:35,880 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:11:35,880 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 21:11:35,881 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:11:35,886 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:11:36,455 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 21:11:36,886 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:11:37,455 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 21:11:38,302 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:11:38,302 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:11:38,302 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:11:39,617 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:11:39,618 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 21:11:39,618 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:11:39,623 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:11:40,460 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 21:11:41,463 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 21:11:42,663 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:11:42,699 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:11:42,700 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 21:11:42,700 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:11:42,705 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:11:43,464 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 21:11:45,468 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 21:11:46,839 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: partial_history +2024-03-12 21:11:46,840 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 21:11:46,840 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:11:46,847 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:11:47,470 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 21:11:47,847 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:11:49,474 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 21:11:52,848 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:11:53,302 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:11:53,302 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:11:53,303 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:11:58,579 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:12:03,580 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:12:03,861 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 21:12:08,767 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:12:08,862 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:12:09,160 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:12:09,160 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:12:14,458 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:12:19,459 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:12:24,460 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:12:25,478 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:12:25,479 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:12:25,795 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:12:29,870 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:12:33,863 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 21:12:35,865 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:12:40,865 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:12:45,866 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:12:50,867 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:12:54,187 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:12:54,187 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:12:54,188 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:12:56,425 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:13:01,425 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:13:03,869 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 21:13:06,870 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:13:09,187 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:13:09,187 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:13:09,187 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:13:12,418 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:13:17,419 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:13:22,419 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:13:24,187 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:13:24,187 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:13:24,187 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:13:27,489 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:13:32,489 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:13:33,867 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 21:13:37,869 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:13:39,187 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:13:39,187 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:13:39,187 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:13:43,480 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:13:48,481 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:13:53,481 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:13:54,187 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:13:54,187 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:13:54,188 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:13:58,483 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:14:03,483 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:14:03,870 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 21:14:08,872 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:14:09,187 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:14:09,187 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:14:09,187 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:14:14,395 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:14:19,395 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:14:24,187 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:14:24,187 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:14:24,187 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:14:24,446 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:14:29,447 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:14:33,877 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 21:14:34,878 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:14:39,187 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:14:39,187 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:14:39,187 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:14:40,488 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:14:45,488 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:14:50,489 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:14:54,187 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:14:54,187 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:14:54,187 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:14:56,468 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:15:01,468 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:15:03,873 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 21:15:06,875 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:15:09,187 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:15:09,187 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:15:09,188 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:15:12,463 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:15:17,463 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:15:22,464 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:15:24,187 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:15:24,187 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:15:24,188 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:15:28,399 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:15:33,399 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:15:33,876 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 21:15:38,878 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:15:39,188 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:15:39,188 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:15:39,188 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:15:44,470 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:15:49,471 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:15:54,188 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: stop_status +2024-03-12 21:15:54,188 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:15:54,188 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: stop_status +2024-03-12 21:15:55,457 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:16:00,457 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:16:03,283 DEBUG SenderThread:2571781 [sender.py:send():382] send: telemetry +2024-03-12 21:16:03,283 DEBUG SenderThread:2571781 [sender.py:send():382] send: exit +2024-03-12 21:16:03,283 INFO SenderThread:2571781 [sender.py:send_exit():589] handling exit code: 0 +2024-03-12 21:16:03,283 INFO SenderThread:2571781 [sender.py:send_exit():591] handling runtime: 24451 +2024-03-12 21:16:03,289 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:16:03,289 INFO SenderThread:2571781 [sender.py:send_exit():597] send defer +2024-03-12 21:16:03,290 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: defer +2024-03-12 21:16:03,290 INFO HandlerThread:2571781 [handler.py:handle_request_defer():172] handle defer: 0 +2024-03-12 21:16:03,290 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: defer +2024-03-12 21:16:03,290 INFO SenderThread:2571781 [sender.py:send_request_defer():613] handle sender defer: 0 +2024-03-12 21:16:03,290 INFO SenderThread:2571781 [sender.py:transition_state():617] send defer: 1 +2024-03-12 21:16:03,290 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: defer +2024-03-12 21:16:03,290 INFO HandlerThread:2571781 [handler.py:handle_request_defer():172] handle defer: 1 +2024-03-12 21:16:03,290 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: defer +2024-03-12 21:16:03,290 INFO SenderThread:2571781 [sender.py:send_request_defer():613] handle sender defer: 1 +2024-03-12 21:16:03,290 INFO SenderThread:2571781 [sender.py:transition_state():617] send defer: 2 +2024-03-12 21:16:03,290 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: defer +2024-03-12 21:16:03,290 INFO HandlerThread:2571781 [handler.py:handle_request_defer():172] handle defer: 2 +2024-03-12 21:16:03,290 INFO HandlerThread:2571781 [system_monitor.py:finish():203] Stopping system monitor +2024-03-12 21:16:03,290 DEBUG SystemMonitor:2571781 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2024-03-12 21:16:03,291 DEBUG SystemMonitor:2571781 [system_monitor.py:_start():183] Publishing last batch of metrics +2024-03-12 21:16:03,294 INFO HandlerThread:2571781 [interfaces.py:finish():202] Joined cpu monitor +2024-03-12 21:16:03,294 INFO HandlerThread:2571781 [interfaces.py:finish():202] Joined disk monitor +2024-03-12 21:16:04,166 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 21:16:04,167 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 21:16:04,808 INFO HandlerThread:2571781 [interfaces.py:finish():202] Joined gpu monitor +2024-03-12 21:16:04,808 INFO HandlerThread:2571781 [interfaces.py:finish():202] Joined memory monitor +2024-03-12 21:16:04,809 INFO HandlerThread:2571781 [interfaces.py:finish():202] Joined network monitor +2024-03-12 21:16:04,809 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: poll_exit +2024-03-12 21:16:04,809 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: defer +2024-03-12 21:16:04,809 INFO SenderThread:2571781 [sender.py:send_request_defer():613] handle sender defer: 2 +2024-03-12 21:16:04,809 INFO SenderThread:2571781 [sender.py:transition_state():617] send defer: 3 +2024-03-12 21:16:04,809 DEBUG SenderThread:2571781 [sender.py:send():382] send: stats +2024-03-12 21:16:04,810 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: poll_exit +2024-03-12 21:16:04,810 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: defer +2024-03-12 21:16:04,811 INFO HandlerThread:2571781 [handler.py:handle_request_defer():172] handle defer: 3 +2024-03-12 21:16:04,812 DEBUG SenderThread:2571781 [sender.py:send():382] send: history +2024-03-12 21:16:04,812 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: summary_record +2024-03-12 21:16:04,817 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:16:04,818 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: defer +2024-03-12 21:16:04,818 INFO SenderThread:2571781 [sender.py:send_request_defer():613] handle sender defer: 3 +2024-03-12 21:16:04,818 INFO SenderThread:2571781 [sender.py:transition_state():617] send defer: 4 +2024-03-12 21:16:04,818 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: defer +2024-03-12 21:16:04,818 INFO HandlerThread:2571781 [handler.py:handle_request_defer():172] handle defer: 4 +2024-03-12 21:16:04,818 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: defer +2024-03-12 21:16:04,818 INFO SenderThread:2571781 [sender.py:send_request_defer():613] handle sender defer: 4 +2024-03-12 21:16:04,818 INFO SenderThread:2571781 [sender.py:transition_state():617] send defer: 5 +2024-03-12 21:16:04,818 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: defer +2024-03-12 21:16:04,818 INFO HandlerThread:2571781 [handler.py:handle_request_defer():172] handle defer: 5 +2024-03-12 21:16:04,821 DEBUG SenderThread:2571781 [sender.py:send():382] send: summary +2024-03-12 21:16:04,827 INFO SenderThread:2571781 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-03-12 21:16:04,827 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: defer +2024-03-12 21:16:04,827 INFO SenderThread:2571781 [sender.py:send_request_defer():613] handle sender defer: 5 +2024-03-12 21:16:04,827 INFO SenderThread:2571781 [sender.py:transition_state():617] send defer: 6 +2024-03-12 21:16:04,827 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: defer +2024-03-12 21:16:04,828 INFO HandlerThread:2571781 [handler.py:handle_request_defer():172] handle defer: 6 +2024-03-12 21:16:04,828 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: defer +2024-03-12 21:16:04,828 INFO SenderThread:2571781 [sender.py:send_request_defer():613] handle sender defer: 6 +2024-03-12 21:16:04,830 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:16:05,167 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 21:16:05,258 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: poll_exit +2024-03-12 21:16:06,170 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 21:16:07,325 INFO SenderThread:2571781 [sender.py:transition_state():617] send defer: 7 +2024-03-12 21:16:07,325 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: poll_exit +2024-03-12 21:16:07,325 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: defer +2024-03-12 21:16:07,325 INFO HandlerThread:2571781 [handler.py:handle_request_defer():172] handle defer: 7 +2024-03-12 21:16:07,326 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: defer +2024-03-12 21:16:07,326 INFO SenderThread:2571781 [sender.py:send_request_defer():613] handle sender defer: 7 +2024-03-12 21:16:08,172 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/config.yaml +2024-03-12 21:16:08,259 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: poll_exit +2024-03-12 21:16:09,176 INFO SenderThread:2571781 [sender.py:transition_state():617] send defer: 8 +2024-03-12 21:16:09,176 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: poll_exit +2024-03-12 21:16:09,177 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: defer +2024-03-12 21:16:09,177 INFO HandlerThread:2571781 [handler.py:handle_request_defer():172] handle defer: 8 +2024-03-12 21:16:09,177 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: defer +2024-03-12 21:16:09,177 INFO SenderThread:2571781 [sender.py:send_request_defer():613] handle sender defer: 8 +2024-03-12 21:16:09,177 INFO SenderThread:2571781 [job_builder.py:build():296] Attempting to build job artifact +2024-03-12 21:16:09,178 INFO SenderThread:2571781 [job_builder.py:_get_source_type():426] is repo sourced job +2024-03-12 21:16:09,249 INFO SenderThread:2571781 [job_builder.py:build():402] adding wandb-job metadata file +2024-03-12 21:16:09,259 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: poll_exit +2024-03-12 21:16:09,280 INFO SenderThread:2571781 [sender.py:transition_state():617] send defer: 9 +2024-03-12 21:16:09,280 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: poll_exit +2024-03-12 21:16:09,280 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: defer +2024-03-12 21:16:09,280 DEBUG SenderThread:2571781 [sender.py:send():382] send: artifact +2024-03-12 21:16:09,280 INFO HandlerThread:2571781 [handler.py:handle_request_defer():172] handle defer: 9 +2024-03-12 21:16:10,173 INFO Thread-12 :2571781 [dir_watcher.py:_on_file_modified():288] file/dir modified: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 21:16:10,260 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: poll_exit +2024-03-12 21:16:10,332 INFO SenderThread:2571781 [sender.py:send_artifact():1494] sent artifact job-https___github.com_TRI-ML_prismatic-vlms.git_scripts_pretrain.py - {'id': 'QXJ0aWZhY3Q6NzUzNTUzNzEx', 'state': 'COMMITTED', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjE0MzcwMzk1OA==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6NzUzNTUzNzEx', 'versionIndex': 7}}} +2024-03-12 21:16:10,332 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: defer +2024-03-12 21:16:10,332 INFO SenderThread:2571781 [sender.py:send_request_defer():613] handle sender defer: 9 +2024-03-12 21:16:10,332 INFO SenderThread:2571781 [dir_watcher.py:finish():358] shutting down directory watcher +2024-03-12 21:16:11,175 INFO SenderThread:2571781 [dir_watcher.py:finish():388] scan: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files +2024-03-12 21:16:11,175 INFO SenderThread:2571781 [dir_watcher.py:finish():402] scan save: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/config.yaml config.yaml +2024-03-12 21:16:11,175 INFO SenderThread:2571781 [dir_watcher.py:finish():402] scan save: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/conda-environment.yaml conda-environment.yaml +2024-03-12 21:16:11,177 INFO SenderThread:2571781 [dir_watcher.py:finish():402] scan save: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-metadata.json wandb-metadata.json +2024-03-12 21:16:11,177 INFO SenderThread:2571781 [dir_watcher.py:finish():402] scan save: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/requirements.txt requirements.txt +2024-03-12 21:16:11,177 INFO SenderThread:2571781 [dir_watcher.py:finish():402] scan save: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log output.log +2024-03-12 21:16:11,179 INFO SenderThread:2571781 [dir_watcher.py:finish():402] scan save: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json wandb-summary.json +2024-03-12 21:16:11,181 INFO SenderThread:2571781 [sender.py:transition_state():617] send defer: 10 +2024-03-12 21:16:11,182 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: poll_exit +2024-03-12 21:16:11,182 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: defer +2024-03-12 21:16:11,182 INFO HandlerThread:2571781 [handler.py:handle_request_defer():172] handle defer: 10 +2024-03-12 21:16:11,185 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: defer +2024-03-12 21:16:11,185 INFO SenderThread:2571781 [sender.py:send_request_defer():613] handle sender defer: 10 +2024-03-12 21:16:11,185 INFO SenderThread:2571781 [file_pusher.py:finish():172] shutting down file pusher +2024-03-12 21:16:11,260 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: poll_exit +2024-03-12 21:16:11,260 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: poll_exit +2024-03-12 21:16:11,920 INFO wandb-upload_0:2571781 [upload_job.py:push():131] Uploaded file runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/config.yaml +2024-03-12 21:16:12,127 INFO wandb-upload_2:2571781 [upload_job.py:push():131] Uploaded file runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/requirements.txt +2024-03-12 21:16:12,143 INFO wandb-upload_4:2571781 [upload_job.py:push():131] Uploaded file runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/wandb-summary.json +2024-03-12 21:16:12,150 INFO wandb-upload_1:2571781 [upload_job.py:push():131] Uploaded file runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/conda-environment.yaml +2024-03-12 21:16:12,192 INFO wandb-upload_3:2571781 [upload_job.py:push():131] Uploaded file runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/files/output.log +2024-03-12 21:16:12,260 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: poll_exit +2024-03-12 21:16:12,260 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: poll_exit +2024-03-12 21:16:12,392 INFO Thread-11 (_thread_body):2571781 [sender.py:transition_state():617] send defer: 11 +2024-03-12 21:16:12,392 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: defer +2024-03-12 21:16:12,392 INFO HandlerThread:2571781 [handler.py:handle_request_defer():172] handle defer: 11 +2024-03-12 21:16:12,393 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: defer +2024-03-12 21:16:12,393 INFO SenderThread:2571781 [sender.py:send_request_defer():613] handle sender defer: 11 +2024-03-12 21:16:12,393 INFO SenderThread:2571781 [file_pusher.py:join():178] waiting for file pusher +2024-03-12 21:16:12,393 INFO SenderThread:2571781 [sender.py:transition_state():617] send defer: 12 +2024-03-12 21:16:12,393 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: defer +2024-03-12 21:16:12,393 INFO HandlerThread:2571781 [handler.py:handle_request_defer():172] handle defer: 12 +2024-03-12 21:16:12,393 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: status_report +2024-03-12 21:16:12,393 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: defer +2024-03-12 21:16:12,394 INFO SenderThread:2571781 [sender.py:send_request_defer():613] handle sender defer: 12 +2024-03-12 21:16:12,394 INFO SenderThread:2571781 [file_stream.py:finish():595] file stream finish called +2024-03-12 21:16:13,061 INFO SenderThread:2571781 [file_stream.py:finish():599] file stream finish is done +2024-03-12 21:16:13,061 INFO SenderThread:2571781 [sender.py:transition_state():617] send defer: 13 +2024-03-12 21:16:13,061 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: defer +2024-03-12 21:16:13,061 INFO HandlerThread:2571781 [handler.py:handle_request_defer():172] handle defer: 13 +2024-03-12 21:16:13,061 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: defer +2024-03-12 21:16:13,061 INFO SenderThread:2571781 [sender.py:send_request_defer():613] handle sender defer: 13 +2024-03-12 21:16:13,061 INFO SenderThread:2571781 [sender.py:transition_state():617] send defer: 14 +2024-03-12 21:16:13,062 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: defer +2024-03-12 21:16:13,062 INFO HandlerThread:2571781 [handler.py:handle_request_defer():172] handle defer: 14 +2024-03-12 21:16:13,062 DEBUG SenderThread:2571781 [sender.py:send():382] send: final +2024-03-12 21:16:13,062 DEBUG SenderThread:2571781 [sender.py:send():382] send: footer +2024-03-12 21:16:13,062 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: defer +2024-03-12 21:16:13,062 INFO SenderThread:2571781 [sender.py:send_request_defer():613] handle sender defer: 14 +2024-03-12 21:16:13,062 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: poll_exit +2024-03-12 21:16:13,063 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: poll_exit +2024-03-12 21:16:13,063 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: internal_messages +2024-03-12 21:16:13,063 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: server_info +2024-03-12 21:16:13,063 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: server_info +2024-03-12 21:16:13,107 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: get_summary +2024-03-12 21:16:13,107 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: sampled_history +2024-03-12 21:16:13,108 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: job_info +2024-03-12 21:16:13,267 DEBUG SenderThread:2571781 [sender.py:send_request():409] send_request: job_info +2024-03-12 21:16:13,267 DEBUG HandlerThread:2571781 [handler.py:handle_request():146] handle_request: shutdown +2024-03-12 21:16:13,268 INFO HandlerThread:2571781 [handler.py:finish():869] shutting down handler +2024-03-12 21:16:14,108 INFO WriterThread:2571781 [datastore.py:close():296] close: runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/run-b1zz0q9t.wandb +2024-03-12 21:16:14,267 INFO SenderThread:2571781 [sender.py:finish():1572] shutting down sender +2024-03-12 21:16:14,267 INFO SenderThread:2571781 [file_pusher.py:finish():172] shutting down file pusher +2024-03-12 21:16:14,267 INFO SenderThread:2571781 [file_pusher.py:join():178] waiting for file pusher diff --git a/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/logs/debug.log b/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..e632f72244730b53ad6b600500b6144ec809f002 --- /dev/null +++ b/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/logs/debug.log @@ -0,0 +1,34 @@ +2024-03-12 14:28:31,451 INFO MainThread:2568751 [wandb_setup.py:_flush():76] Current SDK version is 0.16.3 +2024-03-12 14:28:31,451 INFO MainThread:2568751 [wandb_setup.py:_flush():76] Configure stats pid to 2568751 +2024-03-12 14:28:31,451 INFO MainThread:2568751 [wandb_setup.py:_flush():76] Loading settings from /home/lilei/.config/wandb/settings +2024-03-12 14:28:31,451 INFO MainThread:2568751 [wandb_setup.py:_flush():76] Loading settings from /home/lilei/prismatic-vlms/wandb/settings +2024-03-12 14:28:31,451 INFO MainThread:2568751 [wandb_setup.py:_flush():76] Loading settings from environment variables: {} +2024-03-12 14:28:31,451 INFO MainThread:2568751 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False} +2024-03-12 14:28:31,451 INFO MainThread:2568751 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'scripts/pretrain.py', 'program_abspath': '/home/lilei/prismatic-vlms/scripts/pretrain.py', 'program': '/home/lilei/prismatic-vlms/scripts/pretrain.py'} +2024-03-12 14:28:31,451 INFO MainThread:2568751 [wandb_init.py:_log_setup():526] Logging user logs to runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/logs/debug.log +2024-03-12 14:28:31,451 INFO MainThread:2568751 [wandb_init.py:_log_setup():527] Logging internal logs to runs/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/logs/debug-internal.log +2024-03-12 14:28:31,451 INFO MainThread:2568751 [wandb_init.py:init():566] calling init triggers +2024-03-12 14:28:31,451 INFO MainThread:2568751 [wandb_init.py:init():573] wandb.init called with sweep_config: {} +config: {'model': {'type': 'one-stage+7b', 'model_id': 'one-stage+7b_qformer2_64', 'arch_specifier': 'qformer2_64', 'vision_backbone_id': 'clip-vit-l-336px', 'llm_backbone_id': 'vicuna-v15-7b', 'image_resize_strategy': 'letterbox', 'llm_max_length': 2048, 'align_epochs': 1, 'align_max_steps': None, 'align_global_batch_size': 256, 'align_per_device_batch_size': 16, 'align_learning_rate': 0.001, 'align_weight_decay': 0.0, 'align_max_grad_norm': 1.0, 'align_lr_scheduler_type': 'linear-warmup+cosine-decay', 'align_warmup_ratio': 0.03, 'align_train_strategy': 'fsdp-shard-grad-op', 'finetune_epochs': 1, 'finetune_max_steps': None, 'finetune_global_batch_size': 128, 'finetune_per_device_batch_size': 16, 'finetune_learning_rate': 2e-05, 'finetune_weight_decay': 0.1, 'finetune_max_grad_norm': 1.0, 'finetune_lr_scheduler_type': 'linear-warmup+cosine-decay', 'finetune_warmup_ratio': 0.03, 'finetune_train_strategy': 'fsdp-full-shard', 'enable_gradient_checkpointing': True, 'enable_mixed_precision_training': True, 'reduce_in_full_precision': False}, 'dataset': {'type': 'llava-v15', 'dataset_id': 'llava-v15', 'align_stage_components': ['download/llava-laion-cc-sbu-558k/chat.json', 'download/llava-laion-cc-sbu-558k'], 'finetune_stage_components': ['download/llava-v1.5-instruct/llava_v1_5_mix665k.json', 'download/llava-v1.5-instruct'], 'dataset_root_dir': 'data'}, 'stage': 'finetune', 'pretrained_checkpoint': None, 'run_id': 'qformer2_64', 'run_root_dir': 'runs', 'seed': 7, 'hf_token': '.hf_token', 'trackers': ['jsonl', 'wandb'], 'wandb_project': 'hf-vlms', 'wandb_entity': 'lilei_stones', 'max_length': 4096} +2024-03-12 14:28:31,452 INFO MainThread:2568751 [wandb_init.py:init():616] starting backend +2024-03-12 14:28:31,452 INFO MainThread:2568751 [wandb_init.py:init():620] setting up manager +2024-03-12 14:28:31,459 INFO MainThread:2568751 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2024-03-12 14:28:31,461 INFO MainThread:2568751 [wandb_init.py:init():628] backend started and connected +2024-03-12 14:28:31,474 INFO MainThread:2568751 [wandb_init.py:init():720] updated telemetry +2024-03-12 14:28:31,492 INFO MainThread:2568751 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout +2024-03-12 14:28:32,019 INFO MainThread:2568751 [wandb_run.py:_on_init():2262] communicating current version +2024-03-12 14:28:32,065 INFO MainThread:2568751 [wandb_run.py:_on_init():2271] got version response upgrade_message: "wandb version 0.16.4 is available! To upgrade, please run:\n $ pip install wandb --upgrade" + +2024-03-12 14:28:32,065 INFO MainThread:2568751 [wandb_init.py:init():804] starting run threads in backend +2024-03-12 14:28:36,927 INFO MainThread:2568751 [wandb_run.py:_console_start():2241] atexit reg +2024-03-12 14:28:36,927 INFO MainThread:2568751 [wandb_run.py:_redirect():2096] redirect: wrap_raw +2024-03-12 14:28:36,927 INFO MainThread:2568751 [wandb_run.py:_redirect():2161] Wrapping output streams. +2024-03-12 14:28:36,927 INFO MainThread:2568751 [wandb_run.py:_redirect():2186] Redirects installed. +2024-03-12 14:28:36,928 INFO MainThread:2568751 [wandb_init.py:init():847] run started, returning control to user process +2024-03-12 21:16:03,257 INFO MainThread:2568751 [wandb_run.py:_finish():1970] finishing run lilei_stones/hf-vlms/b1zz0q9t +2024-03-12 21:16:03,257 INFO MainThread:2568751 [wandb_run.py:_atexit_cleanup():2210] got exitcode: 0 +2024-03-12 21:16:03,258 INFO MainThread:2568751 [wandb_run.py:_restore():2193] restore +2024-03-12 21:16:03,258 INFO MainThread:2568751 [wandb_run.py:_restore():2199] restore done +2024-03-12 21:16:14,269 INFO MainThread:2568751 [wandb_run.py:_footer_history_summary_info():3866] rendering history +2024-03-12 21:16:14,269 INFO MainThread:2568751 [wandb_run.py:_footer_history_summary_info():3898] rendering summary +2024-03-12 21:16:14,280 INFO MainThread:2568751 [wandb_run.py:_footer_sync_info():3825] logging synced files diff --git a/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/run-b1zz0q9t.wandb b/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/run-b1zz0q9t.wandb new file mode 100644 index 0000000000000000000000000000000000000000..c4b5c5ffa4f4ab6ce384e04ef0c2a690f74ceda1 --- /dev/null +++ b/qformer2_64/wandb/run-20240312_142831-b1zz0q9t/run-b1zz0q9t.wandb @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9ad4aff7a896fcc1248d625c0cb2136f523eca702e66b78d215ea25360d63dd9 +size 12124943